Fixes issue 37108: import.sample.data fails if a file cannot be imported 3.0PR17Q3.1
authorAugusto Mauch <augusto.mauch@openbravo.com>
Thu, 19 Oct 2017 09:40:51 +0200
changeset 1120 a6b6565a165c
parent 1066 7cdd957ca879
child 1121 28a8922dfe69
Fixes issue 37108: import.sample.data fails if a file cannot be imported

If a file cannot be imported the import.sample.data process will stop immediately and will show log describing the problem.

To be able to catch the exceptions thrown in another thread, now the logic to import files is wrapped in a Callable instead of a Runnable.
src/org/apache/ddlutils/platform/postgresql/PostgreSqlDatabaseDataIO.java
src/org/openbravo/ddlutils/task/ImportSampledata.java
--- a/src/org/apache/ddlutils/platform/postgresql/PostgreSqlDatabaseDataIO.java	Fri Sep 22 12:56:42 2017 +0200
+++ b/src/org/apache/ddlutils/platform/postgresql/PostgreSqlDatabaseDataIO.java	Thu Oct 19 09:40:51 2017 +0200
@@ -16,11 +16,13 @@
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
+import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -28,7 +30,6 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.ddlutils.DdlUtilsException;
 import org.apache.ddlutils.Platform;
 import org.apache.ddlutils.io.DataSetTableExporter;
 import org.apache.ddlutils.io.DataSetTableQueryGenerator;
@@ -117,10 +118,12 @@
    *          The file being imported
    * @param platform
    *          Platform instance that will be used to retrieve the database connection
-   * @throws DdlUtilsException
-   *           if the file cannot be imported using PostgreSQL's COPY
+   * @throws IOException
+   *           if there is a problem reading the copy file
+   * @throws SQLException
+   *           if there is a problem importing the file contents
    */
-  public void importCopyFile(File file, Platform platform) throws DdlUtilsException {
+  public void importCopyFile(File file, Platform platform) throws IOException, SQLException {
     String tableName = getTableName(file);
     Connection connection = platform.borrowConnection();
     try (InputStream inputStream = new FileInputStream(file);) {
@@ -132,8 +135,6 @@
       copyCommand.append("(" + getColumnNames(file) + " ) ");
       copyCommand.append("FROM STDIN WITH (FORMAT CSV, HEADER TRUE) ");
       copyManager.copyIn(copyCommand.toString(), bufferedInStream);
-    } catch (Exception e) {
-      log.error("Error while importing file " + file.getName(), e);
     } finally {
       platform.returnConnection(connection);
     }
--- a/src/org/openbravo/ddlutils/task/ImportSampledata.java	Fri Sep 22 12:56:42 2017 +0200
+++ b/src/org/openbravo/ddlutils/task/ImportSampledata.java	Thu Oct 19 09:40:51 2017 +0200
@@ -20,16 +20,20 @@
 import java.io.InputStreamReader;
 import java.nio.charset.StandardCharsets;
 import java.sql.Connection;
+import java.sql.SQLException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Vector;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorCompletionService;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.Future;
 
 import org.apache.commons.beanutils.DynaBean;
 import org.apache.commons.dbcp.BasicDataSource;
+import org.apache.ddlutils.DdlUtilsException;
 import org.apache.ddlutils.Platform;
 import org.apache.ddlutils.PlatformFactory;
 import org.apache.ddlutils.io.DataReader;
@@ -154,22 +158,17 @@
 
           getLog().info("Inserting data into the database...");
           ExecutorService es = Executors.newFixedThreadPool(platform.getMaxThreads());
+
+          ExecutorCompletionService<Void> ecs = new ExecutorCompletionService<>(es);
           for (int i = 0; i < files.size(); i++) {
             File f = files.get(i);
             getLog().debug("Queueing data import from file: " + files.get(i).getName());
             ImportRunner ir = new ImportRunner(getLog(), platform, db, f, rdbms);
-            es.execute(ir);
+            ecs.submit(ir);
           }
-          es.shutdown();
-          boolean ok;
-          try {
-            // Wait until all the tables have been imported, or until 24 hours have passed
-            ok = es.awaitTermination(24, TimeUnit.HOURS);
-          } catch (InterruptedException e) {
-            throw new RuntimeException("InterruptedException in ");
-          }
-          if (!ok) {
-            throw new RuntimeException("Didn't finish in timeout");
+          for (int i = 0; i < files.size(); i++) {
+            Future<Void> future = ecs.take();
+            future.get();
           }
         }
       }
@@ -206,7 +205,7 @@
       }
 
     } catch (final Exception e) {
-      e.printStackTrace();
+      log.error("Error while importing the sample data", e);
       throw new BuildException(e);
     }
   }
@@ -317,7 +316,7 @@
    * Runnable class that will read data from a file and will write it in the database
    *
    */
-  private static class ImportRunner implements Runnable {
+  private static class ImportRunner implements Callable<Void> {
     private final Logger log;
     private final Platform platform;
     private final Database db;
@@ -333,7 +332,7 @@
     }
 
     @Override
-    public void run() {
+    public Void call() {
       String fileName = file.getName();
       log.debug("Importing data from file: " + fileName);
       if (fileName.endsWith(".xml")) {
@@ -345,6 +344,7 @@
           log.warn("File " + fileName + " cannot be imported in Oracle");
         }
       }
+      return null;
     }
 
     private void importXmlFile() {
@@ -359,7 +359,11 @@
 
     private void importPgCopyFile() {
       final PostgreSqlDatabaseDataIO dbdio = new PostgreSqlDatabaseDataIO();
-      dbdio.importCopyFile(file, platform);
+      try {
+        dbdio.importCopyFile(file, platform);
+      } catch (IOException | SQLException e) {
+        throw new DdlUtilsException("Error while importing file " + file.getAbsolutePath(), e);
+      }
     }
   }