fixed bug 36755: inefficient translate task
authorAsier Lostalé <asier.lostale@openbravo.com>
Fri, 11 Aug 2017 16:21:42 +0200
changeset 32608 f8c3d0a4f7a4
parent 32607 b9227826b5dc
child 32609 74017a22fc13
fixed bug 36755: inefficient translate task

Several improvements applied:
* A single invocation is performed instead one per file extension.
* Look for labels just once regardless number of installed translations.
* Cache all labels in memory so that a single query is performed to look for
them up instead of one per each label found in files.
* Execute all DB updates in a single transaction
* Reduced log vebosity (specially when no modules in development)
build.xml
src-trl/src/org/openbravo/translate/Translate.java
src-trl/src/org/openbravo/translate/Translate_data.xsql
src/build.xml
--- a/build.xml	Wed Aug 30 14:10:12 2017 +0200
+++ b/build.xml	Fri Aug 11 16:21:42 2017 +0200
@@ -712,10 +712,6 @@
     <ant dir="${base.src}" target="translate" inheritAll="true" inheritRefs="true" />
   </target>
 
-  <target name="translate.modules" depends="init">
-    <ant dir="${base.src}" target="translate.modules" inheritAll="true" inheritRefs="true" />
-  </target>
-
   <target name="installWebService" depends="init">
     <ant dir="${base.src}" target="installWebService" inheritAll="true" inheritRefs="true">
       <property name="wsdd" value="1" />
--- a/src-trl/src/org/openbravo/translate/Translate.java	Wed Aug 30 14:10:12 2017 +0200
+++ b/src-trl/src/org/openbravo/translate/Translate.java	Fri Aug 11 16:21:42 2017 +0200
@@ -18,13 +18,27 @@
  */
 package org.openbravo.translate;
 
-import java.io.File;
-import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.FileSystems;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.nio.file.Paths;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.sql.Connection;
+import java.sql.SQLException;
 import java.util.ArrayList;
-import java.util.StringTokenizer;
-import java.util.Vector;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
 
 import javax.servlet.ServletException;
 
@@ -33,74 +47,63 @@
 import org.apache.xerces.parsers.SAXParser;
 import org.openbravo.database.CPStandAlone;
 import org.openbravo.database.SessionInfo;
-import org.openbravo.utils.DirFilter;
+import org.openbravo.exception.NoConnectionAvailableException;
 import org.xml.sax.Attributes;
 import org.xml.sax.InputSource;
 import org.xml.sax.XMLReader;
-import org.xml.sax.ext.LexicalHandler;
 import org.xml.sax.helpers.DefaultHandler;
 
 /**
+ * Looks for translatable elements in html, fo, srpt and jrxml files inserting them in
+ * ad_textinterfaces table.
+ * 
  * @author Fernando Iriazabal
- * 
- *         Translate the HTML file of the folder especified
  **/
-public class Translate extends DefaultHandler implements LexicalHandler {
-  protected static CPStandAlone pool;
-  static XMLReader parser;
-  static TranslateData[] toLanguage;
-  static String actualLanguage;
-  static String fileTermination;
+public class Translate extends DefaultHandler {
+  private static CPStandAlone pool;
+  private static final Pattern LETTER_PATTERN = Pattern.compile("[a-zA-Z]");
+  private static final List<String> translatableExtensions = Arrays.asList("html", "fo", "srpt",
+      "jrxml");
+  private static final Logger log = Logger.getLogger(Translate.class);
 
-  static boolean isHtml = false;
+  private XMLReader parser;
+  private String extension;
 
-  static String actualTag;
-  static String actualFile;
-  static String actualPrefix;
-  static StringBuffer translationText;
-  static int count = 0;
-  static ArrayList<String> moduleDirectories;
-  static String moduleName = "";
-  static String moduleLang = "";
-  static String moduleID = "";
-  static boolean translateModule = true;
-  static final String[] tokens = { "-", ":" };
+  private String actualTag;
+  private String actualFile;
+  private String actualPrefix;
+  private StringBuilder translationText;
+  private int count = 0;
 
-  static Logger log4j = Logger.getLogger(Translate.class);
+  private String moduleLang = "";
+  private String moduleID = "";
+  private Path moduleBasePath;
+  private Connection conn;
+  private boolean canTranslateModule = true;
 
-  /**
-   * Constructor
-   * 
-   * @param xmlPoolFile
-   *          Path to the Openbravo.properties file.
-   * @throws ServletException
-   */
-  public Translate(String xmlPoolFile) throws ServletException {
+  private static Map<String, List<TranslateData>> allLabels;
+  private static HashSet<String> modsInDev = new HashSet<>();
+
+  private static void init(String xmlPoolFile) {
     pool = new CPStandAlone(xmlPoolFile);
   }
 
+  private Translate() {
+  }
+
   /**
-   * Constructor
-   * 
-   * @param xmlPoolFile
-   *          Path to the Openbravo.properties file.
    * @param _fileTermination
    *          File extension to filter.
-   * @throws ServletException
    */
-  public Translate(String xmlPoolFile, String _fileTermination) throws ServletException {
-    this(xmlPoolFile);
-    fileTermination = _fileTermination;
-    isHtml = fileTermination.toLowerCase().endsWith("html");
+  public Translate(String _fileTermination) throws ServletException {
+    extension = _fileTermination;
+    boolean isHtml = extension.toLowerCase().endsWith("html");
     if (isHtml)
       parser = new org.cyberneko.html.parsers.SAXParser();
     else
       parser = new SAXParser();
     parser.setEntityResolver(new LocalEntityResolver());
     parser.setContentHandler(this);
-    toLanguage = TranslateData.systemLanguage(pool);
-    if (toLanguage.length == 0)
-      log4j.warn("No system languages defined, translation will parse all files.");
   }
 
   /**
@@ -110,66 +113,71 @@
    *          List of arguments. There is 2 call ways, with 2 arguments; the first one is the
    *          attribute to indicate if the AD_TEXTINTERFACES must be cleaned ("clean") and the
    *          second one is the Openbravo.properties path. The other way is with more arguments,
-   *          where: 0- Openbravo.properties path. 1- File extension. 2- Path where are the files to
-   *          translate. 3- Relative path.
-   * @throws Exception
+   *          where: 0- Openbravo.properties path. 1- Path where are the files to translate.
    */
   public static void main(String argv[]) throws Exception {
     PropertyConfigurator.configure("log4j.lcf");
-    String dirIni;
-    boolean boolFilter;
-    DirFilter dirFilter = null;
-    String relativePath = "";
 
-    if ((argv.length == 2)) {
-      if (argv[0].equals("clean")) {
-        log4j.debug("clean AD_TEXTINTERFACES");
-        final Translate translate = new Translate(argv[1]);
-        translate.clean();
-        return;
-      } else if (argv[0].equals("remove")) {
-        log4j.debug("remove AD_TEXTINTERFACES");
-        final Translate translate = new Translate(argv[1]);
-        translate.remove();
-        return;
+    if (argv.length != 2) {
+      log.error("Usage: Translate Openbravo.properties [clean|remove|sourceDir]");
+      log.error("Received: " + Arrays.asList(argv));
+      return;
+    }
+
+    init(argv[0]);
+    Translate translate;
+    switch (argv[1]) {
+    case "clean":
+      log.debug("clean AD_TEXTINTERFACES");
+      translate = new Translate();
+      translate.clean();
+      return;
+    case "remove":
+      log.debug("remove AD_TEXTINTERFACES");
+      translate = new Translate();
+      translate.remove();
+      return;
+    }
+
+    Path obPath = Paths.get(argv[1]).normalize();
+
+    TranslateData[] mods = TranslateData.getModulesInDevelopment(pool);
+    for (TranslateData mod : mods) {
+      modsInDev.add(mod.id);
+    }
+    for (TranslateData mod : mods) {
+      Path path;
+      if ("0".equals(mod.id)) {
+        path = obPath.resolve("src");
+      } else {
+        path = obPath.resolve(Paths.get("modules", mod.javapackage, "src"));
+        if (!Files.exists(path)) {
+          continue;
+        }
+      }
+      log.info("Looking for translatable elements in " + mod.javapackage + " (" + mod.name + ")");
+
+      for (String extension : translatableExtensions) {
+        translate = new Translate(extension);
+        translate.moduleLang = mod.lang;
+        translate.moduleID = mod.id;
+        translate.moduleBasePath = path;
+
+        translate.execute();
+        if (translate.count > 0) {
+          log.info("  parsed " + translate.count + " " + extension + " files");
+        }
+
+        if (!translate.canTranslateModule) {
+          break;
+        }
       }
     }
 
-    if (argv.length < 3) {
-      log4j.error("Usage: java Translate Openbravo.properties fileTermination sourceDir");
-      return;
+    if (mods.length == 0) {
+      log.info("No modules in development to look for translatable elements.");
     }
-
-    final Translate translate = new Translate(argv[0], argv[1]);
-
-    dirIni = argv[2].replace("\\", "/");
-
-    if (argv.length > 3) {
-      moduleDirectories = getDirectories(argv[3]);
-      log4j.info("Translation for modules");
-    }
-    boolFilter = true;
-    dirFilter = new DirFilter(fileTermination);
-    log4j.info("directory source: " + dirIni);
-    log4j.info("file termination: " + fileTermination);
-
-    final File path = new File(dirIni, relativePath);
-    if (!path.exists()) {
-      log4j.error("Can't find directory: " + dirIni);
-      translate.destroy();
-      return;
-    }
-    if (moduleDirectories == null) {
-      if (TranslateData.isInDevelopmentModule(pool, "0")) {
-        listDir(path, boolFilter, dirFilter, relativePath, true, "", 0, "");
-        log4j.info("Translated files for " + fileTermination + ": " + count);
-      } else
-        log4j.info("Core is not in development: skipping it");
-    } else {
-      listDir(path, boolFilter, dirFilter, relativePath, false, "", 0, "");
-      log4j.info("Translated files for " + fileTermination + ": " + count);
-    }
-    translate.destroy();
+    destroy();
   }
 
   /**
@@ -179,115 +187,52 @@
     try {
       TranslateData.clean(pool);
     } catch (final Exception e) {
-      log4j.error("clean error", e);
+      log.error("clean error", e);
     }
   }
 
   private void remove() {
     try {
-      TranslateData.remove(pool);
+      int n = TranslateData.remove(pool);
+      if (n > 0) {
+        log.info("Removed " + n + " unused elements");
+      }
     } catch (final Exception e) {
-      log4j.error("remove error", e);
+      log.error("remove error", e);
     }
   }
 
-  /**
-   * Receives a list of directories and returns this list as an ArrayList
-   * 
-   */
-  private static ArrayList<String> getDirectories(String s) {
-    final ArrayList<String> l = new ArrayList<String>();
-    final StringTokenizer tok = new StringTokenizer(s, "/");
-    while (tok.hasMoreTokens())
-      l.add(tok.nextToken());
-    return l;
+  /** Looks for translatable elements for a given module and extension */
+  private void execute() throws IOException {
+    final PathMatcher matcher = FileSystems.getDefault().getPathMatcher("regex:.*\\." + extension);
+    Files.walkFileTree(moduleBasePath, new SimpleFileVisitor<Path>() {
+      @Override
+      public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+        if (matcher.matches(file)) {
+          if (!canTranslateModule()) {
+            log.error("  Module is not set as translatable or has not defined language, but has translatable elements");
+            log.error("  No translations will be inserted. Set the module as 'is translation requiered' and select a language");
+            log.error("  then execute translation again.");
+            canTranslateModule = false;
+            return FileVisitResult.TERMINATE;
+          }
+          parseFile(file);
+        }
+        return FileVisitResult.CONTINUE;
+      }
+    });
+
+    if (conn != null) {
+      try {
+        pool.releaseCommitConnection(conn);
+      } catch (SQLException e) {
+        log.error("Error commiting changes to database", e);
+      }
+    }
   }
 
-  /**
-   * List all the files and folders in the selected path.
-   * 
-   * @param file
-   *          The selected path to list.
-   * @param boolFilter
-   *          If is filtered.
-   * @param dirFilter
-   *          Filter to apply.
-   * @param _relativePath
-   *          The relative path.
-   */
-  private static void listDir(File file, boolean boolFilter, DirFilter dirFilter,
-      String _relativePath, boolean parse, String parent, int level, String module) {
-    String relativePath = _relativePath;
-    File[] list;
-    if (boolFilter)
-      list = file.listFiles(dirFilter);
-    else
-      list = file.listFiles();
-    for (int i = 0; i < list.length; i++) {
-      final File fileItem = list[i];
-      if (fileItem.isDirectory()) {
-        // if it is a subdirectory then list recursively
-        final String prevRelativePath = new String(relativePath);
-        relativePath += "/" + fileItem.getName();
-
-        if (log4j.isDebugEnabled())
-          log4j.debug("dir: "
-              + fileItem.getName()
-              + " - parse:"
-              + parse
-              + " - parent:"
-              + parent
-              + " - level:"
-              + level
-              + " - include:"
-              + (moduleDirectories != null && moduleDirectories.size() > level ? moduleDirectories
-                  .get(level) : "--"));
-        if (parse)
-          listDir(fileItem, boolFilter, dirFilter, relativePath, true, parent, level + 1, module);
-        else {
-          if ((moduleDirectories.size() == level + 1)
-              && (moduleDirectories.get(level).equals("*") || moduleDirectories.get(level).equals(
-                  fileItem.getName()))) {
-            log4j.info("Start parsing module: " + parent.replace("/", ""));
-            translateModule = true;
-            try {
-              if (TranslateData.isInDevelopmentModulePack(pool, parent.replace("/", "")))
-                listDir(fileItem, boolFilter, dirFilter, relativePath, true, parent + "/"
-                    + fileItem.getName(), level + 1, parent.replace("/", ""));
-              else
-                log4j.info("Module is not in development: skipping it");
-            } catch (final Exception e) {
-              e.printStackTrace();
-            }
-          } else if (moduleDirectories.size() > level
-              && (moduleDirectories.get(level).equals("*") || moduleDirectories.get(level).equals(
-                  fileItem.getName())))
-            listDir(fileItem, boolFilter, dirFilter, relativePath, false,
-                parent + "/" + fileItem.getName(), level + 1, module);
-          // other case don't follow deeping into the tree
-        }
-
-        relativePath = prevRelativePath;
-      } else {
-        try {
-          if (parse) {
-            if (log4j.isDebugEnabled())
-              log4j.debug(list[i] + " Parent: " + fileItem.getParent() + " getName() "
-                  + fileItem.getName() + " canonical: " + fileItem.getCanonicalPath());
-            for (int h = 0; h < toLanguage.length; h++) {
-              actualLanguage = toLanguage[h].name;
-              parseFile(list[i], relativePath, parent, module);
-            }
-            if (toLanguage.length == 0) {
-              actualLanguage = "";
-              parseFile(list[i], relativePath, parent, module);
-            }
-          }
-        } catch (final IOException e) {
-          log4j.error("IOException: " + e);
-        }
-      }
-    }
+  private boolean canTranslateModule() {
+    return !(moduleLang == null || moduleLang.equals(""));
   }
 
   /**
@@ -295,66 +240,18 @@
    * 
    * @param fileParsing
    *          File to parse.
-   * @param _relativePath
-   *          The relative path.
    */
-  private static void parseFile(File fileParsing, String _relativePath, String parent, String module) {
-    if (!translateModule)
-      return;
-    String relativePath = _relativePath;
-    final String strFileName = fileParsing.getName();
-    if (log4j.isDebugEnabled())
-      log4j.debug("Parsing of " + strFileName);
-    final int pos = strFileName.indexOf(fileTermination);
-    if (pos == -1) {
-      log4j.error("File " + strFileName + " don't have termination " + fileTermination);
-      return;
-    }
-    final String strFileWithoutTermination = strFileName.substring(0, pos);
-    if (log4j.isDebugEnabled())
-      log4j.debug("File without termination: " + strFileWithoutTermination);
+  private void parseFile(Path fileParsing) {
+    actualFile = "/" + moduleBasePath.relativize(fileParsing).toString().replace("\\", "/");
 
-    // In case moduleDirectories has value remove parent from path to keep
-    // clean the package
-    try {
-      moduleName = module.equals("") ? "org.openbravo" : module;
-      moduleID = TranslateData.getModuleID(pool, moduleName);
-      if (moduleID == null || moduleID.equals("")) {
-        log4j.error("Trying to insert element in module " + moduleName
-            + " which has no ID, it will be set to core");
-        moduleName = "CORE";
-        moduleID = "0";
-      }
-      moduleLang = TranslateData.getModuleLang(pool, moduleID);
-    } catch (final ServletException e) {
-      e.printStackTrace();
-    }
+    log.debug("File: " + actualFile);
 
-    if (log4j.isDebugEnabled())
-      log4j.debug("Module name: " + module + " - oldRelativePath:" + relativePath + " - parent:"
-          + parent);
-    if (moduleDirectories != null && relativePath.startsWith(parent)) {
-      relativePath = relativePath.substring(parent.length());
-      if (log4j.isDebugEnabled())
-        log4j.debug("new relativePath:" + relativePath);
-    }
-
-    actualFile = relativePath + "/" + strFileName;
-
-    log4j.debug("File: " + fileParsing);
-
-    try {
-      FileInputStream fis = new FileInputStream(fileParsing);
-      InputStreamReader reader = new InputStreamReader(fis, "UTF-8");
+    try (InputStream in = Files.newInputStream(fileParsing);
+        InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) {
       parser.parse(new InputSource(reader));
-      if (translateModule)
-        count++;
-    } catch (final IOException e) {
-      log4j.error("file: " + actualFile);
-      e.printStackTrace();
+      count++;
     } catch (final Exception e) {
-      log4j.error("file: " + actualFile);
-      e.printStackTrace();
+      log.error("file: " + actualFile, e);
     }
   }
 
@@ -366,8 +263,6 @@
    *          Attributes of the element.
    */
   private void parseAttributes(Attributes amap) {
-    if (!translateModule)
-      return;
     String type = "";
     String value = "";
     for (int i = 0; i < amap.getLength(); i++) {
@@ -391,9 +286,8 @@
         translate(strAux);
       }
     }
-    if (value != null && !value.equals("")) {
-      if (type.equalsIgnoreCase("button"))
-        translate(value);
+    if (value != null && !value.equals("") && type.equalsIgnoreCase("button")) {
+      translate(value);
     }
   }
 
@@ -419,10 +313,10 @@
    *          Name of the element.
    * @return True if the element is parseable, false if not.
    */
-  private static boolean isParseable(String tagname) {
+  private boolean isParseable(String tagname) {
     if (tagname.equalsIgnoreCase("script"))
       return false;
-    else if (fileTermination.equalsIgnoreCase("jrxml")) {
+    else if (extension.equalsIgnoreCase("jrxml")) {
       if (!tagname.equalsIgnoreCase("text") && !tagname.equalsIgnoreCase("textFieldExpression"))
         return false;
     }
@@ -434,15 +328,9 @@
    * this method.
    */
   @Override
-  public void startElement(String uri, String name, String qName, Attributes amap) {// (String name,
-    // AttributeList
-    // amap) throws
-    // SAXException {
-    if (!translateModule)
-      return;
-    if (log4j.isDebugEnabled())
-      log4j.info("Configuration: startElement is called: element name=" + qName + " actualtag"
-          + actualTag + " trlTxt" + translationText);
+  public void startElement(String uri, String name, String qName, Attributes amap) {
+    log.debug("startElement element name=" + qName + " actualtag" + actualTag + " trlTxt"
+        + translationText);
     if (actualTag != null && isParseable(actualTag) && translationText != null) {
       translate(translationText.toString());
     }
@@ -454,49 +342,13 @@
     actualTag = name.trim().toUpperCase();
   }
 
-  public void comment(char[] ch, int start, int length) {
-  }
-
-  public void endDTD() {
-  }
-
-  public void endEntity(String name) {
-  }
-
-  public void startDTD(String name, String publicId, String systemId) {
-  }
-
-  public void startEntity(String name) {
-  }
-
-  /**
-   * Method to insert begining of CDATA expresions.
-   */
-  public void startCDATA() {
-
-  }
-
-  /**
-   * Method to insert ends of CDATA expresions.
-   */
-  public void endCDATA() {
-
-  }
-
   /**
    * End of an element of the file. When the parser finds the end of an element in the file, it
    * calls to this method.
    */
   @Override
-  public void endElement(String uri, String name, String qName) {// (String
-    // name)
-    // throws
-    // SAXException
-    // {
-    if (!translateModule)
-      return;
-    if (log4j.isDebugEnabled())
-      log4j.debug("Configuration: endElement is called: " + qName);
+  public void endElement(String uri, String name, String qName) {
+    log.debug("endElement : " + qName);
 
     if (isParseable(actualTag) && translationText != null) {
       translate(translationText.toString());
@@ -510,24 +362,22 @@
    * element's tags.
    */
   @Override
-  public void characters(char[] ch, int start, int length) {// throws
-    // SAXException {
+  public void characters(char[] ch, int start, int length) {
     final String chars = new String(ch, start, length);
-    if (log4j.isDebugEnabled())
-      log4j.debug("Configuration(characters) is called: " + chars);
+    log.debug("characters: " + chars);
     if (translationText == null)
-      translationText = new StringBuffer();
+      translationText = new StringBuilder();
     translationText.append(chars);
   }
 
   /**
    * This method is the one in charge of the translation of the found text.
    * 
-   * @param ini
+   * @param txt
    *          String with the text to translate.
    */
-  private void translate(String ini) {
-    translate(ini, false);
+  private void translate(String txt) {
+    translate(txt, false);
   }
 
   /**
@@ -540,63 +390,125 @@
    *          content.
    */
   private void translate(String input, boolean isPartial) {
-    if (!translateModule)
-      return;
-    String ini = input;
-    ini = replace(replace(ini.trim(), "\r", ""), "\n", " ");
-    ini = ini.trim();
-    ini = delSp(ini);
+    String txt = input.replace("\r", "").replace("\n", " ").replaceAll(" ( )+", " ").trim();
+
     if (!isPartial && actualTag.equalsIgnoreCase("textFieldExpression")) {
-      int pos = ini.indexOf("\"");
+      int pos = txt.indexOf("\"");
       while (pos != -1) {
-        ini = ini.substring(pos + 1);
-        pos = ini.indexOf("\"");
+        txt = txt.substring(pos + 1);
+        pos = txt.indexOf("\"");
         if (pos != -1) {
-          translate(ini.substring(0, pos), true);
-          ini = ini.substring(pos + 1);
+          translate(txt.substring(0, pos), true);
+          txt = txt.substring(pos + 1);
         } else
           break;
-        pos = ini.indexOf("\"");
+        pos = txt.indexOf("\"");
       }
       return;
     }
-    final Vector<String> translated = new Vector<String>(0);
-    boolean aux = true;
-    translated.addElement("Y");
-    if (!ini.equals("") && !ini.toLowerCase().startsWith("xx") && !isNumeric(ini)) {
-      log4j.debug("Translating " + ini + " for file" + actualFile + " moduleLang:" + moduleLang);
-      tokenize(ini, 0, translated);
-      try {
-        aux = translated.elementAt(0).equals("Y");
-        if (moduleLang == null || moduleLang.equals("")) {
-          log4j
-              .error("Module is not set as translateable or has not defined language, but has translateable elements");
-          log4j
-              .error("No translations will be inserted. Set the module as 'is translation requiered' and select a language");
-          log4j.error("then execute translation again.");
-          translateModule = false;
-          return;
+    boolean translatableTxt = !(txt.equals("") || txt.toLowerCase().startsWith("xx") || isNumeric(txt));
+    if (!translatableTxt) {
+      return;
+    }
+
+    log.debug("Checking [" + txt + "] from file" + actualFile + " - language: " + moduleLang);
+    TranslateData t = getLabel(txt);
+    if (t != null) {
+      if ("N".equals(t.tr) && t.module.equals(moduleID) && t.id != null) {
+        try {
+          TranslateData.update(getConnection(), pool, t.id);
+          t.tr = "Y";
+        } catch (ServletException e) {
+          log.error("Could not set label as used [" + txt + "]");
         }
-        if (!aux) {
-          if (TranslateData.existsExpresionModFile(pool, ini, actualFile, moduleLang) == 0
-              && TranslateData.existsExpresionModNoFile(pool, ini, moduleLang) == 0
-              && TranslateData.existsExpresionNoModFile(pool, ini, actualFile, moduleLang) == 0
-              && TranslateData.existsExpresionNoModNoFile(pool, ini, moduleLang) == 0) {
+      }
+    } else {
+      createLabel(txt);
+    }
+  }
 
-            if (!TranslateData.isInDevelopmentModule(pool, moduleID))
-              log4j.error("Module  is not in development, it will be inserted anyway");
-            if (log4j.isDebugEnabled())
-              log4j.debug("inserting in module:" + moduleName + " - ID:" + moduleID);
-            TranslateData.insert(pool, ini, actualFile, moduleID);
-            log4j.info("Inserting text: " + ini + "from file: " + actualFile
-                + "into ad_textinterfaces");
-          }
-        }
-      } catch (final ServletException e) {
-        e.printStackTrace();
+  /** Returns a suitable text interface element for the given txt or null if there is none. */
+  private TranslateData getLabel(String txt) {
+    if (allLabels == null) {
+      // lazy initialization: in case none of the modules in development has elements to translate
+      // it won't be initialized
+      initializeAllLabels();
+    }
+
+    List<TranslateData> labels = allLabels.get(txt);
+    if (labels == null) {
+      return null;
+    }
+
+    TranslateData selected = null;
+    for (TranslateData label : labels) {
+      if (!moduleLang.equals(label.lang)) {
+        continue;
+      }
+      if (actualFile.equals(label.filename)) {
+        return label;
+      } else if (label.filename == null || "".equals(label.filename)) {
+        selected = label;
       }
     }
-    return;
+    return selected;
+  }
+
+  /**
+   * Loads in memory all text interfaces. Typically this is faster than querying them each time a
+   * translatable element is found in a file by reducing significantly the number of queries.
+   */
+  private void initializeAllLabels() {
+    TranslateData[] all;
+    try {
+      all = TranslateData.getTextInterfaces(pool);
+    } catch (ServletException e) {
+      throw new RuntimeException("Error loading text interfaces from DB", e);
+    }
+    allLabels = new HashMap<>(all.length);
+    String currentTxt = null;
+    List<TranslateData> currentList = null;
+    for (TranslateData label : all) {
+      if (!label.text.equals(currentTxt)) {
+        currentTxt = label.text;
+        currentList = new ArrayList<>();
+        allLabels.put(currentTxt, currentList);
+      }
+      currentList.add(label);
+    }
+  }
+
+  private void createLabel(String txt) {
+    try {
+      TranslateData.insert(getConnection(), pool, txt, actualFile, moduleID);
+      log.info("    New label found [" + txt + "] in " + actualFile);
+    } catch (ServletException e) {
+      log.error("   Could not insert label [" + txt + "]", e);
+    }
+
+    List<TranslateData> labelsForTxt = allLabels.get(txt);
+    if (labelsForTxt == null) {
+      labelsForTxt = new ArrayList<>();
+      allLabels.put(txt, labelsForTxt);
+    }
+
+    TranslateData newElement = new TranslateData();
+    newElement.text = txt;
+    newElement.filename = actualFile;
+    newElement.lang = moduleLang;
+
+    labelsForTxt.add(newElement);
+  }
+
+  private Connection getConnection() {
+    if (conn == null) {
+      try {
+        conn = pool.getTransactionConnection();
+      } catch (NoConnectionAvailableException | SQLException e) {
+        throw new RuntimeException("Could not get a DB connection", e);
+      }
+    }
+    return conn;
   }
 
   /**
@@ -607,129 +519,13 @@
    * @return True if has no letter in the text or false if has any letter.
    */
   private static boolean isNumeric(String ini) {
-    boolean isNumericData = true;
-    for (int i = 0; i < ini.length(); i++) {
-      if (Character.isLetter(ini.charAt(i))) {
-        isNumericData = false;
-        break;
-      }
-    }
-    return isNumericData;
+    return !LETTER_PATTERN.matcher(ini).find();
   }
 
-  /**
-   * Replace a char, inside a given text, with another char.
-   * 
-   * @param strInicial
-   *          Text where is the char to replace.
-   * @param strReplaceWhat
-   *          Char to replace.
-   * @param strReplaceWith
-   *          Char to replace with.
-   * @return String with the replaced text.
-   */
-  private static String replace(String strInicial, String strReplaceWhat, String strReplaceWith) {
-    int index = 0;
-    int pos;
-    final StringBuffer strFinal = new StringBuffer("");
-    do {
-      pos = strInicial.indexOf(strReplaceWhat, index);
-      if (pos != -1) {
-        strFinal.append(strInicial.substring(index, pos) + strReplaceWith);
-        index = pos + strReplaceWhat.length();
-      } else {
-        strFinal.append(strInicial.substring(index));
-      }
-    } while (index < strInicial.length() && pos != -1);
-    return strFinal.toString();
-  }
-
-  /**
-   * This method remove all the spaces in the string.
-   * 
-   * @param strIni
-   *          String to clean.
-   * @return String without spaces.
-   */
-  private static String delSp(String strIni) {
-    boolean sp = false;
-    String strFin = "";
-    for (int i = 0; i < strIni.length(); i++) {
-      if (!sp || strIni.charAt(i) != ' ')
-        strFin += strIni.charAt(i);
-      sp = (strIni.charAt(i) == ' ');
-    }
-    return strFin;
-  }
-
-  /**
-   * This method splits the main string into shortest fragments to translate them separately.
-   * 
-   * @param ini
-   *          String to split
-   * @param indice
-   *          Index of the separator array to use.
-   * @param isTranslated
-   *          Indicates if the text has been translated.
-   * @return String translated.
-   */
-  private String tokenize(String ini, int indice, Vector<String> isTranslated) {
-    final StringBuffer fin = new StringBuffer();
-    try {
-      boolean first = true;
-      String translated = null;
-      TranslateData[] dataTranslated = TranslateData.select(pool, ini.trim(), actualFile,
-          actualLanguage);
-      if (dataTranslated != null && dataTranslated.length > 0) {
-        translated = dataTranslated[0].tr;
-        // TranslateData.update(pool,
-        // dataTranslated[0].baseDictionaryEntryId, actualLanguage);
-        TranslateData.update(pool, dataTranslated[0].adTextinterfacesId);
-      }
-      if (translated != null && translated.length() > 0) {
-        fin.append(translated);
-        return fin.toString();
-      }
-      final StringTokenizer st = new StringTokenizer(ini, tokens[indice], false);
-      while (st.hasMoreTokens()) {
-        if (first) {
-          first = false;
-        } else {
-          fin.append(tokens[indice]);
-        }
-        String token = st.nextToken();
-        token = token.trim();
-        if (log4j.isDebugEnabled())
-          log4j.debug("Token of " + ini + " : -" + token + "-");
-        translated = null;
-        dataTranslated = TranslateData.select(pool, token.trim(), actualFile, actualLanguage);
-        if (dataTranslated != null && dataTranslated.length > 0) {
-          translated = dataTranslated[0].tr;
-          // TranslateData.update(pool,
-          // dataTranslated[0].baseDictionaryEntryId, actualLanguage);
-          TranslateData.update(pool, dataTranslated[0].adTextinterfacesId);
-        }
-        if ((translated == null || translated.equals("")) && indice < (tokens.length - 1))
-          translated = tokenize(token, indice + 1, isTranslated);
-        if (translated == null || translated.equals("")) {
-          fin.append(token);
-          isTranslated.set(0, "N");
-        } else
-          fin.append(translated);
-      }
-    } catch (final Exception e) {
-      e.printStackTrace();
-    }
-    return fin.toString();
-  }
-
-  /**
-   * The method to close database connection.
-   */
-  private void destroy() {
+  /** Closes database connection. */
+  private static void destroy() {
     // remove cached connection from thread local
     SessionInfo.init();
-
     pool.destroy();
   }
 }
--- a/src-trl/src/org/openbravo/translate/Translate_data.xsql	Wed Aug 30 14:10:12 2017 +0200
+++ b/src-trl/src/org/openbravo/translate/Translate_data.xsql	Fri Aug 11 16:21:42 2017 +0200
@@ -12,7 +12,7 @@
  * under the License. 
  * The Original Code is Openbravo ERP. 
  * The Initial Developer of the Original Code is Openbravo SLU 
- * All portions are Copyright (C) 2001-2010 Openbravo SLU 
+ * All portions are Copyright (C) 2001-2017 Openbravo SLU 
  * All Rights Reserved. 
  * Contributor(s):  ______________________________________.
  ************************************************************************
@@ -20,166 +20,25 @@
 
 
 <SqlClass name="TranslateData" package="org.openbravo.translate">
-  <SqlMethod name="select" type="preparedStatement" return="multiple">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-      select A.ad_textinterfaces_id, A.tr, '' AS NAME, '' AS TOTAL
-        from (select tt.ad_textinterfaces_id AS ad_textinterfaces_id, trim(tt.text) AS tr, 1 AS o
-                from ad_textinterfaces_trl tt
-               where tt.ad_textinterfaces_id =  (select min(ad_textinterfaces_id)
-                                                   from ad_textinterfaces t
-                                                  where t.text     = ?
-                                                    and t.filename = ?)
-                 and tt.ad_language = ?
-               union
-              select tt.ad_textinterfaces_id AS id, trim(tt.text) AS tr, 2 AS o
-                from ad_textinterfaces_trl tt
-               where tt.ad_textinterfaces_id =  (select min(ad_textinterfaces_id)
-                                                   from ad_textinterfaces t
-                                                  where t.text =  ?
-                                                    and t.filename is null)
-                 and tt.ad_language = ?
-               order by o) A
-      group by A.ad_textinterfaces_id, A.tr, A.o
-      order by A.o
+  <SqlMethod name="getTextInterfaces" type="preparedStatement" return="multiple">
+      <Sql>
+      select ad_textinterfaces_id as id, isUsed as tr, text, t.ad_module_id as module, filename, m.ad_language as lang, '' as name,'' as javapackage
+        from ad_textinterfaces t, ad_module m
+       where t.ad_module_id = m.ad_module_id
+       order by text, case when filename is not null then 0 else 1 end
     </Sql>
-    <Parameter name="string"/>
-    <Parameter name="filename"/>
-    <Parameter name="languajeFin"/>
-    <Parameter name="string"/>
-    <Parameter name="languajeFin"/>
-  </SqlMethod>
-
-  <SqlMethod name="existsExpresionModFile" type="preparedStatement" return="rowCount">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-         update ad_textinterfaces
-           set isUsed = 'Y'
-         where ad_textinterfaces_id = ( select min(A.ad_textinterfaces_id)
-                                          from (select B.*
-                                                from (select min(ad_textinterfaces_id) AS ad_textinterfaces_id, 1 AS o
-                                                          from ad_textinterfaces t,
-                                                               ad_module m
-                                                         where text     = ?
-                                                           and filename = ?
-                                                           and m.ad_module_id = t.ad_module_id
-                                                           and m.ad_language = ?
-                                                         order by o) B
-                                                 where ad_textinterfaces_id is not null) A)
-    </Sql>
-    <Parameter name="text"/>
-    <Parameter name="filename"/>
-    <Parameter name="moduleLang"/>
   </SqlMethod>
   
-  <SqlMethod name="existsExpresionNoModFile" type="preparedStatement" return="rowCount">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
+  <SqlMethod name="getModulesInDevelopment" type="preparedStatement" return="multiple">
     <Sql>
-         update ad_textinterfaces
-           set isUsed = 'Y'
-         where ad_textinterfaces_id = ( select min(A.ad_textinterfaces_id)
-                                          from (select B.*
-                                                from (select min(t.ad_textinterfaces_id) AS ad_textinterfaces_id, 2 AS o
-                                                          from ad_textinterfaces t, ad_textinterfaces_trl trl
-                                                         where trl.text     = ?
-                                                           and t.filename = ?
-                                                           and trl.ad_textinterfaces_id = t.ad_textinterfaces_id
-                                                           and trl.ad_language = ?
-                                                         order by o) B
-                                                 where ad_textinterfaces_id is not null) A)
+      select AD_Module_id as id, name, javapackage, AD_LANGUAGE as lang
+        from AD_Module
+       where isInDevelopment = 'Y'
+       order by 1
     </Sql>
-    
-    <Parameter name="text"/>
-    <Parameter name="filename"/>
-    <Parameter name="moduleLang"/>
   </SqlMethod>
   
-  <SqlMethod name="existsExpresionModNoFile" type="preparedStatement" return="rowCount">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-         update ad_textinterfaces
-           set isUsed = 'Y'
-         where ad_textinterfaces_id = ( select min(A.ad_textinterfaces_id)
-                                          from (select B.*
-                                                from (select min(ad_textinterfaces_id) AS ad_textinterfaces_id, 3 AS o
-                                                          from ad_textinterfaces t,
-                                                               ad_module m
-                                                         where text = ?
-                                                           and filename is null
-                                                           and m.ad_module_id = t.ad_module_id
-                                                           and m.ad_language = ?
-                                                         order by o) B
-                                                 where ad_textinterfaces_id is not null) A)
-    </Sql>
-    <Parameter name="text"/>
-    <Parameter name="moduleLang"/>
-  </SqlMethod>
-  
-  <SqlMethod name="existsExpresionNoModNoFile" type="preparedStatement" return="rowCount">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-         update ad_textinterfaces
-           set isUsed = 'Y'
-         where ad_textinterfaces_id = ( select min(A.ad_textinterfaces_id)
-                                          from (select B.*
-                                                from (select min(t.ad_textinterfaces_id) AS ad_textinterfaces_id, 4 AS o
-                                                          from ad_textinterfaces t, ad_textinterfaces_trl trl
-                                                         where trl.text = ?
-                                                           and t.filename is null
-                                                           and trl.ad_textinterfaces_id = t.ad_textinterfaces_id
-                                                           and trl.ad_language = ?
-                                                         order by o) B
-                                                 where ad_textinterfaces_id is not null) A)
-    </Sql>
-    
-    <Parameter name="text"/>
-    <Parameter name="moduleLang"/>
-  </SqlMethod>
-
-   <SqlMethod name="getModuleID" type="preparedStatement" return="string">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-      SELECT AD_MODULE_ID
-        FROM AD_MODULE
-       WHERE JavaPackage = ?
-    </Sql>
-    <Parameter name="name"/>
-  </SqlMethod>
-  
-  <SqlMethod name="getModuleLang" type="preparedStatement" return="string">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-      SELECT AD_LANGUAGE
-        FROM AD_MODULE
-       WHERE AD_MODULE_ID = ?
-    </Sql>
-    <Parameter name="module_id"/>
-  </SqlMethod>
-  
-  <SqlMethod name="isInDevelopmentModule" type="preparedStatement" return="boolean">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-      SELECT COUNT(*) AS TOTAL
-        FROM AD_MODULE
-       WHERE AD_MODULE_ID = ?
-         AND ISINDEVELOPMENT = 'Y'
-    </Sql>
-    <Parameter name="name"/>
-  </SqlMethod>
-
-  <SqlMethod name="isInDevelopmentModulePack" type="preparedStatement" return="boolean">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-      SELECT COUNT(*) AS TOTAL
-        FROM AD_MODULE
-       WHERE JAVAPACKAGE = ?
-         AND ISINDEVELOPMENT = 'Y'
-    </Sql>
-    <Parameter name="pack"/>
-  </SqlMethod>
-  
-  <SqlMethod name="insert" type="preparedStatement" return="rowCount">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
+  <SqlMethod name="insert" type="preparedStatement" connection="true" return="rowCount">
     <Sql>
       insert into ad_textinterfaces
         (AD_TEXTINTERFACES_ID, AD_CLIENT_ID, AD_ORG_ID, ISACTIVE, CREATED, CREATEDBY, UPDATED, UPDATEDBY, TEXT, FILENAME, ISUSED, AD_MODULE_ID)
@@ -191,32 +50,19 @@
     <Parameter name="module"/>
   </SqlMethod>
 
-  <SqlMethod name="update" type="preparedStatement" return="rowCount">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
+  <SqlMethod name="update" type="preparedStatement" connection="true" return="rowCount">
     <Sql>
       update ad_textinterfaces t
-         set isUsed='Y'
+         set isUsed='Y', updated=now(), updatedBy = '0'
        where ad_textinterfaces_id = ?
-         and exists (select 1 
-                       from ad_module m
-                      where m.ad_module_id = t.ad_module_id
-                        and m.isInDevelopment = 'Y')
     </Sql>
     <Parameter name="adTextinterfacesId"/>
   </SqlMethod>
 
-  <SqlMethod name="systemLanguage" type="preparedStatement" return="multiple">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
-    <Sql>
-      SELECT AD_LANGUAGE AS NAME FROM AD_LANGUAGE WHERE ISACTIVE='Y' AND ISSYSTEMLANGUAGE='Y' AND ISBASELANGUAGE='N'
-    </Sql>
-  </SqlMethod>
-  
   <SqlMethod name="clean" type="preparedStatement" return="rowCount">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
     <Sql>
       update ad_textinterfaces t
-         set isUsed='N'
+         set isUsed='N', updated=now(), updatedBy = '0'
        where exists (select 1 
                        from ad_module m
                       where m.ad_module_id = t.ad_module_id
@@ -225,7 +71,6 @@
   </SqlMethod>
   
   <SqlMethod name="remove" type="preparedStatement" return="rowCount">
-    <SqlMethodComment>Return the translation of a text</SqlMethodComment>
     <Sql>
       delete from ad_textinterfaces t
        where isUsed='N'
--- a/src/build.xml	Wed Aug 30 14:10:12 2017 +0200
+++ b/src/build.xml	Fri Aug 11 16:21:42 2017 +0200
@@ -40,7 +40,6 @@
 compile: compiles specified WAD window and src and also translates.
 compile.complete: compiles all WAD windows and src and also translates.
 translate: Translate the modified files.
-compile.translate: Translate the specified extensions files.
 installWebService: install the web services configuration file.
 uninstallWebService: uninstall the web services configuration file.
 copy.files: copy the local files to the context.
@@ -70,7 +69,6 @@
   <property name="tab" value="%" />
 
   <property name="base.translate.structure" value="org/openbravo/erpWindows" />
-  <property name="extension" value="html" />
   <property name="src" value="." />
   <property name="tr" value="yes" />
 
@@ -124,7 +122,7 @@
 
   <target name="trl.clean" if="translation">
     <java classname="org.openbravo.translate.Translate" jvm="${env.JAVA_HOME}/bin/java" fork="yes" maxmemory="${build.maxmemory}">
-      <arg line="clean '${base.config}/Openbravo.properties'" />
+      <arg line="'${base.config}/Openbravo.properties' clean" />
       <classpath refid="project.class.path" />
       <syspropertyset>
          <propertyref name="java.security.egd" />
@@ -134,7 +132,7 @@
   
   <target name="trl.remove.unused" if="translation">
     <java classname="org.openbravo.translate.Translate" jvm="${env.JAVA_HOME}/bin/java" fork="yes" maxmemory="${build.maxmemory}">
-      <arg line="remove '${base.config}/Openbravo.properties'" />
+      <arg line="'${base.config}/Openbravo.properties' remove" />
       <classpath refid="project.class.path" />
       <syspropertyset>
          <propertyref name="java.security.egd" />
@@ -689,42 +687,8 @@
   </target>
 
   <target name="translate" if="translation">
-    <antcall inheritall="true" inheritrefs="true" target="compile.translate">
-      <param name="extension" value="html" />
-    </antcall>
-    <antcall inheritall="true" inheritrefs="true" target="compile.translate">
-      <param name="extension" value="fo" />
-    </antcall>
-    <antcall inheritall="true" inheritrefs="true" target="compile.translate">
-      <param name="extension" value="srpt" />
-    </antcall>
-    <antcall inheritall="true" inheritrefs="true" target="compile.translate">
-      <param name="extension" value="jrxml" />
-    </antcall>
-    <antcall inheritall="true" inheritrefs="true" target="translate.modules" />
-  </target>
-
-  <target name="compile" depends="wad,compileSqlc, postsrc, translate">
-  </target>
-
-  <target name="translate.modules" if="translation">
-    <antcall inheritall="true" inheritrefs="true" target="compile.translate.modules">
-      <param name="extension" value="html" />
-    </antcall>
-    <antcall inheritall="true" inheritrefs="true" target="compile.translate.modules">
-      <param name="extension" value="fo" />
-    </antcall>
-    <antcall inheritall="true" inheritrefs="true" target="compile.translate.modules">
-      <param name="extension" value="srpt" />
-    </antcall>
-    <antcall inheritall="true" inheritrefs="true" target="compile.translate.modules">
-      <param name="extension" value="jrxml" />
-    </antcall>
-  </target>
-
-  <target name="compile.translate.modules">
     <java classname="org.openbravo.translate.Translate" jvm="${env.JAVA_HOME}/bin/java" fork="yes" maxmemory="${build.maxmemory}">
-      <arg line="'${base.config}/Openbravo.properties' ${extension} '${base.modules}' '*/src'" />
+      <arg line="'${base.config}/Openbravo.properties' '${basedir}/..'" />
       <classpath refid="project.class.path" />
       <syspropertyset>
          <propertyref name="java.security.egd" />
@@ -732,6 +696,9 @@
     </java>
   </target>
 
+  <target name="compile" depends="wad,compileSqlc, postsrc, translate">
+  </target>
+
   <target name="installWebService" depends="init" if="wsdd">
     <taskdef name="wsAdmin" classname="org.openbravo.erpCommon.utility.WebServiceAdmin">
       <classpath refid="project.class.path" />
@@ -750,16 +717,6 @@
              action="undeploy"/>
   </target>
 
-  <target name="compile.translate">
-    <java classname="org.openbravo.translate.Translate" jvm="${env.JAVA_HOME}/bin/java" fork="yes" maxmemory="${build.maxmemory}">
-      <arg line="'${base.config}/Openbravo.properties' ${extension} '${basedir}'" />
-      <classpath refid="project.class.path" />
-      <syspropertyset>
-         <propertyref name="java.security.egd" />
-      </syspropertyset>
-    </java>
-  </target>
-
   <!-- Note (see issue 15709 for details):
        orphan jarfiles are not deleted from the webapp as deployed in tomcat when
        this rebuild is running inside the rebuild UI (started from MMC). See also the related