BUG 9720 Proceso de actualizacióe ProcessMaker es lento SOLVED

- ProcessMaker update process is slow for query.
- Was optimized queries and was modified the  rebuild "Table content".

workflow/engine/bin/cli.php

@@ -50,7 +50,7 @@
   
   $e_all  = defined('E_DEPRECATED') ? E_ALL & ~E_DEPRECATED : E_ALL;
   $e_all  = defined('E_STRICT')     ? E_ALL & ~E_STRICT     : $e_all;
-  $e_all  = $e_all & ~E_NOTICE; // don't notices
+  //$e_all  = $e_all & ~E_NOTICE; // don't notices
 
   // Do not change any of these settings directly, use env.ini instead
   ini_set('display_errors', $config['debug']);

workflow/engine/bin/tasks/cliUpgrade.php

@@ -30,12 +30,13 @@ G::LoadClass("wsTools");
 
 CLI::taskName('upgrade');
 CLI::taskDescription(<<<EOT
-  Upgrade workspaces.
+    Upgrade workspaces.
 
-  This command should be run after ProcessMaker files are upgraded so that all
+    This command should be run after ProcessMaker files are upgraded so that all
-  workspaces are upgraded to the current version.
+    workspaces are upgraded to the current version.
 EOT
 );
+CLI::taskOpt("buildAPV", "If the option is enabled, performs the Build Cache View.", "APV", "buildAPV");
 CLI::taskRun(run_upgrade);
 
 
@@ -45,72 +46,76 @@ CLI::taskRun(run_upgrade);
  * @param  string $filename directory or file to remove
  * @param  bool $filesOnly either to remove the containing directory as well or not
  */
-function rm_dir($filename, $filesOnly = false) {
+function rm_dir($filename, $filesOnly = false)
-  if (is_file($filename)) {
+{
-    @unlink($filename) or CLI::logging(CLI::error("Could not remove file $filename")."\n");
+    if (is_file($filename)) {
-  } else {
+        @unlink($filename) or CLI::logging(CLI::error("Could not remove file $filename")."\n");
-    foreach(glob("$filename/*") as $f) {
+    } else {
-      rm_dir($f);
+        foreach (glob("$filename/*") as $f) {
+            rm_dir($f);
+        }
+        if (!$filesOnly) {
+            @rmdir($filename) or CLI::logging(CLI::error("Could not remove directory $filename")."\n");
+        }
     }
-    if (!$filesOnly)
-      @rmdir($filename) or CLI::logging(CLI::error("Could not remove directory $filename")."\n");
-  }
 }
 
-function run_upgrade($command, $args) {
+function run_upgrade($command, $args)
-  CLI::logging("UPGRADE", PROCESSMAKER_PATH . "upgrade.log");
+{
-  CLI::logging("Checking files integrity...\n");
+    CLI::logging("UPGRADE", PROCESSMAKER_PATH . "upgrade.log");
-  $checksum = System::verifyChecksum();
+    CLI::logging("Checking files integrity...\n");
-  if ($checksum === false) {
+    $checksum = System::verifyChecksum();
-    CLI::logging(CLI::error("checksum.txt not found, integrity check is not possible") . "\n");
+    if ($checksum === false) {
-    if (!CLI::question("Integrity check failed, do you want to continue the upgrade?")) {
+        CLI::logging(CLI::error("checksum.txt not found, integrity check is not possible") . "\n");
-      CLI::logging("Upgrade failed\n");
+        if (!CLI::question("Integrity check failed, do you want to continue the upgrade?")) {
-      die();
+            CLI::logging("Upgrade failed\n");
+            die();
+        }
+    } else {
+        if (!empty($checksum['missing'])) {
+            CLI::logging(CLI::error("The following files were not found in the installation:")."\n");
+            foreach ($checksum['missing'] as $missing) {
+                CLI::logging(" $missing\n");
+            }
+        }
+        if (!empty($checksum['diff'])) {
+            CLI::logging(CLI::error("The following files have modifications:")."\n");
+            foreach ($checksum['diff'] as $diff) {
+                CLI::logging(" $diff\n");
+            }
+        }
+        if (!(empty($checksum['missing']) || empty($checksum['diff']))) {
+            if (!CLI::question("Integrity check failed, do you want to continue the upgrade?")) {
+                CLI::logging("Upgrade failed\n");
+                die();
+            }
+        }
     }
-  } else {
+    CLI::logging("Clearing cache...\n");
-    if (!empty($checksum['missing'])) {
+    if (defined('PATH_C')) {
-      CLI::logging(CLI::error("The following files were not found in the installation:")."\n");
+        rm_dir(PATH_C, true);
-      foreach($checksum['missing'] as $missing) {
-        CLI::logging(" $missing\n");
-      }
     }
-    if (!empty($checksum['diff'])) {
+    $workspaces = get_workspaces_from_args($args);
-      CLI::logging(CLI::error("The following files have modifications:")."\n");
+    $count = count($workspaces);
-      foreach($checksum['diff'] as $diff) {
+    $first = true;
-        CLI::logging(" $diff\n");
+    $errors = false;
-      }
+    $buildCacheView = array_key_exists("buildAPV", $command);
+    foreach ($workspaces as $index => $workspace) {
+        try {
+            CLI::logging("Upgrading workspaces ($index/$count): " . CLI::info($workspace->name) . "\n");
+            $workspace->upgrade($first, $buildCacheView);
+            $workspace->close();
+            $first = false;
+        } catch (Exception $e) {
+            CLI::logging("Errors upgrading workspace " . CLI::info($workspace->name) . ": " . CLI::error($e->getMessage()) . "\n");
+            $errors = true;
+        }
     }
-    if (!(empty($checksum['missing']) || empty($checksum['diff']))) {
+    if ($errors) {
-      if (!CLI::question("Integrity check failed, do you want to continue the upgrade?")) {
+        CLI::logging("Upgrade finished but there were errors upgrading workspaces.\n");
-        CLI::logging("Upgrade failed\n");
+        CLI::logging(CLI::error("Please check the log above to correct any issues.")."\n");
-        die();
+    } else {
-      }
+        CLI::logging("Upgrade successful\n");
     }
-  }
-  CLI::logging("Clearing cache...\n");
-  if(defined('PATH_C'))
-    rm_dir(PATH_C, true);
-  $workspaces = get_workspaces_from_args($args);
-  $count = count($workspaces);
-  $first = true;
-  $errors = false;
-  foreach ($workspaces as $index => $workspace) {
-    try {
-      CLI::logging("Upgrading workspaces ($index/$count): " . CLI::info($workspace->name) . "\n");
-      $workspace->upgrade($first);
-      $workspace->close();
-      $first = false;
-    } catch (Exception $e) {
-      CLI::logging("Errors upgrading workspace " . CLI::info($workspace->name) . ": " . CLI::error($e->getMessage()) . "\n");
-      $errors = true;
-    }
-  }
-  if ($errors) {
-    CLI::logging("Upgrade finished but there were errors upgrading workspaces.\n");
-    CLI::logging(CLI::error("Please check the log above to correct any issues.")."\n");
-  } else {
-    CLI::logging("Upgrade successful\n");
-  }
 }
 
-?>

workflow/engine/classes/class.wsTools.php

@@ -49,19 +49,42 @@ class workspaceTools {
     return (file_exists($this->path) && file_exists($this->dbPath));
   }
 
-  /**
+    /**
-   * Upgrade this workspace to the latest system version
+     * Upgrade this workspace to the latest system version
-   *
+     *
-   * @param   bool $first true if this is the first workspace to be upgrade
+     * @param   bool $first true if this is the first workspace to be upgrade
-   */
+     */
-  public function upgrade($first = false) {
+    public function upgrade($first=false, $buildCacheView=false)
-    CLI::logging("> Updating database...\n");
+    {
-    $this->upgradeDatabase();
+        $start = microtime(true);
-    CLI::logging("> Updating translations...\n");
+        CLI::logging("> Updating database...\n");
-    $this->upgradeTranslation($first);
+        $this->upgradeDatabase();
-    CLI::logging("> Updating cache view...\n");
+        $this->upgradeDatabase();
-    $this->upgradeCacheView();
+        $stop = microtime(true);
-  }
+        $final = $stop - $start;
+        CLI::logging("<*>   Process Updating database carried out in $final seconds.\n");
+
+        $start = microtime(true);
+        CLI::logging("> Updating translations...\n");
+        $this->upgradeTranslation($first);
+        $stop = microtime(true);
+        $final = $stop - $start;
+        CLI::logging("<*>   Process Updating translations carried out in $final seconds.\n");
+
+        $start = microtime(true);
+        CLI::logging("> Updating Content...\n");
+        $this->upgradeContent();
+        $stop = microtime(true);
+        $final = $stop - $start;
+        CLI::logging("<*>   Process Updating Content carried out in $final seconds.\n");
+
+        $start = microtime(true);
+        CLI::logging("> Updating cache view...\n");
+        $this->upgradeCacheView($buildCacheView);
+        $stop = microtime(true);
+        $final = $stop - $start;
+        CLI::logging("<*>   Process Updating cache view carried out in $final seconds.\n");
+    }
 
   /**
    * Scan the db.php file for database information and return it as an array
@@ -264,6 +287,27 @@ class workspaceTools {
     $this->initPropelRoot = false;
   }
 
+    /**
+     * Upgrade this workspace Content.
+     *
+     */
+    public function upgradeContent() {
+        $this->initPropel(true);
+        require_once('classes/model/Language.php');
+        G::LoadThirdParty('pear/json', 'class.json');
+        $lang = array();
+        foreach (System::listPoFiles() as $poFile) {
+            $poName = basename($poFile);
+            $names = explode(".", basename($poFile));
+            $extension = array_pop($names);
+            $langid = array_pop($names);
+            $arrayLang[] = $langid;
+        }
+        require_once('classes/model/Content.php');
+        $regenerateContent = new Content();
+        $regenerateContent->regenerateContent($arrayLang);
+    }
+
   /**
    * Upgrade this workspace translations from all avaliable languages.
    *
@@ -387,7 +431,7 @@ class workspaceTools {
    * @param   bool $checkOnly only check if the upgrade is needed if true
    * @param  string $lang not currently used
    */
-  public function upgradeCacheView($fill = true) {
+  public function upgradeCacheView($fill=true) {
     $this->initPropel(true);
 
     $lang = "en";
@@ -430,7 +474,7 @@ class workspaceTools {
     $triggers[] = $appCache->triggerContentUpdate($lang, $checkOnly);
 
     if ($fill) {
-      CLI::logging("-> Filling cache view\n");
+      CLI::logging("-> Rebuild Cache View\n");
       //build using the method in AppCacheView Class
       $res = $appCache->fillAppCacheView($lang);
       //set status in config table

workflow/engine/classes/model/Content.php

@@ -39,6 +39,12 @@ require_once 'classes/model/om/BaseContent.php';
  */
 class Content extends BaseContent {
   
+        public $langs;
+        public $rowsProcessed;
+        public $rowsInserted;
+        public $rowsUnchanged;
+        public $rowsClustered;
+        public $langsAsoc;
   /*
   * Load the content row specified by the parameters: 
   * @param string $sUID
@@ -264,24 +270,117 @@ class Content extends BaseContent {
       throw($oError);
     }
   }
-  
+
-  function regenerateContent($langId)
+    /*
-  {
+    * Regenerate Table Content
-    $oCriteria = new Criteria('workflow');
+    *
-    $oCriteria->addSelectColumn(ContentPeer::CON_CATEGORY);
+    * @param  array  $langs
-    $oCriteria->addSelectColumn(ContentPeer::CON_ID);
+    */
-    $oCriteria->addSelectColumn(ContentPeer::CON_VALUE);
+    function regenerateContent($langId)
-    $oCriteria->add(ContentPeer::CON_LANG, 'en');
+    {
-    $oCriteria->add(ContentPeer::CON_VALUE, '', Criteria::NOT_EQUAL );
+        //Search the language
-    $oDataset = ContentPeer::doSelectRS($oCriteria);
+        $key = array_search('en',$langs);
-    $oDataset->setFetchmode(ResultSet::FETCHMODE_ASSOC);
+        if ($key === false) {
-    $oDataset->next();
+            $key = array_search(SYS_LANG,$langs);
-    $oContent = new Content();
+            if ($key === false) {
-    while ($aRow = $oDataset->getRow()) {
+                $key = '0';
-      $oContent->load($aRow['CON_CATEGORY'], '', $aRow['CON_ID'], $langId);
+            }
-      $oDataset->next();
+        }
+        $this->langsAsoc = array();
+        foreach ($langs as $key=>$value) {
+            $this->langsAsoc[$value] = $value;
+        }
+        
+        $this->langs = $langs;
+        $this->rowsProcessed = 0;
+        $this->rowsInserted  = 0;
+        $this->rowsUnchanged = 0;
+        $this->rowsClustered = 0;
+        
+        $con = Propel::getConnection('workflow');
+        $sql = " SELECT DISTINCT CON_LANG
+                FROM CONTENT ";
+        $stmt = $con->createStatement();
+        $rs = $stmt->executeQuery($sql, ResultSet::FETCHMODE_ASSOC);
+        while ($rs->next()) {
+            $row = $rs->getRow();
+            $language = $row['CON_LANG'];
+            if (array_search($row['CON_LANG'],$langs) === false) {
+                Content::removeLanguageContent($row['CON_LANG']);
+            }
+        }
+
+        $sql = " SELECT CON_ID, CON_CATEGORY, CON_LANG, CON_PARENT, CON_VALUE
+                FROM CONTENT
+                ORDER BY CON_ID, CON_CATEGORY, CON_LANG, CON_PARENT ";
+        mysql_query('SET OPTION SQL_BIG_SELECTS=1');
+        $result = mysql_unbuffered_query($sql);
+        $list = array();
+        $default = array();
+        $sw = array('CON_ID'=>'','CON_CATEGORY'=>'','CON_PARENT'=>'');
+        while ($row = mysql_fetch_assoc($result)) {
+            if ($sw['CON_ID'] == $row['CON_ID'] && $sw['CON_CATEGORY'] == $row['CON_CATEGORY'] && $sw['CON_PARENT'] == $row['CON_PARENT']) {
+                $list[] = $row;
+                if ($sw['CON_LANG'] == $langs[$key]) {
+                    $default = $row;
+                }
+            } else {
+                $this->rowsClustered++;
+                if (count($langs) != count($list)) {
+                    $this->checkLanguage($list, $default);
+                } else {
+                    $this->rowsUnchanged = $this->rowsUnchanged + count($langs);
+                }
+                $sw = array();
+                $sw['CON_ID'] = $row['CON_ID'];
+                $sw['CON_CATEGORY'] = $row['CON_CATEGORY'];
+                $sw['CON_LANG'] = $row['CON_LANG'];
+                $sw['CON_PARENT'] = $row['CON_PARENT'];
+                unset($list);
+                unset($default);
+                $list = array();
+                $default = array();
+                $list[] = $row;
+            }
+            $this->rowsProcessed++;
+        }
+        if (count($langs) != count($list)) {
+            $this->checkLanguage($list, $default);
+        } else {
+            $this->rowsUnchanged = $this->rowsUnchanged + count($langs);
+        }
+        mysql_free_result($result);
+        $total = $this->rowsProcessed + $this->rowsInserted;
+
+        CLI::logging("Rows Processed ---> $this->rowsProcessed ..... \n");
+        CLI::logging("Rows Clustered ---> $this->rowsClustered ..... \n");
+        CLI::logging("Rows Unchanged ---> $this->rowsUnchanged ..... \n");
+        CLI::logging("Rows Inserted  ---> $this->rowsInserted ..... \n");
+        CLI::logging("Rows Total     ---> $total ..... \n");
+    }
+
+    function checkLanguage($content, $default)
+    {
+        if (count($content)>0) {
+            $langs = $this->langs;
+            $langsAsoc = $this->langsAsoc;
+            //Element default
+            $default = (count($default)>0) ? $default : $content[0];
+            foreach ($content as $key => $value) {
+                unset($langsAsoc[$value['CON_LANG']]);
+            }
+            foreach ($langsAsoc as $key) {
+                $this->rowsInserted++;
+                $this->fastInsertContent(
+                    $default['CON_CATEGORY'],
+                    $default['CON_PARENT'],
+                    $default['CON_ID'],
+                    $lang,
+                    $default['CON_VALUE']
+                    );
+            }
+        }
     }
-  }
 
   function removeLanguageContent($lanId) {
     try {

workflow/engine/classes/model/Language.php

@@ -268,9 +268,6 @@ class Language extends BaseLanguage {
       $trn->generateFileTranslation($LOCALE);
       $trn->addTranslationEnvironment($LOCALE, $POHeaders, $countItemsSuccess);
 
-      $content = new Content();
-      $content->regenerateContent($languageID);
-
       //fill the results
       $results = new stdClass();
       $results->recordsCount        = $countItems;

workflow/engine/methods/setup/languages_Import.php

@@ -71,6 +71,10 @@ try {
   $configuration = new Configurations;
   $importResults = $language->import($languageFile);
   
+    G::LoadClass("wsTools");
+    $renegerateContent = new workspaceTools();
+    $renegerateContent->upgradeContent();
+  
   $result->msg = G::LoadTranslation('IMPORT_LANGUAGE_SUCCESS') . "\n";
   $result->msg .= "PO File num. records: " . $importResults->recordsCount . "\n";
   $result->msg .= "Success Records: " . $importResults->recordsCountSuccess . "\n";

workflow/engine/methods/setup/setupSchemas/app_cache_view_insert.sql

@@ -35,14 +35,14 @@ SELECT
   APPLICATION.APP_UPDATE_DATE,
   APP_DELEGATION.APP_OVERDUE_PERCENTAGE
 FROM
-  APPLICATION
+  APPLICATION FORCE INDEX (PRIMARY)
   LEFT JOIN APP_DELEGATION ON (APPLICATION.APP_UID=APP_DELEGATION.APP_UID)
   LEFT JOIN TASK ON (APP_DELEGATION.TAS_UID=TASK.TAS_UID)
   LEFT JOIN USERS ON (APP_DELEGATION.USR_UID=USERS.USR_UID)
   LEFT JOIN APP_THREAD ON (APPLICATION.APP_UID=APP_THREAD.APP_UID AND APP_DELEGATION.DEL_THREAD=APP_THREAD.APP_THREAD_INDEX)
-  LEFT JOIN CONTENT APP_TITLE ON (APPLICATION.APP_UID=APP_TITLE.CON_ID AND APP_TITLE.CON_CATEGORY='APP_TITLE' AND APP_TITLE.CON_LANG = @DEFAULT_LANG)
+  LEFT JOIN CONTENT APP_TITLE FORCE INDEX (indexUid) ON (APPLICATION.APP_UID=APP_TITLE.CON_ID AND APP_TITLE.CON_CATEGORY='APP_TITLE' AND APP_TITLE.CON_LANG = @DEFAULT_LANG)
-  LEFT JOIN CONTENT PRO_TITLE ON (APPLICATION.PRO_UID=PRO_TITLE.CON_ID AND PRO_TITLE.CON_CATEGORY='PRO_TITLE' AND PRO_TITLE.CON_LANG = @DEFAULT_LANG)
+  LEFT JOIN CONTENT PRO_TITLE FORCE INDEX (indexUid) ON (APPLICATION.PRO_UID=PRO_TITLE.CON_ID AND PRO_TITLE.CON_CATEGORY='PRO_TITLE' AND PRO_TITLE.CON_LANG = @DEFAULT_LANG)
-  LEFT JOIN CONTENT TAS_TITLE ON (APP_DELEGATION.TAS_UID=TAS_TITLE.CON_ID AND TAS_TITLE.CON_CATEGORY='TAS_TITLE' AND TAS_TITLE.CON_LANG = @DEFAULT_LANG)
+  LEFT JOIN CONTENT TAS_TITLE FORCE INDEX (indexUid) ON (APP_DELEGATION.TAS_UID=TAS_TITLE.CON_ID AND TAS_TITLE.CON_CATEGORY='TAS_TITLE' AND TAS_TITLE.CON_LANG = @DEFAULT_LANG)
   LEFT JOIN APP_DELEGATION APP_PREV_DEL ON (APPLICATION.APP_UID=APP_PREV_DEL.APP_UID AND APP_PREV_DEL.DEL_INDEX=APP_DELEGATION.DEL_PREVIOUS)
   LEFT JOIN USERS APP_LAST_USER ON (APP_PREV_DEL.USR_UID=APP_LAST_USER.USR_UID)
 WHERE