BUG 9720 Proceso de actualizacióe ProcessMaker es lento SOLVED

- ProcessMaker update process is slow for query.
- Was optimized queries and was modified the  rebuild "Table content".

workflow/engine/bin/cli.php

@@ -50,7 +50,7 @@
   
   $e_all  = defined('E_DEPRECATED') ? E_ALL & ~E_DEPRECATED : E_ALL;
   $e_all  = defined('E_STRICT')     ? E_ALL & ~E_STRICT     : $e_all;
-  $e_all  = $e_all & ~E_NOTICE; // don't notices
+  //$e_all  = $e_all & ~E_NOTICE; // don't notices
 
   // Do not change any of these settings directly, use env.ini instead
   ini_set('display_errors', $config['debug']);

workflow/engine/bin/tasks/cliUpgrade.php

@@ -36,6 +36,7 @@ CLI::taskDescription(<<<EOT
     workspaces are upgraded to the current version.
 EOT
 );
+CLI::taskOpt("buildAPV", "If the option is enabled, performs the Build Cache View.", "APV", "buildAPV");
 CLI::taskRun(run_upgrade);
 
 
@@ -45,19 +46,22 @@ CLI::taskRun(run_upgrade);
  * @param  string $filename directory or file to remove
  * @param  bool $filesOnly either to remove the containing directory as well or not
  */
-function rm_dir($filename, $filesOnly = false) {
+function rm_dir($filename, $filesOnly = false)
+{
     if (is_file($filename)) {
         @unlink($filename) or CLI::logging(CLI::error("Could not remove file $filename")."\n");
     } else {
-    foreach(glob("$filename/*") as $f) {
+        foreach (glob("$filename/*") as $f) {
             rm_dir($f);
         }
-    if (!$filesOnly)
+        if (!$filesOnly) {
             @rmdir($filename) or CLI::logging(CLI::error("Could not remove directory $filename")."\n");
         }
+    }
 }
 
-function run_upgrade($command, $args) {
+function run_upgrade($command, $args)
+{
     CLI::logging("UPGRADE", PROCESSMAKER_PATH . "upgrade.log");
     CLI::logging("Checking files integrity...\n");
     $checksum = System::verifyChecksum();
@@ -70,13 +74,13 @@ function run_upgrade($command, $args) {
     } else {
         if (!empty($checksum['missing'])) {
             CLI::logging(CLI::error("The following files were not found in the installation:")."\n");
-      foreach($checksum['missing'] as $missing) {
+            foreach ($checksum['missing'] as $missing) {
                 CLI::logging(" $missing\n");
             }
         }
         if (!empty($checksum['diff'])) {
             CLI::logging(CLI::error("The following files have modifications:")."\n");
-      foreach($checksum['diff'] as $diff) {
+            foreach ($checksum['diff'] as $diff) {
                 CLI::logging(" $diff\n");
             }
         }
@@ -88,16 +92,18 @@ function run_upgrade($command, $args) {
         }
     }
     CLI::logging("Clearing cache...\n");
-  if(defined('PATH_C'))
+    if (defined('PATH_C')) {
         rm_dir(PATH_C, true);
+    }
     $workspaces = get_workspaces_from_args($args);
     $count = count($workspaces);
     $first = true;
     $errors = false;
+    $buildCacheView = array_key_exists("buildAPV", $command);
     foreach ($workspaces as $index => $workspace) {
         try {
             CLI::logging("Upgrading workspaces ($index/$count): " . CLI::info($workspace->name) . "\n");
-      $workspace->upgrade($first);
+            $workspace->upgrade($first, $buildCacheView);
             $workspace->close();
             $first = false;
         } catch (Exception $e) {
@@ -113,4 +119,3 @@ function run_upgrade($command, $args) {
     }
 }
 
-?>

workflow/engine/classes/class.wsTools.php

@@ -54,13 +54,36 @@ class workspaceTools {
      *
      * @param   bool $first true if this is the first workspace to be upgrade
      */
-  public function upgrade($first = false) {
+    public function upgrade($first=false, $buildCacheView=false)
+    {
+        $start = microtime(true);
         CLI::logging("> Updating database...\n");
         $this->upgradeDatabase();
+        $this->upgradeDatabase();
+        $stop = microtime(true);
+        $final = $stop - $start;
+        CLI::logging("<*>   Process Updating database carried out in $final seconds.\n");
+
+        $start = microtime(true);
         CLI::logging("> Updating translations...\n");
         $this->upgradeTranslation($first);
+        $stop = microtime(true);
+        $final = $stop - $start;
+        CLI::logging("<*>   Process Updating translations carried out in $final seconds.\n");
+
+        $start = microtime(true);
+        CLI::logging("> Updating Content...\n");
+        $this->upgradeContent();
+        $stop = microtime(true);
+        $final = $stop - $start;
+        CLI::logging("<*>   Process Updating Content carried out in $final seconds.\n");
+
+        $start = microtime(true);
         CLI::logging("> Updating cache view...\n");
-    $this->upgradeCacheView();
+        $this->upgradeCacheView($buildCacheView);
+        $stop = microtime(true);
+        $final = $stop - $start;
+        CLI::logging("<*>   Process Updating cache view carried out in $final seconds.\n");
     }
 
   /**
@@ -264,6 +287,27 @@ class workspaceTools {
     $this->initPropelRoot = false;
   }
 
+    /**
+     * Upgrade this workspace Content.
+     *
+     */
+    public function upgradeContent() {
+        $this->initPropel(true);
+        require_once('classes/model/Language.php');
+        G::LoadThirdParty('pear/json', 'class.json');
+        $lang = array();
+        foreach (System::listPoFiles() as $poFile) {
+            $poName = basename($poFile);
+            $names = explode(".", basename($poFile));
+            $extension = array_pop($names);
+            $langid = array_pop($names);
+            $arrayLang[] = $langid;
+        }
+        require_once('classes/model/Content.php');
+        $regenerateContent = new Content();
+        $regenerateContent->regenerateContent($arrayLang);
+    }
+
   /**
    * Upgrade this workspace translations from all avaliable languages.
    *
@@ -387,7 +431,7 @@ class workspaceTools {
    * @param   bool $checkOnly only check if the upgrade is needed if true
    * @param  string $lang not currently used
    */
-  public function upgradeCacheView($fill = true) {
+  public function upgradeCacheView($fill=true) {
     $this->initPropel(true);
 
     $lang = "en";
@@ -430,7 +474,7 @@ class workspaceTools {
     $triggers[] = $appCache->triggerContentUpdate($lang, $checkOnly);
 
     if ($fill) {
-      CLI::logging("-> Filling cache view\n");
+      CLI::logging("-> Rebuild Cache View\n");
       //build using the method in AppCacheView Class
       $res = $appCache->fillAppCacheView($lang);
       //set status in config table

workflow/engine/classes/model/Content.php

@@ -39,6 +39,12 @@ require_once 'classes/model/om/BaseContent.php';
  */
 class Content extends BaseContent {
   
+        public $langs;
+        public $rowsProcessed;
+        public $rowsInserted;
+        public $rowsUnchanged;
+        public $rowsClustered;
+        public $langsAsoc;
   /*
   * Load the content row specified by the parameters: 
   * @param string $sUID
@@ -265,21 +271,114 @@ class Content extends BaseContent {
     }
   }
 
+    /*
+    * Regenerate Table Content
+    *
+    * @param  array  $langs
+    */
     function regenerateContent($langId)
     {
-    $oCriteria = new Criteria('workflow');
+        //Search the language
-    $oCriteria->addSelectColumn(ContentPeer::CON_CATEGORY);
+        $key = array_search('en',$langs);
-    $oCriteria->addSelectColumn(ContentPeer::CON_ID);
+        if ($key === false) {
-    $oCriteria->addSelectColumn(ContentPeer::CON_VALUE);
+            $key = array_search(SYS_LANG,$langs);
-    $oCriteria->add(ContentPeer::CON_LANG, 'en');
+            if ($key === false) {
-    $oCriteria->add(ContentPeer::CON_VALUE, '', Criteria::NOT_EQUAL );
+                $key = '0';
-    $oDataset = ContentPeer::doSelectRS($oCriteria);
+            }
-    $oDataset->setFetchmode(ResultSet::FETCHMODE_ASSOC);
+        }
-    $oDataset->next();
+        $this->langsAsoc = array();
-    $oContent = new Content();
+        foreach ($langs as $key=>$value) {
-    while ($aRow = $oDataset->getRow()) {
+            $this->langsAsoc[$value] = $value;
-      $oContent->load($aRow['CON_CATEGORY'], '', $aRow['CON_ID'], $langId);
+        }
-      $oDataset->next();
+        
+        $this->langs = $langs;
+        $this->rowsProcessed = 0;
+        $this->rowsInserted  = 0;
+        $this->rowsUnchanged = 0;
+        $this->rowsClustered = 0;
+        
+        $con = Propel::getConnection('workflow');
+        $sql = " SELECT DISTINCT CON_LANG
+                FROM CONTENT ";
+        $stmt = $con->createStatement();
+        $rs = $stmt->executeQuery($sql, ResultSet::FETCHMODE_ASSOC);
+        while ($rs->next()) {
+            $row = $rs->getRow();
+            $language = $row['CON_LANG'];
+            if (array_search($row['CON_LANG'],$langs) === false) {
+                Content::removeLanguageContent($row['CON_LANG']);
+            }
+        }
+
+        $sql = " SELECT CON_ID, CON_CATEGORY, CON_LANG, CON_PARENT, CON_VALUE
+                FROM CONTENT
+                ORDER BY CON_ID, CON_CATEGORY, CON_LANG, CON_PARENT ";
+        mysql_query('SET OPTION SQL_BIG_SELECTS=1');
+        $result = mysql_unbuffered_query($sql);
+        $list = array();
+        $default = array();
+        $sw = array('CON_ID'=>'','CON_CATEGORY'=>'','CON_PARENT'=>'');
+        while ($row = mysql_fetch_assoc($result)) {
+            if ($sw['CON_ID'] == $row['CON_ID'] && $sw['CON_CATEGORY'] == $row['CON_CATEGORY'] && $sw['CON_PARENT'] == $row['CON_PARENT']) {
+                $list[] = $row;
+                if ($sw['CON_LANG'] == $langs[$key]) {
+                    $default = $row;
+                }
+            } else {
+                $this->rowsClustered++;
+                if (count($langs) != count($list)) {
+                    $this->checkLanguage($list, $default);
+                } else {
+                    $this->rowsUnchanged = $this->rowsUnchanged + count($langs);
+                }
+                $sw = array();
+                $sw['CON_ID'] = $row['CON_ID'];
+                $sw['CON_CATEGORY'] = $row['CON_CATEGORY'];
+                $sw['CON_LANG'] = $row['CON_LANG'];
+                $sw['CON_PARENT'] = $row['CON_PARENT'];
+                unset($list);
+                unset($default);
+                $list = array();
+                $default = array();
+                $list[] = $row;
+            }
+            $this->rowsProcessed++;
+        }
+        if (count($langs) != count($list)) {
+            $this->checkLanguage($list, $default);
+        } else {
+            $this->rowsUnchanged = $this->rowsUnchanged + count($langs);
+        }
+        mysql_free_result($result);
+        $total = $this->rowsProcessed + $this->rowsInserted;
+
+        CLI::logging("Rows Processed ---> $this->rowsProcessed ..... \n");
+        CLI::logging("Rows Clustered ---> $this->rowsClustered ..... \n");
+        CLI::logging("Rows Unchanged ---> $this->rowsUnchanged ..... \n");
+        CLI::logging("Rows Inserted  ---> $this->rowsInserted ..... \n");
+        CLI::logging("Rows Total     ---> $total ..... \n");
+    }
+
+    function checkLanguage($content, $default)
+    {
+        if (count($content)>0) {
+            $langs = $this->langs;
+            $langsAsoc = $this->langsAsoc;
+            //Element default
+            $default = (count($default)>0) ? $default : $content[0];
+            foreach ($content as $key => $value) {
+                unset($langsAsoc[$value['CON_LANG']]);
+            }
+            foreach ($langsAsoc as $key) {
+                $this->rowsInserted++;
+                $this->fastInsertContent(
+                    $default['CON_CATEGORY'],
+                    $default['CON_PARENT'],
+                    $default['CON_ID'],
+                    $lang,
+                    $default['CON_VALUE']
+                    );
+            }
         }
     }
 

workflow/engine/classes/model/Language.php

@@ -268,9 +268,6 @@ class Language extends BaseLanguage {
       $trn->generateFileTranslation($LOCALE);
       $trn->addTranslationEnvironment($LOCALE, $POHeaders, $countItemsSuccess);
 
-      $content = new Content();
-      $content->regenerateContent($languageID);
-
       //fill the results
       $results = new stdClass();
       $results->recordsCount        = $countItems;

workflow/engine/methods/setup/languages_Import.php

@@ -71,6 +71,10 @@ try {
   $configuration = new Configurations;
   $importResults = $language->import($languageFile);
   
+    G::LoadClass("wsTools");
+    $renegerateContent = new workspaceTools();
+    $renegerateContent->upgradeContent();
+  
   $result->msg = G::LoadTranslation('IMPORT_LANGUAGE_SUCCESS') . "\n";
   $result->msg .= "PO File num. records: " . $importResults->recordsCount . "\n";
   $result->msg .= "Success Records: " . $importResults->recordsCountSuccess . "\n";

workflow/engine/methods/setup/setupSchemas/app_cache_view_insert.sql

@@ -35,14 +35,14 @@ SELECT
   APPLICATION.APP_UPDATE_DATE,
   APP_DELEGATION.APP_OVERDUE_PERCENTAGE
 FROM
-  APPLICATION
+  APPLICATION FORCE INDEX (PRIMARY)
   LEFT JOIN APP_DELEGATION ON (APPLICATION.APP_UID=APP_DELEGATION.APP_UID)
   LEFT JOIN TASK ON (APP_DELEGATION.TAS_UID=TASK.TAS_UID)
   LEFT JOIN USERS ON (APP_DELEGATION.USR_UID=USERS.USR_UID)
   LEFT JOIN APP_THREAD ON (APPLICATION.APP_UID=APP_THREAD.APP_UID AND APP_DELEGATION.DEL_THREAD=APP_THREAD.APP_THREAD_INDEX)
-  LEFT JOIN CONTENT APP_TITLE ON (APPLICATION.APP_UID=APP_TITLE.CON_ID AND APP_TITLE.CON_CATEGORY='APP_TITLE' AND APP_TITLE.CON_LANG = @DEFAULT_LANG)
+  LEFT JOIN CONTENT APP_TITLE FORCE INDEX (indexUid) ON (APPLICATION.APP_UID=APP_TITLE.CON_ID AND APP_TITLE.CON_CATEGORY='APP_TITLE' AND APP_TITLE.CON_LANG = @DEFAULT_LANG)
-  LEFT JOIN CONTENT PRO_TITLE ON (APPLICATION.PRO_UID=PRO_TITLE.CON_ID AND PRO_TITLE.CON_CATEGORY='PRO_TITLE' AND PRO_TITLE.CON_LANG = @DEFAULT_LANG)
+  LEFT JOIN CONTENT PRO_TITLE FORCE INDEX (indexUid) ON (APPLICATION.PRO_UID=PRO_TITLE.CON_ID AND PRO_TITLE.CON_CATEGORY='PRO_TITLE' AND PRO_TITLE.CON_LANG = @DEFAULT_LANG)
-  LEFT JOIN CONTENT TAS_TITLE ON (APP_DELEGATION.TAS_UID=TAS_TITLE.CON_ID AND TAS_TITLE.CON_CATEGORY='TAS_TITLE' AND TAS_TITLE.CON_LANG = @DEFAULT_LANG)
+  LEFT JOIN CONTENT TAS_TITLE FORCE INDEX (indexUid) ON (APP_DELEGATION.TAS_UID=TAS_TITLE.CON_ID AND TAS_TITLE.CON_CATEGORY='TAS_TITLE' AND TAS_TITLE.CON_LANG = @DEFAULT_LANG)
   LEFT JOIN APP_DELEGATION APP_PREV_DEL ON (APPLICATION.APP_UID=APP_PREV_DEL.APP_UID AND APP_PREV_DEL.DEL_INDEX=APP_DELEGATION.DEL_PREVIOUS)
   LEFT JOIN USERS APP_LAST_USER ON (APP_PREV_DEL.USR_UID=APP_LAST_USER.USR_UID)
 WHERE