From a8cc4a00f90776894c12a3b7c4ee38017d20cae7 Mon Sep 17 00:00:00 2001 From: Marco Antonio Nina Date: Wed, 26 Sep 2012 10:15:20 -0400 Subject: [PATCH] =?UTF-8?q?BUG=209720=20Proceso=20de=20actualizaci=C3=B3e?= =?UTF-8?q?=20ProcessMaker=20es=20lento=20SOLVED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ProcessMaker update process is slow for query. - Was optimized queries and was modified the rebuild "Table content". --- workflow/engine/bin/cli.php | 2 +- workflow/engine/bin/tasks/cliUpgrade.php | 131 ++++++++--------- workflow/engine/classes/class.wsTools.php | 74 ++++++++-- workflow/engine/classes/model/Content.php | 133 +++++++++++++++--- workflow/engine/classes/model/Language.php | 3 - .../engine/methods/setup/languages_Import.php | 4 + .../setupSchemas/app_cache_view_insert.sql | 8 +- 7 files changed, 252 insertions(+), 103 deletions(-) diff --git a/workflow/engine/bin/cli.php b/workflow/engine/bin/cli.php index 9ff1c7147..ca040b93c 100755 --- a/workflow/engine/bin/cli.php +++ b/workflow/engine/bin/cli.php @@ -50,7 +50,7 @@ $e_all = defined('E_DEPRECATED') ? E_ALL & ~E_DEPRECATED : E_ALL; $e_all = defined('E_STRICT') ? E_ALL & ~E_STRICT : $e_all; - $e_all = $e_all & ~E_NOTICE; // don't notices + //$e_all = $e_all & ~E_NOTICE; // don't notices // Do not change any of these settings directly, use env.ini instead ini_set('display_errors', $config['debug']); diff --git a/workflow/engine/bin/tasks/cliUpgrade.php b/workflow/engine/bin/tasks/cliUpgrade.php index c6d9b482e..74e2176ff 100755 --- a/workflow/engine/bin/tasks/cliUpgrade.php +++ b/workflow/engine/bin/tasks/cliUpgrade.php @@ -30,12 +30,13 @@ G::LoadClass("wsTools"); CLI::taskName('upgrade'); CLI::taskDescription(<< $workspace) { + try { + CLI::logging("Upgrading workspaces ($index/$count): " . CLI::info($workspace->name) . "\n"); + $workspace->upgrade($first, $buildCacheView); + $workspace->close(); + $first = false; + } catch (Exception $e) { + CLI::logging("Errors upgrading workspace " . CLI::info($workspace->name) . ": " . CLI::error($e->getMessage()) . "\n"); + $errors = true; + } } - if (!(empty($checksum['missing']) || empty($checksum['diff']))) { - if (!CLI::question("Integrity check failed, do you want to continue the upgrade?")) { - CLI::logging("Upgrade failed\n"); - die(); - } + if ($errors) { + CLI::logging("Upgrade finished but there were errors upgrading workspaces.\n"); + CLI::logging(CLI::error("Please check the log above to correct any issues.")."\n"); + } else { + CLI::logging("Upgrade successful\n"); } - } - CLI::logging("Clearing cache...\n"); - if(defined('PATH_C')) - rm_dir(PATH_C, true); - $workspaces = get_workspaces_from_args($args); - $count = count($workspaces); - $first = true; - $errors = false; - foreach ($workspaces as $index => $workspace) { - try { - CLI::logging("Upgrading workspaces ($index/$count): " . CLI::info($workspace->name) . "\n"); - $workspace->upgrade($first); - $workspace->close(); - $first = false; - } catch (Exception $e) { - CLI::logging("Errors upgrading workspace " . CLI::info($workspace->name) . ": " . CLI::error($e->getMessage()) . "\n"); - $errors = true; - } - } - if ($errors) { - CLI::logging("Upgrade finished but there were errors upgrading workspaces.\n"); - CLI::logging(CLI::error("Please check the log above to correct any issues.")."\n"); - } else { - CLI::logging("Upgrade successful\n"); - } } -?> diff --git a/workflow/engine/classes/class.wsTools.php b/workflow/engine/classes/class.wsTools.php index 5b2404f66..3e3b6219b 100755 --- a/workflow/engine/classes/class.wsTools.php +++ b/workflow/engine/classes/class.wsTools.php @@ -49,19 +49,42 @@ class workspaceTools { return (file_exists($this->path) && file_exists($this->dbPath)); } - /** - * Upgrade this workspace to the latest system version - * - * @param bool $first true if this is the first workspace to be upgrade - */ - public function upgrade($first = false) { - CLI::logging("> Updating database...\n"); - $this->upgradeDatabase(); - CLI::logging("> Updating translations...\n"); - $this->upgradeTranslation($first); - CLI::logging("> Updating cache view...\n"); - $this->upgradeCacheView(); - } + /** + * Upgrade this workspace to the latest system version + * + * @param bool $first true if this is the first workspace to be upgrade + */ + public function upgrade($first=false, $buildCacheView=false) + { + $start = microtime(true); + CLI::logging("> Updating database...\n"); + $this->upgradeDatabase(); + $this->upgradeDatabase(); + $stop = microtime(true); + $final = $stop - $start; + CLI::logging("<*> Process Updating database carried out in $final seconds.\n"); + + $start = microtime(true); + CLI::logging("> Updating translations...\n"); + $this->upgradeTranslation($first); + $stop = microtime(true); + $final = $stop - $start; + CLI::logging("<*> Process Updating translations carried out in $final seconds.\n"); + + $start = microtime(true); + CLI::logging("> Updating Content...\n"); + $this->upgradeContent(); + $stop = microtime(true); + $final = $stop - $start; + CLI::logging("<*> Process Updating Content carried out in $final seconds.\n"); + + $start = microtime(true); + CLI::logging("> Updating cache view...\n"); + $this->upgradeCacheView($buildCacheView); + $stop = microtime(true); + $final = $stop - $start; + CLI::logging("<*> Process Updating cache view carried out in $final seconds.\n"); + } /** * Scan the db.php file for database information and return it as an array @@ -264,6 +287,27 @@ class workspaceTools { $this->initPropelRoot = false; } + /** + * Upgrade this workspace Content. + * + */ + public function upgradeContent() { + $this->initPropel(true); + require_once('classes/model/Language.php'); + G::LoadThirdParty('pear/json', 'class.json'); + $lang = array(); + foreach (System::listPoFiles() as $poFile) { + $poName = basename($poFile); + $names = explode(".", basename($poFile)); + $extension = array_pop($names); + $langid = array_pop($names); + $arrayLang[] = $langid; + } + require_once('classes/model/Content.php'); + $regenerateContent = new Content(); + $regenerateContent->regenerateContent($arrayLang); + } + /** * Upgrade this workspace translations from all avaliable languages. * @@ -387,7 +431,7 @@ class workspaceTools { * @param bool $checkOnly only check if the upgrade is needed if true * @param string $lang not currently used */ - public function upgradeCacheView($fill = true) { + public function upgradeCacheView($fill=true) { $this->initPropel(true); $lang = "en"; @@ -430,7 +474,7 @@ class workspaceTools { $triggers[] = $appCache->triggerContentUpdate($lang, $checkOnly); if ($fill) { - CLI::logging("-> Filling cache view\n"); + CLI::logging("-> Rebuild Cache View\n"); //build using the method in AppCacheView Class $res = $appCache->fillAppCacheView($lang); //set status in config table diff --git a/workflow/engine/classes/model/Content.php b/workflow/engine/classes/model/Content.php index 1103e95ff..574f51dab 100755 --- a/workflow/engine/classes/model/Content.php +++ b/workflow/engine/classes/model/Content.php @@ -39,6 +39,12 @@ require_once 'classes/model/om/BaseContent.php'; */ class Content extends BaseContent { + public $langs; + public $rowsProcessed; + public $rowsInserted; + public $rowsUnchanged; + public $rowsClustered; + public $langsAsoc; /* * Load the content row specified by the parameters: * @param string $sUID @@ -264,24 +270,117 @@ class Content extends BaseContent { throw($oError); } } - - function regenerateContent($langId) - { - $oCriteria = new Criteria('workflow'); - $oCriteria->addSelectColumn(ContentPeer::CON_CATEGORY); - $oCriteria->addSelectColumn(ContentPeer::CON_ID); - $oCriteria->addSelectColumn(ContentPeer::CON_VALUE); - $oCriteria->add(ContentPeer::CON_LANG, 'en'); - $oCriteria->add(ContentPeer::CON_VALUE, '', Criteria::NOT_EQUAL ); - $oDataset = ContentPeer::doSelectRS($oCriteria); - $oDataset->setFetchmode(ResultSet::FETCHMODE_ASSOC); - $oDataset->next(); - $oContent = new Content(); - while ($aRow = $oDataset->getRow()) { - $oContent->load($aRow['CON_CATEGORY'], '', $aRow['CON_ID'], $langId); - $oDataset->next(); + + /* + * Regenerate Table Content + * + * @param array $langs + */ + function regenerateContent($langId) + { + //Search the language + $key = array_search('en',$langs); + if ($key === false) { + $key = array_search(SYS_LANG,$langs); + if ($key === false) { + $key = '0'; + } + } + $this->langsAsoc = array(); + foreach ($langs as $key=>$value) { + $this->langsAsoc[$value] = $value; + } + + $this->langs = $langs; + $this->rowsProcessed = 0; + $this->rowsInserted = 0; + $this->rowsUnchanged = 0; + $this->rowsClustered = 0; + + $con = Propel::getConnection('workflow'); + $sql = " SELECT DISTINCT CON_LANG + FROM CONTENT "; + $stmt = $con->createStatement(); + $rs = $stmt->executeQuery($sql, ResultSet::FETCHMODE_ASSOC); + while ($rs->next()) { + $row = $rs->getRow(); + $language = $row['CON_LANG']; + if (array_search($row['CON_LANG'],$langs) === false) { + Content::removeLanguageContent($row['CON_LANG']); + } + } + + $sql = " SELECT CON_ID, CON_CATEGORY, CON_LANG, CON_PARENT, CON_VALUE + FROM CONTENT + ORDER BY CON_ID, CON_CATEGORY, CON_LANG, CON_PARENT "; + mysql_query('SET OPTION SQL_BIG_SELECTS=1'); + $result = mysql_unbuffered_query($sql); + $list = array(); + $default = array(); + $sw = array('CON_ID'=>'','CON_CATEGORY'=>'','CON_PARENT'=>''); + while ($row = mysql_fetch_assoc($result)) { + if ($sw['CON_ID'] == $row['CON_ID'] && $sw['CON_CATEGORY'] == $row['CON_CATEGORY'] && $sw['CON_PARENT'] == $row['CON_PARENT']) { + $list[] = $row; + if ($sw['CON_LANG'] == $langs[$key]) { + $default = $row; + } + } else { + $this->rowsClustered++; + if (count($langs) != count($list)) { + $this->checkLanguage($list, $default); + } else { + $this->rowsUnchanged = $this->rowsUnchanged + count($langs); + } + $sw = array(); + $sw['CON_ID'] = $row['CON_ID']; + $sw['CON_CATEGORY'] = $row['CON_CATEGORY']; + $sw['CON_LANG'] = $row['CON_LANG']; + $sw['CON_PARENT'] = $row['CON_PARENT']; + unset($list); + unset($default); + $list = array(); + $default = array(); + $list[] = $row; + } + $this->rowsProcessed++; + } + if (count($langs) != count($list)) { + $this->checkLanguage($list, $default); + } else { + $this->rowsUnchanged = $this->rowsUnchanged + count($langs); + } + mysql_free_result($result); + $total = $this->rowsProcessed + $this->rowsInserted; + + CLI::logging("Rows Processed ---> $this->rowsProcessed ..... \n"); + CLI::logging("Rows Clustered ---> $this->rowsClustered ..... \n"); + CLI::logging("Rows Unchanged ---> $this->rowsUnchanged ..... \n"); + CLI::logging("Rows Inserted ---> $this->rowsInserted ..... \n"); + CLI::logging("Rows Total ---> $total ..... \n"); + } + + function checkLanguage($content, $default) + { + if (count($content)>0) { + $langs = $this->langs; + $langsAsoc = $this->langsAsoc; + //Element default + $default = (count($default)>0) ? $default : $content[0]; + foreach ($content as $key => $value) { + unset($langsAsoc[$value['CON_LANG']]); + } + foreach ($langsAsoc as $key) { + $this->rowsInserted++; + $this->fastInsertContent( + $default['CON_CATEGORY'], + $default['CON_PARENT'], + $default['CON_ID'], + $lang, + $default['CON_VALUE'] + ); + } + } } - } function removeLanguageContent($lanId) { try { diff --git a/workflow/engine/classes/model/Language.php b/workflow/engine/classes/model/Language.php index e626e1db2..2f60442be 100755 --- a/workflow/engine/classes/model/Language.php +++ b/workflow/engine/classes/model/Language.php @@ -268,9 +268,6 @@ class Language extends BaseLanguage { $trn->generateFileTranslation($LOCALE); $trn->addTranslationEnvironment($LOCALE, $POHeaders, $countItemsSuccess); - $content = new Content(); - $content->regenerateContent($languageID); - //fill the results $results = new stdClass(); $results->recordsCount = $countItems; diff --git a/workflow/engine/methods/setup/languages_Import.php b/workflow/engine/methods/setup/languages_Import.php index e201761cc..97b3e0630 100755 --- a/workflow/engine/methods/setup/languages_Import.php +++ b/workflow/engine/methods/setup/languages_Import.php @@ -71,6 +71,10 @@ try { $configuration = new Configurations; $importResults = $language->import($languageFile); + G::LoadClass("wsTools"); + $renegerateContent = new workspaceTools(); + $renegerateContent->upgradeContent(); + $result->msg = G::LoadTranslation('IMPORT_LANGUAGE_SUCCESS') . "\n"; $result->msg .= "PO File num. records: " . $importResults->recordsCount . "\n"; $result->msg .= "Success Records: " . $importResults->recordsCountSuccess . "\n"; diff --git a/workflow/engine/methods/setup/setupSchemas/app_cache_view_insert.sql b/workflow/engine/methods/setup/setupSchemas/app_cache_view_insert.sql index 02cf0f165..ed26d9208 100755 --- a/workflow/engine/methods/setup/setupSchemas/app_cache_view_insert.sql +++ b/workflow/engine/methods/setup/setupSchemas/app_cache_view_insert.sql @@ -35,14 +35,14 @@ SELECT APPLICATION.APP_UPDATE_DATE, APP_DELEGATION.APP_OVERDUE_PERCENTAGE FROM - APPLICATION + APPLICATION FORCE INDEX (PRIMARY) LEFT JOIN APP_DELEGATION ON (APPLICATION.APP_UID=APP_DELEGATION.APP_UID) LEFT JOIN TASK ON (APP_DELEGATION.TAS_UID=TASK.TAS_UID) LEFT JOIN USERS ON (APP_DELEGATION.USR_UID=USERS.USR_UID) LEFT JOIN APP_THREAD ON (APPLICATION.APP_UID=APP_THREAD.APP_UID AND APP_DELEGATION.DEL_THREAD=APP_THREAD.APP_THREAD_INDEX) - LEFT JOIN CONTENT APP_TITLE ON (APPLICATION.APP_UID=APP_TITLE.CON_ID AND APP_TITLE.CON_CATEGORY='APP_TITLE' AND APP_TITLE.CON_LANG = @DEFAULT_LANG) - LEFT JOIN CONTENT PRO_TITLE ON (APPLICATION.PRO_UID=PRO_TITLE.CON_ID AND PRO_TITLE.CON_CATEGORY='PRO_TITLE' AND PRO_TITLE.CON_LANG = @DEFAULT_LANG) - LEFT JOIN CONTENT TAS_TITLE ON (APP_DELEGATION.TAS_UID=TAS_TITLE.CON_ID AND TAS_TITLE.CON_CATEGORY='TAS_TITLE' AND TAS_TITLE.CON_LANG = @DEFAULT_LANG) + LEFT JOIN CONTENT APP_TITLE FORCE INDEX (indexUid) ON (APPLICATION.APP_UID=APP_TITLE.CON_ID AND APP_TITLE.CON_CATEGORY='APP_TITLE' AND APP_TITLE.CON_LANG = @DEFAULT_LANG) + LEFT JOIN CONTENT PRO_TITLE FORCE INDEX (indexUid) ON (APPLICATION.PRO_UID=PRO_TITLE.CON_ID AND PRO_TITLE.CON_CATEGORY='PRO_TITLE' AND PRO_TITLE.CON_LANG = @DEFAULT_LANG) + LEFT JOIN CONTENT TAS_TITLE FORCE INDEX (indexUid) ON (APP_DELEGATION.TAS_UID=TAS_TITLE.CON_ID AND TAS_TITLE.CON_CATEGORY='TAS_TITLE' AND TAS_TITLE.CON_LANG = @DEFAULT_LANG) LEFT JOIN APP_DELEGATION APP_PREV_DEL ON (APPLICATION.APP_UID=APP_PREV_DEL.APP_UID AND APP_PREV_DEL.DEL_INDEX=APP_DELEGATION.DEL_PREVIOUS) LEFT JOIN USERS APP_LAST_USER ON (APP_PREV_DEL.USR_UID=APP_LAST_USER.USR_UID) WHERE