fernando/luos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PMCORE-1527 PMCORE-1421 - Regenerate PM Report Tables Asynchronously

Browse Source
This commit is contained in:
Roly Rudy Gutierrez Pinto
2020-06-11 10:44:47 -04:00
parent f1824ae9af
commit 45ba40b5a1
20 changed files with 401 additions and 606 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
workflow/engine/bin/tasks/cliGenerateDataReport.php
View File

@@ -1,74 +0,0 @@
<?php
use Illuminate\Support\Facades\DB;
use ProcessMaker\Core\System;
CLI::taskName("generate-data-report");
CLI::taskRun("generateDataReport");
CLI::taskDescription("\nGenerate data report by process in a respective workspace, you must pass through arguments the project identifier and the processing interval.");
CLI::taskArg("workspace", false);
CLI::taskOpt("uid", "Identifier that represents the process, must be 32 characters.", "", "process=");
CLI::taskOpt("start", "The start option skips so many rows before returning results.", "", "start=");
CLI::taskOpt("limit", "The limit option restricts the number of rows returned.", "", "limit=");
/**
* Generate data report by process in a respective workspace, you must pass through
* arguments the project identifier and the processing interval.
* @param array $options
* @return void
*/
function generateDataReport(array $options): void
{
//get workspace
if (empty($options[0])) {
CLI::logging("Workspace undefined!\n");
return;
}
$workspace = $options[0];
//get options
$parameters = [
"tableName=" => "",
"type=" => "",
"process=" => "",
"gridKey=" => "",
"additionalTable=" => "",
"className=" => "",
"pathWorkspace=" => "",
"start=" => "",
"limit=" => ""
];
foreach ($parameters as $key => $value) {
for ($i = 1; $i < count($options); $i++) {
if (strpos($options[$i], $key) !== false) {
$parameters[$key] = str_replace($key, "", $options[$i]);
break;
}
}
}
//validations
$needed = [
"process="
];
foreach ($needed as $value) {
if (empty($parameters[$value])) {
CLI::logging("Missing options {$value}.\n");
return;
}
}
//run method
$workspaceTools = new WorkspaceTools($workspace);
$workspaceTools->generateDataReport(
$parameters["tableName="],
$parameters["type="],
$parameters["process="],
$parameters["gridKey="],
$parameters["additionalTable="],
$parameters["className="],
$parameters["pathWorkspace="],
(int) $parameters["start="],
(int) $parameters["limit="]
);
}

21
workflow/engine/bin/tasks/cliPopulateTable.php
View File

@@ -1,21 +0,0 @@
<?php
CLI::taskName("populate-table");
CLI::taskRun("populateTable");
CLI::taskDescription("\nThis function populates the report table with the APP_DATA data");
CLI::taskArg("workspace", false);
/**
* This function populates the report table with the APP_DATA data.
* @return void
*/
function populateTable($options): void
{
//get options
$workspaceName = $options[0];
$query = base64_decode($options[1]);
$isRbac = (bool) $options[2];
$workspace = new WorkspaceTools($workspaceName);
$workspace->populateTableReport($query, $isRbac);
}

31
workflow/engine/classes/ReportTables.php
View File

@@ -1,8 +1,8 @@
<?php
use App\Jobs\GenerateReportTable;
use Illuminate\Support\Facades\DB;
use ProcessMaker\Core\MultiProcOpen;
use ProcessMaker\Commands\PopulateTableReport;
use ProcessMaker\Core\JobsManager;
use ProcessMaker\Model\Application;
/**
@@ -205,6 +205,9 @@ class ReportTables
*/
public function populateTable($tableName, $connectionShortName = 'report', $type = 'NORMAL', $fields = [], $proUid = '', $grid = '')
{
$config = System::getSystemConfiguration();
$reportTableBatchRegeneration = $config['report_table_batch_regeneration'];
$tableName = $this->sPrefix . $tableName;
//we have to do the propel connection
$database = $this->chooseDB($connectionShortName);
@@ -222,7 +225,7 @@ class ReportTables
$applications = Application::getByProUid($proUid);
$i = 1;
$queryValues = "";
$numberRecords = 1000;
$numberRecords = $reportTableBatchRegeneration;
$n = count($applications);
foreach ($applications as $application) {
$appData = $case->unserializeData($application->APP_DATA);
@@ -262,11 +265,12 @@ class ReportTables
$queryValues = rtrim($queryValues, ",");
$query = $headQuery . $queryValues;
$queryValues = "";
$workspace = config("system.workspace");
$processesManager = new MultiProcOpen();
$processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) {
return new PopulateTableReport($workspace, $query);
});
//add to queue
$closure = function() use($query) {
DB::insert($query);
};
JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
}
} else {
if (isset($appData[$grid])) {
@@ -304,11 +308,12 @@ class ReportTables
$queryValues = rtrim($queryValues, ",");
$query = $headQuery . $queryValues;
$queryValues = "";
$workspace = config("system.workspace");
$processesManager = new MultiProcOpen();
$processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) {
return new PopulateTableReport($workspace, $query);
});
//add to queue
$closure = function() use($query) {
DB::insert($query);
};
JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
}
}
}

3
workflow/engine/classes/WsBase.php
View File

@@ -1,5 +1,6 @@
<?php
use App\Jobs\EmailEvent;
use Illuminate\Support\Facades\Crypt;
use ProcessMaker\BusinessModel\EmailServer;
/*----------------------------------********---------------------------------*/
@@ -1028,7 +1029,7 @@ class WsBase
switch ($appMsgType) {
case WsBase::MESSAGE_TYPE_EMAIL_EVENT:
case WsBase::MESSAGE_TYPE_PM_FUNCTION:
JobsManager::getSingleton()->dispatch('EmailEvent', $closure);
JobsManager::getSingleton()->dispatch(EmailEvent::class, $closure);
$result = new WsResponse(0, G::loadTranslation('ID_MESSAGE_SENT') . ": " . $to);
break;
default :

44
workflow/engine/classes/model/AdditionalTables.php
View File

@@ -1,8 +1,9 @@
<?php
use App\Jobs\GenerateReportTable;
use Illuminate\Support\Facades\DB;
use ProcessMaker\Commands\GenerateDataReport;
use ProcessMaker\Core\MultiProcOpen;
use ProcessMaker\Core\JobsManager;
use ProcessMaker\Core\System;
use ProcessMaker\Model\Application;
use ProcessMaker\Model\Fields;
@@ -735,28 +736,23 @@ class AdditionalTables extends BaseAdditionalTables
$workspace = config("system.workspace");
$pathWorkspace = PATH_WORKSPACE;
$n = Application::count();
$processesManager = new MultiProcOpen();
$processesManager->chunk($n, 1000, function($size, $start, $limit) use(
$workspace,
$tableName,
$type,
$processUid,
$gridKey,
$addTabUid,
$className,
$pathWorkspace) {
return new GenerateDataReport(
$workspace,
$tableName,
$type,
$processUid,
$gridKey,
$addTabUid,
$className,
$pathWorkspace,
$start,
$limit);
});
//batch process
$config = System::getSystemConfiguration();
$reportTableBatchRegeneration = $config['report_table_batch_regeneration'];
$size = $n;
$start = 0;
$limit = $reportTableBatchRegeneration;
for ($i = 1; $start < $size; $i++) {
$closure = function() use($workspace, $tableName, $type, $processUid, $gridKey, $addTabUid, $className, $pathWorkspace, $start, $limit) {
$workspaceTools = new WorkspaceTools($workspace);
$workspaceTools->generateDataReport($tableName, $type, $processUid, $gridKey, $addTabUid, $className, $pathWorkspace, $start, $limit);
};
JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
$start = $i * $limit;
}
}
/**

61
workflow/engine/config/databases.php
View File

@@ -1,27 +1,4 @@
<?php
/**
* databases.php
*
* ProcessMaker Open Source Edition
* Copyright (C) 2004 - 2008 Colosa Inc.23
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* For more information, contact Colosa Inc, 2566 Le Jeune Rd.,
* Coral Gables, FL, 33134, USA, or email info@colosa.com.
*
*/
if (defined('PATH_DB') && !empty(config("system.workspace"))) {
@@ -29,7 +6,32 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
throw new Exception("Could not find db.php in current workspace " . config("system.workspace"));
}
require_once(PATH_DB . config("system.workspace") . '/db.php');
//These constants must not exist, they will be created by "db.php".
$constants = [
'DB_ADAPTER',
'DB_HOST',
'DB_NAME',
'DB_USER',
'DB_PASS',
'DB_RBAC_HOST',
'DB_RBAC_NAME',
'DB_RBAC_USER',
'DB_RBAC_PASS' ,
'DB_REPORT_HOST',
'DB_REPORT_NAME',
'DB_REPORT_USER',
'DB_REPORT_PASS',
];
$load = true;
foreach ($constants as $value) {
if (defined($value)) {
$load = false;
break;
}
}
if ($load === true) {
require_once(PATH_DB . config("system.workspace") . '/db.php');
}
//to do: enable for other databases
$dbType = DB_ADAPTER;
$dsn = DB_ADAPTER . '://' . DB_USER . ':' . urlencode(DB_PASS) . '@' . DB_HOST . '/' . DB_NAME;
@@ -42,15 +44,12 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
switch (DB_ADAPTER) {
case 'mysql':
$dsn .= '?encoding=utf8';
$dsnRbac .= '?encoding=utf8';
$dsn .= '?encoding=utf8';
$dsnRbac .= '?encoding=utf8';
$dsnReport .= '?encoding=utf8';
break;
case 'mssql':
case 'sqlsrv':
//$dsn .= '?sendStringAsUnicode=false';
//$dsnRbac .= '?sendStringAsUnicode=false';
//$dsnReport .= '?sendStringAsUnicode=false';
break;
default:
break;
@@ -64,7 +63,7 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
$pro ['datasources']['rp']['connection'] = $dsnReport;
$pro ['datasources']['rp']['adapter'] = DB_ADAPTER;
$dbHost = explode(':', DB_HOST);
config(['database.connections.workflow.host' => $dbHost[0]]);
config(['database.connections.workflow.database' => DB_NAME]);
@@ -76,6 +75,6 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
}
$pro ['datasources']['dbarray']['connection'] = 'dbarray://user:pass@localhost/pm_os';
$pro ['datasources']['dbarray']['adapter'] = 'dbarray';
$pro ['datasources']['dbarray']['adapter'] = 'dbarray';
return $pro;

6
workflow/engine/content/translations/english/processmaker.en.po
View File

@@ -25325,6 +25325,12 @@ msgstr "The PHP files execution was disabled please contact the system administr
msgid "Please complete the reassign reason."
msgstr "Please complete the reassign reason."
# TRANSLATION
# LABEL/ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES
#: LABEL/ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES
msgid "The report table is regenerating please come back in a few minutes."
msgstr "The report table is regenerating please come back in a few minutes."
# TRANSLATION
# LABEL/ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED
#: LABEL/ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED

2
workflow/engine/controllers/pmTablesProxy.php
View File

@@ -1215,7 +1215,7 @@ class pmTablesProxy extends HttpProxyController
if (!empty($table) && $table['PRO_UID'] != '') {
try {
$additionalTables->populateReportTable($table['ADD_TAB_NAME'], PmTable::resolveDbSource($table['DBS_UID']), $table['ADD_TAB_TYPE'], $table['PRO_UID'], $table['ADD_TAB_GRID'], $table['ADD_TAB_UID']);
$result->message = 'Generated for table ' . $table['ADD_TAB_NAME'];
$result->message = G::LoadTranslation("ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES");
} catch (Exception $e) {
$context = Bootstrap::getDefaultContextLog();
$context['proUid'] = $table['PRO_UID'];

1
workflow/engine/data/mysql/insert.sql
View File

@@ -61112,6 +61112,7 @@ INSERT INTO TRANSLATION (TRN_CATEGORY,TRN_ID,TRN_LANG,TRN_VALUE,TRN_UPDATE_DATE
( 'LABEL','ID_THE_NAME_CHANGE_MAY_CAUSE_DATA_LOSS','en','The change might cause data loss in the PM table. Do you want to continue?','2017-03-30') ,
( 'LABEL','ID_THE_PHP_FILES_EXECUTION_WAS_DISABLED','en','The PHP files execution was disabled please contact the system administrator.','2018-04-20') ,
( 'LABEL','ID_THE_REASON_REASSIGN_USER_EMPTY','en','Please complete the reassign reason.','2016-10-20') ,
( 'LABEL','ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES','en','The report table is regenerating please come back in a few minutes.','2020-06-01') ,
( 'LABEL','ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED','en','The upload of PHP files was disabled please contact the system administrator.','2018-04-20') ,
( 'LABEL','ID_THE_USERNAME_EMAIL_IS_INCORRECT','en','The username or email is incorrect','2018-01-18') ,
( 'LABEL','ID_THIS_MONTH','en','This Month','2014-01-15') ,

80
workflow/engine/src/ProcessMaker/Commands/GenerateDataReport.php
View File

@@ -1,80 +0,0 @@
<?php
namespace ProcessMaker\Commands;
use ProcessMaker\Core\ProcOpen;
class GenerateDataReport extends ProcOpen
{
private $workspace;
private $tableName;
private $type;
private $processUid;
private $gridKey;
private $addTabUid;
private $className;
private $pathWorkspace;
private $start;
private $limit;
/**
* Initializes the command parameters.
* @param string $workspace
* @param string $tableName
* @param string $type
* @param string $processUid
* @param string $gridKey
* @param string $addTabUid
* @param string $className
* @param string $pathWorkspace
* @param integer $start
* @param integer $limit
*/
public function __construct(
$workspace,
$tableName,
$type = 'NORMAL',
$processUid = '',
$gridKey = '',
$addTabUid = '',
$className = '',
$pathWorkspace,
$start = 0,
$limit = 10)
{
$this->workspace = $workspace;
$this->tableName = $tableName;
$this->type = $type;
$this->processUid = $processUid;
$this->gridKey = $gridKey;
$this->addTabUid = $addTabUid;
$this->className = $className;
$this->pathWorkspace = $pathWorkspace;
$this->start = $start;
$this->limit = $limit;
$this->setCwd(PATH_TRUNK);
parent::__construct($this->buildCommand());
}
/**
* Returns the command to execute.
* @return string
*/
private function buildCommand(): string
{
$command = PHP_BINDIR . "/php "
. "./processmaker "
. "'generate-data-report' "
. "'{$this->workspace}' "
. "'tableName={$this->tableName}' "
. "'type={$this->type}' "
. "'process={$this->processUid}' "
. "'gridKey={$this->gridKey}' "
. "'additionalTable={$this->addTabUid}' "
. "'className={$this->className}' "
. "'pathWorkspace={$this->pathWorkspace}' "
. "'start={$this->start}' "
. "'limit={$this->limit}' ";
return $command;
}
}

42
workflow/engine/src/ProcessMaker/Commands/PopulateTableReport.php
View File

@@ -1,42 +0,0 @@
<?php
namespace ProcessMaker\Commands;
use ProcessMaker\Core\ProcOpen;
class PopulateTableReport extends ProcOpen
{
private $workspace;
private $sql;
private $isRbac;
/**
* Initializes the command parameters.
* @param string $workspace
* @param string $sql
* @param boolean $isRbac
*/
public function __construct($workspace, $sql, $isRbac = false)
{
$this->workspace = $workspace;
$this->sql = $sql;
$this->isRbac = $isRbac;
$this->setCwd(PATH_TRUNK);
parent::__construct($this->buildCommand());
}
/**
* Returns the command to execute.
* @return string
*/
public function buildCommand()
{
$command = PHP_BINDIR . "/php "
. "./processmaker "
. "'populate-table' "
. "'{$this->workspace}' "
. base64_encode($this->sql) . " "
. ($this->isRbac ? "'1'" : "'0'");
return $command;
}
}

3
workflow/engine/src/ProcessMaker/Core/JobsManager.php
View File

@@ -5,7 +5,6 @@ namespace ProcessMaker\Core;
use Bootstrap;
use Exception;
use Illuminate\Support\Facades\Log;
use ProcessMaker\BusinessModel\Factories\Jobs;
use ProcessMaker\Core\System;
use Propel;
@@ -187,7 +186,7 @@ class JobsManager
{
$environment = $this->getDataSnapshot();
$instance = Jobs::create($name, function() use ($callback, $environment) {
$instance = $name::dispatch(function() use ($callback, $environment) {
try {
$this->recoverDataSnapshot($environment);
$callback($environment);

92
workflow/engine/src/ProcessMaker/Core/MultiProcOpen.php
View File

@@ -1,92 +0,0 @@
<?php
namespace ProcessMaker\Core;
class MultiProcOpen
{
/**
* Represents the waiting time before starting the process monitoring.
* @var integer
*/
private $sleepTime = 1;
/**
* This method obtains a paging by returning the start and limit indexes
* compatible with the mysql pagination in its call function.
* The return function must return an instance of the object "ProcessMaker\Core\ProcOpen".
* Returns an array containing the status, content, and errors generated by
* the open process.
* @param int $size
* @param int $chunk
* @param callable $callback
* @return array
*/
public function chunk(int $size, int $chunk, callable $callback): array
{
$start = 0;
$limit = $chunk;
$queries = [];
for ($i = 1; $start < $size; $i++) {
$queries[] = $callback($size, $start, $limit);
$start = $i * $limit;
}
return $this->run($queries);
}
/**
* Open a set of background processes.
* The array must contain one or more instances of the object inherited from
* the class "ProcessMaker\Core\ProcOpen"
* Returns an array containing the status, content, and errors generated by
* the open process.
* @param array $processes
* @return array
*/
public function run(array $processes): array
{
foreach ($processes as $procOpen) {
$procOpen->open();
}
return $this->processMonitoring($processes);
}
/**
* It monitors the open processes, verifying if they have ended or thrown an
* error and later closing the resources related to the process.
* Returns an array containing the status, content, and errors generated by
* the open process.
* @param array $processes
* @return array
*/
private function processMonitoring(array $processes): array
{
sleep($this->sleepTime); //this sleep is very important
$i = 0;
$n = count($processes);
if ($n === 0) {
return [];
}
$outputs = [];
do {
$index = $i % $n;
if (isset($processes[$index])) {
$procOpen = $processes[$index];
$status = $procOpen->getStatus();
$contents = $procOpen->getContents();
$errors = $procOpen->getErrors();
if ($status->running === false || !empty($errors)) {
$outputs[] = [
"status" => $status,
"contents" => $contents,
"errors" => $errors,
];
$procOpen->terminate();
$procOpen->close();
unset($processes[$index]);
}
}
$i = $i + 1;
} while (!empty($processes));
return $outputs;
}
}

126
workflow/engine/src/ProcessMaker/Core/ProcOpen.php
View File

@@ -1,126 +0,0 @@
<?php
namespace ProcessMaker\Core;
class ProcOpen
{
private $command;
private $resource;
private $descriptorspec;
private $pipes;
private $cwd;
/**
* This initializes the descriptors and the command for the open process.
* @param string $command
*/
public function __construct(string $command)
{
$this->descriptorspec = [
['pipe', 'r'],
['pipe', 'w'],
['pipe', 'w']
];
$this->command = $command;
}
/**
* Gets the resource that represents the process.
* @return resource
*/
public function getResource()
{
return $this->resource;
}
/**
* Sets the process execution directory.
* @param string $cwd
*/
public function setCwd(string $cwd)
{
$this->cwd = $cwd;
}
/**
* Open a background process.
*/
public function open()
{
if (empty($this->cwd)) {
$this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes);
} else {
$this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes, $this->cwd);
}
}
/**
* Get the content of the process when it is finished.
* @return string
*/
public function getContents()
{
if (is_resource($this->pipes[1])) {
return stream_get_contents($this->pipes[1]);
}
return "";
}
/**
* Get the process errors when it is finished.
* @return string
*/
public function getErrors()
{
if (is_resource($this->pipes[2])) {
return stream_get_contents($this->pipes[2]);
}
return "";
}
/**
* Close the resources related to the open process.
* return void
*/
public function close()
{
if (is_resource($this->resource)) {
foreach ($this->pipes as $value) {
fclose($value);
}
proc_close($this->resource);
}
}
/**
* End the process before it ends.
*/
public function terminate()
{
if (is_resource($this->resource)) {
proc_terminate($this->resource);
}
}
/**
* Gets the status of the process.
* @return object
*/
public function getStatus()
{
$status = [
"command" => $this->command,
"pid" => null,
"running" => false,
"signaled" => false,
"stopped" => false,
"exitcode" => -1,
"termsig" => 0,
"stopsig" => 0
];
if (is_resource($this->resource)) {
$status = proc_get_status($this->resource);
}
return (object) $status;
}
}

3
workflow/engine/src/ProcessMaker/Core/System.php
View File

@@ -78,7 +78,8 @@ class System
'highlight_home_folder_enable' => 0,
'highlight_home_folder_refresh_time' => 10,
'highlight_home_folder_scope' => 'unassigned', // For now only this list is supported
'disable_advanced_search_case_title_fulltext' => 0
'disable_advanced_search_case_title_fulltext' => 0,
'report_table_batch_regeneration' => 1000
];
/**