diff --git a/tests/CreateTestSite.php b/tests/CreateTestSite.php new file mode 100644 index 000000000..a53520e45 --- /dev/null +++ b/tests/CreateTestSite.php @@ -0,0 +1,34 @@ + "new_site"]); $workspace = config("system.workspace"); - - if (!file_exists(PATH_DB . $workspace)) { - G::mk_dir(PATH_DB . $workspace); - } - - if (!file_exists(PATH_DB . $workspace . PATH_SEP . "db.php")) { - $myfile = fopen(PATH_DB . $workspace . PATH_SEP . "db.php", "w"); - fwrite($myfile, "createDBFile($workspace); } /** diff --git a/tests/unit/workflow/engine/classes/ReportTablesTest.php b/tests/unit/workflow/engine/classes/ReportTablesTest.php index 559478dc8..db67e90ad 100644 --- a/tests/unit/workflow/engine/classes/ReportTablesTest.php +++ b/tests/unit/workflow/engine/classes/ReportTablesTest.php @@ -7,11 +7,12 @@ use ProcessMaker\Model\Application; use ProcessMaker\Model\Process; use ProcessMaker\Model\Task; use ProcessMaker\Model\User; +use Tests\CreateTestSite; use Tests\TestCase; class ReportTablesTest extends TestCase { - + use CreateTestSite; use DatabaseTransactions; /** @@ -20,7 +21,14 @@ class ReportTablesTest extends TestCase public function setUp() { parent::setUp(); + $this->markTestIncomplete("" + . "This test has started using the ./processmaker command, this " + . "command requires the file 'paths_installed.php', that is, a " + . "valid installation of processmaker."); $_SERVER["REQUEST_URI"] = ""; + config(["system.workspace" => "test"]); + $workspace = config("system.workspace"); + $this->createDBFile($workspace); } /** diff --git a/tests/unit/workflow/engine/controllers/PmTablesProxyTest.php b/tests/unit/workflow/engine/controllers/PmTablesProxyTest.php index 1af876066..1d77b851b 100644 --- a/tests/unit/workflow/engine/controllers/PmTablesProxyTest.php +++ b/tests/unit/workflow/engine/controllers/PmTablesProxyTest.php @@ -7,6 +7,7 @@ use G; use Illuminate\Foundation\Testing\DatabaseTransactions; use pmTablesProxy; use ProcessMaker\BusinessModel\ReportTable; +use Tests\CreateTestSite; use Tests\TestCase; /** @@ -14,6 +15,7 @@ use Tests\TestCase; */ class PmTablesProxyTest extends TestCase { + use CreateTestSite; use DatabaseTransactions; protected $preserveGlobalState = false; @@ -40,30 +42,8 @@ class PmTablesProxyTest extends TestCase parent::setUp(); config(["system.workspace" => SYS_SYS]); - $workspace = config("system.workspace"); - - if (!file_exists(PATH_DB . $workspace)) { - G::mk_dir(PATH_DB . $workspace); - } - - if (!file_exists(PATH_DB . $workspace . PATH_SEP . "db.php")) { - $myfile = fopen(PATH_DB . $workspace . PATH_SEP . "db.php", "w"); - fwrite($myfile, "createDBFile($workspace); //Set the user logged as the admin $_SESSION['USER_LOGGED'] = "00000000000000000000000000000001"; diff --git a/tests/unit/workflow/engine/src/ProcessMaker/Importer/ImporterTest.php b/tests/unit/workflow/engine/src/ProcessMaker/Importer/ImporterTest.php index e453f09f2..fe93291c4 100644 --- a/tests/unit/workflow/engine/src/ProcessMaker/Importer/ImporterTest.php +++ b/tests/unit/workflow/engine/src/ProcessMaker/Importer/ImporterTest.php @@ -8,10 +8,13 @@ use ProcessMaker\Model\BpmnProject; use ProcessMaker\Model\Process; use ProcessMaker\Model\User; use ReflectionClass; +use Tests\CreateTestSite; use Tests\TestCase; class ImporterTest extends TestCase { + use CreateTestSite; + /** * Declared to avoid the incompatibility exception */ @@ -21,28 +24,7 @@ class ImporterTest extends TestCase error_reporting(E_ALL & ~E_DEPRECATED & ~E_STRICT); config(["system.workspace" => "test"]); $workspace = config("system.workspace"); - - if (!file_exists(PATH_DB . $workspace)) { - mkdir(PATH_DB . $workspace); - } - - if (!file_exists(PATH_DB . $workspace . PATH_SEP . "db.php")) { - $myfile = fopen(PATH_DB . $workspace . PATH_SEP . "db.php", "w"); - fwrite($myfile, "createDBFile($workspace); } /** diff --git a/workflow/engine/bin/tasks/cliGenerateDataReport.php b/workflow/engine/bin/tasks/cliGenerateDataReport.php new file mode 100644 index 000000000..0dcdea02f --- /dev/null +++ b/workflow/engine/bin/tasks/cliGenerateDataReport.php @@ -0,0 +1,74 @@ + "", + "type=" => "", + "process=" => "", + "gridKey=" => "", + "additionalTable=" => "", + "className=" => "", + "pathWorkspace=" => "", + "start=" => "", + "limit=" => "" + ]; + foreach ($parameters as $key => $value) { + for ($i = 1; $i < count($options); $i++) { + if (strpos($options[$i], $key) !== false) { + $parameters[$key] = str_replace($key, "", $options[$i]); + break; + } + } + } + + //validations + $needed = [ + "process=" + ]; + foreach ($needed as $value) { + if (empty($parameters[$value])) { + CLI::logging("Missing options {$value}.\n"); + return; + } + } + + //run method + $workspaceTools = new WorkspaceTools($workspace); + $workspaceTools->generateDataReport( + $parameters["tableName="], + $parameters["type="], + $parameters["process="], + $parameters["gridKey="], + $parameters["additionalTable="], + $parameters["className="], + $parameters["pathWorkspace="], + (int) $parameters["start="], + (int) $parameters["limit="] + ); +} diff --git a/workflow/engine/bin/tasks/cliPopulateTable.php b/workflow/engine/bin/tasks/cliPopulateTable.php new file mode 100644 index 000000000..c99584c00 --- /dev/null +++ b/workflow/engine/bin/tasks/cliPopulateTable.php @@ -0,0 +1,21 @@ +populateTableReport($query, $isRbac); +} diff --git a/workflow/engine/classes/ReportTables.php b/workflow/engine/classes/ReportTables.php index 155e9c9b0..73cba7357 100644 --- a/workflow/engine/classes/ReportTables.php +++ b/workflow/engine/classes/ReportTables.php @@ -1,6 +1,8 @@ chooseDB($connectionShortName); $connection = Propel::getConnection($database); - $statement = $connection->createStatement(); if ($type == 'GRID') { $aux = explode('-', $grid); $grid = $aux[0]; } $case = new Cases(); try { - switch (DB_ADAPTER) { - case 'mysql': - $applications = Application::getByProUid($proUid); - foreach ($applications as $application) { - $appData = $case->unserializeData($application->APP_DATA); - DB::delete("DELETE FROM `{$tableName}` WHERE APP_UID = '{$application->APP_UID}'"); - if ($type == 'NORMAL') { - $query = 'INSERT INTO `' . $tableName . '` ('; - $query .= '`APP_UID`,`APP_NUMBER`'; - foreach ($fields as $field) { - $query .= ',`' . $field['sFieldName'] . '`'; - } - $query .= ") VALUES ('" . $application->APP_UID . "'," . $application->APP_NUMBER; + $applicationTableName = (new Application())->getTable(); + $deleteQuery = "TRUNCATE TABLE `{$tableName}` "; + DB::delete($deleteQuery); + + $applications = Application::getByProUid($proUid); + $i = 1; + $queryValues = ""; + $numberRecords = 1000; + $n = count($applications); + foreach ($applications as $application) { + $appData = $case->unserializeData($application->APP_DATA); + if ($type == 'NORMAL') { + $query = 'INSERT INTO `' . $tableName . '` ('; + $query .= '`APP_UID`,`APP_NUMBER`'; + foreach ($fields as $field) { + $query .= ',`' . $field['sFieldName'] . '`'; + } + $headQuery = $query . ") VALUES "; + $query = "('" . $application->APP_UID . "'," . $application->APP_NUMBER; + foreach ($fields as $field) { + switch ($field['sType']) { + case 'number': + $query .= ',' . (isset($appData[$field['sFieldName']]) ? (float) str_replace(',', '', $appData[$field['sFieldName']]) : '0'); + break; + case 'char': + case 'text': + if (!isset($appData[$field['sFieldName']])) { + $appData[$field['sFieldName']] = ''; + } + $string = $appData[$field['sFieldName']]; + if (is_array($string)) { + $string = implode($string, ","); + } + $query .= ",'" . (isset($appData[$field['sFieldName']]) ? mysqli_real_escape_string($connection->getResource(), $string) : '') . "'"; + break; + case 'date': + $value = (isset($appData[$field['sFieldName']]) && trim($appData[$field['sFieldName']])) != '' ? "'" . $appData[$field['sFieldName']] . "'" : 'NULL'; + $query .= "," . $value; + break; + } + } + $query .= ')'; + $queryValues = $queryValues . $query . ","; + if ($i % $numberRecords === 0 || $i === $n) { + $queryValues = rtrim($queryValues, ","); + $query = $headQuery . $queryValues; + $queryValues = ""; + $workspace = config("system.workspace"); + $processesManager = new MultiProcOpen(); + $processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) { + return new PopulateTableReport($workspace, $query); + }); + } + } else { + if (isset($appData[$grid])) { + $query = 'INSERT INTO `' . $tableName . '` ('; + $query .= '`APP_UID`,`APP_NUMBER`,`ROW`'; + foreach ($fields as $field) { + $query .= ',`' . $field['sFieldName'] . '`'; + } + $headQuery = $query . ") VALUES "; + foreach ($appData[$grid] as $indexRow => $gridRow) { + $query = "('" . $application->APP_UID . "'," . (int) $application->APP_NUMBER . ',' . $indexRow; foreach ($fields as $field) { switch ($field['sType']) { case 'number': - $query .= ',' . (isset($appData[$field['sFieldName']]) ? (float) str_replace(',', '', $appData[$field['sFieldName']]) : '0'); + $query .= ',' . (isset($gridRow[$field['sFieldName']]) ? (float) str_replace(',', '', $gridRow[$field['sFieldName']]) : '0'); break; case 'char': case 'text': - if (!isset($appData[$field['sFieldName']])) { - $appData[$field['sFieldName']] = ''; + if (!isset($gridRow[$field['sFieldName']])) { + $gridRow[$field['sFieldName']] = ''; } - $string = $appData[$field['sFieldName']]; - if (is_array($string)) { - $string = implode($string, ","); - } - $query .= ",'" . (isset($appData[$field['sFieldName']]) ? mysqli_real_escape_string($connection->getResource(), $string) : '') . "'"; + $stringEscape = mysqli_real_escape_string($connection->getResource(), $gridRow[$field['sFieldName']]); + $query .= ",'" . (isset($gridRow[$field['sFieldName']]) ? $stringEscape : '') . "'"; break; case 'date': - $value = (isset($appData[$field['sFieldName']]) && trim($appData[$field['sFieldName']])) != '' ? "'" . $appData[$field['sFieldName']] . "'" : 'NULL'; + $value = (isset($gridRow[$field['sFieldName']]) && trim($gridRow[$field['sFieldName']])) != '' ? "'" . $gridRow[$field['sFieldName']] . "'" : 'NULL'; $query .= "," . $value; break; } } $query .= ')'; - DB::insert($query); - } else { - if (isset($appData[$grid])) { - foreach ($appData[$grid] as $indexRow => $gridRow) { - $query = 'INSERT INTO `' . $tableName . '` ('; - $query .= '`APP_UID`,`APP_NUMBER`,`ROW`'; - foreach ($fields as $field) { - $query .= ',`' . $field['sFieldName'] . '`'; - } - $query .= ") VALUES ('" . $application->APP_UID . "'," . (int) $application->APP_NUMBER . ',' . $indexRow; - foreach ($fields as $field) { - switch ($field['sType']) { - case 'number': - $query .= ',' . (isset($gridRow[$field['sFieldName']]) ? (float) str_replace(',', '', $gridRow[$field['sFieldName']]) : '0'); - break; - case 'char': - case 'text': - if (!isset($gridRow[$field['sFieldName']])) { - $gridRow[$field['sFieldName']] = ''; - } - $stringEscape = mysqli_real_escape_string($connection->getResource(), $gridRow[$field['sFieldName']]); - $query .= ",'" . (isset($gridRow[$field['sFieldName']]) ? $stringEscape : '') . "'"; - break; - case 'date': - $value = (isset($gridRow[$field['sFieldName']]) && trim($gridRow[$field['sFieldName']])) != '' ? "'" . $gridRow[$field['sFieldName']] . "'" : 'NULL'; - $query .= "," . $value; - break; - } - } - $query .= ')'; - DB::insert($query); - } - } + $queryValues = $queryValues . $query . ","; + } + if ($i % $numberRecords === 0 || $i === $n) { + $queryValues = rtrim($queryValues, ","); + $query = $headQuery . $queryValues; + $queryValues = ""; + $workspace = config("system.workspace"); + $processesManager = new MultiProcOpen(); + $processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) { + return new PopulateTableReport($workspace, $query); + }); } } - break; - - /** - * For SQLServer code - * Note: It is only possible to create Report Tables in MySQL databases. The list will only show connections to those databases even if the project has connections to other DBMS. - * This section is not used and has been marked for deletion. - * @link https://wiki.processmaker.com/3.0/Report_Tables#Creating_Report_Tables - * @deprecated - */ - case 'mssql': - $criteria = new Criteria('workflow'); - $criteria->add(ApplicationPeer::PRO_UID, $proUid); - $criteria->addAscendingOrderByColumn(ApplicationPeer::APP_NUMBER); - $dataset = ApplicationPeer::doSelectRS($criteria); - $dataset->setFetchmode(ResultSet::FETCHMODE_ASSOC); - $dataset->next(); - while ($row = $dataset->getRow()) { - $appData = unserialize($row['APP_DATA']); - //verify use mssql - mysqli_query( - $connection->getResource(), 'DELETE FROM [' . $tableName . "] WHERE APP_UID = '" . $row['APP_UID'] . "'" - ); - if ($type == 'NORMAL') { - $query = 'INSERT INTO [' . $tableName . '] ('; - $query .= '[APP_UID],[APP_NUMBER]'; - foreach ($fields as $field) { - $query .= ',[' . $field['sFieldName'] . ']'; - } - $query .= ") VALUES ('" . $row['APP_UID'] . "'," . (int) $row['APP_NUMBER']; - foreach ($fields as $field) { - switch ($field['sType']) { - case 'number': - $query .= ',' . (isset($appData[$field['sFieldName']]) ? (float) str_replace(',', '', $appData[$field['sFieldName']]) : '0'); - break; - case 'char': - case 'text': - if (!isset($appData[$field['sFieldName']])) { - $appData[$field['sFieldName']] = ''; - } - $stringEscape = mysqli_real_escape_string($connection->getResource(), $appData[$field['sFieldName']]); - $query .= ",'" . (isset($appData[$field['sFieldName']]) ? $stringEscape : '') . "'"; - break; - case 'date': - $query .= ",'" . (isset($appData[$field['sFieldName']]) ? $appData[$field['sFieldName']] : '') . "'"; - break; - } - } - $query .= ')'; - $rs = $statement->executeQuery($query); - } else { - if (isset($appData[$grid])) { - foreach ($appData[$grid] as $indexRow => $gridRow) { - $query = 'INSERT INTO [' . $tableName . '] ('; - $query .= '`APP_UID`,`APP_NUMBER`,`ROW`'; - foreach ($fields as $field) { - $query .= ',[' . $field['sFieldName'] . ']'; - } - $query .= ") VALUES ('" . $row['APP_UID'] . "'," . (int) $row['APP_NUMBER'] . ',' . $indexRow; - foreach ($fields as $field) { - switch ($field['sType']) { - case 'number': - $query .= ',' . (isset($gridRow[$field['sFieldName']]) ? (float) str_replace(',', '', $gridRow[$field['sFieldName']]) : '0'); - break; - case 'char': - case 'text': - if (!isset($gridRow[$field['sFieldName']])) { - $gridRow[$field['sFieldName']] = ''; - } - $stringEscape = mysqli_real_escape_string($connection->getResource(), $gridRow[$field['sFieldName']]); - $query .= ",'" . (isset($gridRow[$field['sFieldName']]) ? $stringEscape : '') . "'"; - break; - case 'date': - $query .= ",'" . (isset($gridRow[$field['sFieldName']]) ? $gridRow[$field['sFieldName']] : '') . "'"; - break; - } - } - $query .= ')'; - $rs = $statement->executeQuery($query); - } - } - } - $dataset->next(); - } - break; + } + $i = $i + 1; } } catch (Exception $oError) { throw ($oError); diff --git a/workflow/engine/classes/WorkspaceTools.php b/workflow/engine/classes/WorkspaceTools.php index 90c87b991..24112d1dd 100644 --- a/workflow/engine/classes/WorkspaceTools.php +++ b/workflow/engine/classes/WorkspaceTools.php @@ -9,6 +9,8 @@ use ProcessMaker\ChangeLog\ChangeLog; use ProcessMaker\Core\Installer; use ProcessMaker\Core\ProcessesManager; use ProcessMaker\Core\System; +use ProcessMaker\Model\Application; +use ProcessMaker\Model\Fields; use ProcessMaker\Plugins\Adapters\PluginAdapter; use ProcessMaker\Project\Adapter\BpmnWorkflow; use ProcessMaker\Upgrade\RunProcessUpgradeQuery; @@ -4821,4 +4823,167 @@ class WorkspaceTools $database = $this->getDatabase($rbac); $database->executeQuery($query, true); } + + /** + * This method regenerates data report with the APP_DATA data. + * @param string $tableName + * @param string $type + * @param string $processUid + * @param string $gridKey + * @param string $addTabUid + * @param string $className + * @param string $pathWorkspace + * @param int $start + * @param int $limit + * @throws Exception + */ + public function generateDataReport( + $tableName, + $type = 'NORMAL', + $processUid = '', + $gridKey = '', + $addTabUid = '', + $className = '', + $pathWorkspace, + int $start = 0, + int $limit = 10) + { + $this->initPropel(); + $dbHost = explode(':', $this->dbHost); + config(['database.connections.workflow.host' => $dbHost[0]]); + config(['database.connections.workflow.database' => $this->dbName]); + config(['database.connections.workflow.username' => $this->dbUser]); + config(['database.connections.workflow.password' => $this->dbPass]); + if (count($dbHost) > 1) { + config(['database.connections.workflow.port' => $dbHost[1]]); + } + + //require file + set_include_path(get_include_path() . PATH_SEPARATOR . $pathWorkspace); + if (!file_exists($pathWorkspace . 'classes/' . $className . '.php')) { + throw new Exception("ERROR: " . $pathWorkspace . 'classes/' . $className . '.php' . " class file doesn't exit!"); + } + require_once 'classes/model/AdditionalTables.php'; + require_once $pathWorkspace . 'classes/' . $className . '.php'; + + //get fields + $fieldTypes = []; + if ($addTabUid != '') { + $fields = Fields::where('ADD_TAB_UID', '=', $addTabUid)->get(); + foreach ($fields as $field) { + switch ($field->FLD_TYPE) { + case 'FLOAT': + case 'DOUBLE': + case 'INTEGER': + $fieldTypes[] = [$field->FLD_NAME => $field->FLD_TYPE]; + break; + default: + break; + } + } + } + + $context = Bootstrap::getDefaultContextLog(); + $case = new Cases(); + + //select cases for this Process, ordered by APP_NUMBER + $applications = Application::where('PRO_UID', '=', $processUid) + ->orderBy('APP_NUMBER', 'asc') + ->offset($start) + ->limit($limit) + ->get(); + foreach ($applications as $application) { + //getting the case data + $appData = $case->unserializeData($application->APP_DATA); + + //quick fix, map all empty values as NULL for Database + foreach ($appData as $appDataKey => $appDataValue) { + if (is_array($appDataValue) && count($appDataValue)) { + $j = key($appDataValue); + $appDataValue = is_array($appDataValue[$j]) ? $appDataValue : $appDataValue[$j]; + } + if (is_string($appDataValue)) { + foreach ($fieldTypes as $key => $fieldType) { + foreach ($fieldType as $fieldTypeKey => $fieldTypeValue) { + if (strtoupper($appDataKey) == $fieldTypeKey) { + $appData[$appDataKey] = validateType($appDataValue, $fieldTypeValue); + unset($fieldTypeKey); + } + } + } + // normal fields + if (trim($appDataValue) === '') { + $appData[$appDataKey] = null; + } + } else { + // grids + if (is_array($appData[$appDataKey])) { + foreach ($appData[$appDataKey] as $dIndex => $dRow) { + if (is_array($dRow)) { + foreach ($dRow as $k => $v) { + if (is_string($v) && trim($v) === '') { + $appData[$appDataKey][$dIndex][$k] = null; + } + } + } + } + } + } + } + + //populate data + if ($type === 'GRID') { + list($gridName, $gridUid) = explode('-', $gridKey); + $gridData = isset($appData[$gridName]) ? $appData[$gridName] : []; + foreach ($gridData as $i => $gridRow) { + try { + $obj = new $className(); + $obj->fromArray($appData, BasePeer::TYPE_FIELDNAME); + $obj->setAppUid($application->APP_UID); + $obj->setAppNumber($application->APP_NUMBER); + if (method_exists($obj, 'setAppStatus')) { + $obj->setAppStatus($application->APP_STATUS); + } + $obj->fromArray(array_change_key_case($gridRow, CASE_UPPER), BasePeer::TYPE_FIELDNAME); + $obj->setRow($i); + $obj->save(); + } catch (Exception $e) { + $context["message"] = $e->getMessage(); + $context["tableName"] = $tableName; + $context["appUid"] = $application->APP_UID; + Bootstrap::registerMonolog("sqlExecution", 500, "Sql Execution", $context, $context["workspace"], "processmaker.log"); + } + unset($obj); + } + } else { + try { + $obj = new $className(); + $obj->fromArray(array_change_key_case($appData, CASE_UPPER), BasePeer::TYPE_FIELDNAME); + $obj->setAppUid($application->APP_UID); + $obj->setAppNumber($application->APP_NUMBER); + if (method_exists($obj, 'setAppStatus')) { + $obj->setAppStatus($application->APP_STATUS); + } + $obj->save(); + } catch (Exception $e) { + $context["message"] = $e->getMessage(); + $context["tableName"] = $tableName; + $context["appUid"] = $application->APP_UID; + Bootstrap::registerMonolog("sqlExecution", 500, "Sql Execution", $context, $context["workspace"], "processmaker.log"); + } + unset($obj); + } + } + } + + /** + * This method populates the table with a query string. + * @param string $query + * @param boolean $rbac + */ + public function populateTableReport($query, $rbac) + { + $database = $this->getDatabase($rbac); + $database->executeQuery($query, true); + } } diff --git a/workflow/engine/classes/model/AdditionalTables.php b/workflow/engine/classes/model/AdditionalTables.php index d896ee0b7..22c1f967a 100644 --- a/workflow/engine/classes/model/AdditionalTables.php +++ b/workflow/engine/classes/model/AdditionalTables.php @@ -1,23 +1,12 @@ - * - * @package workflow.engine.classes.model - */ +require_once 'classes/model/om/BaseAdditionalTables.php'; function validateType($value, $type) { @@ -740,136 +729,34 @@ class AdditionalTables extends BaseAdditionalTables */ public function populateReportTable($tableName, $sConnection = 'rp', $type = 'NORMAL', $processUid = '', $gridKey = '', $addTabUid = '') { - $this->className = $this->getPHPName($tableName); - $this->classPeerName = $this->className . 'Peer'; - - if (!file_exists(PATH_WORKSPACE . 'classes/' . $this->className . '.php')) { - throw new Exception("ERROR: " . PATH_WORKSPACE . 'classes/' . $this->className . '.php' . " class file doesn't exit!"); - } - - require_once PATH_WORKSPACE . 'classes/' . $this->className . '.php'; - - //get fields - $fieldTypes = []; - if ($addTabUid != '') { - $criteria = new Criteria('workflow'); - $criteria->add(FieldsPeer::ADD_TAB_UID, $addTabUid); - $dataset = FieldsPeer::doSelectRS($criteria); - $dataset->setFetchmode(ResultSet::FETCHMODE_ASSOC); - while ($dataset->next()) { - $row = $dataset->getRow(); - switch ($row['FLD_TYPE']) { - case 'FLOAT': - case 'DOUBLE': - case 'INTEGER': - $fieldTypes[] = array($row['FLD_NAME'] => $row['FLD_TYPE']); - break; - default: - break; - } - } - } - - //remove old applications references - $connection = Propel::getConnection($sConnection); - $statement = $connection->createStatement(); - $sql = "TRUNCATE " . $tableName; - $statement->executeQuery($sql); - - $case = new Cases(); - $context = Bootstrap::getDefaultContextLog(); - - //select cases for this Process, ordered by APP_NUMBER - $criteria = new Criteria('workflow'); - $criteria->add(ApplicationPeer::PRO_UID, $processUid); - $criteria->addAscendingOrderByColumn(ApplicationPeer::APP_NUMBER); - $dataset = ApplicationPeer::doSelectRS($criteria); - $dataset->setFetchmode(ResultSet::FETCHMODE_ASSOC); - while ($dataset->next()) { - $row = $dataset->getRow(); - - //getting the case data - $appData = $case->unserializeData($row['APP_DATA']); - - //quick fix, map all empty values as NULL for Database - foreach ($appData as $appDataKey => $appDataValue) { - if (is_array($appDataValue) && count($appDataValue)) { - $j = key($appDataValue); - $appDataValue = is_array($appDataValue[$j]) ? $appDataValue : $appDataValue[$j]; - } - if (is_string($appDataValue)) { - foreach ($fieldTypes as $key => $fieldType) { - foreach ($fieldType as $fieldTypeKey => $fieldTypeValue) { - if (strtoupper($appDataKey) == $fieldTypeKey) { - $appData[$appDataKey] = validateType($appDataValue, $fieldTypeValue); - unset($fieldTypeKey); - } - } - } - // normal fields - if (trim($appDataValue) === '') { - $appData[$appDataKey] = null; - } - } else { - // grids - if (is_array($appData[$appDataKey])) { - foreach ($appData[$appDataKey] as $dIndex => $dRow) { - if (is_array($dRow)) { - foreach ($dRow as $k => $v) { - if (is_string($v) && trim($v) === '') { - $appData[$appDataKey][$dIndex][$k] = null; - } - } - } - } - } - } - } - - //populate data - $className = $this->className; - if ($type === 'GRID') { - list($gridName, $gridUid) = explode('-', $gridKey); - $gridData = isset($appData[$gridName]) ? $appData[$gridName] : []; - foreach ($gridData as $i => $gridRow) { - try { - $obj = new $className(); - $obj->fromArray($appData, BasePeer::TYPE_FIELDNAME); - $obj->setAppUid($row['APP_UID']); - $obj->setAppNumber($row['APP_NUMBER']); - if (method_exists($obj, 'setAppStatus')) { - $obj->setAppStatus($row['APP_STATUS']); - } - $obj->fromArray(array_change_key_case($gridRow, CASE_UPPER), BasePeer::TYPE_FIELDNAME); - $obj->setRow($i); - $obj->save(); - } catch (Exception $e) { - $context["message"] = $e->getMessage(); - $context["tableName"] = $tableName; - $context["appUid"] = $row['APP_UID']; - Bootstrap::registerMonolog("sqlExecution", 500, "Sql Execution", $context, $context["workspace"], "processmaker.log"); - } - unset($obj); - } - } else { - try { - $obj = new $className(); - $obj->fromArray(array_change_key_case($appData, CASE_UPPER), BasePeer::TYPE_FIELDNAME); - $obj->setAppUid($row['APP_UID']); - $obj->setAppNumber($row['APP_NUMBER']); - if (method_exists($obj, 'setAppStatus')) { - $obj->setAppStatus($row['APP_STATUS']); - } - $obj->save(); - } catch (Exception $e) { - $context["message"] = $e->getMessage(); - $context["tableName"] = $tableName; - $context["appUid"] = $row['APP_UID']; - Bootstrap::registerMonolog("sqlExecution", 500, "Sql Execution", $context, $context["workspace"], "processmaker.log"); - } - unset($obj); - } - } + $this->className = $className = $this->getPHPName($tableName); + $this->classPeerName = $classPeerName = $className . 'Peer'; + DB::statement("TRUNCATE " . $tableName); + $workspace = config("system.workspace"); + $pathWorkspace = PATH_WORKSPACE; + $n = Application::count(); + $processesManager = new MultiProcOpen(); + $processesManager->chunk($n, 1000, function($size, $start, $limit) use( + $workspace, + $tableName, + $type, + $processUid, + $gridKey, + $addTabUid, + $className, + $pathWorkspace) { + return new GenerateDataReport( + $workspace, + $tableName, + $type, + $processUid, + $gridKey, + $addTabUid, + $className, + $pathWorkspace, + $start, + $limit); + }); } /** diff --git a/workflow/engine/src/ProcessMaker/Commands/GenerateDataReport.php b/workflow/engine/src/ProcessMaker/Commands/GenerateDataReport.php new file mode 100644 index 000000000..b14301794 --- /dev/null +++ b/workflow/engine/src/ProcessMaker/Commands/GenerateDataReport.php @@ -0,0 +1,80 @@ +workspace = $workspace; + $this->tableName = $tableName; + $this->type = $type; + $this->processUid = $processUid; + $this->gridKey = $gridKey; + $this->addTabUid = $addTabUid; + $this->className = $className; + $this->pathWorkspace = $pathWorkspace; + $this->start = $start; + $this->limit = $limit; + $this->setCwd(PATH_TRUNK); + parent::__construct($this->buildCommand()); + } + + /** + * Returns the command to execute. + * @return string + */ + private function buildCommand(): string + { + $command = PHP_BINDIR . "/php " + . "./processmaker " + . "'generate-data-report' " + . "'{$this->workspace}' " + . "'tableName={$this->tableName}' " + . "'type={$this->type}' " + . "'process={$this->processUid}' " + . "'gridKey={$this->gridKey}' " + . "'additionalTable={$this->addTabUid}' " + . "'className={$this->className}' " + . "'pathWorkspace={$this->pathWorkspace}' " + . "'start={$this->start}' " + . "'limit={$this->limit}' "; + return $command; + } +} diff --git a/workflow/engine/src/ProcessMaker/Commands/PopulateTableReport.php b/workflow/engine/src/ProcessMaker/Commands/PopulateTableReport.php new file mode 100644 index 000000000..310d2e948 --- /dev/null +++ b/workflow/engine/src/ProcessMaker/Commands/PopulateTableReport.php @@ -0,0 +1,42 @@ +workspace = $workspace; + $this->sql = $sql; + $this->isRbac = $isRbac; + $this->setCwd(PATH_TRUNK); + parent::__construct($this->buildCommand()); + } + + /** + * Returns the command to execute. + * @return string + */ + public function buildCommand() + { + $command = PHP_BINDIR . "/php " + . "./processmaker " + . "'populate-table' " + . "'{$this->workspace}' " + . base64_encode($this->sql) . " " + . ($this->isRbac ? "'1'" : "'0'"); + return $command; + } +} diff --git a/workflow/engine/src/ProcessMaker/Core/MultiProcOpen.php b/workflow/engine/src/ProcessMaker/Core/MultiProcOpen.php new file mode 100644 index 000000000..f288b781d --- /dev/null +++ b/workflow/engine/src/ProcessMaker/Core/MultiProcOpen.php @@ -0,0 +1,89 @@ +run($queries); + } + + /** + * Open a set of background processes. + * The array must contain one or more instances of the object inherited from + * the class "ProcessMaker\Core\ProcOpen" + * Returns an array containing the status, content, and errors generated by + * the open process. + * @param array $processes + * @return array + */ + public function run(array $processes): array + { + foreach ($processes as $procOpen) { + $procOpen->open(); + } + return $this->processMonitoring($processes); + } + + /** + * It monitors the open processes, verifying if they have ended or thrown an + * error and later closing the resources related to the process. + * Returns an array containing the status, content, and errors generated by + * the open process. + * @param array $processes + * @return array + */ + private function processMonitoring(array $processes): array + { + sleep($this->sleepTime); //this sleep is very important + $i = 0; + $n = count($processes); + $outputs = []; + do { + $index = $i % $n; + if (isset($processes[$index])) { + $procOpen = $processes[$index]; + $status = $procOpen->getStatus(); + $contents = $procOpen->getContents(); + $errors = $procOpen->getErrors(); + if ($status->running === false || !empty($errors)) { + $outputs[] = [ + "status" => $status, + "contents" => $contents, + "errors" => $errors, + ]; + $procOpen->terminate(); + $procOpen->close(); + unset($processes[$index]); + } + } + $i = $i + 1; + } while (!empty($processes)); + return $outputs; + } +} diff --git a/workflow/engine/src/ProcessMaker/Core/ProcOpen.php b/workflow/engine/src/ProcessMaker/Core/ProcOpen.php new file mode 100644 index 000000000..9840db670 --- /dev/null +++ b/workflow/engine/src/ProcessMaker/Core/ProcOpen.php @@ -0,0 +1,126 @@ +descriptorspec = [ + ['pipe', 'r'], + ['pipe', 'w'], + ['pipe', 'w'] + ]; + $this->command = $command; + } + + /** + * Gets the resource that represents the process. + * @return resource + */ + public function getResource() + { + return $this->resource; + } + + /** + * Sets the process execution directory. + * @param string $cwd + */ + public function setCwd(string $cwd) + { + $this->cwd = $cwd; + } + + /** + * Open a background process. + */ + public function open() + { + if (empty($this->cwd)) { + $this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes); + } else { + $this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes, $this->cwd); + } + } + + /** + * Get the content of the process when it is finished. + * @return string + */ + public function getContents() + { + if (is_resource($this->pipes[1])) { + return stream_get_contents($this->pipes[1]); + } + return ""; + } + + /** + * Get the process errors when it is finished. + * @return string + */ + public function getErrors() + { + if (is_resource($this->pipes[2])) { + return stream_get_contents($this->pipes[2]); + } + return ""; + } + + /** + * Close the resources related to the open process. + * return void + */ + public function close() + { + if (is_resource($this->resource)) { + foreach ($this->pipes as $value) { + fclose($value); + } + proc_close($this->resource); + } + } + + /** + * End the process before it ends. + */ + public function terminate() + { + if (is_resource($this->resource)) { + proc_terminate($this->resource); + } + } + + /** + * Gets the status of the process. + * @return object + */ + public function getStatus() + { + $status = [ + "command" => $this->command, + "pid" => null, + "running" => false, + "signaled" => false, + "stopped" => false, + "exitcode" => -1, + "termsig" => 0, + "stopsig" => 0 + ]; + if (is_resource($this->resource)) { + $status = proc_get_status($this->resource); + } + return (object) $status; + } +} diff --git a/workflow/engine/src/ProcessMaker/Model/Application.php b/workflow/engine/src/ProcessMaker/Model/Application.php index d9393bc57..b2cca4de3 100644 --- a/workflow/engine/src/ProcessMaker/Model/Application.php +++ b/workflow/engine/src/ProcessMaker/Model/Application.php @@ -8,6 +8,8 @@ use Illuminate\Support\Facades\DB; class Application extends Model { protected $table = "APPLICATION"; + protected $primaryKey = 'APP_NUMBER'; + public $incrementing = false; // No timestamps public $timestamps = false;