PMCORE-1203 Help on how batch routing feature works and help to debug the code with a client

tests/CreateTestSite.php

@@ -0,0 +1,34 @@
+<?php
+
+namespace Tests;
+
+trait CreateTestSite
+{
+
+    public function createDBFile(string $workspace)
+    {
+        if (!file_exists(PATH_DB . $workspace)) {
+            mkdir(PATH_DB . $workspace);
+        }
+
+        if (!file_exists(PATH_DB . $workspace . PATH_SEP . "db.php")) {
+            $myfile = fopen(PATH_DB . $workspace . PATH_SEP . "db.php", "w");
+            $content = ""
+                    . "<?php\n"
+                    . "define ('DB_ADAPTER',     'mysql' );\n"
+                    . "define ('DB_HOST',        '" . env('DB_HOST') . "' );\n"
+                    . "define ('DB_NAME',        '" . env('DB_DATABASE') . "' );\n"
+                    . "define ('DB_USER',        '" . env('DB_USERNAME') . "' );\n"
+                    . "define ('DB_PASS',        '" . env('DB_PASSWORD') . "' );\n"
+                    . "define ('DB_RBAC_HOST',   '" . env('DB_HOST') . "' );\n"
+                    . "define ('DB_RBAC_NAME',   '" . env('DB_DATABASE') . "' );\n"
+                    . "define ('DB_RBAC_USER',   '" . env('DB_USERNAME') . "' );\n"
+                    . "define ('DB_RBAC_PASS',   '" . env('DB_PASSWORD') . "' );\n"
+                    . "define ('DB_REPORT_HOST', '" . env('DB_HOST') . "' );\n"
+                    . "define ('DB_REPORT_NAME', '" . env('DB_DATABASE') . "' );\n"
+                    . "define ('DB_REPORT_USER', '" . env('DB_USERNAME') . "' );\n"
+                    . "define ('DB_REPORT_PASS', '" . env('DB_PASSWORD') . "' );\n";
+            fwrite($myfile, $content);
+        }
+    }
+}

tests/unit/workflow/engine/classes/DerivationTest.php

@@ -13,11 +13,13 @@ use ProcessMaker\Model\Route;
 use ProcessMaker\Model\Task;
 use ProcessMaker\Model\TaskUser;
 use ProcessMaker\Model\User;
+use Tests\CreateTestSite;
 use Tests\TestCase;
 
 class DerivationTest extends TestCase
 {
-
+    use CreateTestSite;
+    
     /**
      * Call the setUp parent method
      */
@@ -30,28 +32,7 @@ class DerivationTest extends TestCase
 
         config(["system.workspace" => "new_site"]);
         $workspace = config("system.workspace");
-
+        $this->createDBFile($workspace);
-        if (!file_exists(PATH_DB . $workspace)) {
-            G::mk_dir(PATH_DB . $workspace);
-        }
-
-        if (!file_exists(PATH_DB . $workspace . PATH_SEP . "db.php")) {
-            $myfile = fopen(PATH_DB . $workspace . PATH_SEP . "db.php", "w");
-            fwrite($myfile, "<?php
-define ('DB_ADAPTER',     'mysql' );
-define ('DB_HOST',        '" . env('DB_HOST') . "' );
-define ('DB_NAME',        '" . env('DB_DATABASE') . "' );
-define ('DB_USER',        '" . env('DB_USERNAME') . "' );
-define ('DB_PASS',        '" . env('DB_PASSWORD') . "' );
-define ('DB_RBAC_HOST',   '" . env('DB_HOST') . "' );
-define ('DB_RBAC_NAME',   '" . env('DB_DATABASE') . "' );
-define ('DB_RBAC_USER',   '" . env('DB_USERNAME') . "' );
-define ('DB_RBAC_PASS',   '" . env('DB_PASSWORD') . "' );
-define ('DB_REPORT_HOST', '" . env('DB_HOST') . "' );
-define ('DB_REPORT_NAME', '" . env('DB_DATABASE') . "' );
-define ('DB_REPORT_USER', '" . env('DB_USERNAME') . "' );
-define ('DB_REPORT_PASS', '" . env('DB_PASSWORD') . "' );");
-        }
     }
 
     /**

tests/unit/workflow/engine/classes/ReportTablesTest.php

@@ -7,11 +7,12 @@ use ProcessMaker\Model\Application;
 use ProcessMaker\Model\Process;
 use ProcessMaker\Model\Task;
 use ProcessMaker\Model\User;
+use Tests\CreateTestSite;
 use Tests\TestCase;
 
 class ReportTablesTest extends TestCase
 {
-
+    use CreateTestSite;
     use DatabaseTransactions;
 
     /**
@@ -20,7 +21,14 @@ class ReportTablesTest extends TestCase
     public function setUp()
     {
         parent::setUp();
+        $this->markTestIncomplete(""
+                . "This test has started using the ./processmaker command, this "
+                . "command requires the file 'paths_installed.php', that is, a "
+                . "valid installation of processmaker.");
         $_SERVER["REQUEST_URI"] = "";
+        config(["system.workspace" => "test"]);
+        $workspace = config("system.workspace");
+        $this->createDBFile($workspace);
     }
 
     /**

tests/unit/workflow/engine/controllers/PmTablesProxyTest.php

@@ -7,6 +7,7 @@ use G;
 use Illuminate\Foundation\Testing\DatabaseTransactions;
 use pmTablesProxy;
 use ProcessMaker\BusinessModel\ReportTable;
+use Tests\CreateTestSite;
 use Tests\TestCase;
 
 /**
@@ -14,6 +15,7 @@ use Tests\TestCase;
  */
 class PmTablesProxyTest extends TestCase
 {
+    use CreateTestSite;
     use DatabaseTransactions;
 
     protected $preserveGlobalState = false;
@@ -40,30 +42,8 @@ class PmTablesProxyTest extends TestCase
         parent::setUp();
 
         config(["system.workspace" => SYS_SYS]);
-
         $workspace = config("system.workspace");
-
+        $this->createDBFile($workspace);
-        if (!file_exists(PATH_DB . $workspace)) {
-            G::mk_dir(PATH_DB . $workspace);
-        }
-
-        if (!file_exists(PATH_DB . $workspace . PATH_SEP . "db.php")) {
-            $myfile = fopen(PATH_DB . $workspace . PATH_SEP . "db.php", "w");
-            fwrite($myfile, "<?php
-define ('DB_ADAPTER',     'mysql' );
-define ('DB_HOST',        '" . env('DB_HOST') . "' );
-define ('DB_NAME',        '" . env('DB_DATABASE') . "' );
-define ('DB_USER',        '" . env('DB_USERNAME') . "' );
-define ('DB_PASS',        '" . env('DB_PASSWORD') . "' );
-define ('DB_RBAC_HOST',   '" . env('DB_HOST') . "' );
-define ('DB_RBAC_NAME',   '" . env('DB_DATABASE') . "' );
-define ('DB_RBAC_USER',   '" . env('DB_USERNAME') . "' );
-define ('DB_RBAC_PASS',   '" . env('DB_PASSWORD') . "' );
-define ('DB_REPORT_HOST', '" . env('DB_HOST') . "' );
-define ('DB_REPORT_NAME', '" . env('DB_DATABASE') . "' );
-define ('DB_REPORT_USER', '" . env('DB_USERNAME') . "' );
-define ('DB_REPORT_PASS', '" . env('DB_PASSWORD') . "' );");
-        }
 
         //Set the user logged as the admin
         $_SESSION['USER_LOGGED'] = "00000000000000000000000000000001";

tests/unit/workflow/engine/src/ProcessMaker/Importer/ImporterTest.php

@@ -8,10 +8,13 @@ use ProcessMaker\Model\BpmnProject;
 use ProcessMaker\Model\Process;
 use ProcessMaker\Model\User;
 use ReflectionClass;
+use Tests\CreateTestSite;
 use Tests\TestCase;
 
 class ImporterTest extends TestCase
 {
+    use CreateTestSite;
+
     /**
      * Declared to avoid the incompatibility exception
      */
@@ -21,28 +24,7 @@ class ImporterTest extends TestCase
         error_reporting(E_ALL & ~E_DEPRECATED & ~E_STRICT);
         config(["system.workspace" => "test"]);
         $workspace = config("system.workspace");
-
+        $this->createDBFile($workspace);
-        if (!file_exists(PATH_DB . $workspace)) {
-            mkdir(PATH_DB . $workspace);
-        }
-
-        if (!file_exists(PATH_DB . $workspace . PATH_SEP . "db.php")) {
-            $myfile = fopen(PATH_DB . $workspace . PATH_SEP . "db.php", "w");
-            fwrite($myfile, "<?php
-define ('DB_ADAPTER',     'mysql' );
-define ('DB_HOST',        '" . env('DB_HOST') . "' );
-define ('DB_NAME',        '" . env('DB_DATABASE') . "' );
-define ('DB_USER',        '" . env('DB_USERNAME') . "' );
-define ('DB_PASS',        '" . env('DB_PASSWORD') . "' );
-define ('DB_RBAC_HOST',   '" . env('DB_HOST') . "' );
-define ('DB_RBAC_NAME',   '" . env('DB_DATABASE') . "' );
-define ('DB_RBAC_USER',   '" . env('DB_USERNAME') . "' );
-define ('DB_RBAC_PASS',   '" . env('DB_PASSWORD') . "' );
-define ('DB_REPORT_HOST', '" . env('DB_HOST') . "' );
-define ('DB_REPORT_NAME', '" . env('DB_DATABASE') . "' );
-define ('DB_REPORT_USER', '" . env('DB_USERNAME') . "' );
-define ('DB_REPORT_PASS', '" . env('DB_PASSWORD') . "' );");
-        }
     }
 
     /**

workflow/engine/bin/tasks/cliGenerateDataReport.php

@@ -0,0 +1,74 @@
+<?php
+
+use Illuminate\Support\Facades\DB;
+use ProcessMaker\Core\System;
+
+CLI::taskName("generate-data-report");
+CLI::taskRun("generateDataReport");
+CLI::taskDescription("\nGenerate data report by process in a respective workspace, you must pass through arguments the project identifier and the processing interval.");
+CLI::taskArg("workspace", false);
+CLI::taskOpt("uid", "Identifier that represents the process, must be 32 characters.", "", "process=");
+CLI::taskOpt("start", "The start option skips so many rows before returning results.", "", "start=");
+CLI::taskOpt("limit", "The limit option restricts the number of rows returned.", "", "limit=");
+
+/**
+ * Generate data report by process in a respective workspace, you must pass through 
+ * arguments the project identifier and the processing interval.
+ * @param array $options
+ * @return void
+ */
+function generateDataReport(array $options): void
+{
+    //get workspace
+    if (empty($options[0])) {
+        CLI::logging("Workspace undefined!\n");
+        return;
+    }
+    $workspace = $options[0];
+
+    //get options
+    $parameters = [
+        "tableName=" => "",
+        "type=" => "",
+        "process=" => "",
+        "gridKey=" => "",
+        "additionalTable=" => "",
+        "className=" => "",
+        "pathWorkspace=" => "",
+        "start=" => "",
+        "limit=" => ""
+    ];
+    foreach ($parameters as $key => $value) {
+        for ($i = 1; $i < count($options); $i++) {
+            if (strpos($options[$i], $key) !== false) {
+                $parameters[$key] = str_replace($key, "", $options[$i]);
+                break;
+            }
+        }
+    }
+
+    //validations
+    $needed = [
+        "process="
+    ];
+    foreach ($needed as $value) {
+        if (empty($parameters[$value])) {
+            CLI::logging("Missing options {$value}.\n");
+            return;
+        }
+    }
+
+    //run method
+    $workspaceTools = new WorkspaceTools($workspace);
+    $workspaceTools->generateDataReport(
+            $parameters["tableName="],
+            $parameters["type="],
+            $parameters["process="],
+            $parameters["gridKey="],
+            $parameters["additionalTable="],
+            $parameters["className="],
+            $parameters["pathWorkspace="],
+            (int) $parameters["start="],
+            (int) $parameters["limit="]
+    );
+}

workflow/engine/bin/tasks/cliPopulateTable.php

@@ -0,0 +1,21 @@
+<?php
+
+CLI::taskName("populate-table");
+CLI::taskRun("populateTable");
+CLI::taskDescription("\nThis function populates the report table with the APP_DATA data");
+CLI::taskArg("workspace", false);
+
+/**
+ * This function populates the report table with the APP_DATA data.
+ * @return void
+ */
+function populateTable($options): void
+{
+    //get options
+    $workspaceName = $options[0];
+    $query = base64_decode($options[1]);
+    $isRbac = (bool) $options[2];
+
+    $workspace = new WorkspaceTools($workspaceName);
+    $workspace->populateTableReport($query, $isRbac);
+}

workflow/engine/classes/ReportTables.php

@@ -1,6 +1,8 @@
 <?php
 
 use Illuminate\Support\Facades\DB;
+use ProcessMaker\Core\MultiProcOpen;
+use ProcessMaker\Commands\PopulateTableReport;
 use ProcessMaker\Model\Application;
 
 /**
@@ -207,168 +209,110 @@ class ReportTables
         //we have to do the propel connection
         $database = $this->chooseDB($connectionShortName);
         $connection = Propel::getConnection($database);
-        $statement = $connection->createStatement();
         if ($type == 'GRID') {
             $aux = explode('-', $grid);
             $grid = $aux[0];
         }
         $case = new Cases();
         try {
-            switch (DB_ADAPTER) {
+            $applicationTableName = (new Application())->getTable();
-                case 'mysql':
+            $deleteQuery = "TRUNCATE TABLE `{$tableName}` ";
-                    $applications = Application::getByProUid($proUid);
+            DB::delete($deleteQuery);
-                    foreach ($applications as $application) {
+
-                        $appData = $case->unserializeData($application->APP_DATA);
+            $applications = Application::getByProUid($proUid);
-                        DB::delete("DELETE FROM `{$tableName}` WHERE APP_UID = '{$application->APP_UID}'");
+            $i = 1;
-                        if ($type == 'NORMAL') {
+            $queryValues = "";
-                            $query = 'INSERT INTO `' . $tableName . '` (';
+            $numberRecords = 1000;
-                            $query .= '`APP_UID`,`APP_NUMBER`';
+            $n = count($applications);
-                            foreach ($fields as $field) {
+            foreach ($applications as $application) {
-                                $query .= ',`' . $field['sFieldName'] . '`';
+                $appData = $case->unserializeData($application->APP_DATA);
-                            }
+                if ($type == 'NORMAL') {
-                            $query .= ") VALUES ('" . $application->APP_UID . "'," . $application->APP_NUMBER;
+                    $query = 'INSERT INTO `' . $tableName . '` (';
+                    $query .= '`APP_UID`,`APP_NUMBER`';
+                    foreach ($fields as $field) {
+                        $query .= ',`' . $field['sFieldName'] . '`';
+                    }
+                    $headQuery = $query . ") VALUES ";
+                    $query = "('" . $application->APP_UID . "'," . $application->APP_NUMBER;
+                    foreach ($fields as $field) {
+                        switch ($field['sType']) {
+                            case 'number':
+                                $query .= ',' . (isset($appData[$field['sFieldName']]) ? (float) str_replace(',', '', $appData[$field['sFieldName']]) : '0');
+                                break;
+                            case 'char':
+                            case 'text':
+                                if (!isset($appData[$field['sFieldName']])) {
+                                    $appData[$field['sFieldName']] = '';
+                                }
+                                $string = $appData[$field['sFieldName']];
+                                if (is_array($string)) {
+                                    $string = implode($string, ",");
+                                }
+                                $query .= ",'" . (isset($appData[$field['sFieldName']]) ? mysqli_real_escape_string($connection->getResource(), $string) : '') . "'";
+                                break;
+                            case 'date':
+                                $value = (isset($appData[$field['sFieldName']]) && trim($appData[$field['sFieldName']])) != '' ? "'" . $appData[$field['sFieldName']] . "'" : 'NULL';
+                                $query .= "," . $value;
+                                break;
+                        }
+                    }
+                    $query .= ')';
+                    $queryValues = $queryValues . $query . ",";
+                    if ($i % $numberRecords === 0 || $i === $n) {
+                        $queryValues = rtrim($queryValues, ",");
+                        $query = $headQuery . $queryValues;
+                        $queryValues = "";
+                        $workspace = config("system.workspace");
+                        $processesManager = new MultiProcOpen();
+                        $processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) {
+                            return new PopulateTableReport($workspace, $query);
+                        });
+                    }
+                } else {
+                    if (isset($appData[$grid])) {
+                        $query = 'INSERT INTO `' . $tableName . '` (';
+                        $query .= '`APP_UID`,`APP_NUMBER`,`ROW`';
+                        foreach ($fields as $field) {
+                            $query .= ',`' . $field['sFieldName'] . '`';
+                        }
+                        $headQuery = $query . ") VALUES ";
+                        foreach ($appData[$grid] as $indexRow => $gridRow) {
+                            $query = "('" . $application->APP_UID . "'," . (int) $application->APP_NUMBER . ',' . $indexRow;
                             foreach ($fields as $field) {
                                 switch ($field['sType']) {
                                     case 'number':
-                                        $query .= ',' . (isset($appData[$field['sFieldName']]) ? (float) str_replace(',', '', $appData[$field['sFieldName']]) : '0');
+                                        $query .= ',' . (isset($gridRow[$field['sFieldName']]) ? (float) str_replace(',', '', $gridRow[$field['sFieldName']]) : '0');
                                         break;
                                     case 'char':
                                     case 'text':
-                                        if (!isset($appData[$field['sFieldName']])) {
+                                        if (!isset($gridRow[$field['sFieldName']])) {
-                                            $appData[$field['sFieldName']] = '';
+                                            $gridRow[$field['sFieldName']] = '';
                                         }
-                                        $string = $appData[$field['sFieldName']];
+                                        $stringEscape = mysqli_real_escape_string($connection->getResource(), $gridRow[$field['sFieldName']]);
-                                        if (is_array($string)) {
+                                        $query .= ",'" . (isset($gridRow[$field['sFieldName']]) ? $stringEscape : '') . "'";
-                                            $string = implode($string, ",");
-                                        }
-                                        $query .= ",'" . (isset($appData[$field['sFieldName']]) ? mysqli_real_escape_string($connection->getResource(), $string) : '') . "'";
                                         break;
                                     case 'date':
-                                        $value = (isset($appData[$field['sFieldName']]) && trim($appData[$field['sFieldName']])) != '' ? "'" . $appData[$field['sFieldName']] . "'" : 'NULL';
+                                        $value = (isset($gridRow[$field['sFieldName']]) && trim($gridRow[$field['sFieldName']])) != '' ? "'" . $gridRow[$field['sFieldName']] . "'" : 'NULL';
                                         $query .= "," . $value;
                                         break;
                                 }
                             }
                             $query .= ')';
-                            DB::insert($query);
+                            $queryValues = $queryValues . $query . ",";
-                        } else {
+                        }
-                            if (isset($appData[$grid])) {
+                        if ($i % $numberRecords === 0 || $i === $n) {
-                                foreach ($appData[$grid] as $indexRow => $gridRow) {
+                            $queryValues = rtrim($queryValues, ",");
-                                    $query = 'INSERT INTO `' . $tableName . '` (';
+                            $query = $headQuery . $queryValues;
-                                    $query .= '`APP_UID`,`APP_NUMBER`,`ROW`';
+                            $queryValues = "";
-                                    foreach ($fields as $field) {
+                            $workspace = config("system.workspace");
-                                        $query .= ',`' . $field['sFieldName'] . '`';
+                            $processesManager = new MultiProcOpen();
-                                    }
+                            $processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) {
-                                    $query .= ") VALUES ('" . $application->APP_UID . "'," . (int) $application->APP_NUMBER . ',' . $indexRow;
+                                return new PopulateTableReport($workspace, $query);
-                                    foreach ($fields as $field) {
+                            });
-                                        switch ($field['sType']) {
-                                            case 'number':
-                                                $query .= ',' . (isset($gridRow[$field['sFieldName']]) ? (float) str_replace(',', '', $gridRow[$field['sFieldName']]) : '0');
-                                                break;
-                                            case 'char':
-                                            case 'text':
-                                                if (!isset($gridRow[$field['sFieldName']])) {
-                                                    $gridRow[$field['sFieldName']] = '';
-                                                }
-                                                $stringEscape = mysqli_real_escape_string($connection->getResource(), $gridRow[$field['sFieldName']]);
-                                                $query .= ",'" . (isset($gridRow[$field['sFieldName']]) ? $stringEscape : '') . "'";
-                                                break;
-                                            case 'date':
-                                                $value = (isset($gridRow[$field['sFieldName']]) && trim($gridRow[$field['sFieldName']])) != '' ? "'" . $gridRow[$field['sFieldName']] . "'" : 'NULL';
-                                                $query .= "," . $value;
-                                                break;
-                                        }
-                                    }
-                                    $query .= ')';
-                                    DB::insert($query);
-                                }
-                            }
                         }
                     }
-                    break;
+                }
-
+                $i = $i + 1;
-                /**
-                 * For SQLServer code
-                 * Note: It is only possible to create Report Tables in MySQL databases. The list will only show connections to those databases even if the project has connections to other DBMS.
-                 * This section is not used and has been marked for deletion.
-                 * @link https://wiki.processmaker.com/3.0/Report_Tables#Creating_Report_Tables
-                 * @deprecated
-                 */
-                case 'mssql':
-                    $criteria = new Criteria('workflow');
-                    $criteria->add(ApplicationPeer::PRO_UID, $proUid);
-                    $criteria->addAscendingOrderByColumn(ApplicationPeer::APP_NUMBER);
-                    $dataset = ApplicationPeer::doSelectRS($criteria);
-                    $dataset->setFetchmode(ResultSet::FETCHMODE_ASSOC);
-                    $dataset->next();
-                    while ($row = $dataset->getRow()) {
-                        $appData = unserialize($row['APP_DATA']);
-                        //verify use mssql
-                        mysqli_query(
-                                $connection->getResource(), 'DELETE FROM [' . $tableName . "] WHERE APP_UID = '" . $row['APP_UID'] . "'"
-                        );
-                        if ($type == 'NORMAL') {
-                            $query = 'INSERT INTO [' . $tableName . '] (';
-                            $query .= '[APP_UID],[APP_NUMBER]';
-                            foreach ($fields as $field) {
-                                $query .= ',[' . $field['sFieldName'] . ']';
-                            }
-                            $query .= ") VALUES ('" . $row['APP_UID'] . "'," . (int) $row['APP_NUMBER'];
-                            foreach ($fields as $field) {
-                                switch ($field['sType']) {
-                                    case 'number':
-                                        $query .= ',' . (isset($appData[$field['sFieldName']]) ? (float) str_replace(',', '', $appData[$field['sFieldName']]) : '0');
-                                        break;
-                                    case 'char':
-                                    case 'text':
-                                        if (!isset($appData[$field['sFieldName']])) {
-                                            $appData[$field['sFieldName']] = '';
-                                        }
-                                        $stringEscape = mysqli_real_escape_string($connection->getResource(), $appData[$field['sFieldName']]);
-                                        $query .= ",'" . (isset($appData[$field['sFieldName']]) ? $stringEscape : '') . "'";
-                                        break;
-                                    case 'date':
-                                        $query .= ",'" . (isset($appData[$field['sFieldName']]) ? $appData[$field['sFieldName']] : '') . "'";
-                                        break;
-                                }
-                            }
-                            $query .= ')';
-                            $rs = $statement->executeQuery($query);
-                        } else {
-                            if (isset($appData[$grid])) {
-                                foreach ($appData[$grid] as $indexRow => $gridRow) {
-                                    $query = 'INSERT INTO [' . $tableName . '] (';
-                                    $query .= '`APP_UID`,`APP_NUMBER`,`ROW`';
-                                    foreach ($fields as $field) {
-                                        $query .= ',[' . $field['sFieldName'] . ']';
-                                    }
-                                    $query .= ") VALUES ('" . $row['APP_UID'] . "'," . (int) $row['APP_NUMBER'] . ',' . $indexRow;
-                                    foreach ($fields as $field) {
-                                        switch ($field['sType']) {
-                                            case 'number':
-                                                $query .= ',' . (isset($gridRow[$field['sFieldName']]) ? (float) str_replace(',', '', $gridRow[$field['sFieldName']]) : '0');
-                                                break;
-                                            case 'char':
-                                            case 'text':
-                                                if (!isset($gridRow[$field['sFieldName']])) {
-                                                    $gridRow[$field['sFieldName']] = '';
-                                                }
-                                                $stringEscape = mysqli_real_escape_string($connection->getResource(), $gridRow[$field['sFieldName']]);
-                                                $query .= ",'" . (isset($gridRow[$field['sFieldName']]) ? $stringEscape : '') . "'";
-                                                break;
-                                            case 'date':
-                                                $query .= ",'" . (isset($gridRow[$field['sFieldName']]) ? $gridRow[$field['sFieldName']] : '') . "'";
-                                                break;
-                                        }
-                                    }
-                                    $query .= ')';
-                                    $rs = $statement->executeQuery($query);
-                                }
-                            }
-                        }
-                        $dataset->next();
-                    }
-                    break;
             }
         } catch (Exception $oError) {
             throw ($oError);

workflow/engine/classes/WorkspaceTools.php

@@ -9,6 +9,8 @@ use ProcessMaker\ChangeLog\ChangeLog;
 use ProcessMaker\Core\Installer;
 use ProcessMaker\Core\ProcessesManager;
 use ProcessMaker\Core\System;
+use ProcessMaker\Model\Application;
+use ProcessMaker\Model\Fields;
 use ProcessMaker\Plugins\Adapters\PluginAdapter;
 use ProcessMaker\Project\Adapter\BpmnWorkflow;
 use ProcessMaker\Upgrade\RunProcessUpgradeQuery;
@@ -4821,4 +4823,167 @@ class WorkspaceTools
         $database = $this->getDatabase($rbac);
         $database->executeQuery($query, true);
     }
+    
+    /**
+     * This method regenerates data report with the APP_DATA data.
+     * @param string $tableName
+     * @param string $type
+     * @param string $processUid
+     * @param string $gridKey
+     * @param string $addTabUid
+     * @param string $className
+     * @param string $pathWorkspace
+     * @param int $start
+     * @param int $limit
+     * @throws Exception
+     */
+    public function generateDataReport(
+            $tableName, 
+            $type = 'NORMAL', 
+            $processUid = '', 
+            $gridKey = '', 
+            $addTabUid = '', 
+            $className = '', 
+            $pathWorkspace, 
+            int $start = 0, 
+            int $limit = 10)
+    {
+        $this->initPropel();
+        $dbHost = explode(':', $this->dbHost);
+        config(['database.connections.workflow.host' => $dbHost[0]]);
+        config(['database.connections.workflow.database' => $this->dbName]);
+        config(['database.connections.workflow.username' => $this->dbUser]);
+        config(['database.connections.workflow.password' => $this->dbPass]);
+        if (count($dbHost) > 1) {
+            config(['database.connections.workflow.port' => $dbHost[1]]);
+        }
+
+        //require file
+        set_include_path(get_include_path() . PATH_SEPARATOR . $pathWorkspace);
+        if (!file_exists($pathWorkspace . 'classes/' . $className . '.php')) {
+            throw new Exception("ERROR: " . $pathWorkspace . 'classes/' . $className . '.php' . " class file doesn't exit!");
+        }
+        require_once 'classes/model/AdditionalTables.php';
+        require_once $pathWorkspace . 'classes/' . $className . '.php';
+
+        //get fields
+        $fieldTypes = [];
+        if ($addTabUid != '') {
+            $fields = Fields::where('ADD_TAB_UID', '=', $addTabUid)->get();
+            foreach ($fields as $field) {
+                switch ($field->FLD_TYPE) {
+                    case 'FLOAT':
+                    case 'DOUBLE':
+                    case 'INTEGER':
+                        $fieldTypes[] = [$field->FLD_NAME => $field->FLD_TYPE];
+                        break;
+                    default:
+                        break;
+                }
+            }
+        }
+
+        $context = Bootstrap::getDefaultContextLog();
+        $case = new Cases();
+
+        //select cases for this Process, ordered by APP_NUMBER
+        $applications = Application::where('PRO_UID', '=', $processUid)
+                ->orderBy('APP_NUMBER', 'asc')
+                ->offset($start)
+                ->limit($limit)
+                ->get();
+        foreach ($applications as $application) {
+            //getting the case data
+            $appData = $case->unserializeData($application->APP_DATA);
+
+            //quick fix, map all empty values as NULL for Database
+            foreach ($appData as $appDataKey => $appDataValue) {
+                if (is_array($appDataValue) && count($appDataValue)) {
+                    $j = key($appDataValue);
+                    $appDataValue = is_array($appDataValue[$j]) ? $appDataValue : $appDataValue[$j];
+                }
+                if (is_string($appDataValue)) {
+                    foreach ($fieldTypes as $key => $fieldType) {
+                        foreach ($fieldType as $fieldTypeKey => $fieldTypeValue) {
+                            if (strtoupper($appDataKey) == $fieldTypeKey) {
+                                $appData[$appDataKey] = validateType($appDataValue, $fieldTypeValue);
+                                unset($fieldTypeKey);
+                            }
+                        }
+                    }
+                    // normal fields
+                    if (trim($appDataValue) === '') {
+                        $appData[$appDataKey] = null;
+                    }
+                } else {
+                    // grids
+                    if (is_array($appData[$appDataKey])) {
+                        foreach ($appData[$appDataKey] as $dIndex => $dRow) {
+                            if (is_array($dRow)) {
+                                foreach ($dRow as $k => $v) {
+                                    if (is_string($v) && trim($v) === '') {
+                                        $appData[$appDataKey][$dIndex][$k] = null;
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+
+            //populate data
+            if ($type === 'GRID') {
+                list($gridName, $gridUid) = explode('-', $gridKey);
+                $gridData = isset($appData[$gridName]) ? $appData[$gridName] : [];
+                foreach ($gridData as $i => $gridRow) {
+                    try {
+                        $obj = new $className();
+                        $obj->fromArray($appData, BasePeer::TYPE_FIELDNAME);
+                        $obj->setAppUid($application->APP_UID);
+                        $obj->setAppNumber($application->APP_NUMBER);
+                        if (method_exists($obj, 'setAppStatus')) {
+                            $obj->setAppStatus($application->APP_STATUS);
+                        }
+                        $obj->fromArray(array_change_key_case($gridRow, CASE_UPPER), BasePeer::TYPE_FIELDNAME);
+                        $obj->setRow($i);
+                        $obj->save();
+                    } catch (Exception $e) {
+                        $context["message"] = $e->getMessage();
+                        $context["tableName"] = $tableName;
+                        $context["appUid"] = $application->APP_UID;
+                        Bootstrap::registerMonolog("sqlExecution", 500, "Sql Execution", $context, $context["workspace"], "processmaker.log");
+                    }
+                    unset($obj);
+                }
+            } else {
+                try {
+                    $obj = new $className();
+                    $obj->fromArray(array_change_key_case($appData, CASE_UPPER), BasePeer::TYPE_FIELDNAME);
+                    $obj->setAppUid($application->APP_UID);
+                    $obj->setAppNumber($application->APP_NUMBER);
+                    if (method_exists($obj, 'setAppStatus')) {
+                        $obj->setAppStatus($application->APP_STATUS);
+                    }
+                    $obj->save();
+                } catch (Exception $e) {
+                    $context["message"] = $e->getMessage();
+                    $context["tableName"] = $tableName;
+                    $context["appUid"] = $application->APP_UID;
+                    Bootstrap::registerMonolog("sqlExecution", 500, "Sql Execution", $context, $context["workspace"], "processmaker.log");
+                }
+                unset($obj);
+            }
+        }
+    }
+
+    /**
+     * This method populates the table with a query string.
+     * @param string $query
+     * @param boolean $rbac
+     */
+    public function populateTableReport($query, $rbac)
+    {
+        $database = $this->getDatabase($rbac);
+        $database->executeQuery($query, true);
+    }
 }

workflow/engine/classes/model/AdditionalTables.php

@@ -1,23 +1,12 @@
 <?php
 
-/**
+use Illuminate\Support\Facades\DB;
- * AdditionalTables.php
+use ProcessMaker\Commands\GenerateDataReport;
- * @package    workflow.engine.classes.model
+use ProcessMaker\Core\MultiProcOpen;
- */
+use ProcessMaker\Model\Application;
-require_once 'classes/model/om/BaseAdditionalTables.php';
+use ProcessMaker\Model\Fields;
 
-/**
+require_once 'classes/model/om/BaseAdditionalTables.php';
- * Skeleton subclass for representing a row from the 'ADDITIONAL_TABLES' table.
- *
- *
- *
- * You should add additional methods to this class to meet the
- * application requirements.  This class will only be generated as
- * long as it does not already exist in the output directory.
- * <juliocesar@colosa.com, julces2000@gmail.com>
- *
- * @package    workflow.engine.classes.model
- */
 
 function validateType($value, $type)
 {
@@ -740,136 +729,34 @@ class AdditionalTables extends BaseAdditionalTables
      */
     public function populateReportTable($tableName, $sConnection = 'rp', $type = 'NORMAL', $processUid = '', $gridKey = '', $addTabUid = '')
     {
-        $this->className = $this->getPHPName($tableName);
+        $this->className = $className = $this->getPHPName($tableName);
-        $this->classPeerName = $this->className . 'Peer';
+        $this->classPeerName = $classPeerName = $className . 'Peer';
-
+        DB::statement("TRUNCATE " . $tableName);
-        if (!file_exists(PATH_WORKSPACE . 'classes/' . $this->className . '.php')) {
+        $workspace = config("system.workspace");
-            throw new Exception("ERROR: " . PATH_WORKSPACE . 'classes/' . $this->className . '.php' . " class file doesn't exit!");
+        $pathWorkspace = PATH_WORKSPACE;
-        }
+        $n = Application::count();
-
+        $processesManager = new MultiProcOpen();
-        require_once PATH_WORKSPACE . 'classes/' . $this->className . '.php';
+        $processesManager->chunk($n, 1000, function($size, $start, $limit) use(
-
+                $workspace, 
-        //get fields
+                $tableName, 
-        $fieldTypes = [];
+                $type, 
-        if ($addTabUid != '') {
+                $processUid, 
-            $criteria = new Criteria('workflow');
+                $gridKey, 
-            $criteria->add(FieldsPeer::ADD_TAB_UID, $addTabUid);
+                $addTabUid, 
-            $dataset = FieldsPeer::doSelectRS($criteria);
+                $className, 
-            $dataset->setFetchmode(ResultSet::FETCHMODE_ASSOC);
+                $pathWorkspace) {
-            while ($dataset->next()) {
+            return new GenerateDataReport(
-                $row = $dataset->getRow();
+                    $workspace, 
-                switch ($row['FLD_TYPE']) {
+                    $tableName, 
-                    case 'FLOAT':
+                    $type, 
-                    case 'DOUBLE':
+                    $processUid, 
-                    case 'INTEGER':
+                    $gridKey, 
-                        $fieldTypes[] = array($row['FLD_NAME'] => $row['FLD_TYPE']);
+                    $addTabUid, 
-                        break;
+                    $className, 
-                    default:
+                    $pathWorkspace, 
-                        break;
+                    $start, 
-                }
+                    $limit);
-            }
+        });
-        }
-
-        //remove old applications references
-        $connection = Propel::getConnection($sConnection);
-        $statement = $connection->createStatement();
-        $sql = "TRUNCATE " . $tableName;
-        $statement->executeQuery($sql);
-
-        $case = new Cases();
-        $context = Bootstrap::getDefaultContextLog();
-
-        //select cases for this Process, ordered by APP_NUMBER
-        $criteria = new Criteria('workflow');
-        $criteria->add(ApplicationPeer::PRO_UID, $processUid);
-        $criteria->addAscendingOrderByColumn(ApplicationPeer::APP_NUMBER);
-        $dataset = ApplicationPeer::doSelectRS($criteria);
-        $dataset->setFetchmode(ResultSet::FETCHMODE_ASSOC);
-        while ($dataset->next()) {
-            $row = $dataset->getRow();
-
-            //getting the case data
-            $appData = $case->unserializeData($row['APP_DATA']);
-
-            //quick fix, map all empty values as NULL for Database
-            foreach ($appData as $appDataKey => $appDataValue) {
-                if (is_array($appDataValue) && count($appDataValue)) {
-                    $j = key($appDataValue);
-                    $appDataValue = is_array($appDataValue[$j]) ? $appDataValue : $appDataValue[$j];
-                }
-                if (is_string($appDataValue)) {
-                    foreach ($fieldTypes as $key => $fieldType) {
-                        foreach ($fieldType as $fieldTypeKey => $fieldTypeValue) {
-                            if (strtoupper($appDataKey) == $fieldTypeKey) {
-                                $appData[$appDataKey] = validateType($appDataValue, $fieldTypeValue);
-                                unset($fieldTypeKey);
-                            }
-                        }
-                    }
-                    // normal fields
-                    if (trim($appDataValue) === '') {
-                        $appData[$appDataKey] = null;
-                    }
-                } else {
-                    // grids
-                    if (is_array($appData[$appDataKey])) {
-                        foreach ($appData[$appDataKey] as $dIndex => $dRow) {
-                            if (is_array($dRow)) {
-                                foreach ($dRow as $k => $v) {
-                                    if (is_string($v) && trim($v) === '') {
-                                        $appData[$appDataKey][$dIndex][$k] = null;
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-
-            //populate data
-            $className = $this->className;
-            if ($type === 'GRID') {
-                list($gridName, $gridUid) = explode('-', $gridKey);
-                $gridData = isset($appData[$gridName]) ? $appData[$gridName] : [];
-                foreach ($gridData as $i => $gridRow) {
-                    try {
-                        $obj = new $className();
-                        $obj->fromArray($appData, BasePeer::TYPE_FIELDNAME);
-                        $obj->setAppUid($row['APP_UID']);
-                        $obj->setAppNumber($row['APP_NUMBER']);
-                        if (method_exists($obj, 'setAppStatus')) {
-                            $obj->setAppStatus($row['APP_STATUS']);
-                        }
-                        $obj->fromArray(array_change_key_case($gridRow, CASE_UPPER), BasePeer::TYPE_FIELDNAME);
-                        $obj->setRow($i);
-                        $obj->save();
-                    } catch (Exception $e) {
-                        $context["message"] = $e->getMessage();
-                        $context["tableName"] = $tableName;
-                        $context["appUid"] = $row['APP_UID'];
-                        Bootstrap::registerMonolog("sqlExecution", 500, "Sql Execution", $context, $context["workspace"], "processmaker.log");
-                    }
-                    unset($obj);
-                }
-            } else {
-                try {
-                    $obj = new $className();
-                    $obj->fromArray(array_change_key_case($appData, CASE_UPPER), BasePeer::TYPE_FIELDNAME);
-                    $obj->setAppUid($row['APP_UID']);
-                    $obj->setAppNumber($row['APP_NUMBER']);
-                    if (method_exists($obj, 'setAppStatus')) {
-                        $obj->setAppStatus($row['APP_STATUS']);
-                    }
-                    $obj->save();
-                } catch (Exception $e) {
-                    $context["message"] = $e->getMessage();
-                    $context["tableName"] = $tableName;
-                    $context["appUid"] = $row['APP_UID'];
-                    Bootstrap::registerMonolog("sqlExecution", 500, "Sql Execution", $context, $context["workspace"], "processmaker.log");
-                }
-                unset($obj);
-            }
-        }
     }
 
     /**

workflow/engine/src/ProcessMaker/Commands/GenerateDataReport.php

@@ -0,0 +1,80 @@
+<?php
+
+namespace ProcessMaker\Commands;
+
+use ProcessMaker\Core\ProcOpen;
+
+class GenerateDataReport extends ProcOpen
+{
+    private $workspace;
+    private $tableName;
+    private $type;
+    private $processUid;
+    private $gridKey;
+    private $addTabUid;
+    private $className;
+    private $pathWorkspace;
+    private $start;
+    private $limit;
+
+    /**
+     * Initializes the command parameters.
+     * @param string $workspace
+     * @param string $tableName
+     * @param string $type
+     * @param string $processUid
+     * @param string $gridKey
+     * @param string $addTabUid
+     * @param string $className
+     * @param string $pathWorkspace
+     * @param integer $start
+     * @param integer $limit
+     */
+    public function __construct(
+            $workspace, 
+            $tableName, 
+            $type = 'NORMAL', 
+            $processUid = '', 
+            $gridKey = '', 
+            $addTabUid = '', 
+            $className = '', 
+            $pathWorkspace, 
+            $start = 0, 
+            $limit = 10)
+    {
+        $this->workspace = $workspace;
+        $this->tableName = $tableName;
+        $this->type = $type;
+        $this->processUid = $processUid;
+        $this->gridKey = $gridKey;
+        $this->addTabUid = $addTabUid;
+        $this->className = $className;
+        $this->pathWorkspace = $pathWorkspace;
+        $this->start = $start;
+        $this->limit = $limit;
+        $this->setCwd(PATH_TRUNK);
+        parent::__construct($this->buildCommand());
+    }
+
+    /**
+     * Returns the command to execute.
+     * @return string
+     */
+    private function buildCommand(): string
+    {
+        $command = PHP_BINDIR . "/php "
+                . "./processmaker "
+                . "'generate-data-report' "
+                . "'{$this->workspace}' "
+                . "'tableName={$this->tableName}' "
+                . "'type={$this->type}' "
+                . "'process={$this->processUid}' "
+                . "'gridKey={$this->gridKey}' "
+                . "'additionalTable={$this->addTabUid}' "
+                . "'className={$this->className}' "
+                . "'pathWorkspace={$this->pathWorkspace}' "
+                . "'start={$this->start}' "
+                . "'limit={$this->limit}' ";
+        return $command;
+    }
+}

workflow/engine/src/ProcessMaker/Commands/PopulateTableReport.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace ProcessMaker\Commands;
+
+use ProcessMaker\Core\ProcOpen;
+
+class PopulateTableReport extends ProcOpen
+{
+    private $workspace;
+    private $sql;
+    private $isRbac;
+
+    /**
+     * Initializes the command parameters.
+     * @param string $workspace
+     * @param string $sql
+     * @param boolean $isRbac
+     */
+    public function __construct($workspace, $sql, $isRbac = false)
+    {
+        $this->workspace = $workspace;
+        $this->sql = $sql;
+        $this->isRbac = $isRbac;
+        $this->setCwd(PATH_TRUNK);
+        parent::__construct($this->buildCommand());
+    }
+
+    /**
+     * Returns the command to execute.
+     * @return string
+     */
+    public function buildCommand()
+    {
+        $command = PHP_BINDIR . "/php "
+                . "./processmaker "
+                . "'populate-table' "
+                . "'{$this->workspace}' "
+                . base64_encode($this->sql) . " "
+                . ($this->isRbac ? "'1'" : "'0'");
+        return $command;
+    }
+}

workflow/engine/src/ProcessMaker/Core/MultiProcOpen.php

@@ -0,0 +1,89 @@
+<?php
+
+namespace ProcessMaker\Core;
+
+class MultiProcOpen
+{
+    /**
+     * Represents the waiting time before starting the process monitoring.
+     * @var integer 
+     */
+    private $sleepTime = 1;
+
+    /**
+     * This method obtains a paging by returning the start and limit indexes 
+     * compatible with the mysql pagination in its call function.
+     * The return function must return an instance of the object "ProcessMaker\Core\ProcOpen".
+     * Returns an array containing the status, content, and errors generated by 
+     * the open process.
+     * @param int $size
+     * @param int $chunk
+     * @param callable $callback
+     * @return array
+     */
+    public function chunk(int $size, int $chunk, callable $callback): array
+    {
+        $start = 0;
+        $limit = $chunk;
+        $queries = [];
+        for ($i = 1; $start < $size; $i++) {
+            $queries[] = $callback($size, $start, $limit);
+            $start = $i * $limit;
+        }
+        return $this->run($queries);
+    }
+
+    /**
+     * Open a set of background processes.
+     * The array must contain one or more instances of the object inherited from 
+     * the class "ProcessMaker\Core\ProcOpen"
+     * Returns an array containing the status, content, and errors generated by 
+     * the open process.
+     * @param array $processes
+     * @return array
+     */
+    public function run(array $processes): array
+    {
+        foreach ($processes as $procOpen) {
+            $procOpen->open();
+        }
+        return $this->processMonitoring($processes);
+    }
+
+    /**
+     * It monitors the open processes, verifying if they have ended or thrown an 
+     * error and later closing the resources related to the process.
+     * Returns an array containing the status, content, and errors generated by 
+     * the open process.
+     * @param array $processes
+     * @return array
+     */
+    private function processMonitoring(array $processes): array
+    {
+        sleep($this->sleepTime); //this sleep is very important
+        $i = 0;
+        $n = count($processes);
+        $outputs = [];
+        do {
+            $index = $i % $n;
+            if (isset($processes[$index])) {
+                $procOpen = $processes[$index];
+                $status = $procOpen->getStatus();
+                $contents = $procOpen->getContents();
+                $errors = $procOpen->getErrors();
+                if ($status->running === false || !empty($errors)) {
+                    $outputs[] = [
+                        "status" => $status,
+                        "contents" => $contents,
+                        "errors" => $errors,
+                    ];
+                    $procOpen->terminate();
+                    $procOpen->close();
+                    unset($processes[$index]);
+                }
+            }
+            $i = $i + 1;
+        } while (!empty($processes));
+        return $outputs;
+    }
+}

workflow/engine/src/ProcessMaker/Core/ProcOpen.php

@@ -0,0 +1,126 @@
+<?php
+
+namespace ProcessMaker\Core;
+
+class ProcOpen
+{
+    private $command;
+    private $resource;
+    private $descriptorspec;
+    private $pipes;
+    private $cwd;
+
+    /**
+     * This initializes the descriptors and the command for the open process.
+     * @param string $command
+     */
+    public function __construct(string $command)
+    {
+        $this->descriptorspec = [
+            ['pipe', 'r'],
+            ['pipe', 'w'],
+            ['pipe', 'w']
+        ];
+        $this->command = $command;
+    }
+
+    /**
+     * Gets the resource that represents the process.
+     * @return resource
+     */
+    public function getResource()
+    {
+        return $this->resource;
+    }
+
+    /**
+     * Sets the process execution directory.
+     * @param string $cwd
+     */
+    public function setCwd(string $cwd)
+    {
+        $this->cwd = $cwd;
+    }
+
+    /**
+     * Open a background process.
+     */
+    public function open()
+    {
+        if (empty($this->cwd)) {
+            $this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes);
+        } else {
+            $this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes, $this->cwd);
+        }
+    }
+
+    /**
+     * Get the content of the process when it is finished.
+     * @return string
+     */
+    public function getContents()
+    {
+        if (is_resource($this->pipes[1])) {
+            return stream_get_contents($this->pipes[1]);
+        }
+        return "";
+    }
+
+    /**
+     * Get the process errors when it is finished.
+     * @return string
+     */
+    public function getErrors()
+    {
+        if (is_resource($this->pipes[2])) {
+            return stream_get_contents($this->pipes[2]);
+        }
+        return "";
+    }
+
+    /**
+     * Close the resources related to the open process.
+     * return void
+     */
+    public function close()
+    {
+        if (is_resource($this->resource)) {
+            foreach ($this->pipes as $value) {
+                fclose($value);
+            }
+            proc_close($this->resource);
+        }
+    }
+
+    /**
+     * End the process before it ends.
+     */
+    public function terminate()
+    {
+        if (is_resource($this->resource)) {
+            proc_terminate($this->resource);
+        }
+    }
+
+    /**
+     * Gets the status of the process.
+     * @return object
+     */
+    public function getStatus()
+    {
+        $status = [
+            "command" => $this->command,
+            "pid" => null,
+            "running" => false,
+            "signaled" => false,
+            "stopped" => false,
+            "exitcode" => -1,
+            "termsig" => 0,
+            "stopsig" => 0
+        ];
+        if (is_resource($this->resource)) {
+            $status = proc_get_status($this->resource);
+        }
+        return (object) $status;
+    }
+}

workflow/engine/src/ProcessMaker/Model/Application.php

@@ -8,6 +8,8 @@ use Illuminate\Support\Facades\DB;
 class Application extends Model
 {
     protected $table = "APPLICATION";
+    protected $primaryKey = 'APP_NUMBER';
+    public $incrementing = false;
     // No timestamps
     public $timestamps = false;