PMCORE-1527 PMCORE-1421 - Regenerate PM Report Tables Asynchronously

app/Jobs/GenerateReportTable.php

@@ -0,0 +1,14 @@
+<?php
+
+namespace App\Jobs;
+
+class GenerateReportTable extends QueuedClosure
+{
+    /**
+     * The number of times the job may be attempted.
+     *
+     * @var int
+     */
+    public $tries = 1;
+
+}

tests/resources/structureReportTable.json

@@ -0,0 +1,33 @@
+{
+    "data": {
+        "var_Text1": "5298067855ed6e6893e1424067710552",
+        "var_Textarea1": "4835809915ed6e6893e14a6079656976",
+        "var_Dropdown1": "5287566085ed6e6893e14e0083475368",
+        "var_Suggest1": "4813021735ed6e6893e1520081779769",
+        "var_DateTime1": "1975-04-04 16:34:22",
+        "var_String1": "7984290665ed6e6893e15b2092042769",
+        "var_Integer1": "44"
+    },
+    "mapping": [{
+            "sFieldName": "var_Text1",
+            "sType": "char"
+        }, {
+            "sFieldName": "var_Textarea1",
+            "sType": "text"
+        }, {
+            "sFieldName": "var_Dropdown1",
+            "sType": "char"
+        }, {
+            "sFieldName": "var_Suggest1",
+            "sType": "char"
+        }, {
+            "sFieldName": "var_DateTime1",
+            "sType": "date"
+        }, {
+            "sFieldName": "var_String1",
+            "sType": "char"
+        }, {
+            "sFieldName": "var_Integer1",
+            "sType": "number"
+        }]
+}

tests/unit/workflow/engine/classes/ReportTablesTest.php

@@ -12,6 +12,7 @@ use Tests\TestCase;
 
 class ReportTablesTest extends TestCase
 {
+
     use CreateTestSite;
     use DatabaseTransactions;
 
@@ -21,14 +22,12 @@ class ReportTablesTest extends TestCase
     public function setUp()
     {
         parent::setUp();
-        $this->markTestIncomplete(""
-                . "This test has started using the ./processmaker command, this "
-                . "command requires the file 'paths_installed.php', that is, a "
-                . "valid installation of processmaker.");
         $_SERVER["REQUEST_URI"] = "";
+        config(['queue.default' => 'sync']);
         config(["system.workspace" => "test"]);
         $workspace = config("system.workspace");
         $this->createDBFile($workspace);
+        $this->createConstantsOfConnection();
     }
 
     /**
@@ -39,6 +38,33 @@ class ReportTablesTest extends TestCase
         parent::tearDown();
     }
 
+    /**
+     * Create constants of connection to databases.
+     */
+    private function createConstantsOfConnection()
+    {
+        $constants = [
+            'DB_ADAPTER' => env('mysql'),
+            'DB_HOST' => env('DB_HOST'),
+            'DB_NAME' => env('DB_DATABASE'),
+            'DB_USER' => env('DB_USERNAME'),
+            'DB_PASS' => env('DB_PASSWORD'),
+            'DB_RBAC_HOST' => env('DB_HOST'),
+            'DB_RBAC_NAME' => env('DB_DATABASE'),
+            'DB_RBAC_USER' => env('DB_USERNAME'),
+            'DB_RBAC_PASS' => env('DB_PASSWORD'),
+            'DB_REPORT_HOST' => env('DB_HOST'),
+            'DB_REPORT_NAME' => env('DB_DATABASE'),
+            'DB_REPORT_USER' => env('DB_USERNAME'),
+            'DB_REPORT_PASS' => env('DB_PASSWORD'),
+        ];
+        foreach ($constants as $key => $value) {
+            if (!defined($key)) {
+                define($key, $value);
+            }
+        }
+    }
+
     /**
      * Check if the "populateTable" function returns an array value if entered all parameters.
      * @test
@@ -565,97 +591,6 @@ class ReportTablesTest extends TestCase
         $this->assertEquals($expected, $actual);
     }
 
-    /**
-     * Get mapping fields supported by report table.
-     * @return array
-     */
-    private function getMapFields()
-    {
-        return [
-            [
-                'sFieldName' => 'var_Text1',
-                'sType' => 'char'
-            ],
-            [
-                'sFieldName' => 'var_Textarea1',
-                'sType' => 'text'
-            ],
-            [
-                'sFieldName' => 'var_Dropdown1',
-                'sType' => 'char'
-            ],
-            [
-                'sFieldName' => 'var_Suggest1',
-                'sType' => 'char'
-            ],
-            [
-                'sFieldName' => 'var_DateTime1',
-                'sType' => 'date'
-            ],
-            [
-                'sFieldName' => 'var_String1',
-                'sType' => 'char'
-            ],
-            [
-                'sFieldName' => 'var_Integer1',
-                'sType' => 'number'
-            ],
-            [
-                'sFieldName' => 'var_Boolean1',
-                'sType' => 'boolean'
-            ],
-            [
-                'sFieldName' => 'var_Array1',
-                'sType' => 'array'
-            ]
-        ];
-    }
-
-    /**
-     * Create fields data by type supported.
-     * @param array $types
-     * @return array
-     */
-    private function createFieldsByType($types = [])
-    {
-        $fields = [];
-        $mapping = [];
-        $faker = Faker\Factory::create();
-        $date = $faker->dateTime();
-        $mapFields = $this->getMapFields();
-        foreach ($mapFields as $key => $value) {
-            if (!in_array($value['sType'], $types)) {
-                continue;
-            }
-            switch ($value['sType']) {
-                case 'number':
-                    $mapping[] = $value;
-                    $fields[$value['sFieldName']] = (string) random_int(0, 100);
-                    break;
-                case 'char':
-                    $mapping[] = $value;
-                    $fields[$value['sFieldName']] = G::generateUniqueID();
-                    break;
-                case 'text':
-                    $mapping[] = $value;
-                    $fields[$value['sFieldName']] = G::generateUniqueID();
-                    break;
-                case 'date':
-                    $mapping[] = $value;
-                    $fields[$value['sFieldName']] = $date->format('Y-m-d H:i:s');
-                    break;
-                case 'boolean':
-                    $mapping[] = $value;
-                    $fields[$value['sFieldName']] = ['0' => 0];
-                    break;
-            }
-        }
-        return [
-            'data' => $fields,
-            'mapping' => $mapping
-        ];
-    }
-
     /**
      * Prepare data initial for test, the grid parameter is optional if you want 
      * to create a grid type field.
@@ -665,7 +600,7 @@ class ReportTablesTest extends TestCase
      * @param boolean $grid
      * @return object
      */
-    private function prepareData($tableName, $grid = null)
+    private function prepareData($tableName, $grid = null, $structure = [])
     {
         $applicationNumber = Application::max('APP_NUMBER');
         if (is_null($applicationNumber)) {
@@ -681,7 +616,10 @@ class ReportTablesTest extends TestCase
         $taskUid = G::generateUniqueID();
         $applicationUid = G::generateUniqueID();
 
-        $structure = $this->createFieldsByType(['number', 'char', 'text', 'date']);
+        if (empty($structure)) {
+            $structure = $this->getDataFromFile('structureReportTable.json');
+        }
+
         $fields = $structure['mapping'];
         $dataFields = $structure['data'];
         $appData = [
@@ -753,4 +691,157 @@ class ReportTablesTest extends TestCase
         $result->application = $application;
         return $result;
     }
+
+    /**
+     * Check if the "populateTable" method is it filling with missing values into app_data.
+     * @test
+     * @covers ReportTables::populateTable
+     */
+    public function it_should_populating_data_with_fields_missing_in_to_app_data()
+    {
+        $tableName = 'TestReportTable';
+        $result = $this->prepareData($tableName);
+        $connectionShortName = 'wf';
+        $type = 'NORMAL';
+        $fields = $result->fields;
+        $proUid = $result->processUid;
+        $grid = '';
+
+        $app = Application::where('APP_UID', '=', $result->applicationUid)->get()->first();
+        $appData = unserialize($app->APP_DATA);
+        unset($appData['var_Textarea1']);
+        $appData = serialize($appData);
+        Application::where('APP_UID', '=', $result->applicationUid)->update(['APP_DATA' => $appData]);
+
+        $reportTables = new ReportTables();
+        $reportTables->populateTable($tableName, $connectionShortName, $type, $fields, $proUid, $grid);
+
+        $expected = $result->dataFields;
+        $expected['APP_UID'] = $result->applicationUid;
+        $expected['APP_NUMBER'] = $result->applicationNumber;
+        $expected['var_Textarea1'] = '';
+
+        $actual = (array) DB::table($tableName)
+                        ->select()
+                        ->first();
+
+        $this->assertEquals($expected, $actual);
+    }
+
+    /**
+     * Check if the "populateTable" method is it filling with arrays values.
+     * @test
+     * @covers ReportTables::populateTable
+     */
+    public function it_should_populating_data_with_arrays_values()
+    {
+        $tableName = 'TestReportTable';
+        $result = $this->prepareData($tableName);
+        $connectionShortName = 'wf';
+        $type = 'NORMAL';
+        $fields = $result->fields;
+        $proUid = $result->processUid;
+        $grid = '';
+
+        $app = Application::where('APP_UID', '=', $result->applicationUid)->get()->first();
+        $appData = unserialize($app->APP_DATA);
+        $appData['var_Textarea1'] = [];
+        $appData = serialize($appData);
+        Application::where('APP_UID', '=', $result->applicationUid)->update(['APP_DATA' => $appData]);
+
+        $reportTables = new ReportTables();
+        $reportTables->populateTable($tableName, $connectionShortName, $type, $fields, $proUid, $grid);
+
+        $expected = $result->dataFields;
+        $expected['APP_UID'] = $result->applicationUid;
+        $expected['APP_NUMBER'] = $result->applicationNumber;
+        $expected['var_Textarea1'] = '';
+
+        $actual = (array) DB::table($tableName)
+                        ->select()
+                        ->first();
+
+        $this->assertEquals($expected, $actual);
+    }
+
+    /**
+     * Check if the "populateTable" method is it filling with missing values into app_data for grids control.
+     * parameters and type and grid are correct values.
+     * @test
+     * @covers ReportTables::populateTable
+     */
+    public function it_should_populating_data_with_all_parameters_with_type_is_grid_fields_missing_in_to_app_data()
+    {
+        $tableName = 'TestReportTable';
+        $result = $this->prepareData($tableName, true);
+        $connectionShortName = 'wf';
+        $type = 'GRID';
+        $fields = $result->fields;
+        $proUid = $result->processUid;
+        $grid = 'var_Grid1';
+
+        $app = Application::where('APP_UID', '=', $result->applicationUid)->get()->first();
+        $appData = unserialize($app->APP_DATA);
+        unset($appData['var_Grid1'][1]['var_Textarea1']);
+        $appData = serialize($appData);
+        Application::where('APP_UID', '=', $result->applicationUid)->update(['APP_DATA' => $appData]);
+
+        $reportTables = new ReportTables();
+        $reportTables->populateTable($tableName, $connectionShortName, $type, $fields, $proUid, $grid);
+
+        $indexRow = 1;
+        $expected = $result->appData[$grid];
+        foreach ($expected as &$row) {
+            $row['APP_UID'] = $result->applicationUid;
+            $row['APP_NUMBER'] = $result->applicationNumber;
+            $row['ROW'] = (string) ($indexRow++);
+        }
+        $expected = array_values($expected);
+        $expected[0]['var_Textarea1'] = '';
+
+        $actual = DB::table($tableName)
+                ->select()
+                ->get();
+        $actual->transform(function ($item, $key) {
+            return (array) $item;
+        });
+        $actual = $actual->toArray();
+
+        $this->assertEquals($expected, $actual);
+    }
+
+    /**
+     * Check an exception if the input parameters are wrong.
+     * @test
+     * @covers ReportTables::populateTable
+     */
+    public function it_should_catch_an_exception()
+    {
+        $tableName = 'TestReportTable';
+        $result = $this->prepareData($tableName, true);
+        $connectionShortName = 'wf';
+        $type = 'GRID';
+        $fields = $result->fields;
+        $proUid = $result->processUid;
+        $grid = 'var_Grid1';
+
+        //assert exception
+        $this->expectException(Exception::class);
+
+        $reportTables = new ReportTables();
+        $reportTables->populateTable($tableName, $connectionShortName, $type, null, $proUid, $grid);
+    }
+
+    /**
+     * This gets data from a json file.
+     * @param string $pathData
+     * @return array
+     */
+    private function getDataFromFile(string $pathData): array
+    {
+        $pathData = PATH_TRUNK . "tests/resources/{$pathData}";
+        $data = file_get_contents($pathData);
+        $result = json_decode($data, JSON_OBJECT_AS_ARRAY);
+        return $result;
+    }
 }

tests/unit/workflow/engine/classes/model/AdditionalTablesTest.php

@@ -3,17 +3,32 @@
 namespace Tests\unit\workflow\engine\classes\model;
 
 use AdditionalTables;
+use App\Jobs\GenerateReportTable;
 use Exception;
 use G;
 use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Queue;
 use Illuminate\Support\Facades\Schema;
 use ProcessMaker\BusinessModel\ReportTable;
 use ProcessMaker\Model\AdditionalTables as AdditionalTablesModel;
+use ProcessMaker\Model\Application;
+use ProcessMaker\Model\DbSource;
+use ProcessMaker\Model\Delegation;
+use ProcessMaker\Model\Process;
+use ProcessMaker\Model\Task;
 use Tests\TestCase;
 
 class AdditionalTablesTest extends TestCase
 {
 
+    /**
+     * Set up method.
+     */
+    public function setUp()
+    {
+        parent::setUp();
+    }
+
     /**
      * This tests the creation of a PMTable.
      * @test
@@ -198,6 +213,74 @@ class AdditionalTablesTest extends TestCase
         $this->assertContains($actual[0], $expected, false);
     }
 
+    /**
+     * Check if populate report table is added to job queue.
+     * @test
+     * @covers \AdditionalTables::populateReportTable
+     */
+    public function it_should_test_populate_report_table()
+    {
+        $proUid = factory(Process::class)->create()->PRO_UID;
+
+        $task = factory(Task::class)->create([
+            'PRO_UID' => $proUid
+        ]);
+
+        //local connections
+        $dbSource = factory(DbSource::class)->create([
+            'PRO_UID' => $proUid,
+            'DBS_SERVER' => env('DB_HOST'),
+            'DBS_DATABASE_NAME' => env('DB_DATABASE'),
+            'DBS_USERNAME' => env('DB_USERNAME'),
+            'DBS_PASSWORD' => G::encrypt(env('DB_PASSWORD'), env('DB_DATABASE')) . "_2NnV3ujj3w",
+            'DBS_PORT' => '3306',
+            'DBS_CONNECTION_TYPE' => 'NORMAL'
+        ]);
+        $additionalTable = factory(AdditionalTablesModel::class)->create([
+            'PRO_UID' => $proUid,
+            'DBS_UID' => $dbSource->DBS_UID,
+        ]);
+        $tableName = $additionalTable->ADD_TAB_NAME;
+        $name = $additionalTable->ADD_TAB_CLASS_NAME;
+        $this->createSchema($dbSource->DBS_DATABASE_NAME, $tableName, $name, $dbSource->DBS_UID);
+
+        //external connection
+        $dbSource = factory(DbSource::class)->create([
+            'PRO_UID' => $proUid,
+            'DBS_SERVER' => config('database.connections.testexternal.host'),
+            'DBS_DATABASE_NAME' => config('database.connections.testexternal.database'),
+            'DBS_USERNAME' => config('database.connections.testexternal.username'),
+            'DBS_PASSWORD' => G::encrypt(config('database.connections.testexternal.password'), config('database.connections.testexternal.database')) . "_2NnV3ujj3w",
+            'DBS_PORT' => '3306',
+            'DBS_CONNECTION_TYPE' => 'NORMAL'
+        ]);
+        $additionalTable = factory(AdditionalTablesModel::class)->create([
+            'PRO_UID' => $proUid,
+            'DBS_UID' => $dbSource->DBS_UID,
+        ]);
+        $tableNameExternal = $additionalTable->ADD_TAB_NAME;
+        $nameExternal = $additionalTable->ADD_TAB_CLASS_NAME;
+        $this->createSchema($dbSource->DBS_DATABASE_NAME, $tableNameExternal, $nameExternal, $dbSource->DBS_UID);
+
+        $application = factory(Application::class)->create([
+            'PRO_UID' => $proUid
+        ]);
+        factory(Delegation::class)->create([
+            'DEL_THREAD_STATUS' => 'CLOSED',
+            'APP_NUMBER' => $application->APP_NUMBER,
+            'TAS_UID' => $task->TAS_UID,
+        ]);
+
+        //assertions
+        Queue::fake();
+        Queue::assertNothingPushed();
+
+        $additionalTables = new AdditionalTables();
+        $additionalTables->populateReportTable($tableName, 'workflow', 'NORMAL', $proUid, '', $additionalTable->ADD_TAB_UID);
+
+        Queue::assertPushed(GenerateReportTable::class);
+    }
+
     /**
      * This gets the content from template file.
      * @param string $pathData

tests/unit/workflow/engine/src/ProcessMaker/Core/JobsManagerTest.php

@@ -2,6 +2,7 @@
 
 namespace ProcessMaker\Core;
 
+use App\Jobs\Email;
 use Tests\TestCase;
 
 class JobsManagerTest extends TestCase
@@ -116,7 +117,7 @@ class JobsManagerTest extends TestCase
         $callback = function() {
         };
 
-        $actual = $this->object->dispatch('Email', $callback);
+        $actual = $this->object->dispatch(Email::class, $callback);
 
         $this->assertInstanceOf(\Illuminate\Foundation\Bus\PendingDispatch::class, $actual);
     }

workflow/engine/bin/tasks/cliGenerateDataReport.php

@@ -1,74 +0,0 @@
-<?php
-
-use Illuminate\Support\Facades\DB;
-use ProcessMaker\Core\System;
-
-CLI::taskName("generate-data-report");
-CLI::taskRun("generateDataReport");
-CLI::taskDescription("\nGenerate data report by process in a respective workspace, you must pass through arguments the project identifier and the processing interval.");
-CLI::taskArg("workspace", false);
-CLI::taskOpt("uid", "Identifier that represents the process, must be 32 characters.", "", "process=");
-CLI::taskOpt("start", "The start option skips so many rows before returning results.", "", "start=");
-CLI::taskOpt("limit", "The limit option restricts the number of rows returned.", "", "limit=");
-
-/**
- * Generate data report by process in a respective workspace, you must pass through 
- * arguments the project identifier and the processing interval.
- * @param array $options
- * @return void
- */
-function generateDataReport(array $options): void
-{
-    //get workspace
-    if (empty($options[0])) {
-        CLI::logging("Workspace undefined!\n");
-        return;
-    }
-    $workspace = $options[0];
-
-    //get options
-    $parameters = [
-        "tableName=" => "",
-        "type=" => "",
-        "process=" => "",
-        "gridKey=" => "",
-        "additionalTable=" => "",
-        "className=" => "",
-        "pathWorkspace=" => "",
-        "start=" => "",
-        "limit=" => ""
-    ];
-    foreach ($parameters as $key => $value) {
-        for ($i = 1; $i < count($options); $i++) {
-            if (strpos($options[$i], $key) !== false) {
-                $parameters[$key] = str_replace($key, "", $options[$i]);
-                break;
-            }
-        }
-    }
-
-    //validations
-    $needed = [
-        "process="
-    ];
-    foreach ($needed as $value) {
-        if (empty($parameters[$value])) {
-            CLI::logging("Missing options {$value}.\n");
-            return;
-        }
-    }
-
-    //run method
-    $workspaceTools = new WorkspaceTools($workspace);
-    $workspaceTools->generateDataReport(
-            $parameters["tableName="],
-            $parameters["type="],
-            $parameters["process="],
-            $parameters["gridKey="],
-            $parameters["additionalTable="],
-            $parameters["className="],
-            $parameters["pathWorkspace="],
-            (int) $parameters["start="],
-            (int) $parameters["limit="]
-    );
-}

workflow/engine/bin/tasks/cliPopulateTable.php

@@ -1,21 +0,0 @@
-<?php
-
-CLI::taskName("populate-table");
-CLI::taskRun("populateTable");
-CLI::taskDescription("\nThis function populates the report table with the APP_DATA data");
-CLI::taskArg("workspace", false);
-
-/**
- * This function populates the report table with the APP_DATA data.
- * @return void
- */
-function populateTable($options): void
-{
-    //get options
-    $workspaceName = $options[0];
-    $query = base64_decode($options[1]);
-    $isRbac = (bool) $options[2];
-
-    $workspace = new WorkspaceTools($workspaceName);
-    $workspace->populateTableReport($query, $isRbac);
-}

workflow/engine/classes/ReportTables.php

@@ -1,8 +1,8 @@
 <?php
 
+use App\Jobs\GenerateReportTable;
 use Illuminate\Support\Facades\DB;
-use ProcessMaker\Core\MultiProcOpen;
+use ProcessMaker\Core\JobsManager;
-use ProcessMaker\Commands\PopulateTableReport;
 use ProcessMaker\Model\Application;
 
 /**
@@ -205,6 +205,9 @@ class ReportTables
      */
     public function populateTable($tableName, $connectionShortName = 'report', $type = 'NORMAL', $fields = [], $proUid = '', $grid = '')
     {
+        $config = System::getSystemConfiguration();
+        $reportTableBatchRegeneration = $config['report_table_batch_regeneration'];
+
         $tableName = $this->sPrefix . $tableName;
         //we have to do the propel connection
         $database = $this->chooseDB($connectionShortName);
@@ -222,7 +225,7 @@ class ReportTables
             $applications = Application::getByProUid($proUid);
             $i = 1;
             $queryValues = "";
-            $numberRecords = 1000;
+            $numberRecords = $reportTableBatchRegeneration;
             $n = count($applications);
             foreach ($applications as $application) {
                 $appData = $case->unserializeData($application->APP_DATA);
@@ -262,11 +265,12 @@ class ReportTables
                         $queryValues = rtrim($queryValues, ",");
                         $query = $headQuery . $queryValues;
                         $queryValues = "";
-                        $workspace = config("system.workspace");
+
-                        $processesManager = new MultiProcOpen();
+                        //add to queue
-                        $processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) {
+                        $closure = function() use($query) {
-                            return new PopulateTableReport($workspace, $query);
+                            DB::insert($query);
-                        });
+                        };
+                        JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
                     }
                 } else {
                     if (isset($appData[$grid])) {
@@ -304,11 +308,12 @@ class ReportTables
                             $queryValues = rtrim($queryValues, ",");
                             $query = $headQuery . $queryValues;
                             $queryValues = "";
-                            $workspace = config("system.workspace");
+
-                            $processesManager = new MultiProcOpen();
+                            //add to queue
-                            $processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) {
+                            $closure = function() use($query) {
-                                return new PopulateTableReport($workspace, $query);
+                                DB::insert($query);
-                            });
+                            };
+                            JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
                         }
                     }
                 }

workflow/engine/classes/WsBase.php

@@ -1,5 +1,6 @@
 <?php
 
+use App\Jobs\EmailEvent;
 use Illuminate\Support\Facades\Crypt;
 use ProcessMaker\BusinessModel\EmailServer;
 /*----------------------------------********---------------------------------*/
@@ -1028,7 +1029,7 @@ class WsBase
             switch ($appMsgType) {
                 case WsBase::MESSAGE_TYPE_EMAIL_EVENT:
                 case WsBase::MESSAGE_TYPE_PM_FUNCTION:
-                    JobsManager::getSingleton()->dispatch('EmailEvent', $closure);
+                    JobsManager::getSingleton()->dispatch(EmailEvent::class, $closure);
                     $result = new WsResponse(0, G::loadTranslation('ID_MESSAGE_SENT') . ": " . $to);
                     break;
                 default :

workflow/engine/classes/model/AdditionalTables.php

@@ -1,8 +1,9 @@
 <?php
 
+use App\Jobs\GenerateReportTable;
 use Illuminate\Support\Facades\DB;
-use ProcessMaker\Commands\GenerateDataReport;
+use ProcessMaker\Core\JobsManager;
-use ProcessMaker\Core\MultiProcOpen;
+use ProcessMaker\Core\System;
 use ProcessMaker\Model\Application;
 use ProcessMaker\Model\Fields;
 
@@ -735,28 +736,23 @@ class AdditionalTables extends BaseAdditionalTables
         $workspace = config("system.workspace");
         $pathWorkspace = PATH_WORKSPACE;
         $n = Application::count();
-        $processesManager = new MultiProcOpen();
+
-        $processesManager->chunk($n, 1000, function($size, $start, $limit) use(
+        //batch process
-                $workspace, 
+        $config = System::getSystemConfiguration();
-                $tableName, 
+        $reportTableBatchRegeneration = $config['report_table_batch_regeneration'];
-                $type, 
+
-                $processUid, 
+        $size = $n;
-                $gridKey, 
+        $start = 0;
-                $addTabUid, 
+        $limit = $reportTableBatchRegeneration;
-                $className, 
+
-                $pathWorkspace) {
+        for ($i = 1; $start < $size; $i++) {
-            return new GenerateDataReport(
+            $closure = function() use($workspace, $tableName, $type, $processUid, $gridKey, $addTabUid, $className, $pathWorkspace, $start, $limit) {
-                    $workspace, 
+                $workspaceTools = new WorkspaceTools($workspace);
-                    $tableName, 
+                $workspaceTools->generateDataReport($tableName, $type, $processUid, $gridKey, $addTabUid, $className, $pathWorkspace, $start, $limit);
-                    $type, 
+            };
-                    $processUid, 
+            JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
-                    $gridKey, 
+            $start = $i * $limit;
-                    $addTabUid, 
+        }
-                    $className, 
-                    $pathWorkspace, 
-                    $start, 
-                    $limit);
-        });
     }
 
     /**

workflow/engine/config/databases.php

@@ -1,27 +1,4 @@
 <?php
-/**
- * databases.php
- *
- * ProcessMaker Open Source Edition
- * Copyright (C) 2004 - 2008 Colosa Inc.23
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * For more information, contact Colosa Inc, 2566 Le Jeune Rd.,
- * Coral Gables, FL, 33134, USA, or email info@colosa.com.
- *
- */
 
 if (defined('PATH_DB') && !empty(config("system.workspace"))) {
 
@@ -29,7 +6,32 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
         throw new Exception("Could not find db.php in current workspace " . config("system.workspace"));
     }
 
-    require_once(PATH_DB . config("system.workspace") . '/db.php');
+    //These constants must not exist, they will be created by "db.php".
+    $constants = [
+        'DB_ADAPTER',
+        'DB_HOST',
+        'DB_NAME',
+        'DB_USER',
+        'DB_PASS',
+        'DB_RBAC_HOST',
+        'DB_RBAC_NAME',
+        'DB_RBAC_USER',
+        'DB_RBAC_PASS' ,
+        'DB_REPORT_HOST',
+        'DB_REPORT_NAME',
+        'DB_REPORT_USER',
+        'DB_REPORT_PASS',
+    ];
+    $load = true;
+    foreach ($constants as $value) {
+        if (defined($value)) {
+            $load = false;
+            break;
+        }
+    }
+    if ($load === true) {
+        require_once(PATH_DB . config("system.workspace") . '/db.php');
+    }
     //to do: enable for other databases
     $dbType = DB_ADAPTER;
     $dsn = DB_ADAPTER . '://' . DB_USER . ':' . urlencode(DB_PASS) . '@' . DB_HOST . '/' . DB_NAME;
@@ -42,15 +44,12 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
 
     switch (DB_ADAPTER) {
         case 'mysql':
-            $dsn       .= '?encoding=utf8';
+            $dsn .= '?encoding=utf8';
-            $dsnRbac   .= '?encoding=utf8';
+            $dsnRbac .= '?encoding=utf8';
             $dsnReport .= '?encoding=utf8';
             break;
         case 'mssql':
         case 'sqlsrv':
-            //$dsn       .= '?sendStringAsUnicode=false';
-            //$dsnRbac   .= '?sendStringAsUnicode=false';
-            //$dsnReport .= '?sendStringAsUnicode=false';
             break;
         default:
             break;
@@ -64,7 +63,7 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
 
     $pro ['datasources']['rp']['connection'] = $dsnReport;
     $pro ['datasources']['rp']['adapter'] = DB_ADAPTER;
-    
+
     $dbHost = explode(':', DB_HOST);
     config(['database.connections.workflow.host' => $dbHost[0]]);
     config(['database.connections.workflow.database' => DB_NAME]);
@@ -76,6 +75,6 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
 }
 
 $pro ['datasources']['dbarray']['connection'] = 'dbarray://user:pass@localhost/pm_os';
-$pro ['datasources']['dbarray']['adapter']    = 'dbarray';
+$pro ['datasources']['dbarray']['adapter'] = 'dbarray';
 
 return $pro;

workflow/engine/content/translations/english/processmaker.en.po

@@ -25325,6 +25325,12 @@ msgstr "The PHP files execution was disabled please contact the system administr
 msgid "Please complete the reassign reason."
 msgstr "Please complete the reassign reason."
 
+# TRANSLATION
+# LABEL/ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES
+#: LABEL/ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES
+msgid "The report table is regenerating please come back in a few minutes."
+msgstr "The report table is regenerating please come back in a few minutes."
+
 # TRANSLATION
 # LABEL/ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED
 #: LABEL/ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED

workflow/engine/controllers/pmTablesProxy.php

@@ -1215,7 +1215,7 @@ class pmTablesProxy extends HttpProxyController
         if (!empty($table) && $table['PRO_UID'] != '') {
             try {
                 $additionalTables->populateReportTable($table['ADD_TAB_NAME'], PmTable::resolveDbSource($table['DBS_UID']), $table['ADD_TAB_TYPE'], $table['PRO_UID'], $table['ADD_TAB_GRID'], $table['ADD_TAB_UID']);
-                $result->message = 'Generated for table ' . $table['ADD_TAB_NAME'];
+                $result->message = G::LoadTranslation("ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES");
             } catch (Exception $e) {
                 $context = Bootstrap::getDefaultContextLog();
                 $context['proUid'] = $table['PRO_UID'];

workflow/engine/data/mysql/insert.sql

@@ -61112,6 +61112,7 @@ INSERT INTO TRANSLATION (TRN_CATEGORY,TRN_ID,TRN_LANG,TRN_VALUE,TRN_UPDATE_DATE
 ( 'LABEL','ID_THE_NAME_CHANGE_MAY_CAUSE_DATA_LOSS','en','The change might cause data loss in the PM table. Do you want to continue?','2017-03-30') ,
 ( 'LABEL','ID_THE_PHP_FILES_EXECUTION_WAS_DISABLED','en','The PHP files execution was disabled please contact the system administrator.','2018-04-20') ,
 ( 'LABEL','ID_THE_REASON_REASSIGN_USER_EMPTY','en','Please complete the reassign reason.','2016-10-20') ,
+( 'LABEL','ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES','en','The report table is regenerating please come back in a few minutes.','2020-06-01') ,
 ( 'LABEL','ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED','en','The upload of PHP files was disabled please contact the system administrator.','2018-04-20') ,
 ( 'LABEL','ID_THE_USERNAME_EMAIL_IS_INCORRECT','en','The username or email is incorrect','2018-01-18') ,
 ( 'LABEL','ID_THIS_MONTH','en','This Month','2014-01-15') ,

workflow/engine/src/ProcessMaker/Commands/GenerateDataReport.php

@@ -1,80 +0,0 @@
-<?php
-
-namespace ProcessMaker\Commands;
-
-use ProcessMaker\Core\ProcOpen;
-
-class GenerateDataReport extends ProcOpen
-{
-    private $workspace;
-    private $tableName;
-    private $type;
-    private $processUid;
-    private $gridKey;
-    private $addTabUid;
-    private $className;
-    private $pathWorkspace;
-    private $start;
-    private $limit;
-
-    /**
-     * Initializes the command parameters.
-     * @param string $workspace
-     * @param string $tableName
-     * @param string $type
-     * @param string $processUid
-     * @param string $gridKey
-     * @param string $addTabUid
-     * @param string $className
-     * @param string $pathWorkspace
-     * @param integer $start
-     * @param integer $limit
-     */
-    public function __construct(
-            $workspace, 
-            $tableName, 
-            $type = 'NORMAL', 
-            $processUid = '', 
-            $gridKey = '', 
-            $addTabUid = '', 
-            $className = '', 
-            $pathWorkspace, 
-            $start = 0, 
-            $limit = 10)
-    {
-        $this->workspace = $workspace;
-        $this->tableName = $tableName;
-        $this->type = $type;
-        $this->processUid = $processUid;
-        $this->gridKey = $gridKey;
-        $this->addTabUid = $addTabUid;
-        $this->className = $className;
-        $this->pathWorkspace = $pathWorkspace;
-        $this->start = $start;
-        $this->limit = $limit;
-        $this->setCwd(PATH_TRUNK);
-        parent::__construct($this->buildCommand());
-    }
-
-    /**
-     * Returns the command to execute.
-     * @return string
-     */
-    private function buildCommand(): string
-    {
-        $command = PHP_BINDIR . "/php "
-                . "./processmaker "
-                . "'generate-data-report' "
-                . "'{$this->workspace}' "
-                . "'tableName={$this->tableName}' "
-                . "'type={$this->type}' "
-                . "'process={$this->processUid}' "
-                . "'gridKey={$this->gridKey}' "
-                . "'additionalTable={$this->addTabUid}' "
-                . "'className={$this->className}' "
-                . "'pathWorkspace={$this->pathWorkspace}' "
-                . "'start={$this->start}' "
-                . "'limit={$this->limit}' ";
-        return $command;
-    }
-}

workflow/engine/src/ProcessMaker/Commands/PopulateTableReport.php

@@ -1,42 +0,0 @@
-<?php
-
-namespace ProcessMaker\Commands;
-
-use ProcessMaker\Core\ProcOpen;
-
-class PopulateTableReport extends ProcOpen
-{
-    private $workspace;
-    private $sql;
-    private $isRbac;
-
-    /**
-     * Initializes the command parameters.
-     * @param string $workspace
-     * @param string $sql
-     * @param boolean $isRbac
-     */
-    public function __construct($workspace, $sql, $isRbac = false)
-    {
-        $this->workspace = $workspace;
-        $this->sql = $sql;
-        $this->isRbac = $isRbac;
-        $this->setCwd(PATH_TRUNK);
-        parent::__construct($this->buildCommand());
-    }
-
-    /**
-     * Returns the command to execute.
-     * @return string
-     */
-    public function buildCommand()
-    {
-        $command = PHP_BINDIR . "/php "
-                . "./processmaker "
-                . "'populate-table' "
-                . "'{$this->workspace}' "
-                . base64_encode($this->sql) . " "
-                . ($this->isRbac ? "'1'" : "'0'");
-        return $command;
-    }
-}

workflow/engine/src/ProcessMaker/Core/JobsManager.php

@@ -5,7 +5,6 @@ namespace ProcessMaker\Core;
 use Bootstrap;
 use Exception;
 use Illuminate\Support\Facades\Log;
-use ProcessMaker\BusinessModel\Factories\Jobs;
 use ProcessMaker\Core\System;
 use Propel;
 
@@ -187,7 +186,7 @@ class JobsManager
     {
         $environment = $this->getDataSnapshot();
 
-        $instance = Jobs::create($name, function() use ($callback, $environment) {
+        $instance = $name::dispatch(function() use ($callback, $environment) {
                     try {
                         $this->recoverDataSnapshot($environment);
                         $callback($environment);

workflow/engine/src/ProcessMaker/Core/MultiProcOpen.php

@@ -1,92 +0,0 @@
-<?php
-
-namespace ProcessMaker\Core;
-
-class MultiProcOpen
-{
-    /**
-     * Represents the waiting time before starting the process monitoring.
-     * @var integer 
-     */
-    private $sleepTime = 1;
-
-    /**
-     * This method obtains a paging by returning the start and limit indexes 
-     * compatible with the mysql pagination in its call function.
-     * The return function must return an instance of the object "ProcessMaker\Core\ProcOpen".
-     * Returns an array containing the status, content, and errors generated by 
-     * the open process.
-     * @param int $size
-     * @param int $chunk
-     * @param callable $callback
-     * @return array
-     */
-    public function chunk(int $size, int $chunk, callable $callback): array
-    {
-        $start = 0;
-        $limit = $chunk;
-        $queries = [];
-        for ($i = 1; $start < $size; $i++) {
-            $queries[] = $callback($size, $start, $limit);
-            $start = $i * $limit;
-        }
-        return $this->run($queries);
-    }
-
-    /**
-     * Open a set of background processes.
-     * The array must contain one or more instances of the object inherited from 
-     * the class "ProcessMaker\Core\ProcOpen"
-     * Returns an array containing the status, content, and errors generated by 
-     * the open process.
-     * @param array $processes
-     * @return array
-     */
-    public function run(array $processes): array
-    {
-        foreach ($processes as $procOpen) {
-            $procOpen->open();
-        }
-        return $this->processMonitoring($processes);
-    }
-
-    /**
-     * It monitors the open processes, verifying if they have ended or thrown an 
-     * error and later closing the resources related to the process.
-     * Returns an array containing the status, content, and errors generated by 
-     * the open process.
-     * @param array $processes
-     * @return array
-     */
-    private function processMonitoring(array $processes): array
-    {
-        sleep($this->sleepTime); //this sleep is very important
-        $i = 0;
-        $n = count($processes);
-        if ($n === 0) {
-            return [];
-        }
-        $outputs = [];
-        do {
-            $index = $i % $n;
-            if (isset($processes[$index])) {
-                $procOpen = $processes[$index];
-                $status = $procOpen->getStatus();
-                $contents = $procOpen->getContents();
-                $errors = $procOpen->getErrors();
-                if ($status->running === false || !empty($errors)) {
-                    $outputs[] = [
-                        "status" => $status,
-                        "contents" => $contents,
-                        "errors" => $errors,
-                    ];
-                    $procOpen->terminate();
-                    $procOpen->close();
-                    unset($processes[$index]);
-                }
-            }
-            $i = $i + 1;
-        } while (!empty($processes));
-        return $outputs;
-    }
-}

workflow/engine/src/ProcessMaker/Core/ProcOpen.php

@@ -1,126 +0,0 @@
-<?php
-
-namespace ProcessMaker\Core;
-
-class ProcOpen
-{
-    private $command;
-    private $resource;
-    private $descriptorspec;
-    private $pipes;
-    private $cwd;
-
-    /**
-     * This initializes the descriptors and the command for the open process.
-     * @param string $command
-     */
-    public function __construct(string $command)
-    {
-        $this->descriptorspec = [
-            ['pipe', 'r'],
-            ['pipe', 'w'],
-            ['pipe', 'w']
-        ];
-        $this->command = $command;
-    }
-
-    /**
-     * Gets the resource that represents the process.
-     * @return resource
-     */
-    public function getResource()
-    {
-        return $this->resource;
-    }
-
-    /**
-     * Sets the process execution directory.
-     * @param string $cwd
-     */
-    public function setCwd(string $cwd)
-    {
-        $this->cwd = $cwd;
-    }
-
-    /**
-     * Open a background process.
-     */
-    public function open()
-    {
-        if (empty($this->cwd)) {
-            $this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes);
-        } else {
-            $this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes, $this->cwd);
-        }
-    }
-
-    /**
-     * Get the content of the process when it is finished.
-     * @return string
-     */
-    public function getContents()
-    {
-        if (is_resource($this->pipes[1])) {
-            return stream_get_contents($this->pipes[1]);
-        }
-        return "";
-    }
-
-    /**
-     * Get the process errors when it is finished.
-     * @return string
-     */
-    public function getErrors()
-    {
-        if (is_resource($this->pipes[2])) {
-            return stream_get_contents($this->pipes[2]);
-        }
-        return "";
-    }
-
-    /**
-     * Close the resources related to the open process.
-     * return void
-     */
-    public function close()
-    {
-        if (is_resource($this->resource)) {
-            foreach ($this->pipes as $value) {
-                fclose($value);
-            }
-            proc_close($this->resource);
-        }
-    }
-
-    /**
-     * End the process before it ends.
-     */
-    public function terminate()
-    {
-        if (is_resource($this->resource)) {
-            proc_terminate($this->resource);
-        }
-    }
-
-    /**
-     * Gets the status of the process.
-     * @return object
-     */
-    public function getStatus()
-    {
-        $status = [
-            "command" => $this->command,
-            "pid" => null,
-            "running" => false,
-            "signaled" => false,
-            "stopped" => false,
-            "exitcode" => -1,
-            "termsig" => 0,
-            "stopsig" => 0
-        ];
-        if (is_resource($this->resource)) {
-            $status = proc_get_status($this->resource);
-        }
-        return (object) $status;
-    }
-}

workflow/engine/src/ProcessMaker/Core/System.php

@@ -78,7 +78,8 @@ class System
         'highlight_home_folder_enable' => 0,
         'highlight_home_folder_refresh_time' => 10,
         'highlight_home_folder_scope' => 'unassigned', // For now only this list is supported
-        'disable_advanced_search_case_title_fulltext' => 0
+        'disable_advanced_search_case_title_fulltext' => 0,
+        'report_table_batch_regeneration' => 1000
     ];
 
     /**