fernando/luos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PMCORE-1527 PMCORE-1421 - Regenerate PM Report Tables Asynchronously

Browse Source
This commit is contained in:
Roly Rudy Gutierrez Pinto
2020-06-11 10:44:47 -04:00
parent f1824ae9af
commit 45ba40b5a1
20 changed files with 401 additions and 606 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
app/Jobs/GenerateReportTable.php Normal file
View File

@@ -0,0 +1,14 @@
<?php
namespace App\Jobs;
class GenerateReportTable extends QueuedClosure
{
/**
* The number of times the job may be attempted.
*
* @var int
*/
public $tries = 1;
}

33
tests/resources/structureReportTable.json Normal file
View File

@@ -0,0 +1,33 @@
{
"data": {
"var_Text1": "5298067855ed6e6893e1424067710552",
"var_Textarea1": "4835809915ed6e6893e14a6079656976",
"var_Dropdown1": "5287566085ed6e6893e14e0083475368",
"var_Suggest1": "4813021735ed6e6893e1520081779769",
"var_DateTime1": "1975-04-04 16:34:22",
"var_String1": "7984290665ed6e6893e15b2092042769",
"var_Integer1": "44"
},
"mapping": [{
"sFieldName": "var_Text1",
"sType": "char"
}, {
"sFieldName": "var_Textarea1",
"sType": "text"
}, {
"sFieldName": "var_Dropdown1",
"sType": "char"
}, {
"sFieldName": "var_Suggest1",
"sType": "char"
}, {
"sFieldName": "var_DateTime1",
"sType": "date"
}, {
"sFieldName": "var_String1",
"sType": "char"
}, {
"sFieldName": "var_Integer1",
"sType": "number"
}]
}

285
tests/unit/workflow/engine/classes/ReportTablesTest.php
View File

@@ -12,6 +12,7 @@ use Tests\TestCase;
class ReportTablesTest extends TestCase class ReportTablesTest extends TestCase
{ {
use CreateTestSite; use CreateTestSite;
use DatabaseTransactions; use DatabaseTransactions;
@@ -21,14 +22,12 @@ class ReportTablesTest extends TestCase
public function setUp() public function setUp()
{ {
parent::setUp(); parent::setUp();
$this->markTestIncomplete(""
. "This test has started using the ./processmaker command, this "
. "command requires the file 'paths_installed.php', that is, a "
. "valid installation of processmaker.");
$_SERVER["REQUEST_URI"] = ""; $_SERVER["REQUEST_URI"] = "";
config(['queue.default' => 'sync']);
config(["system.workspace" => "test"]); config(["system.workspace" => "test"]);
$workspace = config("system.workspace"); $workspace = config("system.workspace");
$this->createDBFile($workspace); $this->createDBFile($workspace);
$this->createConstantsOfConnection();
} }
/** /**
@@ -39,6 +38,33 @@ class ReportTablesTest extends TestCase
parent::tearDown(); parent::tearDown();
} }
/**
* Create constants of connection to databases.
*/
private function createConstantsOfConnection()
{
$constants = [
'DB_ADAPTER' => env('mysql'),
'DB_HOST' => env('DB_HOST'),
'DB_NAME' => env('DB_DATABASE'),
'DB_USER' => env('DB_USERNAME'),
'DB_PASS' => env('DB_PASSWORD'),
'DB_RBAC_HOST' => env('DB_HOST'),
'DB_RBAC_NAME' => env('DB_DATABASE'),
'DB_RBAC_USER' => env('DB_USERNAME'),
'DB_RBAC_PASS' => env('DB_PASSWORD'),
'DB_REPORT_HOST' => env('DB_HOST'),
'DB_REPORT_NAME' => env('DB_DATABASE'),
'DB_REPORT_USER' => env('DB_USERNAME'),
'DB_REPORT_PASS' => env('DB_PASSWORD'),
];
foreach ($constants as $key => $value) {
if (!defined($key)) {
define($key, $value);
}
}
}
/** /**
* Check if the "populateTable" function returns an array value if entered all parameters. * Check if the "populateTable" function returns an array value if entered all parameters.
* @test * @test
@@ -565,97 +591,6 @@ class ReportTablesTest extends TestCase
$this->assertEquals($expected, $actual); $this->assertEquals($expected, $actual);
} }
/**
* Get mapping fields supported by report table.
* @return array
*/
private function getMapFields()
{
return [
[
'sFieldName' => 'var_Text1',
'sType' => 'char'
],
[
'sFieldName' => 'var_Textarea1',
'sType' => 'text'
],
[
'sFieldName' => 'var_Dropdown1',
'sType' => 'char'
],
[
'sFieldName' => 'var_Suggest1',
'sType' => 'char'
],
[
'sFieldName' => 'var_DateTime1',
'sType' => 'date'
],
[
'sFieldName' => 'var_String1',
'sType' => 'char'
],
[
'sFieldName' => 'var_Integer1',
'sType' => 'number'
],
[
'sFieldName' => 'var_Boolean1',
'sType' => 'boolean'
],
[
'sFieldName' => 'var_Array1',
'sType' => 'array'
]
];
}
/**
* Create fields data by type supported.
* @param array $types
* @return array
*/
private function createFieldsByType($types = [])
{
$fields = [];
$mapping = [];
$faker = Faker\Factory::create();
$date = $faker->dateTime();
$mapFields = $this->getMapFields();
foreach ($mapFields as $key => $value) {
if (!in_array($value['sType'], $types)) {
continue;
}
switch ($value['sType']) {
case 'number':
$mapping[] = $value;
$fields[$value['sFieldName']] = (string) random_int(0, 100);
break;
case 'char':
$mapping[] = $value;
$fields[$value['sFieldName']] = G::generateUniqueID();
break;
case 'text':
$mapping[] = $value;
$fields[$value['sFieldName']] = G::generateUniqueID();
break;
case 'date':
$mapping[] = $value;
$fields[$value['sFieldName']] = $date->format('Y-m-d H:i:s');
break;
case 'boolean':
$mapping[] = $value;
$fields[$value['sFieldName']] = ['0' => 0];
break;
}
}
return [
'data' => $fields,
'mapping' => $mapping
];
}
/** /**
* Prepare data initial for test, the grid parameter is optional if you want * Prepare data initial for test, the grid parameter is optional if you want
* to create a grid type field. * to create a grid type field.
@@ -665,7 +600,7 @@ class ReportTablesTest extends TestCase
* @param boolean $grid * @param boolean $grid
* @return object * @return object
*/ */
private function prepareData($tableName, $grid = null) private function prepareData($tableName, $grid = null, $structure = [])
{ {
$applicationNumber = Application::max('APP_NUMBER'); $applicationNumber = Application::max('APP_NUMBER');
if (is_null($applicationNumber)) { if (is_null($applicationNumber)) {
@@ -681,7 +616,10 @@ class ReportTablesTest extends TestCase
$taskUid = G::generateUniqueID(); $taskUid = G::generateUniqueID();
$applicationUid = G::generateUniqueID(); $applicationUid = G::generateUniqueID();
$structure = $this->createFieldsByType(['number', 'char', 'text', 'date']); if (empty($structure)) {
$structure = $this->getDataFromFile('structureReportTable.json');
}
$fields = $structure['mapping']; $fields = $structure['mapping'];
$dataFields = $structure['data']; $dataFields = $structure['data'];
$appData = [ $appData = [
@@ -753,4 +691,157 @@ class ReportTablesTest extends TestCase
$result->application = $application; $result->application = $application;
return $result; return $result;
} }
/**
* Check if the "populateTable" method is it filling with missing values into app_data.
* @test
* @covers ReportTables::populateTable
*/
public function it_should_populating_data_with_fields_missing_in_to_app_data()
{
$tableName = 'TestReportTable';
$result = $this->prepareData($tableName);
$connectionShortName = 'wf';
$type = 'NORMAL';
$fields = $result->fields;
$proUid = $result->processUid;
$grid = '';
$app = Application::where('APP_UID', '=', $result->applicationUid)->get()->first();
$appData = unserialize($app->APP_DATA);
unset($appData['var_Textarea1']);
$appData = serialize($appData);
Application::where('APP_UID', '=', $result->applicationUid)->update(['APP_DATA' => $appData]);
$reportTables = new ReportTables();
$reportTables->populateTable($tableName, $connectionShortName, $type, $fields, $proUid, $grid);
$expected = $result->dataFields;
$expected['APP_UID'] = $result->applicationUid;
$expected['APP_NUMBER'] = $result->applicationNumber;
$expected['var_Textarea1'] = '';
$actual = (array) DB::table($tableName)
->select()
->first();
$this->assertEquals($expected, $actual);
}
/**
* Check if the "populateTable" method is it filling with arrays values.
* @test
* @covers ReportTables::populateTable
*/
public function it_should_populating_data_with_arrays_values()
{
$tableName = 'TestReportTable';
$result = $this->prepareData($tableName);
$connectionShortName = 'wf';
$type = 'NORMAL';
$fields = $result->fields;
$proUid = $result->processUid;
$grid = '';
$app = Application::where('APP_UID', '=', $result->applicationUid)->get()->first();
$appData = unserialize($app->APP_DATA);
$appData['var_Textarea1'] = [];
$appData = serialize($appData);
Application::where('APP_UID', '=', $result->applicationUid)->update(['APP_DATA' => $appData]);
$reportTables = new ReportTables();
$reportTables->populateTable($tableName, $connectionShortName, $type, $fields, $proUid, $grid);
$expected = $result->dataFields;
$expected['APP_UID'] = $result->applicationUid;
$expected['APP_NUMBER'] = $result->applicationNumber;
$expected['var_Textarea1'] = '';
$actual = (array) DB::table($tableName)
->select()
->first();
$this->assertEquals($expected, $actual);
}
/**
* Check if the "populateTable" method is it filling with missing values into app_data for grids control.
* parameters and type and grid are correct values.
* @test
* @covers ReportTables::populateTable
*/
public function it_should_populating_data_with_all_parameters_with_type_is_grid_fields_missing_in_to_app_data()
{
$tableName = 'TestReportTable';
$result = $this->prepareData($tableName, true);
$connectionShortName = 'wf';
$type = 'GRID';
$fields = $result->fields;
$proUid = $result->processUid;
$grid = 'var_Grid1';
$app = Application::where('APP_UID', '=', $result->applicationUid)->get()->first();
$appData = unserialize($app->APP_DATA);
unset($appData['var_Grid1'][1]['var_Textarea1']);
$appData = serialize($appData);
Application::where('APP_UID', '=', $result->applicationUid)->update(['APP_DATA' => $appData]);
$reportTables = new ReportTables();
$reportTables->populateTable($tableName, $connectionShortName, $type, $fields, $proUid, $grid);
$indexRow = 1;
$expected = $result->appData[$grid];
foreach ($expected as &$row) {
$row['APP_UID'] = $result->applicationUid;
$row['APP_NUMBER'] = $result->applicationNumber;
$row['ROW'] = (string) ($indexRow++);
}
$expected = array_values($expected);
$expected[0]['var_Textarea1'] = '';
$actual = DB::table($tableName)
->select()
->get();
$actual->transform(function ($item, $key) {
return (array) $item;
});
$actual = $actual->toArray();
$this->assertEquals($expected, $actual);
}
/**
* Check an exception if the input parameters are wrong.
* @test
* @covers ReportTables::populateTable
*/
public function it_should_catch_an_exception()
{
$tableName = 'TestReportTable';
$result = $this->prepareData($tableName, true);
$connectionShortName = 'wf';
$type = 'GRID';
$fields = $result->fields;
$proUid = $result->processUid;
$grid = 'var_Grid1';
//assert exception
$this->expectException(Exception::class);
$reportTables = new ReportTables();
$reportTables->populateTable($tableName, $connectionShortName, $type, null, $proUid, $grid);
}
/**
* This gets data from a json file.
* @param string $pathData
* @return array
*/
private function getDataFromFile(string $pathData): array
{
$pathData = PATH_TRUNK . "tests/resources/{$pathData}";
$data = file_get_contents($pathData);
$result = json_decode($data, JSON_OBJECT_AS_ARRAY);
return $result;
}
} }

83
tests/unit/workflow/engine/classes/model/AdditionalTablesTest.php
View File

@@ -3,17 +3,32 @@
namespace Tests\unit\workflow\engine\classes\model; namespace Tests\unit\workflow\engine\classes\model;
use AdditionalTables; use AdditionalTables;
use App\Jobs\GenerateReportTable;
use Exception; use Exception;
use G; use G;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Queue;
use Illuminate\Support\Facades\Schema; use Illuminate\Support\Facades\Schema;
use ProcessMaker\BusinessModel\ReportTable; use ProcessMaker\BusinessModel\ReportTable;
use ProcessMaker\Model\AdditionalTables as AdditionalTablesModel; use ProcessMaker\Model\AdditionalTables as AdditionalTablesModel;
use ProcessMaker\Model\Application;
use ProcessMaker\Model\DbSource;
use ProcessMaker\Model\Delegation;
use ProcessMaker\Model\Process;
use ProcessMaker\Model\Task;
use Tests\TestCase; use Tests\TestCase;
class AdditionalTablesTest extends TestCase class AdditionalTablesTest extends TestCase
{ {
/**
* Set up method.
*/
public function setUp()
{
parent::setUp();
}
/** /**
* This tests the creation of a PMTable. * This tests the creation of a PMTable.
* @test * @test
@@ -198,6 +213,74 @@ class AdditionalTablesTest extends TestCase
$this->assertContains($actual[0], $expected, false); $this->assertContains($actual[0], $expected, false);
} }
/**
* Check if populate report table is added to job queue.
* @test
* @covers \AdditionalTables::populateReportTable
*/
public function it_should_test_populate_report_table()
{
$proUid = factory(Process::class)->create()->PRO_UID;
$task = factory(Task::class)->create([
'PRO_UID' => $proUid
]);
//local connections
$dbSource = factory(DbSource::class)->create([
'PRO_UID' => $proUid,
'DBS_SERVER' => env('DB_HOST'),
'DBS_DATABASE_NAME' => env('DB_DATABASE'),
'DBS_USERNAME' => env('DB_USERNAME'),
'DBS_PASSWORD' => G::encrypt(env('DB_PASSWORD'), env('DB_DATABASE')) . "_2NnV3ujj3w",
'DBS_PORT' => '3306',
'DBS_CONNECTION_TYPE' => 'NORMAL'
]);
$additionalTable = factory(AdditionalTablesModel::class)->create([
'PRO_UID' => $proUid,
'DBS_UID' => $dbSource->DBS_UID,
]);
$tableName = $additionalTable->ADD_TAB_NAME;
$name = $additionalTable->ADD_TAB_CLASS_NAME;
$this->createSchema($dbSource->DBS_DATABASE_NAME, $tableName, $name, $dbSource->DBS_UID);
//external connection
$dbSource = factory(DbSource::class)->create([
'PRO_UID' => $proUid,
'DBS_SERVER' => config('database.connections.testexternal.host'),
'DBS_DATABASE_NAME' => config('database.connections.testexternal.database'),
'DBS_USERNAME' => config('database.connections.testexternal.username'),
'DBS_PASSWORD' => G::encrypt(config('database.connections.testexternal.password'), config('database.connections.testexternal.database')) . "_2NnV3ujj3w",
'DBS_PORT' => '3306',
'DBS_CONNECTION_TYPE' => 'NORMAL'
]);
$additionalTable = factory(AdditionalTablesModel::class)->create([
'PRO_UID' => $proUid,
'DBS_UID' => $dbSource->DBS_UID,
]);
$tableNameExternal = $additionalTable->ADD_TAB_NAME;
$nameExternal = $additionalTable->ADD_TAB_CLASS_NAME;
$this->createSchema($dbSource->DBS_DATABASE_NAME, $tableNameExternal, $nameExternal, $dbSource->DBS_UID);
$application = factory(Application::class)->create([
'PRO_UID' => $proUid
]);
factory(Delegation::class)->create([
'DEL_THREAD_STATUS' => 'CLOSED',
'APP_NUMBER' => $application->APP_NUMBER,
'TAS_UID' => $task->TAS_UID,
]);
//assertions
Queue::fake();
Queue::assertNothingPushed();
$additionalTables = new AdditionalTables();
$additionalTables->populateReportTable($tableName, 'workflow', 'NORMAL', $proUid, '', $additionalTable->ADD_TAB_UID);
Queue::assertPushed(GenerateReportTable::class);
}
/** /**
* This gets the content from template file. * This gets the content from template file.
* @param string $pathData * @param string $pathData

3
tests/unit/workflow/engine/src/ProcessMaker/Core/JobsManagerTest.php
View File

@@ -2,6 +2,7 @@
namespace ProcessMaker\Core; namespace ProcessMaker\Core;
use App\Jobs\Email;
use Tests\TestCase; use Tests\TestCase;
class JobsManagerTest extends TestCase class JobsManagerTest extends TestCase
@@ -116,7 +117,7 @@ class JobsManagerTest extends TestCase
$callback = function() { $callback = function() {
}; };
$actual = $this->object->dispatch('Email', $callback); $actual = $this->object->dispatch(Email::class, $callback);
$this->assertInstanceOf(\Illuminate\Foundation\Bus\PendingDispatch::class, $actual); $this->assertInstanceOf(\Illuminate\Foundation\Bus\PendingDispatch::class, $actual);
} }

74
workflow/engine/bin/tasks/cliGenerateDataReport.php
View File

@@ -1,74 +0,0 @@
<?php
use Illuminate\Support\Facades\DB;
use ProcessMaker\Core\System;
CLI::taskName("generate-data-report");
CLI::taskRun("generateDataReport");
CLI::taskDescription("\nGenerate data report by process in a respective workspace, you must pass through arguments the project identifier and the processing interval.");
CLI::taskArg("workspace", false);
CLI::taskOpt("uid", "Identifier that represents the process, must be 32 characters.", "", "process=");
CLI::taskOpt("start", "The start option skips so many rows before returning results.", "", "start=");
CLI::taskOpt("limit", "The limit option restricts the number of rows returned.", "", "limit=");
/**
* Generate data report by process in a respective workspace, you must pass through
* arguments the project identifier and the processing interval.
* @param array $options
* @return void
*/
function generateDataReport(array $options): void
{
//get workspace
if (empty($options[0])) {
CLI::logging("Workspace undefined!\n");
return;
}
$workspace = $options[0];
//get options
$parameters = [
"tableName=" => "",
"type=" => "",
"process=" => "",
"gridKey=" => "",
"additionalTable=" => "",
"className=" => "",
"pathWorkspace=" => "",
"start=" => "",
"limit=" => ""
];
foreach ($parameters as $key => $value) {
for ($i = 1; $i < count($options); $i++) {
if (strpos($options[$i], $key) !== false) {
$parameters[$key] = str_replace($key, "", $options[$i]);
break;
}
}
}
//validations
$needed = [
"process="
];
foreach ($needed as $value) {
if (empty($parameters[$value])) {
CLI::logging("Missing options {$value}.\n");
return;
}
}
//run method
$workspaceTools = new WorkspaceTools($workspace);
$workspaceTools->generateDataReport(
$parameters["tableName="],
$parameters["type="],
$parameters["process="],
$parameters["gridKey="],
$parameters["additionalTable="],
$parameters["className="],
$parameters["pathWorkspace="],
(int) $parameters["start="],
(int) $parameters["limit="]
);
}

21
workflow/engine/bin/tasks/cliPopulateTable.php
View File

@@ -1,21 +0,0 @@
<?php
CLI::taskName("populate-table");
CLI::taskRun("populateTable");
CLI::taskDescription("\nThis function populates the report table with the APP_DATA data");
CLI::taskArg("workspace", false);
/**
* This function populates the report table with the APP_DATA data.
* @return void
*/
function populateTable($options): void
{
//get options
$workspaceName = $options[0];
$query = base64_decode($options[1]);
$isRbac = (bool) $options[2];
$workspace = new WorkspaceTools($workspaceName);
$workspace->populateTableReport($query, $isRbac);
}

31
workflow/engine/classes/ReportTables.php
View File

@@ -1,8 +1,8 @@
<?php <?php
use App\Jobs\GenerateReportTable;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use ProcessMaker\Core\MultiProcOpen; use ProcessMaker\Core\JobsManager;
use ProcessMaker\Commands\PopulateTableReport;
use ProcessMaker\Model\Application; use ProcessMaker\Model\Application;
/** /**
@@ -205,6 +205,9 @@ class ReportTables
*/ */
public function populateTable($tableName, $connectionShortName = 'report', $type = 'NORMAL', $fields = [], $proUid = '', $grid = '') public function populateTable($tableName, $connectionShortName = 'report', $type = 'NORMAL', $fields = [], $proUid = '', $grid = '')
{ {
$config = System::getSystemConfiguration();
$reportTableBatchRegeneration = $config['report_table_batch_regeneration'];
$tableName = $this->sPrefix . $tableName; $tableName = $this->sPrefix . $tableName;
//we have to do the propel connection //we have to do the propel connection
$database = $this->chooseDB($connectionShortName); $database = $this->chooseDB($connectionShortName);
@@ -222,7 +225,7 @@ class ReportTables
$applications = Application::getByProUid($proUid); $applications = Application::getByProUid($proUid);
$i = 1; $i = 1;
$queryValues = ""; $queryValues = "";
$numberRecords = 1000; $numberRecords = $reportTableBatchRegeneration;
$n = count($applications); $n = count($applications);
foreach ($applications as $application) { foreach ($applications as $application) {
$appData = $case->unserializeData($application->APP_DATA); $appData = $case->unserializeData($application->APP_DATA);
@@ -262,11 +265,12 @@ class ReportTables
$queryValues = rtrim($queryValues, ","); $queryValues = rtrim($queryValues, ",");
$query = $headQuery . $queryValues; $query = $headQuery . $queryValues;
$queryValues = ""; $queryValues = "";
$workspace = config("system.workspace");
$processesManager = new MultiProcOpen(); //add to queue
$processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) { $closure = function() use($query) {
return new PopulateTableReport($workspace, $query); DB::insert($query);
}); };
JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
} }
} else { } else {
if (isset($appData[$grid])) { if (isset($appData[$grid])) {
@@ -304,11 +308,12 @@ class ReportTables
$queryValues = rtrim($queryValues, ","); $queryValues = rtrim($queryValues, ",");
$query = $headQuery . $queryValues; $query = $headQuery . $queryValues;
$queryValues = ""; $queryValues = "";
$workspace = config("system.workspace");
$processesManager = new MultiProcOpen(); //add to queue
$processesManager->chunk(1, 1, function($size, $start, $limit) use ($query, $workspace) { $closure = function() use($query) {
return new PopulateTableReport($workspace, $query); DB::insert($query);
}); };
JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
} }
} }
} }

3
workflow/engine/classes/WsBase.php
View File

@@ -1,5 +1,6 @@
<?php <?php
use App\Jobs\EmailEvent;
use Illuminate\Support\Facades\Crypt; use Illuminate\Support\Facades\Crypt;
use ProcessMaker\BusinessModel\EmailServer; use ProcessMaker\BusinessModel\EmailServer;
/*----------------------------------********---------------------------------*/ /*----------------------------------********---------------------------------*/
@@ -1028,7 +1029,7 @@ class WsBase
switch ($appMsgType) { switch ($appMsgType) {
case WsBase::MESSAGE_TYPE_EMAIL_EVENT: case WsBase::MESSAGE_TYPE_EMAIL_EVENT:
case WsBase::MESSAGE_TYPE_PM_FUNCTION: case WsBase::MESSAGE_TYPE_PM_FUNCTION:
JobsManager::getSingleton()->dispatch('EmailEvent', $closure); JobsManager::getSingleton()->dispatch(EmailEvent::class, $closure);
$result = new WsResponse(0, G::loadTranslation('ID_MESSAGE_SENT') . ": " . $to); $result = new WsResponse(0, G::loadTranslation('ID_MESSAGE_SENT') . ": " . $to);
break; break;
default : default :

44
workflow/engine/classes/model/AdditionalTables.php
View File

@@ -1,8 +1,9 @@
<?php <?php
use App\Jobs\GenerateReportTable;
use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\DB;
use ProcessMaker\Commands\GenerateDataReport; use ProcessMaker\Core\JobsManager;
use ProcessMaker\Core\MultiProcOpen; use ProcessMaker\Core\System;
use ProcessMaker\Model\Application; use ProcessMaker\Model\Application;
use ProcessMaker\Model\Fields; use ProcessMaker\Model\Fields;
@@ -735,28 +736,23 @@ class AdditionalTables extends BaseAdditionalTables
$workspace = config("system.workspace"); $workspace = config("system.workspace");
$pathWorkspace = PATH_WORKSPACE; $pathWorkspace = PATH_WORKSPACE;
$n = Application::count(); $n = Application::count();
$processesManager = new MultiProcOpen();
$processesManager->chunk($n, 1000, function($size, $start, $limit) use( //batch process
$workspace, $config = System::getSystemConfiguration();
$tableName, $reportTableBatchRegeneration = $config['report_table_batch_regeneration'];
$type,
$processUid, $size = $n;
$gridKey, $start = 0;
$addTabUid, $limit = $reportTableBatchRegeneration;
$className,
$pathWorkspace) { for ($i = 1; $start < $size; $i++) {
return new GenerateDataReport( $closure = function() use($workspace, $tableName, $type, $processUid, $gridKey, $addTabUid, $className, $pathWorkspace, $start, $limit) {
$workspace, $workspaceTools = new WorkspaceTools($workspace);
$tableName, $workspaceTools->generateDataReport($tableName, $type, $processUid, $gridKey, $addTabUid, $className, $pathWorkspace, $start, $limit);
$type, };
$processUid, JobsManager::getSingleton()->dispatch(GenerateReportTable::class, $closure);
$gridKey, $start = $i * $limit;
$addTabUid, }
$className,
$pathWorkspace,
$start,
$limit);
});
} }
/** /**

59
workflow/engine/config/databases.php
View File

@@ -1,27 +1,4 @@
<?php <?php
/**
* databases.php
*
* ProcessMaker Open Source Edition
* Copyright (C) 2004 - 2008 Colosa Inc.23
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* For more information, contact Colosa Inc, 2566 Le Jeune Rd.,
* Coral Gables, FL, 33134, USA, or email info@colosa.com.
*
*/
if (defined('PATH_DB') && !empty(config("system.workspace"))) { if (defined('PATH_DB') && !empty(config("system.workspace"))) {
@@ -29,7 +6,32 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
throw new Exception("Could not find db.php in current workspace " . config("system.workspace")); throw new Exception("Could not find db.php in current workspace " . config("system.workspace"));
} }
require_once(PATH_DB . config("system.workspace") . '/db.php'); //These constants must not exist, they will be created by "db.php".
$constants = [
'DB_ADAPTER',
'DB_HOST',
'DB_NAME',
'DB_USER',
'DB_PASS',
'DB_RBAC_HOST',
'DB_RBAC_NAME',
'DB_RBAC_USER',
'DB_RBAC_PASS' ,
'DB_REPORT_HOST',
'DB_REPORT_NAME',
'DB_REPORT_USER',
'DB_REPORT_PASS',
];
$load = true;
foreach ($constants as $value) {
if (defined($value)) {
$load = false;
break;
}
}
if ($load === true) {
require_once(PATH_DB . config("system.workspace") . '/db.php');
}
//to do: enable for other databases //to do: enable for other databases
$dbType = DB_ADAPTER; $dbType = DB_ADAPTER;
$dsn = DB_ADAPTER . '://' . DB_USER . ':' . urlencode(DB_PASS) . '@' . DB_HOST . '/' . DB_NAME; $dsn = DB_ADAPTER . '://' . DB_USER . ':' . urlencode(DB_PASS) . '@' . DB_HOST . '/' . DB_NAME;
@@ -42,15 +44,12 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
switch (DB_ADAPTER) { switch (DB_ADAPTER) {
case 'mysql': case 'mysql':
$dsn .= '?encoding=utf8'; $dsn .= '?encoding=utf8';
$dsnRbac .= '?encoding=utf8'; $dsnRbac .= '?encoding=utf8';
$dsnReport .= '?encoding=utf8'; $dsnReport .= '?encoding=utf8';
break; break;
case 'mssql': case 'mssql':
case 'sqlsrv': case 'sqlsrv':
//$dsn .= '?sendStringAsUnicode=false';
//$dsnRbac .= '?sendStringAsUnicode=false';
//$dsnReport .= '?sendStringAsUnicode=false';
break; break;
default: default:
break; break;
@@ -76,6 +75,6 @@ if (defined('PATH_DB') && !empty(config("system.workspace"))) {
} }
$pro ['datasources']['dbarray']['connection'] = 'dbarray://user:pass@localhost/pm_os'; $pro ['datasources']['dbarray']['connection'] = 'dbarray://user:pass@localhost/pm_os';
$pro ['datasources']['dbarray']['adapter'] = 'dbarray'; $pro ['datasources']['dbarray']['adapter'] = 'dbarray';
return $pro; return $pro;

6
workflow/engine/content/translations/english/processmaker.en.po
View File

@@ -25325,6 +25325,12 @@ msgstr "The PHP files execution was disabled please contact the system administr
msgid "Please complete the reassign reason." msgid "Please complete the reassign reason."
msgstr "Please complete the reassign reason." msgstr "Please complete the reassign reason."
# TRANSLATION
# LABEL/ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES
#: LABEL/ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES
msgid "The report table is regenerating please come back in a few minutes."
msgstr "The report table is regenerating please come back in a few minutes."
# TRANSLATION # TRANSLATION
# LABEL/ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED # LABEL/ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED
#: LABEL/ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED #: LABEL/ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED

2
workflow/engine/controllers/pmTablesProxy.php
View File

@@ -1215,7 +1215,7 @@ class pmTablesProxy extends HttpProxyController
if (!empty($table) && $table['PRO_UID'] != '') { if (!empty($table) && $table['PRO_UID'] != '') {
try { try {
$additionalTables->populateReportTable($table['ADD_TAB_NAME'], PmTable::resolveDbSource($table['DBS_UID']), $table['ADD_TAB_TYPE'], $table['PRO_UID'], $table['ADD_TAB_GRID'], $table['ADD_TAB_UID']); $additionalTables->populateReportTable($table['ADD_TAB_NAME'], PmTable::resolveDbSource($table['DBS_UID']), $table['ADD_TAB_TYPE'], $table['PRO_UID'], $table['ADD_TAB_GRID'], $table['ADD_TAB_UID']);
$result->message = 'Generated for table ' . $table['ADD_TAB_NAME']; $result->message = G::LoadTranslation("ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES");
} catch (Exception $e) { } catch (Exception $e) {
$context = Bootstrap::getDefaultContextLog(); $context = Bootstrap::getDefaultContextLog();
$context['proUid'] = $table['PRO_UID']; $context['proUid'] = $table['PRO_UID'];

1
workflow/engine/data/mysql/insert.sql
View File

@@ -61112,6 +61112,7 @@ INSERT INTO TRANSLATION (TRN_CATEGORY,TRN_ID,TRN_LANG,TRN_VALUE,TRN_UPDATE_DATE
( 'LABEL','ID_THE_NAME_CHANGE_MAY_CAUSE_DATA_LOSS','en','The change might cause data loss in the PM table. Do you want to continue?','2017-03-30') , ( 'LABEL','ID_THE_NAME_CHANGE_MAY_CAUSE_DATA_LOSS','en','The change might cause data loss in the PM table. Do you want to continue?','2017-03-30') ,
( 'LABEL','ID_THE_PHP_FILES_EXECUTION_WAS_DISABLED','en','The PHP files execution was disabled please contact the system administrator.','2018-04-20') , ( 'LABEL','ID_THE_PHP_FILES_EXECUTION_WAS_DISABLED','en','The PHP files execution was disabled please contact the system administrator.','2018-04-20') ,
( 'LABEL','ID_THE_REASON_REASSIGN_USER_EMPTY','en','Please complete the reassign reason.','2016-10-20') , ( 'LABEL','ID_THE_REASON_REASSIGN_USER_EMPTY','en','Please complete the reassign reason.','2016-10-20') ,
( 'LABEL','ID_THE_REPORT_TABLE_IS_REGENERATING_PLEASE_COME_BACK_IN_A_FEW_MINUTES','en','The report table is regenerating please come back in a few minutes.','2020-06-01') ,
( 'LABEL','ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED','en','The upload of PHP files was disabled please contact the system administrator.','2018-04-20') , ( 'LABEL','ID_THE_UPLOAD_OF_PHP_FILES_WAS_DISABLED','en','The upload of PHP files was disabled please contact the system administrator.','2018-04-20') ,
( 'LABEL','ID_THE_USERNAME_EMAIL_IS_INCORRECT','en','The username or email is incorrect','2018-01-18') , ( 'LABEL','ID_THE_USERNAME_EMAIL_IS_INCORRECT','en','The username or email is incorrect','2018-01-18') ,
( 'LABEL','ID_THIS_MONTH','en','This Month','2014-01-15') , ( 'LABEL','ID_THIS_MONTH','en','This Month','2014-01-15') ,

80
workflow/engine/src/ProcessMaker/Commands/GenerateDataReport.php
View File

@@ -1,80 +0,0 @@
<?php
namespace ProcessMaker\Commands;
use ProcessMaker\Core\ProcOpen;
class GenerateDataReport extends ProcOpen
{
private $workspace;
private $tableName;
private $type;
private $processUid;
private $gridKey;
private $addTabUid;
private $className;
private $pathWorkspace;
private $start;
private $limit;
/**
* Initializes the command parameters.
* @param string $workspace
* @param string $tableName
* @param string $type
* @param string $processUid
* @param string $gridKey
* @param string $addTabUid
* @param string $className
* @param string $pathWorkspace
* @param integer $start
* @param integer $limit
*/
public function __construct(
$workspace,
$tableName,
$type = 'NORMAL',
$processUid = '',
$gridKey = '',
$addTabUid = '',
$className = '',
$pathWorkspace,
$start = 0,
$limit = 10)
{
$this->workspace = $workspace;
$this->tableName = $tableName;
$this->type = $type;
$this->processUid = $processUid;
$this->gridKey = $gridKey;
$this->addTabUid = $addTabUid;
$this->className = $className;
$this->pathWorkspace = $pathWorkspace;
$this->start = $start;
$this->limit = $limit;
$this->setCwd(PATH_TRUNK);
parent::__construct($this->buildCommand());
}
/**
* Returns the command to execute.
* @return string
*/
private function buildCommand(): string
{
$command = PHP_BINDIR . "/php "
. "./processmaker "
. "'generate-data-report' "
. "'{$this->workspace}' "
. "'tableName={$this->tableName}' "
. "'type={$this->type}' "
. "'process={$this->processUid}' "
. "'gridKey={$this->gridKey}' "
. "'additionalTable={$this->addTabUid}' "
. "'className={$this->className}' "
. "'pathWorkspace={$this->pathWorkspace}' "
. "'start={$this->start}' "
. "'limit={$this->limit}' ";
return $command;
}
}

42
workflow/engine/src/ProcessMaker/Commands/PopulateTableReport.php
View File

@@ -1,42 +0,0 @@
<?php
namespace ProcessMaker\Commands;
use ProcessMaker\Core\ProcOpen;
class PopulateTableReport extends ProcOpen
{
private $workspace;
private $sql;
private $isRbac;
/**
* Initializes the command parameters.
* @param string $workspace
* @param string $sql
* @param boolean $isRbac
*/
public function __construct($workspace, $sql, $isRbac = false)
{
$this->workspace = $workspace;
$this->sql = $sql;
$this->isRbac = $isRbac;
$this->setCwd(PATH_TRUNK);
parent::__construct($this->buildCommand());
}
/**
* Returns the command to execute.
* @return string
*/
public function buildCommand()
{
$command = PHP_BINDIR . "/php "
. "./processmaker "
. "'populate-table' "
. "'{$this->workspace}' "
. base64_encode($this->sql) . " "
. ($this->isRbac ? "'1'" : "'0'");
return $command;
}
}

3
workflow/engine/src/ProcessMaker/Core/JobsManager.php
View File

@@ -5,7 +5,6 @@ namespace ProcessMaker\Core;
use Bootstrap; use Bootstrap;
use Exception; use Exception;
use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Log;
use ProcessMaker\BusinessModel\Factories\Jobs;
use ProcessMaker\Core\System; use ProcessMaker\Core\System;
use Propel; use Propel;
@@ -187,7 +186,7 @@ class JobsManager
{ {
$environment = $this->getDataSnapshot(); $environment = $this->getDataSnapshot();
$instance = Jobs::create($name, function() use ($callback, $environment) { $instance = $name::dispatch(function() use ($callback, $environment) {
try { try {
$this->recoverDataSnapshot($environment); $this->recoverDataSnapshot($environment);
$callback($environment); $callback($environment);

92
workflow/engine/src/ProcessMaker/Core/MultiProcOpen.php
View File

@@ -1,92 +0,0 @@
<?php
namespace ProcessMaker\Core;
class MultiProcOpen
{
/**
* Represents the waiting time before starting the process monitoring.
* @var integer
*/
private $sleepTime = 1;
/**
* This method obtains a paging by returning the start and limit indexes
* compatible with the mysql pagination in its call function.
* The return function must return an instance of the object "ProcessMaker\Core\ProcOpen".
* Returns an array containing the status, content, and errors generated by
* the open process.
* @param int $size
* @param int $chunk
* @param callable $callback
* @return array
*/
public function chunk(int $size, int $chunk, callable $callback): array
{
$start = 0;
$limit = $chunk;
$queries = [];
for ($i = 1; $start < $size; $i++) {
$queries[] = $callback($size, $start, $limit);
$start = $i * $limit;
}
return $this->run($queries);
}
/**
* Open a set of background processes.
* The array must contain one or more instances of the object inherited from
* the class "ProcessMaker\Core\ProcOpen"
* Returns an array containing the status, content, and errors generated by
* the open process.
* @param array $processes
* @return array
*/
public function run(array $processes): array
{
foreach ($processes as $procOpen) {
$procOpen->open();
}
return $this->processMonitoring($processes);
}
/**
* It monitors the open processes, verifying if they have ended or thrown an
* error and later closing the resources related to the process.
* Returns an array containing the status, content, and errors generated by
* the open process.
* @param array $processes
* @return array
*/
private function processMonitoring(array $processes): array
{
sleep($this->sleepTime); //this sleep is very important
$i = 0;
$n = count($processes);
if ($n === 0) {
return [];
}
$outputs = [];
do {
$index = $i % $n;
if (isset($processes[$index])) {
$procOpen = $processes[$index];
$status = $procOpen->getStatus();
$contents = $procOpen->getContents();
$errors = $procOpen->getErrors();
if ($status->running === false || !empty($errors)) {
$outputs[] = [
"status" => $status,
"contents" => $contents,
"errors" => $errors,
];
$procOpen->terminate();
$procOpen->close();
unset($processes[$index]);
}
}
$i = $i + 1;
} while (!empty($processes));
return $outputs;
}
}

126
workflow/engine/src/ProcessMaker/Core/ProcOpen.php
View File

@@ -1,126 +0,0 @@
<?php
namespace ProcessMaker\Core;
class ProcOpen
{
private $command;
private $resource;
private $descriptorspec;
private $pipes;
private $cwd;
/**
* This initializes the descriptors and the command for the open process.
* @param string $command
*/
public function __construct(string $command)
{
$this->descriptorspec = [
['pipe', 'r'],
['pipe', 'w'],
['pipe', 'w']
];
$this->command = $command;
}
/**
* Gets the resource that represents the process.
* @return resource
*/
public function getResource()
{
return $this->resource;
}
/**
* Sets the process execution directory.
* @param string $cwd
*/
public function setCwd(string $cwd)
{
$this->cwd = $cwd;
}
/**
* Open a background process.
*/
public function open()
{
if (empty($this->cwd)) {
$this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes);
} else {
$this->resource = proc_open($this->command, $this->descriptorspec, $this->pipes, $this->cwd);
}
}
/**
* Get the content of the process when it is finished.
* @return string
*/
public function getContents()
{
if (is_resource($this->pipes[1])) {
return stream_get_contents($this->pipes[1]);
}
return "";
}
/**
* Get the process errors when it is finished.
* @return string
*/
public function getErrors()
{
if (is_resource($this->pipes[2])) {
return stream_get_contents($this->pipes[2]);
}
return "";
}
/**
* Close the resources related to the open process.
* return void
*/
public function close()
{
if (is_resource($this->resource)) {
foreach ($this->pipes as $value) {
fclose($value);
}
proc_close($this->resource);
}
}
/**
* End the process before it ends.
*/
public function terminate()
{
if (is_resource($this->resource)) {
proc_terminate($this->resource);
}
}
/**
* Gets the status of the process.
* @return object
*/
public function getStatus()
{
$status = [
"command" => $this->command,
"pid" => null,
"running" => false,
"signaled" => false,
"stopped" => false,
"exitcode" => -1,
"termsig" => 0,
"stopsig" => 0
];
if (is_resource($this->resource)) {
$status = proc_get_status($this->resource);
}
return (object) $status;
}
}

3
workflow/engine/src/ProcessMaker/Core/System.php
View File

@@ -78,7 +78,8 @@ class System
'highlight_home_folder_enable' => 0, 'highlight_home_folder_enable' => 0,
'highlight_home_folder_refresh_time' => 10, 'highlight_home_folder_refresh_time' => 10,
'highlight_home_folder_scope' => 'unassigned', // For now only this list is supported 'highlight_home_folder_scope' => 'unassigned', // For now only this list is supported
'disable_advanced_search_case_title_fulltext' => 0 'disable_advanced_search_case_title_fulltext' => 0,
'report_table_batch_regeneration' => 1000
]; ];
/** /**