BUG-9482 Solr reindex error

Add catch exceptions to continue the index beside errors.
The found errors are logged.
Correct bug in unassigned users and groups.
Add functionality to reindex script:
- Add skip functionality to skip records to index
- Add definition of trunk size used to send docs to index server
- Add optimization utility to optimize index
This commit is contained in:
Herbert Saal Gutierrez
2012-08-03 18:54:16 -04:00
parent 629ea73a15
commit b40a4d593e
3 changed files with 210 additions and 52 deletions

View File

@@ -23,14 +23,51 @@
*/ */
// check script parameters // check script parameters
// php reindex_solr.php workspacename [reindexall|reindexmissing] // php reindex_solr.php workspacename [reindexall|reindexmissing|optimizeindex] [-skip 1005] [-reindextrunksize 1000]
// var_dump($argv); // var_dump($argv);
if (count ($argv) != 3) { //(count ($argv) == 4) || ((count ($argv) == 5) && ($argv [3] != '-skip'))
print "Invalid command line arguments: \n syntax: php reindex_solr.php [workspace_name] [reindexall|reindexmissing] \n" . " Where reindexall : reindex all the database \n" . " reindexmissing: reindex only the missing records stored in database.\n"; $commandLineSyntaxMsg = "Invalid command line arguments: \n " .
"syntax: ".
"php reindex_solr.php [workspace_name] [reindexall|reindexmissing|optimizeindex] [-skip {record_number}] [-reindextrunksize {trunk_size}]\n" .
" Where \n".
" reindexall : reindex all the database. \n" .
" reindexmissing: reindex only the missing records stored in database. \n".
" (records defined in APP_SOLR_QUEUE table are required)\n" .
" optimizeindex: optimize the changes in the search index. (used to get faster results) \n" .
" Optional Options: \n" .
" -skip {record_number}: used to skip a number of records. \n ex: -skip 10000 //skips the first 10000 records. \n" .
" -reindextrunksize {trunk_size}: specify the number of records sent to index each time. \n ex: -reindextrunksize 100 //(default = 1000) \n Reduce the trunk if using big documents, and memory is not enough. \n";
if ( (count ($argv) < 3) || ((count ($argv) % 2) == 0) ||
($argv [2] != 'reindexall' && $argv [2] != 'reindexmissing' && $argv [2] != 'optimizeindex')) {
print $commandLineSyntaxMsg;
die (); die ();
} }
$workspaceName = $argv [1]; $workspaceName = $argv [1];
$ScriptAction = $argv [2]; $ScriptAction = $argv [2];
$SkipRecords = 0;
$TrunkSize = 1000;
//3 5 7
if(count ($argv) > 3) {
for($argNumber = 3 ; $argNumber < count ($argv) ; $argNumber += 2) {
if(($argv [$argNumber] == '-skip' || $argv [$argNumber] == '-reindextrunksize')) {
//get options
if($argv [$argNumber] == '-skip') {
//use skip option
$SkipRecords = intval($argv [$argNumber + 1]);
}
if($argv [$argNumber] == '-reindextrunksize') {
//use skip option
$TrunkSize = intval($argv [$argNumber + 1]);
}
}
else {
print $commandLineSyntaxMsg;
die ();
}
}
}
ini_set ('display_errors', 1); ini_set ('display_errors', 1);
error_reporting (E_ALL); error_reporting (E_ALL);
@@ -237,6 +274,8 @@ function processWorkspace()
{ {
global $sLastExecution; global $sLastExecution;
global $ScriptAction; global $ScriptAction;
global $SkipRecords;
global $TrunkSize;
try { try {
@@ -249,11 +288,14 @@ function processWorkspace()
$oAppSolr = new AppSolr ($solrConf ['solr_enabled'], $solrConf ['solr_host'], $solrConf ['solr_instance']); $oAppSolr = new AppSolr ($solrConf ['solr_enabled'], $solrConf ['solr_host'], $solrConf ['solr_instance']);
if ($ScriptAction == "reindexall") { if ($ScriptAction == "reindexall") {
$oAppSolr->reindexAllApplications (); $oAppSolr->reindexAllApplications ($SkipRecords, $TrunkSize);
} }
if ($ScriptAction == "reindexmissing") { if ($ScriptAction == "reindexmissing") {
$oAppSolr->synchronizePendingApplications (); $oAppSolr->synchronizePendingApplications ();
} }
if ($ScriptAction == "optimizeindex") {
$oAppSolr->optimizeSearchIndex ();
}
} }
else { else {
print "Incomplete Solr configuration. See configuration file: " . PATH_DATA_SITE . "env.ini"; print "Incomplete Solr configuration. See configuration file: " . PATH_DATA_SITE . "env.ini";

View File

@@ -85,6 +85,31 @@ class ApplicationWithoutDelegationRecordsException extends Exception
} }
} }
/**
* Dynaform file corrupt
*
* @author Herbert Saal Gutierrez
*
* @category Colosa
* @copyright Copyright (c) 2005-2012 Colosa Inc. (http://www.colosa.com)
*/
class ApplicationWithCorruptDynaformException extends Exception
{
// Redefine the exception so message isn't optional
public function __construct($message, $code = 0)
{
// some code
// make sure everything is assigned properly
parent::__construct ($message, $code);
}
// custom string representation of object
public function __toString()
{
return __CLASS__ . ": [{$this->code}]: {$this->message}\n";
}
}
/** /**
* Application APP_DATA could not be unserialized exception * Application APP_DATA could not be unserialized exception
* *
@@ -219,6 +244,7 @@ class AppSolr
$swErrorInSearchText = false; $swErrorInSearchText = false;
$solrQueryResult = null; $solrQueryResult = null;
$aPriorities = array('1'=>'VL', '2'=>'L', '3'=>'N', '4'=>'H', '5'=>'VH'); $aPriorities = array('1'=>'VL', '2'=>'L', '3'=>'N', '4'=>'H', '5'=>'VH');
$delegationIndexes = array();
$result = array (); $result = array ();
$result ['totalCount'] = 0; $result ['totalCount'] = 0;
@@ -321,30 +347,30 @@ class AppSolr
if ($userUid != null && $action == 'todo') { if ($userUid != null && $action == 'todo') {
if ($filter == 'read') { if ($filter == 'read') {
$solrSearchText .= "APP_ASSIGNED_USERS_READ:" . $userUid . " AND "; $solrSearchText .= "APP_ASSIGNED_USERS_READ:" . $userUid . " AND ";
$delIndexDynaField = "APP_ASSIGNED_USER_READ_DEL_INDEX_" . trim ($userUid) . '_txt'; $delegationIndexes[] = "APP_ASSIGNED_USER_READ_DEL_INDEX_" . trim ($userUid) . '_txt';
} }
elseif ($filter == 'unread') { elseif ($filter == 'unread') {
$solrSearchText .= "APP_ASSIGNED_USERS_UNREAD:" . $userUid . " AND "; $solrSearchText .= "APP_ASSIGNED_USERS_UNREAD:" . $userUid . " AND ";
$delIndexDynaField = "APP_ASSIGNED_USER_UNREAD_DEL_INDEX_" . trim ($userUid) . '_txt'; $delegationIndexes[] = "APP_ASSIGNED_USER_UNREAD_DEL_INDEX_" . trim ($userUid) . '_txt';
} }
else { else {
$solrSearchText .= "APP_ASSIGNED_USERS:" . $userUid . " AND "; $solrSearchText .= "APP_ASSIGNED_USERS:" . $userUid . " AND ";
$delIndexDynaField = "APP_ASSIGNED_USER_DEL_INDEX_" . trim ($userUid) . '_txt'; $delegationIndexes[] = "APP_ASSIGNED_USER_DEL_INDEX_" . trim ($userUid) . '_txt';
} }
} }
// participated, add condition // participated, add condition
if ($userUid != null && $action == 'sent') { if ($userUid != null && $action == 'sent') {
if ($filter == 'started') { if ($filter == 'started') {
$solrSearchText .= "APP_PARTICIPATED_USERS_STARTED:" . $userUid . " AND "; $solrSearchText .= "APP_PARTICIPATED_USERS_STARTED:" . $userUid . " AND ";
$delIndexDynaField = "APP_PARTICIPATED_USER_STARTED_DEL_INDEX_" . trim ($userUid) . '_txt'; $delegationIndexes[] = "APP_PARTICIPATED_USER_STARTED_DEL_INDEX_" . trim ($userUid) . '_txt';
} }
elseif ($filter == 'completed') { elseif ($filter == 'completed') {
$solrSearchText .= "APP_PARTICIPATED_USERS_COMPLETED:" . $userUid . " AND "; $solrSearchText .= "APP_PARTICIPATED_USERS_COMPLETED:" . $userUid . " AND ";
$delIndexDynaField = "APP_PARTICIPATED_USER_COMPLETED_DEL_INDEX_" . trim ($userUid) . '_txt'; $delegationIndexes[] = "APP_PARTICIPATED_USER_COMPLETED_DEL_INDEX_" . trim ($userUid) . '_txt';
} }
else { else {
$solrSearchText .= "APP_PARTICIPATED_USERS:" . $userUid . " AND "; $solrSearchText .= "APP_PARTICIPATED_USERS:" . $userUid . " AND ";
$delIndexDynaField = "APP_PARTICIPATED_USER_DEL_INDEX_" . trim ($userUid) . '_txt'; $delegationIndexes[] = "APP_PARTICIPATED_USER_DEL_INDEX_" . trim ($userUid) . '_txt';
} }
} }
// draft, add condition // draft, add condition
@@ -356,6 +382,7 @@ class AppSolr
if ($userUid != null && $action == 'unassigned') { if ($userUid != null && $action == 'unassigned') {
// get the list of groups to which belongs the user. // get the list of groups to which belongs the user.
$userGroups = $this->getUserGroups ($userUid); $userGroups = $this->getUserGroups ($userUid);
$solrSearchText .= "(APP_UNASSIGNED_USERS:" . $userUid; $solrSearchText .= "(APP_UNASSIGNED_USERS:" . $userUid;
if (count ($userGroups) > 0) { if (count ($userGroups) > 0) {
$solrSearchText .= " OR "; $solrSearchText .= " OR ";
@@ -370,7 +397,10 @@ class AppSolr
} }
$solrSearchText .= ") AND "; $solrSearchText .= ") AND ";
$delIndexDynaField = "APP_UNASSIGNED_USER_GROUP_DEL_INDEX_" . trim ($userUid) . '_txt'; $delegationIndexes[] = "APP_UNASSIGNED_USER_GROUP_DEL_INDEX_" . trim ($userUid) . '_txt';
foreach ($userGroups as $group) {
$delegationIndexes[] = "APP_UNASSIGNED_USER_GROUP_DEL_INDEX_" . trim ($group ['GRP_UID']) . '_txt';
}
} }
// remove last AND in condition // remove last AND in condition
@@ -392,10 +422,9 @@ class AppSolr
if ($search != "") if ($search != "")
$solrSearchText .= "(" . $search . ")"; $solrSearchText .= "(" . $search . ")";
} }
// add del_index dynamic field to list of resulting columns // add del_index dynamic fields to list of resulting columns
$columsToInclude = array_merge ($columsToInclude, array ( // the fields begin in the 30th column and more
$delIndexDynaField $columsToInclude = array_merge ($columsToInclude, $delegationIndexes);
));
// if is a counter no records are returned // if is a counter no records are returned
if ($doCount) { if ($doCount) {
@@ -420,6 +449,7 @@ class AppSolr
'resultFormat' => 'json' 'resultFormat' => 'json'
); );
$solrRequestData = Entity_SolrRequestData::createForRequestPagination ($data); $solrRequestData = Entity_SolrRequestData::createForRequestPagination ($data);
// use search index to return list of cases // use search index to return list of cases
$searchIndex = new BpmnEngine_Services_SearchIndex ($this->_solrIsEnabled, $this->_solrHost); $searchIndex = new BpmnEngine_Services_SearchIndex ($this->_solrIsEnabled, $this->_solrHost);
@@ -465,13 +495,24 @@ class AppSolr
$result ['totalCount'] = $solrQueryResult->iTotalDisplayRecords; $result ['totalCount'] = $solrQueryResult->iTotalDisplayRecords;
// complete the missing data to display it in the grid. // complete the missing data to display it in the grid.
$delIndexes = array(); //store all the delegation indexes
foreach ($solrQueryResult->aaData as $i => $data) { foreach ($solrQueryResult->aaData as $i => $data) {
//initialize array
$delIndexes = array();
// complete empty values // complete empty values
$appUID = $data [11]; $appUID = $data [11];
$delIndexes = $data [30]; //get all the delindexes
for($i = 30 ; $i < count($data) ; $i++) {
if (is_array ($data [$i])) {
foreach($data [$i] as $delIndex){
$delIndexes[] = $delIndex;
}
}
}
// verify if the delindex is an array // verify if the delindex is an array
// if is not an array all the indexed must be returned // if is not check different types of repositories
if (! is_array ($delIndexes)) { // the delegation index must always be defined.
if (count($delIndexes) == 0) {
// if is draft // if is draft
if ($action == 'draft') { if ($action == 'draft') {
$delIndexes [] = 1; // the first default index $delIndexes [] = 1; // the first default index
@@ -480,9 +521,14 @@ class AppSolr
// get all the indexes // get all the indexes
$delIndexes = $this->getApplicationDelegationsIndex ($appUID); $delIndexes = $this->getApplicationDelegationsIndex ($appUID);
} }
else { else {
$delIndexes = array(); //error an index must always be defined
print "Delegation not defined\n";
} }
/*
elseif ($action == 'unassigned'){
$delIndexes = $this->getApplicationDelegationsIndex ($appUID);
}*/
} }
foreach ($delIndexes as $delIndex) { foreach ($delIndexes as $delIndex) {
$aRow = array (); $aRow = array ();
@@ -535,14 +581,14 @@ class AppSolr
return $result; return $result;
} // end try } // end try
catch ( InvalidIndexSearchTextException $e ) { catch ( InvalidIndexSearchTextException $ex ) {
// return empty result with description of error // return empty result with description of error
$result = array (); $result = array ();
$result ['totalCount'] = 0; $result ['totalCount'] = 0;
$result ['data'] = array (); $result ['data'] = array ();
$result ['success'] = true; $result ['success'] = true;
$result ['result'] = false; $result ['result'] = false;
$result ['message'] = $e->getMessage (); $result ['message'] = $ex->getMessage ();
return $result; return $result;
} }
} }
@@ -928,6 +974,7 @@ class AppSolr
} }
/** /**
* Update the information of the specified applications in Solr * Update the information of the specified applications in Solr
* *
@@ -954,6 +1001,7 @@ class AppSolr
foreach ($aaAPPUIDs as $aAPPUID) { foreach ($aaAPPUIDs as $aAPPUID) {
$this->applicationChangedUpdateSolrQueue ($aAPPUID ['APP_UID'], true); $this->applicationChangedUpdateSolrQueue ($aAPPUID ['APP_UID'], true);
} }
return;
} }
// create XML document // create XML document
$xmlDoc = $this->createSolrXMLDocument ($aaAPPUIDs); $xmlDoc = $this->createSolrXMLDocument ($aaAPPUIDs);
@@ -974,8 +1022,8 @@ class AppSolr
$oSearchIndex->updateIndexDocument ($oSolrUpdateDocument); $oSearchIndex->updateIndexDocument ($oSolrUpdateDocument);
// commit changes // commit changes
$oSearchIndex->commitIndexChanges ($this->_solrInstance); $oSearchIndex->commitIndexChanges ($this->_solrInstance);
} catch(Exception $ex) }
{ catch(Exception $ex) {
//print "Excepcion indexing data: " . $ex->getMessage() . "\n"; die; //print "Excepcion indexing data: " . $ex->getMessage() . "\n"; die;
$fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors."); $fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors.");
fwrite($fh, $ex->getMessage()); fwrite($fh, $ex->getMessage());
@@ -1032,11 +1080,27 @@ class AppSolr
try { try {
$result = $this->getApplicationIndexData ($aAPPUID ['APP_UID']); $result = $this->getApplicationIndexData ($aAPPUID ['APP_UID']);
} }
catch ( ApplicationWithoutDelegationRecordsException $e ) { catch ( ApplicationWithoutDelegationRecordsException $ex ) {
// exception trying to get application information // exception trying to get application information
// skip and continue with the next application // skip and continue with the next application
$fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors.");
fwrite($fh, $ex->getMessage());
fclose($fh);
continue; continue;
} }
catch( ApplicationWithCorruptDynaformException $ex) {
$fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors.");
fwrite($fh, $ex->getMessage());
fclose($fh);
continue;
}
catch (Exception $ex) {
$fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors.");
fwrite($fh, "getApplicationIndexData " . $aAPPUID . ":" . $ex->getMessage() . "\n");
fclose($fh);
continue;
}
$documentInformation = $result [0]; $documentInformation = $result [0];
$dynaformFieldTypes = $result [1]; $dynaformFieldTypes = $result [1];
$lastUpdateDate = $result [2]; $lastUpdateDate = $result [2];
@@ -1058,14 +1122,20 @@ class AppSolr
$draftUser, $participatedUsers, $participatedUsersStartedByUser, $participatedUsersCompletedByUser, $draftUser, $participatedUsers, $participatedUsersStartedByUser, $participatedUsersCompletedByUser,
$unassignedUsers, $unassignedGroups); $unassignedUsers, $unassignedGroups);
} }
catch ( ApplicationAPP_DATAUnserializeException $e ) { catch ( ApplicationAPP_DATAUnserializeException $ex ) {
// exception trying to get application information // exception trying to get application information
$fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors."); $fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors.");
fwrite($fh, $e->getMessage()); fwrite($fh, $ex->getMessage());
fclose($fh); fclose($fh);
// skip and continue with the next application // skip and continue with the next application
continue; continue;
} }
catch (Exception $ex) {
$fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors.");
fwrite($fh, "buildSearchIndexDocumentPMOS2 " . $aAPPUID . ":" . $ex->getMessage() . "\n");
fclose($fh);
continue;
}
} }
@@ -1315,8 +1385,8 @@ class AppSolr
// add dynamic field for del_index information // add dynamic field for del_index information
$writer->startElement ("field"); $writer->startElement ("field");
$writer->writeAttribute ('name', 'APP_UNASSIGNED_USER_GROUP_DEL_INDEX_' . trim ($userUID ['USR_UID']) . '_txt'); $writer->writeAttribute ('name', 'APP_UNASSIGNED_USER_GROUP_DEL_INDEX_' . trim ($groupUID ['USR_UID']) . '_txt');
$writer->text ($userUID ['DEL_INDEX']); $writer->text ($groupUID ['DEL_INDEX']);
$writer->endElement (); $writer->endElement ();
} }
} }
@@ -1333,7 +1403,7 @@ class AppSolr
if (! $UnSerializedCaseData) { if (! $UnSerializedCaseData) {
// error unserializing // error unserializing
throw new ApplicationAPP_DATAUnserializeException ("Could not unserialize APP_DATA of APP_UID: " . $documentData ['APP_UID']); throw new ApplicationAPP_DATAUnserializeException ("Could not unserialize APP_DATA of APP_UID: " . $documentData ['APP_UID'] . "\n");
} }
else { else {
foreach ($UnSerializedCaseData as $k => $value) { foreach ($UnSerializedCaseData as $k => $value) {
@@ -1347,14 +1417,14 @@ class AppSolr
$typeSufix = '_t'; $typeSufix = '_t';
break; break;
case 'Int' : case 'Int' :
if(intval ($value) > 2147483647) { if(intval ($value) > 2147483647) {
$typeSufix = '_tl'; //for long values $typeSufix = '_tl'; //for long values
$value = intval ($value); $value = intval ($value);
} }
else { else {
$typeSufix = '_ti'; $typeSufix = '_ti';
$value = intval ($value); $value = intval ($value);
} }
break; break;
case 'Real' : case 'Real' :
$typeSufix = '_td'; $typeSufix = '_td';
@@ -1397,7 +1467,7 @@ class AppSolr
break; break;
case 'currency' : case 'currency' :
$typeSufix = '_td'; $typeSufix = '_td';
$value = floatval ($value); $value = floatval ($value);
break; break;
case 'percentage' : case 'percentage' :
$typeSufix = '_t'; $typeSufix = '_t';
@@ -1481,7 +1551,7 @@ class AppSolr
// this case occurs when the application doesn't have related delegation // this case occurs when the application doesn't have related delegation
// records. // records.
if (empty ($allAppDbData) || ! isset ($allAppDbData [0])) { if (empty ($allAppDbData) || ! isset ($allAppDbData [0])) {
throw new ApplicationWithoutDelegationRecordsException ("Application without delegation records. APP_UID: " . $AppUID); throw new ApplicationWithoutDelegationRecordsException ("Application without delegation records. APP_UID: " . $AppUID . "\n");
} }
// copy the application information // copy the application information
@@ -1593,33 +1663,42 @@ class AppSolr
// search information of unassigned users // search information of unassigned users
// the unassigned users are the self service users and groups. // the unassigned users are the self service users and groups.
// the self service users are defined in the TASKs of the PROCESS. // the self service users are defined in the TASKs of the PROCESS.
foreach ($allAppDbData as $row) { $unassignedUsers = array ();
$unassignedGroups = array ();
//filter only the delegations that are in selfservice status
// `USR_UID` = '' AND `DEL_FINISH_DATE` IS NULL
$indexes = $this->aaSearchRecords ($allAppDbData, array (
'USR_UID' => 'NULL',
'DEL_FINISH_DATE' => 'NULL',
'APP_THREAD_STATUS' => 'OPEN'
));
foreach ($indexes as $index) {
$unassignedUsersGroups = array (); $unassignedUsersGroups = array ();
// use cache // use cache
$oMemcache = PMmemcached::getSingleton ($this->_solrInstance); $oMemcache = PMmemcached::getSingleton ($this->_solrInstance);
$unassignedUsersGroups = $oMemcache->get ($row ['PRO_UID'] . "_" . $row ['TAS_UID']); $unassignedUsersGroups = $oMemcache->get ($allAppDbData [$index] ['PRO_UID'] . "_" . $allAppDbData [$index] ['TAS_UID']);
if (! $unassignedUsersGroups) { if (! $unassignedUsersGroups) {
$unassignedUsersGroups = $this->getTaskUnassignedUsersGroupsData ($row ['PRO_UID'], $row ['TAS_UID']); $unassignedUsersGroups = $this->getTaskUnassignedUsersGroupsData ($allAppDbData [$index] ['PRO_UID'], $allAppDbData [$index] ['TAS_UID']);
// add del_index // if the task has unassigned users or groups add del_index of delegation
foreach ($unassignedUsersGroups as $i => $newRow) { foreach ($unassignedUsersGroups as $i => $newRow) {
$unassignedUsersGroups [$i] ['DEL_INDEX'] = $row ['DEL_INDEX']; $unassignedUsersGroups [$i] ['DEL_INDEX'] = $allAppDbData [$index] ['DEL_INDEX'];
} }
// store in cache // store in cache
$oMemcache->set ($row ['PRO_UID'] . "_" . $row ['TAS_UID'], $unassignedUsersGroups); $oMemcache->set ($allAppDbData [$index] ['PRO_UID'] . "_" . $allAppDbData [$index] ['TAS_UID'], $unassignedUsersGroups);
} }
// copy list of unassigned users and groups // copy list of unassigned users and groups
$unassignedUsers = array ();
$unassignedGroups = array ();
foreach ($unassignedUsersGroups as $unassignedUserGroup) { foreach ($unassignedUsersGroups as $unassignedUserGroup) {
//unassigned users
if ($unassignedUserGroup ['TU_RELATION'] == 1) { if ($unassignedUserGroup ['TU_RELATION'] == 1) {
$unassignedUsers [] = array ( $unassignedUsers [] = array (
'USR_UID' => $unassignedUserGroup ['USR_UID'], 'USR_UID' => $unassignedUserGroup ['USR_UID'],
'DEL_INDEX' => $unassignedUserGroup ['DEL_INDEX'] 'DEL_INDEX' => $unassignedUserGroup ['DEL_INDEX']
); );
} }
//unassigned groups
elseif ($unassignedUserGroup ['TU_RELATION'] == 2) { elseif ($unassignedUserGroup ['TU_RELATION'] == 2) {
$unassignedGroups [] = array ( $unassignedGroups [] = array (
'USR_UID' => $unassignedUserGroup ['USR_UID'], 'USR_UID' => $unassignedUserGroup ['USR_UID'],
@@ -1644,12 +1723,17 @@ class AppSolr
G::LoadClass ('dynaformhandler'); G::LoadClass ('dynaformhandler');
$dynaformFileNames = $this->getProcessDynaformFileNames ($documentInformation ['PRO_UID']); $dynaformFileNames = $this->getProcessDynaformFileNames ($documentInformation ['PRO_UID']);
$dynaformFields = array (); $dynaformFields = array ();
foreach ($dynaformFileNames as $dynaformFileName) { foreach ($dynaformFileNames as $dynaformFileName) {
if (file_exists (PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml') && if (file_exists (PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml') &&
filesize(PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml') >0 ) { filesize(PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml') >0 ) {
$dyn = new dynaFormHandler (PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml'); $dyn = new dynaFormHandler (PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml');
$dynaformFields [] = $dyn->getFields (); $dynaformFields [] = $dyn->getFields ();
} }
if (file_exists (PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml') &&
filesize(PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml') == 0 ) {
throw new ApplicationWithCorruptDynaformException("Application with corrupt dynaform. APP_UID: " . $AppUID . "\n");
}
} }
foreach ($dynaformFields as $aDynFormFields) { foreach ($dynaformFields as $aDynFormFields) {
@@ -1787,10 +1871,10 @@ class AppSolr
* Search array of indexes that fullfill the conditions * Search array of indexes that fullfill the conditions
* *
* @param * @param
* array o arrays $arr contains the arrays that are searched * array of arrays $arr contains the arrays that are searched
* @param array $andColumnsConditions * @param array $andColumnsConditions
* contain the conditions that must fullfill 'Column'=>'Condition' * contain the conditions that must fullfill 'Column'=>'Condition'
* @return array array of indexes with the found records * @return array of indexes with the found records
*/ */
public function aaSearchRecords($arr, $andColumnsConditions) public function aaSearchRecords($arr, $andColumnsConditions)
{ {
@@ -2092,11 +2176,28 @@ class AppSolr
*/ */
public function getCountApplicationsSearchIndex() public function getCountApplicationsSearchIndex()
{ {
G::LoadClass ('searchIndex');
$searchIndex = new BpmnEngine_Services_SearchIndex ($this->_solrIsEnabled, $this->_solrHost); $searchIndex = new BpmnEngine_Services_SearchIndex ($this->_solrIsEnabled, $this->_solrHost);
// execute query // execute query
$count = $searchIndex->getNumberDocuments ($this->_solrInstance); $count = $searchIndex->getNumberDocuments ($this->_solrInstance);
return $count; return $count;
}
/**
* Optimize the records in search index
*
* @return
*/
public function optimizeSearchIndex()
{
G::LoadClass ('searchIndex');
$searchIndex = new BpmnEngine_Services_SearchIndex ($this->_solrIsEnabled, $this->_solrHost);
// execute query
$searchIndex->optimizeIndexChanges ($this->_solrInstance);
} }
/** /**
@@ -2135,9 +2236,9 @@ class AppSolr
* Reindex all the application records in Solr server * Reindex all the application records in Solr server
* update applications in groups of 1000 * update applications in groups of 1000
*/ */
public function reindexAllApplications() public function reindexAllApplications($SkipRecords = 0, $indexTrunkSize = 1000)
{ {
$trunk = 1000; $trunk = $indexTrunkSize;
// delete all documents to begin reindex // delete all documents to begin reindex
// deleteAllDocuments(); // deleteAllDocuments();
// commitChanges(); // commitChanges();
@@ -2147,8 +2248,8 @@ class AppSolr
print "Total number of records: " . $numRows . "\n"; print "Total number of records: " . $numRows . "\n";
// //
$initTimeAll = microtime (true); $initTimeAll = microtime (true);
// $numRows = 15;
for ($skip = 0; $skip <= $numRows;) { for ($skip = $SkipRecords; $skip <= $numRows;) {
$aaAPPUIds = $this->getPagedApplicationUids ($skip, $trunk); $aaAPPUIds = $this->getPagedApplicationUids ($skip, $trunk);
printf ("Indexing %d to %d \n", $skip, $skip + $trunk); printf ("Indexing %d to %d \n", $skip, $skip + $trunk);
@@ -2162,6 +2263,7 @@ class AppSolr
$curTimeDoc = gmdate ('H:i:s', (microtime (true) - $initTimeAll)); $curTimeDoc = gmdate ('H:i:s', (microtime (true) - $initTimeAll));
printf ("Total reindex time: %s \n", $curTimeDoc); printf ("Total reindex time: %s \n", $curTimeDoc);
printf ("Reindex completed successfully!!.\n");
} }
} }

View File

@@ -331,6 +331,20 @@ class BpmnEngine_Services_SearchIndex
$solr->commitChanges ($workspace); $solr->commitChanges ($workspace);
} }
/**
* Optimize index changes
* @param string $workspace
*/
public function optimizeIndexChanges($workspace)
{
G::LoadClass ('solr');
$solr = new BpmnEngine_SearchIndexAccess_Solr ($this->_solrIsEnabled, $this->_solrHost);
// commit
$solr->optimizeChanges ($workspace);
}
/** /**
* Call Solr server to return the list of paginated pages. * Call Solr server to return the list of paginated pages.
* @param FacetRequest $solrRequestData * @param FacetRequest $solrRequestData