fernando/luos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

BUG-9482 Solr reindex error solved

Browse Source 
Error
-----
running reindex script to reindex Solr don't finish

php reindex_solr.php workflow reindexall

Solution
--------
Add validation of empty dynaform XML files.
Add validation of big integer values.
Add validation of currency values.
Add Solr response information to exceptions in Solr.
write log of not indexed files.
This commit is contained in:
Herbert Saal Gutierrez
2012-07-19 19:59:26 -04:00
parent 6df4a049ff
commit 629ea73a15
2 changed files with 52 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
workflow/engine/classes/class.AppSolr.php
View File

@@ -463,6 +463,7 @@ class AppSolr
$rows = array ();
// number of found records
$result ['totalCount'] = $solrQueryResult->iTotalDisplayRecords;
// complete the missing data to display it in the grid.
foreach ($solrQueryResult->aaData as $i => $data) {
// complete empty values
@@ -479,6 +480,9 @@ class AppSolr
// get all the indexes
$delIndexes = $this->getApplicationDelegationsIndex ($appUID);
}
else {
$delIndexes = array();
}
}
foreach ($delIndexes as $delIndex) {
$aRow = array ();
@@ -953,7 +957,7 @@ class AppSolr
}
// create XML document
$xmlDoc = $this->createSolrXMLDocument ($aaAPPUIDs);
// update document
$data = array (
'workspace' => $this->_solrInstance,
@@ -965,11 +969,18 @@ class AppSolr
G::LoadClass ('searchIndex');
$oSearchIndex = new BpmnEngine_Services_SearchIndex ($this->_solrIsEnabled, $this->_solrHost);
$oSearchIndex->updateIndexDocument ($oSolrUpdateDocument);
// commit changes
$oSearchIndex->commitIndexChanges ($this->_solrInstance);
try{
$oSearchIndex->updateIndexDocument ($oSolrUpdateDocument);
// commit changes
$oSearchIndex->commitIndexChanges ($this->_solrInstance);
} catch(Exception $ex)
{
//print "Excepcion indexing data: " . $ex->getMessage() . "\n"; die;
$fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors.");
fwrite($fh, $ex->getMessage());
fclose($fh);
}
}
/**
@@ -1016,7 +1027,7 @@ class AppSolr
// search data from DB
$xmlDoc = "<?xml version='1.0' encoding='UTF-8'?>\n";
$xmlDoc .= "<add>\n";
// echo "APP Uids to index \n";
foreach ($aaAPPUIDs as $aAPPUID) {
try {
$result = $this->getApplicationIndexData ($aAPPUID ['APP_UID']);
@@ -1039,7 +1050,7 @@ class AppSolr
$participatedUsersCompletedByUser = $result [10];
$unassignedUsers = $result [11];
$unassignedGroups = $result [12];
try {
// create document
$xmlDoc .= $this->buildSearchIndexDocumentPMOS2 ($documentInformation, $dynaformFieldTypes,
@@ -1049,11 +1060,9 @@ class AppSolr
}
catch ( ApplicationAPP_DATAUnserializeException $e ) {
// exception trying to get application information
//print $e->message +" \n";
//$fh = fopen("./UnserializeError_APP_DATA".".txt", 'a') or die("can't open file");
//fwrite($fh, $e->message . "\n");
//fclose($fh);
$fh = fopen("./SolrIndexErrors.txt", 'a') or die("can't open file to store Solr index errors.");
fwrite($fh, $e->getMessage());
fclose($fh);
// skip and continue with the next application
continue;
}
@@ -1061,7 +1070,6 @@ class AppSolr
}
$xmlDoc .= "</add>\n";
return $xmlDoc;
}
@@ -1282,7 +1290,7 @@ class AppSolr
$writer->endElement ();
}
}
if (is_array ($unassignedUsers) && ! empty ($unassignedUsers)) {
foreach ($unassignedUsers as $userUID) {
$writer->startElement ("field");
@@ -1312,7 +1320,7 @@ class AppSolr
$writer->endElement ();
}
}
// get the serialized fields
if (! empty ($documentData ['APP_DATA']) && $documentData ['APP_DATA'] != "N;" ) {
@@ -1325,7 +1333,7 @@ class AppSolr
if (! $UnSerializedCaseData) {
// error unserializing
throw new ApplicationAPP_DATAUnserializeException ("Could not unserialize APP_DATA of APP_UID: " . $documentData ['APP_UID']);
throw new ApplicationAPP_DATAUnserializeException ("Could not unserialize APP_DATA of APP_UID: " . $documentData ['APP_UID']);
}
else {
foreach ($UnSerializedCaseData as $k => $value) {
@@ -1339,8 +1347,14 @@ class AppSolr
$typeSufix = '_t';
break;
case 'Int' :
$typeSufix = '_ti';
$value = intval ($value);
if(intval ($value) > 2147483647) {
$typeSufix = '_tl'; //for long values
$value = intval ($value);
}
else {
$typeSufix = '_ti';
$value = intval ($value);
}
break;
case 'Real' :
$typeSufix = '_td';
@@ -1383,6 +1397,7 @@ class AppSolr
break;
case 'currency' :
$typeSufix = '_td';
$value = floatval ($value);
break;
case 'percentage' :
$typeSufix = '_t';
@@ -1459,7 +1474,7 @@ class AppSolr
public function getApplicationIndexData($AppUID)
{
G::LoadClass ('memcached');
// get all the application data
$allAppDbData = $this->getApplicationDelegationData ($AppUID);
// check if the application record was found
@@ -1471,7 +1486,7 @@ class AppSolr
// copy the application information
$documentInformation = $allAppDbData [0];
// get the last delegate date using the del_delegate_date
$index = $this->aaGetMaximun ($allAppDbData, 'DEL_DELEGATE_DATE', 'DATE');
@@ -1620,15 +1635,18 @@ class AppSolr
// All the datatypes of the process => all variables in all dynaforms in the
// process
$dynaformFieldTypes = array ();
// get cache instance
$oMemcache = PMmemcached::getSingleton ($this->_solrInstance);
$dynaformFieldTypes = $oMemcache->get ($documentInformation ['PRO_UID']);
if (! $dynaformFieldTypes) {
G::LoadClass ('dynaformhandler');
$dynaformFileNames = $this->getProcessDynaformFileNames ($documentInformation ['PRO_UID']);
$dynaformFields = array ();
foreach ($dynaformFileNames as $dynaformFileName) {
if (file_exists (PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml')) {
foreach ($dynaformFileNames as $dynaformFileName) {
if (file_exists (PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml') &&
filesize(PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml') >0 ) {
$dyn = new dynaFormHandler (PATH_DATA . '/sites/workflow/xmlForms/' . $dynaformFileName ['DYN_FILENAME'] . '.xml');
$dynaformFields [] = $dyn->getFields ();
}
@@ -1651,7 +1669,6 @@ class AppSolr
// create cache of dynaformfields
$oMemcache->set ($documentInformation ['PRO_UID'], $dynaformFieldTypes);
}
// return result values
$result = array (
$documentInformation,

22
workflow/engine/classes/class.solr.php
View File

@@ -87,7 +87,7 @@ class BpmnEngine_SearchIndexAccess_Solr
// verify the result of solr
$responseSolrTotal = G::json_decode ($responseTotal);
if ($responseSolrTotal->responseHeader->status != 0) {
throw new Exception ("Error returning the total number of documents in Solr." . $solrIntruct);
throw new Exception ("Error returning the total number of documents in Solr." . $solrIntruct . " response error: " . $response . "\n");
}
$numTotalDocs = $responseSolrTotal->response->numFound;
return $numTotalDocs;
@@ -156,7 +156,7 @@ class BpmnEngine_SearchIndexAccess_Solr
// decode
$responseSolr = G::json_decode ($response);
if ($responseSolr->responseHeader->status != 0) {
throw new Exception ("Error executing query to Solr." . $solrIntruct);
throw new Exception ("Error executing query to Solr." . $solrIntruct . " response error: " . $response . "\n");
}
return $responseSolr;
@@ -192,7 +192,7 @@ class BpmnEngine_SearchIndexAccess_Solr
$swOk = strpos ($response, '<int name="status">0</int>');
if (! $swOk) {
throw new Exception ("Error updating document in Solr." . $solrIntruct);
throw new Exception ("Error updating document in Solr." . $solrIntruct . " response error: " . $response . "\n");
}
}
@@ -226,7 +226,7 @@ class BpmnEngine_SearchIndexAccess_Solr
$swOk = strpos ($response, '<int name="status">0</int>');
if (! $swOk) {
throw new Exception ("Error commiting changes in Solr." . $solrIntruct);
throw new Exception ("Error commiting changes in Solr." . $solrIntruct . " response error: " . $response . "\n");
}
}
@@ -261,7 +261,7 @@ class BpmnEngine_SearchIndexAccess_Solr
$swOk = strpos ($response, '<int name="status">0</int>');
if (! $swOk) {
throw new Exception ("Error rolling back changes in Solr." . $solrIntruct);
throw new Exception ("Error rolling back changes in Solr." . $solrIntruct . " response error: " . $response . "\n");
}
}
@@ -296,7 +296,7 @@ class BpmnEngine_SearchIndexAccess_Solr
$swOk = strpos ($response, '<int name="status">0</int>');
if (! $swOk) {
throw new Exception ("Error optimizing changes in Solr." . $solrIntruct);
throw new Exception ("Error optimizing changes in Solr." . $solrIntruct . " response error: " . $response . "\n");
}
}
@@ -326,7 +326,7 @@ class BpmnEngine_SearchIndexAccess_Solr
// decode
$responseSolr = G::json_decode ($response);
if ($responseSolr->responseHeader->status != 0) {
throw new Exception ("Error getting index fields in Solr." . $solrIntruct);
throw new Exception ("Error getting index fields in Solr." . $solrIntruct . " response error: " . $response . "\n");
}
return $responseSolr;
}
@@ -364,7 +364,7 @@ class BpmnEngine_SearchIndexAccess_Solr
$swOk = strpos ($response, '<int name="status">0</int>');
if (! $swOk) {
throw new Exception ("Error deleting all documents in Solr." . $solrIntruct);
throw new Exception ("Error deleting all documents in Solr." . $solrIntruct . " response error: " . $response . "\n");
}
}
@@ -401,7 +401,7 @@ class BpmnEngine_SearchIndexAccess_Solr
$swOk = strpos ($response, '<int name="status">0</int>');
if (! $swOk) {
throw new Exception ("Error deleting document in Solr." . $solrIntruct);
throw new Exception ("Error deleting document in Solr." . $solrIntruct . " response error: " . $response . "\n");
}
}
@@ -472,9 +472,9 @@ class BpmnEngine_SearchIndexAccess_Solr
// decode
$responseSolr = G::json_decode ($response);
if ($responseSolr->responseHeader->status != 0) {
throw new Exception ("Error getting faceted list from Solr." . $solrIntruct);
throw new Exception ("Error getting faceted list from Solr." . $solrIntruct . " response error: " . $response . "\n");
}
return $responseSolr;
}
}
}