From e4722622a5d4e977cc26f2dc758951159c70138d Mon Sep 17 00:00:00 2001 From: Julio Cesar Laura Date: Mon, 15 Oct 2012 14:36:26 -0400 Subject: [PATCH] Bug 8626 Backup issues SOLVED PROBLEM the backup is too bigger to be handle in some os configurations systema files SOLUTION A new class will help to compress into several files instead one. to use the new feature the admin would use and extra option as follow: processmaker workspace-backup -s {} to restore may use : processmaker workspace-restore -m In both case there are to new arguments -s and -m, these are for use the new feature only, if they are not added the old way is still available. --- gulliver/system/class.g.php | 46 +++- workflow/engine/bin/tasks/cliWorkspaces.php | 54 ++++- .../classes/class.multipleFilesBackup.php | 227 ++++++++++++++++++ workflow/engine/classes/class.wsTools.php | 2 +- 4 files changed, 321 insertions(+), 8 deletions(-) create mode 100644 workflow/engine/classes/class.multipleFilesBackup.php diff --git a/gulliver/system/class.g.php b/gulliver/system/class.g.php index 15ae084d3..6c0cca0ea 100755 --- a/gulliver/system/class.g.php +++ b/gulliver/system/class.g.php @@ -5430,7 +5430,51 @@ function getDirectorySize($path,$maxmtime=0) return $reservedWordsSql; } - + + /* Friendly functions. + /** + * isWinOs + * + * @return true if the 3 first letters of PHP_OS got 'WIN', otherwise false. + */ + function isWinOs() + { + return strtoupper(substr(PHP_OS, 0, 3)) == "WIN"; + } + + /** + * isNTOs + * + * @return true if PHP_OS is 'WINNT', otherwise false. + */ + function isNTOs() + { + return PHP_OS == "WINNT"; + } + + /** + * isLinuxOs + * + * @return true if PHP_OS (upper text) got 'LINUX', otherwise false. + */ + function isLinuxOs() + { + return strtoupper(PHP_OS) == "LINUX"; + } + + /** + * getDirSize + * + * @path of the directory that want to check. smaller than getDirectorySize function. + */ + function getDirSize($path) + { + $io = popen('/usr/bin/du -sb '.$path, 'r'); + $size = intval(fgets($io,80)); + pclose($io); + return $size; + } + /** * getMinText, Used with longer size labels to minimize them, don't like it but works. * diff --git a/workflow/engine/bin/tasks/cliWorkspaces.php b/workflow/engine/bin/tasks/cliWorkspaces.php index a73f674e0..3d6b0bb8c 100755 --- a/workflow/engine/bin/tasks/cliWorkspaces.php +++ b/workflow/engine/bin/tasks/cliWorkspaces.php @@ -52,6 +52,7 @@ EOT ); CLI::taskArg('workspace', false); CLI::taskArg('backup-file', true); +CLI::taskOpt("filesize", "Set the max size of the compressed splitted files, by default the max is 1000 Mb.", "s:","filesize="); CLI::taskRun(run_workspace_backup); CLI::taskName('workspace-restore'); @@ -70,6 +71,7 @@ CLI::taskArg('backup-file', false); CLI::taskArg('workspace', true); CLI::taskOpt("overwrite", "If a workspace already exists, overwrite it.", "o", "overwrite"); CLI::taskOpt("info", "Only shows information about a backup archive.", "i"); +CLI::taskOpt("multiple", "Restore from multiple compresed enumerated files.", "m"); CLI::taskOpt("workspace", "Select which workspace to restore if multiple workspaces are present in the archive.", "w:", "workspace="); CLI::taskRun(run_workspace_restore); @@ -374,14 +376,33 @@ function run_workspace_backup($args, $opts) { if (!$workspace->workspaceExists()) throw new Exception("Workspace '{$workspace->name}' not found"); //If this is a relative path, put the file in the backups directory - if (strpos($filename, "/") === false && strpos($filename, '\\') === false) + if (strpos($filename, "/") === false && strpos($filename, '\\') === false){ $filename = PATH_DATA . "backups/$filename"; + } CLI::logging("Backing up to $filename\n"); - $backup = workspaceTools::createBackup($filename); - - foreach ($workspaces as $workspace) - $workspace->backup($backup); + + $filesize = array_key_exists("filesize", $opts) ? $opts['filesize'] : -1; + if($filesize >= 0) + { + if(!G::isLinuxOs()){ + CLI::error("This is not a Linux enviroment, cannot use this filesize [-s] feature.\n"); + return; + } + $multipleBackup = new multipleFilesBackup ($filename,$filesize);//if filesize is 0 the default size will be took + //using new method + foreach ($workspaces as $workspace){ + $multipleBackup->addToBackup($workspace); + } + $multipleBackup->letsBackup(); + } + else + { + //ansient method to backup into one large file + $backup = workspaceTools::createBackup($filename); + foreach ($workspaces as $workspace) + $workspace->backup($backup); + } CLI::logging("\n"); workspaceTools::printSysInfo(); foreach ($workspaces as $workspace) { @@ -405,8 +426,29 @@ function run_workspace_restore($args, $opts) { CLI::logging("Restoring from $filename\n"); $workspace = array_key_exists("workspace", $opts) ? $opts['workspace'] : NULL; $overwrite = array_key_exists("overwrite", $opts); + $multiple = array_key_exists("multiple", $opts); $dstWorkspace = $args[1]; - workspaceTools::restore($filename, $workspace, $dstWorkspace, $overwrite); + if(!empty($multiple)){ + if(!G::isLinuxOs()){ + CLI::error("This is not a Linux enviroment, cannot use this multiple [-m] feature.\n"); + return; + } + multipleFilesBackup::letsRestore ($filename,$workspace,$dstWorkspace,$overwrite); + } + else{ + $anotherExtention = ".*"; //if there are files with and extra extention: e.g. .tar.number + $multiplefiles = glob($filename . $anotherExtention);// example: //shared/workflow_data/backups/myWorkspace.tar.* + if(count($multiplefiles) > 0) + { + CLI::error("Processmaker found these files: .\n"); + foreach($multiplefiles as $index => $value){ + CLI::logging($value . "\n"); + } + CLI::error("Please, you should use -m parameter to restore them.\n"); + return; + } + workspaceTools::restore($filename, $workspace, $dstWorkspace, $overwrite); + } } } diff --git a/workflow/engine/classes/class.multipleFilesBackup.php b/workflow/engine/classes/class.multipleFilesBackup.php new file mode 100644 index 000000000..a3f2be169 --- /dev/null +++ b/workflow/engine/classes/class.multipleFilesBackup.php @@ -0,0 +1,227 @@ +filename = $filename; + } + if(!empty($size) && (int)$size > 0){ + $this->fileSize = $size; + } + } + /* Gets workspace information enough to make its backup. + * @workspace contains the workspace to be add to the commpression process. + */ + public function addToBackup($workspace) + { + //verifing if workspace exists. + if (!$workspace->workspaceExists()) { + echo "Workspace {$workspace->name} not found\n"; + return false; + } + //create destination path + if (!file_exists(PATH_DATA . "upgrade/")){ + mkdir(PATH_DATA . "upgrade/"); + } + $tempDirectory = PATH_DATA . "upgrade/" . basename(tempnam(__FILE__, '')); + mkdir($tempDirectory); + $metadata = $workspace->getMetadata(); + CLI::logging("Temporing up database...\n"); + $metadata["databases"] = $workspace->exportDatabase($tempDirectory); + $metadata["directories"] = array("{$workspace->name}.files"); + $metadata["version"] = 1; + $metaFilename = "$tempDirectory/{$workspace->name}.meta"; + if (!file_put_contents($metaFilename, + str_replace(array(",", "{", "}"), array(",\n ", "{\n ", "\n}\n"), + G::json_encode($metadata)))) { + CLI::logging("Could not create backup metadata"); + } + CLI::logging("Adding database to backup...\n"); + $this->addDirToBackup($tempDirectory); + CLI::logging("Adding files to backup...\n"); + $this->addDirToBackup($workspace->path); + $this->tempDirectories[] = $tempDirectory; + } + + /* Add a directory containing Db files or info files to be commpressed + * @directory the name and path of the directory to be add to the commpression process. + */ + private function addDirToBackup($directory) + { + if(!empty($directory)){ + $this->dir_to_compress .= $directory . " "; + } + } + + /* Commpress the DB and files into a single or several files with numerical series extentions + */ + public function letsBackup() + { + // creating command + $CommpressCommand = "tar czv "; + $CommpressCommand .= $this->dir_to_compress; + $CommpressCommand .= "| split -b "; + $CommpressCommand .= $this->fileSize; + $CommpressCommand .= "m -d - "; + $CommpressCommand .= $this->filename . "."; + //executing command to create the files + echo exec($CommpressCommand); + //Remove leftovers dirs. + foreach($this->tempDirectories as $tempDirectory) + { + CLI::logging("Deleting: ".$tempDirectory."\n"); + G::rm_dir($tempDirectory); + } + } + /* Restore from file(s) commpressed by letsBackup function, into a temporary directory + * @ filename got the name and path of the compressed file(s), if there are many files with file extention as a numerical series, the extention should be discriminated. + * @ srcWorkspace contains the workspace to be restored. + * @ dstWorkspace contains the workspace to be overwriting. + * @ overwrite got the option true if the workspace will be overwrite. + */ + static public function letsRestore($filename, $srcWorkspace, $dstWorkspace = NULL, $overwrite = true) + { + // Needed info: + // TEMPDIR /shared/workflow_data/upgrade/ + // BACKUPS /shared/workflow_data/backups/ + + // Creating command cat myfiles_split.tgz_* | tar xz + $DecommpressCommand = "cat " . $filename . ".* "; + $DecommpressCommand .= " | tar xzv"; + + $tempDirectory = PATH_DATA . "upgrade/" . basename(tempnam(__FILE__, '')); + $parentDirectory = PATH_DATA . "upgrade"; + if (is_writable($parentDirectory)) { + mkdir($tempDirectory); + } else { + throw new Exception("Could not create directory:" . $parentDirectory); + } + //Extract all backup files, including database scripts and workspace files + CLI::logging("Restoring into ".$tempDirectory."\n"); + chdir($tempDirectory); + echo exec($DecommpressCommand); + CLI::logging("\nUncompressed into: ".$tempDirectory."\n"); + + //Search for metafiles in the new standard (the old standard would contain meta files. + $metaFiles = glob($tempDirectory . "/*.meta"); + if (empty($metaFiles)) { + $metaFiles = glob($tempDirectory . "/*.txt"); + if (!empty($metaFiles)){ + return workspaceTools::restoreLegacy($tempDirectory); + } + else{ + throw new Exception("No metadata found in backup"); + } + } + else { + CLI::logging("Found " . count($metaFiles) . " workspaces in backup:\n"); + foreach ($metaFiles as $metafile){ + CLI::logging("-> " . basename($metafile) . "\n"); + } + } + if (count($metaFiles) > 1 && (!isset($srcWorkspace))){ + throw new Exception("Multiple workspaces in backup but no workspace specified to restore"); + } + if (isset($srcWorkspace) && !in_array("$srcWorkspace.meta", array_map(basename, $metaFiles))){ + throw new Exception("Workspace $srcWorkspace not found in backup"); + } + foreach ($metaFiles as $metaFile) { + $metadata = G::json_decode(file_get_contents($metaFile)); + if ($metadata->version != 1){ + throw new Exception("Backup version {$metadata->version} not supported"); + } + $backupWorkspace = $metadata->WORKSPACE_NAME; + if (isset($dstWorkspace)) { + $workspaceName = $dstWorkspace; + $createWorkspace = true; + } + else { + $workspaceName = $metadata->WORKSPACE_NAME; + $createWorkspace = false; + } + if (isset($srcWorkspace) && strcmp($metadata->WORKSPACE_NAME, $srcWorkspace) != 0) { + CLI::logging(CLI::warning("> Workspace $backupWorkspace found, but not restoring.") . "\n"); + continue; + } + else { + CLI::logging("> Restoring " . CLI::info($backupWorkspace) . " to " . CLI::info($workspaceName) . "\n"); + } + $workspace = new workspaceTools($workspaceName); + if ($workspace->workspaceExists()){ + if ($overwrite){ + CLI::logging(CLI::warning("> Workspace $workspaceName already exist, overwriting!") . "\n"); + } + else{ + throw new Exception("Destination workspace already exist (use -o to overwrite)"); + } + } + if (file_exists($workspace->path)) { + G::rm_dir($workspace->path); + } + foreach ($metadata->directories as $dir) { + CLI::logging("+> Restoring directory '$dir'\n"); + if (!rename("$tempDirectory/$dir", $workspace->path)) { + throw new Exception("There was an error copying the backup files ($tempDirectory/$dir) to the workspace directory {$workspace->path}."); + } + } + + CLI::logging("> Changing file permissions\n"); + $shared_stat = stat(PATH_DATA); + if ($shared_stat !== false){ + workspaceTools::dirPerms($workspace->path, $shared_stat['uid'], $shared_stat['gid'], $shared_stat['mode']); + } + else{ + CLI::logging(CLI::error ("Could not get the shared folder permissions, not changing workspace permissions") . "\n"); + } + + list($dbHost, $dbUser, $dbPass) = @explode(SYSTEM_HASH, G::decrypt(HASH_INSTALLATION, SYSTEM_HASH)); + + CLI::logging("> Connecting to system database in '$dbHost'\n"); + $link = mysql_connect($dbHost, $dbUser, $dbPass); + @mysql_query("SET NAMES 'utf8';"); + if (!$link){ + throw new Exception('Could not connect to system database: ' . mysql_error()); + } + + $newDBNames = $workspace->resetDBInfo($dbHost, $createWorkspace); + + foreach ($metadata->databases as $db) { + $dbName = $newDBNames[$db->name]; + CLI::logging("+> Restoring database {$db->name} to $dbName\n"); + $workspace->executeSQLScript($dbName, "$tempDirectory/{$db->name}.sql"); + $workspace->createDBUser($dbName, $db->pass, "localhost", $dbName); + $workspace->createDBUser($dbName, $db->pass, "%", $dbName); + } + $workspace->upgradeCacheView(false); + mysql_close($link); + + } + CLI::logging("Removing temporary files\n"); + G::rm_dir($tempDirectory); + CLI::logging(CLI::info("Done restoring") . "\n"); + } +} + +?> diff --git a/workflow/engine/classes/class.wsTools.php b/workflow/engine/classes/class.wsTools.php index fb41599ee..3841b45ed 100755 --- a/workflow/engine/classes/class.wsTools.php +++ b/workflow/engine/classes/class.wsTools.php @@ -7,7 +7,7 @@ G::LoadSystem('dbMaintenance'); G::LoadClass("cli"); - +G::LoadClass("multipleFilesBackup"); /** * class workspaceTools * @package workflow.engine.classes