-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add backup folder to persistent volume for services (#267)
Co-authored-by: Thomas Fink <[email protected]>
- Loading branch information
1 parent
107d2fb
commit 77876b2
Showing
4 changed files
with
79 additions
and
87 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,124 +1,117 @@ | ||
| #!/usr/bin/env php | ||
| <?php | ||
| /** | ||
| * Download and validate json files | ||
| * Download and validate JSON files, and handle backups if changes are detected. | ||
| */ | ||
| $localInclude = __DIR__ . DIRECTORY_SEPARATOR . '..' . DIRECTORY_SEPARATOR . 'vendor'; | ||
| $modulInclude = __DIR__ . str_repeat(DIRECTORY_SEPARATOR . '..', 3); | ||
| set_include_path(get_include_path() . PATH_SEPARATOR . $localInclude . PATH_SEPARATOR . $modulInclude); | ||
| require('autoload.php'); | ||
| require 'vendor/autoload.php'; | ||
|
|
||
| use Garden\Cli\Cli; | ||
| use \Httpful\Request; | ||
| use Httpful\Request; | ||
|
|
||
| $cli = new Cli(); | ||
|
|
||
| $cli->description("Downloads json files from the official site and validates them and replaces the old files.") | ||
| $cli->description("Downloads JSON files from the official site, validates them, replaces the old files, and handles backups if changes are detected.") | ||
| ->opt('output:o', 'Path to save files', true) | ||
| ->opt('proxy:p', 'Proxy server like "proxy:3128"', false) | ||
| ->opt('base:b', 'Base download url', false) | ||
| ; | ||
| ->opt('proxy:p', 'Proxy server like \"proxy:3128\"', false) | ||
| ->opt('base:b', 'Base download url', true); | ||
| $args = $cli->parse($argv, true); | ||
|
|
||
| $destinationPath = realpath($args->getOpt('output')); | ||
| $destinationPath = $args->getOpt('output'); | ||
| $backupPath = $destinationPath . '/backups'; | ||
|
|
||
| if (!is_writeable($destinationPath) || !is_dir($destinationPath)) { | ||
| echo $cli->red("$destinationPath should be a writeable directory\n"); | ||
| // Ensure the destination directory exists | ||
| if (!is_dir($destinationPath) && !mkdir($destinationPath, 0777, true) && !is_dir($destinationPath)) { | ||
| echo $cli->red("Failed to create the destination directory at $destinationPath\n"); | ||
| exit(1); | ||
| } | ||
|
|
||
| $baseDomain = $args->getOpt("base"); | ||
| $client = Request::init() | ||
| ->withoutAutoParsing() | ||
| ->followRedirects() | ||
| ->timeout(30); | ||
| $proxy = $args->getOpt('proxy') ? $args->getOpt('proxy') : getenv('HTTP_PROXY'); | ||
| $baseDomain = $args->getOpt('base'); | ||
| $proxy = $args->getOpt('proxy') ?: getenv('HTTP_PROXY'); | ||
| $client = Request::init()->withoutAutoParsing()->followRedirects()->timeout(30); | ||
|
|
||
| if ($proxy) { | ||
| $proxyconf = array(); | ||
| preg_match('#^(?:(?P<user>[^:]+):(?P<pass>[^@]+)@)?(?P<host>[^:]+):?(?P<port>\d+)?$#', $proxy, $proxyconf); | ||
| $proxyconf = array_merge(array('host' => '','port' => '80','user' => false,'pass' => false), $proxyconf); | ||
| if ($proxyconf['user']) { | ||
| $client->useProxy( | ||
| $proxyconf['host'], | ||
| $proxyconf['port'], | ||
| CURLAUTH_BASIC, | ||
| $proxyconf['user'], | ||
| $proxyconf['pass'] | ||
| ); | ||
| } else { | ||
| $client->useProxy($proxyconf['host'], $proxyconf['port']); | ||
| } | ||
| $client->useProxy($proxy); | ||
| } | ||
|
|
||
| Request::ini($client); | ||
|
|
||
| $downloads = array( | ||
| array( | ||
| $downloads = [ | ||
| [ | ||
| 'url' => '/export/standorte/json/', | ||
| 'file' => 'locations_de.json', | ||
| ), | ||
| array( | ||
| ], | ||
| [ | ||
| 'url' => '/export/standorte/json/en/', | ||
| 'file' => 'locations_en.json', | ||
| ), | ||
| array( | ||
| ], | ||
| [ | ||
| 'url' => '/export/dienstleistungen/json/', | ||
| 'file' => 'services_de.json', | ||
| ), | ||
| array( | ||
| ], | ||
| [ | ||
| 'url' => '/export/dienstleistungen/json/en/', | ||
| 'file' => 'services_en.json', | ||
| ), | ||
| array( | ||
| ], | ||
| [ | ||
| 'url' => '/export/themen/json/', | ||
| 'file' => 'topic_de.json', | ||
| ), | ||
| array( | ||
| ], | ||
| [ | ||
| 'url' => '/export/behoerden/json/', | ||
| 'file' => 'authority_de.json', | ||
| ), | ||
| array( | ||
| ], | ||
| [ | ||
| 'url' => '/export/settings/json/', | ||
| 'file' => 'settings.json', | ||
| ), | ||
| ); | ||
| ], | ||
| ]; | ||
|
|
||
| // Backup logic | ||
| $backupRequired = false; | ||
| foreach ($downloads as $download) { | ||
| $dest = $destinationPath . DIRECTORY_SEPARATOR . $download['file']; | ||
| $destFile = $destinationPath . '/' . $download['file']; | ||
| $request = Request::get($baseDomain . $download['url'])->send(); | ||
|
|
||
| if (file_exists($dest)) { | ||
| $oldContentJson = file_get_contents($dest); | ||
| } else { | ||
| $oldContentJson = '{}'; | ||
| if ($request->code !== 200 || (isset($request->body->error) && $request->body->error)) { | ||
| echo $cli->red("Failed to download or validate {$download['file']}\n"); | ||
| continue; | ||
| } | ||
| $oldContent = json_decode($oldContentJson, true); | ||
| $oldContentJson = null; | ||
|
|
||
| $request = Request::get($baseDomain . $download['url']); | ||
|
|
||
| if (isset($oldContent['hash']) && !empty($oldContent['hash'])) { | ||
| $request->addHeader('If-None-Match', $oldContent['hash']); | ||
| $newContent = $request->raw_body; | ||
| if (file_exists($destFile) && md5_file($destFile) !== md5($newContent)) { | ||
| $backupRequired = true; | ||
| } | ||
| $oldContent = null; | ||
|
|
||
| $response = $request | ||
| //->followRedirects() | ||
| ->send() | ||
| ; | ||
|
|
||
| file_put_contents($destFile, $newContent); | ||
| } | ||
|
|
||
| if (304 == $response->code) { | ||
| continue; | ||
| // Perform backup if necessary | ||
| if ($backupRequired) { | ||
| $timestamp = date('Y-m-d'); | ||
| $backupDir = $backupPath . '/' . $timestamp; | ||
|
|
||
| if (!mkdir($backupDir, 0777, true) && !is_dir($backupDir)) { | ||
| echo $cli->red("Failed to create backup directory at $backupDir\n"); | ||
| exit(1); | ||
| } | ||
|
|
||
| // Copy updated files to backup | ||
| foreach (glob($destinationPath . '/*.json') as $file) { | ||
| if (strpos($file, '/backups/') === false && !copy($file, $backupDir . '/' . basename($file))) { | ||
| echo $cli->red("Failed to backup $file\n"); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| $json = $response->body; | ||
| $tmp = $destinationPath . DIRECTORY_SEPARATOR . 'tmp.' . $download['file']; | ||
| file_put_contents($tmp, $json); | ||
| $data = json_decode($json, true); | ||
| $json = null; | ||
| $response = null; | ||
| if (array_key_exists('error', $data) && $data['error'] === false) { | ||
| rename($tmp, $dest); | ||
| } else { | ||
| echo $cli->red($baseDomain . $download['url'] . " did not validate, see temp file $tmp \n"); | ||
| exit(1); | ||
| // Logic to delete backups older than 7 days | ||
| $sevenDaysAgo = time() - (7 * 24 * 60 * 60); | ||
| foreach (glob($backupPath . '/*', GLOB_ONLYDIR) as $dir) { | ||
| if (filemtime($dir) < $sevenDaysAgo) { | ||
| foreach (new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir, FilesystemIterator::SKIP_DOTS), RecursiveIteratorIterator::CHILD_FIRST) as $item) { | ||
| $item->isDir() ? rmdir($item->getRealPath()) : unlink($item->getRealPath()); | ||
| } | ||
| rmdir($dir); | ||
| echo "Deleted old backup: $dir\n"; | ||
| } | ||
| $data = null; | ||
| } | ||
|
|
||
| echo "Downloads and backups (if necessary) completed.\n"; |