diff --git a/postgres-appliance/Dockerfile b/postgres-appliance/Dockerfile index 89298181b..51fe19d1b 100644 --- a/postgres-appliance/Dockerfile +++ b/postgres-appliance/Dockerfile @@ -19,7 +19,7 @@ FROM $BASE_IMAGE as dependencies-builder ARG DEMO -ENV WALG_VERSION=v2.0.1 +ENV WALG_VERSION=v3.0.3 COPY build_scripts/dependencies.sh /builddeps/ diff --git a/postgres-appliance/bootstrap/clone_with_wale.py b/postgres-appliance/bootstrap/clone_with_wale.py index e8d31962d..595d19e38 100755 --- a/postgres-appliance/bootstrap/clone_with_wale.py +++ b/postgres-appliance/bootstrap/clone_with_wale.py @@ -2,6 +2,7 @@ import argparse import csv +import json import logging import os import re @@ -40,7 +41,7 @@ def read_configuration(): return options(args.scope, args.datadir, recovery_target_time, args.dry_run) -def build_wale_command(command, datadir=None, backup=None): +def build_wale_command(command, datadir=None, backup=None, extra_args=None): cmd = ['wal-g' if os.getenv('USE_WALG_RESTORE') == 'true' else 'wal-e'] + [command] if command == 'backup-fetch': if datadir is None or backup is None: @@ -48,6 +49,8 @@ def build_wale_command(command, datadir=None, backup=None): cmd.extend([datadir, backup]) elif command != 'backup-list': raise Exception("invalid {0} command {1}".format(cmd[0], command)) + if extra_args: + cmd.extend(extra_args) return cmd @@ -57,8 +60,7 @@ def fix_output(output): started = None for line in output.decode('utf-8').splitlines(): if not started: - started = re.match(r'^name\s+last_modified\s+', line) - started = re.match(r'^name\s+last_modified\s+', line) or re.match(r'^name\s+modified\s+', line) + started = re.match(r'^(backup_)?name\s+(last_)?modified\s+', line) if started: line = line.replace(' modified ', ' last_modified ') if started: @@ -70,20 +72,25 @@ def choose_backup(backup_list, recovery_target_time): match_timestamp = match = None for backup in backup_list: - last_modified = parse(backup['last_modified']) + last_modified = parse(backup['finish_time' if os.getenv('USE_WALG_RESTORE') == 'true' else 'last_modified']) if last_modified < recovery_target_time: if match is None or last_modified > match_timestamp: match = backup match_timestamp = last_modified if match is not None: - return match['name'] + return match.get('name', match['backup_name']) def list_backups(env): - backup_list_cmd = build_wale_command('backup-list') - output = subprocess.check_output(backup_list_cmd, env=env) - reader = csv.DictReader(fix_output(output), dialect='excel-tab') - return list(reader) + if os.getenv('USE_WALG_RESTORE') == 'true': + backup_list_cmd = build_wale_command('backup-list', extra_args=['--detail', '--json']) + output = subprocess.check_output(backup_list_cmd, env=env) + return json.loads(output or 'null') + else: + backup_list_cmd = build_wale_command('backup-list') + output = subprocess.check_output(backup_list_cmd, env=env) + reader = csv.DictReader(fix_output(output), dialect='excel-tab') + return list(reader) def get_clone_envdir(): diff --git a/postgres-appliance/scripts/postgres_backup.sh b/postgres-appliance/scripts/postgres_backup.sh index 3216ae4ed..f26cc08e9 100755 --- a/postgres-appliance/scripts/postgres_backup.sh +++ b/postgres-appliance/scripts/postgres_backup.sh @@ -39,29 +39,31 @@ else POOL_SIZE=(--pool-size "$POOL_SIZE") fi -BEFORE="" -LEFT=0 - NOW=$(date +%s -u) readonly NOW -while read -r name last_modified rest; do - last_modified=$(date +%s -ud "$last_modified") - if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then - if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then - BEFORE_TIME=$last_modified - BEFORE=$name + +if [[ "$USE_WALG_BACKUP" == "true" ]]; then + AFTER=$((NOW-(DAYS_TO_RETAIN*86400))) + $WAL_E delete --use-sentinel-time --confirm retain FIND_FULL "$DAYS_TO_RETAIN" --after "$(date --iso-8601=seconds -d @"${AFTER}")" +else + BEFORE="" + LEFT=0 + + while read -r name last_modified rest; do + last_modified=$(date +%s -ud "$last_modified") + if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then + if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then + BEFORE_TIME=$last_modified + BEFORE=$name + fi + else + # count how many backups will remain after we remove everything up to certain date + ((LEFT=LEFT+1)) fi - else - # count how many backups will remain after we remove everything up to certain date - ((LEFT=LEFT+1)) - fi -done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d') + done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d') -# we want keep at least N backups even if the number of days exceeded -if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then - if [[ "$USE_WALG_BACKUP" == "true" ]]; then - $WAL_E delete before FIND_FULL "$BEFORE" --confirm - else + # we want keep at least N backups even if the number of days exceeded + if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then $WAL_E delete --confirm before "$BEFORE" fi fi diff --git a/postgres-appliance/scripts/wale_restore.sh b/postgres-appliance/scripts/wale_restore.sh index 138dfecae..d641efb91 100755 --- a/postgres-appliance/scripts/wale_restore.sh +++ b/postgres-appliance/scripts/wale_restore.sh @@ -43,7 +43,7 @@ ATTEMPT=0 server_version="-1" while true; do [[ -z $wal_segment_backup_start ]] && wal_segment_backup_start=$($WAL_E backup-list 2> /dev/null \ - | sed '0,/^name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//') + | sed '0,/^(backup_\)?name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//') [[ -n "$CONNSTR" && $server_version == "-1" ]] && server_version=$(psql -d "$CONNSTR" -tAc 'show server_version_num' 2> /dev/null || echo "-1")