Choose WAL-G backup by start time when cloning and deleting

postgres-appliance/Dockerfile

@@ -19,7 +19,7 @@ FROM $BASE_IMAGE as dependencies-builder
 
 ARG DEMO
 
-ENV WALG_VERSION=v2.0.1
+ENV WALG_VERSION=v3.0.3
 
 COPY build_scripts/dependencies.sh /builddeps/
 

postgres-appliance/bootstrap/clone_with_wale.py

@@ -2,6 +2,7 @@
 
 import argparse
 import csv
+import json
 import logging
 import os
 import re
@@ -40,14 +41,16 @@ def read_configuration():
     return options(args.scope, args.datadir, recovery_target_time, args.dry_run)
 
 
-def build_wale_command(command, datadir=None, backup=None):
+def build_wale_command(command, datadir=None, backup=None, extra_args=None):
     cmd = ['wal-g' if os.getenv('USE_WALG_RESTORE') == 'true' else 'wal-e'] + [command]
     if command == 'backup-fetch':
         if datadir is None or backup is None:
             raise Exception("backup-fetch requires datadir and backup arguments")
         cmd.extend([datadir, backup])
     elif command != 'backup-list':
         raise Exception("invalid {0} command {1}".format(cmd[0], command))
+    if extra_args:
+        cmd.extend(extra_args)
     return cmd
 
 
@@ -57,8 +60,7 @@ def fix_output(output):
     started = None
     for line in output.decode('utf-8').splitlines():
         if not started:
-            started = re.match(r'^name\s+last_modified\s+', line)
-            started = re.match(r'^name\s+last_modified\s+', line) or re.match(r'^name\s+modified\s+', line)
+            started = re.match(r'^(backup_)?name\s+(last_)?modified\s+', line)
             if started:
                 line = line.replace(' modified ', ' last_modified ')
         if started:
@@ -70,20 +72,25 @@ def choose_backup(backup_list, recovery_target_time):
 
     match_timestamp = match = None
     for backup in backup_list:
-        last_modified = parse(backup['last_modified'])
+        last_modified = parse(backup['finish_time' if os.getenv('USE_WALG_RESTORE') == 'true' else 'last_modified'])
         if last_modified < recovery_target_time:
             if match is None or last_modified > match_timestamp:
                 match = backup
                 match_timestamp = last_modified
     if match is not None:
-        return match['name']
+        return match.get('name', match['backup_name'])
 
 
 def list_backups(env):
-    backup_list_cmd = build_wale_command('backup-list')
-    output = subprocess.check_output(backup_list_cmd, env=env)
-    reader = csv.DictReader(fix_output(output), dialect='excel-tab')
-    return list(reader)
+    if os.getenv('USE_WALG_RESTORE') == 'true':
+        backup_list_cmd = build_wale_command('backup-list', extra_args=['--detail', '--json'])
+        output = subprocess.check_output(backup_list_cmd, env=env)
+        return json.loads(output or 'null')
+    else:
+        backup_list_cmd = build_wale_command('backup-list')
+        output = subprocess.check_output(backup_list_cmd, env=env)
+        reader = csv.DictReader(fix_output(output), dialect='excel-tab')
+        return list(reader)
 
 
 def get_clone_envdir():

postgres-appliance/scripts/postgres_backup.sh

@@ -39,29 +39,31 @@ else
     POOL_SIZE=(--pool-size "$POOL_SIZE")
 fi
 
-BEFORE=""
-LEFT=0
-
 NOW=$(date +%s -u)
 readonly NOW
-while read -r name last_modified rest; do
-    last_modified=$(date +%s -ud "$last_modified")
-    if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then
-        if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then
-            BEFORE_TIME=$last_modified
-            BEFORE=$name
+
+if [[ "$USE_WALG_BACKUP" == "true" ]]; then
+    AFTER=$((NOW-(DAYS_TO_RETAIN*86400)))
+    $WAL_E delete --use-sentinel-time --confirm retain FIND_FULL "$DAYS_TO_RETAIN" --after "$(date --iso-8601=seconds -d @"${AFTER}")"
+else
+    BEFORE=""
+    LEFT=0
+
+    while read -r name last_modified rest; do
+        last_modified=$(date +%s -ud "$last_modified")
+        if [ $(((NOW-last_modified)/86400)) -ge $DAYS_TO_RETAIN ]; then
+            if [ -z "$BEFORE" ] || [ "$last_modified" -gt "$BEFORE_TIME" ]; then
+                BEFORE_TIME=$last_modified
+                BEFORE=$name
+            fi
+        else
+            # count how many backups will remain after we remove everything up to certain date
+            ((LEFT=LEFT+1))
         fi
-    else
-        # count how many backups will remain after we remove everything up to certain date
-        ((LEFT=LEFT+1))
-    fi
-done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d')
+    done < <($WAL_E backup-list 2> /dev/null | sed '0,/^name\s*\(last_\)\?modified\s*/d')
 
-# we want keep at least N backups even if the number of days exceeded
-if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then
-    if [[ "$USE_WALG_BACKUP" == "true" ]]; then
-        $WAL_E delete before FIND_FULL "$BEFORE" --confirm
-    else
+    # we want keep at least N backups even if the number of days exceeded
+    if [ -n "$BEFORE" ] && [ $LEFT -ge $DAYS_TO_RETAIN ]; then
         $WAL_E delete --confirm before "$BEFORE"
     fi
 fi

postgres-appliance/scripts/wale_restore.sh

@@ -43,7 +43,7 @@ ATTEMPT=0
 server_version="-1"
 while true; do
     [[ -z $wal_segment_backup_start ]] && wal_segment_backup_start=$($WAL_E backup-list 2> /dev/null \
-        | sed '0,/^name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//')
+        | sed '0,/^(backup_\)?name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//')
 
     [[ -n "$CONNSTR" && $server_version == "-1" ]] && server_version=$(psql -d "$CONNSTR" -tAc 'show server_version_num' 2> /dev/null || echo "-1")