Add additional function

.gitignore

@@ -7,3 +7,6 @@ __pycache__/
 /build/
 /*.egg-info
 /*.egg
+env/
+venv/
+pyznap.local.conf

CHANGELOG.md

@@ -5,6 +5,26 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [1.6.0+local-b2] - 2021-07-10
+### Added
+- Finely grained processing via ZFS user properties in the "pyznap:" domain
+  + pyznap:exclude=[true|false]
+    use pyznap:exclude to skip ZFS *send* processing of tagged datasets
+  + pyznap:max_size=[size <units>] units=B|KB|MB|TB|PB
+    use pyznap:max_size=500M to skip processing of tagged datasets
+
+
+Note that the default has changed to NO
+## [1.6.0+local-b1] - 2021-05-15
+### Added
+- Allow in-line comments in configuration file
+- Added command line `dry-run` and config `dry_run` options
+Use the `dry-run` option to see what changes would be made, but not make them
+- Added option `prune_sanoid`
+Use the `prune_sanoid` option to control if pyznap prunes existing sanoid snapshots
+Note that the default has changed to NO
+
+
 ## [1.6.0] - 2020-09-22
 ### Added
 - Added resumable send/receive.

README.md

@@ -56,7 +56,7 @@ Before you can use pyznap, you will need to create a config file. For initial se
     pyznap setup [-p PATH]
 
 This will create a directory `PATH` (default is `/etc/pyznap/`) and copy a sample config there. A
-config for your system might look like this (remove the comments):
+config for your system might look like this:
 
     [rpool/filesystem]
     frequent = 4                          # Keep 4 frequent snapshots
@@ -67,6 +67,8 @@ config for your system might look like this (remove the comments):
     yearly = 1                            # Keep 1 yearly snapshot
     snap = yes                            # Take snapshots on this filesystem
     clean = yes                           # Delete old snapshots on this filesystem
+    prune_sanoid = no                     # Don't delete old sanoid snapshots on this filesystem
+    dry_run = no                          # Run in normal mode, don't use dry_run
     dest = backup/filesystem              # Backup this filesystem on this location
     exclude = rpool/filesystem/data/*     # Exclude these datasets for pyznap send
 
@@ -76,20 +78,28 @@ Then set up a cronjob by creating a file under `/etc/cron.d/`
 
 and let pyznap run regularly by adding the following lines
 
-    SHELL=/bin/sh
+    SHELL=/bin/bash
     PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
 
     */15 * * * *   root    /path/to/pyznap snap >> /var/log/pyznap.log 2>&1
 
 This will run pyznap every quarter hour to take and delete snapshots. 'frequent' snapshots can be
 taken up to once per minute, so adjust your cronjob accordingly.
 
+For installation in a python virtual environment (venv) you can replace the line above with:
+
+    */15 * * * *   root   /path/to/pyznap/bin/python /path/to/pyznap/bin/pyznap snap >> /var/log/pyznap.log 2>&1
+
 If you also want to send your filesystems to another location you can add a line
 
     0 0 * * *   root    /path/to/pyznap send >> /var/log/pyznap.log 2>&1
 
 This will backup your data once per day at 12am.
 
+Or for a venv:
+
+    0 0 * * *   root    /path/to/pyznap/bin/python /path/to/pyznap/bin/pyznap send >> /var/log/pyznap.log 2>&1
+
 You can also manage, send to and pull from remote ssh locations. Always specify ssh locations with
 
     ssh:port:user@host:rpool/data
@@ -143,6 +153,8 @@ Here is a list of all options you can set in the config fie:
 | `yearly`           | Integer         | Number of yearly snapshots |
 | `snap`             | yes/no          | Should snapshots be taken |
 | `clean`            | yes/no          | Should snapshots be cleaned |
+| `prune_sanoid`     | yes/no          | Should sanoid snapshots be cleaned |
+| `dry_run`          | yes/no          | Display commands/changes, but don't update filesystem |
 | `dest`             | List of string  | Comma-separated list of destinations where to send source filesystem |
 | `dest_key`         | List of string  | Path to ssh keyfile for dest. Comma-separated list for multiple dest |
 | `compress`         | List of string  | Compression to use over ssh, supported are gzip, lzop, bzip2, pigz, xz & lz4. Default is lzop. Comma-separated list for multiple dest |
@@ -166,6 +178,10 @@ Run `pyznap -h` to see all available options.
 
   Print more verbose output.
 
++ -n, --dry-run
+
+  Use dry-run mode.
+
 + setup [-p PATH]
 
   Initial setup. Creates a config dir and puts a sample config file there. You can specify the path
@@ -211,6 +227,25 @@ Run `pyznap -h` to see all available options.
     but you can set the `--dest-auto-create` flag to automatically create it.
 
 
+#### ZFS User Properties ####
+pyznap now supports finely grained settings via ZFS user properties in the "pyznap:" domain. The following properties are in use:
+
++ pyznap:exclude [true|false]
+  Setting this property to true will caused the dateset to be excluded during *send*. Any other value, including the default, is assume to be "false"
+
+    + Example: Set the dataset to be excluded by pyznap send
+
+      `ZFS set pyznap:exclude=true tank/some_dataset`
+
++ pyznap:max_size [size]
+  Sets the maximum dataset size to be processed during this run. The following suffixes are supported:
+
+    `B, KB, MB, GB, TB, PB`
+
+    + Example: Set the maximum dataset processing size to 60GB
+
+      `ZFS set pyznap:max_size=60G tank/some_other_dataset`
+
 #### Usage examples ####
 
 + Take snapshots according to policy in default config file:
@@ -233,6 +268,10 @@ Run `pyznap -h` to see all available options.
 
     `pyznap send -s tank/data -d backup/data`
 
++ Backup a single filesystem locally, display ZFS commands, and run in dry-run (dummy) mode:
+
+    `pyznap --verbose --dry-run send -s tank/data -d backup/data`
+
 + Send a single filesystem to a remote location, using `pigz` compression:
 
     `pyznap send -s tank/data -d ssh:20022:[email protected]:backup/data -i /root/.ssh/id_rsa -c pigz`

pyznap/__init__.py

@@ -9,4 +9,4 @@
 """
 
 
-__version__ = '1.6.0'
+__version__ = 'v1.6.0+local-b2'

pyznap/clean.py

@@ -27,10 +27,11 @@ def clean_snap(snap):
     """
 
     logger = logging.getLogger(__name__)
-
-    logger.info('Deleting snapshot {}...'.format(snap))
+    dry_run = snap.dry_run == True
+    dry_msg = '*** DRY RUN ***' if dry_run else ''
+    logger.info('Deleting snapshot {}... {}'.format(snap, dry_msg))
     try:
-        snap.destroy()
+      snap.destroy(dry_run=dry_run)
     except DatasetBusyError as err:
         logger.error(err)
     except CalledProcessError as err:
@@ -54,7 +55,8 @@ def clean_filesystem(filesystem, conf):
     """
 
     logger = logging.getLogger(__name__)
-    logger.debug('Cleaning snapshots on {}...'.format(filesystem))
+    prunes = 'pyznap' if (conf.get('prune_sanoid', None) == False) else ('autosnap', 'pyznap')
+    logger.debug("Cleaning snapshots on {}... prunes={}".format(filesystem, prunes ) )
 
     snapshots = {'frequent': [], 'hourly': [], 'daily': [], 'weekly': [], 'monthly': [], 'yearly': []}
     # catch exception if dataset was destroyed since pyznap was started
@@ -65,11 +67,12 @@ def clean_filesystem(filesystem, conf):
         return 1
     # categorize snapshots
     for snap in fs_snapshots:
-        # Ignore snapshots not taken with pyznap or sanoid
-        if not snap.name.split('@')[1].startswith(('pyznap', 'autosnap')):
+        # Ignore snapshots not taken with pyznap or sanoid, depending on configuration
+        if not snap.name.split('@')[1].startswith(prunes):
             continue
         try:
             snap_type = snap.name.split('_')[-1]
+            snap.dry_run = conf.get('dry_run', None)
             snapshots[snap_type].append(snap)
         except (ValueError, KeyError):
             continue

pyznap/config/etc/cron.d/pyznap

@@ -0,0 +1,9 @@
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+# Take pyznap snapshots every 15 minutes
+*/15 * * * *  root  /root/pyznap/bin/python /root/pyznap/bin/pyznap snap >> /var/log/pyznap.log 2>&1
+
+# Copy to pyznap backup at midnight
+0 0  * * *    root  /root/pyznap/bin/python /root/pyznap/bin/pyznap send >> /var/log/pyznap.log 2>&1
+

pyznap/config/etc/logrotate.d/pyznap

@@ -0,0 +1,7 @@
+/var/log/pyznap.log {
+	daily
+	rotate 14
+	compress
+	missingok
+	notifempty
+}

pyznap/config/pyznap.conf → pyznap/config/etc/pyznap.conf

@@ -4,8 +4,7 @@
 ## filesystem. For remote syncronisation always keep enough snapshots on the destination. If there
 ## are no common snapshots the destination has to be destroyed and a full stream has to be sent.
 ## ssh locations are always specified with 'ssh:port:user@host:poolname/filesystem'.
-## Remove the comments at the end of the lines in your config, as they will not be ignored. Only
-## lines starting with '#' will be ignored.
+## Lines starting with '#' will be ignored.
 #
 #
 #
@@ -21,6 +20,8 @@
 # yearly = 1                            # Keep 1 yearly snapshot
 # snap = yes                            # Take snapshots on this filesystem
 # clean = yes                           # Delete old snapshots on this filesystem
+# prune_sanoid = no                     # Prune sanoid snapshots yes|no
+# dry_run = yes                         # Use dry-run mode? yes|no
 # dest = backup/filesystem              # Backup this filesystem on this location
 # exclude = rpool/filesystem/data/*     # Exclude these datasets for pyznap send
 #

pyznap/main.py

@@ -19,6 +19,7 @@
 from .clean import clean_config
 from .take import take_config
 from .send import send_config
+from pyznap import __version__
 
 
 DIRNAME = os.path.dirname(os.path.abspath(__file__))
@@ -33,7 +34,10 @@ def _main():
         Exit code
     """
 
-    parser = ArgumentParser(prog='pyznap', description='ZFS snapshot tool written in python')
+    parser = ArgumentParser(prog='pyznap', 
+                            description='ZFS snapshot tool written in python {}'.format(__version__))
+    parser.add_argument('-n', '--dry-run', action="store_true",
+                        dest="dry_run", help="Dry-run, don't execute commands")
     parser.add_argument('-v', '--verbose', action="store_true",
                         dest="verbose", help='print more verbose output')
     parser.add_argument('--config', action="store",
@@ -80,6 +84,8 @@ def _main():
     parser_send.add_argument('--retry-interval', action="store", type=int,
                              dest='retry_interval', default=10,
                              help='interval in seconds between retries. default is 10')
+
+    parser.epilog = "ZFS properties: [pyznap:exclude, pyznap:max_size]"
 
     if len(sys.argv)==1:
         parser.print_help(sys.stderr)
@@ -91,14 +97,21 @@ def _main():
                         datefmt='%b %d %H:%M:%S', stream=sys.stdout)
     logger = logging.getLogger(__name__)
 
-    logger.info('Starting pyznap...')
+    logger.info('Starting pyznap {}...'.format(__version__))
 
     if args.command in ('snap', 'send'):
         config_path = args.config if args.config else os.path.join(CONFIG_DIR, 'pyznap.conf')
         config = read_config(config_path)
         if config == None:
             return 1
 
+    # Append global dry_run flag don't override existing dry-run = yes
+    try:
+      for conf in config: 
+        conf['dry_run'] = True if (args.dry_run or conf.get('dry_run', None)) else False
+    except UnboundLocalError:
+      pass
+
     if args.command == 'setup':
         path = args.path if args.path else CONFIG_DIR
         create_config(path)
@@ -143,7 +156,8 @@ def _main():
             send_config([{'name': args.source, 'dest': [args.dest], 'key': source_key,
                           'dest_keys': dest_key, 'compress': compress, 'exclude': exclude,
                           'raw_send': raw, 'resume': resume, 'dest_auto_create': dest_auto_create,
-                          'retries': retries, 'retry_interval': retry_interval}])
+                          'retries': retries, 'retry_interval': retry_interval,
+                          'dry_run': dry_run}])
 
         elif args.source and not args.dest:
             logger.error('Missing dest...')

pyznap/process.py

@@ -10,6 +10,7 @@
 
 import re
 import errno as _errno
+import logging
 import subprocess as sp
 import socket
 
@@ -93,13 +94,15 @@ def check_output(*popenargs, timeout=None, ssh=None, **kwargs):
         List of all lines from the output, seperated at '\t' into lists
     """
 
+    logger = logging.getLogger(__name__)
     if 'stdout' in kwargs:
         raise ValueError('stdout argument not allowed, it will be overridden.')
     if 'universal_newlines' in kwargs:
         raise ValueError('universal_newlines argument not allowed, it will be overridden.')
     if 'input' in kwargs:
         raise ValueError('input argument not allowed, it will be overridden.')
 
+    logger.debug("'{}'...".format(' '.join(*popenargs)))
     ret = run(*popenargs, stdout=PIPE, stderr=PIPE, timeout=timeout,
               universal_newlines=True, ssh=ssh, **kwargs)
     ret.check_returncode()

pyznap/pyzfs.py

@@ -119,7 +119,7 @@ def roots(ssh=None):
     return find(ssh=ssh, max_depth=0)
 
 # note: force means create missing parent filesystems
-def create(name, ssh=None, type='filesystem', props={}, force=False):
+def create(name, ssh=None, type='filesystem', props={}, force=False, dry_run=False):
     cmd = ['zfs', 'create']
 
     if type == 'volume':
@@ -130,6 +130,9 @@ def create(name, ssh=None, type='filesystem', props={}, force=False):
     if force:
         cmd.append('-p')
 
+    if dry_run:
+        cmd.append('-n')
+
     for prop, value in props.items():
         cmd.append('-o')
         cmd.append(prop + '=' + str(value))
@@ -142,7 +145,7 @@ def create(name, ssh=None, type='filesystem', props={}, force=False):
 
 
 def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_path=False,
-            force=False, nomount=False, stream_size=0, raw=False, resume=False):
+            force=False, nomount=False, stream_size=0, raw=False, resume=False, dry_run=False):
     """Returns Popen instance for zfs receive"""
     logger = logging.getLogger(__name__)
 
@@ -181,6 +184,8 @@ def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_pa
         cmd.append('-u')
     if resume:
         cmd.append('-s')
+    if dry_run:
+        cmd.append('-n')
 
     cmd.append(quote(name)) # use shlex to quote the name
 
@@ -196,6 +201,7 @@ def receive(name, stdin, ssh=None, ssh_source=None, append_name=False, append_pa
     # execute command with shell (sh or ssh)
     cmd = shell + [' '.join(cmd)]
 
+    logger.debug("'{}'...".format(' '.join(cmd)))
     return sp.Popen(cmd, stdin=stdin, stderr=sp.PIPE) # zfs receive process
 
 
@@ -232,7 +238,7 @@ def dependents(self):
 
     # TODO: split force to allow -f, -r and -R to be specified individually
     # TODO: remove or ignore defer option for non-snapshot datasets
-    def destroy(self, defer=False, force=False):
+    def destroy(self, defer=False, force=False, dry_run=False):
         cmd = ['zfs', 'destroy']
 
         cmd.append('-v')
@@ -244,6 +250,9 @@ def destroy(self, defer=False, force=False):
             cmd.append('-f')
             cmd.append('-R')
 
+        if dry_run:
+            cmd.append('-n')
+
         cmd.append(self.name)
 
         check_output(cmd, ssh=self.ssh)