#!/usr/bin/python
"""
Manage rsync --link-dest based snapshots.
rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current.
Updates symlink <dst>/current -> <dst>/snapshots/...
Then archives <dst>/current to <dst>/<period>/<date> using --link-dest.
"""
from pvl.backup import rsync
import optparse, ConfigParser
import os, os.path, stat
import shutil
import datetime
import logging
log = logging.getLogger()
# command-line options
options = None
def parse_options (argv, defaults) :
"""
Parse command-line arguments.
"""
parser = optparse.OptionParser(
prog = argv[0],
usage = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]',
# module docstring
# XXX: breaks multi-line descriptions..
description = __doc__,
)
# logging
general = optparse.OptionGroup(parser, "General Options")
general.add_option('-q', '--quiet', dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")
general.add_option('-v', '--verbose', dest='loglevel', action='store_const', const=logging.INFO, help="More output")
general.add_option('-D', '--debug', dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")
parser.add_option_group(general)
# rsync
rsync = optparse.OptionGroup(parser, "rsync Options")
rsync.add_option('--exclude-from', metavar='FILE',
help="Read exclude rules from given file")
rsync.add_option('--include-from', metavar='FILE',
help="Read include rules from given file")
parser.add_option_group(rsync)
# global
parser.add_option('--clean-intervals', action='store_true',
help="Clean out old interval links")
parser.add_option('--clean-snapshots', action='store_true',
help="Clean out unused snapshots (those not linked to)")
parser.add_option('--clean', action='store_true',
help="Clean out both intervals and snapshots")
parser.add_option('-n', '--dry-run', action='store_true',
help="Don't actually clean anything")
#
parser.add_option('-c', '--config', metavar='FILE',
help="Load configuration file")
#
parser.add_option('-T', '--target', metavar='PATH',
help="Target path")
parser.add_option('-s', '--source', metavar='RSYNC-PATH', dest='target_source', default=False,
help="Run target backup from source in rsync-syntax")
parser.add_option('--interval', metavar='NAME', action='append', dest='target_intervals',
help="Run target with given given interval(s)")
# defaults
parser.set_defaults(
loglevel = logging.INFO,
target_intervals = [],
)
parser.set_defaults(**defaults)
# parse
options, args = parser.parse_args(argv[1:])
# configure
logging.basicConfig(
format = '%(processName)s: %(name)s: %(levelname)s %(funcName)s : %(message)s',
level = options.loglevel,
)
if options.clean :
options.clean_intervals = options.clean_snapshots = options.clean
if options.include_from :
options.rsync_options['include-from'] = options.include_from
if options.exclude_from :
options.rsync_options['exclude-from'] = options.exclude_from
return options, args
## Configuration
class ConfigError (Exception) :
pass
def process_config_name (name) :
"""
Process config file name into python version
"""
return name.replace('-', '_')
def parse_config (path, defaults) :
"""
Parse given config file
"""
log.debug("loading config: %s", path)
config = dict(defaults)
config_file = ConfigParser.RawConfigParser()
config_file.read([path])
# handle each section
for section in config_file.sections() :
# mangle
section_name = process_config_name(section)
log.debug("section: %s", section_name)
# subsections
section_path = section_name.split(':')
# lookup section dict from config
lookup = config
# XXX: sections are not in order, so we can't rely on the parent section being created before we handle the sub-section
for name in section_path :
# possibly create
if name not in lookup :
lookup[name] = {}
lookup = lookup[name]
# found dict for this section
config_section = lookup
# values
for name, value in config_file.items(section) :
# mangle
name = process_config_name(name)
log.debug("section: %s: %s = %s", '/'.join(section_path), name, value)
config_section[name] = value
log.debug("config: %s", config)
return config
def config_bool (name, value, strict=True) :
if value.lower() in ('yes', 'true', '1', 'on') :
return True
elif value.lower() in ('no', 'false', '0', 'off') :
return False
elif strict :
raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value))
else :
# allow non-boolean values
return value
def config_int (name, value) :
try :
return int(value)
except ValueError, e:
raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value))
def config_list (name, value) :
return value.split()
def walk_symlinks (tree, ignore=False) :
"""
Walk through all symlinks in given dir, yielding:
(dirpath, name, target)
Passes through errors from os.listdir/os.lstat.
"""
for name in os.listdir(tree) :
if ignore and name in ignore :
log.debug("%s: ignore: %s", tree, name)
continue
path = os.path.join(tree, name)
# stat symlink itself
st = os.lstat(path)
if stat.S_ISDIR(st.st_mode) :
# recurse
log.debug("%s: tree: %s", tree, name)
for item in walk_symlinks(path) :
yield item
elif stat.S_ISLNK(st.st_mode) :
# found
target = os.readlink(path)
log.debug("%s: link: %s -> %s", tree, name, target)
yield tree, name, target
else :
log.debug("%s: skip: %s", tree, name)
class Interval (object) :
"""
An interval definition.
"""
@classmethod
def from_config (cls, options, name,
format,
keep,
) :
if not format :
# magic to use snapshot name
_format = None
else :
_format = format
return cls(name,
format = _format,
keep = config_int('keep', keep),
)
@classmethod
def from_target_config (cls, name, base, arg) :
if isinstance(arg, dict) :
# full instance
return cls(name,
format = arg.get('format', base.format if base else None),
keep = arg.get('keep', base.keep if base else None),
)
else :
# partial instance with keep
return cls(name,
format = base.format,
keep = config_int('keep', arg) if arg else base.keep,
)
def __init__ (self, name, format, keep) :
self.name = name
self.format = format
self.keep = keep
def __str__ (self) :
return self.name
class Target (object) :
"""
A target run, i.e. a rsync-snapshot destination dir
[target:...]
"""
@classmethod
def from_config (cls, options, name,
path = False,
source = None,
enable = 'no',
exclude_from = None,
# subsections
intervals = None,
rsync_options = None,
) :
if not source and source is not False :
raise ConfigError("Missing required option: source for [target:{name}]".format(name=name))
# global defaults
_rsync_options = dict(options.rsync_options)
if rsync_options :
# override
_rsync_options.update([
# parse
(option, config_bool(option, value, strict=False)) for option, value in rsync_options.iteritems()
])
# lookup intervals
_intervals = [
(
# lookup base from options.intervals
Interval.from_target_config(name, options.intervals.get(name), arg)
) for name, arg in intervals.iteritems()
]
return cls(name,
path = path if path else name,
source = source,
enable = config_bool('enable', enable),
intervals = _intervals,
rsync_options = _rsync_options,
exclude_from = exclude_from,
)
def __init__ (self, name,
path,
source,
enable = False,
intervals = [],
rsync_options = {},
exclude_from = None
) :
self.name = name
self.path = path
self.source = source
self.enable = enable
self.intervals = intervals
self.rsync_options = rsync_options
self.exclude_from = exclude_from
# this snapshot?
self.snapshots_dir = os.path.join(self.path, 'snapshots')
# 'current' symlink
self.current_path = os.path.join(self.path, 'current')
def prepare (self, options) :
"""
Prepare dir for usage
"""
if not os.path.exists(self.path) :
raise Exception("Missing target dir: {path}".format(path=self.path))
if not os.path.exists(self.snapshots_dir) :
log.warn("Creating snapshots dir: %s", self.snapshots_dir)
os.mkdir(self.snapshots_dir)
def snapshot (self, options, now) :
"""
Perform the rsync from our source to self.snapshot_dir.
XXX: allocate snapshot_name here?
"""
# new snapshot
snapshot_name = now.strftime(options.snapshot_format)
snapshot_path = os.path.join(self.snapshots_dir, snapshot_name)
temp_path = os.path.join(self.snapshots_dir, 'tmp')
if os.path.exists(temp_path) :
raise Exception("Old temp snapshot dir remains, please clean up: {path}".format(path=temp_path))
log.info("Perform main snapshot: %s -> %s", self.source, snapshot_path)
# build rsync options
opts = dict(self.rsync_options)
if os.path.exists(self.current_path) :
# real path to target
target = os.readlink(self.current_path)
target_path = os.path.join(os.path.dirname(self.current_path), target)
target_abs = os.path.abspath(target_path)
log.info("Using current -> %s as base", target_path)
# use as link-dest base; hardlinks unchanged files; target directory must be empty
# rsync links absolute paths..
opts['link-dest'] = target_abs
# go
log.debug("rsync %s -> %s", self.source, temp_path)
rsync.rsync(self.source, temp_path, **opts)
# move in to final name
log.debug("rename %s -> %s", temp_path, snapshot_path)
os.rename(temp_path, snapshot_path)
return snapshot_name
def update_interval (self, options, interval, now, snapshot_name) :
"""
Update given <interval>/... links for this target, using the given new snapshot
"""
dir_path = os.path.join(self.path, interval.name)
if not os.path.exists(dir_path) :
log.warn("Creating interval dir: %s", dir_path)
os.mkdir(dir_path)
# name
if interval.format is None :
# per-snapshot
name = snapshot_name
log.debug("%s: using snapshot_name: %s", interval, name)
else :
# by date
name = now.strftime(interval.format)
log.debug("%s: using interval.format: %s -> %s", interval, interval.format, name)
# path
path_name = os.path.join(interval.name, name)
path = os.path.join(self.path, path_name)
log.debug("%s: processing %s", interval, path_name)
# already there?
if os.path.exists(path) :
target = os.readlink(path)
log.info("%s: Found existing: %s -> %s", interval, name, target)
else :
# update
target = os.path.join('..', 'snapshots', snapshot_name)
log.info("%s: Updating: %s -> %s", interval, name, target)
log.debug("%s -> %s", path, target)
os.symlink(target, path)
def clean_interval (self, options, interval) :
"""
Clean out given <interval>/... dir for this target.
"""
# path
dir_path = os.path.join(self.path, interval.name)
if not os.path.exists(dir_path) :
log.warn("%s: Skipping, no interval dir: %s", interval, dir_path)
return
# configured
keep = interval.keep
# clean?
items = os.listdir(dir_path)
items.sort()
log.info("%s: Have %d / %d items", interval, len(items), keep)
log.debug("%s: items: %s", interval, ' '.join(items))
if len(items) > keep :
# clean out
clean = items[keep:]
log.info("%s: Cleaning out %d items", interval, len(clean))
log.debug("%s: cleaning out: %s", interval, ' '.join(clean))
for item in clean :
path = os.path.join(dir_path, item)
log.info("%s: Clean: %s", interval, path)
os.unlink(path)
def clean_snapshots (self, options) :
"""
Clean out all snapshots for this target not linked to from within our root.
Fails without doing anything if unable to read the destination dir.
"""
# real path to snapshots
snapshots_path = os.path.realpath(os.path.abspath(self.snapshots_dir))
log.debug("real snapshots_path: %s", snapshots_path)
# set of found targets
found = set()
# walk all symlinks
for dirpath, name, target in walk_symlinks(self.path, ignore=set(['snapshots'])) :
# target dir
target_path = os.path.realpath(os.path.join(dirpath, target))
target_dir = os.path.dirname(target_path)
target_name = os.path.basename(target_path)
if target_dir == snapshots_path :
log.debug("%s: found: %s -> %s", dirpath, name, target_name)
found.add(target_name)
else :
log.debug("%s: ignore: %s -> %s", dirpath, name, target_path)
# discover all snapshots
snapshots = set(os.listdir(snapshots_path))
# clean out special names
snapshots = snapshots - set(['new'])
## compare
used = snapshots & found
unused = snapshots - found
broken = found - snapshots
log.info("Found used=%d, unused=%d, broken=%d snapshot symlinks", len(used), len(unused), len(broken))
log.debug("used=%s, unused=%s", used, unused)
if broken :
log.warn("Found broken symlinks to snapshots: %s", ' '.join(broken))
if unused :
log.info("Clean out unused snapshots: %s", ' '.join(unused))
for name in unused :
path = os.path.join(snapshots_path, name)
log.info("Clean: %s", name)
if not options.dry_run :
log.debug("rmtree: %s", path)
# nuke
shutil.rmtree(path)
else :
log.debug("dry-run: %s", path)
def run_snapshot (self, options, now) :
"""
Run snapshot + update current.
"""
# initial rsync
snapshot_name = self.snapshot(options, now)
# update current
log.info("Updating current -> %s", snapshot_name)
if os.path.islink(self.current_path) :
# replace
os.unlink(self.current_path)
os.symlink(os.path.join('snapshots', snapshot_name), self.current_path)
return snapshot_name
def run_intervals (self, options, now, snapshot_name) :
"""
Run our intervals.
"""
if not self.intervals :
log.info("No intervals given; not running any")
else :
# maintain intervals
log.info("Updating %d intervals...", len(self.intervals))
for interval in self.intervals :
log.debug("%s", interval)
log.info("Updating interval: %s", interval)
# update
self.update_interval(options, interval, now, snapshot_name)
def run (self, options) :
"""
Execute
"""
# prep
self.prepare(options)
# clean intervals?
if options.clean_intervals:
for interval in self.intervals :
log.info("Cleaning interval: %s...", interval)
self.clean_interval(options, interval)
# clean snapshots?
if options.clean_snapshots :
log.info("Cleaning snapshots...")
self.clean_snapshots(options)
# snapshot from source?
if self.source :
# timestamp for run
now = datetime.datetime.now()
log.info("Started snapshot run at: %s", now)
# snapshot + current
snapshot_name = self.run_snapshot(options, now)
# intervals?
self.run_intervals(options, now, snapshot_name)
# ok
return 1
def __str__ (self) :
return self.name
def run (options, targets) :
# default config
config = dict(
rsync_options = {},
intervals = {},
targets = {},
)
if options.config :
# load
try :
config = parse_config(options.config, config)
except ConfigError as e:
log.error("Configuration error: %s: %s", options.config, e)
return 2
# manual?
if options.target :
config['targets'][options.target] = dict(
path = options.target,
source = options.target_source,
intervals = dict((name, None) for name in options.target_intervals),
)
# intervals
for name in config['intervals'] :
interval_config = config['intervals'][name]
# parse
interval = Interval.from_config(options, name, **interval_config)
log.debug("config interval: %s", name)
# store
options.intervals[name] = interval
for option in config['rsync_options'] :
value = config['rsync_options'][option]
# parse, allowing non-boolean values as well...
value = config_bool(option, value, strict=False)
log.debug("rsync option: %s=%s", option, value)
# store
options.rsync_options[option] = value
# what targets?
if not targets :
# default to all defined targets
targets = list(config['targets'])
else :
# given ones, but verify they exist
for target in targets :
if target not in config['targets'] :
log.error("Unknown target given: %s", target)
log.info("Defined targets: %s", ' '.join(config['targets']))
return 2
# targets
for name in targets :
target_config = config['targets'][name]
# parse
target = Target.from_config(options, name, **target_config)
log.info("Config target: %s", name)
# run
target.run(options)
# ok
return 0
def config_defaults () :
return dict(
# snapshots/ naming
snapshot_format = '%Y%m%d-%H%M%S',
# rsync options, in invoke.optargs format
rsync_options = {
'archive': True,
'hard-links': True,
'one-file-system': True,
'numeric-ids': True,
'delete': True,
},
# defined intervals
intervals = dict((i.name, i) for i in [
Interval('recent',
format = None,
keep = 4,
),
Interval('day',
format = '%Y-%m-%d',
keep = 7,
),
Interval('week',
format = '%Y-%W',
keep = 4,
),
Interval('month',
format = '%Y-%m',
keep = 4,
),
Interval('year',
format = '%Y',
keep = 1,
)
]),
)
def main (argv) :
global options
# option defaults
defaults = config_defaults()
# global options + args
options, args = parse_options(argv, defaults)
# args: filter targets
# XXX: fix name mangling
targets = [target.replace('-', '_') for target in args]
try :
# handle it
return run(options, targets)
except Exception, e:
log.error("Internal error:", exc_info=e)
return 3
# ok
return 0
if __name__ == '__main__' :
import sys
sys.exit(main(sys.argv))