pvl.backup-rsync: implement local rsync command execution, rename from pvlbackup-rsync-wrapper
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/pvl.backup-rsync Sun Apr 22 12:56:53 2012 +0300
@@ -0,0 +1,195 @@
+#!/usr/bin/python
+
+"""
+ SSH authorized_keys command="..." wrapper for rsync.
+
+ Testing goes something like:
+ sudo PYTHONPATH=. ./bin/pvlbackup-rsync-wrapper --command 'rsync --server --sender -ax . lvm:asdf:test' -vD
+
+ sudo sh -c "PYTHONPATH=. rsync -e './bin/pvlbackup-rsync-wrapper --debug -C --' -ax testing:lvm:asdf:test test/tmp"
+"""
+
+from pvl.backup import __version__
+from pvl.backup.rsync import RSyncCommandFormatError
+from pvl.backup.invoke import InvokeError
+from pvl.backup import rsync, lvm
+
+import optparse
+import shlex
+import os
+import logging
+
+log = logging.getLogger()
+
+def parse_options (argv) :
+ """
+ Parse command-line arguments.
+ """
+
+# import sys; sys.stderr.write("%s\n" % (argv, ))
+
+ parser = optparse.OptionParser(
+ prog = argv[0],
+
+ # module docstring
+ description = __doc__,
+ version = __version__,
+ )
+
+ # logging
+ general = optparse.OptionGroup(parser, "General Options")
+
+ general.add_option('-q', '--quiet', dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")
+ general.add_option('-v', '--verbose', dest='loglevel', action='store_const', const=logging.INFO, help="More output")
+ general.add_option('-D', '--debug', dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")
+ general.add_option('--debug-for', action='append', metavar='MODULE', help="Enable logging for the given logger/module name")
+
+ parser.add_option_group(general)
+
+ #
+ parser.add_option('-c', '--command', metavar='CMD', default=os.environ.get('SSH_ORIGINAL_COMMAND'),
+ help="rsync command to execute")
+
+ parser.add_option('-C', '--given-command', action='store_true', default=False,
+ help="use given command in `rsync -e %prog` format")
+
+ parser.add_option('-n', '--noop', action='store_true', default=False,
+ help="Parse command, but do not execute")
+
+ parser.add_option('-R', '--readonly', action='store_true', default=False,
+ help="restrict to read/source mode")
+
+ parser.add_option('-P', '--restrict-path', metavar='PATH', default=False,
+ help="restrict to given path prefix")
+
+ # lvm options
+ parser.add_option('-L', '--snapshot-size', metavar='SIZE', default=lvm.LVM_SNAPSHOT_SIZE,
+ help="create snapshot with given LV size (used to store writes during backup)")
+
+ parser.add_option('--snapshot-wait', metavar='SECONDS', default=lvm.LVM_SNAPSHOT_WAIT, type='float',
+ help="wait for snapshot to settle after unmounting")
+
+ # defaults
+ parser.set_defaults(
+ debug_for = [],
+ loglevel = logging.INFO,
+ )
+
+ # parse
+ options, args = parser.parse_args(argv[1:])
+
+ # configure
+ logging.basicConfig(
+ format = '%(levelname)6s %(name)s : %(funcName)s : %(message)s',
+ level = options.loglevel,
+ )
+
+ # enable debugging for specific targets
+ for target in options.debug_for :
+ logging.getLogger(target).setLevel(logging.DEBUG)
+
+ return options, args
+
+def rsync_wrapper (command, options, local=False) :
+ """
+ Wrap given rsync command.
+
+ Parses the command, the source path, and then executes rsync within the source path (which may be a special
+ pseudo-path with additional handling).
+
+ local - the given command is a full local command, not an --server mode operation
+ """
+
+ # parse the rsync command sent by the client
+ try :
+ # path = the real source path
+ rsync_cmd, rsync_options, path, srcdst = rsync.parse_command(command,
+ restrict_server = not local,
+ restrict_readonly = options.readonly,
+ )
+
+ except RSyncCommandFormatError, e:
+ log.error("invalid rsync command: %r: %s", command, e)
+ return 2
+
+
+ # parse source
+ try :
+ # parse the source path as given by the client, may be a real path or pseudo-path
+ source = rsync.parse_source(path,
+ restrict_path = options.restrict_path,
+ lvm_opts = dict(size = options.snapshot_size, wait = options.snapshot_wait),
+ )
+
+ except RSyncCommandFormatError, e:
+ log.error("invalid rsync source: %r: %s", path, e)
+ return 2
+
+ # noop?
+ if options.noop :
+ log.info("noop: %r -> %r: execute(%r, %r)", path, source, rsync_options, srcdst)
+ return 0
+
+ # execute
+ try :
+ # run rsync within the source (may perform additional stuff like snapshotting...)
+ source.execute(rsync_options, srcdst)
+
+ except InvokeError, e:
+ log.error("%s failed: %d", e.cmd, e.exit)
+ return e.exit
+
+
+ # ok
+ return 0
+
+def main (argv) :
+ """
+ Run, with full argv
+ """
+
+ # global options + args
+ options, args = parse_options(argv)
+
+ # was a local command, not remote?
+ local = False
+
+ # from args (as given by `rsync -e pvlbackup-rsync-wrapper`) -> 'pvlbackup-rsync-wrapper <host> (<command> ...)'
+ if options.given_command :
+ host = args.pop(0)
+ command_parts = args
+
+ log.debug("host=%r, using command from args: %r", host, command_parts)
+
+ # from env/--command
+ elif options.command:
+ # as given
+ command_parts = shlex.split(options.command)
+
+ # normal args
+ elif args :
+ command_parts = [argv[0]] + args
+ local = True
+
+ log.debug("using given rsync args: %r", command_parts)
+
+ else :
+ log.error("SSH_ORIGINAL_COMMAND not given")
+ return 2
+
+ # run
+ try :
+ return rsync_wrapper(command_parts, options, local)
+
+ except Exception, e:
+ log.error("Internal error:", exc_info=e)
+ return 3
+
+ # ok
+ return 0
+
+if __name__ == '__main__' :
+ import sys
+
+ sys.exit(main(sys.argv))
+
--- a/bin/pvlbackup-rsync-wrapper Sun Apr 22 12:31:26 2012 +0300
+++ /dev/null Thu Jan 01 00:00:00 1970 +0000
@@ -1,179 +0,0 @@
-#!/usr/bin/python
-
-"""
- SSH authorized_keys command="..." wrapper for rsync.
-
- Testing goes something like:
- sudo PYTHONPATH=. ./bin/pvlbackup-rsync-wrapper --command 'rsync --server --sender -ax . lvm:asdf:test' -vD
-
- sudo sh -c "PYTHONPATH=. rsync -e './bin/pvlbackup-rsync-wrapper --debug -C --' -ax testing:lvm:asdf:test test/tmp"
-"""
-
-from pvl.backup import __version__
-from pvl.backup.rsync import RSyncCommandFormatError
-from pvl.backup.invoke import InvokeError
-from pvl.backup import rsync, lvm
-
-import optparse
-import shlex
-import os
-import logging
-
-log = logging.getLogger()
-
-# command-line options
-options = None
-
-def parse_options (argv) :
- """
- Parse command-line arguments.
- """
-
-# import sys; sys.stderr.write("%s\n" % (argv, ))
-
- parser = optparse.OptionParser(
- prog = argv[0],
-
- # module docstring
- description = __doc__,
- version = __version__,
- )
-
- # logging
- general = optparse.OptionGroup(parser, "General Options")
-
- general.add_option('-q', '--quiet', dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")
- general.add_option('-v', '--verbose', dest='loglevel', action='store_const', const=logging.INFO, help="More output")
- general.add_option('-D', '--debug', dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")
- general.add_option('--debug-for', action='append', metavar='MODULE', help="Enable logging for the given logger/module name")
-
- parser.add_option_group(general)
-
- #
- parser.add_option('-c', '--command', metavar='CMD', default=os.environ.get('SSH_ORIGINAL_COMMAND'),
- help="rsync command to execute")
-
- parser.add_option('-C', '--given-command', action='store_true', default=False,
- help="use given command in `rsync -e %prog` format")
-
- parser.add_option('-R', '--readonly', action='store_true', default=False,
- help="restrict to read/source mode")
-
- parser.add_option('-P', '--restrict-path', metavar='PATH', default=False,
- help="restrict to given path prefix")
-
- # lvm options
- parser.add_option('-L', '--snapshot-size', metavar='SIZE', default=lvm.LVM_SNAPSHOT_SIZE,
- help="create snapshot with given LV size (used to store writes during backup)")
-
- parser.add_option('--snapshot-wait', metavar='SECONDS', default=lvm.LVM_SNAPSHOT_WAIT, type='float',
- help="wait for snapshot to settle after unmounting")
-
- # defaults
- parser.set_defaults(
- debug_for = [],
- loglevel = logging.INFO,
- )
-
- # parse
- options, args = parser.parse_args(argv[1:])
-
- # configure
- logging.basicConfig(
- format = '%(levelname)6s %(name)s : %(funcName)s : %(message)s',
- level = options.loglevel,
- )
-
- # enable debugging for specific targets
- for target in options.debug_for :
- logging.getLogger(target).setLevel(logging.DEBUG)
-
- return options, args
-
-def rsync_wrapper (command, restrict='lvm:') :
- """
- Wrap given rsync command.
-
- Parses the command, the source path, and then executes rsync within the source path (which may be a special
- pseudo-path with additional handling).
- """
-
- try :
- # parse the rsync command sent by the client
- rsync_cmd, rsync_options, source_path, dest_path = rsync.parse_command(command,
- restrict_readonly = options.readonly,
- )
-
- except RSyncCommandFormatError, e:
- log.error("invalid rsync command: %r: %s", command, e)
- return 2
-
- # XXX: the real path is always given second..
- path = dest_path
-
- try :
- # parse the source path as given by the client, may be a real path or pseudo-path
- source = rsync.parse_source(path,
- restrict_path = options.restrict_path,
- lvm_opts = dict(size = options.snapshot_size, wait = options.snapshot_wait),
- )
-
- except RSyncCommandFormatError, e:
- log.error("invalid rsync source: %r: %s", path, e)
- return 2
-
- try :
- # run rsync within the source (may perform additional stuff like snapshotting...)
- source.execute(rsync_options)
-
- except InvokeError, e:
- log.error("%s failed: %d", e.cmd, e.exit)
- return e.exit
-
- # ok
- return 0
-
-def main (argv) :
- global options
-
- # global options + args
- options, args = parse_options(argv)
-
- # command required
- if options.given_command :
- # from args (as given by `rsync -e pvlbackup-rsync-wrapper`) -> 'pvlbackup-rsync-wrapper <host> (<command> ...)'
- host = args.pop(0)
- command_parts = args
-
- log.debug("host=%r, using command from args: %r", host, command_parts)
-
- # args
- elif args :
- log.error("No arguments are handled")
- return 2
-
- elif options.command:
- # as given
- command_parts = shlex.split(options.command)
-
- else :
- log.error("SSH_ORIGINAL_COMMAND not given")
- return 2
-
-
- # run
- try :
- return rsync_wrapper(command_parts)
-
- except Exception, e:
- log.error("Internal error:", exc_info=e)
- return 3
-
- # ok
- return 0
-
-if __name__ == '__main__' :
- import sys
-
- sys.exit(main(sys.argv))
-
--- a/pvl/backup/rsync.py Sun Apr 22 12:31:26 2012 +0300
+++ b/pvl/backup/rsync.py Sun Apr 22 12:56:53 2012 +0300
@@ -36,13 +36,26 @@
rsync server-mode execution.
"""
- def _execute (self, options, path) :
+ def _execute (self, options, srcdst, path) :
"""
Underlying rsync just reads from filesystem.
+
+ options - list of rsync options
+ srcdst - the (source, dest) pair with None placeholder, as returned by parse_command
+ path - the real path to replace None with
"""
+
+ # one of this will be None
+ src, dst = srcdst
+
+ # replace None -> path
+ src = src or path
+ dst = dst or path
+
+ log.debug("%r -> %r", src, dst)
# invoke directly, no option-handling, nor stdin/out redirection
- invoke.invoke(RSYNC, options + ['.', path], data=False)
+ invoke.invoke(RSYNC, options + [ src, dst ], data=False)
class RSyncFSServer (RSyncServer) :
"""
@@ -54,8 +67,13 @@
self.path = path
- def execute (self, options) :
- return self._execute(options, self.path)
+ def execute (self, options, srcdst) :
+ """
+ options - list of rsync options
+ srcdst - the (source, dest) pair with None placeholder, as returned by parse_command
+ """
+
+ return self._execute(options, srcdst, self.path)
class RSyncLVMServer (RSyncServer) :
"""
@@ -73,11 +91,12 @@
self.volume = volume
self.snapshot_opts = opts
- def execute (self, options) :
+ def execute (self, options, srcdst) :
"""
Snapshot, mount, execute
options - list of rsync options
+ srcdst - the (source, dest) pair with None placeholder, as returned by parse_command
"""
# backup target from LVM command
@@ -97,7 +116,7 @@
log.info("Running rsync: %s", mountpoint)
# with trailing slash
- return self._execute(options, mountpoint.path + '/')
+ return self._execute(options, srcdst, mountpoint.path + '/')
# cleanup
# cleanup
@@ -110,9 +129,15 @@
restrict_server - restrict to server-mode
restrict_readonly - restrict to read/send-mode
+ In server mode, source will always be '.', and dest the source/dest.
+
Returns:
- (cmd, options, source, dest)
+ (cmd, options, path, (source, dest))
+
+ path -> the real source path
+ (source, dest) -> combination of None for path, and the real source/dest
+
"""
cmd = None
@@ -139,34 +164,49 @@
have_sender = ('--sender' in options)
# verify
- if not have_server :
+ if restrict_server and not have_server :
raise RSyncCommandFormatError("Missing --server")
if restrict_readonly and not have_sender :
raise RSyncCommandFormatError("Missing --sender for readonly")
- # parse path
+ if not source :
+ raise RSyncCommandFormatError("Missing source path")
+
+ if not dest:
+ raise RSyncCommandFormatError("Missing dest path")
+
+
+ # parse real source
if have_sender :
- # read
- # XXX: which way does the dot go?
+ # read; first arg will always be .
if source != '.' :
raise RSyncCommandFormatError("Invalid dest for sender")
-
+
path = dest
+ dest = None
+
+ log.debug("using server/sender source path: %s", path)
- else :
+ elif have_server :
# write
if source != '.' :
raise RSyncCommandFormatError("Invalid source for reciever")
path = dest
+ dest = None
+
+ log.debug("using server dest path: %s", path)
- if not path :
- raise RSyncCommandFormatError("Missing path")
+ else :
+ # local src -> dst
+ path = source
+ source = None
+
+ log.debug("using local src path: %s -> %s", path, dest)
# ok
- return cmd, options, source, dest
-
+ return cmd, options, path, (source, dest)
def parse_source (path, restrict_path=False, lvm_opts={}) :
"""
@@ -198,8 +238,7 @@
except ValueError, e:
raise RSyncCommandFormatError("Invalid lvm pseudo-path: {error}".format(error=e))
- # XXX: validate
-
+ # XXX: validate?
log.info("LVM: %s/%s", vg, lv)
# open
--- a/setup.py Sun Apr 22 12:31:26 2012 +0300
+++ b/setup.py Sun Apr 22 12:56:53 2012 +0300
@@ -16,7 +16,7 @@
# binaries
scripts = [
- 'bin/pvlbackup-rsync-wrapper',
+ 'bin/pvl.backup-rsync',
'bin/pvlbackup-rsync-snapshot',
],
)