# HG changeset patch # User Tero Marttila # Date 1335088613 -10800 # Node ID 2911d4dd5a472e32ee366a3c933eac9af51bb1fe # Parent 43e27a3e9efe87dbd9c09e482ead6a5dffb93350 pvl.backup-rsync: implement local rsync command execution, rename from pvlbackup-rsync-wrapper diff -r 43e27a3e9efe -r 2911d4dd5a47 bin/pvl.backup-rsync --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/pvl.backup-rsync Sun Apr 22 12:56:53 2012 +0300 @@ -0,0 +1,195 @@ +#!/usr/bin/python + +""" + SSH authorized_keys command="..." wrapper for rsync. + + Testing goes something like: + sudo PYTHONPATH=. ./bin/pvlbackup-rsync-wrapper --command 'rsync --server --sender -ax . lvm:asdf:test' -vD + + sudo sh -c "PYTHONPATH=. rsync -e './bin/pvlbackup-rsync-wrapper --debug -C --' -ax testing:lvm:asdf:test test/tmp" +""" + +from pvl.backup import __version__ +from pvl.backup.rsync import RSyncCommandFormatError +from pvl.backup.invoke import InvokeError +from pvl.backup import rsync, lvm + +import optparse +import shlex +import os +import logging + +log = logging.getLogger() + +def parse_options (argv) : + """ + Parse command-line arguments. + """ + +# import sys; sys.stderr.write("%s\n" % (argv, )) + + parser = optparse.OptionParser( + prog = argv[0], + + # module docstring + description = __doc__, + version = __version__, + ) + + # logging + general = optparse.OptionGroup(parser, "General Options") + + general.add_option('-q', '--quiet', dest='loglevel', action='store_const', const=logging.WARNING, help="Less output") + general.add_option('-v', '--verbose', dest='loglevel', action='store_const', const=logging.INFO, help="More output") + general.add_option('-D', '--debug', dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output") + general.add_option('--debug-for', action='append', metavar='MODULE', help="Enable logging for the given logger/module name") + + parser.add_option_group(general) + + # + parser.add_option('-c', '--command', metavar='CMD', default=os.environ.get('SSH_ORIGINAL_COMMAND'), + help="rsync command to execute") + + parser.add_option('-C', '--given-command', action='store_true', default=False, + help="use given command in `rsync -e %prog` format") + + parser.add_option('-n', '--noop', action='store_true', default=False, + help="Parse command, but do not execute") + + parser.add_option('-R', '--readonly', action='store_true', default=False, + help="restrict to read/source mode") + + parser.add_option('-P', '--restrict-path', metavar='PATH', default=False, + help="restrict to given path prefix") + + # lvm options + parser.add_option('-L', '--snapshot-size', metavar='SIZE', default=lvm.LVM_SNAPSHOT_SIZE, + help="create snapshot with given LV size (used to store writes during backup)") + + parser.add_option('--snapshot-wait', metavar='SECONDS', default=lvm.LVM_SNAPSHOT_WAIT, type='float', + help="wait for snapshot to settle after unmounting") + + # defaults + parser.set_defaults( + debug_for = [], + loglevel = logging.INFO, + ) + + # parse + options, args = parser.parse_args(argv[1:]) + + # configure + logging.basicConfig( + format = '%(levelname)6s %(name)s : %(funcName)s : %(message)s', + level = options.loglevel, + ) + + # enable debugging for specific targets + for target in options.debug_for : + logging.getLogger(target).setLevel(logging.DEBUG) + + return options, args + +def rsync_wrapper (command, options, local=False) : + """ + Wrap given rsync command. + + Parses the command, the source path, and then executes rsync within the source path (which may be a special + pseudo-path with additional handling). + + local - the given command is a full local command, not an --server mode operation + """ + + # parse the rsync command sent by the client + try : + # path = the real source path + rsync_cmd, rsync_options, path, srcdst = rsync.parse_command(command, + restrict_server = not local, + restrict_readonly = options.readonly, + ) + + except RSyncCommandFormatError, e: + log.error("invalid rsync command: %r: %s", command, e) + return 2 + + + # parse source + try : + # parse the source path as given by the client, may be a real path or pseudo-path + source = rsync.parse_source(path, + restrict_path = options.restrict_path, + lvm_opts = dict(size = options.snapshot_size, wait = options.snapshot_wait), + ) + + except RSyncCommandFormatError, e: + log.error("invalid rsync source: %r: %s", path, e) + return 2 + + # noop? + if options.noop : + log.info("noop: %r -> %r: execute(%r, %r)", path, source, rsync_options, srcdst) + return 0 + + # execute + try : + # run rsync within the source (may perform additional stuff like snapshotting...) + source.execute(rsync_options, srcdst) + + except InvokeError, e: + log.error("%s failed: %d", e.cmd, e.exit) + return e.exit + + + # ok + return 0 + +def main (argv) : + """ + Run, with full argv + """ + + # global options + args + options, args = parse_options(argv) + + # was a local command, not remote? + local = False + + # from args (as given by `rsync -e pvlbackup-rsync-wrapper`) -> 'pvlbackup-rsync-wrapper ( ...)' + if options.given_command : + host = args.pop(0) + command_parts = args + + log.debug("host=%r, using command from args: %r", host, command_parts) + + # from env/--command + elif options.command: + # as given + command_parts = shlex.split(options.command) + + # normal args + elif args : + command_parts = [argv[0]] + args + local = True + + log.debug("using given rsync args: %r", command_parts) + + else : + log.error("SSH_ORIGINAL_COMMAND not given") + return 2 + + # run + try : + return rsync_wrapper(command_parts, options, local) + + except Exception, e: + log.error("Internal error:", exc_info=e) + return 3 + + # ok + return 0 + +if __name__ == '__main__' : + import sys + + sys.exit(main(sys.argv)) + diff -r 43e27a3e9efe -r 2911d4dd5a47 bin/pvlbackup-rsync-wrapper --- a/bin/pvlbackup-rsync-wrapper Sun Apr 22 12:31:26 2012 +0300 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,179 +0,0 @@ -#!/usr/bin/python - -""" - SSH authorized_keys command="..." wrapper for rsync. - - Testing goes something like: - sudo PYTHONPATH=. ./bin/pvlbackup-rsync-wrapper --command 'rsync --server --sender -ax . lvm:asdf:test' -vD - - sudo sh -c "PYTHONPATH=. rsync -e './bin/pvlbackup-rsync-wrapper --debug -C --' -ax testing:lvm:asdf:test test/tmp" -""" - -from pvl.backup import __version__ -from pvl.backup.rsync import RSyncCommandFormatError -from pvl.backup.invoke import InvokeError -from pvl.backup import rsync, lvm - -import optparse -import shlex -import os -import logging - -log = logging.getLogger() - -# command-line options -options = None - -def parse_options (argv) : - """ - Parse command-line arguments. - """ - -# import sys; sys.stderr.write("%s\n" % (argv, )) - - parser = optparse.OptionParser( - prog = argv[0], - - # module docstring - description = __doc__, - version = __version__, - ) - - # logging - general = optparse.OptionGroup(parser, "General Options") - - general.add_option('-q', '--quiet', dest='loglevel', action='store_const', const=logging.WARNING, help="Less output") - general.add_option('-v', '--verbose', dest='loglevel', action='store_const', const=logging.INFO, help="More output") - general.add_option('-D', '--debug', dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output") - general.add_option('--debug-for', action='append', metavar='MODULE', help="Enable logging for the given logger/module name") - - parser.add_option_group(general) - - # - parser.add_option('-c', '--command', metavar='CMD', default=os.environ.get('SSH_ORIGINAL_COMMAND'), - help="rsync command to execute") - - parser.add_option('-C', '--given-command', action='store_true', default=False, - help="use given command in `rsync -e %prog` format") - - parser.add_option('-R', '--readonly', action='store_true', default=False, - help="restrict to read/source mode") - - parser.add_option('-P', '--restrict-path', metavar='PATH', default=False, - help="restrict to given path prefix") - - # lvm options - parser.add_option('-L', '--snapshot-size', metavar='SIZE', default=lvm.LVM_SNAPSHOT_SIZE, - help="create snapshot with given LV size (used to store writes during backup)") - - parser.add_option('--snapshot-wait', metavar='SECONDS', default=lvm.LVM_SNAPSHOT_WAIT, type='float', - help="wait for snapshot to settle after unmounting") - - # defaults - parser.set_defaults( - debug_for = [], - loglevel = logging.INFO, - ) - - # parse - options, args = parser.parse_args(argv[1:]) - - # configure - logging.basicConfig( - format = '%(levelname)6s %(name)s : %(funcName)s : %(message)s', - level = options.loglevel, - ) - - # enable debugging for specific targets - for target in options.debug_for : - logging.getLogger(target).setLevel(logging.DEBUG) - - return options, args - -def rsync_wrapper (command, restrict='lvm:') : - """ - Wrap given rsync command. - - Parses the command, the source path, and then executes rsync within the source path (which may be a special - pseudo-path with additional handling). - """ - - try : - # parse the rsync command sent by the client - rsync_cmd, rsync_options, source_path, dest_path = rsync.parse_command(command, - restrict_readonly = options.readonly, - ) - - except RSyncCommandFormatError, e: - log.error("invalid rsync command: %r: %s", command, e) - return 2 - - # XXX: the real path is always given second.. - path = dest_path - - try : - # parse the source path as given by the client, may be a real path or pseudo-path - source = rsync.parse_source(path, - restrict_path = options.restrict_path, - lvm_opts = dict(size = options.snapshot_size, wait = options.snapshot_wait), - ) - - except RSyncCommandFormatError, e: - log.error("invalid rsync source: %r: %s", path, e) - return 2 - - try : - # run rsync within the source (may perform additional stuff like snapshotting...) - source.execute(rsync_options) - - except InvokeError, e: - log.error("%s failed: %d", e.cmd, e.exit) - return e.exit - - # ok - return 0 - -def main (argv) : - global options - - # global options + args - options, args = parse_options(argv) - - # command required - if options.given_command : - # from args (as given by `rsync -e pvlbackup-rsync-wrapper`) -> 'pvlbackup-rsync-wrapper ( ...)' - host = args.pop(0) - command_parts = args - - log.debug("host=%r, using command from args: %r", host, command_parts) - - # args - elif args : - log.error("No arguments are handled") - return 2 - - elif options.command: - # as given - command_parts = shlex.split(options.command) - - else : - log.error("SSH_ORIGINAL_COMMAND not given") - return 2 - - - # run - try : - return rsync_wrapper(command_parts) - - except Exception, e: - log.error("Internal error:", exc_info=e) - return 3 - - # ok - return 0 - -if __name__ == '__main__' : - import sys - - sys.exit(main(sys.argv)) - diff -r 43e27a3e9efe -r 2911d4dd5a47 pvl/backup/rsync.py --- a/pvl/backup/rsync.py Sun Apr 22 12:31:26 2012 +0300 +++ b/pvl/backup/rsync.py Sun Apr 22 12:56:53 2012 +0300 @@ -36,13 +36,26 @@ rsync server-mode execution. """ - def _execute (self, options, path) : + def _execute (self, options, srcdst, path) : """ Underlying rsync just reads from filesystem. + + options - list of rsync options + srcdst - the (source, dest) pair with None placeholder, as returned by parse_command + path - the real path to replace None with """ + + # one of this will be None + src, dst = srcdst + + # replace None -> path + src = src or path + dst = dst or path + + log.debug("%r -> %r", src, dst) # invoke directly, no option-handling, nor stdin/out redirection - invoke.invoke(RSYNC, options + ['.', path], data=False) + invoke.invoke(RSYNC, options + [ src, dst ], data=False) class RSyncFSServer (RSyncServer) : """ @@ -54,8 +67,13 @@ self.path = path - def execute (self, options) : - return self._execute(options, self.path) + def execute (self, options, srcdst) : + """ + options - list of rsync options + srcdst - the (source, dest) pair with None placeholder, as returned by parse_command + """ + + return self._execute(options, srcdst, self.path) class RSyncLVMServer (RSyncServer) : """ @@ -73,11 +91,12 @@ self.volume = volume self.snapshot_opts = opts - def execute (self, options) : + def execute (self, options, srcdst) : """ Snapshot, mount, execute options - list of rsync options + srcdst - the (source, dest) pair with None placeholder, as returned by parse_command """ # backup target from LVM command @@ -97,7 +116,7 @@ log.info("Running rsync: %s", mountpoint) # with trailing slash - return self._execute(options, mountpoint.path + '/') + return self._execute(options, srcdst, mountpoint.path + '/') # cleanup # cleanup @@ -110,9 +129,15 @@ restrict_server - restrict to server-mode restrict_readonly - restrict to read/send-mode + In server mode, source will always be '.', and dest the source/dest. + Returns: - (cmd, options, source, dest) + (cmd, options, path, (source, dest)) + + path -> the real source path + (source, dest) -> combination of None for path, and the real source/dest + """ cmd = None @@ -139,34 +164,49 @@ have_sender = ('--sender' in options) # verify - if not have_server : + if restrict_server and not have_server : raise RSyncCommandFormatError("Missing --server") if restrict_readonly and not have_sender : raise RSyncCommandFormatError("Missing --sender for readonly") - # parse path + if not source : + raise RSyncCommandFormatError("Missing source path") + + if not dest: + raise RSyncCommandFormatError("Missing dest path") + + + # parse real source if have_sender : - # read - # XXX: which way does the dot go? + # read; first arg will always be . if source != '.' : raise RSyncCommandFormatError("Invalid dest for sender") - + path = dest + dest = None + + log.debug("using server/sender source path: %s", path) - else : + elif have_server : # write if source != '.' : raise RSyncCommandFormatError("Invalid source for reciever") path = dest + dest = None + + log.debug("using server dest path: %s", path) - if not path : - raise RSyncCommandFormatError("Missing path") + else : + # local src -> dst + path = source + source = None + + log.debug("using local src path: %s -> %s", path, dest) # ok - return cmd, options, source, dest - + return cmd, options, path, (source, dest) def parse_source (path, restrict_path=False, lvm_opts={}) : """ @@ -198,8 +238,7 @@ except ValueError, e: raise RSyncCommandFormatError("Invalid lvm pseudo-path: {error}".format(error=e)) - # XXX: validate - + # XXX: validate? log.info("LVM: %s/%s", vg, lv) # open diff -r 43e27a3e9efe -r 2911d4dd5a47 setup.py --- a/setup.py Sun Apr 22 12:31:26 2012 +0300 +++ b/setup.py Sun Apr 22 12:56:53 2012 +0300 @@ -16,7 +16,7 @@ # binaries scripts = [ - 'bin/pvlbackup-rsync-wrapper', + 'bin/pvl.backup-rsync', 'bin/pvlbackup-rsync-snapshot', ], )