pvl.backup-rsync: implement local rsync command execution, rename from pvlbackup-rsync-wrapper
authorTero Marttila <terom@paivola.fi>
Sun, 22 Apr 2012 12:56:53 +0300
changeset 43 2911d4dd5a47
parent 42 43e27a3e9efe
child 44 7069af6b7025
pvl.backup-rsync: implement local rsync command execution, rename from pvlbackup-rsync-wrapper
bin/pvl.backup-rsync
bin/pvlbackup-rsync-wrapper
pvl/backup/rsync.py
setup.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/bin/pvl.backup-rsync	Sun Apr 22 12:56:53 2012 +0300
@@ -0,0 +1,195 @@
+#!/usr/bin/python
+
+"""
+    SSH authorized_keys command="..." wrapper for rsync.
+
+    Testing goes something like:
+        sudo PYTHONPATH=. ./bin/pvlbackup-rsync-wrapper --command 'rsync --server --sender -ax . lvm:asdf:test' -vD
+
+        sudo sh -c "PYTHONPATH=. rsync -e './bin/pvlbackup-rsync-wrapper --debug -C --' -ax testing:lvm:asdf:test test/tmp"
+"""
+
+from pvl.backup import __version__
+from pvl.backup.rsync import RSyncCommandFormatError
+from pvl.backup.invoke import InvokeError
+from pvl.backup import rsync, lvm
+
+import optparse
+import shlex
+import os
+import logging
+
+log = logging.getLogger()
+
+def parse_options (argv) :
+    """
+        Parse command-line arguments.
+    """
+
+#    import sys; sys.stderr.write("%s\n" % (argv, ))
+
+    parser = optparse.OptionParser(
+            prog        = argv[0],
+
+            # module docstring
+            description = __doc__,
+            version     = __version__,
+    )
+
+    # logging
+    general = optparse.OptionGroup(parser, "General Options")
+
+    general.add_option('-q', '--quiet',      dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")
+    general.add_option('-v', '--verbose',    dest='loglevel', action='store_const', const=logging.INFO,  help="More output")
+    general.add_option('-D', '--debug',      dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")
+    general.add_option('--debug-for',        action='append', metavar='MODULE', help="Enable logging for the given logger/module name")
+
+    parser.add_option_group(general)
+
+    #
+    parser.add_option('-c', '--command',    metavar='CMD', default=os.environ.get('SSH_ORIGINAL_COMMAND'),
+            help="rsync command to execute")
+
+    parser.add_option('-C', '--given-command', action='store_true', default=False,
+            help="use given command in `rsync -e %prog` format")
+
+    parser.add_option('-n', '--noop', action='store_true', default=False,
+            help="Parse command, but do not execute")
+
+    parser.add_option('-R', '--readonly',   action='store_true', default=False,
+            help="restrict to read/source mode")
+
+    parser.add_option('-P', '--restrict-path', metavar='PATH', default=False,
+            help="restrict to given path prefix")
+
+    # lvm options
+    parser.add_option('-L', '--snapshot-size', metavar='SIZE', default=lvm.LVM_SNAPSHOT_SIZE,
+            help="create snapshot with given LV size (used to store writes during backup)")
+
+    parser.add_option('--snapshot-wait', metavar='SECONDS', default=lvm.LVM_SNAPSHOT_WAIT, type='float',
+            help="wait for snapshot to settle after unmounting")
+
+    # defaults
+    parser.set_defaults(
+        debug_for   = [],
+        loglevel    = logging.INFO,
+    )
+
+    # parse
+    options, args = parser.parse_args(argv[1:])
+
+    # configure
+    logging.basicConfig(
+        format  = '%(levelname)6s %(name)s : %(funcName)s : %(message)s',
+        level   = options.loglevel,
+    )
+
+    # enable debugging for specific targets
+    for target in options.debug_for :
+        logging.getLogger(target).setLevel(logging.DEBUG)
+
+    return options, args
+
+def rsync_wrapper (command, options, local=False) :
+    """
+        Wrap given rsync command.
+
+        Parses the command, the source path, and then executes rsync within the source path (which may be a special
+        pseudo-path with additional handling).
+
+            local       - the given command is a full local command, not an --server mode operation
+    """
+
+    # parse the rsync command sent by the client
+    try :
+        # path = the real source path
+        rsync_cmd, rsync_options, path, srcdst = rsync.parse_command(command, 
+                restrict_server     = not local,
+                restrict_readonly   = options.readonly,
+            )
+
+    except RSyncCommandFormatError, e:
+        log.error("invalid rsync command: %r: %s", command, e)
+        return 2
+
+
+    # parse source
+    try :
+        # parse the source path as given by the client, may be a real path or pseudo-path
+        source = rsync.parse_source(path,
+                restrict_path       = options.restrict_path,
+                lvm_opts            = dict(size = options.snapshot_size, wait = options.snapshot_wait),
+            )
+
+    except RSyncCommandFormatError, e:
+        log.error("invalid rsync source: %r: %s", path, e)
+        return 2
+
+    # noop?
+    if options.noop :
+        log.info("noop: %r -> %r: execute(%r, %r)", path, source, rsync_options, srcdst)
+        return 0
+
+    # execute
+    try :
+        # run rsync within the source (may perform additional stuff like snapshotting...)
+        source.execute(rsync_options, srcdst)
+
+    except InvokeError, e:
+        log.error("%s failed: %d", e.cmd, e.exit)
+        return e.exit
+
+
+    # ok
+    return 0
+
+def main (argv) :
+    """
+        Run, with full argv
+    """
+
+    # global options + args
+    options, args = parse_options(argv)
+
+    # was a local command, not remote?
+    local = False
+
+    # from args (as given by `rsync -e pvlbackup-rsync-wrapper`) -> 'pvlbackup-rsync-wrapper <host> (<command> ...)'
+    if options.given_command :
+        host = args.pop(0)
+        command_parts = args
+
+        log.debug("host=%r, using command from args: %r", host, command_parts)
+    
+    # from env/--command
+    elif options.command:
+        # as given
+        command_parts = shlex.split(options.command)
+
+    # normal args
+    elif args :
+        command_parts = [argv[0]] + args
+        local = True
+
+        log.debug("using given rsync args: %r", command_parts)
+
+    else :
+        log.error("SSH_ORIGINAL_COMMAND not given")
+        return 2
+
+    # run
+    try :
+        return rsync_wrapper(command_parts, options, local)
+
+    except Exception, e:
+        log.error("Internal error:", exc_info=e)
+        return 3
+
+    # ok
+    return 0
+
+if __name__ == '__main__' :
+    import sys
+
+    sys.exit(main(sys.argv))
+
--- a/bin/pvlbackup-rsync-wrapper	Sun Apr 22 12:31:26 2012 +0300
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,179 +0,0 @@
-#!/usr/bin/python
-
-"""
-    SSH authorized_keys command="..." wrapper for rsync.
-
-    Testing goes something like:
-        sudo PYTHONPATH=. ./bin/pvlbackup-rsync-wrapper --command 'rsync --server --sender -ax . lvm:asdf:test' -vD
-
-        sudo sh -c "PYTHONPATH=. rsync -e './bin/pvlbackup-rsync-wrapper --debug -C --' -ax testing:lvm:asdf:test test/tmp"
-"""
-
-from pvl.backup import __version__
-from pvl.backup.rsync import RSyncCommandFormatError
-from pvl.backup.invoke import InvokeError
-from pvl.backup import rsync, lvm
-
-import optparse
-import shlex
-import os
-import logging
-
-log = logging.getLogger()
-
-# command-line options
-options = None
-
-def parse_options (argv) :
-    """
-        Parse command-line arguments.
-    """
-
-#    import sys; sys.stderr.write("%s\n" % (argv, ))
-
-    parser = optparse.OptionParser(
-            prog        = argv[0],
-
-            # module docstring
-            description = __doc__,
-            version     = __version__,
-    )
-
-    # logging
-    general = optparse.OptionGroup(parser, "General Options")
-
-    general.add_option('-q', '--quiet',      dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")
-    general.add_option('-v', '--verbose',    dest='loglevel', action='store_const', const=logging.INFO,  help="More output")
-    general.add_option('-D', '--debug',      dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")
-    general.add_option('--debug-for',        action='append', metavar='MODULE', help="Enable logging for the given logger/module name")
-
-    parser.add_option_group(general)
-
-    #
-    parser.add_option('-c', '--command',    metavar='CMD', default=os.environ.get('SSH_ORIGINAL_COMMAND'),
-            help="rsync command to execute")
-
-    parser.add_option('-C', '--given-command', action='store_true', default=False,
-            help="use given command in `rsync -e %prog` format")
-
-    parser.add_option('-R', '--readonly',   action='store_true', default=False,
-            help="restrict to read/source mode")
-
-    parser.add_option('-P', '--restrict-path', metavar='PATH', default=False,
-            help="restrict to given path prefix")
-
-    # lvm options
-    parser.add_option('-L', '--snapshot-size', metavar='SIZE', default=lvm.LVM_SNAPSHOT_SIZE,
-            help="create snapshot with given LV size (used to store writes during backup)")
-
-    parser.add_option('--snapshot-wait', metavar='SECONDS', default=lvm.LVM_SNAPSHOT_WAIT, type='float',
-            help="wait for snapshot to settle after unmounting")
-
-    # defaults
-    parser.set_defaults(
-        debug_for   = [],
-        loglevel    = logging.INFO,
-    )
-
-    # parse
-    options, args = parser.parse_args(argv[1:])
-
-    # configure
-    logging.basicConfig(
-        format  = '%(levelname)6s %(name)s : %(funcName)s : %(message)s',
-        level   = options.loglevel,
-    )
-
-    # enable debugging for specific targets
-    for target in options.debug_for :
-        logging.getLogger(target).setLevel(logging.DEBUG)
-
-    return options, args
-
-def rsync_wrapper (command, restrict='lvm:') :
-    """
-        Wrap given rsync command.
-
-        Parses the command, the source path, and then executes rsync within the source path (which may be a special
-        pseudo-path with additional handling).
-    """
-
-    try :
-        # parse the rsync command sent by the client
-        rsync_cmd, rsync_options, source_path, dest_path = rsync.parse_command(command, 
-                restrict_readonly   = options.readonly,
-            )
-
-    except RSyncCommandFormatError, e:
-        log.error("invalid rsync command: %r: %s", command, e)
-        return 2
-
-    # XXX: the real path is always given second..
-    path = dest_path
-
-    try :
-        # parse the source path as given by the client, may be a real path or pseudo-path
-        source = rsync.parse_source(path,
-                restrict_path       = options.restrict_path,
-                lvm_opts            = dict(size = options.snapshot_size, wait = options.snapshot_wait),
-            )
-
-    except RSyncCommandFormatError, e:
-        log.error("invalid rsync source: %r: %s", path, e)
-        return 2
-
-    try :
-        # run rsync within the source (may perform additional stuff like snapshotting...)
-        source.execute(rsync_options)
-
-    except InvokeError, e:
-        log.error("%s failed: %d", e.cmd, e.exit)
-        return e.exit
-
-    # ok
-    return 0
-
-def main (argv) :
-    global options
-
-    # global options + args
-    options, args = parse_options(argv)
-
-    # command required
-    if options.given_command :
-        # from args (as given by `rsync -e pvlbackup-rsync-wrapper`) -> 'pvlbackup-rsync-wrapper <host> (<command> ...)'
-        host = args.pop(0)
-        command_parts = args
-
-        log.debug("host=%r, using command from args: %r", host, command_parts)
-
-    # args
-    elif args :
-        log.error("No arguments are handled")
-        return 2
-
-    elif options.command:
-        # as given
-        command_parts = shlex.split(options.command)
-
-    else :
-        log.error("SSH_ORIGINAL_COMMAND not given")
-        return 2
-
-
-    # run
-    try :
-        return rsync_wrapper(command_parts)
-
-    except Exception, e:
-        log.error("Internal error:", exc_info=e)
-        return 3
-
-    # ok
-    return 0
-
-if __name__ == '__main__' :
-    import sys
-
-    sys.exit(main(sys.argv))
-
--- a/pvl/backup/rsync.py	Sun Apr 22 12:31:26 2012 +0300
+++ b/pvl/backup/rsync.py	Sun Apr 22 12:56:53 2012 +0300
@@ -36,13 +36,26 @@
         rsync server-mode execution.
     """
 
-    def _execute (self, options, path) :
+    def _execute (self, options, srcdst, path) :
         """
             Underlying rsync just reads from filesystem.
+
+                options     - list of rsync options
+                srcdst      - the (source, dest) pair with None placeholder, as returned by parse_command
+                path        - the real path to replace None with
         """
+    
+        # one of this will be None
+        src, dst = srcdst
+
+        # replace None -> path
+        src = src or path
+        dst = dst or path
+
+        log.debug("%r -> %r", src, dst)
         
         # invoke directly, no option-handling, nor stdin/out redirection
-        invoke.invoke(RSYNC, options + ['.', path], data=False)
+        invoke.invoke(RSYNC, options + [ src, dst ], data=False)
 
 class RSyncFSServer (RSyncServer) :
     """
@@ -54,8 +67,13 @@
 
         self.path = path
 
-    def execute (self, options) :
-        return self._execute(options, self.path)
+    def execute (self, options, srcdst) :
+        """
+                options     - list of rsync options
+                srcdst      - the (source, dest) pair with None placeholder, as returned by parse_command
+        """
+
+        return self._execute(options, srcdst, self.path)
 
 class RSyncLVMServer (RSyncServer) :
     """
@@ -73,11 +91,12 @@
         self.volume = volume
         self.snapshot_opts = opts
  
-    def execute (self, options) :
+    def execute (self, options, srcdst) :
         """
             Snapshot, mount, execute
 
                 options     - list of rsync options
+                srcdst      - the (source, dest) pair with None placeholder, as returned by parse_command
         """
         
         # backup target from LVM command
@@ -97,7 +116,7 @@
                 log.info("Running rsync: %s", mountpoint)
 
                 # with trailing slash
-                return self._execute(options, mountpoint.path + '/')
+                return self._execute(options, srcdst, mountpoint.path + '/')
 
             # cleanup
         # cleanup
@@ -110,9 +129,15 @@
             restrict_server     - restrict to server-mode
             restrict_readonly   - restrict to read/send-mode
         
+        In server mode, source will always be '.', and dest the source/dest.
+        
         Returns:
 
-            (cmd, options, source, dest)
+            (cmd, options, path, (source, dest))
+
+            path            -> the real source path
+            (source, dest)  -> combination of None for path, and the real source/dest
+
     """
 
     cmd = None
@@ -139,34 +164,49 @@
     have_sender = ('--sender' in options)
 
     # verify
-    if not have_server :
+    if restrict_server and not have_server :
         raise RSyncCommandFormatError("Missing --server")
 
     if restrict_readonly and not have_sender :
         raise RSyncCommandFormatError("Missing --sender for readonly")
 
-    # parse path
+    if not source :
+        raise RSyncCommandFormatError("Missing source path")
+        
+    if not dest:
+        raise RSyncCommandFormatError("Missing dest path")
+
+
+    # parse real source
     if have_sender :
-        # read
-        # XXX: which way does the dot go?
+        # read; first arg will always be .
         if source != '.' :
             raise RSyncCommandFormatError("Invalid dest for sender")
-        
+
         path = dest
+        dest = None
+        
+        log.debug("using server/sender source path: %s", path)
 
-    else :
+    elif have_server :
         # write
         if source != '.' :
             raise RSyncCommandFormatError("Invalid source for reciever")
 
         path = dest
+        dest = None
+        
+        log.debug("using server dest path: %s", path)
 
-    if not path :
-        raise RSyncCommandFormatError("Missing path")
+    else :
+        # local src -> dst
+        path = source
+        source = None
+
+        log.debug("using local src path: %s -> %s", path, dest)
 
     # ok
-    return cmd, options, source, dest
-
+    return cmd, options, path, (source, dest)
       
 def parse_source (path, restrict_path=False, lvm_opts={}) :
     """
@@ -198,8 +238,7 @@
         except ValueError, e:
             raise RSyncCommandFormatError("Invalid lvm pseudo-path: {error}".format(error=e))
         
-        # XXX: validate
-
+        # XXX: validate?
         log.info("LVM: %s/%s", vg, lv)
 
         # open
--- a/setup.py	Sun Apr 22 12:31:26 2012 +0300
+++ b/setup.py	Sun Apr 22 12:56:53 2012 +0300
@@ -16,7 +16,7 @@
 
     # binaries
     scripts         = [
-        'bin/pvlbackup-rsync-wrapper', 
+        'bin/pvl.backup-rsync', 
         'bin/pvlbackup-rsync-snapshot',
     ],
 )