terom@5: """
terom@5:     rsync handling.
terom@5: 
terom@5:     Apologies for the 'RSync' nomenclature
terom@5: """
terom@5: 
terom@5: from pvl.backup.lvm import LVM, LVMVolume, LVMSnapshot
terom@5: from pvl.backup.mount import mount
terom@12: from pvl.backup import invoke
terom@5: 
terom@5: import os.path
terom@5: 
terom@5: import logging
terom@5: 
terom@5: log = logging.getLogger('pvl.backup.rsync')
terom@5: 
terom@12: RSYNC = '/usr/bin/rsync'
terom@12: 
terom@12: def rsync (source, dest, **opts) :
terom@12:     """
terom@12:         Run rsync.
terom@64: 
terom@64:         Raises RsyncError if rsync fails.
terom@64: 
terom@64:         XXX: never used anywhere?
terom@12:     """
terom@12: 
terom@64:     try :
terom@64:         invoke.command(RSYNC, source, dest, **opts)
terom@64: 
terom@64:     except invoke.InvokeError as ex :
terom@64:         raise RsyncError(ex)
terom@5: 
terom@5: class RSyncCommandFormatError (Exception) :
terom@5:     """
terom@64:         Improper rsync command/source.
terom@5:     """
terom@5: 
terom@5:     pass
terom@5: 
terom@64: class RsyncError (Exception) :
terom@64:     """
terom@64:         Rsync command invocation failed.
terom@64:     """
terom@64:     
terom@64:     pass
terom@64: 
terom@64: 
terom@12: class RSyncServer (object) :
terom@12:     """
terom@12:         rsync server-mode execution.
terom@12:     """
terom@5: 
terom@70:     def _execute (self, options, srcdst, path, sudo=False) :
terom@5:         """
terom@5:             Underlying rsync just reads from filesystem.
terom@43: 
terom@43:                 options     - list of rsync options
terom@43:                 srcdst      - the (source, dest) pair with None placeholder, as returned by parse_command
terom@43:                 path        - the real path to replace None with
terom@70: 
terom@70:                 sudo        - execute rsync using sudo
terom@5:         """
terom@43:     
terom@43:         # one of this will be None
terom@43:         src, dst = srcdst
terom@43: 
terom@43:         # replace None -> path
terom@43:         src = src or path
terom@43:         dst = dst or path
terom@43: 
terom@67:         log.info("rsync %s %s %s", ' '.join(options), src, dst)
terom@70: 
terom@64:         try :
terom@70:             # invoke directly; no option-handling, nor stdin/out redirection
terom@70:             invoke.invoke(RSYNC, options + [ src, dst ], data=False, sudo=sudo)
terom@64: 
terom@64:         except invoke.InvokeError as ex :
terom@64:             raise RsyncError(ex)
terom@5: 
terom@12: class RSyncFSServer (RSyncServer) :
terom@5:     """
terom@5:         Normal filesystem backup.
terom@5:     """
terom@5: 
terom@5:     def __init__ (self, path) :
terom@12:         RSyncServer.__init__(self)
terom@5: 
terom@5:         self.path = path
terom@5: 
terom@70:     def execute (self, options, srcdst, **opts) :
terom@43:         """
terom@43:                 options     - list of rsync options
terom@43:                 srcdst      - the (source, dest) pair with None placeholder, as returned by parse_command
terom@43:         """
terom@43: 
terom@70:         return self._execute(options, srcdst, self.path, **opts)
terom@5: 
terom@44:     def __str__ (self) :
terom@44:         return self.path
terom@44:     
terom@44: class RSyncRemoteServer (RSyncServer) :
terom@44:     """
terom@44:         Remote filesystem backup.
terom@44:     """
terom@44: 
terom@44:     def __init__ (self, host, path) :
terom@44:         """
terom@44:             host        - remote SSH host
terom@44:             path        - remote path
terom@44:         """
terom@44: 
terom@44:         RSyncServer.__init__(self)
terom@44:     
terom@44:         # glue
terom@50:         self.path = host + ':' + path
terom@44: 
terom@70:     def execute (self, options, srcdst, **opts) :
terom@44:         """
terom@44:                 options     - list of rsync options
terom@44:                 srcdst      - the (source, dest) pair with None placeholder, as returned by parse_command
terom@44:         """
terom@44: 
terom@70:         return self._execute(options, srcdst, self.path, **opts)
terom@44: 
terom@44:     def __str__ (self) :
terom@44:         return self.path
terom@44:  
terom@12: class RSyncLVMServer (RSyncServer) :
terom@5:     """
terom@5:         Backup LVM LV by snapshotting + mounting it.
terom@5:     """
terom@5: 
terom@70:     def __init__ (self, vg, lv, sudo=None, **opts) :
terom@42:         """
terom@42:             **opts      - options for LVM.snapshot
terom@42:         """
terom@42: 
terom@12:         RSyncServer.__init__(self)
terom@70:         
terom@70:         # lvm
terom@70:         self.lvm = LVM(vg, sudo=sudo)
terom@70:         self.volume = self.lvm.volume(lv)
terom@5: 
terom@70:         self.sudo = sudo
terom@42:         self.snapshot_opts = opts
terom@5:  
terom@70:     def execute (self, options, srcdst, sudo=False, **opts) :
terom@5:         """
terom@5:             Snapshot, mount, execute
terom@42: 
terom@42:                 options     - list of rsync options
terom@43:                 srcdst      - the (source, dest) pair with None placeholder, as returned by parse_command
terom@5:         """
terom@5:         
terom@5:         # backup target from LVM command
terom@5:         lvm = self.volume.lvm
terom@5:         volume = self.volume
terom@5: 
terom@5:         # snapshot
terom@34:         log.info("Open snapshot: %s", volume)
terom@5: 
terom@5:         # XXX: generate snapshot nametag to be unique?
terom@42:         with lvm.snapshot(volume, tag='backup', **self.snapshot_opts) as snapshot:
terom@5:             # mount
terom@33:             log.info("Mounting snapshot: %s", snapshot)
terom@5: 
terom@70:             with mount(snapshot.dev_path, name_hint=('lvm_' + snapshot.name + '_'), readonly=True, sudo=sudo) as mountpoint:
terom@5:                 # rsync!
terom@34:                 log.info("Running rsync: %s", mountpoint)
terom@5: 
terom@9:                 # with trailing slash
terom@70:                 return self._execute(options, srcdst, mountpoint.path + '/', sudo=sudo, **opts)
terom@5: 
terom@5:             # cleanup
terom@5:         # cleanup
terom@5:  
terom@44:     def __str__ (self) :
terom@44:         return 'lvm:{volume}'.format(volume=self.volume)
terom@44:  
terom@28: def parse_command (command_parts, restrict_server=True, restrict_readonly=True) :
terom@5:     """
terom@5:         Parse given rsync server command into bits. 
terom@5: 
terom@28:             command_parts       - the command-list sent by rsync
terom@5:             restrict_server     - restrict to server-mode
terom@5:             restrict_readonly   - restrict to read/send-mode
terom@5:         
terom@43:         In server mode, source will always be '.', and dest the source/dest.
terom@43:         
terom@5:         Returns:
terom@5: 
terom@43:             (cmd, options, path, (source, dest))
terom@67:             
terom@67:             options         -> list of -options
terom@67:             path            -> real source path
terom@43:             (source, dest)  -> combination of None for path, and the real source/dest
terom@43: 
terom@5:     """
terom@5: 
terom@5:     cmd = None
terom@5:     options = []
terom@5:     source = None
terom@5:     dest = None
terom@5: 
terom@5:     # parse
terom@28:     for part in command_parts :
terom@5:         if cmd is None :
terom@5:             cmd = part
terom@5: 
terom@5:         elif part.startswith('-') :
terom@5:             options.append(part)
terom@5: 
terom@5:         elif source is None :
terom@5:             source = part
terom@5: 
terom@5:         elif dest is None :
terom@5:             dest = part
terom@5: 
terom@67:     log.debug("%s: %s", cmd, options)
terom@67: 
terom@5:     # options
terom@5:     have_server = ('--server' in options)
terom@5:     have_sender = ('--sender' in options)
terom@5: 
terom@5:     # verify
terom@43:     if restrict_server and not have_server :
terom@5:         raise RSyncCommandFormatError("Missing --server")
terom@5: 
terom@5:     if restrict_readonly and not have_sender :
terom@5:         raise RSyncCommandFormatError("Missing --sender for readonly")
terom@5: 
terom@43:     if not source :
terom@43:         raise RSyncCommandFormatError("Missing source path")
terom@43:         
terom@43:     if not dest:
terom@43:         raise RSyncCommandFormatError("Missing dest path")
terom@43: 
terom@43: 
terom@43:     # parse real source
terom@5:     if have_sender :
terom@43:         # read; first arg will always be .
terom@5:         if source != '.' :
terom@5:             raise RSyncCommandFormatError("Invalid dest for sender")
terom@43: 
terom@5:         path = dest
terom@43:         dest = None
terom@43:         
terom@43:         log.debug("using server/sender source path: %s", path)
terom@5: 
terom@43:     elif have_server :
terom@5:         # write
terom@5:         if source != '.' :
terom@5:             raise RSyncCommandFormatError("Invalid source for reciever")
terom@5: 
terom@5:         path = dest
terom@43:         dest = None
terom@43:         
terom@43:         log.debug("using server dest path: %s", path)
terom@5: 
terom@43:     else :
terom@43:         # local src -> dst
terom@43:         path = source
terom@43:         source = None
terom@43: 
terom@43:         log.debug("using local src path: %s -> %s", path, dest)
terom@6: 
terom@5:     # ok
terom@43:     return cmd, options, path, (source, dest)
terom@5:       
terom@67: def parse_source (path, restrict_paths=None, allow_remote=True, lvm_opts={}) :
terom@5:     """
terom@5:         Figure out source to rsync from, based on pseudo-path given in rsync command.
terom@67:             
terom@67:             restrict_paths  - raise RsyncCommandFormatError if source path is not under any of the given sources.
terom@67:             allow_remote    - allow remote backups?
terom@42:             lvm_opts        - dict of **opts for RSyncLVMServer
terom@5:     """
terom@51: 
terom@51:     endslash = path.endswith('/')
terom@5:         
terom@5:     # normalize
terom@5:     path = os.path.normpath(path)
terom@5: 
terom@51:     if endslash and not path.endswith('/') :
terom@51:         # add it back in
terom@51:         # happens for 'foo:/' and such
terom@51:         path += '/'
terom@51: 
terom@5:     # verify path
terom@67:     if restrict_paths :
terom@67:         for restrict_path in restrict_paths :
terom@67:             if path.startswith(restrict_path) :
terom@67:                 # ok
terom@67:                 break
terom@67:         else :
terom@67:             # fail
terom@67:             raise RSyncCommandFormatError("Restricted path".format())
terom@5: 
terom@5:     if path.startswith('/') :
terom@5:         # direct filesystem path
terom@57:         log.debug("filesystem: %s", path)
terom@5: 
terom@12:         return RSyncFSServer(path)
terom@5: 
terom@5:     elif path.startswith('lvm:') :
terom@49:         _, lvm = path.split(':', 1)
terom@49: 
terom@5:         # LVM LV
terom@5:         try :
terom@49:             if ':' in lvm :
terom@49:                 vg, lv = lvm.split(':', 1)
terom@49: 
terom@49:                 log.warn("old lvm: syntax: lvm:%s; use: lvm:%s/%s", path, vg, lv)
terom@49: 
terom@49:             elif '/' in lvm:
terom@49:                 vg, lv = lvm.split('/', 1)
terom@49: 
terom@49:             else :
terom@49:                 raise RSyncCommandFormatError("Invalid lvm pseudo-path: {lvm}: unrecognized vg/lv separator".format(lvm=lvm))
terom@5: 
terom@5:         except ValueError, e:
terom@49:             raise RSyncCommandFormatError("Invalid lvm pseudo-path: {lvm}: {error}".format(lvm=lvm, error=e))
terom@5:         
terom@43:         # XXX: validate?
terom@57:         log.debug("LVM: %s/%s", vg, lv)
terom@5: 
terom@5:         # open
terom@70:         return RSyncLVMServer(vg, lv, **lvm_opts)
terom@44: 
terom@67:     elif ':' in path and allow_remote :
terom@44:         host, path = path.split(':', 1)
terom@44: 
terom@44:         # remote host
terom@57:         log.debug("remote: %s:%s", host, path)
terom@44: 
terom@44:         return RSyncRemoteServer(host, path)
terom@5:        
terom@5:     else :
terom@5:         # invalid
terom@48:         raise RSyncCommandFormatError("Unrecognized backup path: {path}".format(path=path))
terom@5: