1

#!/usr/bin/python

     2

     3

"""

     4

    Manage rsync --link-dest based snapshots.

     5

     6

    rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current.

     7

     8

    Updates symlink <dst>/current -> <dst>/snapshots/...

     9

    10

    Then archives <dst>/current to <dst>/<period>/<date> using --link-dest.

    11

"""

    12

    13

from pvl.backup import __version__

    14

from pvl.backup import rsync, invoke

    15

    16

import optparse, ConfigParser

    17

import os, os.path, stat

    18

import shutil, glob

    19

import datetime

    20

import logging

    21

    22

log = logging.getLogger('main')

    23

    24

# command-line options, global state

    25

options = None

    26

    27

def parse_options (argv, defaults) :

    28

    """

    29

        Parse command-line arguments.

    30

    """

    31

    32

    parser = optparse.OptionParser(

    33

            prog        = argv[0],

    34

            usage       = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]',

    35

            version     = __version__,

    36

    37

            # module docstring

    38

            # XXX: breaks multi-line descriptions..

    39

            description = __doc__,

    40

    )

    41

    42

    # logging

    43

    general = optparse.OptionGroup(parser, "General Options")

    44

    45

    general.add_option('-q', '--quiet',      dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")

    46

    general.add_option('-v', '--verbose',    dest='loglevel', action='store_const', const=logging.INFO,  help="More output")

    47

    general.add_option('-D', '--debug',      dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")

    48

    49

    parser.add_option_group(general)

    50

    51

    # rsync

    52

    rsync = optparse.OptionGroup(parser, "rsync Options")

    53

    54

    rsync.add_option('--exclude-from',       metavar='FILE',

    55

        help="Read exclude rules from given file")

    56

    57

    rsync.add_option('--include-from',       metavar='FILE',

    58

        help="Read include rules from given file")

    59

    60

    parser.add_option_group(rsync)

    61

    62

    # global

    63

    parser.add_option('--clean-intervals',  action='store_true',

    64

        help="Clean out old interval links")

    65

    66

    parser.add_option('--clean-snapshots',  action='store_true',

    67

        help="Clean out unused snapshots (those not linked to)")

    68

    69

    parser.add_option('--clean',             action='store_true',

    70

        help="Clean out both intervals and snapshots")

    71

    72

    parser.add_option('-n', '--noop',       action='store_true',

    73

        help="Don't actually clean anything")

    74

    75

    #

    76

    parser.add_option('-c', '--config',     metavar='FILE/DIR', action='append',    # multi

    77

        help="Load configuration file(s)")

    78

    79

    parser.add_option('-r', '--run',        metavar='NAME',

    80

        help="Run given set of targets, per config [run/...]")

    81

    82

    #

    83

    parser.add_option('-T', '--target',    metavar='PATH',

    84

        help="Target path")

    85

    86

    parser.add_option('-s', '--source',     metavar='RSYNC-PATH', dest='target_source', default=False,

    87

        help="Run target backup from source in rsync-syntax")

    88

    89

    parser.add_option('--interval',         metavar='NAME', action='append', dest='target_intervals',

    90

        help="Run target with given given interval(s)")

    91

    92

    93

    # defaults

    94

    parser.set_defaults(

    95

        loglevel            = logging.INFO,

    96

    97

        config              = [],

    98

        target_intervals    = [],

    99

    )

   100

    parser.set_defaults(**defaults)

   101

   102

   103

    # parse

   104

    options, args = parser.parse_args(argv[1:])

   105

   106

    # configure

   107

    logging.basicConfig(

   108

        format  = argv[0] + ': %(name)s: %(levelname)s %(funcName)s : %(message)s',

   109

        level   = options.loglevel,

   110

    )

   111

   112

    if options.clean :

   113

        options.clean_intervals = options.clean_snapshots = options.clean

   114

   115

    if options.include_from :

   116

        options.rsync_options['include-from'] = options.include_from

   117

   118

    if options.exclude_from :

   119

        options.rsync_options['exclude-from'] = options.exclude_from

   120

   121

    return options, args

   122

   123

## Configuration

   124

class ConfigError (Exception) :

   125

    pass

   126

   127

def process_config_name (name) :

   128

    """

   129

        Process config file name into python version

   130

    """

   131

   132

    return name.replace('-', '_')

   133

   134

def parse_config (path, defaults) :

   135

    """

   136

        Parse given config file, returning updated set of configs based on given defaults.

   137

    """

   138

   139

    log.debug("loading config: %s", path)

   140

   141

    config = dict(defaults)

   142

    config_file = ConfigParser.RawConfigParser()

   143

    config_file.read([path])

   144

   145

    # handle each section

   146

    for section in config_file.sections() :

   147

        # mangle

   148

        section_name = process_config_name(section)

   149

   150

        log.debug("section: %s", section_name)

   151

   152

        # subsections

   153

        if ':' in section_name :

   154

            # legacy!

   155

            section_path = section_name.split(':')

   156

        else :

   157

            # new! shiny!

   158

            section_path = section_name.split('/')

   159

   160

        # lookup section dict from config

   161

        lookup = config

   162

   163

        # XXX: sections are not in order, so we can't rely on the parent section being created before we handle the sub-section

   164

        for name in section_path :

   165

            # possibly create

   166

            if name not in lookup :

   167

                lookup[name] = {}

   168

   169

            lookup = lookup[name]

   170

   171

        # found dict for this section

   172

        config_section = lookup

   173

   174

        # values

   175

        for name, value in config_file.items(section) :

   176

            # mangle

   177

            name = process_config_name(name)

   178

   179

            log.debug("section: %s: %s = %s", '/'.join(section_path), name, value)

   180

   181

            config_section[name] = value

   182

   183

    log.debug("config: %s", config)

   184

   185

    return config

   186

   187

def config_bool (name, value, strict=True) :

   188

    if value.lower() in ('yes', 'true', '1', 'on') :

   189

        return True

   190

   191

    elif value.lower() in ('no', 'false', '0', 'off') :

   192

        return False

   193

   194

    elif strict :

   195

        raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value))

   196

   197

    else :

   198

        # allow non-boolean values

   199

        return value

   200

   201

def config_int (name, value, default=False) :

   202

    if not value and default is not False:

   203

        # returning default value if one is given

   204

        return default

   205

   206

    try :

   207

        return int(value)

   208

   209

    except ValueError, e:

   210

        raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value))

   211

   212

def config_list (name, value) :

   213

    return value.split()

   214

   215

def walk_symlinks (tree, ignore=False) :

   216

    """

   217

        Walk through all symlinks in given dir, yielding:

   218

   219

            (dirpath, name, target)

   220

   221

        Passes through errors from os.listdir/os.lstat.

   222

    """

   223

   224

    for name in os.listdir(tree) :

   225

        if ignore and name in ignore :

   226

            log.debug("%s: ignore: %s", tree, name)

   227

            continue

   228

   229

        path = os.path.join(tree, name)

   230

   231

        # stat symlink itself

   232

        st = os.lstat(path)

   233

   234

        if stat.S_ISDIR(st.st_mode) :

   235

            # recurse

   236

            log.debug("%s: tree: %s", tree, name)

   237

   238

            for item in walk_symlinks(path) :

   239

                yield item

   240

   241

        elif stat.S_ISLNK(st.st_mode) :

   242

            # found

   243

            target = os.readlink(path)

   244

   245

            log.debug("%s: link: %s -> %s", tree, name, target)

   246

   247

            yield tree, name, target

   248

   249

        else :

   250

            log.debug("%s: skip: %s", tree, name)

   251

   252

   253

class Interval (object) :

   254

    """

   255

        An interval definition.

   256

    """

   257

   258

    @classmethod

   259

    def from_config (cls, options, name,

   260

        format,

   261

   262

        # deprecated

   263

        keep    = None,

   264

    ) :

   265

        if not format :

   266

            # magic to use snapshot name

   267

            _format = None

   268

        else :

   269

            _format = format

   270

   271

        return cls(name, 

   272

            format  = _format, 

   273

            keep    = config_int('keep', keep, default=None),

   274

        )

   275

   276

    @classmethod

   277

    def from_target_config (cls, name, base, arg) :

   278

        if isinstance(arg, dict) :

   279

            # full instance

   280

            return cls(name,

   281

                format  = arg.get('format', base.format if base else None),

   282

                keep    = arg.get('keep', base.keep if base else None),

   283

            )

   284

        else :

   285

            # partial instance with keep

   286

            return cls(name,

   287

                format  = base.format,

   288

                keep    = config_int('keep', arg) if arg else base.keep,

   289

            )

   290

   291

    def __init__ (self, name, format, keep) :

   292

        self.name = name

   293

        self.format = format

   294

        self.keep = keep

   295

   296

    def __str__ (self) :

   297

        return self.name

   298

   306

class Target (object) :

   307

    """

   308

        A target run, i.e. a rsync-snapshot destination dir

   309

   310

        [target:...]

   311

    """

   312

   313

    @classmethod

   314

    def config_intervals (cls, name, intervals) :

   315

        for interval, arg in intervals.iteritems() :

   316

            # lookup base from options.intervals

   317

            try :

   318

                base = options.intervals[process_config_name(interval)]

   319

            except KeyError:

   320

                raise ConfigError("Unknown interval for [target/{target}]: {interval}".format(target=name, interval=interval))

   321

   322

            # parse

   323

            yield Interval.from_target_config(interval, base, arg)

   324

   325

    # type() mapping for lvm_options

   326

    LVM_OPTIONS = dict(

   327

        wait    = float,

   328

        size    = str,

   329

    )

   330

   331

    @classmethod

   332

    def from_config (cls, options, name,

   333

        path            = None,

   334

        source          = None,

   335

        enable          = 'no',

   336

        exclude_from    = None,

   337

   338

        # subsections

   339

        intervals       = None,

   340

        rsync_options   = None,

   341

        lvm_options     = {},

   342

    ) :

   343

        if not source and source is not False :

   344

            raise ConfigError("Missing required option: source for [target/{name}]".format(name=name))

   345

   346

        # process lvm opts by LVM_OPTIONS types

   347

        lvm_options = dict((opt, cls.LVM_OPTIONS[opt](value)) for opt, value in lvm_options.iteritems())

   348

   349

        # parse source -> rsync.RSyncServer

   350

        source_path = source

   351

        source = rsync.parse_source(source, lvm_opts=lvm_options)

   352

   353

        log.debug("parse source: %r -> %s", source_path, source)

   354

   355

        # global defaults

   356

        _rsync_options = dict(options.rsync_options)

   357

   358

        if rsync_options :

   359

            # override

   360

            _rsync_options.update([

   361

                # parse

   362

                (option, config_bool(option, value, strict=False)) for option, value in rsync_options.iteritems()

   363

            ])

   364

   365

        if not intervals :

   366

            raise ConfigError("Missing required [target/{name}/intervals]".format(name=name))

   367

   368

        # lookup intervals

   369

        _intervals = list(cls.config_intervals(name, intervals))

   370

   371

        return cls(name, 

   372

            path            = path if path else name,

   373

            source          = source,

   374

            enable          = config_bool('enable', enable),

   375

            intervals       = _intervals,

   376

            rsync_options   = _rsync_options,

   377

            exclude_from    = exclude_from,

   378

        )

   379

   380

    def __init__ (self, name,

   381

        path,

   382

        source, 

   383

        enable          = False, 

   384

        intervals       = [],

   385

        rsync_options   = {},

   386

   387

        # XXX: not implemented?

   388

        exclude_from    = None

   389

    ) :

   390

        self.name = name

   391

   392

        self.path = path

   393

        self.source = source

   394

        self.enable = enable

   395

   396

        self.intervals = intervals

   397

   398

        self.rsync_options = rsync_options

   399

        self.exclude_from = exclude_from

   400

   401

        # this snapshot?

   402

        self.snapshots_dir = os.path.join(self.path, 'snapshots')

   403

   404

        # 'current' symlink

   405

        self.current_path = os.path.join(self.path, 'current')

   406

   407

    def prepare (self, options) :

   408

        """

   409

            Prepare dir for usage

   410

        """

   411

   412

        if not os.path.exists(self.path) :

   413

            raise Exception("Missing target dir: {path}".format(path=self.path))

   414

   415

        if not os.path.exists(self.snapshots_dir) :

   416

            log.warn("Creating snapshots dir: %s", self.snapshots_dir)

   417

            os.mkdir(self.snapshots_dir)

   418

   419

    def snapshot (self, options, now) :

   420

        """

   421

            Perform the rsync from our source to self.snapshot_dir.

   424

        """

   425

   426

        # new snapshot

   427

        snapshot_name = now.strftime(options.snapshot_format)

   428

        snapshot_path = os.path.join(self.snapshots_dir, snapshot_name)

   429

        temp_path = os.path.join(self.snapshots_dir, 'tmp')

   430

   431

        if os.path.exists(temp_path) :

   433

   434

        # link-dest from current?

   435

        if os.path.exists(self.current_path) :

   436

            # real path to target

   437

            target = os.readlink(self.current_path)

   438

            target_path = os.path.join(os.path.dirname(self.current_path), target)

   439

   440

            log.debug("%s: link-dest: %s", self, target_path)

   441

   442

            # use as link-dest base; hardlinks unchanged files; target directory must be empty

   443

            link_dest = target_path

   444

   445

        else :

   446

            link_dest = None

   447

   448

        # log

   449

        log.info("%s: %s -> %s <- %s", self, self.source, snapshot_path, link_dest)

   450

   451

        # build rsync options

   452

        opts = dict(self.rsync_options)

   453

   454

        if link_dest :

   455

            # rsync links absolute paths..

   456

            opts['link-dest'] = os.path.abspath(link_dest)

   457

   458

        # to tempdir

   459

        log.debug("rsync %s -> %s", self.source, temp_path)

   460

   464

   469

   479

   480

    def interval (self, options, interval, now, snapshot_name) :

   481

        """

   482

            Update given <interval>/... links for this target, using the given new snapshot

   483

        """

   484

   485

        dir_path = os.path.join(self.path, interval.name)

   486

   487

        if not os.path.exists(dir_path) :

   488

            log.warn("%s/%s: Creating interval dir: %s", self, interval, dir_path)

   489

            os.mkdir(dir_path)

   490

   491

   492

        # name

   493

        if interval.format is None :

   494

            # per-snapshot