1

#!/usr/bin/python

     2

     3

"""

     4

    Manage rsync --link-dest based snapshots.

     5

     6

    rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current.

     7

     8

    Updates symlink <dst>/current -> <dst>/snapshots/...

     9

    10

    Then archives <dst>/current to <dst>/<period>/<date> using --link-dest.

    11

"""

    12

    14

from pvl.backup import __version__

    15

from pvl.backup import rsync, invoke

    16

    17

import optparse, ConfigParser

    18

import os, os.path, stat

    19

import shutil, glob

    20

import datetime

    21

import logging

    22

    23

log = logging.getLogger('main')

    24

    25

# command-line options, global state

    26

options = None

    27

    28

def parse_options (argv, defaults) :

    29

    """

    30

        Parse command-line arguments.

    31

    """

    32

    33

    parser = optparse.OptionParser(

    34

            prog        = argv[0],

    35

            usage       = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]',

    36

            version     = __version__,

    37

    38

            # module docstring

    39

            # XXX: breaks multi-line descriptions..

    40

            description = __doc__,

    41

    )

    42

    44

    45

    # rsync

    46

    rsync = optparse.OptionGroup(parser, "rsync Options")

    47

    48

    rsync.add_option('--exclude-from',       metavar='FILE',

    49

        help="Read exclude rules from given file")

    50

    51

    rsync.add_option('--include-from',       metavar='FILE',

    52

        help="Read include rules from given file")

    53

    54

    parser.add_option_group(rsync)

    55

    56

    # global

    57

    parser.add_option('--clean-intervals',  action='store_true',

    58

        help="Clean out old interval links")

    59

    60

    parser.add_option('--clean-snapshots',  action='store_true',

    61

        help="Clean out unused snapshots (those not linked to)")

    62

    63

    parser.add_option('--clean',             action='store_true',

    64

        help="Clean out both intervals and snapshots")

    65

    66

    parser.add_option('-n', '--noop',       action='store_true',

    67

        help="Don't actually clean anything")

    68

    69

    #

    71

        help="Load configuration file(s)")

    72

    73

    parser.add_option('-r', '--run',        metavar='NAME',

    74

        help="Run given set of targets, per config [run/...]")

    75

    76

    #

    77

    parser.add_option('-T', '--target',    metavar='PATH',

    78

        help="Target path")

    79

    80

    parser.add_option('-s', '--source',     metavar='RSYNC-PATH', dest='target_source', default=False,

    81

        help="Run target backup from source in rsync-syntax")

    82

    83

    parser.add_option('--interval',         metavar='NAME', action='append', dest='target_intervals',

    84

        help="Run target with given given interval(s)")

    85

    86

    87

    # defaults

    88

    parser.set_defaults(

    89

        config              = [],

    90

        target_intervals    = [],

    91

    )

    92

    parser.set_defaults(**defaults)

    93

    94

    # parse

    95

    options, args = parser.parse_args(argv[1:])

    99

   100

    if options.clean :

   101

        options.clean_intervals = options.clean_snapshots = options.clean

   102

   103

    if options.include_from :

   104

        options.rsync_options['include-from'] = options.include_from

   105

   106

    if options.exclude_from :

   107

        options.rsync_options['exclude-from'] = options.exclude_from

   108

   109

    return options, args

   110

   111

## Configuration

   112

class ConfigError (Exception) :

   113

    pass

   114

   115

def process_config_name (name) :

   116

    """

   117

        Process config file name into python version

   118

    """

   119

   120

    return name.replace('-', '_')

   121

   122

def parse_config (path, defaults) :

   123

    """

   124

        Parse given config file, returning updated set of configs based on given defaults.

   125

    """

   126

   127

    log.debug("loading config: %s", path)

   128

   129

    config = dict(defaults)

   130

    config_file = ConfigParser.RawConfigParser()

   131

    config_file.read([path])

   132

   133

    # handle each section

   134

    for section in config_file.sections() :

   135

        # mangle

   136

        section_name = process_config_name(section)

   137

   138

        log.debug("section: %s", section_name)

   139

   140

        # subsections

   141

        if ':' in section_name :

   142

            # legacy!

   143

            section_path = section_name.split(':')

   144

        else :

   145

            # new! shiny!

   146

            section_path = section_name.split('/')

   147

   148

        # lookup section dict from config

   149

        lookup = config

   150

   151

        # XXX: sections are not in order, so we can't rely on the parent section being created before we handle the sub-section

   152

        for name in section_path :

   153

            # possibly create

   154

            if name not in lookup :

   155

                lookup[name] = {}

   156

   157

            lookup = lookup[name]

   158

   159

        # found dict for this section

   160

        config_section = lookup

   161

   162

        # values

   163

        for name, value in config_file.items(section) :

   164

            # mangle

   165

            name = process_config_name(name)

   166

   167

            log.debug("section: %s: %s = %s", '/'.join(section_path), name, value)

   168

   169

            config_section[name] = value

   170

   171

    log.debug("config: %s", config)

   172

   173

    return config

   174

   175

def config_bool (name, value, strict=True) :

   176

    if value.lower() in ('yes', 'true', '1', 'on') :

   177

        return True

   178

   179

    elif value.lower() in ('no', 'false', '0', 'off') :

   180

        return False

   181

   182

    elif strict :

   183

        raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value))

   184

   185

    else :

   186

        # allow non-boolean values

   187

        return value

   188

   189

def config_int (name, value, default=False) :

   190

    if not value and default is not False:

   191

        # returning default value if one is given

   192

        return default

   193

   194

    try :

   195

        return int(value)

   196

   197

    except ValueError, e:

   198

        raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value))

   199

   200

def config_list (name, value) :

   201

    return value.split()

   202

   203

def walk_symlinks (tree, ignore=False) :

   204

    """

   205

        Walk through all symlinks in given dir, yielding:

   206

   207

            (dirpath, name, target)

   208

   209

        Passes through errors from os.listdir/os.lstat.

   210

    """

   211

   212

    for name in os.listdir(tree) :

   213

        if ignore and name in ignore :

   214

            log.debug("%s: ignore: %s", tree, name)

   215

            continue

   216

   217

        path = os.path.join(tree, name)

   218

   219

        # stat symlink itself

   220

        st = os.lstat(path)

   221

   222

        if stat.S_ISDIR(st.st_mode) :

   223

            # recurse

   224

            log.debug("%s: tree: %s", tree, name)

   225

   226

            for item in walk_symlinks(path) :

   227

                yield item

   228

   229

        elif stat.S_ISLNK(st.st_mode) :

   230

            # found

   231

            target = os.readlink(path)

   232

   233

            log.debug("%s: link: %s -> %s", tree, name, target)

   234

   235

            yield tree, name, target

   236

   237

        else :

   238

            log.debug("%s: skip: %s", tree, name)

   239

   240

   241

class Interval (object) :

   242

    """

   243

        An interval definition.

   244

    """

   245

   246

    @classmethod

   247

    def from_config (cls, options, name,

   248

        format,

   249

   250

        # deprecated

   251

        keep    = None,

   252

    ) :

   253

        if not format :

   254

            # magic to use snapshot name

   255

            _format = None

   256

        else :

   257

            _format = format

   258

   259

        return cls(name, 

   260

            format  = _format, 

   261

            keep    = config_int('keep', keep, default=None),

   262

        )

   263

   264

    @classmethod

   265

    def from_target_config (cls, name, base, arg) :

   266

        if isinstance(arg, dict) :

   267

            # full instance

   268

            return cls(name,

   269

                format  = arg.get('format', base.format if base else None),

   270

                keep    = arg.get('keep', base.keep if base else None),

   271

            )

   272

        else :

   273

            # partial instance with keep

   274

            return cls(name,

   275

                format  = base.format,

   276

                keep    = config_int('keep', arg) if arg else base.keep,

   277

            )

   278

   279

    def __init__ (self, name, format, keep) :

   280

        self.name = name

   281

        self.format = format

   282

        self.keep = keep

   283

   284

    def __str__ (self) :

   285

        return self.name

   286

   287

class SnapshotError (Exception) :

   288

    """

   289

        An error handling Target.snapshot()

   290

    """

   291

   292

    pass

   293

   294

class Target (object) :

   295

    """

   296

        A target run, i.e. a rsync-snapshot destination dir

   297

   298

        [target:...]

   299

    """

   300

   301

    @classmethod

   302

    def config_intervals (cls, name, intervals) :

   303

        for interval, arg in intervals.iteritems() :

   304

            # lookup base from options.intervals

   305

            try :

   306

                base = options.intervals[process_config_name(interval)]

   307

            except KeyError:

   308

                raise ConfigError("Unknown interval for [target/{target}]: {interval}".format(target=name, interval=interval))

   309

   310

            # parse

   311

            yield Interval.from_target_config(interval, base, arg)

   312

   313

    # type() mapping for lvm_options

   314

    LVM_OPTIONS = dict(

   315

        wait    = float,

   316

        size    = str,

   317

    )

   318

   319

    @classmethod

   320

    def from_config (cls, options, name,

   321

        path            = None,

   322

        source          = None,

   323

        enable          = 'no',

   324

        exclude_from    = None,

   325

   326

        # subsections

   327

        intervals       = None,

   328

        rsync_options   = None,

   329

        lvm_options     = {},

   330

    ) :

   331

        if not source and source is not False :

   332

            raise ConfigError("Missing required option: source for [target/{name}]".format(name=name))

   333

   334

        # process lvm opts by LVM_OPTIONS types

   335

        lvm_options = dict((opt, cls.LVM_OPTIONS[opt](value)) for opt, value in lvm_options.iteritems())

   336

   337

        # parse source -> rsync.RSyncServer

   338

        source_path = source

   339

        source = rsync.parse_source(source, lvm_opts=lvm_options)

   340

   341

        log.debug("parse source: %r -> %s", source_path, source)

   342

   343

        # global defaults

   344

        _rsync_options = dict(options.rsync_options)

   345

   346

        if rsync_options :

   347

            # override

   348

            _rsync_options.update([

   349

                # parse

   350

                (option, config_bool(option, value, strict=False)) for option, value in rsync_options.iteritems()

   351

            ])

   352

   353

        if not intervals :

   354

            raise ConfigError("Missing required [target/{name}/intervals]".format(name=name))

   355

   356

        # lookup intervals

   357

        _intervals = list(cls.config_intervals(name, intervals))

   358

   359

        return cls(name, 

   360

            path            = path if path else name,

   361

            source          = source,

   362

            enable          = config_bool('enable', enable),

   363

            intervals       = _intervals,

   364

            rsync_options   = _rsync_options,

   365

            exclude_from    = exclude_from,

   366

        )

   367

   368

    def __init__ (self, name,

   369

        path,

   370

        source, 

   371

        enable          = False, 

   372

        intervals       = [],

   373

        rsync_options   = {},

   374

   375

        # XXX: not implemented?

   376

        exclude_from    = None

   377

    ) :

   378

        self.name = name

   379

   380

        self.path = path

   381

        self.source = source

   382

        self.enable = enable

   383

   384

        self.intervals = intervals

   385

   386

        self.rsync_options = rsync_options

   387

        self.exclude_from = exclude_from

   388

   389

        # this snapshot?

   390

        self.snapshots_dir = os.path.join(self.path, 'snapshots')

   391

   392

        # 'current' symlink

   393

        self.current_path = os.path.join(self.path, 'current')

   394

   395

    def prepare (self, options) :

   396

        """

   397

            Prepare dir for usage

   398

        """

   399

   400

        if not os.path.exists(self.path) :

   401

            raise Exception("Missing target dir: {path}".format(path=self.path))

   402

   403

        if not os.path.exists(self.snapshots_dir) :

   404

            log.warn("Creating snapshots dir: %s", self.snapshots_dir)

   405

            os.mkdir(self.snapshots_dir)

   406

   407

    def snapshot (self, options, now) :

   408

        """

   409

            Perform the rsync from our source to self.snapshot_dir.

   410

   411

            Raises rsync.RsyncError or SnapshotError.

   412

        """

   413

   414

        # new snapshot

   415

        snapshot_name = now.strftime(options.snapshot_format)

   416

        snapshot_path = os.path.join(self.snapshots_dir, snapshot_name)

   417

        temp_path = os.path.join(self.snapshots_dir, 'tmp')

   418

   419

        if os.path.exists(temp_path) :

   420

            raise SnapshotError("Old temp snapshot dir remains, please clean up: {path}".format(path=temp_path))

   421

   422

        # link-dest from current?

   423

        if os.path.exists(self.current_path) :

   424

            # real path to target

   425

            target = os.readlink(self.current_path)

   426

            target_path = os.path.join(os.path.dirname(self.current_path), target)

   427

   428

            log.debug("%s: link-dest: %s", self, target_path)

   429

   430

            # use as link-dest base; hardlinks unchanged files; target directory must be empty

   431

            link_dest = target_path

   432

   433

        else :

   434

            link_dest = None

   435

   436

        # log

   437

        log.info("%s: %s -> %s <- %s", self, self.source, snapshot_path, link_dest)

   438

   439

        # build rsync options

   440

        opts = dict(self.rsync_options)

   441

   442

        if link_dest :

   443

            # rsync links absolute paths..

   444

            opts['link-dest'] = os.path.abspath(link_dest)

   445

   446

        # to tempdir

   447

        log.debug("rsync %s -> %s", self.source, temp_path)

   448

   449

        try :

   450

            # run the rsync.RSyncServer; None as a placeholder will get replaced with the actual source

   451

            self.source.execute(invoke.optargs(**opts), srcdst=(None, temp_path))

   452

   453

        except rsync.RsyncError as ex :

   454

            # XXX:  leaves temp_path in place, which must be removed or cleaned up..

   455

            #       maybe use {snapshot_name}.tmp instead?

   456

            log.warn("%s: rsync failed:", self, exc_info=ex)

   457

   458

            # run() handles this

   459

            raise

   460

   461

        else :

   462

            # move in to final name

   463

            log.debug("rename %s -> %s", temp_path, snapshot_path)

   464

            os.rename(temp_path, snapshot_path)

   465

   466

            return snapshot_name

   467

   468

    def interval (self, options, interval, now, snapshot_name) :

   469

        """

   470

            Update given <interval>/... links for this target, using the given new snapshot

   471

        """

   472

   473

        dir_path = os.path.join(self.path, interval.name)

   474

   475

        if not os.path.exists(dir_path) :

   476

            log.warn("%s/%s: Creating interval dir: %s", self, interval, dir_path)

   477

            os.mkdir(dir_path)

   478

   479

   480

        # name

   481

        if interval.format is None :

   482

            # per-snapshot

   483

            name = snapshot_name

   484

   485

            log.debug("%s: using snapshot_name: %s", interval, name)

   486

   487

        else :

   488

            # by date

   489

            name = now.strftime(interval.format)

   490

   491

            log.debug("%s: using interval.format: %s -> %s", interval, interval.format, name)

   492

   493

        # path

   494