1

#!/usr/bin/python

     2

     3

"""

     4

    Manage rsync --link-dest based snapshots.

     5

     6

    rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current.

     7

     8

    Updates symlink <dst>/current -> <dst>/snapshots/...

     9

    10

    Then archives <dst>/current to <dst>/<period>/<date> using --link-dest.

    11

"""

    12

    13

from pvl.backup import __version__

    14

from pvl.backup import rsync

    15

    16

import optparse, ConfigParser

    17

import os, os.path, stat

    18

import shutil

    19

import datetime

    20

import logging

    21

    22

log = logging.getLogger()

    23

    24

# command-line options, global state

    25

options = None

    26

    27

def parse_options (argv, defaults) :

    28

    """

    29

        Parse command-line arguments.

    30

    """

    31

    32

    parser = optparse.OptionParser(

    33

            prog        = argv[0],

    34

            usage       = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]',

    35

            version     = __version__,

    36

    37

            # module docstring

    38

            # XXX: breaks multi-line descriptions..

    39

            description = __doc__,

    40

    )

    41

    42

    # logging

    43

    general = optparse.OptionGroup(parser, "General Options")

    44

    45

    general.add_option('-q', '--quiet',      dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")

    46

    general.add_option('-v', '--verbose',    dest='loglevel', action='store_const', const=logging.INFO,  help="More output")

    47

    general.add_option('-D', '--debug',      dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")

    48

    49

    parser.add_option_group(general)

    50

    51

    # rsync

    52

    rsync = optparse.OptionGroup(parser, "rsync Options")

    53

    54

    rsync.add_option('--exclude-from',       metavar='FILE',

    55

        help="Read exclude rules from given file")

    56

    57

    rsync.add_option('--include-from',       metavar='FILE',

    58

        help="Read include rules from given file")

    59

    60

    parser.add_option_group(rsync)

    61

    62

    # global

    63

    parser.add_option('--clean-intervals',  action='store_true',

    64

        help="Clean out old interval links")

    65

    66

    parser.add_option('--clean-snapshots',  action='store_true',

    67

        help="Clean out unused snapshots (those not linked to)")

    68

    69

    parser.add_option('--clean',             action='store_true',

    70

        help="Clean out both intervals and snapshots")

    71

    72

    parser.add_option('-n', '--dry-run',    action='store_true',

    73

        help="Don't actually clean anything")

    74

    75

    #

    76

    parser.add_option('-c', '--config',     metavar='FILE',

    77

        help="Load configuration file")

    78

    79

    parser.add_option('-r', '--run',        metavar='NAME',

    80

        help="Run given set of targets, per config [run/...]")

    81

    82

    #

    83

    parser.add_option('-T', '--target',    metavar='PATH',

    84

        help="Target path")

    85

    86

    parser.add_option('-s', '--source',     metavar='RSYNC-PATH', dest='target_source', default=False,

    87

        help="Run target backup from source in rsync-syntax")

    88

    89

    parser.add_option('--interval',         metavar='NAME', action='append', dest='target_intervals',

    90

        help="Run target with given given interval(s)")

    91

    92

    93

    # defaults

    94

    parser.set_defaults(

    95

        loglevel            = logging.INFO,

    96

    97

        target_intervals    = [],

    98

    )

    99

    parser.set_defaults(**defaults)

   100

   101

   102

    # parse

   103

    options, args = parser.parse_args(argv[1:])

   104

   105

    # configure

   106

    logging.basicConfig(

   107

        format  = '%(processName)s: %(name)s: %(levelname)s %(funcName)s : %(message)s',

   108

        level   = options.loglevel,

   109

    )

   110

   111

    if options.clean :

   112

        options.clean_intervals = options.clean_snapshots = options.clean

   113

   114

    if options.include_from :

   115

        options.rsync_options['include-from'] = options.include_from

   116

   117

    if options.exclude_from :

   118

        options.rsync_options['exclude-from'] = options.exclude_from

   119

   120

    return options, args

   121

   122

## Configuration

   123

class ConfigError (Exception) :

   124

    pass

   125

   126

def process_config_name (name) :

   127

    """

   128

        Process config file name into python version

   129

    """

   130

   131

    return name.replace('-', '_')

   132

   133

def parse_config (path, defaults) :

   134

    """

   135

        Parse given config file

   136

    """

   137

   138

    log.debug("loading config: %s", path)

   139

   140

    config = dict(defaults)

   141

    config_file = ConfigParser.RawConfigParser()

   142

    config_file.read([path])

   143

   144

    # handle each section

   145

    for section in config_file.sections() :

   146

        # mangle

   147

        section_name = process_config_name(section)

   148

   149

        log.debug("section: %s", section_name)

   150

   151

        # subsections

   152

        if ':' in section_name :

   153

            # legacy!

   154

            section_path = section_name.split(':')

   155

        else :

   156

            # new! shiny!

   157

            section_path = section_name.split('/')

   158

   159

        # lookup section dict from config

   160

        lookup = config

   161

   162

        # XXX: sections are not in order, so we can't rely on the parent section being created before we handle the sub-section

   163

        for name in section_path :

   164

            # possibly create

   165

            if name not in lookup :

   166

                lookup[name] = {}

   167

   168

            lookup = lookup[name]

   169

   170

        # found dict for this section

   171

        config_section = lookup

   172

   173

        # values

   174

        for name, value in config_file.items(section) :

   175

            # mangle

   176

            name = process_config_name(name)

   177

   178

            log.debug("section: %s: %s = %s", '/'.join(section_path), name, value)

   179

   180

            config_section[name] = value

   181

   182

    log.debug("config: %s", config)

   183

   184

    return config

   185

   186

def config_bool (name, value, strict=True) :

   187

    if value.lower() in ('yes', 'true', '1', 'on') :

   188

        return True

   189

   190

    elif value.lower() in ('no', 'false', '0', 'off') :

   191

        return False

   192

   193

    elif strict :

   194

        raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value))

   195

   196

    else :

   197

        # allow non-boolean values

   198

        return value

   199

   200

def config_int (name, value, default=False) :

   201

    if not value and default is not False:

   202

        # returning default value if one is given

   203

        return default

   204

   205

    try :

   206

        return int(value)

   207

   208

    except ValueError, e:

   209

        raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value))

   210

   211

def config_list (name, value) :

   212

    return value.split()

   213

   214

def walk_symlinks (tree, ignore=False) :

   215

    """

   216

        Walk through all symlinks in given dir, yielding:

   217

   218

            (dirpath, name, target)

   219

   220

        Passes through errors from os.listdir/os.lstat.

   221

    """

   222

   223

    for name in os.listdir(tree) :

   224

        if ignore and name in ignore :

   225

            log.debug("%s: ignore: %s", tree, name)

   226

            continue

   227

   228

        path = os.path.join(tree, name)

   229

   230

        # stat symlink itself

   231

        st = os.lstat(path)

   232

   233

        if stat.S_ISDIR(st.st_mode) :

   234

            # recurse

   235

            log.debug("%s: tree: %s", tree, name)

   236

   237

            for item in walk_symlinks(path) :

   238

                yield item

   239

   240

        elif stat.S_ISLNK(st.st_mode) :

   241

            # found

   242

            target = os.readlink(path)

   243

   244

            log.debug("%s: link: %s -> %s", tree, name, target)

   245

   246

            yield tree, name, target

   247

   248

        else :

   249

            log.debug("%s: skip: %s", tree, name)

   250

   251

   252

class Interval (object) :

   253

    """

   254

        An interval definition.

   255

    """

   256

   257

    @classmethod

   258

    def from_config (cls, options, name,

   259

        format,

   260

   261

        # deprecated

   262

        keep    = None,

   263

    ) :

   264

        if not format :

   265

            # magic to use snapshot name

   266

            _format = None

   267

        else :

   268

            _format = format

   269

   270

        return cls(name, 

   271

            format  = _format, 

   272

            keep    = config_int('keep', keep, default=None),

   273

        )

   274

   275

    @classmethod

   276

    def from_target_config (cls, name, base, arg) :

   277

        if isinstance(arg, dict) :

   278

            # full instance

   279

            return cls(name,

   280

                format  = arg.get('format', base.format if base else None),

   281

                keep    = arg.get('keep', base.keep if base else None),

   282

            )

   283

        else :

   284

            # partial instance with keep

   285

            return cls(name,

   286

                format  = base.format,

   287

                keep    = config_int('keep', arg) if arg else base.keep,

   288

            )

   289

   290

    def __init__ (self, name, format, keep) :

   291

        self.name = name

   292

        self.format = format

   293

        self.keep = keep

   294

   295

    def __str__ (self) :

   296

        return self.name

   297

   298

class Target (object) :

   299

    """

   300

        A target run, i.e. a rsync-snapshot destination dir

   301

   302

        [target:...]

   303

    """

   304

   305

    @classmethod

   306

    def config_intervals (cls, name, intervals) :

   307

        for interval, arg in intervals.iteritems() :

   308

            # lookup base from options.intervals

   309

            try :

   310

                base = options.intervals[process_config_name(interval)]

   311

            except KeyError:

   312

                raise ConfigError("Unknown interval for [target/{target}]: {interval}".format(target=name, interval=interval))

   313

   314

            # parse

   315

            yield Interval.from_target_config(interval, base, arg)

   316

   317

    @classmethod

   318

    def from_config (cls, options, name,

   319

        path            = False,

   320

        source          = None,

   321

        enable          = 'no',

   322

        exclude_from    = None,

   323

   324

        # subsections

   325

        intervals       = None,

   326

        rsync_options   = None,

   327

    ) :

   328

        if not source and source is not False :

   329

            raise ConfigError("Missing required option: source for [target/{name}]".format(name=name))

   330

   331

        # global defaults

   332

        _rsync_options = dict(options.rsync_options)

   333

   334

        if rsync_options :

   335

            # override

   336

            _rsync_options.update([

   337

                # parse

   338

                (option, config_bool(option, value, strict=False)) for option, value in rsync_options.iteritems()

   339

            ])

   340

   341

        if not intervals :

   342

            raise ConfigError("Missing required [target/{name}/intervals]".format(name=name))

   343

   344

        # lookup intervals

   345

        _intervals = list(cls.config_intervals(name, intervals))

   346

   347

        return cls(name, 

   348

            path            = path if path else name,

   349

            source          = source,

   350

            enable          = config_bool('enable', enable),

   351

            intervals       = _intervals,

   352

            rsync_options   = _rsync_options,

   353

            exclude_from    = exclude_from,

   354

        )

   355

   356

    def __init__ (self, name,

   357

        path,

   358

        source, 

   359

        enable          = False, 

   360

        intervals       = [],

   361

        rsync_options   = {},

   362

        exclude_from    = None

   363

    ) :

   364

        self.name = name

   365

   366

        self.path = path

   367

        self.source = source

   368

        self.enable = enable

   369

   370

        self.intervals = intervals

   371

   372

        self.rsync_options = rsync_options

   373

        self.exclude_from = exclude_from

   374

   375

        # this snapshot?

   376

        self.snapshots_dir = os.path.join(self.path, 'snapshots')

   377

   378

        # 'current' symlink

   379

        self.current_path = os.path.join(self.path, 'current')

   380

   381

    def prepare (self, options) :

   382

        """

   383

            Prepare dir for usage

   384

        """

   385

   386

        if not os.path.exists(self.path) :

   387

            raise Exception("Missing target dir: {path}".format(path=self.path))

   388

   389

        if not os.path.exists(self.snapshots_dir) :

   390

            log.warn("Creating snapshots dir: %s", self.snapshots_dir)

   391

            os.mkdir(self.snapshots_dir)

   392

   393

    def snapshot (self, options, now) :

   394

        """

   395

            Perform the rsync from our source to self.snapshot_dir.

   396

   397

            XXX: allocate snapshot_name here?

   398

        """

   399

   400

        # new snapshot

   401

        snapshot_name = now.strftime(options.snapshot_format)

   402

        snapshot_path = os.path.join(self.snapshots_dir, snapshot_name)

   403

        temp_path = os.path.join(self.snapshots_dir, 'tmp')

   404

   405

        if os.path.exists(temp_path) :

   406

            raise Exception("Old temp snapshot dir remains, please clean up: {path}".format(path=temp_path))

   407

   408

        log.info("Perform main snapshot: %s -> %s", self.source, snapshot_path)

   409

   410

        # build rsync options

   411

        opts = dict(self.rsync_options)

   412

   413

        if os.path.exists(self.current_path) :

   414

            # real path to target

   415

            target = os.readlink(self.current_path)

   416

            target_path = os.path.join(os.path.dirname(self.current_path), target)

   417

            target_abs = os.path.abspath(target_path)

   418

   419

            log.info("Using current -> %s as base", target_path)

   420

   421

            # use as link-dest base; hardlinks unchanged files; target directory must be empty

   422

            # rsync links absolute paths..

   423

            opts['link-dest'] = target_abs

   424

   425

        # go

   426

        log.debug("rsync %s -> %s", self.source, temp_path)

   427

        rsync.rsync(self.source, temp_path, **opts)

   428

   429

        # move in to final name

   430

        log.debug("rename %s -> %s", temp_path, snapshot_path)

   431

        os.rename(temp_path, snapshot_path)

   432

   433

        return snapshot_name

   434

   435

    def update_interval (self, options, interval, now, snapshot_name) :

   436

        """

   437

            Update given <interval>/... links for this target, using the given new snapshot

   438

        """

   439

   440

        dir_path = os.path.join(self.path, interval.name)

   441

   442

        if not os.path.exists(dir_path) :

   443

            log.warn("Creating interval dir: %s", dir_path)

   444

            os.mkdir(dir_path)

   445

   446

   447

        # name

   448

        if interval.format is None :

   449

            # per-snapshot

   450

            name = snapshot_name

   451

   452

            log.debug("%s: using snapshot_name: %s", interval, name)

   453

   454

        else :

   455

            # by date

   456

            name = now.strftime(interval.format)

   457

   458

            log.debug("%s: using interval.format: %s -> %s", interval, interval.format, name)

   459

   460

        # path

   461

        path_name = os.path.join(interval.name, name)

   462

        path = os.path.join(self.path, path_name)

   463

   464

        log.debug("%s: processing %s", interval, path_name)

   465

   466

        # already there?

   467

        if os.path.exists(path) :

   468

            target = os.readlink(path)

   469

   470

            log.info("%s: Keeping existing: %s -> %s", interval, name, target)

   471

   472

        else :

   473

            # update

   474

            target = os.path.join('..', 'snapshots', snapshot_name)

   475

   476

            log.info("%s: Updating: %s -> %s", interval, name, target)

   477

            log.debug("%s -> %s", path, target)

   478

   479

            os.symlink(target, path)

   480

   481

   482

    def clean_interval (self, options, interval) :

   483

        """

   484

            Clean out given <interval>/... dir for this target.

   485

        """

   486

   487

        # path

   488

        dir_path = os.path.join(self.path, interval.name)

   489

   490

        if not os.path.exists(dir_path) :

   491

            log.warn("%s: Skipping, no interval dir: %s", interval, dir_path)

   492

            return

   493

   494

        # configured

   495

        keep = interval.keep

   496

   497

        if not keep :