1

#!/usr/bin/python

     2

     3

"""

     4

    Manage rsync --link-dest based snapshots.

     5

     6

    rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current.

     7

     8

    Updates symlink <dst>/current -> <dst>/snapshots/...

     9

    10

    Then archives <dst>/current to <dst>/<period>/<date> using --link-dest.

    11

"""

    12

    13

from pvl.backup import __version__

    14

from pvl.backup import rsync

    15

    16

import optparse, ConfigParser

    17

import os, os.path, stat

    18

import shutil

    19

import datetime

    20

import logging

    21

    22

log = logging.getLogger()

    23

    24

# command-line options, global state

    25

options = None

    26

    27

def parse_options (argv, defaults) :

    28

    """

    29

        Parse command-line arguments.

    30

    """

    31

    32

    parser = optparse.OptionParser(

    33

            prog        = argv[0],

    34

            usage       = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]',

    35

            version     = __version__,

    36

    37

            # module docstring

    38

            # XXX: breaks multi-line descriptions..

    39

            description = __doc__,

    40

    )

    41

    42

    # logging

    43

    general = optparse.OptionGroup(parser, "General Options")

    44

    45

    general.add_option('-q', '--quiet',      dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")

    46

    general.add_option('-v', '--verbose',    dest='loglevel', action='store_const', const=logging.INFO,  help="More output")

    47

    general.add_option('-D', '--debug',      dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")

    48

    49

    parser.add_option_group(general)

    50

    51

    # rsync

    52

    rsync = optparse.OptionGroup(parser, "rsync Options")

    53

    54

    rsync.add_option('--exclude-from',       metavar='FILE',

    55

        help="Read exclude rules from given file")

    56

    57

    rsync.add_option('--include-from',       metavar='FILE',

    58

        help="Read include rules from given file")

    59

    60

    parser.add_option_group(rsync)

    61

    62

    # global

    63

    parser.add_option('--clean-intervals',  action='store_true',

    64

        help="Clean out old interval links")

    65

    66

    parser.add_option('--clean-snapshots',  action='store_true',

    67

        help="Clean out unused snapshots (those not linked to)")

    68

    69

    parser.add_option('--clean',             action='store_true',

    70

        help="Clean out both intervals and snapshots")

    71

    72

    parser.add_option('-n', '--dry-run',    action='store_true',

    73

        help="Don't actually clean anything")

    74

    75

    #

    76

    parser.add_option('-c', '--config',     metavar='FILE',

    77

        help="Load configuration file")

    78

    79

    #

    80

    parser.add_option('-T', '--target',    metavar='PATH',

    81

        help="Target path")

    82

    83

    parser.add_option('-s', '--source',     metavar='RSYNC-PATH', dest='target_source', default=False,

    84

        help="Run target backup from source in rsync-syntax")

    85

    86

    parser.add_option('--interval',         metavar='NAME', action='append', dest='target_intervals',

    87

        help="Run target with given given interval(s)")

    88

    89

    90

    # defaults

    91

    parser.set_defaults(

    92

        loglevel            = logging.INFO,

    93

    94

        target_intervals    = [],

    95

    )

    96

    parser.set_defaults(**defaults)

    97

    98

    99

    # parse

   100

    options, args = parser.parse_args(argv[1:])

   101

   102

    # configure

   103

    logging.basicConfig(

   104

        format  = '%(processName)s: %(name)s: %(levelname)s %(funcName)s : %(message)s',

   105

        level   = options.loglevel,

   106

    )

   107

   108

    if options.clean :

   109

        options.clean_intervals = options.clean_snapshots = options.clean

   110

   111

    if options.include_from :

   112

        options.rsync_options['include-from'] = options.include_from

   113

   114

    if options.exclude_from :

   115

        options.rsync_options['exclude-from'] = options.exclude_from

   116

   117

    return options, args

   118

   119

## Configuration

   120

class ConfigError (Exception) :

   121

    pass

   122

   123

def process_config_name (name) :

   124

    """

   125

        Process config file name into python version

   126

    """

   127

   128

    return name.replace('-', '_')

   129

   130

def parse_config (path, defaults) :

   131

    """

   132

        Parse given config file

   133

    """

   134

   135

    log.debug("loading config: %s", path)

   136

   137

    config = dict(defaults)

   138

    config_file = ConfigParser.RawConfigParser()

   139

    config_file.read([path])

   140

   141

    # handle each section

   142

    for section in config_file.sections() :

   143

        # mangle

   144

        section_name = process_config_name(section)

   145

   146

        log.debug("section: %s", section_name)

   147

   148

        # subsections

   149

        if ':' in section_name :

   150

            # legacy!

   151

            section_path = section_name.split(':')

   152

        else :

   153

            # new! shiny!

   154

            section_path = section_name.split('/')

   155

   156

        # lookup section dict from config

   157

        lookup = config

   158

   159

        # XXX: sections are not in order, so we can't rely on the parent section being created before we handle the sub-section

   160

        for name in section_path :

   161

            # possibly create

   162

            if name not in lookup :

   163

                lookup[name] = {}

   164

   165

            lookup = lookup[name]

   166

   167

        # found dict for this section

   168

        config_section = lookup

   169

   170

        # values

   171

        for name, value in config_file.items(section) :

   172

            # mangle

   173

            name = process_config_name(name)

   174

   175

            log.debug("section: %s: %s = %s", '/'.join(section_path), name, value)

   176

   177

            config_section[name] = value

   178

   179

    log.debug("config: %s", config)

   180

   181

    return config

   182

   183

def config_bool (name, value, strict=True) :

   184

    if value.lower() in ('yes', 'true', '1', 'on') :

   185

        return True

   186

   187

    elif value.lower() in ('no', 'false', '0', 'off') :

   188

        return False

   189

   190

    elif strict :

   191

        raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value))

   192

   193

    else :

   194

        # allow non-boolean values

   195

        return value

   196

   202

    try :

   203

        return int(value)

   204

   205

    except ValueError, e:

   206

        raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value))

   207

   208

def config_list (name, value) :

   209

    return value.split()

   210

   211

def walk_symlinks (tree, ignore=False) :

   212

    """

   213

        Walk through all symlinks in given dir, yielding:

   214

   215

            (dirpath, name, target)

   216

   217

        Passes through errors from os.listdir/os.lstat.

   218

    """

   219

   220

    for name in os.listdir(tree) :

   221

        if ignore and name in ignore :

   222

            log.debug("%s: ignore: %s", tree, name)

   223

            continue

   224

   225

        path = os.path.join(tree, name)

   226

   227

        # stat symlink itself

   228

        st = os.lstat(path)

   229

   230

        if stat.S_ISDIR(st.st_mode) :

   231

            # recurse

   232

            log.debug("%s: tree: %s", tree, name)

   233

   234

            for item in walk_symlinks(path) :

   235

                yield item

   236

   237

        elif stat.S_ISLNK(st.st_mode) :

   238

            # found

   239

            target = os.readlink(path)

   240

   241

            log.debug("%s: link: %s -> %s", tree, name, target)

   242

   243

            yield tree, name, target

   244

   245

        else :

   246

            log.debug("%s: skip: %s", tree, name)

   247

   248

   249

class Interval (object) :

   250

    """

   251

        An interval definition.

   252

    """

   253

   254

    @classmethod

   255

    def from_config (cls, options, name,

   256

        format,

   260

    ) :

   261

        if not format :

   262

            # magic to use snapshot name

   263

            _format = None

   264

        else :

   265

            _format = format

   266

   267

        return cls(name, 

   268

            format  = _format, 

   270

        )

   271

   272

    @classmethod

   273

    def from_target_config (cls, name, base, arg) :

   274

        if isinstance(arg, dict) :

   275

            # full instance

   276

            return cls(name,

   277

                format  = arg.get('format', base.format if base else None),

   278

                keep    = arg.get('keep', base.keep if base else None),

   279

            )

   280

        else :

   281

            # partial instance with keep

   282

            return cls(name,

   283

                format  = base.format,

   284

                keep    = config_int('keep', arg) if arg else base.keep,

   285

            )

   286

   287

    def __init__ (self, name, format, keep) :

   288

        self.name = name

   289

        self.format = format

   290

        self.keep = keep

   291

   292

    def __str__ (self) :

   293

        return self.name

   294

   295

class Target (object) :

   296

    """

   297

        A target run, i.e. a rsync-snapshot destination dir

   298

   299

        [target:...]

   300

    """

   301

   302

    @classmethod

   303

    def config_intervals (cls, name, intervals) :

   304

        for interval, arg in intervals.iteritems() :

   305

            # lookup base from options.intervals

   306

            try :

   307

                base = options.intervals[interval]

   308

            except KeyError:

   309

                raise ConfigError("Unknown interval for [target/{target}]: {interval}".format(target=name, interval=interval))

   310

   311

            # parse

   312

            yield Interval.from_target_config(interval, base, arg)

   313

   314

    @classmethod

   315

    def from_config (cls, options, name,

   316

        path            = False,

   317

        source          = None,

   318

        enable          = 'no',

   319

        exclude_from    = None,

   320

   321

        # subsections

   322

        intervals       = None,

   323

        rsync_options   = None,

   324

    ) :

   325

        if not source and source is not False :

   326

            raise ConfigError("Missing required option: source for [target/{name}]".format(name=name))

   327

   328

        # global defaults

   329

        _rsync_options = dict(options.rsync_options)

   330

   331

        if rsync_options :

   332

            # override

   333

            _rsync_options.update([

   334

                # parse

   335

                (option, config_bool(option, value, strict=False)) for option, value in rsync_options.iteritems()

   336

            ])

   337

   341

        # lookup intervals

   342

        _intervals = list(cls.config_intervals(name, intervals))

   343

   344

        return cls(name, 

   345

            path            = path if path else name,

   346

            source          = source,

   347

            enable          = config_bool('enable', enable),

   348

            intervals       = _intervals,

   349

            rsync_options   = _rsync_options,

   350

            exclude_from    = exclude_from,

   351

        )

   352

   353

    def __init__ (self, name,

   354

        path,

   355

        source, 

   356

        enable          = False, 

   357

        intervals       = [],

   358

        rsync_options   = {},

   359

        exclude_from    = None

   360

    ) :

   361

        self.name = name

   362

   363

        self.path = path

   364

        self.source = source

   365

        self.enable = enable

   366

   367

        self.intervals = intervals

   368

   369

        self.rsync_options = rsync_options

   370

        self.exclude_from = exclude_from

   371

   372

        # this snapshot?

   373

        self.snapshots_dir = os.path.join(self.path, 'snapshots')

   374

   375

        # 'current' symlink

   376

        self.current_path = os.path.join(self.path, 'current')

   377

   378

    def prepare (self, options) :

   379

        """

   380

            Prepare dir for usage

   381

        """

   382

   383

        if not os.path.exists(self.path) :

   384

            raise Exception("Missing target dir: {path}".format(path=self.path))

   385

   386

        if not os.path.exists(self.snapshots_dir) :

   387

            log.warn("Creating snapshots dir: %s", self.snapshots_dir)

   388

            os.mkdir(self.snapshots_dir)

   389

   390

    def snapshot (self, options, now) :

   391

        """

   392

            Perform the rsync from our source to self.snapshot_dir.

   393

   394

            XXX: allocate snapshot_name here?

   395

        """

   396

   397

        # new snapshot

   398

        snapshot_name = now.strftime(options.snapshot_format)

   399

        snapshot_path = os.path.join(self.snapshots_dir, snapshot_name)

   400

        temp_path = os.path.join(self.snapshots_dir, 'tmp')

   401

   402

        if os.path.exists(temp_path) :

   403

            raise Exception("Old temp snapshot dir remains, please clean up: {path}".format(path=temp_path))

   404

   405

        log.info("Perform main snapshot: %s -> %s", self.source, snapshot_path)

   406

   407

        # build rsync options

   408

        opts = dict(self.rsync_options)

   409

   410

        if os.path.exists(self.current_path) :

   411

            # real path to target

   412

            target = os.readlink(self.current_path)

   413

            target_path = os.path.join(os.path.dirname(self.current_path), target)

   414

            target_abs = os.path.abspath(target_path)

   415

   416

            log.info("Using current -> %s as base", target_path)

   417

   418

            # use as link-dest base; hardlinks unchanged files; target directory must be empty

   419

            # rsync links absolute paths..

   420

            opts['link-dest'] = target_abs

   421

   422

        # go

   423

        log.debug("rsync %s -> %s", self.source, temp_path)

   424

        rsync.rsync(self.source, temp_path, **opts)

   425

   426

        # move in to final name

   427

        log.debug("rename %s -> %s", temp_path, snapshot_path)

   428

        os.rename(temp_path, snapshot_path)

   429

   430

        return snapshot_name

   431

   432

    def update_interval (self, options, interval, now, snapshot_name) :

   433

        """

   434

            Update given <interval>/... links for this target, using the given new snapshot

   435

        """

   436

   437

        dir_path = os.path.join(self.path, interval.name)

   438

   439

        if not os.path.exists(dir_path) :

   440

            log.warn("Creating interval dir: %s", dir_path)

   441

            os.mkdir(dir_path)

   442

   443

   444

        # name

   445

        if interval.format is None :

   446

            # per-snapshot

   447

            name = snapshot_name

   448

   449

            log.debug("%s: using snapshot_name: %s", interval, name)

   450

   451

        else :

   452

            # by date

   453

            name = now.strftime(interval.format)

   454

   455

            log.debug("%s: using interval.format: %s -> %s", interval, interval.format, name)

   456

   457

        # path

   458

        path_name = os.path.join(interval.name, name)

   459

        path = os.path.join(self.path, path_name)

   460

   461

        log.debug("%s: processing %s", interval, path_name)

   462

   463

        # already there?

   464

        if os.path.exists(path) :

   465

            target = os.readlink(path)

   466

   467

            log.info("%s: Keeping existing: %s -> %s", interval, name, target)

   468

   469

        else :

   470

            # update

   471

            target = os.path.join('..', 'snapshots', snapshot_name)

   472

   473

            log.info("%s: Updating: %s -> %s", interval, name, target)

   474

            log.debug("%s -> %s", path, target)

   475

   476

            os.symlink(target, path)

   477

   478

   479

    def clean_interval (self, options, interval) :

   480

        """

   481

            Clean out given <interval>/... dir for this target.

   482

        """

   483

   484

        # path

   485

        dir_path = os.path.join(self.path, interval.name)

   486

   487

        if not os.path.exists(dir_path) :

   488

            log.warn("%s: Skipping, no interval dir: %s", interval, dir_path)

   489

            return

   490

   491

        # configured

   492

        keep = interval.keep

   493