1

#!/usr/bin/python

     2

     3

"""

     4

    Manage rsync --link-dest based snapshots.

     5

     6

    rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current.

     7

     8

    Updates symlink <dst>/current -> <dst>/snapshots/...

     9

    10

    Then archives <dst>/current to <dst>/<period>/<date> using --link-dest.

    11

"""

    12

    13

from pvl.backup import rsync

    14

    15

import optparse, ConfigParser

    16

import os, os.path, stat

    17

import shutil

    18

import datetime

    19

import logging

    20

    21

log = logging.getLogger()

    22

    23

# command-line options

    24

options = None

    25

    26

def parse_options (argv, defaults) :

    27

    """

    28

        Parse command-line arguments.

    29

    """

    30

    31

    parser = optparse.OptionParser(

    32

            prog        = argv[0],

    33

            usage       = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]',

    34

    35

            # module docstring

    36

            # XXX: breaks multi-line descriptions..

    37

            description = __doc__,

    38

    )

    39

    40

    # logging

    41

    general = optparse.OptionGroup(parser, "General Options")

    42

    43

    general.add_option('-q', '--quiet',      dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")

    44

    general.add_option('-v', '--verbose',    dest='loglevel', action='store_const', const=logging.INFO,  help="More output")

    45

    general.add_option('-D', '--debug',      dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")

    46

    47

    parser.add_option_group(general)

    48

    49

    # rsync

    50

    rsync = optparse.OptionGroup(parser, "rsync Options")

    51

    52

    rsync.add_option('--exclude-from',       metavar='FILE',

    53

        help="Read exclude rules from given file")

    54

    55

    rsync.add_option('--include-from',       metavar='FILE',

    56

        help="Read include rules from given file")

    57

    58

    parser.add_option_group(rsync)

    59

    60

    # global

    61

    parser.add_option('--clean-intervals',  action='store_true',

    62

        help="Clean out old interval links")

    63

    64

    parser.add_option('--clean-snapshots',  action='store_true',

    65

        help="Clean out unused snapshots (those not linked to)")

    66

    67

    parser.add_option('--clean',             action='store_true',

    68

        help="Clean out both intervals and snapshots")

    69

    70

    parser.add_option('-n', '--dry-run',    action='store_true',

    71

        help="Don't actually clean anything")

    72

    73

    #

    74

    parser.add_option('-c', '--config',     metavar='FILE',

    75

        help="Load configuration file")

    76

    77

    #

    78

    parser.add_option('-T', '--target',    metavar='PATH',

    79

        help="Target path")

    80

    81

    parser.add_option('-s', '--source',     metavar='RSYNC-PATH', dest='target_source', default=False,

    82

        help="Run target backup from source in rsync-syntax")

    83

    84

    parser.add_option('--interval',         metavar='NAME', action='append', dest='target_intervals',

    85

        help="Run target with given given interval(s)")

    86

    87

    88

    # defaults

    89

    parser.set_defaults(

    90

        loglevel            = logging.INFO,

    91

    92

        target_intervals    = [],

    93

    )

    94

    parser.set_defaults(**defaults)

    95

    96

    97

    # parse

    98

    options, args = parser.parse_args(argv[1:])

    99

   100

    # configure

   101

    logging.basicConfig(

   102

        format  = '%(processName)s: %(name)s: %(levelname)s %(funcName)s : %(message)s',

   103

        level   = options.loglevel,

   104

    )

   105

   106

    if options.clean :

   107

        options.clean_intervals = options.clean_snapshots = options.clean

   108

   109

    if options.include_from :

   110

        options.rsync_options['include-from'] = options.include_from

   111

   112

    if options.exclude_from :

   113

        options.rsync_options['exclude-from'] = options.exclude_from

   114

   115

    return options, args

   116

   117

## Configuration

   118

class ConfigError (Exception) :

   119

    pass

   120

   121

def process_config_name (name) :

   122

    """

   123

        Process config file name into python version

   124

    """

   125

   126

    return name.replace('-', '_')

   127

   128

def parse_config (path, defaults) :

   129

    """

   130

        Parse given config file

   131

    """

   132

   133

    log.debug("loading config: %s", path)

   134

   135

    config = dict(defaults)

   136

    config_file = ConfigParser.RawConfigParser()

   137

    config_file.read([path])

   138

   139

    # handle each section

   140

    for section in config_file.sections() :

   141

        # mangle

   142

        section_name = process_config_name(section)

   143

   144

        log.debug("section: %s", section_name)

   145

   146

        # subsections

   147

        section_path = section_name.split(':')

   148

   150

        lookup = config

   151

   153

        for name in section_path :

   155

            if name not in lookup :

   156

                lookup[name] = {}

   158

            lookup = lookup[name]

   160

        # found dict for this section

   161

        config_section = lookup

   162

   163

        # values

   164

        for name, value in config_file.items(section) :

   165

            # mangle

   166

            name = process_config_name(name)

   167

   168

            log.debug("section: %s: %s = %s", '/'.join(section_path), name, value)

   169

   170

            config_section[name] = value

   171

   172

    log.debug("config: %s", config)

   173

   174

    return config

   175

   176

def config_bool (name, value, strict=True) :

   177

    if value.lower() in ('yes', 'true', '1', 'on') :

   178

        return True

   179

   180

    elif value.lower() in ('no', 'false', '0', 'off') :

   181

        return False

   182

   183

    elif strict :

   184

        raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value))

   185

   186

    else :

   187

        # allow non-boolean values

   188

        return value

   189

   190

def config_int (name, value) :

   191

    try :

   192

        return int(value)

   193

   194

    except ValueError, e:

   195

        raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value))

   196

   197

def config_list (name, value) :

   198

    return value.split()

   199

   200

def walk_symlinks (tree, ignore=False) :

   201

    """

   202

        Walk through all symlinks in given dir, yielding:

   203

   204

            (dirpath, name, target)

   205

   206

        Passes through errors from os.listdir/os.lstat.

   207

    """

   208

   209

    for name in os.listdir(tree) :

   210

        if ignore and name in ignore :

   211

            log.debug("%s: ignore: %s", tree, name)

   212

            continue

   213

   214

        path = os.path.join(tree, name)

   215

   216

        # stat symlink itself

   217

        st = os.lstat(path)

   218

   219

        if stat.S_ISDIR(st.st_mode) :

   220

            # recurse

   221

            log.debug("%s: tree: %s", tree, name)

   222

   223

            for item in walk_symlinks(path) :

   224

                yield item

   225

   226

        elif stat.S_ISLNK(st.st_mode) :

   227

            # found

   228

            target = os.readlink(path)

   229

   230

            log.debug("%s: link: %s -> %s", tree, name, target)

   231

   232

            yield tree, name, target

   233

   234

        else :

   235

            log.debug("%s: skip: %s", tree, name)

   236

   237

   238

class Interval (object) :

   239

    """

   240

        An interval definition.

   241

    """

   242

   243

    @classmethod

   244

    def from_config (cls, options, name,

   245

        format,

   246

        keep,

   247

    ) :

   248

        if not format :

   249

            # magic to use snapshot name

   250

            _format = None

   251

        else :

   252

            _format = format

   253

   254

        return cls(name, 

   255

            format  = _format, 

   256

            keep    = config_int('keep', keep),

   257

        )

   258

   259

    @classmethod

   260

    def from_target_config (cls, name, base, arg) :

   261

        if isinstance(arg, dict) :

   262

            # full instance

   263

            return cls(name,

   264

                format  = arg.get('format', base.format if base else None),

   265

                keep    = arg.get('keep', base.keep if base else None),

   266

            )

   267

        else :

   268

            # partial instance with keep

   269

            return cls(name,

   270

                format  = base.format,

   271

                keep    = config_int('keep', arg) if arg else base.keep,

   272

            )

   273

   274

    def __init__ (self, name, format, keep) :

   275

        self.name = name

   276

        self.format = format

   277

        self.keep = keep

   278

   279

    def __str__ (self) :

   280

        return self.name

   281

   282

class Target (object) :

   283

    """

   284

        A target run, i.e. a rsync-snapshot destination dir

   285

   286

        [target:...]

   287

    """

   288

   289

    @classmethod

   290

    def from_config (cls, options, name,

   291

        path            = False,

   292

        source          = None,

   293

        enable          = 'no',

   294

        exclude_from    = None,

   295

   296

        # subsections

   297

        intervals       = None,

   298

        rsync_options   = None,

   299

    ) :

   300

        if not source and source is not False :

   301

            raise ConfigError("Missing required option: source for [target:{name}]".format(name=name))

   302

   303

        # global defaults

   304

        _rsync_options = dict(options.rsync_options)

   305

   306

        if rsync_options :

   307

            # override

   308

            _rsync_options.update([

   309

                # parse

   310

                (option, config_bool(option, value, strict=False)) for option, value in rsync_options.iteritems()

   311

            ])

   312

   313

        # lookup intervals

   314

        _intervals = [

   315

            (

   316

                # lookup base from options.intervals

   317

                Interval.from_target_config(name, options.intervals.get(name), arg)

   318

            ) for name, arg in intervals.iteritems()

   319

        ]

   320

   321

        return cls(name, 

   322

            path            = path if path else name,

   323

            source          = source,

   324

            enable          = config_bool('enable', enable),

   325

            intervals       = _intervals,

   326

            rsync_options   = _rsync_options,

   327

            exclude_from    = exclude_from,

   328

        )

   329

   330

    def __init__ (self, name,

   331

        path,

   332

        source, 

   333

        enable          = False, 

   334

        intervals       = [],

   335

        rsync_options   = {},

   336

        exclude_from    = None

   337

    ) :

   338

        self.name = name

   339

   340

        self.path = path

   341

        self.source = source

   342

        self.enable = enable

   343

   344

        self.intervals = intervals

   345

   346

        self.rsync_options = rsync_options

   347

        self.exclude_from = exclude_from

   348

   349

        # this snapshot?

   350

        self.snapshots_dir = os.path.join(self.path, 'snapshots')

   351

   352

        # 'current' symlink

   353

        self.current_path = os.path.join(self.path, 'current')

   354

   355

    def prepare (self, options) :

   356

        """

   357

            Prepare dir for usage

   358

        """

   359

   360

        if not os.path.exists(self.path) :

   361

            raise Exception("Missing target dir: {path}".format(path=self.path))

   362

   363

        if not os.path.exists(self.snapshots_dir) :

   364

            log.warn("Creating snapshots dir: %s", self.snapshots_dir)

   365

            os.mkdir(self.snapshots_dir)

   366

   367

    def snapshot (self, options, now) :

   368

        """

   369

            Perform the rsync from our source to self.snapshot_dir.

   370

   371

            XXX: allocate snapshot_name here?

   372

        """

   373

   374

        # new snapshot

   375

        snapshot_name = now.strftime(options.snapshot_format)

   376

        snapshot_path = os.path.join(self.snapshots_dir, snapshot_name)

   377

        temp_path = os.path.join(self.snapshots_dir, 'tmp')

   378

   379

        if os.path.exists(temp_path) :

   380

            raise Exception("Old temp snapshot dir remains, please clean up: {path}".format(path=temp_path))

   381

   383

   384

        # build rsync options

   385

        opts = dict(self.rsync_options)

   386

   387

        if os.path.exists(self.current_path) :

   388

            # real path to target

   389

            target = os.readlink(self.current_path)

   390

            target_path = os.path.join(os.path.dirname(self.current_path), target)

   391

            target_abs = os.path.abspath(target_path)

   392

   393

            log.info("Using current -> %s as base", target_path)

   394

   395

            # use as link-dest base; hardlinks unchanged files; target directory must be empty

   396

            # rsync links absolute paths..

   397

            opts['link-dest'] = target_abs

   398

   399

        # go

   400

        log.debug("rsync %s -> %s", self.source, temp_path)

   401

        rsync.rsync(self.source, temp_path, **opts)

   402

   403

        # move in to final name

   404

        log.debug("rename %s -> %s", temp_path, snapshot_path)

   405

        os.rename(temp_path, snapshot_path)

   406

   407

        return snapshot_name

   408

   409

    def update_interval (self, options, interval, now, snapshot_name) :

   410

        """

   411

            Update given <interval>/... links for this target, using the given new snapshot

   412

        """

   413

   414

        dir_path = os.path.join(self.path, interval.name)

   415

   416

        if not os.path.exists(dir_path) :

   417

            log.warn("Creating interval dir: %s", dir_path)

   418

            os.mkdir(dir_path)

   419

   420

   421

        # name

   422

        if interval.format is None :

   423

            # per-snapshot

   424

            name = snapshot_name

   425

   426

            log.debug("%s: using snapshot_name: %s", interval, name)

   427

   428

        else :

   429

            # by date

   430

            name = now.strftime(interval.format)

   431

   432

            log.debug("%s: using interval.format: %s -> %s", interval, interval.format, name)

   433

   434

        # path

   435

        path_name = os.path.join(interval.name, name)

   436

        path = os.path.join(self.path, path_name)

   437

   438

        log.debug("%s: processing %s", interval, path_name)

   439

   440

        # already there?

   441

        if os.path.exists(path) :

   442

            target = os.readlink(path)

   443

   444

            log.info("%s: Found existing: %s -> %s", interval, name, target)

   445

   446

        else :

   447

            # update

   448

            target = os.path.join('..', 'snapshots', snapshot_name)

   449

   450

            log.info("%s: Updating: %s -> %s", interval, name, target)

   451

            log.debug("%s -> %s", path, target)

   452

   453

            os.symlink(target, path)

   454

   455

   456

    def clean_interval (self, options, interval) :

   457

        """

   458

            Clean out given <interval>/... dir for this target.

   459

        """

   460

   461

        # path

   462

        dir_path = os.path.join(self.path, interval.name)

   463

   464

        if not os.path.exists(dir_path) :

   465

            log.warn("%s: Skipping, no interval dir: %s", interval, dir_path)

   466

            return

   467

   468

        # configured

   469

        keep = interval.keep

   470

   471

        # clean?

   472

        items = os.listdir(dir_path)

   473

        items.sort()

   474

   475

        log.info("%s: Have %d / %d items", interval, len(items), keep)

   476

        log.debug("%s: items: %s", interval, ' '.join(items))

   477

   478

        if len(items) > keep :

   479

            # clean out

   480

            clean = items[keep:]

   481

   482

            log.info("%s: Cleaning out %d items", interval, len(clean))

   483

            log.debug("%s: cleaning out: %s", interval, ' '.join(clean))

   484

   485

            for item in clean :

   486

                path = os.path.join(dir_path, item)

   487

   488

                log.info("%s: Clean: %s", interval, path)

   489

   490

                os.unlink(path)

   491

   492

   493

    def clean_snapshots (self, options) :

   494

        """

   495

            Clean out all snapshots for this target not linked to from within our root.

   496

   497

            Fails without doing anything if unable to read the destination dir.

   498

        """

   499