1

#!/usr/bin/python

     2

     3

"""

     4

    Manage rsync --link-dest based snapshots.

     5

     6

    rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current.

     7

     8

    Updates symlink <dst>/current -> <dst>/snapshots/...

     9

    10

    Then archives <dst>/current to <dst>/<period>/<date> using --link-dest.

    11

"""

    12

    13

from pvl.backup import rsync

    14

    15

import optparse, ConfigParser

    16

import os, os.path, stat

    17

import shutil

    18

import datetime

    19

import logging

    20

    21

log = logging.getLogger()

    22

    23

# command-line options

    24

options = None

    25

    26

def parse_options (argv, defaults) :

    27

    """

    28

        Parse command-line arguments.

    29

    """

    30

    31

    parser = optparse.OptionParser(

    32

            prog        = argv[0],

    33

            usage       = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]',

    34

    35

            # module docstring

    36

            # XXX: breaks multi-line descriptions..

    37

            description = __doc__,

    38

    )

    39

    40

    # logging

    41

    general = optparse.OptionGroup(parser, "General Options")

    42

    43

    general.add_option('-q', '--quiet',      dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")

    44

    general.add_option('-v', '--verbose',    dest='loglevel', action='store_const', const=logging.INFO,  help="More output")

    45

    general.add_option('-D', '--debug',      dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")

    46

    47

    parser.add_option_group(general)

    48

    49

    # rsync

    50

    rsync = optparse.OptionGroup(parser, "rsync Options")

    51

    52

    rsync.add_option('--exclude-from',       metavar='FILE',

    53

        help="Read exclude rules from given file")

    54

    55

    rsync.add_option('--include-from',       metavar='FILE',

    56

        help="Read include rules from given file")

    57

    58

    parser.add_option_group(rsync)

    59

    60

    # global

    61

    parser.add_option('--clean-intervals',  action='store_true',

    62

        help="Clean out old interval links")

    63

    64

    parser.add_option('--clean-snapshots',  action='store_true',

    65

        help="Clean out unused snapshots (those not linked to)")

    66

    67

    parser.add_option('--clean',             action='store_true',

    68

        help="Clean out both intervals and snapshots")

    69

    70

    parser.add_option('-n', '--dry-run',    action='store_true',

    71

        help="Don't actually clean anything")

    72

    73

    #

    74

    parser.add_option('-c', '--config',     metavar='FILE',

    75

        help="Load configuration file")

    76

    77

    #

    78

    parser.add_option('-T', '--target',    metavar='PATH',

    79

        help="Target path")

    80

    81

    parser.add_option('-s', '--source',     metavar='RSYNC-PATH', dest='target_source', default=False,

    82

        help="Run target backup from source in rsync-syntax")

    83

    84

    parser.add_option('--interval',         metavar='NAME', action='append', dest='target_intervals',

    85

        help="Run target with given given interval(s)")

    86

    87

    88

    # defaults

    89

    parser.set_defaults(

    90

        loglevel            = logging.WARNING,

    91

    92

        target_intervals    = [],

    93

    )

    94

    parser.set_defaults(**defaults)

    95

    96

    97

    # parse

    98

    options, args = parser.parse_args(argv[1:])

    99

   100

    # configure

   101

    logging.basicConfig(

   102

        format  = '%(processName)s: %(name)s: %(levelname)s %(funcName)s : %(message)s',

   103

        level   = options.loglevel,

   104

    )

   105

   106

    if options.clean :

   107

        options.clean_intervals = options.clean_snapshots = options.clean

   108

   109

    if options.include_from :

   110

        options.rsync_options['include-from'] = options.include_from

   111

   112

    if options.exclude_from :

   113

        options.rsync_options['exclude-from'] = options.exclude_from

   114

   115

    return options, args

   116

   117

def process_config_name (name) :

   118

    """

   119

        Process config file name into python version

   120

    """

   121

   122

    return name.replace('-', '_')

   123

   124

def parse_config (path, defaults) :

   125

    """

   126

        Parse given config file

   127

    """

   128

   129

    log.debug("loading config: %s", path)

   130

   131

    config = dict(defaults)

   132

    config_file = ConfigParser.RawConfigParser()

   133

    config_file.read([path])

   134

   135

    # handle each section

   136

    for section in config_file.sections() :

   137

        # mangle

   138

        section_name = process_config_name(section)

   139

   140

        log.debug("section: %s", section_name)

   141

   142

        # subsections

   143

        section_path = section_name.split(':')

   144

   145

        # lookup dict

   146

        lookup = config

   147

   148

        for name in section_path :

   149

            if name not in lookup :

   150

                lookup[name] = {}

   151

   152

            lookup = lookup[name]

   153

   154

        # found dict for this section

   155

        config_section = lookup

   156

   157

        # values

   158

        for name, value in config_file.items(section) :

   159

            # mangle

   160

            name = process_config_name(name)

   161

   162

            log.debug("section: %s: %s = %s", '/'.join(section_path), name, value)

   163

   164

            config_section[name] = value

   165

   166

    log.debug("config: %s", config)

   167

   168

    return config

   169

   170

def config_bool (name, value) :

   171

    if value.lower() in ('yes', 'true', '1', 'on') :

   172

        return True

   173

   174

    elif value.lower() in ('no', 'false', '0', 'off') :

   175

        return False

   176

   177

    else :

   178

        raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value))

   179

   180

def config_int (name, value) :

   181

    try :

   182

        return int(value)

   183

   184

    except ValueError, e:

   185

        raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value))

   186

   187

def config_list (name, value) :

   188

    return value.split()

   189

   190

def walk_symlinks (tree, ignore=False) :

   191

    """

   192

        Walk through all symlinks in given dir, yielding:

   193

   194

            (dirpath, name, target)

   195

   196

        Passes through errors from os.listdir/os.lstat.

   197

    """

   198

   199

    for name in os.listdir(tree) :

   200

        if ignore and name in ignore :

   201

            log.debug("%s: ignore: %s", tree, name)

   202

            continue

   203

   204

        path = os.path.join(tree, name)

   205

   206

        # stat symlink itself

   207

        st = os.lstat(path)

   208

   209

        if stat.S_ISDIR(st.st_mode) :

   210

            # recurse

   211

            log.debug("%s: tree: %s", tree, name)

   212

   213

            for item in walk_symlinks(path) :

   214

                yield item

   215

   216

        elif stat.S_ISLNK(st.st_mode) :

   217

            # found

   218

            target = os.readlink(path)

   219

   220

            log.debug("%s: link: %s -> %s", tree, name, target)

   221

   222

            yield tree, name, target

   223

   224

        else :

   225

            log.debug("%s: skip: %s", tree, name)

   226

   227

   228

class Interval (object) :

   229

    """

   230

        An interval definition.

   231

    """

   232

   233

    @classmethod

   234

    def from_config (cls, options, name,

   235

        format,

   236

        keep,

   237

    ) :

   238

        if not format :

   239

            # magic to use snapshot name

   240

            _format = None

   241

        else :

   242

            _format = format

   243

   244

        return cls(name, 

   245

            format  = _format, 

   246

            keep    = config_int('keep', keep),

   247

        )

   248

   249

    @classmethod

   250

    def from_target_config (cls, name, base, arg) :

   251

        if isinstance(arg, dict) :

   252

            # full instance

   253

            return cls(name,

   254

                format  = arg.get('format', base.format if base else None),

   255

                keep    = arg.get('keep', base.keep if base else None),

   256

            )

   257

        else :

   258

            # partial instance with keep

   259

            return cls(name,

   260

                format  = base.format,

   261

                keep    = config_int('keep', arg) if arg else base.keep,

   262

            )

   263

   264

    def __init__ (self, name, format, keep) :

   265

        self.name = name

   266

        self.format = format

   267

        self.keep = keep

   268

   269

    def __str__ (self) :

   270

        return self.name

   271

   272

class Target (object) :

   273

    """

   274

        A target run, i.e. a rsync-snapshot destination dir

   275

   276

        [target:...]

   277

    """

   278

   279

    @classmethod

   280

    def from_config (cls, options, name,

   281

        path            = False,

   282

        source          = None,

   283

        enable          = 'no',

   284

        exclude_from    = None,

   285

   286

        # subsections

   287

        intervals       = None,

   288

        rsync_options   = None,

   289

    ) :

   290

        if not source and source is not False :

   291

            raise ConfigError("Missing required option: source for [target:{name}]".format(name=name))

   292

   293

        # global defaults

   294

        _rsync_options = dict(options.rsync_options)

   295

   296

        if rsync_options :

   297

            # override

   298

            _rsync_options.update(rsync_options)

   299

   300

        # lookup intervals

   301

        _intervals = [

   302

            (

   303

                # lookup base from options.intervals

   304

                Interval.from_target_config(name, options.intervals.get(name), arg)

   305

            ) for name, arg in intervals.iteritems()

   306

        ]

   307

   308

        return cls(name, 

   309

            path            = path if path else name,

   310

            source          = source,

   311

            enable          = config_bool('enable', enable),

   312

            intervals       = _intervals,

   313

            rsync_options   = _rsync_options,

   314

            exclude_from    = exclude_from,

   315

        )

   316

   317

    def __init__ (self, name,

   318

        path,

   319

        source, 

   320

        enable          = False, 

   321

        intervals       = [],

   322

        rsync_options   = {},

   323

        exclude_from    = None

   324

    ) :

   325

        self.name = name

   326

   327

        self.path = path

   328

        self.source = source

   329

        self.enable = enable

   330

   331

        self.intervals = intervals

   332

   333

        self.rsync_options = rsync_options

   334

        self.exclude_from = exclude_from

   335

   336

        # this snapshot?

   337

        self.snapshots_dir = os.path.join(self.path, 'snapshots')

   338

   339

        # 'current' symlink

   340

        self.current_path = os.path.join(self.path, 'current')

   341

   342

    def prepare (self, options) :

   343

        """

   344

            Prepare dir for usage

   345

        """

   346

   347

        if not os.path.exists(self.path) :

   348

            raise Exception("Missing target dir: {path}".format(path=self.path))

   349

   350

        if not os.path.exists(self.snapshots_dir) :

   351

            log.warn("Creating snapshots dir: %s", self.snapshots_dir)

   352

            os.mkdir(self.snapshots_dir)

   353

   354

    def snapshot (self, options, now) :

   355

        """

   356

            Perform the rsync from our source to self.snapshot_dir.

   357

   358

            XXX: allocate snapshot_name here?

   359

        """

   360

   361

        # new snapshot

   362

        snapshot_name = now.strftime(options.snapshot_format)

   363

        snapshot_path = os.path.join(self.snapshots_dir, snapshot_name)

   364

        temp_path = os.path.join(self.snapshots_dir, 'tmp')

   365

   366

        if os.path.exists(temp_path) :

   367

            raise Exception("Old temp snapshot dir remains, please clean up: {path}".format(path=temp_path))

   368

   369

        log.info("Perform main snapshot: %s", snapshot_path)

   370

   371

        # build rsync options

   372

        opts = dict(self.rsync_options)

   373

   374

        if os.path.exists(self.current_path) :

   375

            # real path to target

   376

            target = os.readlink(self.current_path)

   377

            target_path = os.path.join(os.path.dirname(self.current_path), target)

   378

            target_abs = os.path.abspath(target_path)

   379

   380

            log.info("Using current -> %s as base", target_path)

   381

   382

            # use as link-dest base; hardlinks unchanged files; target directory must be empty

   383

            # rsync links absolute paths..

   384

            opts['link-dest'] = target_abs

   385

   386

        # go

   387

        log.debug("rsync %s -> %s", self.source, temp_path)

   388

        rsync.rsync(self.source, temp_path, **opts)

   389

   390

        # move in to final name

   391

        log.debug("rename %s -> %s", temp_path, snapshot_path)

   392

        os.rename(temp_path, snapshot_path)

   393

   394

        return snapshot_name

   395

   396

    def update_interval (self, options, interval, now, snapshot_name) :

   397

        """

   398

            Update given <interval>/... links for this target, using the given new snapshot

   399

        """

   400

   401

        dir_path = os.path.join(self.path, interval.name)

   402

   403

        if not os.path.exists(dir_path) :

   404

            log.warn("Creating interval dir: %s", dir_path)

   405

            os.mkdir(dir_path)

   406

   407

   408

        # name

   409

        if interval.format is None :

   410

            # per-snapshot

   411

            name = snapshot_name

   412

   413

            log.debug("%s: using snapshot_name: %s", interval, name)

   414

   415

        else :

   416

            # by date

   417

            name = now.strftime(interval.format)

   418

   419

            log.debug("%s: using interval.format: %s -> %s", interval, interval.format, name)

   420

   421

        # path

   422

        path_name = os.path.join(interval.name, name)

   423

        path = os.path.join(self.path, path_name)

   424

   425

        log.debug("%s: processing %s", interval, path_name)

   426

   427

        # already there?

   428

        if os.path.exists(path) :

   429

            target = os.readlink(path)

   430

   431

            log.info("%s: Found existing: %s -> %s", interval, name, target)

   432

   433

        else :

   434

            # update

   435

            target = os.path.join('..', 'snapshots', snapshot_name)

   436

   437

            log.info("%s: Updating: %s -> %s", interval, name, target)

   438

            log.debug("%s -> %s", path, target)

   439

   440

            os.symlink(target, path)

   441

   442

   443

    def clean_interval (self, options, interval) :

   444

        """

   445

            Clean out given <interval>/... dir for this target.

   446

        """

   447

   448

        # path

   449

        dir_path = os.path.join(self.path, interval.name)

   450

   451

        if not os.path.exists(dir_path) :

   452

            log.warn("%s: Skipping, no interval dir: %s", interval, dir_path)

   453

            return

   454

   455

        # configured

   456

        keep = interval.keep

   457

   458

        # clean?

   459

        items = os.listdir(dir_path)

   460

        items.sort()

   461

   462

        log.info("%s: Have %d / %d items", interval, len(items), keep)

   463

        log.debug("%s: items: %s", interval, ' '.join(items))

   464

   465

        if len(items) > keep :

   466

            # clean out

   467

            clean = items[keep:]

   468

   469

            log.info("%s: Cleaning out %d items", interval, len(clean))

   470

            log.debug("%s: cleaning out: %s", interval, ' '.join(clean))

   471

   472

            for item in clean :

   473

                path = os.path.join(dir_path, item)

   474

   475

                log.info("%s: Clean: %s", interval, path)

   476

   477

                os.unlink(path)

   478

   479

   480

    def clean_snapshots (self, options) :

   481

        """

   482

            Clean out all snapshots for this target not linked to from within our root.

   483

   484

            Fails without doing anything if unable to read the destination dir.

   485

        """

   486

   487

        # real path to snapshots

   488

        snapshots_path = os.path.realpath(os.path.abspath(self.snapshots_dir))

   489

        log.debug("real snapshots_path: %s", snapshots_path)

   490

   491

        # set of found targets

   492

        found = set()

   493

   494

        # walk all symlinks

   495

        for dirpath, name, target in walk_symlinks(self.path, ignore=set(['snapshots'])) :

   496

            # target dir

   497

            target_path = os.path.realpath(os.path.join(dirpath, target))

   498

            target_dir = os.path.dirname(target_path)

   499

            target_name = os.path.basename(target_path)

   500