1

#!/usr/bin/python

     2

     3

"""

     4

    Manage rsync --link-dest based snapshots.

     5

     6

    rsync's from <src> to <dst>/snapshots/YYYY-MM-DD-HH-MM-SS using --link-dest <dst>/current.

     7

     8

    Updates symlink <dst>/current -> <dst>/snapshots/...

     9

    10

    Then archives <dst>/current to <dst>/<period>/<date> using --link-dest.

    11

"""

    12

    13

from pvl.backup import __version__

    14

from pvl.backup import rsync

    15

    16

import optparse, ConfigParser

    17

import os, os.path, stat

    18

import shutil

    19

import datetime

    20

import logging

    21

    22

log = logging.getLogger()

    23

    24

# command-line options

    25

options = None

    26

    27

def parse_options (argv, defaults) :

    28

    """

    29

        Parse command-line arguments.

    30

    """

    31

    32

    parser = optparse.OptionParser(

    33

            prog        = argv[0],

    34

            usage       = '%prog: [options] [ --config <path> | --target <path> [ --source <src> ] [ --interval <name> ] ]',

    35

            version     = __version__,

    36

    37

            # module docstring

    38

            # XXX: breaks multi-line descriptions..

    39

            description = __doc__,

    40

    )

    41

    42

    # logging

    43

    general = optparse.OptionGroup(parser, "General Options")

    44

    45

    general.add_option('-q', '--quiet',      dest='loglevel', action='store_const', const=logging.WARNING, help="Less output")

    46

    general.add_option('-v', '--verbose',    dest='loglevel', action='store_const', const=logging.INFO,  help="More output")

    47

    general.add_option('-D', '--debug',      dest='loglevel', action='store_const', const=logging.DEBUG, help="Even more output")

    48

    49

    parser.add_option_group(general)

    50

    51

    # rsync

    52

    rsync = optparse.OptionGroup(parser, "rsync Options")

    53

    54

    rsync.add_option('--exclude-from',       metavar='FILE',

    55

        help="Read exclude rules from given file")

    56

    57

    rsync.add_option('--include-from',       metavar='FILE',

    58

        help="Read include rules from given file")

    59

    60

    parser.add_option_group(rsync)

    61

    62

    # global

    63

    parser.add_option('--clean-intervals',  action='store_true',

    64

        help="Clean out old interval links")

    65

    66

    parser.add_option('--clean-snapshots',  action='store_true',

    67

        help="Clean out unused snapshots (those not linked to)")

    68

    69

    parser.add_option('--clean',             action='store_true',

    70

        help="Clean out both intervals and snapshots")

    71

    72

    parser.add_option('-n', '--dry-run',    action='store_true',

    73

        help="Don't actually clean anything")

    74

    75

    #

    76

    parser.add_option('-c', '--config',     metavar='FILE',

    77

        help="Load configuration file")

    78

    79

    #

    80

    parser.add_option('-T', '--target',    metavar='PATH',

    81

        help="Target path")

    82

    83

    parser.add_option('-s', '--source',     metavar='RSYNC-PATH', dest='target_source', default=False,

    84

        help="Run target backup from source in rsync-syntax")

    85

    86

    parser.add_option('--interval',         metavar='NAME', action='append', dest='target_intervals',

    87

        help="Run target with given given interval(s)")

    88

    89

    90

    # defaults

    91

    parser.set_defaults(

    92

        loglevel            = logging.INFO,

    93

    94

        target_intervals    = [],

    95

    )

    96

    parser.set_defaults(**defaults)

    97

    98

    99

    # parse

   100

    options, args = parser.parse_args(argv[1:])

   101

   102

    # configure

   103

    logging.basicConfig(

   104

        format  = '%(processName)s: %(name)s: %(levelname)s %(funcName)s : %(message)s',

   105

        level   = options.loglevel,

   106

    )

   107

   108

    if options.clean :

   109

        options.clean_intervals = options.clean_snapshots = options.clean

   110

   111

    if options.include_from :

   112

        options.rsync_options['include-from'] = options.include_from

   113

   114

    if options.exclude_from :

   115

        options.rsync_options['exclude-from'] = options.exclude_from

   116

   117

    return options, args

   118

   119

## Configuration

   120

class ConfigError (Exception) :

   121

    pass

   122

   123

def process_config_name (name) :

   124

    """

   125

        Process config file name into python version

   126

    """

   127

   128

    return name.replace('-', '_')

   129

   130

def parse_config (path, defaults) :

   131

    """

   132

        Parse given config file

   133

    """

   134

   135

    log.debug("loading config: %s", path)

   136

   137

    config = dict(defaults)

   138

    config_file = ConfigParser.RawConfigParser()

   139

    config_file.read([path])

   140

   141

    # handle each section

   142

    for section in config_file.sections() :

   143

        # mangle

   144

        section_name = process_config_name(section)

   145

   146

        log.debug("section: %s", section_name)

   147

   148

        # subsections

   149

        section_path = section_name.split(':')

   150

   151

        # lookup section dict from config

   152

        lookup = config

   153

   154

        # XXX: sections are not in order, so we can't rely on the parent section being created before we handle the sub-section

   155

        for name in section_path :

   156

            # possibly create

   157

            if name not in lookup :

   158

                lookup[name] = {}

   159

   160

            lookup = lookup[name]

   161

   162

        # found dict for this section

   163

        config_section = lookup

   164

   165

        # values

   166

        for name, value in config_file.items(section) :

   167

            # mangle

   168

            name = process_config_name(name)

   169

   170

            log.debug("section: %s: %s = %s", '/'.join(section_path), name, value)

   171

   172

            config_section[name] = value

   173

   174

    log.debug("config: %s", config)

   175

   176

    return config

   177

   178

def config_bool (name, value, strict=True) :

   179

    if value.lower() in ('yes', 'true', '1', 'on') :

   180

        return True

   181

   182

    elif value.lower() in ('no', 'false', '0', 'off') :

   183

        return False

   184

   185

    elif strict :

   186

        raise ConfigError("Unrecognized boolean value: {name} = {value}".format(name=name, value=value))

   187

   188

    else :

   189

        # allow non-boolean values

   190

        return value

   191

   192

def config_int (name, value) :

   193

    try :

   194

        return int(value)

   195

   196

    except ValueError, e:

   197

        raise ConfigError("Invalid integer value: {name} = {value}".format(name=name, value=value))

   198

   199

def config_list (name, value) :

   200

    return value.split()

   201

   202

def walk_symlinks (tree, ignore=False) :

   203

    """

   204

        Walk through all symlinks in given dir, yielding:

   205

   206

            (dirpath, name, target)

   207

   208

        Passes through errors from os.listdir/os.lstat.

   209

    """

   210

   211

    for name in os.listdir(tree) :

   212

        if ignore and name in ignore :

   213

            log.debug("%s: ignore: %s", tree, name)

   214

            continue

   215

   216

        path = os.path.join(tree, name)

   217

   218

        # stat symlink itself

   219

        st = os.lstat(path)

   220

   221

        if stat.S_ISDIR(st.st_mode) :

   222

            # recurse

   223

            log.debug("%s: tree: %s", tree, name)

   224

   225

            for item in walk_symlinks(path) :

   226

                yield item

   227

   228

        elif stat.S_ISLNK(st.st_mode) :

   229

            # found

   230

            target = os.readlink(path)

   231

   232

            log.debug("%s: link: %s -> %s", tree, name, target)

   233

   234

            yield tree, name, target

   235

   236

        else :

   237

            log.debug("%s: skip: %s", tree, name)

   238

   239

   240

class Interval (object) :

   241

    """

   242

        An interval definition.

   243

    """

   244

   245

    @classmethod

   246

    def from_config (cls, options, name,

   247

        format,

   248

        keep,

   249

    ) :

   250

        if not format :

   251

            # magic to use snapshot name

   252

            _format = None

   253

        else :

   254

            _format = format

   255

   256

        return cls(name, 

   257

            format  = _format, 

   258

            keep    = config_int('keep', keep),

   259

        )

   260

   261

    @classmethod

   262

    def from_target_config (cls, name, base, arg) :

   263

        if isinstance(arg, dict) :

   264

            # full instance

   265

            return cls(name,

   266

                format  = arg.get('format', base.format if base else None),

   267

                keep    = arg.get('keep', base.keep if base else None),

   268

            )

   269

        else :

   270

            # partial instance with keep

   271

            return cls(name,

   272

                format  = base.format,

   273

                keep    = config_int('keep', arg) if arg else base.keep,

   274

            )

   275

   276

    def __init__ (self, name, format, keep) :

   277

        self.name = name

   278

        self.format = format

   279

        self.keep = keep

   280

   281

    def __str__ (self) :

   282

        return self.name

   283

   284

class Target (object) :

   285

    """

   286

        A target run, i.e. a rsync-snapshot destination dir

   287

   288

        [target:...]

   289

    """

   290

   291

    @classmethod

   292

    def from_config (cls, options, name,

   293

        path            = False,

   294

        source          = None,

   295

        enable          = 'no',

   296

        exclude_from    = None,

   297

   298

        # subsections

   299

        intervals       = None,

   300

        rsync_options   = None,

   301

    ) :

   302

        if not source and source is not False :

   303

            raise ConfigError("Missing required option: source for [target:{name}]".format(name=name))

   304

   305

        # global defaults

   306

        _rsync_options = dict(options.rsync_options)

   307

   308

        if rsync_options :

   309

            # override

   310

            _rsync_options.update([

   311

                # parse

   312

                (option, config_bool(option, value, strict=False)) for option, value in rsync_options.iteritems()

   313

            ])

   314

   315

        # lookup intervals

   316

        _intervals = [

   317

            (

   318

                # lookup base from options.intervals

   319

                Interval.from_target_config(name, options.intervals.get(name), arg)

   320

            ) for name, arg in intervals.iteritems()

   321

        ]

   322

   323

        return cls(name, 

   324

            path            = path if path else name,

   325

            source          = source,

   326

            enable          = config_bool('enable', enable),

   327

            intervals       = _intervals,

   328

            rsync_options   = _rsync_options,

   329

            exclude_from    = exclude_from,

   330

        )

   331

   332

    def __init__ (self, name,

   333

        path,

   334

        source, 

   335

        enable          = False, 

   336

        intervals       = [],

   337

        rsync_options   = {},

   338

        exclude_from    = None

   339

    ) :

   340

        self.name = name

   341

   342

        self.path = path

   343

        self.source = source

   344

        self.enable = enable

   345

   346

        self.intervals = intervals

   347

   348

        self.rsync_options = rsync_options

   349

        self.exclude_from = exclude_from

   350

   351

        # this snapshot?

   352

        self.snapshots_dir = os.path.join(self.path, 'snapshots')

   353

   354

        # 'current' symlink

   355

        self.current_path = os.path.join(self.path, 'current')

   356

   357

    def prepare (self, options) :

   358

        """

   359

            Prepare dir for usage

   360

        """

   361

   362

        if not os.path.exists(self.path) :

   363

            raise Exception("Missing target dir: {path}".format(path=self.path))

   364

   365

        if not os.path.exists(self.snapshots_dir) :

   366

            log.warn("Creating snapshots dir: %s", self.snapshots_dir)

   367

            os.mkdir(self.snapshots_dir)

   368

   369

    def snapshot (self, options, now) :

   370

        """

   371

            Perform the rsync from our source to self.snapshot_dir.

   372

   373

            XXX: allocate snapshot_name here?

   374

        """

   375

   376

        # new snapshot

   377

        snapshot_name = now.strftime(options.snapshot_format)

   378

        snapshot_path = os.path.join(self.snapshots_dir, snapshot_name)

   379

        temp_path = os.path.join(self.snapshots_dir, 'tmp')

   380

   381

        if os.path.exists(temp_path) :

   382

            raise Exception("Old temp snapshot dir remains, please clean up: {path}".format(path=temp_path))

   383

   384

        log.info("Perform main snapshot: %s -> %s", self.source, snapshot_path)

   385

   386

        # build rsync options

   387

        opts = dict(self.rsync_options)

   388

   389

        if os.path.exists(self.current_path) :

   390

            # real path to target

   391

            target = os.readlink(self.current_path)

   392

            target_path = os.path.join(os.path.dirname(self.current_path), target)

   393

            target_abs = os.path.abspath(target_path)

   394

   395

            log.info("Using current -> %s as base", target_path)

   396

   397

            # use as link-dest base; hardlinks unchanged files; target directory must be empty

   398

            # rsync links absolute paths..

   399

            opts['link-dest'] = target_abs

   400

   401

        # go

   402

        log.debug("rsync %s -> %s", self.source, temp_path)

   403

        rsync.rsync(self.source, temp_path, **opts)

   404

   405

        # move in to final name

   406

        log.debug("rename %s -> %s", temp_path, snapshot_path)

   407

        os.rename(temp_path, snapshot_path)

   408

   409

        return snapshot_name

   410

   411

    def update_interval (self, options, interval, now, snapshot_name) :

   412

        """

   413

            Update given <interval>/... links for this target, using the given new snapshot

   414

        """

   415

   416

        dir_path = os.path.join(self.path, interval.name)

   417

   418

        if not os.path.exists(dir_path) :

   419

            log.warn("Creating interval dir: %s", dir_path)

   420

            os.mkdir(dir_path)

   421

   422

   423

        # name

   424

        if interval.format is None :

   425

            # per-snapshot

   426

            name = snapshot_name

   427

   428

            log.debug("%s: using snapshot_name: %s", interval, name)

   429

   430

        else :

   431

            # by date

   432

            name = now.strftime(interval.format)

   433

   434

            log.debug("%s: using interval.format: %s -> %s", interval, interval.format, name)

   435

   436

        # path

   437

        path_name = os.path.join(interval.name, name)

   438

        path = os.path.join(self.path, path_name)

   439

   440

        log.debug("%s: processing %s", interval, path_name)

   441

   442

        # already there?

   443

        if os.path.exists(path) :

   444

            target = os.readlink(path)

   445

   446

            log.info("%s: Found existing: %s -> %s", interval, name, target)

   447

   448

        else :

   449

            # update

   450

            target = os.path.join('..', 'snapshots', snapshot_name)

   451

   452

            log.info("%s: Updating: %s -> %s", interval, name, target)

   453

            log.debug("%s -> %s", path, target)

   454

   455

            os.symlink(target, path)

   456

   457

   458

    def clean_interval (self, options, interval) :

   459

        """

   460

            Clean out given <interval>/... dir for this target.

   461

        """

   462

   463

        # path

   464

        dir_path = os.path.join(self.path, interval.name)

   465

   466

        if not os.path.exists(dir_path) :

   467

            log.warn("%s: Skipping, no interval dir: %s", interval, dir_path)

   468

            return

   469

   470

        # configured

   471

        keep = interval.keep

   472

   474

        items = os.listdir(dir_path)

   478

   479

        log.info("%s: Have %d / %d items", interval, len(items), keep)

   480

        log.debug("%s: items: %s", interval, ' '.join(items))

   481

   482

        if len(items) > keep :

   484

            clean = items[keep:]

   485

   486

            log.info("%s: Cleaning out %d items", interval, len(clean))

   487

            log.debug("%s: cleaning out: %s", interval, ' '.join(clean))

   488

   489

            for item in clean :

   490

                path = os.path.join(dir_path, item)

   491

   492

                log.info("%s: Clean: %s", interval, path)

   493