bin/update
author Tero Marttila <terom@paivola.fi>
Tue, 20 Mar 2012 13:43:44 +0200
changeset 46 3613e93e4fd7
parent 45 3d6bf3864d8a
child 47 b4b590efe3ef
permissions -rwxr-xr-x
update: rndc: fail if no read perms on $RNDC_KEY
#!/bin/bash
# vim: set ft=sh :

set -ue


### Paths
ROOT=$(pwd)

# resolve $0
self=$0
while [ -L $self ]; do
    tgt=$(readlink $self)

    if [ "${tgt:0:1}" == "/" ]; then
        self=$tgt
    else
        self=$(dirname $self)/$tgt
    fi
done

# Our bin dir, with scripts
BIN=$(dirname $self)

# Data files
DATA=settings
ZONES=zones
SERIALS=$DATA
REPO=

# Script/data args
PROCESS_ARGS='--input-charset latin-1'

# external progs
NAMED_CHECKZONE=/usr/sbin/named-checkzone
HG=/usr/bin/hg
RNDC=/usr/sbin/rndc
RNDC_KEY=/etc/bind/rndc.key

### Command-line argument handling

IS_TTY=

## Options
LOG_ERROR=y
LOG_WARN=y
LOG=y
LOG_FORCE=y
LOG_UPDATE=y
LOG_NOOP=y
LOG_SKIP=
LOG_DEBUG=
LOG_CMD=

UPDATE_FORCE=
UPDATE_NOOP=
UPDATE_DIFF=

SERIAL_NOOP=
SERIAL_FORCE=

COMMIT_SKIP=
COMMIT_FORCE=
COMMIT_MSG=' '

DEPLOY_SKIP=

## Output command-line argument help.
function help_args {
    local prog=$1

    cat <<END
Usage: $prog [options]

General:
    -h      display this help text
    -d DIR  datadir

Logging:    
    -q      quiet
    -v      verbose
    -D      debug
    -V      debug commands
    
Updates:
    -p      show changes
    -F      force-updates without checking src mtime
    -S      do not update serial
    -s      update serials
    -n      no-op/mock-update; don't actually change anything; implies -SpC

Commit    
    -C      do not commit changes
    -c      commit changes
    -m MSG  commit message
END
}

## Parse any command-line arguments, setting the global options vars.
function parse_args {
    OPTIND=1

    while getopts 'hd:qvDVpFSsnCcm:' opt "$@"; do
        case $opt in
            h)  
                help_args $0
                exit 0
            ;;

            d)  ROOT="$OPTARG" ;;

            q)  
                LOG= 
                LOG_WARN=
                LOG_UPDATE=
                LOG_FORCE=
                LOG_NOOP=
                ;;

            v)  LOG_SKIP=y ;;
            D)  
                LOG_DEBUG=y
                LOG_INFO=y
                ;;
            V)  LOG_CMD=y ;;

            p)  UPDATE_DIFF=y   ;;
            F)  UPDATE_FORCE=y  ;;
            S)  SERIAL_NOOP=y   ;;
            s)  SERIAL_FORCE=y  ;;
 
            n)  
                UPDATE_NOOP=y 
                # implies -Sp
                UPDATE_DIFF=y
                SERIAL_NOUPDATE=y
                COMMIT_SKIP=y
                DEPLOY_SKIP=y
                ;;

            C)  COMMIT_SKIP=y ;;
            c)  COMMIT_FORCE=y ;;
            m)  COMMIT_MSG="$OPTARG" ;;
           
            ?)  
                die 
            ;;
        esac

    done
}

### Logging
# Output message to stderr.
function log_msg {
    echo "$*" >&2
}

# Output message to stderr, optionally with given color, if TTY.
function log_color {
    local code=$1; shift

    if [ $IS_TTY ]; then
        echo $'\e['${code}'m'"$*"$'\e[00m' >&2
    else
        echo "$*" >&2
    fi
}

## Log at various log-levels

function log_error {
    [ $LOG_ERROR ] && log_color '31' "$*"
}

function log_warn {
    [ $LOG_WARN ] && log_color '33' "$*" || true
}

# plain
function log {
    [ $LOG ] && log_msg "$*" || true
}

function log_force {
    [ $LOG_FORCE ] && log_color '2;33' "  $*" || true
}

function log_update {
    [ $LOG_UPDATE ] && log_color '36' "  $*" || true
}

function log_noop {
    [ $LOG_NOOP ] && log_color '2;34' "  $*" || true
}

function log_skip {
    [ $LOG_SKIP ] && log_color '1;34' "  $*" || true
}

function log_debug {
    [ $LOG_DEBUG ] && log_color 32 "    $*" || true
}

function log_cmd {
    [ $LOG_CMD ] && log_color 35 "        \$ $*" || true
}

# Output stacktrace, broken.
function log_stack {
    local level=1

    while info=$(caller $level); do
        echo $info | read line sub file

        log_msg "$file:$lineno $sub()"

        level=$(($level + 1))
    done
}

# Output calling function's name.
function func_caller {
    caller 1 | cut -d ' ' -f 2
}

### High-level logging output
# Log with func_caller at log_debug
function debug {
    printf -v prefix "%s" $(func_caller)

    log_debug "$prefix: $*"
}

# Log with func_caller at log_error and exit, intended for internal errors...
function fail {
    log_error "$(func_caller): $*"

    exit 2
}

# Log at log_error and exit
function die {
    log_error "$*"
    exit 1
}

### Command execution
## Execute command, possibly logging its execution.
#
#   cmd     $cmd...
#
# Fails if the command returns an error exit code.
function cmd {
    log_cmd "$@"

    "$@" || die "Failed"
}

function indent () {
    local indent=$1; shift

    log_cmd "$@"

    "$@" | sed "s/^/$indent/"

    return ${PIPESTATUS[0]}
}


### FS utils
# Create dir in $ROOT if not exists.
function ensure_dir {
    local dir=$1

    if [ ! -d $ROOT/$dir ]; then
        log_warn "Creating output dir: $dir"
        cmd mkdir $ROOT/$dir
    fi
}

## Output absolute path from $ROOT:
#
#   abspath $path
#
function abspath () {
    local path=$1

    echo "$ROOT/$path"
}

### HG wrappers
# Run `hg ...` within $REPO.
function hg {
    local repo=$REPO; shift

    cmd $HG -R $ROOT/$repo "$@"
}

# Does the repo have local modifications?
function hg_modified {
    hg id | grep -q '+'
}

# Output possible -u flag for commit.
function hg_user {
    if [ ${SUDO_USER:-} ]; then
        echo '-u' "$SUDO_USER"

    elif [ $HOME ] && [ -e $HOME/.hgrc ]; then
        debug "using .hgrc user"
        echo ''

    else
        echo '-u' "$USER"
    fi
}

# Show changes in repo
function hg_diff {
    hg diff
}

## Commit changes in repo, with given message:
#
#   hg_commit   $msg
#
function hg_commit {
    local msg=$1
    local user_opt=$(hg_user)
    
    debug "$user_opt: $msg"
    hg commit $user_opt -m "$msg"
}


### Dependency-based updates

## Compare the given output file with all given source files:
#
#   check_update $out ${deps[@]} && do_update $out ... || ...
#
# Returns true if the output file needs to be updated.
function check_update {
    # target
    local out=$1; shift

    debug "$out"

    # need update?
    local update=

    if [ ${#@} == 0 ]; then
        debug "  update: unknown deps"
        update=y

    elif [ ! -e $out ]; then
        debug "  update: dest missing"
        update=y
        
    elif [ $UPDATE_FORCE ]; then
        debug "  update: forced"
        update=y
    fi

    # check deps
    for dep in "$@"; do
        # don't bother checking if already figured out
        [ $update ] && continue

        # check
        if [ ! -e $ROOT/$dep ]; then
            fail "$dst: Missing source: $dep"

        elif [ $ROOT/$out -ot $ROOT/$dep ]; then
            debug "  update: $dep"
            update=y
        else
            debug "  check: $dep"
        fi
    done

    [ ! $update ] && debug "  up-to-date"

    # return
    [ $update ]
}

## Generate updated output file from given command's stdout:
#
#   do_update $out $BIN/cmd --args
#
# Writes output to a temporary .new file, optionally shows a diff of changes, and commits
# the new version to $out (unless noop'd).
function do_update {
    local out=$1; shift
    local tmp=$out.new

    debug "$out"
    cmd "$@" > $ROOT/$tmp

    # compare
    if [ -e $ROOT/$out ] && [ $UPDATE_DIFF ]; then
        debug "  changes:"

        # terse
        indent "        " diff --unified=1 $ROOT/$out $ROOT/$tmp || true
    fi
    
    # deploy
    if [ $UPDATE_NOOP ]; then
        # cleanup
        debug "  no-op"

        cmd rm $ROOT/$tmp
    else
        # commit
        debug "  deploy"

        cmd mv $ROOT/$tmp $ROOT/$out
    fi
}

## Look for a link target:
#
#   find_link   $lnk    $tgt...
#
# Outputs the first given target to exist, skipping any that are the same as the given $lnk.
# If no $tgt matches, outputs the last one, or '-'.
function choose_link {
    local lnk=$1; shift
    local tgt=-

    for tgt in "$@"; do
        [ $tgt != $out ] && [ -e $ROOT/$tgt ] && break
    done
    
    echo $tgt
}


## Compare symlink to target:
#
#   check_link $lnk $tgt && do_link $lnk $tgt || ...
#
# Tests if the symlink exists, and the target matches.
# Fails if the target does not exist.
function check_link {
    local lnk=$1
    local tgt=$2

    [ ! -e $ROOT/$tgt ] && fail "$tgt: target does not exist"
    
    [ ! -e $ROOT/$lnk ] || [ $(readlink $ROOT/$lnk) != $ROOT/$tgt ]
}

## Update symlink to point to target:
#
#   do_link $lnk $tgt
#
function do_link {
    local lnk=$1
    local tgt=$2

    cmd ln -sf $ROOT/$tgt $ROOT/$lnk
}

## Update .serial number:
#
#   do_update_serial $serial
#
# Shows old/new serial on debug.
function do_update_serial {
    local serial=$1

    # read
    local old=$(test -e $ROOT/$serial && cat $ROOT/$serial || echo '')


    cmd $BIN/update-serial $ROOT/$serial
    
    # read
    local new=$(cat $ROOT/$serial)
        
    debug "  $old -> $new"
}

## Perform `hg commit` for $DATA
function do_commit {
    local msg=$1

    indent "    " hg_diff

    hg_commit "$msg"
}

### Hosts
## Update hosts from verbatim from input zone data:
#
#   copy_hosts      $ZONES/$zone    $DATA/$base
#
# Writes updated zone to $zone, deps on $base.
function copy_hosts {
    local zone=$1
    local base=$2

    if check_update $zone $base; then
        log_update "Copying hosts $zone <- $base..."

        do_update $zone \
            cat $ROOT/$base
    else
        log_skip "Copying hosts $zone <- $base: not changed"
    fi
}

## Generate hosts from input zone data using $BIN/process-zone:
#
#   update_hosts    $ZONES/$zone    $DATA/$base
#
# Writes process-zone'd data to $zone, deps on $base.
function update_hosts {
    local zone=$1; shift
    local base=$1; shift

    if check_update $zone $base; then
        log_update "Generating hosts $zone <- $base..."

        do_update $zone \
            $BIN/process-zone $PROCESS_ARGS $ROOT/$base "$@"
    else
        log_skip "Generating hosts $zone <- $base: not changed"
    fi
}

## Generate new serial for zone using $BIN/update-serial, if the zone data has changed:
#
#   update_serial   $zone   $deps...
#
# Supports SERIAL_FORCE/NOOP.
# Updates $SERIALS/$zone.serial.
function update_serial {
    local zone=$1; shift
    
    local serial=$SERIALS/$zone.serial

    # test
    if [ $SERIAL_FORCE ]; then
        log_force "Updating $serial: forced"

        do_update_serial $serial

    elif ! check_update $serial "$@"; then
        log_skip "Updating $serial: not changed"

    elif [ $SERIAL_NOOP ]; then
        log_noop "Updating $serial: skipped"

    else
        log_update "Updating $serial..."

        do_update_serial $serial
    fi
}

## Link serial for zone from given base-zone:
#
#   link_serial $zone $base
function link_serial {
    local zone=$1
    local base=$2

    local lnk=$SERIALS/$zone.serial
    local tgt=$SERIALS/$base.serial

    if check_link $lnk $tgt; then
        log_update "Linking $lnk -> $tgt..."

        do_link $lnk $tgt

    else
        log_skip "Linking $lnk -> $tgt: not changed"
    fi
}

## Update zone file verbatim from source:
#
#   copy_zone   $view   $zone   [$base]
#
# Copies changed $DATA/$base zone data to $ZONES/$view/$zone.
function copy_zone {
    local view=$1
    local zone=$2
    local base=${3:-$zone}

    local out=$ZONES/$view/$zone
    local src=$DATA/$base

    if check_update $out $src; then
        log_update "Copying $out <- $src..."

        do_update $out \
            cat $ROOT/$src
    else
        log_skip "Copying $out <- $src: not changed"
    fi
}

## Expand zone file from source using $BIN/expand-zone:
#
#   update_zone $view   $zone   [$base]
#
# Processed $DATA/$base zone data through $BIN/expand-zone, writing output to $ZONES/$view/$zone.
function update_zone {
    local view=$1
    local zone=$2
    local base=${3:-$zone}

    local out=$ZONES/$view/$zone
    local src=$DATA/$base.zone
    local lnk=$ZONES/$base

    local serial=$SERIALS/$base.serial

    if check_update $out $src $serial; then
        log_update "Generating $out <- $src..." 

        do_update $out \
            $BIN/expand-zone $ROOT/$src \
                --serial $ROOT/$serial              \
                --expand zones=$(abspath $ZONES)    \
                --expand view=$view
    else
        log_skip "Generating $out <- $src: not changed" 
    fi
}

## Link zone file to ues given shared zone.
#
#   link_zone   $view   $zone   [$base]
#
# Looks for shared zone at:
#   $ZONES/$view/$base
#   $ZONES/common/$base
function link_zone {
    local view=$1
    local zone=$2
    local base=${3:-$zone}

    local out=$ZONES/$view/$zone
    local tgt=$(choose_link $out $ZONES/$view/$base $ZONES/common/$base)

    if check_link $out $tgt; then
        log_update "Linking $out -> $tgt..."

        do_link $out $tgt

    else
        log_skip "Linking $out -> $tgt: not changed"
    fi
}

## Test hosts zone for validity:
#
#   check_hosts     $DATA/$hosts    --check-exempt ...
#
# Fails if the check fails.
function check_hosts {
    local hosts=$1; shift 1

    local cmd=($BIN/process-zone $PROCESS_ARGS $ROOT/$hosts --check-hosts "$@")

    if "${cmd[@]}" -q; then
        log_skip "Check $hosts: OK"
    else
        log_error "  Check $hosts: Failed"

        indent "    " "${cmd[@]}"

        exit 1
    fi
}

## Test zone file for validity using named-checkzone:
#
#   check_zone      $view       $zone       $origin
#
# Uses the zonefile at $ZONES/$view/$zone, loading it with given initial $ORIGIN.
# Fails if the check fails.
function check_zone {
    local view=$1
    local zone=$2
    local origin=$3

    local src=$ZONES/$view/$zone

    local cmd=($NAMED_CHECKZONE $origin $ROOT/$src)

    # test
    # XXX: checkzone is very specific about the order of arguments, -q must be first
    if $NAMED_CHECKZONE -q $origin $ROOT/$src; then
        log_skip "Check $src ($origin): OK"
    else
        log_error "  Check $src ($origin): Failed:"

        indent "    " "${cmd[@]}"
        
        exit 1
    fi
}

## Load update zonefiles into bind:
#
#   deploy_zones    
#
# Invokes `rndc reload`, showing its output.
function deploy_zones {
    local msg="Reload zones"

    if [ $DEPLOY_SKIP ]; then
        log_skip    "$msg: skipped"
    
    elif [ ! -r $RNDC_KEY ]; then
        log_error   "  $msg: rndc: permission denied: $RNDC_KEY"

    else
        log_update  "$msg..."

        # run
        indent "        rndc: " \
            $RNDC reload
    fi
}
## Commit changes in $DATA to version control:
#
#   commit_data
#
# Invokes `hg commit` in the $REPO, first showing the diff.
function commit_data {
    local repo=$REPO
    local commit_msg="$COMMIT_MSG"

    local msg="Commit changes in $repo"

    # operate?
    if [ $COMMIT_FORCE ]; then
        log_force   "$msg..."

        do_commit "$commit_msg"

    elif ! hg_modified; then
        log_skip    "$msg: no changes"

    elif [ $COMMIT_SKIP ]; then
        log_noop    "$msg: skipped"

    else
        log_update  "msg..."

        do_commit "$commit_msg"
    fi
}

## Site settings, used as arguments to scripts
# MX record to generate in hosts --forward-zone
FORWARD_MX=mail

# IP network to generate reverse records for in --reverse-zone
REVERSE_ZONE=194.197.235

# Origin domain to generate reverse records for in --reverse-zone
REVERSE_DOMAIN=paivola.fi

# Views used
VIEWS=(internal external)

# Base domain zone for domains
DOMAIN_BASE=paivola

# List of actual domains used; will be linked to $DOMAIN_BASE
DOMAINS=(paivola.fi paivola.net paivola.org paivola.info paivola.mobi xn--pivl-load8j.fi)

## Operate!
function run {
    ## Hosts
    # test
    log "Testing hosts..."
        #                   data                            args...
        check_hosts         $DATA/paivola.txt               --check-exempt ufc

    # update
    log "Generating host zones..."
        #                   hosts                           data                args...
        update_hosts        $ZONES/hosts/paivola:internal   $DATA/paivola.txt   --forward-zone --forward-txt --forward-mx $FORWARD_MX
        update_hosts        $ZONES/hosts/paivola:external   $DATA/paivola.txt   --forward-zone
        update_hosts        $ZONES/hosts/194.197.235        $DATA/paivola.txt   --reverse-zone $REVERSE_ZONE --reverse-domain $REVERSE_DOMAIN

        
        update_hosts        $ZONES/hosts/10                 $DATA/pvl.txt       --reverse-zone 10 --reverse-domain pvl -q
        update_hosts        $ZONES/hosts/192.168            $DATA/pvl.txt       --reverse-zone 192.168 --reverse-domain pvl -q

        # XXX: unsupported --forward-zone with pvl.txt
        # update_hosts    $ZONES/hosts/pvl                    $DATA/pvl.txt      --forward-zone
        copy_hosts          $ZONES/hosts/pvl                $DATA/pvl.txt

    ## Includes
    log "Copying zone includes..."
        #                   view            zone                    base
        copy_zone           includes        paivola:internal        paivola.zone.internal
        copy_zone           includes        paivola:external        paivola.zone.external
        copy_zone           includes        paivola.auto            paivola.zone.auto
        copy_zone           includes        paivola.services        paivola.zone.services

    ## Serials
    log "Updating serials..."

        #                   zone            deps...
        #   includes...
        update_serial       pvl             $ZONES/hosts/pvl            $DATA/pvl.zone
        update_serial       10              $ZONES/hosts/10             $DATA/10.zone
        update_serial       192.168         $ZONES/hosts/192.168        $DATA/192.168.zone

        update_serial       paivola         $ZONES/hosts/paivola:*      $DATA/paivola.zone          \
            $ZONES/includes/paivola:*       \
            $ZONES/includes/paivola.*

        update_serial       194.197.235     $ZONES/hosts/194.197.235    $DATA/194.197.235.zone          

    ## Zones
    log "Updating zones..."
        #                   view        zone            base
        update_zone         internal    pvl
        update_zone         internal    paivola
        update_zone         external    paivola

        update_zone         internal    10
        update_zone         internal    192.168

        update_zone         common      194.197.235
        link_zone           internal    194.197.235
        link_zone           external    194.197.235

    ## Test
    log "Testing zones..."
        #                   view        zone            origin
        check_zone          internal    paivola         paivola.fi
        check_zone          external    paivola         paivola.fi

        check_zone          internal    10              10.in-addr.arpa
        check_zone          internal    192.168         192.168.in-addr.arpa
        check_zone          common      194.197.235     235.197.194.in-addr.arpa

    ## Domains...
    log "Linking domains..."
        for view in "${VIEWS[@]}"; do
            for zone in "${DOMAINS[@]}"; do
                # link
                link_zone       $view       $zone           $DOMAIN_BASE

                # test
                check_zone      $view       $zone           $zone
            done
        done

    ## Deploy
    log "Deploy zones..."
        deploy_zones

    ## Commit
    log "Commit data..."
        commit_data
}

## Main entry point
function main {
    # test tty
    [ -t 1 ] && IS_TTY=y
    
    parse_args "$@"

    ## test env
    [ -d $ROOT/$DATA ] || die "Missing data: $ROOT/$DATA"
    ensure_dir  $ZONES
    
    ## Output dirs
    for dir in "common" "hosts" "includes" "${VIEWS[@]}"; do
        ensure_dir $ZONES/$dir
    done

    ## Go
    run
}

main "$@"