1

#!/usr/bin/env python

     2

     3

"""

     4

    Import hosts from existing BIND or dhcpd files.

     5

"""

     6

     7

import pvl.args

     8

import pvl.dns.zone

     9

import pvl.dhcp.config

    10

import pvl.ldap.args

    11

    12

import ipaddr

    13

import optparse

    14

import collections

    15

import re

    16

import logging; log = logging.getLogger('pvl.hosts-import')

    17

    18

__version__ = '0.1'

    19

    20

def parse_options (argv) :

    21

    """

    22

        Parse command-line arguments.

    23

    """

    24

    25

    parser = optparse.OptionParser(

    26

            prog        = argv[0],

    27

            usage       = '%prog: [options]',

    28

            version     = __version__,

    29

    30

            # module docstring

    31

            description = __doc__,

    32

    )

    33

    34

    # logging

    35

    parser.add_option_group(pvl.args.parser(parser))

    36

    parser.add_option_group(pvl.ldap.args.parser(parser))

    37

    38

    parser.add_option('-c', '--input-charset',  metavar='CHARSET',  default='utf-8', 

    39

            help="Encoding used for input files")

    40

    41

    parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 

    42

            help="Encoding used for output files")

    43

    44

    # input

    45

    parser.add_option('--import-zone-hosts',    metavar='FILE',

    46

            help="Load hosts from DNS zone")

    47

    48

    parser.add_option('--import-dhcp-hosts',    metavar='FILE',

    49

            help="Load hosts from DHCP config")

    50

    51

    parser.add_option('--import-dhcp-boot-server',      metavar='NEXT-SERVER',

    52

            help="Default boot_server for dpc hosts")

    53

    54

    parser.add_option('--zone-comments-default-owner',  action='store_const',

    55

            dest='zone_comments_default', const='owner',

    56

            help="Import DNS zone comment as owner comment")

    57

    58

    parser.add_option('--zone-comments-default-host',   action='store_const',

    59

            dest='zone_comments_default', const='host',

    60

            help="Import DNS zone comment as host comment")

    61

    62

    parser.add_option('--dump-host-comments',   action='store_true',

    63

            help="Dump out info on imported host comments")

    64

    65

    # defaults

    66

    parser.add_option('--hosts-domain',         metavar='DOMAIN',

    67

            help="Default domain for hosts")

    68

    69

    parser.add_option('--zone-unused',          metavar='HOST',

    70

            help="DNS name for unallocated hosts")

    71

    72

    # output

    73

    parser.add_option('--output-hosts',         metavar='FILE',

    74

            help="Output hosts file")

    75

    76

    parser.add_option('--output-prefix',        metavar='PREFIX',

    77

            help="Select hosts by ip prefix")

    78

    79

    # defaults

    80

    parser.set_defaults(

    81

    82

    )

    83

    84

    # parse

    85

    options, args = parser.parse_args(argv[1:])

    86

    87

    # apply

    88

    pvl.args.apply(options, argv[0])

    89

    90

    return options, args

    91

    92

def import_zone_hosts (options, file) :

    93

    """

    94

        Yield host info from zonefile records.

    95

    """

    96

    97

    for rr in pvl.dns.zone.ZoneRecord.load(file,

    98

            # generated hosts need to imported by hand...

    99

            expand_generate = False,

   100

    ) :

   101

        if options.zone_unused and rr.name == options.zone_unused :

   102

            log.debug("%s: skip %s", rr.name, rr)

   103

            continue

   104

   105

        elif rr.type in ('A', 'AAAA') :

   106

            ip, = rr.data

   107

   111

   112

            if rr.comment :

   113

                yield rr.name, 'comment', rr.comment

   114

   115

            if rr.origin :

   116

                yield rr.name, 'domain', rr.origin

   117

   118

        elif rr.type == 'CNAME' :

   119

            host, = rr.data

   120

   121

            yield host, 'alias', rr.name

   122

   123

        else :

   124

            log.warn("%s: unknown rr: %s", rr.name, rr)

   125

   126

def import_dhcp_host (options, host, items) :

   127

    """

   128

        Yield host infos from a dhcp host ... { ... }

   129

    """

   130

   131

    hostname = None

   132

    ethernet = []

   133

    fixed_address = None

   134

   135

    boot_server = options.import_dhcp_boot_server

   136

    boot_filename = None

   137

   138

    for item in items :

   139

        item, args = item[0], item[1:]

   140

   141

        if item == 'hardware' :

   142

            _ethernet, ethernet = args

   143

            assert _ethernet == 'ethernet'

   144

        elif item == 'fixed-address' :

   145

            fixed_address, = args

   146

        elif item == 'option' :

   147

            option = args.pop(0)

   148

   149

            if option == 'host-name' :

   150

                hostname, = args

   151

            else :

   152

                log.warn("host %s: ignore unknown option: %s", host, option)

   153

        elif item == 'next-sever' :

   154

            boot_server, = args

   155

        elif item == 'filename' :

   156

            boot_filename, = args

   157

        else :

   158

            log.warn("host %s: ignore unknown item: %s", host, item)

   159

   160

    # determine hostname

   161

    suffix = None

   162

   163

    if '-' in host :

   164

        hostname, suffix = host.rsplit('-', 1)

   165

    else :

   166

        hostname = host

   167

   168

    if fixed_address and not re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :

   169

        hostname, domain = fixed_address.split('.', 1)

   170

   171

    if not (hostname or ethernet) :

   172

        log.warn("%s: no hostname/ethernet: %s/%s", host, hostname, ethernet)

   173

        return

   174

   175

    yield hostname, 'ethernet', ethernet

   176

    #if suffix :

   177

    #    yield hostname, ('ethernet', suffix), ethernet

   178

   179

    if boot_server and boot_filename :

   180

        yield hostname, 'boot', "{server}:{filename}".format(

   181

                server      = boot_server,

   182

                filename    = boot_filename,

   183

        )

   184

    elif boot_filename :

   185

        yield hostname, 'boot', "{filename}".format(filename=boot_filename)

   186

   187

def import_dhcp_hosts (options, blocks) :

   188

    """

   189

        Process hosts from a parsed block

   190

    """

   191

   192

    for block, items, blocks in blocks :

   193

        log.info("%s", block)

   194

   195

        block, args = block[0], block[1:]

   196

   197

        if block == 'group' :

   198

            for info in import_dhcp_hosts(options, blocks) :

   199

                yield info

   200

        elif block == 'host' :

   201

            host, = args

   202

   203

            try :

   204

                for info in import_dhcp_host(options, host, items) :

   205

                    yield info

   206

            except ValueError as error :

   207

                log.exception("%s: invalid host: %s", host, error)

   208

        else:

   209

            log.warn("ignore unknown block: %s", block)

   210

   211

def import_dhcp_conf (options, file) :

   212

    items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)

   213

   214

    for item in items :

   215

        item, args = item[0], item[1:]

   216

   217

        if item == 'include' :

   218

            include, = args

   219

            for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :

   220

                yield info

   221

        else :

   222

            log.warn("ignore unknown item: %s", item)

   223

   224

    for info in import_dhcp_hosts(options, blocks) :

   225

        yield info

   226

   227

ZONE_COMMENTS = (

   228

        re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),

   229

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),

   230

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),

   231

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),

   232

)

   233

   234

ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')

   235

   236

def process_zone_comment (options, hostname, comment) :

   237

    """

   238

        Attempt to parse a host comment field... :D

   239

   240

        Yields (field, value) bits

   241

    """

   242

   243

    for regex in ZONE_COMMENTS :

   244

        match = regex.match(comment)

   245

   246

        if match :

   247

            matches = match.groupdict()

   248

   249

            log.info("%s: matched comment: %s", hostname, comment)

   250

            break

   251

    else :

   252

        if options.zone_comments_default :

   253

            log.info("%s: default comment: %s", hostname, comment)

   254

            matches = { options.zone_comments_default: comment }

   255

        else :

   256

            log.warn("%s: unknown comment: %s", hostname, comment)

   257

            return

   258

   259

    owner = matches.pop('owner', None)

   260

   261

    if owner :

   262

        mail_match = ZONE_OWNER_MAIL.match(owner)

   263

   264

        if mail_match :

   265

            mail_matches = mail_match.groupdict()

   266

   267

            owner = mail_matches['owner']

   268

            yield 'mail', mail_matches['mail'].strip()

   269

   270

        yield 'owner', owner.strip()

   271

   272

    for field, value in matches.iteritems() :

   273

        if value :

   274

            yield field, value.strip()

   275

   276

NONE_OWNERS = set((

   277

    u'tech',

   278

    u'atk',

   279

    u'toimisto',

   280

))

   281

   282

def process_host_owner_ldap (options, host, info) :

   283

    """

   284

        Yield guesses for user from LDAP.

   285

    """

   286

   287

    if info.get('mail') :

   288

        for user in options.ldap.users.filter(

   289

                { 'mailLocalAddress': info['mail'] },

   290

                { 'uid': info['mail'] },

   291

        ) :

   292

            yield user, None

   293

   294

    if info.get('group') and info.get('owner') :

   295

        groups = options.ldap.groups.filter(cn=info['group'])

   296

   297

        for group in groups :

   298

            for user in options.ldap.users.filter({

   299

                'gidNumber': group['gidNumber'],

   300

                'cn': info['owner'],

   301

            }) :

   302

                yield user, group

   303

   304

    if info.get('owner') :

   305

            for user in options.ldap.users.filter({

   306

                'cn': info['owner'],

   307

            }) :

   308

                yield user, None

   309

   310

def process_host_owner (options, host, info) :

   311

    """

   312

        Return (owner, comment) for host based on info, or None.

   313

    """

   314

   315

    owner = info.get('owner')

   316

   317

    if owner and owner.lower() in NONE_OWNERS :

   318

        return False

   319

   320

    # from ldap?

   321

    for ldap in process_host_owner_ldap(options, host, info) :

   322

        user, group = ldap

   323

   324

        if not group :

   325

            # get group from ldap

   326

            group = options.ldap.users.group(user)

   327

   328

        return user['uid'], u"{group} / {user}".format(

   329

                user    = user.getunicode('cn'),

   330

                group   = group.getunicode('cn'),

   331

        )

   332

   333

def process_host_comments (options, host, info) :

   334

    """

   335

        Process host fields from comment.

   336

   337

        Attempts to find owner from LDAP..

   338

    """

   339

   340

    log.debug("%s: %s", host, info)

   341

   342

    owner = process_host_owner(options, host, info) 

   343

   344

    if owner is False :

   345

        # do not mark any owner

   346

        pass

   347

   348

    elif owner :

   349

        owner, comment = owner

   350

   351

        log.info("%s: %s (%s)", host, owner, comment)

   352

   353

        yield 'comment-owner', comment

   354

        yield 'owner', owner,

   355

   356

    elif 'group' in info or 'owner' in info :

   357

        log.warn("%s: unknown owner: %s", host, info)

   358

        yield 'comment-owner', "{group} / {owner}".format(

   359

                group   = info.get('group', ''),

   360

                owner   = info.get('owner', ''),

   361

        )

   362

   363

    if info.get('host') :

   364

        yield 'comment-host', info['host']

   365

   366

def process_hosts_comments (options, import_hosts) :

   367

    """

   368

        Parse out comments from host imports..

   369

    """

   370

   371

    for host, field, value in import_hosts :

   372

        if field != 'comment':

   373

            yield host, field, value

   374

            continue

   375

   376

        fields = dict(process_zone_comment(options, host, value))

   377

   378

        if options.dump_host_comments :

   379

            print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(

   380

                    host        = host,

   381

                    comment     = value,

   382

                    group       = fields.get('group', ''),

   383

                    owner       = fields.get('owner', ''),

   384

                    mail        = fields.get('mail', ''),

   385

                    hostinfo    = fields.get('host', ''),

   386

            ).encode('utf-8')

   387

   388

   389

        for field, value in process_host_comments(options, host, fields) :

   390

            yield host, field, value

   391

   392

def apply_hosts_import (options) :

   393

    """

   394

        Import host infos from given files.

   395

    """

   396

   397

    if options.import_zone_hosts:

   398

        for info in import_zone_hosts(options,

   399

                pvl.args.apply_file(options.import_zone_hosts, 'r', options.input_charset)) :

   400

            yield info

   401

   402

    if options.import_dhcp_hosts:

   403

        for info in import_dhcp_conf(options,

   404

                pvl.args.apply_file(options.import_dhcp_hosts, 'r', options.input_charset)) :

   405

            yield info

   406

   407

def import_hosts (options) :

   408

    """

   409

        Import hosts from dns/dhcp.

   410

    """

   411

   412

    import_hosts = apply_hosts_import(options)

   413

    import_hosts = process_hosts_comments(options, import_hosts)

   414

   415

    # gather

   416

    hosts = collections.defaultdict(lambda: collections.defaultdict(list))

   417

   418

    for host, field, value in import_hosts :

   419

        hosts[host][field].append(value)

   420

   421

    return hosts.iteritems()

   422

   423

def process_export_hosts (options, hosts) :

   424

    if options.output_prefix :

   425

        prefix = ipaddr.IPNetwork(options.output_prefix)

   426

    else :

   427

        prefix = None

   428

   429

    for host, fields in hosts :

   430

        ip = fields.get('ip')

   431

   432

        # sort by IP

   433

        if ip :

   434

            sort = ip[0]

   435

        else :

   436

            # fake, to sort correctly

   437

            sort = ipaddr.IPAddress(0)

   438

   439

        # select

   440

        if prefix:

   441

            if not (ip and ip in prefix) :

   442

                continue

   443

   444

        yield sort, host, fields

   445

   446

def export_hosts (options, hosts) :

   447

    """

   448

        Export hosts to file.

   449

    """

   450

   451

    file = pvl.args.apply_file(options.output_hosts, 'w', options.output_charset)

   452

   453

    # filter + sort

   454

    hosts = [(host, fields) for sort, host, fields in sorted(process_export_hosts(options, hosts))]

   455

   456

    for host, fields in hosts :

   457

        for comment in fields.get('comment-host', ()):

   458

            print >>file, u"# {comment}".format(comment=comment)

   459

   460

        print >>file, u"[{host}]".format(host=host)

   461

   462

        for field, fmt in (

   463

                ('ip',              None),

   465

                ('ethernet',        None),

   466

                ('owner',           u"\t{field:15} = {value} # {fields[comment-owner][0]}"),

   467

                ('alias',           None),

   468

                ('boot',            None),

   469

        ) :

   470

            if not fmt :

   471

                fmt = u"\t{field:15} = {value}"

   472

   473

            values = fields.get(field, ())

   474

   475

            if len(values) > 1 :

   476

                for index, value in enumerate(values, 1) :

   477

                    print >>file, fmt.format(

   478

                            field   = "{field}.{index}".format(field=field, index=index),

   479

                            value   = value,

   480

                            fields  = fields

   481

                    )

   482

            elif len(values) > 0 :

   483

                value, = values

   484

                print >>file, fmt.format(field=field, value=value, fields=fields)

   485

   486

        print >>file

   487

   488

def main (argv) :

   489

    options, args = parse_options(argv)

   490

   491

    options.ldap = pvl.ldap.args.apply(options)

   492

   493

    if args :

   494

        # direct from file

   495

        hosts = pvl.args.apply_files(args, 'r', options.input_charset)

   496

    else :

   497

        # import

   498

        hosts = import_hosts(options)

   499