1

#!/usr/bin/env python

     2

     3

"""

     4

    Import hosts from existing BIND or dhcpd files.

     5

"""

     6

     7

import pvl.args

     8

import pvl.dns.zone

     9

import pvl.dhcp.config

    11

import pvl.ldap.args

    12

    13

import collections

    14

import ipaddr

    15

import logging; log = logging.getLogger('pvl.hosts-import')

    16

import optparse

    17

import os.path

    18

import re

    19

    20

__version__ = '0.1'

    21

    22

def parse_options (argv) :

    23

    """

    24

        Parse command-line arguments.

    25

    """

    26

    27

    parser = optparse.OptionParser(

    28

            prog        = argv[0],

    29

            usage       = '%prog: [options]',

    30

            version     = __version__,

    31

    32

            # module docstring

    33

            description = __doc__,

    34

    )

    35

    36

    # logging

    37

    parser.add_option_group(pvl.args.parser(parser))

    38

    parser.add_option_group(pvl.ldap.args.parser(parser))

    39

    40

    parser.add_option('-c', '--input-charset',  metavar='CHARSET',  default='utf-8', 

    41

            help="Encoding used for input files")

    42

    43

    parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 

    44

            help="Encoding used for output files")

    45

    46

    # input

    47

    parser.add_option('--import-zone-hosts',    metavar='FILE',     action='append',

    48

            help="Load hosts from DNS zone")

    49

    50

    parser.add_option('--import-zone-origin',   metavar='ORIGIN',

    51

            help="Initial origin for given zone file; default is basename")

    52

    53

    parser.add_option('--import-zone-comments-owner',  action='store_const',

    54

            dest='import_zone_comments', const='owner',

    55

            help="Import DNS zone comment as owner comment")

    56

    57

    parser.add_option('--import-zone-comments-host',   action='store_const',

    58

            dest='import_zone_comments', const='host',

    59

            help="Import DNS zone comment as host comment")

    60

    61

    parser.add_option('--import-dhcp-hosts',    metavar='FILE',     action='append',

    62

            help="Load hosts from DHCP config")

    63

    64

    parser.add_option('--import-dhcp-boot-server',      metavar='NEXT-SERVER',

    65

            help="Default boot_server for dpc hosts")

    66

    67

    68

    parser.add_option('--dump-host-comments',   action='store_true',

    69

            help="Dump out info on imported host comments")

    70

    71

    # defaults

    72

    parser.add_option('--zone-unused',          metavar='HOST',

    73

            help="DNS name for unallocated hosts")

    74

    75

    # output

    76

    parser.add_option('--output-hosts',         metavar='FILE',         default='-',

    77

            help="Output hosts file")

    78

    79

    parser.add_option('--output-prefix',        metavar='PREFIX',

    80

            help="Select hosts by ip prefix")

    81

    82

    parser.add_option('--output-domain',        metavar='DOMAIN',

    83

            help="Select hosts by domain")

    84

    85

    # defaults

    86

    parser.set_defaults(

    87

        import_zone_hosts   = [],

    88

        import_dhcp_hosts   = [],

    89

    )

    90

    91

    # parse

    92

    options, args = parser.parse_args(argv[1:])

    93

    94

    # apply

    95

    pvl.args.apply(options, argv[0])

    96

    97

    return options, args

    98

    99

def import_zone_host_name (options, name, origin) :

   100

    """

   101

        Import zone name from rr

   102

    """

   103

   104

    if '.' in name :

   105

        host, domain = name.split('.', 1)

   106

        domain = pvl.dns.join(domain, origin)

   107

    else :

   108

        host = name

   109

        domain = origin

   110

   111

    if domain :

   112

        # not a fqdn

   113

        domain = domain.rstrip('.')

   114

   115

        log.info("%s: %s@%s", name, host, domain)

   116

    else :

   117

        log.warn("%s: no domain", name)

   118

   119

    return host, domain

   120

   121

def import_zone_hosts (options, file) :

   122

    """

   123

        Yield host info from zonefile records.

   124

    """

   125

   126

    origin = options.import_zone_origin or os.path.basename(file.name)

   127

   128

    for rr in pvl.dns.zone.ZoneRecord.load(file,

   129

            # used to determine domain

   130

            origin          = origin,

   131

   132

            # generated hosts need to imported by hand...

   133

            expand_generate = False,

   134

    ) :

   135

        host, domain = import_zone_host_name(options, rr.name, rr.origin)

   136

   137

        if options.zone_unused and rr.name == options.zone_unused :

   138

            log.debug("%s: skip %s", rr.name, rr)

   139

            continue

   140

   141

        elif rr.type in ('A', 'AAAA') :

   142

            ip, = rr.data

   143

   144

            type = { 'A': 'ip', 'AAAA': 'ip6' }[rr.type]

   145

   146

            yield (host, domain), type, ipaddr.IPAddress(ip)

   147

   148

            if rr.comment :

   149

                yield (host, domain), 'comment', rr.comment

   150

   151

        elif rr.type == 'CNAME' :

   152

            alias, = rr.data

   153

            alias_host, alias_domain = import_zone_host_name(options, alias, rr.origin)

   154

   155

            if domain == alias_domain :

   156

                yield (alias_host, alias_domain), 'alias', host

   157

            else :

   158

                yield (alias_host, alias_domain), 'alias', pvl.dns.join(host, domain)

   159

   160

        elif rr.type == 'TXT' :

   161

            txt, = rr.data

   162

   163

            yield (host, domain), 'comment', txt

   164

   165

        else :

   166

            log.warn("%s: unknown rr: %s", host, rr)

   167

   168

def import_dhcp_host (options, dhcp_host, items) :

   169

    """

   170

        Yield host infos from a dhcp host ... { ... }

   171

    """

   172

   173

    host_name = None

   174

    ethernet = []

   175

    fixed_address = None

   176

   177

    boot_server = options.import_dhcp_boot_server

   178

    boot_filename = None

   179

   180

    for item in items :

   181

        item, args = item[0], item[1:]

   182

   183

        if item == 'hardware' :

   184

            _ethernet, ethernet = args

   185

            assert _ethernet == 'ethernet'

   186

        elif item == 'fixed-address' :

   187

            fixed_address, = args

   188

        elif item == 'option' :

   189

            option = args.pop(0)

   190

   191

            if option == 'host-name' :

   192

                host_name, = args

   193

            else :

   194

                log.warn("host %s: ignore unknown option: %s", dhcp_host, option)

   195

        elif item == 'next-sever' :

   196

            boot_server, = args

   197

        elif item == 'filename' :

   198

            boot_filename, = args

   199

        else :

   200

            log.warn("host %s: ignore unknown item: %s", dhcp_host, item)

   201

   202

    # determine host

   203

    host = None

   204

    domain = None

   205

    suffix = None

   206

   207

    if not fixed_address :

   208

        log.warn("%s: fixed-address is missing, unable to determine hostname/domainname", dhcp_host)

   209

    elif re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :

   210

        log.warn("%s: fixed-address is an IP, unable to determine hostname/domainname", dhcp_host)

   211

    else :

   212

        host, domain = fixed_address.split('.', 1)

   213

   214

    # XXX: not actually true... eh

   215

    if host and dhcp_host.lower() == host.lower() :

   216

        # do not split suffix from host

   217

        pass

   218

    elif host and '-' in dhcp_host :

   219

        dhcp_host, suffix = dhcp_host.rsplit('-', 1)

   220

    elif '-' in dhcp_host :

   221

        host, suffix = dhcp_host.rsplit('-', 1)

   222

    else :

   223

        host = dhcp_host

   224

   225

    if not (host or ethernet) :

   226

        log.warn("%s: no hostname/ethernet: %s/%s", dhcp_host, hostname, ethernet)

   227

    elif suffix :

   228

        log.info("%s: %s@%s: %s: %s", dhcp_host, host, domain, suffix, ethernet)

   229

        yield (host, domain), 'ethernet.{suffix}'.format(suffix=suffix), ethernet

   230

    else :

   231

        log.info("%s: %s@%s: %s", dhcp_host, host, domain, ethernet)

   232

        yield (host, domain), 'ethernet', ethernet

   233

   234

    if boot_server and boot_filename :

   235

        yield (host, domain), 'boot', "{server}:{filename}".format(

   236

                server      = boot_server,

   237

                filename    = boot_filename,

   238

        )

   239

    elif boot_filename :

   240

        yield (host, domain), 'boot', "{filename}".format(filename=boot_filename)

   241

   242

def import_dhcp_hosts (options, file_name, blocks) :

   243

    """

   244

        Process hosts from a parsed block

   245

    """

   246

   247

    for block, items, blocks in blocks :

   248

   249

        block, args = block[0], block[1:]

   250

   251

        if block == 'group' :

   252

            log.info("%s: group", file_name)

   253

            for info in import_dhcp_hosts(options, file_name, blocks) :

   254

                yield info

   255

        elif block == 'host' :

   256

            host, = args

   257

   258

            log.info("%s: host: %s", file_name, host)

   259

   260

            try :

   261

                for info in import_dhcp_host(options, host, items) :

   262

                    yield info

   263

            except ValueError as error :

   264

                log.exception("%s: invalid host %s: %s", file_name, host, error)

   265

        else:

   266

            log.warn("%s: ignore unknown block: %s", file_name, block)

   267

   268

def import_dhcp_conf (options, file) :

   269

    items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)

   270

   271

    for item in items :

   272

        item, args = item[0], item[1:]

   273

   274

        if item == 'include' :

   275

            include, = args

   276

            for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :

   277

                yield info

   278

        else :

   279

            log.warn("ignore unknown item: %s", item)

   280

   281

    for info in import_dhcp_hosts(options, file.name, blocks) :

   282

        yield info

   283

   284

ZONE_COMMENTS = (

   285

        re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),

   286

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),

   287

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),

   288

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),

   289

)

   290

   291

ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')

   292

   293

def process_zone_comment (options, hostname, comment) :

   294

    """

   295

        Attempt to parse a host comment field... :D

   296

   297

        Yields (field, value) bits

   298

    """

   299

   300

    for regex in ZONE_COMMENTS :

   301

        match = regex.match(comment)

   302

   303

        if match :

   304

            matches = match.groupdict()

   305

   306

            log.info("%s: matched comment: %s", hostname, comment)

   307

            break

   308

    else :

   309

        if options.import_zone_comments :

   310

            log.info("%s: default comment: %s", hostname, comment)

   311

            matches = { options.import_zone_comments: comment }

   312

        else :

   313

            log.warn("%s: unknown comment: %s", hostname, comment)

   314

            return

   315

   316

    owner = matches.pop('owner', None)

   317

   318

    if owner :

   319

        mail_match = ZONE_OWNER_MAIL.match(owner)

   320

   321

        if mail_match :

   322

            mail_matches = mail_match.groupdict()

   323

   324

            owner = mail_matches['owner']

   325

            yield 'mail', mail_matches['mail'].strip()

   326

   327

        yield 'owner', owner.strip()

   328

   329

    for field, value in matches.iteritems() :

   330

        if value :

   331

            yield field, value.strip()

   332

   333

NONE_OWNERS = set((

   334

    u'tech',

   335

    u'atk',

   336

    u'toimisto',

   337

))

   338

   339

def process_host_owner_ldap (options, host, info) :

   340

    """

   341

        Yield guesses for user from LDAP.

   342

    """

   343

   344

    if info.get('mail') :

   345

        for user in options.ldap.users.filter(

   346

                { 'mailLocalAddress': info['mail'] },

   347

                { 'uid': info['mail'] },

   348

        ) :

   349

            yield user, None

   350

   351

    if info.get('group') and info.get('owner') :

   352

        groups = options.ldap.groups.filter(cn=info['group'])

   353

   354

        for group in groups :

   355

            for user in options.ldap.users.filter({

   356

                'gidNumber': group['gidNumber'],

   357

                'cn': info['owner'],

   358

            }) :

   359

                yield user, group

   360

   361

    if info.get('owner') :

   362

            for user in options.ldap.users.filter({

   363

                'cn': info['owner'],

   364

            }) :

   365

                yield user, None

   366

   367

def process_host_owner (options, host, info) :

   368

    """

   369

        Return (owner, comment) for host based on info, or None.

   370

    """

   371

   372

    owner = info.get('owner')

   373

   374

    if owner and owner.lower() in NONE_OWNERS :

   375

        return False

   376

   377

    # from ldap?

   378

    for ldap in process_host_owner_ldap(options, host, info) :

   379

        user, group = ldap

   380

   381

        if not group :

   382

            # get group from ldap

   383

            group = options.ldap.users.group(user)

   384

   385

        return user['uid'], u"{group} / {user}".format(

   386

                user    = user.getunicode('cn'),

   387

                group   = group.getunicode('cn'),

   388

        )

   389

   390

def process_host_comments (options, host, info) :

   391

    """

   392

        Process host fields from comment.

   393

   394

        Attempts to find owner from LDAP..

   395

    """

   396

   397

    log.debug("%s: %s", host, info)

   398

   399

    owner = process_host_owner(options, host, info) 

   400

   401

    if owner is False :

   402

        # do not mark any owner

   403

        pass

   404

   405

    elif owner :

   406

        owner, comment = owner

   407

   408

        log.info("%s: %s (%s)", host, owner, comment)

   409

   410

        yield 'comment.owner', comment

   411

        yield 'owner', owner,

   412

   413

    elif 'group' in info or 'owner' in info :

   414

        log.warn("%s: unknown owner: %s", host, info)

   415

        yield 'comment.owner', "{group} / {owner}".format(

   416

                group   = info.get('group', ''),

   417

                owner   = info.get('owner', ''),

   418

        )

   419

   420

    if info.get('host') :

   421

        yield 'comment.host', info['host']

   422

   423

def process_hosts_comments (options, import_hosts) :

   424

    """

   425

        Parse out comments from host imports..

   426

    """

   427

   428

    for host, field, value in import_hosts :

   429

        if field != 'comment':

   430

            yield host, field, value

   431

            continue

   432

   433

        fields = dict(process_zone_comment(options, host, value))

   434

   435

        if options.dump_host_comments :

   436

            print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(

   437

                    host        = host,

   438

                    comment     = value,

   439

                    group       = fields.get('group', ''),

   440

                    owner       = fields.get('owner', ''),

   441

                    mail        = fields.get('mail', ''),

   442

                    hostinfo    = fields.get('host', ''),

   443

            ).encode('utf-8')

   444

   445

   446

        for field, value in process_host_comments(options, host, fields) :

   447

            yield host, field, value

   448

   449

def import_hosts_files (options, zone_files, dhcp_files) :

   450

    """

   451

        Import host infos from given files.

   452

    """

   453

   454

    for zone_file in zone_files:

   455

        file = pvl.args.apply_file(zone_file, 'r', options.input_charset)

   456

        for info in import_zone_hosts(options, file) :

   457

            yield info

   458

   459

    for dhcp_file in dhcp_files :

   460

        file = pvl.args.apply_file(dhcp_file, 'r', options.input_charset)

   461

        for info in import_dhcp_conf(options, file) :

   462

            yield info

   463

   464

def process_import_hosts (options, import_hosts) :

   465

    """

   466

        Build hosts from imported fields.

   467

   468

        Yields (domain, host), { (field, ...): value }

   469

    """

   470

   471

    # gather

   472

    hosts = collections.defaultdict(lambda: collections.defaultdict(list))

   473

   474

    for (host, domain), field, value in import_hosts :

   475

        hosts[domain, host][tuple(field.split('.'))].append(value)

   476

   477

    # process

   478

    for (domain, host), fields in hosts.iteritems() :

   479

        SINGLE_FIELDS = (

   480

                'ip',

   481

                'ip6',

   482

                'comment.owner',

   483

                'owner',

   484

                'boot',

   485

        )

   486

        MULTI_FIELDS = (

   487

                'comment.host',

   488

                'ethernet',

   489

                'alias',

   490

        ) 

   491

        host_fields = {}

   492

   493

        for field_name in SINGLE_FIELDS :

   494

            field = tuple(field_name.split('.'))