1

#!/usr/bin/env python

     2

     3

"""

     4

    Import hosts from existing BIND or dhcpd files.

     5

"""

     6

     7

import pvl.args

     8

import pvl.dns.zone

     9

import pvl.dhcp.config

    10

import pvl.hosts

    11

import pvl.ldap.args

    12

    13

import collections

    14

import ipaddr

    15

import logging; log = logging.getLogger('pvl.hosts-import')

    16

import optparse

    17

import os.path

    18

import re

    19

    20

__version__ = '0.1'

    21

    22

def parse_options (argv) :

    23

    """

    24

        Parse command-line arguments.

    25

    """

    26

    27

    parser = optparse.OptionParser(

    28

            prog        = argv[0],

    29

            usage       = '%prog: [options]',

    30

            version     = __version__,

    31

    32

            # module docstring

    33

            description = __doc__,

    34

    )

    35

    36

    # logging

    37

    parser.add_option_group(pvl.args.parser(parser))

    38

    parser.add_option_group(pvl.ldap.args.parser(parser))

    39

    40

    parser.add_option('-c', '--input-charset',  metavar='CHARSET',  default='utf-8', 

    41

            help="Encoding used for input files")

    42

    43

    parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 

    44

            help="Encoding used for output files")

    45

    46

    # input

    47

    parser.add_option('--import-zone-hosts',    metavar='FILE',     action='append',

    48

            help="Load hosts from DNS zone")

    49

    50

    parser.add_option('--import-zone-origin',   metavar='ORIGIN',

    51

            help="Initial origin for given zone file; default is basename")

    52

    53

    parser.add_option('--import-zone-comments-owner',  action='store_const',

    54

            dest='import_zone_comments', const='owner',

    55

            help="Import DNS zone comment as owner comment")

    56

    57

    parser.add_option('--import-zone-comments-host',   action='store_const',

    58

            dest='import_zone_comments', const='host',

    59

            help="Import DNS zone comment as host comment")

    60

    61

    parser.add_option('--import-dhcp-hosts',    metavar='FILE',     action='append',

    62

            help="Load hosts from DHCP config")

    63

    64

    parser.add_option('--import-dhcp-boot-server',      metavar='NEXT-SERVER',

    65

            help="Default boot_server for dpc hosts")

    66

    67

    68

    parser.add_option('--dump-host-comments',   action='store_true',

    69

            help="Dump out info on imported host comments")

    70

    71

    # defaults

    72

    parser.add_option('--zone-unused',          metavar='HOST',

    73

            help="DNS name for unallocated hosts")

    74

    75

    # output

    76

    parser.add_option('--output-hosts',         metavar='FILE',         default='-',

    77

            help="Output hosts file")

    78

    79

    parser.add_option('--output-prefix',        metavar='PREFIX',

    80

            help="Select hosts by ip prefix")

    81

    82

    parser.add_option('--output-domain',        metavar='DOMAIN',

    83

            help="Select hosts by domain")

    84

    85

    # defaults

    86

    parser.set_defaults(

    87

        import_zone_hosts   = [],

    88

        import_dhcp_hosts   = [],

    89

    )

    90

    91

    # parse

    92

    options, args = parser.parse_args(argv[1:])

    93

    94

    # apply

    95

    pvl.args.apply(options, argv[0])

    96

    97

    return options, args

    98

    99

def import_zone_host_name (options, name, origin) :

   100

    """

   101

        Import zone name from rr

   102

    """

   103

   104

    if '.' in name :

   105

        host, domain = name.split('.', 1)

   106

        domain = pvl.dns.join(domain, origin)

   107

    else :

   108

        host = name

   109

        domain = origin

   110

   111

    if domain :

   112

        # not a fqdn

   113

        domain = domain.rstrip('.')

   114

   115

        log.info("%s: %s@%s", name, host, domain)

   116

    else :

   117

        log.warn("%s: no domain", name)

   118

   119

    return host, domain

   120

   121

def import_zone_hosts (options, file) :

   122

    """

   123

        Yield host info from zonefile records.

   124

    """

   125

   126

    origin = options.import_zone_origin or os.path.basename(file.name)

   127

   128

    for line, rr in pvl.dns.zone.ZoneLine.load(file,

   129

            # used to determine domain

   130

            origin          = origin,

   131

   132

            # lazy-import generated hosts on demand

   133

            expand_generate = True,

   134

    ) :

   135

        if rr :

   136

            pass

   137

        elif line.parts[0] == '$ORIGIN' :

   138

            # handled by ZoneLine.load

   139

            continue

   140

        else :

   141

            log.warn("%s: skip non-rr line: %s", line, line.line)

   142

            continue

   143

   144

        host, domain = import_zone_host_name(options, rr.name, rr.origin)

   145

   146

        if options.zone_unused and rr.name == options.zone_unused :

   147

            log.debug("%s: skip %s", rr.name, rr)

   148

            continue

   149

   150

        elif rr.type in ('A', 'AAAA') :

   151

            ip, = rr.data

   152

   153

            type = { 'A': 'ip', 'AAAA': 'ip6' }[rr.type]

   154

   155

            yield (host, domain), type, ipaddr.IPAddress(ip)

   156

   157

            if rr.comment :

   158

                yield (host, domain), 'comment', rr.comment

   159

   160

            if line.parts[0] == '$GENERATE' :

   161

                # only import as host if used for other purposes as well

   162

                yield (host, domain), 'lazy-import', True

   163

   164

        elif rr.type == 'CNAME' :

   165

            alias, = rr.data

   166

            alias_host, alias_domain = import_zone_host_name(options, alias, rr.origin)

   167

   168

            if domain == alias_domain :

   169

                yield (alias_host, alias_domain), 'alias', host

   170

            else :

   171

                yield (alias_host, alias_domain), 'alias', pvl.dns.join(host, domain)

   172

   173

        elif rr.type == 'TXT' :

   174

            txt, = rr.data

   175

   176

            yield (host, domain), 'comment', txt

   177

   178

        else :

   179

            log.warn("%s: unknown rr: %s", host, rr)

   180

   181

def import_dhcp_host (options, dhcp_host, items) :

   182

    """

   183

        Yield host infos from a dhcp host ... { ... }

   184

    """

   185

   186

    host_name = None

   187

    ethernet = []

   188

    fixed_address = None

   189

   190

    boot_server = options.import_dhcp_boot_server

   191

    boot_filename = None

   192

   193

    for item in items :

   194

        item, args = item[0], item[1:]

   195

   196

        if item == 'hardware' :

   197

            _ethernet, ethernet = args

   198

            assert _ethernet == 'ethernet'

   199

        elif item == 'fixed-address' :

   200

            fixed_address, = args

   201

        elif item == 'option' :

   202

            option = args.pop(0)

   203

   204

            if option == 'host-name' :

   205

                host_name, = args

   206

            else :

   207

                log.warn("host %s: ignore unknown option: %s", dhcp_host, option)

   208

        elif item == 'next-sever' :

   209

            boot_server, = args

   210

        elif item == 'filename' :

   211

            boot_filename, = args

   212

        else :

   213

            log.warn("host %s: ignore unknown item: %s", dhcp_host, item)

   214

   215

    # determine host

   216

    host = None

   217

    domain = None

   218

    suffix = None

   219

   220

    if not fixed_address :

   221

        log.warn("%s: fixed-address is missing, unable to determine hostname/domainname", dhcp_host)

   222

    elif re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :

   223

        log.warn("%s: fixed-address is an IP, unable to determine hostname/domainname", dhcp_host)

   224

    else :

   225

        host, domain = fixed_address.split('.', 1)

   226

   227

    # XXX: not actually true... eh

   228

    if host and dhcp_host.lower() == host.lower() :

   229

        # do not split suffix from host

   230

        pass

   231

    elif host and '-' in dhcp_host :

   232

        dhcp_host, suffix = dhcp_host.rsplit('-', 1)

   233

    elif '-' in dhcp_host :

   234

        host, suffix = dhcp_host.rsplit('-', 1)

   235

    else :

   236

        host = dhcp_host

   237

   238

    if not (host or ethernet) :

   239

        log.warn("%s: no hostname/ethernet: %s/%s", dhcp_host, hostname, ethernet)

   240

    elif suffix :

   241

        log.info("%s: %s@%s: %s: %s", dhcp_host, host, domain, suffix, ethernet)

   242

        yield (host, domain), 'ethernet.{suffix}'.format(suffix=suffix), ethernet

   243

    else :

   244

        log.info("%s: %s@%s: %s", dhcp_host, host, domain, ethernet)

   245

        yield (host, domain), 'ethernet', ethernet

   246

   247

    if boot_server and boot_filename :

   248

        yield (host, domain), 'boot', "{server}:{filename}".format(

   249

                server      = boot_server,

   250

                filename    = boot_filename,

   251

        )

   252

    elif boot_filename :

   253

        yield (host, domain), 'boot', "{filename}".format(filename=boot_filename)

   254

   255

def import_dhcp_hosts (options, file_name, blocks) :

   256

    """

   257

        Process hosts from a parsed block

   258

    """

   259

   260

    for block, items, blocks in blocks :

   261

   262

        block, args = block[0], block[1:]

   263

   264

        if block == 'group' :

   265

            log.info("%s: group", file_name)

   266

            for info in import_dhcp_hosts(options, file_name, blocks) :

   267

                yield info

   268

        elif block == 'host' :

   269

            host, = args

   270

   271

            log.info("%s: host: %s", file_name, host)

   272

   273

            try :

   274

                for info in import_dhcp_host(options, host, items) :

   275

                    yield info

   276

            except ValueError as error :

   277

                log.exception("%s: invalid host %s: %s", file_name, host, error)

   278

        else:

   279

            log.warn("%s: ignore unknown block: %s", file_name, block)

   280

   281

def import_dhcp_conf (options, file) :

   282

    items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)

   283

   284

    for item in items :

   285

        item, args = item[0], item[1:]

   286

   287

        if item == 'include' :

   288

            include, = args

   289

            for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :

   290

                yield info

   291

        else :

   292

            log.warn("ignore unknown item: %s", item)

   293

   294

    for info in import_dhcp_hosts(options, file.name, blocks) :

   295

        yield info

   296

   297

ZONE_COMMENTS = (

   298

        re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),

   299

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),

   300

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),

   301

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),

   302

)

   303

   304

ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')

   305

   306

def process_zone_comment (options, hostname, comment) :

   307

    """

   308

        Attempt to parse a host comment field... :D

   309

   310

        Yields (field, value) bits

   311

    """

   312

   313

    for regex in ZONE_COMMENTS :

   314

        match = regex.match(comment)

   315

   316

        if match :

   317

            matches = match.groupdict()

   318

   319

            log.info("%s: matched comment: %s", hostname, comment)

   320

            break

   321

    else :

   322

        if options.import_zone_comments :

   323

            log.info("%s: default comment: %s", hostname, comment)

   324

            matches = { options.import_zone_comments: comment }

   325

        else :

   326

            log.warn("%s: unknown comment: %s", hostname, comment)

   327

            return

   328

   329

    owner = matches.pop('owner', None)

   330

   331

    if owner :

   332

        mail_match = ZONE_OWNER_MAIL.match(owner)

   333

   334

        if mail_match :

   335

            mail_matches = mail_match.groupdict()

   336

   337

            owner = mail_matches['owner']

   338

            yield 'mail', mail_matches['mail'].strip()

   339

   340

        yield 'owner', owner.strip()

   341

   342

    for field, value in matches.iteritems() :

   343

        if value :

   344

            yield field, value.strip()

   345

   346

NONE_OWNERS = set((

   347

    u'tech',

   348

    u'atk',

   349

    u'toimisto',

   350

))

   351

   352

def process_host_owner_ldap (options, host, info) :

   353

    """

   354

        Yield guesses for user from LDAP.

   355

    """

   356

   357

    if info.get('mail') :

   358

        for user in options.ldap.users.filter(

   359

                { 'mailLocalAddress': info['mail'] },

   360

                { 'uid': info['mail'] },

   361

        ) :

   362

            yield user, None

   363

   364

    if info.get('group') and info.get('owner') :

   365

        groups = options.ldap.groups.filter(cn=info['group'])

   366

   367

        for group in groups :

   368

            for user in options.ldap.users.filter({

   369

                'gidNumber': group['gidNumber'],

   370

                'cn': info['owner'],

   371

            }) :

   372

                yield user, group

   373

   374

    if info.get('owner') :

   375

            for user in options.ldap.users.filter({

   376

                'cn': info['owner'],

   377

            }) :

   378

                yield user, None

   379

   380

def process_host_owner (options, host, info) :

   381

    """

   382

        Return (owner, comment) for host based on info, or None.

   383

    """

   384

   385

    owner = info.get('owner')

   386

   387

    if owner and owner.lower() in NONE_OWNERS :

   388

        return False

   389

   390

    # from ldap?

   391

    for ldap in process_host_owner_ldap(options, host, info) :

   392

        user, group = ldap

   393

   394

        if not group :

   395

            # get group from ldap

   396

            group = options.ldap.users.group(user)

   397

   398

        return user['uid'], u"{group} / {user}".format(

   399

                user    = user.getunicode('cn'),

   400

                group   = group.getunicode('cn'),

   401

        )

   402

   403

def process_host_comments (options, host, info) :

   404

    """

   405

        Process host fields from comment.

   406

   407

        Attempts to find owner from LDAP..

   408

    """

   409

   410

    log.debug("%s: %s", host, info)

   411

   412

    owner = process_host_owner(options, host, info) 

   413

   414

    if owner is False :

   415

        # do not mark any owner

   416

        pass

   417

   418

    elif owner :

   419

        owner, comment = owner

   420

   421

        log.info("%s: %s (%s)", host, owner, comment)

   422

   423

        yield 'comment.owner', comment

   424

        yield 'owner', owner,

   425

   426

    elif 'group' in info or 'owner' in info :

   427

        log.warn("%s: unknown owner: %s", host, info)

   428

        yield 'comment.owner', "{group} / {owner}".format(

   429

                group   = info.get('group', ''),

   430

                owner   = info.get('owner', ''),

   431

        )

   432

   433

    if info.get('host') :

   434

        yield 'comment.host', info['host']

   435

   436

def process_hosts_comments (options, import_hosts) :

   437

    """

   438

        Parse out comments from host imports..

   439

    """

   440

   441

    for host, field, value in import_hosts :

   442

        if field != 'comment':

   443

            yield host, field, value

   444

            continue

   445

   446

        fields = dict(process_zone_comment(options, host, value))

   447

   448

        if options.dump_host_comments :

   449

            print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(

   450

                    host        = host,

   451

                    comment     = value,

   452

                    group       = fields.get('group', ''),

   453

                    owner       = fields.get('owner', ''),

   454

                    mail        = fields.get('mail', ''),

   455

                    hostinfo    = fields.get('host', ''),

   456

            ).encode('utf-8')

   457

   458

   459

        for field, value in process_host_comments(options, host, fields) :

   460

            yield host, field, value

   461

   462

def import_hosts_files (options, zone_files, dhcp_files) :

   463

    """

   464

        Import host infos from given files.

   465

    """

   466

   467

    for zone_file in zone_files:

   468

        file = pvl.args.apply_file(zone_file, 'r', options.input_charset)

   469

        for info in import_zone_hosts(options, file) :

   470

            yield info

   471

   472

    for dhcp_file in dhcp_files :

   473

        file = pvl.args.apply_file(dhcp_file, 'r', options.input_charset)

   474

        for info in import_dhcp_conf(options, file) :

   475

            yield info

   476

   477

def process_import_hosts (options, import_hosts) :

   478

    """

   479

        Build hosts from imported fields.

   480

   481

        Yields (domain, host), { (field, ...): value }

   482

    """

   483

   484

    # gather

   485

    hosts = collections.defaultdict(lambda: collections.defaultdict(list))

   486

   487

    for (host, domain), field, value in import_hosts :

   488

        hosts[domain, host][tuple(field.split('.'))].append(value)

   489

   490

    # process

   491

    for (domain, host), fields in hosts.iteritems() :

   492

        SINGLE_FIELDS = (

   493

                'ip',

   494

                'ip6',