1

#!/usr/bin/env python

     2

     3

"""

     4

    Import hosts from existing BIND or dhcpd files.

     5

"""

     6

     7

import pvl.args

     8

import pvl.dns.zone

     9

import pvl.dhcp.config

    10

import pvl.hosts

    11

import pvl.ldap.args

    12

    13

import collections

    14

import ipaddr

    15

import logging; log = logging.getLogger('pvl.hosts-import')

    16

import optparse

    17

import os.path

    18

import re

    19

    20

__version__ = '0.1'

    21

    22

def parse_options (argv) :

    23

    """

    24

        Parse command-line arguments.

    25

    """

    26

    27

    parser = optparse.OptionParser(

    28

            prog        = argv[0],

    29

            usage       = '%prog: [options]',

    30

            version     = __version__,

    31

    32

            # module docstring

    33

            description = __doc__,

    34

    )

    35

    36

    # logging

    37

    parser.add_option_group(pvl.args.parser(parser))

    38

    parser.add_option_group(pvl.ldap.args.parser(parser))

    39

    40

    parser.add_option('--input-charset',  metavar='CHARSET',  default='utf-8', 

    41

            help="Encoding used for input files")

    42

    43

    parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 

    44

            help="Encoding used for output files")

    45

    46

    # input

    47

    parser.add_option('--import-zone-hosts',    metavar='FILE',     action='append',

    48

            help="Load hosts from DNS zone")

    49

    50

    parser.add_option('--import-zone-origin',   metavar='ORIGIN',

    51

            help="Initial origin for given zone file; default is basename")

    52

    53

    parser.add_option('--import-zone-comments-owner',  action='store_const',

    54

            dest='import_zone_comments', const='owner',

    55

            help="Import DNS zone comment as owner comment")

    56

    57

    parser.add_option('--import-zone-comments-host',   action='store_const',

    58

            dest='import_zone_comments', const='host',

    59

            help="Import DNS zone comment as host comment")

    60

    61

    parser.add_option('--import-dhcp-hosts',    metavar='FILE',     action='append',

    62

            help="Load hosts from DHCP config")

    63

    64

    parser.add_option('--import-dhcp-boot-server',      metavar='NEXT-SERVER',

    65

            help="Default boot_server for dpc hosts")

    66

    67

    68

    parser.add_option('--dump-host-comments',   action='store_true',

    69

            help="Dump out info on imported host comments")

    70

    71

    # defaults

    72

    parser.add_option('--zone-unused',          metavar='HOST',

    73

            help="DNS name for unallocated hosts")

    74

    75

    # output

    76

    parser.add_option('--output-hosts',         metavar='FILE',         default='-',

    77

            help="Output hosts file")

    78

    79

    parser.add_option('--output-prefix',        metavar='PREFIX',

    80

            help="Select hosts by ip prefix")

    81

    82

    parser.add_option('--output-domain',        metavar='DOMAIN',

    83

            help="Select hosts by domain")

    84

    85

    parser.add_option('--output-others',        action='store_true',

    86

            help="Negate selection")

    87

    88

    # defaults

    89

    parser.set_defaults(

    90

        import_zone_hosts   = [],

    91

        import_dhcp_hosts   = [],

    92

    )

    93

    94

    # parse

    95

    options, args = parser.parse_args(argv[1:])

    96

    97

    # apply

    98

    pvl.args.apply(options, argv[0])

    99

   100

    return options, args

   101

   102

def import_zone_host_name (options, name, origin) :

   103

    """

   104

        Import zone name from rr

   105

    """

   106

   107

    if '.' in name :

   108

        host, domain = name.split('.', 1)

   109

        domain = pvl.dns.join(domain, origin)

   110

    else :

   111

        host = name

   112

        domain = origin

   113

   114

    if domain :

   115

        # not a fqdn

   116

        domain = domain.rstrip('.')

   117

   118

        log.info("%s: %s@%s", name, host, domain)

   119

    else :

   120

        log.warn("%s: no domain", name)

   121

   122

    return host, domain

   123

   124

def import_zone_hosts (options, file) :

   125

    """

   126

        Yield host info from zonefile records.

   127

    """

   128

   129

    origin = options.import_zone_origin or os.path.basename(file.name)

   130

   131

    for line, rr in pvl.dns.zone.ZoneLine.load(file,

   132

            # used to determine domain

   133

            origin          = origin,

   134

   135

            # lazy-import generated hosts on demand

   136

            expand_generate = True,

   137

    ) :

   138

        if rr :

   139

            pass

   140

        elif line.parts[0] == '$ORIGIN' :

   141

            # handled by ZoneLine.load

   142

            continue

   143

        else :

   144

            log.warn("%s: skip non-rr line: %s", line, line.line)

   145

            continue

   146

   147

        host, domain = import_zone_host_name(options, rr.name, rr.origin)

   148

   149

        if rr.type in ('A', 'AAAA') :

   150

            ip, = rr.data

   151

            ip = ipaddr.IPAddress(ip)

   152

   153

            type = { 'A': 'ip', 'AAAA': 'ip6' }[rr.type]

   154

   155

            if options.zone_unused and rr.name == options.zone_unused :

   156

                yield (str(ip), domain), 'ip.unused', ip

   157

            else :

   158

                yield (host, domain), type, ip,

   159

   160

            if rr.comment :

   161

                yield (host, domain), 'comment', rr.comment

   162

   163

            if line.parts[0] == '$GENERATE' :

   164

                # only import as host if used for other purposes as well

   165

                yield (host, domain), 'lazy-import', True

   166

   167

        elif rr.type == 'CNAME' :

   168

            alias, = rr.data

   169

            alias_host, alias_domain = import_zone_host_name(options, alias, rr.origin)

   170

   171

            if domain == alias_domain :

   172

                yield (alias_host, alias_domain), 'alias', host

   173

            elif domain.endswith('.' + alias_domain) :

   174

                yield (alias_host, alias_domain), 'alias', pvl.dns.join(host, domain[:(len(domain) - len(alias_domain) - 1)])

   175

            else :

   176

                log.warn("%s@%s: alias outside of target domain: %s", host, domain, alias_domain)

   177

   178

        elif rr.type == 'TXT' :

   179

            txt, = rr.data

   180

   181

            yield (host, domain), 'comment', txt

   182

   183

        else :

   184

            log.warn("%s: unknown rr: %s", host, rr)

   185

   186

def import_dhcp_host (options, dhcp_host, items) :

   187

    """

   188

        Yield host infos from a dhcp host ... { ... }

   189

    """

   190

   191

    host_name = None

   192

    ethernet = []

   193

    fixed_address = None

   194

   195

    boot_server = options.import_dhcp_boot_server

   196

    boot_filename = None

   197

   198

    for item in items :

   199

        item, args = item[0], item[1:]

   200

   201

        if item == 'hardware' :

   202

            _ethernet, ethernet = args

   203

            assert _ethernet == 'ethernet'

   204

        elif item == 'fixed-address' :

   205

            fixed_address, = args

   206

        elif item == 'option' :

   207

            option = args.pop(0)

   208

   209

            if option == 'host-name' :

   210

                host_name, = args

   211

            else :

   212

                log.warn("host %s: ignore unknown option: %s", dhcp_host, option)

   213

        elif item == 'next-sever' :

   214

            boot_server, = args

   215

        elif item == 'filename' :

   216

            boot_filename, = args

   217

        else :

   218

            log.warn("host %s: ignore unknown item: %s", dhcp_host, item)

   219

   220

    # determine host

   221

    host = None

   222

    domain = None

   223

    suffix = None

   224

   225

    if not fixed_address :

   226

        log.warn("%s: fixed-address is missing, unable to determine hostname/domainname", dhcp_host)

   227

    elif re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :

   228

        log.warn("%s: fixed-address is an IP, unable to determine hostname/domainname", dhcp_host)

   229

    else :

   230

        host, domain = fixed_address.split('.', 1)

   231

   232

    # XXX: not actually true... eh

   233

    if host and dhcp_host.lower() == host.lower() :

   234

        # do not split suffix from host

   235

        pass

   236

    elif host and '-' in dhcp_host :

   237

        dhcp_host, suffix = dhcp_host.rsplit('-', 1)

   238

    elif '-' in dhcp_host :

   239

        host, suffix = dhcp_host.rsplit('-', 1)

   240

    else :

   241

        host = dhcp_host

   242

   243

    if not (host or ethernet) :

   244

        log.warn("%s: no hostname/ethernet: %s/%s", dhcp_host, hostname, ethernet)

   245

    elif suffix :

   246

        log.info("%s: %s@%s: %s: %s", dhcp_host, host, domain, suffix, ethernet)

   247

        yield (host, domain), 'ethernet.{suffix}'.format(suffix=suffix), ethernet

   248

    else :

   249

        log.info("%s: %s@%s: %s", dhcp_host, host, domain, ethernet)

   250

        yield (host, domain), 'ethernet', ethernet

   251

   252

    if boot_server and boot_filename :

   253

        yield (host, domain), 'boot', "{server}:{filename}".format(

   254

                server      = boot_server,

   255

                filename    = boot_filename,

   256

        )

   257

    elif boot_filename :

   258

        yield (host, domain), 'boot', "{filename}".format(filename=boot_filename)

   259

   260

def import_dhcp_hosts (options, file_name, blocks) :

   261

    """

   262

        Process hosts from a parsed block

   263

    """

   264

   265

    for block, items, blocks in blocks :

   266

   267

        block, args = block[0], block[1:]

   268

   269

        if block == 'group' :

   270

            log.info("%s: group", file_name)

   271

            for info in import_dhcp_hosts(options, file_name, blocks) :

   272

                yield info

   273

        elif block == 'host' :

   274

            host, = args

   275

   276

            log.info("%s: host: %s", file_name, host)

   277

   278

            try :

   279

                for info in import_dhcp_host(options, host, items) :

   280

                    yield info

   281

            except ValueError as error :

   282

                log.exception("%s: invalid host %s: %s", file_name, host, error)

   283

        else:

   284

            log.warn("%s: ignore unknown block: %s", file_name, block)

   285

   286

def import_dhcp_conf (options, file) :

   287

    items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)

   288

   289

    for item in items :

   290

        item, args = item[0], item[1:]

   291

   292

        if item == 'include' :

   293

            include, = args

   294

            for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :

   295

                yield info

   296

        else :

   297

            log.warn("ignore unknown item: %s", item)

   298

   299

    for info in import_dhcp_hosts(options, file.name, blocks) :

   300

        yield info

   301

   302

ZONE_COMMENTS = (

   303

        re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),

   304

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),

   305

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),

   306

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),

   307

)

   308

   309

ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')

   310

   311

def process_zone_comment (options, hostname, comment) :

   312

    """

   313

        Attempt to parse a host comment field... :D

   314

   315

        Yields (field, value) bits

   316

    """

   317

   318

    for regex in ZONE_COMMENTS :

   319

        match = regex.match(comment)

   320

   321

        if match :

   322

            matches = match.groupdict()

   323

   324

            log.info("%s: matched comment: %s", hostname, comment)

   325

            break

   326

    else :

   327

        if options.import_zone_comments :

   328

            log.info("%s: default comment: %s", hostname, comment)

   329

            matches = { options.import_zone_comments: comment }

   330

        else :

   331

            log.warn("%s: unknown comment: %s", hostname, comment)

   332

            return

   333

   334

    owner = matches.pop('owner', None)

   335

   336

    if owner :

   337

        mail_match = ZONE_OWNER_MAIL.match(owner)

   338

   339

        if mail_match :

   340

            mail_matches = mail_match.groupdict()

   341

   342

            owner = mail_matches['owner']

   343

            yield 'mail', mail_matches['mail'].strip()

   344

   345

        yield 'owner', owner.strip()

   346

   347

    for field, value in matches.iteritems() :

   348

        if value :

   349

            yield field, value.strip()

   350

   351

NONE_OWNERS = set((

   352

    u'tech',

   353

    u'atk',

   354

    u'toimisto',

   355

))

   356

   357

def process_host_owner_ldap (options, host, info) :

   358

    """

   359

        Yield guesses for user from LDAP.

   360

    """

   361

   362

    if info.get('mail') :

   363

        for user in options.ldap.users.filter(

   364

                { 'mailLocalAddress': info['mail'] },

   365

                { 'uid': info['mail'] },

   366

        ) :

   367

            yield user, None

   368

   369

    if info.get('group') and info.get('owner') :

   370

        groups = options.ldap.groups.filter(cn=info['group'])

   371

   372

        for group in groups :

   373

            for user in options.ldap.users.filter({

   374

                'gidNumber': group['gidNumber'],

   375

                'cn': info['owner'],

   376

            }) :

   377

                yield user, group

   378

   379

    if info.get('owner') :

   380

            for user in options.ldap.users.filter({

   381

                'cn': info['owner'],

   382

            }) :

   383

                yield user, None

   384

   385

def process_host_owner (options, host, info) :

   386

    """

   387

        Return (owner, comment) for host based on info, or None.

   388

    """

   389

   390

    owner = info.get('owner')

   391

   392

    if owner and owner.lower() in NONE_OWNERS :

   393

        return False

   394

   395

    # from ldap?

   396

    for ldap in process_host_owner_ldap(options, host, info) :

   397

        user, group = ldap

   398

   399

        if not group :

   400

            # get group from ldap

   401

            group = options.ldap.users.group(user)

   402

   403

        return user['uid'], u"{group} / {user}".format(

   404

                user    = user.getunicode('cn'),

   405

                group   = group.getunicode('cn'),

   406

        )

   407

   408

def process_host_comments (options, host, info) :

   409

    """

   410

        Process host fields from comment.

   411

   412

        Attempts to find owner from LDAP..

   413

    """

   414

   415

    log.debug("%s: %s", host, info)

   416

   417

    owner = process_host_owner(options, host, info) 

   418

   419

    if owner is False :

   420

        # do not mark any owner

   421

        pass

   422

   423

    elif owner :

   424

        owner, comment = owner

   425

   426

        log.info("%s: %s (%s)", host, owner, comment)

   427

   428

        yield 'comment.owner', comment

   429

        yield 'owner', owner,

   430

   431

    elif 'group' in info or 'owner' in info :

   432

        log.warn("%s: unknown owner: %s", host, info)

   433

        yield 'comment.owner', "{group} / {owner}".format(

   434

                group   = info.get('group', ''),

   435

                owner   = info.get('owner', ''),

   436

        )

   437

   438

    if info.get('host') :

   439

        yield 'comment.host', info['host']

   440

   441

def process_hosts_comments (options, import_hosts) :

   442

    """

   443

        Parse out comments from host imports..

   444

    """

   445

   446

    for host, field, value in import_hosts :

   447

        if field != 'comment':

   448

            yield host, field, value

   449

            continue

   450

   451

        fields = dict(process_zone_comment(options, host, value))

   452

   453

        if options.dump_host_comments :

   454

            print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(

   455

                    host        = host,

   456

                    comment     = value,

   457

                    group       = fields.get('group', ''),

   458

                    owner       = fields.get('owner', ''),

   459

                    mail        = fields.get('mail', ''),

   460

                    hostinfo    = fields.get('host', ''),

   461

            ).encode('utf-8')

   462

   463

   464

        for field, value in process_host_comments(options, host, fields) :

   465

            yield host, field, value

   466

   467

def import_hosts_files (options, zone_files, dhcp_files) :

   468

    """

   469

        Import host infos from given files.

   470

    """

   471

   472

    for zone_file in zone_files:

   473

        file = pvl.args.apply_file(zone_file, 'r', options.input_charset)

   474

        for info in import_zone_hosts(options, file) :

   475

            yield info

   476

   477

    for dhcp_file in dhcp_files :

   478

        file = pvl.args.apply_file(dhcp_file, 'r', options.input_charset)

   479

        for info in import_dhcp_conf(options, file) :

   480

            yield info

   481

   482

def process_import_hosts (options, import_hosts) :

   483

    """

   484

        Build hosts from imported fields.

   485

   486

        Yields (domain, host), { (field, ...): value }

   487

    """

   488

   489

    # gather

   490

    hosts = collections.defaultdict(lambda: collections.defaultdict(list))

   491

   492

    for (host, domain), field, value in import_hosts :

   493

        hosts[domain, host][tuple(field.split('.'))].append(value)

   494