1

#!/usr/bin/env python

     2

     3

"""

     4

    Import hosts from existing BIND or dhcpd files.

     5

"""

     6

     7

import pvl.args

     8

import pvl.dns.zone

     9

import pvl.dhcp.config

    10

import pvl.ldap.args

    11

    12

import collections

    13

import ipaddr

    14

import logging; log = logging.getLogger('pvl.hosts-import')

    15

import optparse

    16

import os.path

    17

import re

    18

    19

__version__ = '0.1'

    20

    21

def parse_options (argv) :

    22

    """

    23

        Parse command-line arguments.

    24

    """

    25

    26

    parser = optparse.OptionParser(

    27

            prog        = argv[0],

    28

            usage       = '%prog: [options]',

    29

            version     = __version__,

    30

    31

            # module docstring

    32

            description = __doc__,

    33

    )

    34

    35

    # logging

    36

    parser.add_option_group(pvl.args.parser(parser))

    37

    parser.add_option_group(pvl.ldap.args.parser(parser))

    38

    39

    parser.add_option('-c', '--input-charset',  metavar='CHARSET',  default='utf-8', 

    40

            help="Encoding used for input files")

    41

    42

    parser.add_option('--output-charset',       metavar='CHARSET',  default='utf-8', 

    43

            help="Encoding used for output files")

    44

    45

    # input

    46

    parser.add_option('--import-zone-hosts',    metavar='FILE',     action='append',

    47

            help="Load hosts from DNS zone")

    48

    49

    parser.add_option('--import-zone-origin',   metavar='ORIGIN',

    50

            help="Initial origin for given zone file; default is basename")

    51

    52

    parser.add_option('--import-zone-comments-owner',  action='store_const',

    53

            dest='import_zone_comments', const='owner',

    54

            help="Import DNS zone comment as owner comment")

    55

    56

    parser.add_option('--import-zone-comments-host',   action='store_const',

    57

            dest='import_zone_comments', const='host',

    58

            help="Import DNS zone comment as host comment")

    59

    60

    parser.add_option('--import-dhcp-hosts',    metavar='FILE',     action='append',

    61

            help="Load hosts from DHCP config")

    62

    63

    parser.add_option('--import-dhcp-boot-server',      metavar='NEXT-SERVER',

    64

            help="Default boot_server for dpc hosts")

    65

    66

    67

    parser.add_option('--dump-host-comments',   action='store_true',

    68

            help="Dump out info on imported host comments")

    69

    70

    # defaults

    71

    parser.add_option('--zone-unused',          metavar='HOST',

    72

            help="DNS name for unallocated hosts")

    73

    74

    # output

    75

    parser.add_option('--output-hosts',         metavar='FILE',

    76

            help="Output hosts file")

    77

    78

    parser.add_option('--output-prefix',        metavar='PREFIX',

    79

            help="Select hosts by ip prefix")

    80

    81

    parser.add_option('--output-domain',        metavar='DOMAIN',

    82

            help="Select hosts by domain")

    83

    84

    # defaults

    85

    parser.set_defaults(

    86

        import_zone_hosts   = [],

    87

        import_dhcp_hosts   = [],

    88

    )

    89

    90

    # parse

    91

    options, args = parser.parse_args(argv[1:])

    92

    93

    # apply

    94

    pvl.args.apply(options, argv[0])

    95

    96

    return options, args

    97

    98

def import_zone_host_name (options, name, origin) :

    99

    """

   100

        Import zone name from rr

   101

    """

   102

   103

    if '.' in name :

   104

        host, domain = name.split('.', 1)

   106

    else :

   107

        host = name

   108

        domain = origin

   109

   110

    if domain :

   111

        # not a fqdn

   112

        domain = domain.rstrip('.')

   113

   114

        log.info("%s: %s@%s", name, host, domain)

   115

    else :

   116

        log.warn("%s: no domain", name)

   117

   118

    return host, domain

   119

   120

def import_zone_hosts (options, file) :

   121

    """

   122

        Yield host info from zonefile records.

   123

    """

   124

   125

    origin = options.import_zone_origin or os.path.basename(file.name)

   126

   127

    for rr in pvl.dns.zone.ZoneRecord.load(file,

   128

            # used to determine domain

   129

            origin          = origin,

   130

   131

            # generated hosts need to imported by hand...

   132

            expand_generate = False,

   133

    ) :

   134

        host, domain = import_zone_host_name(options, rr.name, rr.origin)

   135

   136

        if options.zone_unused and rr.name == options.zone_unused :

   137

            log.debug("%s: skip %s", rr.name, rr)

   138

            continue

   139

   140

        elif rr.type in ('A', 'AAAA') :

   141

            ip, = rr.data

   142

   143

            type = { 'A': 'ip', 'AAAA': 'ip6' }[rr.type]

   144

   145

            yield (host, domain), type, ipaddr.IPAddress(ip)

   146

   147

            if rr.comment :

   148

                yield (host, domain), 'comment', rr.comment

   149

   150

        elif rr.type == 'CNAME' :

   151

            alias, = rr.data

   152

            alias_host, alias_domain = import_zone_host_name(options, alias, rr.origin)

   153

   154

            if domain == alias_domain :

   155

                yield (alias_host, alias_domain), 'alias', host

   156

            else :

   158

   159

        elif rr.type == 'TXT' :

   160

            txt, = rr.data

   161

   162

            yield (host, domain), 'comment', txt

   163

   164

        else :

   165

            log.warn("%s: unknown rr: %s", host, rr)

   166

   167

def import_dhcp_host (options, dhcp_host, items) :

   168

    """

   169

        Yield host infos from a dhcp host ... { ... }

   170

    """

   171

   172

    host_name = None

   173

    ethernet = []

   174

    fixed_address = None

   175

   176

    boot_server = options.import_dhcp_boot_server

   177

    boot_filename = None

   178

   179

    for item in items :

   180

        item, args = item[0], item[1:]

   181

   182

        if item == 'hardware' :

   183

            _ethernet, ethernet = args

   184

            assert _ethernet == 'ethernet'

   185

        elif item == 'fixed-address' :

   186

            fixed_address, = args

   187

        elif item == 'option' :

   188

            option = args.pop(0)

   189

   190

            if option == 'host-name' :

   191

                host_name, = args

   192

            else :

   193

                log.warn("host %s: ignore unknown option: %s", dhcp_host, option)

   194

        elif item == 'next-sever' :

   195

            boot_server, = args

   196

        elif item == 'filename' :

   197

            boot_filename, = args

   198

        else :

   199

            log.warn("host %s: ignore unknown item: %s", dhcp_host, item)

   200

   201

    # determine host

   202

    host = None

   203

    domain = None

   204

    suffix = None

   205

   206

    if not fixed_address :

   207

        log.warn("%s: fixed-address is missing, unable to determine hostname/domainname", dhcp_host)

   208

    elif re.match(r'\d+\.\d+\.\d+.\d+', fixed_address) :

   209

        log.warn("%s: fixed-address is an IP, unable to determine hostname/domainname", dhcp_host)

   210

    else :

   211

        host, domain = fixed_address.split('.', 1)

   212

   213

    # XXX: not actually true... eh

   214

    if host and dhcp_host.lower() == host.lower() :

   215

        # do not split suffix from host

   216

        pass

   217

    elif host and '-' in dhcp_host :

   218

        dhcp_host, suffix = dhcp_host.rsplit('-', 1)

   219

    elif '-' in dhcp_host :

   220

        host, suffix = dhcp_host.rsplit('-', 1)

   221

    else :

   222

        host = dhcp_host

   223

   224

    if not (host or ethernet) :

   225

        log.warn("%s: no hostname/ethernet: %s/%s", dhcp_host, hostname, ethernet)

   226

    elif suffix :

   227

        log.info("%s: %s@%s: %s: %s", dhcp_host, host, domain, suffix, ethernet)

   228

        yield (host, domain), 'ethernet.{suffix}'.format(suffix=suffix), ethernet

   229

    else :

   230

        log.info("%s: %s@%s: %s", dhcp_host, host, domain, ethernet)

   231

        yield (host, domain), 'ethernet', ethernet

   232

   233

    if boot_server and boot_filename :

   234

        yield (host, domain), 'boot', "{server}:{filename}".format(

   235

                server      = boot_server,

   236

                filename    = boot_filename,

   237

        )

   238

    elif boot_filename :

   239

        yield (host, domain), 'boot', "{filename}".format(filename=boot_filename)

   240

   241

def import_dhcp_hosts (options, file_name, blocks) :

   242

    """

   243

        Process hosts from a parsed block

   244

    """

   245

   246

    for block, items, blocks in blocks :

   247

   248

        block, args = block[0], block[1:]

   249

   250

        if block == 'group' :

   251

            log.info("%s: group", file_name)

   252

            for info in import_dhcp_hosts(options, file_name, blocks) :

   253

                yield info

   254

        elif block == 'host' :

   255

            host, = args

   256

   257

            log.info("%s: host: %s", file_name, host)

   258

   259

            try :

   260

                for info in import_dhcp_host(options, host, items) :

   261

                    yield info

   262

            except ValueError as error :

   263

                log.exception("%s: invalid host %s: %s", file_name, host, error)

   264

        else:

   265

            log.warn("%s: ignore unknown block: %s", file_name, block)

   266

   267

def import_dhcp_conf (options, file) :

   268

    items, blocks = pvl.dhcp.config.DHCPConfigParser().load(file)

   269

   270

    for item in items :

   271

        item, args = item[0], item[1:]

   272

   273

        if item == 'include' :

   274

            include, = args

   275

            for info in import_dhcp_conf(options, pvl.args.apply_file(include)) :

   276

                yield info

   277

        else :

   278

            log.warn("ignore unknown item: %s", item)

   279

   280

    for info in import_dhcp_hosts(options, file.name, blocks) :

   281

        yield info

   282

   283

ZONE_COMMENTS = (

   284

        re.compile(r'(?P<owner>[^/]+)\s*-\s+(?P<host>.+)'),

   285

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[/-]\s+(?P<host>.+)'),

   286

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)\s+[(]\s*(?P<host>.+)[)]'),

   287

        re.compile(r'(?P<group>.+?)\s*/\s*(?P<owner>.+)'),

   288

)

   289

   290

ZONE_OWNER_MAIL = re.compile(r'(?P<owner>.*?)\s*<(?P<mail>.+?)>')

   291

   292

def process_zone_comment (options, hostname, comment) :

   293

    """

   294

        Attempt to parse a host comment field... :D

   295

   296

        Yields (field, value) bits

   297

    """

   298

   299

    for regex in ZONE_COMMENTS :

   300

        match = regex.match(comment)

   301

   302

        if match :

   303

            matches = match.groupdict()

   304

   305

            log.info("%s: matched comment: %s", hostname, comment)

   306

            break

   307

    else :

   308

        if options.import_zone_comments :

   309

            log.info("%s: default comment: %s", hostname, comment)

   310

            matches = { options.import_zone_comments: comment }

   311

        else :

   312

            log.warn("%s: unknown comment: %s", hostname, comment)

   313

            return

   314

   315

    owner = matches.pop('owner', None)

   316

   317

    if owner :

   318

        mail_match = ZONE_OWNER_MAIL.match(owner)

   319

   320

        if mail_match :

   321

            mail_matches = mail_match.groupdict()

   322

   323

            owner = mail_matches['owner']

   324

            yield 'mail', mail_matches['mail'].strip()

   325

   326

        yield 'owner', owner.strip()

   327

   328

    for field, value in matches.iteritems() :

   329

        if value :

   330

            yield field, value.strip()

   331

   332

NONE_OWNERS = set((

   333

    u'tech',

   334

    u'atk',

   335

    u'toimisto',

   336

))

   337

   338

def process_host_owner_ldap (options, host, info) :

   339

    """

   340

        Yield guesses for user from LDAP.

   341

    """

   342

   343

    if info.get('mail') :

   344

        for user in options.ldap.users.filter(

   345

                { 'mailLocalAddress': info['mail'] },

   346

                { 'uid': info['mail'] },

   347

        ) :

   348

            yield user, None

   349

   350

    if info.get('group') and info.get('owner') :

   351

        groups = options.ldap.groups.filter(cn=info['group'])

   352

   353

        for group in groups :

   354

            for user in options.ldap.users.filter({

   355

                'gidNumber': group['gidNumber'],

   356

                'cn': info['owner'],

   357

            }) :

   358

                yield user, group

   359

   360

    if info.get('owner') :

   361

            for user in options.ldap.users.filter({

   362

                'cn': info['owner'],

   363

            }) :

   364

                yield user, None

   365

   366

def process_host_owner (options, host, info) :

   367

    """

   368

        Return (owner, comment) for host based on info, or None.

   369

    """

   370

   371

    owner = info.get('owner')

   372

   373

    if owner and owner.lower() in NONE_OWNERS :

   374

        return False

   375

   376

    # from ldap?

   377

    for ldap in process_host_owner_ldap(options, host, info) :

   378

        user, group = ldap

   379

   380

        if not group :

   381

            # get group from ldap

   382

            group = options.ldap.users.group(user)

   383

   384

        return user['uid'], u"{group} / {user}".format(

   385

                user    = user.getunicode('cn'),

   386

                group   = group.getunicode('cn'),

   387

        )

   388

   389

def process_host_comments (options, host, info) :

   390

    """

   391

        Process host fields from comment.

   392

   393

        Attempts to find owner from LDAP..

   394

    """

   395

   396

    log.debug("%s: %s", host, info)

   397

   398

    owner = process_host_owner(options, host, info) 

   399

   400

    if owner is False :

   401

        # do not mark any owner

   402

        pass

   403

   404

    elif owner :

   405

        owner, comment = owner

   406

   407

        log.info("%s: %s (%s)", host, owner, comment)

   408

   409

        yield 'comment.owner', comment

   410

        yield 'owner', owner,

   411

   412

    elif 'group' in info or 'owner' in info :

   413

        log.warn("%s: unknown owner: %s", host, info)

   414

        yield 'comment.owner', "{group} / {owner}".format(

   415

                group   = info.get('group', ''),

   416

                owner   = info.get('owner', ''),

   417

        )

   418

   419

    if info.get('host') :

   420

        yield 'comment.host', info['host']

   421

   422

def process_hosts_comments (options, import_hosts) :

   423

    """

   424

        Parse out comments from host imports..

   425

    """

   426

   427

    for host, field, value in import_hosts :

   428

        if field != 'comment':

   429

            yield host, field, value

   430

            continue

   431

   432

        fields = dict(process_zone_comment(options, host, value))

   433

   434

        if options.dump_host_comments :

   435

            print u"{host:20} {comment:80} = {group:15} / {owner:20} <{mail:20}> / {hostinfo}".format(

   436

                    host        = host,

   437

                    comment     = value,

   438

                    group       = fields.get('group', ''),

   439

                    owner       = fields.get('owner', ''),

   440

                    mail        = fields.get('mail', ''),

   441

                    hostinfo    = fields.get('host', ''),

   442

            ).encode('utf-8')

   443

   444

   445

        for field, value in process_host_comments(options, host, fields) :

   446

            yield host, field, value

   447

   448

def import_hosts_files (options, zone_files, dhcp_files) :

   449

    """

   450

        Import host infos from given files.

   451

    """

   452

   453

    for zone_file in zone_files:

   454

        file = pvl.args.apply_file(zone_file, 'r', options.input_charset)

   455

        for info in import_zone_hosts(options, file) :

   456

            yield info

   457

   458

    for dhcp_file in dhcp_files :

   459

        file = pvl.args.apply_file(dhcp_file, 'r', options.input_charset)

   460

        for info in import_dhcp_conf(options, file) :

   461

            yield info

   462

   463

def process_import_hosts (options, import_hosts) :

   464

    """

   465

        Build hosts from imported fields.

   466

   467

        Yields (domain, host), { (field, ...): value }

   468

    """

   469

   470

    # gather

   471

    hosts = collections.defaultdict(lambda: collections.defaultdict(list))

   472

   473

    for (host, domain), field, value in import_hosts :

   474

        hosts[domain, host][tuple(field.split('.'))].append(value)

   475

   476

    # process

   477

    for (domain, host), fields in hosts.iteritems() :

   478

        SINGLE_FIELDS = (

   479

                'ip',

   480

                'ip6',

   481

                'comment.owner',

   482

                'owner',

   483

                'boot',

   484

        )

   485

        MULTI_FIELDS = (

   486

                'comment.host',

   487

                'ethernet',

   488

                'alias',

   489

        ) 

   490

        host_fields = {}

   491

   492

        for field_name in SINGLE_FIELDS :

   493

            field = tuple(field_name.split('.'))

   494

            values = fields.get(field)

   495