1

#!/usr/bin/env python

     2

     3

"""

     4

    Requirements:

     5

        pydot

     6

"""

     7

     8

import pvl.args

     9

import pvl.hosts

    10

from pvl.invoke import merge

    11

    12

import collections

    13

import logging; log = logging.getLogger('pvl.hosts-graph')

    14

import optparse

    15

    16

class ParseError (Exception) :

    17

    def __init__ (self, file, line, msg) :

    18

        self.file = file

    19

        self.line = line

    20

        self.msg = msg

    21

    22

    def __str__ (self) :

    23

        return "{self.file}:{self.line}: {self.msg}".format(self=self)

    24

    25

def _parse_snmp_part (part) :

    26

    if part.isdigit() :

    27

        return int(part)

    28

    else :

    29

        return part

    30

    31

def _parse_snmp_attr (line) :

    32

    for part in line.split() :

    33

        yield _parse_snmp_part(part)

    34

    35

def _parse_snmp_value (line) :

    36

    if '\t' in line :

    37

        key, value = line.split('\t', 1)

    38

    39

        return { _parse_snmp_part(key): _parse_snmp_part(value) }

    40

    41

    else :

    42

        return set((_parse_snmp_part(line), ))

    43

    44

def _load_snmp_data (options, file) :

    45

    """

    46

        Load a data dict generated by pvl.hosts-snmp from a file.

    47

    48

        Yields (host, attr, value)

    49

    """

    50

    51

    host = None

    52

    attr = None

    53

    value = None

    54

    55

    for idx, line in enumerate(file, 1) :

    56

        indent = 0

    57

    58

        while line.startswith('\t') :

    59

            indent += 1

    60

            line = line[1:]

    61

    62

        line = line.lstrip('\t').rstrip('\n')

    63

    64

        if indent == 0 :

    65

            host = line

    66

            attr = None

    67

            value = None

    68

    69

        elif indent == 1 :

    70

            attr = tuple(_parse_snmp_attr(line))

    71

            value = None

    72

    73

            yield host, attr, None

    74

    75

        elif indent == 2 :

    76

            if not attr :

    77

                raise ParseError(file, line, "[%s] %s: value outside of attr" % (host, attr))

    78

    79

            value = _parse_snmp_value(line)

    80

    81

            yield host, attr, value

    82

    83

def load_snmp_data (options, file, hosts) :

    84

    """

    85

        Load snmp data as dict, from given file path, or stdin.

    86

    """

    87

    88

    if file :

    89

        file = open(file)

    90

    else :

    91

        file = sys.stdin

    92

    93

    root = { }

    94

    95

    hosts_by_namedomain = dict(

    96

        (

    97

            '{host}@{domain}'.format(host=host, domain=host.domain), host

    98

        ) for host in hosts

    99

    )

   100

   101

    for host_domain, attr, value in _load_snmp_data(options, file) :

   102

        host = hosts_by_namedomain.get(host_domain)

   103

   104

        if value :

   105

            log.debug("[%s] %s: %s", host, ' '.join(str(a) for a in attr), value)

   106

        else :

   107

            log.debug("[%s] %s", host, ' '.join(str(a) for a in attr),)

   108

   109

        item = root.setdefault(host, { })

   110

   111

        for a in attr[:-1] :

   112

            item = item.setdefault(a, {})

   113

   114

        a = attr[-1]

   115

   116

        if value is None :

   117

            pass

   118

   119

        elif isinstance(value, set) :

   120

            item.setdefault(a, set()).update(value)

   121

   122

        elif isinstance(value, dict) :

   123

            item.setdefault(a, dict()).update(value)

   124

   125

        else :

   126

            item[a] = value

   127

   128

    return root

   129

   130

def host_vlans (host, host_vlans) :

   131

    """

   132

        {vlan: { tagged/untagged: [port] } } -> (port, (untag, [tag])).

   133

    """

   134

   135

    ports = set()

   136

    vlans_untagged = { }

   137

    vlans_tagged = collections.defaultdict(set)

   138

   139

    for vlan, vlan_attrs in host_vlans.iteritems() :

   140

        for port in vlan_attrs.get('tagged', ()) :

   141

            ports.add(port)

   142

            vlans_tagged[port].add(vlan)

   143

   144

        for port in vlan_attrs.get('untagged', ()) :

   145

            ports.add(port)

   146

            vlans_untagged[port] = vlan

   147

   148

    for port in ports :

   149

        untag = vlans_untagged.get(port)

   150

        tagged = vlans_tagged.get(port, ())

   151

   152

        log.debug("%s: %s: untag=%s tag=%s", host, port, untag, tagged)

   153

   154

        yield port, (untag, tagged)

   155

   156

def build_graph (options, snmp, hosts) :

   157

    """

   158

        Combine given snmp data and { host: Host } into

   159

            { node: label }

   160

            { (remote, remote_port, local_port, local): (local_untag, tagged, remote_untag) }

   161

    """

   162

   163

    nodes = { } # host: label

   164

    links = { } # (local, local_port, remote_port, remote_host): (local_untag, tagged, remote_untag)

   165

   166

    hosts_by_lldp = { } # chassis: host

   167

    hosts_by_ethernet = { } # ethernet: host

   168

    hosts_by_location = { } # (domain, location): host

   169

   170

    nodes_port = { } # (local, int(local_port)): {remote}

   171

    nodes_out = { } # local: {remote}

   172

    nodes_in = { } # remote: {local}

   173

    links_out = { } # (local, remote): local_port

   174

    links_in = { } # (remote, local): remote_port

   175

   176

    # first scan: lldp hosts

   177

    for host, host_attrs in snmp.iteritems() :

   178

        nodes[host] = host.location or str(host)

   179

   180

        if 'lldp' in host_attrs :

   181

            lldp_local = host_attrs['lldp']['local']

   182

   183

            hosts_by_lldp[lldp_local['chassis']] = host

   184

   185

    # second scan: nodes by ethernet

   186

    for host in hosts :

   187

        for ethernet in host.ethernet.itervalues() :

   188

            hosts_by_ethernet[ethernet] = host

   189

   190

        if host.location and host.location_domain:

   191

            hosts_by_location[(host.location_domain, host.location)] = host

   192

        elif host.location:

   193

            hosts_by_location[(host.domain, host.location)] = host

   194

   195

    # first graph: lldp remotes

   196

    for host, host_attrs in snmp.iteritems() :

   197

        local_node = host

   198

   199

        if 'vlan' in host_attrs :

   200

            vlans = dict(host_vlans(host, host_attrs['vlan']))

   201

        else :

   202

            vlans = None

   203

   204

        if 'lldp' in host_attrs :

   205

            lldp = host_attrs['lldp']

   206

   207

            local_lldp = lldp['local']['chassis']

   208

   209

            for port, port_attrs in lldp.get('port', { }).iteritems() :

   210

                local_port = port_attrs['local']['port']

   211

   212

                for remote_lldp, remote_attrs in port_attrs['remote'].iteritems() :

   213

                    # determine remote node

   214

                    remote_label = remote_attrs['sys_name']

   215

   216

                    if remote_lldp in hosts_by_lldp :

   217

                        remote_node = remote_host = hosts_by_lldp[remote_lldp]

   218

   219

                    elif remote_lldp in hosts_by_ethernet :

   220

                        remote_node = remote_host = hosts_by_ethernet[remote_lldp]

   221

   222

                        if remote_host.location :

   223

                            remote_label = remote_host.location

   224

   225

                        log.info("%s:%s: guessing lldp host %s -> %s (%s)", host, port, remote_lldp, remote_host, remote_label)

   226

   227

                    elif options.graph_lldp_unknown :

   228

                        log.warning("%s:%s: unknown lldp remote %s (%s)", host, port, remote_lldp, remote_label)

   229

   230

                        # by chassis id

   231

                        remote_node = remote_lldp

   232

                        remote_host = None

   233

   234

                    else :

   235

                        log.info("%s:%s: unknown lldp remote %s (%s)", host, port, remote_lldp, remote_label)

   236

   237

                        remote_node = remote_host = None

   238

   239

   240

                    if not remote_node :

   241

                        continue

   242

   243

                    # ensure remote node

   244

                    if remote_node not in nodes :

   245

                        log.debug("%s:%s: lazy-add remote %s (%s)", host, port, remote_node, remote_label)

   246

   247

                        nodes[remote_node] = remote_label

   248

   249

                    # local vlans

   250

                    if vlans :

   251

                        port_vlans = vlans.get(port)

   252

                    else :

   253

                        port_vlans = None

   254

   255

                    if port_vlans :

   256

                        local_untag, local_tagged = port_vlans

   257

   258

                    # directional mapping

   259

                    links_out[(local_node, remote_node)] = local_port

   260

                    nodes_port.setdefault((local_node, port), set()).add(remote_node)

   261

                    nodes_out.setdefault(local_node, set()).add(remote_node)

   262

                    nodes_in.setdefault(remote_node, set()).add(local_node)

   263

   264

                    # bidirectional mappings

   265

                    remote_port = remote_attrs['port']

   266

   267

                    links_in[(remote_node, local_node)] = remote_port

   268

   269

                    forward = (local_node, local_port, remote_port, remote_node)

   270

                    reverse = (remote_node, remote_port, local_port, local_node)

   271

   272

                    if reverse not in links :

   273

                        links[forward] = (local_untag, local_tagged, None)

   274

                    else :

   275

                        remote_untag, remote_tagged, _ = links[reverse]

   276

   277

                        # merge

   278

                        if remote_untag != local_untag :

   279

                            log.warning("%s:%s untag %s <=> %s untag %s:%s",

   280

                                    host, local_port, local_untag,

   281

                                    remote_untag, remote_node, remote_port

   282

                            )

   283

   284

                        if remote_tagged != local_tagged :

   285

                            log.warning("%s:%s tagged %s <-> %s tagged %s:%s",

   286

                                    host, local_port, ':'.join(str(x) for x in sorted(local_tagged)),

   287

                                    ':'.join(str(x) for x in sorted(remote_tagged)), remote_node, remote_port

   288

                            )

   289

   290

                        links[reverse] = (remote_untag, remote_tagged, local_untag)

   291

   292

    # second graph: manual ports

   293

    for host in hosts :

   294

        local_node = host

   295

        host_links = host.extensions.get('link')

   296

   297

        # XXX: copy-pasta

   298

        if host in snmp and 'vlan' in snmp[host] :

   299

            vlans = dict(host_vlans(host, snmp[host]['vlan']))

   300

        else :

   301

            vlans = None

   302

   303

        if host_links :

   304

            if local_node not in nodes :

   305

                # XXX: copypasta

   306

                nodes[local_node] = host.location or str(host)

   307

   308

            for link_port, remote in host_links.iteritems() :

   309

                if link_port.isdigit() :

   310

                    port = int(link_port)

   311

                else :

   312

                    port = str(link_port)

   313

   314

                # map remote -> remote_host

   315

                if '@' in remote :

   316

                    remote_location, remote_domain = remote.split('@', 1)

   317

                else :

   318

                    remote_location = remote

   319

                    remote_domain = host.domain

   320

   321

                remote_node = remote_host = hosts_by_location.get((remote_domain, remote_location))

   322

   323

                if not remote_host :

   324

                    log.warning("%s:%s: unknown remote location: %s@%s", host, port, remote_location, remote_domain)

   325

                    continue

   326

   327

                remote_label = remote_host.location or str(remote_host)

   328

   329

                log.info("%s:%s: link -> %s@%s (%s)", host, port, remote_host, remote_host.domain, remote_label)

   330

   331

                if remote_node not in nodes :

   332

                    # XXX: copypasta

   333

                    nodes[remote_node] = remote_label

   334

   335

                # local vlans

   336

                if vlans and port in vlans :

   337

                    local_untag, local_tagged = vlans[port]

   338

   339

                    log.info("%s:%s link vlans (%s) <%s>", host, port, local_untag, ':'.join(str(tag) for tag in local_tagged))

   340

   341

                    link_vlans = (local_untag, local_tagged, None)

   342

                else :

   343

                    # unknown

   344

                    link_vlans = None

   345

   346

                    log.warning("%s:%s unknown vlans", host, port)

   347

   348

                # directional links

   349

                local_port = links_out.get((local_node, remote_node))

   350

   351

                if not local_port :

   352

                    log.info("%s:%s: unconfirmed -> %s", host, port, remote_host)

   353

   354

                elif local_port != port :

   355

                    log.warn("%s:%s: port mismatch %s -> %s", host, port, local_port, remote_host)

   356

   357

                else :

   358

                    log.debug("%s:%s: confirm -> %s", host, port, remote_host)

   359

   360

                links_out[(local_node, remote_node)] = port

   361

                nodes_port.setdefault((local_node, port), set()).add(remote_node)

   362

                nodes_out.setdefault(local_node, set()).add(remote_node)

   363

                nodes_in.setdefault(remote_node, set()).add(local_node)

   364

   365

                # update directional or missing links

   366

                remote_port = links_out.get((remote_node, local_node))

   367

                reverse_port = links_in.get((local_node, remote_node))

   368

   369

                if reverse_port and reverse_port != port :

   370

                    # XXX: this can be caused by str vs int >_>

   371

                    log.warn("%s:%s: reverse port mismatch %s <- %s", host, port, reverse_port, remote_node)

   372

   373

   374

                if local_port and remote_port :

   375

                    log.debug("%s:%s link <-> %s:%s", local_node, local_port, remote_port, remote_node)

   376

   377

                elif local_port :

   378

                    # we have the forward mapping already, so this doesn't add any new info

   379

                    log.debug("%s:%s link -> %s:%s", local_node, local_port, remote_port, remote_node)

   380

   381

                elif remote_port and reverse_port :

   382

                    # we have the bidirectional mapping already, so this doesn't add any new info

   383

                    log.debug("%s:%s link <-> %s:%s", local_node, local_port, remote_port, remote_node)

   384

   385

                elif remote_port :

   386

                    # we had the reverse mapping already, make it bidirectional

   387

                    local_port = port

   388

                    log.info("%s:%s link <- %s:%s", local_node, local_port, remote_port, remote_node)

   389

   390

                    # TODO: update vlan info

   391

                    links[(remote_node, remote_port, local_port, local_node)] = links.pop((remote_node, remote_port, None, local_node))

   392

   393

                else :

   394

                    local_port = port

   395

   396

                    # mapping was completely missing

   397

                    log.info("%s:%s link -> %s", local_node, local_port, remote_node)

   398

   399

                    links[(local_node, local_port, None, remote_node)] = link_vlans

   400

   401

   402

    # verify non-p2p links

   403

    for (node, port), remotes in nodes_port.iteritems() :

   404

        if len(remotes) > 1 :

   405

            log.warning("%s:%s: multiple remotes: %s", node, port, ' '.join(str(host) for host in remotes))

   406

   407

    if options.graph_bridge :

   408

        # scan hosts with bridges

   409

        bridge_hosts = set()

   410

        bridge_ports = { }

   411

   412

        for host, host_attrs in snmp.iteritems() :

   413

            if 'bridge' in host_attrs or any('bridge' in vlan_attrs for vlan_attrs in host_attrs.get('vlan', { }).itervalues()) :

   414

                bridge_hosts.add(host)

   415

   416

        # third graph: bridge

   417

        for host, host_attrs in snmp.iteritems() :

   418

            local_out = nodes_out.get(host)

   419

   420

            if not local_out :

   421

                log.warning("%s: no outgoing links, skipping bridge", host)

   422

                continue

   423

   424

            # scan vlan/port bridge ethers

   425

            bridge = { } # (port, vlan): {ethernet}

   426

   427

            for port, ethernets in host_attrs.get('bridge', { }).iteritems() :

   428

                bridge[(port, None)] = ethernets

   429

   430

            for vlan, vlan_attrs in host_attrs.get('vlan', { }).iteritems() :

   431

                for port, ethernets in vlan_attrs.get('bridge', { }).iteritems()  :

   432

                    bridge[(port, vlan)] = ethernets

   433

   434

            for (port, vlan), ethernets in bridge.iteritems() :

   435

                local_node = host

   436

                local_port = port

   437

   438

                remote_nodes = nodes_port.get((local_node, local_port))

   439

   440

                if not remote_nodes :

   441

                    remote_node = None

   442

                elif len(remote_nodes) == 1 :

   443

                    remote_node, = remote_nodes

   444

                else :

   445

                    log.warning("%s:%s: ignore port with multiple remotes: %s", host, port, ' '.join(str(host) for host in remotes))

   446

                    continue

   447

   448

                if remote_node :

   449

                    remote_in = nodes_in.get(remote_node, set())

   450

                else :

   451

                    remote_in = set()

   452

   453

                remote_leaf = (remote_in == set((host, )))

   454

   455

                # TODO: add ether node and link if remote node also has this ether on this link

   456

                #       also do this if all remote_in's agree that the ether is on the remote_node

   457

                if not remote_node :

   458

                    log.debug("%s:%s: no remote node", host, port)

   459

   460

                elif remote_leaf and (remote_node not in bridge_hosts) and len(ethernets) > 1 :

   461

                    # map onto non-bridge leaf node

   462

                    log.info("%s: <== %s:%s ", remote_node, host, port)

   463

   464

                    # map links out of the assumed remote bridge

   465

                    local_node = remote_node

   466

                    local_port = None

   467

   468

                else :

   469

                    log.debug("%s:%s/%s bridge skip -> %s", host, port, vlan, remote_node)

   470

                    continue

   471

   472

                for ethernet in ethernets :

   473

                    # remote host

   474

                    if ethernet in hosts_by_ethernet :

   475

                        remote_node = remote_host = hosts_by_ethernet[ethernet]

   476

   477

                        remote_label = remote_host.location or str(remote_host)

   478

   479

                        log.debug("%s:%s/%s bridge %s = %s (%s)", host, port, vlan, ethernet, remote_host, remote_label)

   480

   481

                    elif options.graph_bridge_unknown :

   482

                        log.warning("%s:%s/%s bridge unknown host %s", host, port, vlan, ethernet)

   483

   484

                        remote_label = remote_node = ethernet

   485

   486

                        nodes[remote_node] = remote_label

   487

   488

                        remote_host = None

   489

   490

                    else :

   491

                        log.info("%s:%s/%s bridge unknown host %s", host, port, vlan, ethernet)

   492