1

#!/usr/bin/env python

     2

     3

"""

     4

    Requirements:

     5

        pydot

     6

"""

     7

     8

import pvl.args

     9

import pvl.hosts

    10

from pvl.invoke import merge

    11

    12

import collections

    13

import logging; log = logging.getLogger('pvl.hosts-graph')

    14

import optparse

    15

    16

COLOR_VLANS = {

    17

    1:      'grey',         # pvl-lan

    18

    2:      'blue',         # pvl-lan2

    19

    3:      'red',          # pvl-san

    20

    4:      'green',        # pvl-veturi

    21

    7:      'orange',       # pvl-ranssi

    22

    8:      'yellow',       # pvl-mgmt

    23

    10:     'brown',        # pvl-public

    24

    100:    'navyblue',     # pvl-test

    25

    103:    'red4',         # pvl-test-san

    26

    104:    'red2',         # pvl-ganeti

    27

    192:    'purple',       # paivola-services

    28

    255:    'magenta',      # pvl-sonera

    29

}

    30

    31

class ParseError (Exception) :

    32

    def __init__ (self, file, line, msg) :

    33

        self.file = file

    34

        self.line = line

    35

        self.msg = msg

    36

    37

    def __str__ (self) :

    38

        return "{self.file}:{self.line}: {self.msg}".format(self=self)

    39

    40

def _parse_snmp_part (part) :

    41

    if part.isdigit() :

    42

        return int(part)

    43

    else :

    44

        return part

    45

    46

def _parse_snmp_attr (line) :

    47

    for part in line.split() :

    48

        yield _parse_snmp_part(part)

    49

    50

def _parse_snmp_value (line) :

    51

    if '\t' in line :

    52

        key, value = line.split('\t', 1)

    53

    54

        return { _parse_snmp_part(key): _parse_snmp_part(value) }

    55

    56

    else :

    57

        return set((_parse_snmp_part(line), ))

    58

    59

def _load_snmp_data (options, file) :

    60

    """

    61

        Load a data dict generated by pvl.hosts-snmp from a file.

    62

    63

        Yields (host, attr, value)

    64

    """

    65

    66

    host = None

    67

    attr = None

    68

    value = None

    69

    70

    for idx, line in enumerate(file, 1) :

    71

        indent = 0

    72

    73

        while line.startswith('\t') :

    74

            indent += 1

    75

            line = line[1:]

    76

    77

        line = line.lstrip('\t').rstrip('\n')

    78

    79

        if indent == 0 :

    80

            host = line

    81

            attr = None

    82

            value = None

    83

    84

        elif indent == 1 :

    85

            attr = tuple(_parse_snmp_attr(line))

    86

            value = None

    87

    88

            yield host, attr, None

    89

    90

        elif indent == 2 :

    91

            if not attr :

    92

                raise ParseError(file, line, "[%s] %s: value outside of attr" % (host, attr))

    93

    94

            value = _parse_snmp_value(line)

    95

    96

            yield host, attr, value

    97

    98

def load_snmp_data (options, file, hosts) :

    99

    """

   100

        Load snmp data as dict, from given file path, or stdin.

   101

    """

   102

   103

    if file :

   104

        file = open(file)

   105

    else :

   106

        file = sys.stdin

   107

   108

    root = { }

   109

   110

    hosts_by_namedomain = dict(

   111

        (

   112

            '{host}@{domain}'.format(host=host, domain=host.domain), host

   113

        ) for host in hosts

   114

    )

   115

   116

    for host_domain, attr, value in _load_snmp_data(options, file) :

   117

        host = hosts_by_namedomain.get(host_domain)

   118

   119

        if value :

   120

            log.debug("[%s] %s: %s", host, ' '.join(str(a) for a in attr), value)

   121

        else :

   122

            log.debug("[%s] %s", host, ' '.join(str(a) for a in attr),)

   123

   124

        item = root.setdefault(host, { })

   125

   126

        for a in attr[:-1] :

   127

            item = item.setdefault(a, {})

   128

   129

        a = attr[-1]

   130

   131

        if value is None :

   132

            pass

   133

   134

        elif isinstance(value, set) :

   135

            item.setdefault(a, set()).update(value)

   136

   137

        elif isinstance(value, dict) :

   138

            item.setdefault(a, dict()).update(value)

   139

   140

        else :

   141

            item[a] = value

   142

   143

    return root

   144

   145

def host_vlans (host, host_vlans) :

   146

    """

   147

        {vlan: { tagged/untagged: [port] } } -> (port, (untag, [tag])).

   148

    """

   149

   150

    ports = set()

   151

    vlans_untagged = { }

   152

    vlans_tagged = collections.defaultdict(set)

   153

   154

    for vlan, vlan_attrs in host_vlans.iteritems() :

   155

        for port in vlan_attrs.get('tagged', ()) :

   156

            ports.add(port)

   157

            vlans_tagged[port].add(vlan)

   158

   159

        for port in vlan_attrs.get('untagged', ()) :

   160

            ports.add(port)

   161

            vlans_untagged[port] = vlan

   162

   163

    for port in ports :

   164

        untag = vlans_untagged.get(port)

   165

        tagged = vlans_tagged.get(port, ())

   166

   167

        log.debug("%s: %s: untag=%s tag=%s", host, port, untag, tagged)

   168

   169

        yield port, (untag, tagged)

   170

   171

def build_graph (options, snmp, hosts) :

   172

    """

   173

        Combine given snmp data and { host: Host } into

   174

            { node: label }

   175

            { (remote, remote_port, local_port, local): (local_untag, tagged, remote_untag) }

   176

    """

   177

   178

    nodes = { } # host: label

   179

    links = { } # (local, local_port, remote_port, remote_host): (local_untag, tagged, remote_untag)

   180

   181

    hosts_by_lldp = { } # chassis: host

   182

    hosts_by_ethernet = { } # ethernet: host

   183

    hosts_by_location = { } # (domain, location): host

   184

   185

    nodes_port = { } # (local, int(local_port)): {remote}

   186

    nodes_out = { } # local: {remote}

   187

    nodes_in = { } # remote: {local}

   188

    links_out = { } # (local, remote): local_port

   189

    links_in = { } # (remote, local): remote_port

   190

   191

    # first scan: lldp hosts

   192

    for host, host_attrs in snmp.iteritems() :

   193

        nodes[host] = host.location or str(host)

   194

   195

        if 'lldp' in host_attrs :

   196

            lldp_local = host_attrs['lldp']['local']

   197

   198

            hosts_by_lldp[lldp_local['chassis']] = host

   199

   200

    # second scan: nodes by ethernet

   201

    for host in hosts :

   202

        for ethernet in host.ethernet.itervalues() :

   203

            hosts_by_ethernet[ethernet] = host

   204

   205

        if host.location :

   206

            hosts_by_location[(host.domain, host.location)] = host

   207

   208

    # first graph: lldp remotes

   209

    for host, host_attrs in snmp.iteritems() :

   210

        local_node = host

   211

   212

        if 'vlan' in host_attrs :

   213

            vlans = dict(host_vlans(host, host_attrs['vlan']))

   214

        else :

   215

            vlans = None

   216

   217

        if 'lldp' in host_attrs :

   218

            lldp = host_attrs['lldp']

   219

   220

            local_lldp = lldp['local']['chassis']

   221

   222

            for port, port_attrs in lldp.get('port', { }).iteritems() :

   223

                local_port = port_attrs['local']['port']

   224

   225

                for remote_lldp, remote_attrs in port_attrs['remote'].iteritems() :

   226

                    # determine remote node

   227

                    remote_label = remote_attrs['sys_name']

   228

   229

                    if remote_lldp in hosts_by_lldp :

   230

                        remote_node = remote_host = hosts_by_lldp[remote_lldp]

   231

   232

                    elif remote_lldp in hosts_by_ethernet :

   233

                        remote_node = remote_host = hosts_by_ethernet[remote_lldp]

   234

   235

                        if remote_host.location :

   236

                            remote_label = remote_host.location

   237

   238

                        log.info("%s:%s: guessing lldp host %s -> %s (%s)", host, port, remote_lldp, remote_host, remote_label)

   239

   240

                    elif options.graph_lldp_unknown :

   241

                        log.warning("%s:%s: unknown lldp remote %s (%s)", host, port, remote_lldp, remote_label)

   242

   243

                        # by chassis id

   244

                        remote_node = remote_lldp

   245

                        remote_host = None

   246

   247

                    else :

   248

                        log.info("%s:%s: unknown lldp remote %s (%s)", host, port, remote_lldp, remote_label)

   249

   250

                        remote_node = remote_host = None

   251

   252

   253

                    if not remote_node :

   254

                        continue

   255

   256

                    # ensure remote node

   257

                    if remote_node not in nodes :

   258

                        log.debug("%s:%s: lazy-add remote %s (%s)", host, port, remote_node, remote_label)

   259

   260

                        nodes[remote_node] = remote_label

   261

   262

                    # local vlans

   263

                    if vlans :

   264

                        port_vlans = vlans.get(port)

   265

                    else :

   266

                        port_vlans = None

   267

   268

                    if port_vlans :

   269

                        local_untag, local_tagged = port_vlans

   270

   271

                    # directional mapping

   272

                    links_out[(local_node, remote_node)] = local_port

   273

                    nodes_port.setdefault((local_node, port), set()).add(remote_node)

   274

                    nodes_out.setdefault(local_node, set()).add(remote_node)

   275

                    nodes_in.setdefault(remote_node, set()).add(local_node)

   276

   277

                    # bidirectional mappings

   278

                    remote_port = remote_attrs['port']

   279

   280

                    links_in[(remote_node, local_node)] = remote_port

   281

   282

                    forward = (local_node, local_port, remote_port, remote_node)

   283

                    reverse = (remote_node, remote_port, local_port, local_node)

   284

   285

                    if reverse not in links :

   286

                        links[forward] = (local_untag, local_tagged, None)

   287

                    else :

   288

                        remote_untag, remote_tagged, _ = links[reverse]

   289

   290

                        # merge

   291

                        if remote_untag != local_untag :

   292

                            log.warning("%s:%s untag %s <=> %s untag %s:%s",

   293

                                    host, local_port, local_untag,

   294

                                    remote_untag, remote_node, remote_port

   295

                            )

   296

   297

                        if remote_tagged != local_tagged :

   298

                            log.warning("%s:%s tagged %s <-> %s tagged %s:%s",

   299

                                    host, local_port, ':'.join(str(x) for x in sorted(local_tagged)),

   300

                                    ':'.join(str(x) for x in sorted(remote_tagged)), remote_node, remote_port

   301

                            )

   302

   303

                        links[reverse] = (remote_untag, remote_tagged, local_untag)

   304

   305

    # second graph: manual ports

   306

    for host in hosts :

   307

        local_node = host

   308

        host_links = host.extensions.get('link')

   309

   316

        if host_links :

   317

            if local_node not in nodes :

   318

                # XXX: copypasta

   319

                nodes[local_node] = host.location or str(host)

   320

   321

            for link_port, remote in host_links.iteritems() :

   322

                if link_port.isdigit() :

   323

                    port = int(link_port)

   324

                else :

   325

                    port = str(link_port)

   326

   327

                # map remote -> remote_host

   328

                if '@' in remote :

   329

                    remote_location, remote_domain = remote.split('@', 1)

   330

                else :

   331

                    remote_location = remote

   332

                    remote_domain = host.domain

   333

   334

                remote_node = remote_host = hosts_by_location.get((remote_domain, remote_location))

   335

   336

                if not remote_host :

   337

                    log.warning("%s:%s: unknown remote location: %s@%s", host, port, remote_location, remote_domain)

   338

                    continue

   339

   340

                remote_label = remote_host.location or str(remote_host)

   341

   342

                log.info("%s:%s: link -> %s@%s (%s)", host, port, remote_host, remote_host.domain, remote_label)

   343

   344

                if remote_node not in nodes :

   345

                    # XXX: copypasta

   346

                    nodes[remote_node] = remote_label

   347

   361

                # directional links

   362

                local_port = links_out.get((local_node, remote_node))

   363

   364

                if not local_port :

   365

                    log.info("%s:%s: unconfirmed -> %s", host, port, remote_host)

   366

   367

                elif local_port != port :

   368

                    log.warn("%s:%s: port mismatch %s -> %s", host, port, local_port, remote_host)

   369

   370

                else :

   371

                    log.debug("%s:%s: confirm -> %s", host, port, remote_host)

   372

   373

                links_out[(local_node, remote_node)] = port

   374

                nodes_port.setdefault((local_node, port), set()).add(remote_node)

   375

                nodes_out.setdefault(local_node, set()).add(remote_node)

   376

                nodes_in.setdefault(remote_node, set()).add(local_node)

   377

   378

                # update directional or missing links

   379

                remote_port = links_out.get((remote_node, local_node))

   380

                reverse_port = links_in.get((local_node, remote_node))

   381

   382

                if reverse_port and reverse_port != port :

   383

                    # XXX: this can be caused by str vs int >_>

   384

                    log.warn("%s:%s: reverse port mismatch %s <- %s", host, port, reverse_port, remote_node)

   385

   386

   387

                if local_port and remote_port :

   388

                    log.debug("%s:%s link <-> %s:%s", local_node, local_port, remote_port, remote_node)

   389

   390

                elif local_port :

   391

                    # we have the forward mapping already, so this doesn't add any new info

   392

                    log.debug("%s:%s link -> %s:%s", local_node, local_port, remote_port, remote_node)

   393

   394

                elif remote_port and reverse_port :

   395

                    # we have the bidirectional mapping already, so this doesn't add any new info

   396

                    log.debug("%s:%s link <-> %s:%s", local_node, local_port, remote_port, remote_node)

   397

   398

                elif remote_port :

   399

                    # we had the reverse mapping already, make it bidirectional

   400

                    local_port = port

   401

                    log.info("%s:%s link <- %s:%s", local_node, local_port, remote_port, remote_node)

   404

                    links[(remote_node, remote_port, local_port, local_node)] = links.pop((remote_node, remote_port, None, local_node))

   405

   406

                else :

   407

                    local_port = port

   408

   409

                    # mapping was completely missing

   410

                    log.info("%s:%s link -> %s", local_node, local_port, remote_node)

   411

   413

   414

   415

    # verify non-p2p links

   416

    for (node, port), remotes in nodes_port.iteritems() :

   417

        if len(remotes) > 1 :

   418

            log.warning("%s:%s: multiple remotes: %s", node, port, ' '.join(str(host) for host in remotes))

   419

   420

    if options.graph_bridge :

   421

        # scan hosts with bridges

   422

        bridge_hosts = set()

   423

        bridge_ports = { }

   424

   425

        for host, host_attrs in snmp.iteritems() :

   426

            if 'bridge' in host_attrs or any('bridge' in vlan_attrs for vlan_attrs in host_attrs.get('vlan', { }).itervalues()) :

   427

                bridge_hosts.add(host)

   429

        # third graph: bridge

   430

        for host, host_attrs in snmp.iteritems() :

   431

            local_out = nodes_out.get(host)

   432

   433

            if not local_out :

   434

                log.warning("%s: no outgoing links, skipping bridge", host)

   435

                continue

   436

   437

            # scan vlan/port bridge ethers

   438

            bridge = { } # (port, vlan): {ethernet}

   439

   440

            for port, ethernets in host_attrs.get('bridge', { }).iteritems() :

   441

                bridge[(port, None)] = ethernets

   442

   443

            for vlan, vlan_attrs in host_attrs.get('vlan', { }).iteritems() :

   444

                for port, ethernets in vlan_attrs.get('bridge', { }).iteritems()  :

   445

                    bridge[(port, vlan)] = ethernets

   446

   447

            for (port, vlan), ethernets in bridge.iteritems() :

   448

                local_node = host

   449

                local_port = port

   450

   451

                remote_nodes = nodes_port.get((local_node, local_port))

   452

   453

                if not remote_nodes :

   454

                    remote_node = None

   455

                elif len(remote_nodes) == 1 :

   456

                    remote_node, = remote_nodes

   457

                else :

   458

                    log.warning("%s:%s: ignore port with multiple remotes: %s", host, port, ' '.join(str(host) for host in remotes))

   459

                    continue

   460

   461

                if remote_node :

   462

                    remote_in = nodes_in.get(remote_node, set())

   463

                else :

   464

                    remote_in = set()

   465

   466

                remote_leaf = (remote_in == set((host, )))

   467

   468

                # TODO: add ether node and link if remote node also has this ether on this link

   469

                #       also do this if all remote_in's agree that the ether is on the remote_node

   470

                if not remote_node :

   471

                    log.debug("%s:%s: no remote node", host, port)

   472

   473

                elif remote_leaf and (remote_node not in bridge_hosts) and len(ethernets) > 1 :

   474

                    # map onto non-bridge leaf node

   475

                    log.info("%s: <== %s:%s ", remote_node, host, port)

   476

   477

                    # map links out of the assumed remote bridge

   478

                    local_node = remote_node

   479

                    local_port = None

   480

   481

                else :

   482

                    log.debug("%s:%s/%s bridge skip -> %s", host, port, vlan, remote_node)

   483

                    continue

   484

   485

                for ethernet in ethernets :

   486

                    # remote host

   487

                    if ethernet in hosts_by_ethernet :

   488

                        remote_node = remote_host = hosts_by_ethernet[ethernet]

   489

   490

                        remote_label = remote_host.location or str(remote_host)

   491

   492

                        log.debug("%s:%s/%s bridge %s = %s (%s)", host, port, vlan, ethernet, remote_host, remote_label)

   493