1

#!/usr/bin/env python

     2

     3

"""

     4

    Requirements:

     5

        pydot

     6

"""

     7

     8

import pvl.args

     9

import pvl.hosts

    10

from pvl.invoke import merge

    11

    12

import collections

    13

import logging; log = logging.getLogger('pvl.hosts-graph')

    14

import optparse

    15

    16

COLOR_VLANS = {

    17

    1:      'grey',         # pvl-lan

    18

    2:      'blue',         # pvl-lan2

    19

    3:      'red',          # pvl-san

    20

    4:      'green',        # pvl-veturi

    21

    7:      'orange',       # pvl-ranssi

    22

    8:      'yellow',       # pvl-mgmt

    23

    10:     'brown',        # pvl-public

    24

    100:    'navyblue',     # pvl-test

    25

    103:    'red4',         # pvl-test-san

    26

    104:    'red2',         # pvl-ganeti

    27

    192:    'purple',       # paivola-services

    28

    255:    'magenta',      # pvl-sonera

    29

}

    30

    31

class ParseError (Exception) :

    32

    def __init__ (self, file, line, msg) :

    33

        self.file = file

    34

        self.line = line

    35

        self.msg = msg

    36

    37

    def __str__ (self) :

    38

        return "{self.file}:{self.line}: {self.msg}".format(self=self)

    39

    40

def _parse_snmp_part (part) :

    41

    if part.isdigit() :

    42

        return int(part)

    43

    else :

    44

        return part

    45

    46

def _parse_snmp_attr (line) :

    47

    for part in line.split() :

    48

        yield _parse_snmp_part(part)

    49

    50

def _parse_snmp_value (line) :

    51

    if '\t' in line :

    52

        key, value = line.split('\t', 1)

    53

    54

        return { _parse_snmp_part(key): _parse_snmp_part(value) }

    55

    56

    else :

    57

        return set((_parse_snmp_part(line), ))

    58

    59

def _load_snmp_data (options, file) :

    60

    """

    61

        Load a data dict generated by pvl.hosts-snmp from a file.

    62

    63

        Yields (host, attr, value)

    64

    """

    65

    66

    host = None

    67

    attr = None

    68

    value = None

    69

    70

    for idx, line in enumerate(file, 1) :

    71

        indent = 0

    72

    73

        while line.startswith('\t') :

    74

            indent += 1

    75

            line = line[1:]

    76

    77

        line = line.lstrip('\t').rstrip('\n')

    78

    79

        if indent == 0 :

    80

            host = line

    81

            attr = None

    82

            value = None

    83

    84

        elif indent == 1 :

    85

            attr = tuple(_parse_snmp_attr(line))

    86

            value = None

    87

    88

            yield host, attr, None

    89

    90

        elif indent == 2 :

    91

            if not attr :

    92

                raise ParseError(file, line, "[%s] %s: value outside of attr" % (host, attr))

    93

    94

            value = _parse_snmp_value(line)

    95

    96

            yield host, attr, value

    97

    98

def load_snmp_data (options, file, hosts) :

    99

    """

   100

        Load snmp data as dict, from given file path, or stdin.

   101

    """

   102

   103

    if file :

   104

        file = open(file)

   105

    else :

   106

        file = sys.stdin

   107

   108

    root = { }

   109

   110

    hosts_by_namedomain = dict(

   111

        (

   112

            '{host}@{domain}'.format(host=host, domain=host.domain), host

   113

        ) for host in hosts

   114

    )

   115

   116

    for host_domain, attr, value in _load_snmp_data(options, file) :

   117

        host = hosts_by_namedomain.get(host_domain)

   118

   119

        if value :

   120

            log.debug("[%s] %s: %s", host, ' '.join(str(a) for a in attr), value)

   121

        else :

   122

            log.debug("[%s] %s", host, ' '.join(str(a) for a in attr),)

   123

   124

        item = root.setdefault(host, { })

   125

   126

        for a in attr[:-1] :

   127

            item = item.setdefault(a, {})

   128

   129

        a = attr[-1]

   130

   131

        if value is None :

   132

            pass

   133

   134

        elif isinstance(value, set) :

   135

            item.setdefault(a, set()).update(value)

   136

   137

        elif isinstance(value, dict) :

   138

            item.setdefault(a, dict()).update(value)

   139

   140

        else :

   141

            item[a] = value

   142

   143

    return root

   144

   145

def host_vlans (host, host_vlans) :

   146

    """

   147

        {vlan: { tagged/untagged: [port] } } -> (port, (untag, [tag])).

   148

    """

   149

   150

    ports = set()

   151

    vlans_untagged = { }

   152

    vlans_tagged = collections.defaultdict(set)

   153

   154

    for vlan, vlan_attrs in host_vlans.iteritems() :

   155

        for port in vlan_attrs.get('tagged', ()) :

   156

            ports.add(port)

   157

            vlans_tagged[port].add(vlan)

   158

   159

        for port in vlan_attrs.get('untagged', ()) :

   160

            ports.add(port)

   161

            vlans_untagged[port] = vlan

   162

   163

    for port in ports :

   164

        untag = vlans_untagged.get(port)

   165

        tagged = vlans_tagged.get(port, ())

   166

   167

        log.debug("%s: %s: untag=%s tag=%s", host, port, untag, tagged)

   168

   169

        yield port, (untag, tagged)

   170

   171

def build_graph (options, snmp, hosts) :

   172

    """

   173

        Combine given snmp data and { host: Host } into

   174

            { node: label }

   175

            { (remote, remote_port, local_port, local): (local_untag, tagged, remote_untag) }

   176

    """

   177

   178

    nodes = { } # host: label

   179

    links = { } # (local, local_port, remote_port, remote_host): (local_untag, tagged, remote_untag)

   180

   181

    hosts_by_lldp = { } # chassis: host

   182

    hosts_by_ethernet = { } # ethernet: host

   183

    hosts_by_location = { } # (domain, location): host

   184

   185

    nodes_port = { } # (local, int(local_port)): remote

   186

    nodes_out = { } # local: {remote}

   187

    links_out = { } # (local, remote): local_port

   188

    links_in = { } # (remote, local): remote_port

   189

   190

    # first scan: lldp hosts

   191

    for host, host_attrs in snmp.iteritems() :

   192

        nodes[host] = host.location or str(host)

   193

   194

        if 'lldp' in host_attrs :

   195

            lldp_local = host_attrs['lldp']['local']

   196

   197

            hosts_by_lldp[lldp_local['chassis']] = host

   198

   199

    # second scan: nodes by ethernet

   200

    for host in hosts :

   201

        for ethernet in host.ethernet.itervalues() :

   202

            hosts_by_ethernet[ethernet] = host

   203

   204

        if host.location :

   205

            hosts_by_location[(host.domain, host.location)] = host

   206

   207

    # first graph: lldp remotes

   208

    for host, host_attrs in snmp.iteritems() :

   209

        local_node = host

   210

   211

        if 'vlan' in host_attrs :

   212

            vlans = dict(host_vlans(host, host_attrs['vlan']))

   213

        else :

   214

            vlans = None

   215

   216

        if 'lldp' in host_attrs :

   217

            lldp = host_attrs['lldp']

   218

   219

            local_lldp = lldp['local']['chassis']

   220

   221

            for port, port_attrs in lldp.get('port', { }).iteritems() :

   222

                local_port = port_attrs['local']['port']

   223

   224

                for remote_lldp, remote_attrs in port_attrs['remote'].iteritems() :

   225

                    # determine remote node

   226

                    remote_label = remote_attrs['sys_name']

   227

   228

                    if remote_lldp in hosts_by_lldp :

   229

                        remote_node = remote_host = hosts_by_lldp[remote_lldp]

   230

   231

                    elif remote_lldp in hosts_by_ethernet :

   232

                        remote_node = remote_host = hosts_by_ethernet[remote_lldp]

   233

   234

                        if remote_host.location :

   235

                            remote_label = remote_host.location

   236

   237

                        log.info("%s:%s: guessing lldp host %s -> %s (%s)", host, port, remote_lldp, remote_host, remote_label)

   238

   239

                    else :

   251

   252

                    # ensure remote node

   253

                    if remote_node not in nodes :

   254

                        log.debug("%s:%s: lazy-add remote %s (%s)", host, port, remote_node, remote_label)

   255

   256

                        nodes[remote_node] = remote_label

   257

   258

                    # local vlans

   259

                    if vlans :

   260

                        port_vlans = vlans.get(port)

   261

                    else :

   262

                        port_vlans = None

   263

   264

                    if port_vlans :

   265

                        local_untag, local_tagged = port_vlans

   266

   267

                    # directional mapping

   268

                    links_out[(local_node, remote_node)] = local_port

   269

                    nodes_port[(local_node, port)] = remote_node

   270

                    nodes_out.setdefault(local_node, set()).add(remote_node)

   271

   272

                    # bidirectional mappings

   273

                    remote_port = remote_attrs['port']

   274

   275

                    links_in[(remote_node, local_node)] = remote_port

   276

   277

                    forward = (local_node, local_port, remote_port, remote_node)

   278

                    reverse = (remote_node, remote_port, local_port, local_node)

   279

   280

                    if reverse not in links :

   281

                        links[forward] = (local_untag, local_tagged, None)

   282

                    else :

   283

                        remote_untag, remote_tagged, _ = links[reverse]

   284

   285

                        # merge

   286

                        if remote_untag != local_untag :

   287

                            log.warning("%s:%s untag %s <=> %s untag %s:%s",

   288

                                    host, local_port, local_untag,

   289

                                    remote_untag, remote_node, remote_port

   290

                            )

   291

   292

                        if remote_tagged != local_tagged :

   293

                            log.warning("%s:%s tagged %s <-> %s tagged %s:%s",

   294

                                    host, local_port, ':'.join(str(x) for x in sorted(local_tagged)),

   295

                                    ':'.join(str(x) for x in sorted(remote_tagged)), remote_node, remote_port

   296

                            )

   297

   298

                        links[reverse] = (remote_untag, remote_tagged, local_untag)

   299

   300

    # second graph: manual ports

   301

    for host in hosts :

   302

        local_node = host

   303

        host_links = host.extensions.get('link')

   304

   305

        if host_links :

   306

            if local_node not in nodes :

   307

                # XXX: copypasta

   308

                nodes[local_node] = host.location or str(host)

   309

   310

            for link_port, remote in host_links.iteritems() :

   311

                if link_port.isdigit() :

   312

                    port = int(link_port)

   313

                else :

   314

                    port = str(link_port)

   315

   316

                # map remote -> remote_host

   317

                if '@' in remote :

   318

                    remote_location, remote_domain = remote.split('@', 1)

   319

                else :

   320

                    remote_location = remote

   321

                    remote_domain = host.domain

   322

   323

                remote_node = remote_host = hosts_by_location.get((remote_domain, remote_location))

   324

   325

                if not remote_host :

   326

                    log.warning("%s:%s: unknown remote location: %s@%s", host, port, remote_location, remote_domain)

   327

                    continue

   328

   329

                remote_label = remote_host.location or str(remote_host)

   330

   331

                log.info("%s:%s: link -> %s@%s (%s)", host, port, remote_host, remote_host.domain, remote_label)

   332

   333

                if remote_node not in nodes :

   334

                    # XXX: copypasta

   335

                    nodes[remote_node] = remote_label

   336

   337

                # directional links

   338

                local_port = links_out.get((local_node, remote_node))

   339

   340

                if not local_port :

   341

                    log.info("%s:%s: unconfirmed -> %s", host, port, remote_host)

   342

   343

                elif local_port != port :

   344

                    log.warn("%s:%s: port mismatch %s -> %s", host, port, local_port, remote_host)

   345

   346

                else :

   347

                    log.debug("%s:%s: confirm -> %s", host, port, remote_host)

   348

   349

                links_out[(local_node, remote_node)] = port

   350

                nodes_port[(local_node, port)] = remote_node

   351

                nodes_out.setdefault(local_node, set()).add(remote_node)

   352

   353

                # update directional or missing links

   354

                remote_port = links_out.get((remote_node, local_node))

   355

                reverse_port = links_in.get((local_node, remote_node))

   356

   357

                if reverse_port and reverse_port != port :

   358

                    # XXX: this can be caused by str vs int >_>

   359

                    log.warn("%s:%s: reverse port mismatch %s <- %s", host, port, reverse_port, remote_node)

   360

   361

   362

                if local_port and remote_port :

   363

                    log.debug("%s:%s link <-> %s:%s", local_node, local_port, remote_port, remote_node)

   364

   365

                elif local_port :

   366

                    # we have the forward mapping already, so this doesn't add any new info

   367

                    log.debug("%s:%s link -> %s:%s", local_node, local_port, remote_port, remote_node)

   368

   369

                elif remote_port and reverse_port :

   370

                    # we have the bidirectional mapping already, so this doesn't add any new info

   371

                    log.debug("%s:%s link <-> %s:%s", local_node, local_port, remote_port, remote_node)

   372

   373

                elif remote_port :

   374

                    # we had the reverse mapping already, make it bidirectional

   375

                    local_port = port

   376

                    log.info("%s:%s link <- %s:%s", local_node, local_port, remote_port, remote_node)

   377

   378

                    links[(remote_node, remote_port, local_port, local_node)] = links.pop((remote_node, remote_port, None, local_node))

   379

   380

                else :

   381

                    local_port = port

   382

   383

                    # mapping was completely missing

   385

   386

                    links[(local_node, local_port, None, remote_node)] = None

   387

   388

    if options.graph_bridge :

   389

        # third graph: bridge

   390

        for host, host_attrs in snmp.iteritems() :

   391

            local_node = host

   392

   393

            node_out = nodes_out.get(local_node)

   394

   395

            if not node_out :

   396

                log.warning("%s: no outgoing links, skipping bridge", host)

   397

                continue

   398

   399

            bridge = { }

   400

   401

            for port, ethernets in host_attrs.get('bridge', { }).iteritems() :

   402

                bridge[(port, None)] = ethernets

   403

   404

            for vlan, vlan_attrs in host_attrs.get('vlan', { }).iteritems() :

   405

                for port, ethernets in vlan_attrs.get('bridge', { }).iteritems()  :

   406

                    bridge[(port, vlan)] = ethernets

   407

   408

            for (port, vlan), ethernets in bridge.iteritems() :

   409

                local_port = port

   410

   411

                remote_node = nodes_port.get((local_node, local_port))

   412

   413

                if remote_node :

   414

                    log.debug("%s:%s/%s bridge skip -> %s", host, port, vlan, remote_node)

   415

                    continue

   416

   417

                for ethernet in ethernets :

   418

                    # remote host

   419

                    if ethernet in hosts_by_ethernet :

   420

                        remote_node = remote_host = hosts_by_ethernet[ethernet]

   421

   422

                        remote_label = remote_host.location or str(remote_host)

   423

   424

                        log.debug("%s:%s/%s bridge %s = %s (%s)", host, port, vlan, ethernet, remote_host, remote_label)

   425

                    else :

   426

                        remote_node = remote_host = None

   427

   428

                    if not remote_host :

   429

                        continue

   430

   431

                    if remote_node not in nodes :

   432

                        log.debug("%s:%s: lazy-add remote %s (%s)", host, port, remote_node, remote_label)

   433

   434

                        nodes[remote_node] = remote_label

   435

   436

                    # unknown vlans

   437

                    link_vlans = None

   438

   439

                    # directional link

   440

                    links_out[(local_node, remote_node)] = local_port

   441

   442

                    # bidirectional link

   443

                    forward = (local_node, local_port, None, remote_node)

   444

   445

                    # scan for reverse

   446

                    remote_port = links_out.get((remote_node, local_node))

   447

   448

                    if remote_port :

   449

                        reverse = (remote_node, remote_port, None, local_node)

   450

   451

                        log.info("%s:%s <-> %s:%s", local_node, local_port, remote_port, remote_host)

   452

   453

                        # fill in remote_port for bidirectional link

   454

                        del links[reverse]

   455

                        reverse = local_node, local_port, remote_port, remote_node

   456

                        links[reverse] = link_vlans

   457

   458

                    else :

   459

                        log.info("%s:%s -> %s", local_node, local_port, remote_host)

   460

   461

                        links[forward] = link_vlans

   462

   463

   464

   465

    return nodes, links

   466

   467

class GraphVlans (object) :

   468

    """

   469

        Maintain vlan -> dot style/color mappings

   470

    """

   471

   472

    SERIES = 'paired12'

   473

    NONE = 'black'

   474

   475

    def __init__ (self, vlans=None) :

   476

        if vlans :

   477

            self.vlans = dict(vlans)

   478

        else :

   479

            self.vlans = { }

   480

   481

    def color (self, vlan) :

   482

        if vlan in self.vlans :

   483

            return self.vlans[vlan]

   484

   485

        # alloc

   486

        color = '/{series}/{index}'.format(series=self.SERIES, index=len(self.vlans) + 1)

   487

   488

        self.vlans[vlan] = color

   489

   490

        return color

   491

   492

def dot_quote (value) :

   493

    """

   494

        Quote a dot value.

   495

    """

   496