pvl/syslog/parser.py
author Tero Marttila <terom@paivola.fi>
Thu, 10 Jan 2013 17:51:53 +0200
changeset 74 952ee07efd7a
parent 70 c8ec745a2aaa
child 101 6b27d7010bd4
permissions -rw-r--r--
pvl.syslog: implement --syslog-facility, implementing proper glob/regexp support in SyslogFilter
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     1
import datetime, time
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     2
import re
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     3
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     4
import logging; log = logging.getLogger('pvl.syslog.parser')
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     5
70
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
     6
RFC3339_RE = re.compile(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?(Z|[+-]\d{2}:\d{2})?')
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
     7
RFC3339_FMT = '%Y-%m-%dT%H:%M:%S'
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
     8
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
     9
def rfc3339 (timestamp) :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    10
    """
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    11
        RFC3339 timestamps as used in some syslog implementations.
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    12
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    13
        Returns a datetime in some random timezone, possibly localtime.
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    14
    """
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    15
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    16
    match = RFC3339_RE.match(timestamp)
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    17
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    18
    if not match :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    19
        return None
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    20
    
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    21
    # parts
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    22
    dt = datetime.datetime.strptime(match.group(1), RFC3339_FMT)
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    23
    tz = match.group(2)
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    24
    
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    25
    # TODO: timezone?
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    26
    return dt
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    27
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    28
    if not tz :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    29
        # XXX: localtime
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    30
        return dt
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    31
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    32
    elif tz == 'Z' :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    33
        # UTC
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    34
        pass
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    35
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    36
    elif tz[0] in '+-' :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    37
        hours, minutes = tz[1:].split(':')
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    38
        td = datetime.timedelta(hours=int(hours), minutes=int(minutes))
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    39
        
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    40
        if tz[0] == '-' :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    41
            dt += td
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    42
        if tz[0] == '+' :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    43
            dt -= td
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    44
    else :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    45
        raise ValueError("Invalid timezone offset: %s" % timestamp)
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    46
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    47
    # XXX: UTC
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    48
    return dt
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    49
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    50
RFC3164_RE = re.compile(r'\w{3} [0-9 ][0-9] \d{2}:\d{2}:\d{2}')
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    51
RFC3164_FMT = '%b %d %H:%M:%S'
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    52
RFC3164_PRE = '%Y ' # add missing year, assuming current
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    53
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    54
def rfc3164 (timestamp) :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    55
    """
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    56
        Traditional BSD Syslog timestamps.
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    57
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    58
        Returns a datetime assumed to be in localtime.
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    59
    """
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    60
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    61
    if not RFC3164_RE.match(timestamp) :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    62
        return
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    63
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    64
    return datetime.datetime.strptime(time.strftime(RFC3164_PRE) + timestamp, RFC3164_PRE + RFC3164_FMT)
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
    65
       
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    66
class SyslogParser (object) :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    67
    """
44
977442ccb72d pvl.syslog: split out SyslogFilter
Tero Marttila <terom@fixme.fi>
parents: 43
diff changeset
    68
        Parse syslog lines in text format, as used in logfiles/fifos.
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    69
    """
69
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    70
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    71
    SEVERITIES = dict(enumerate((
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    72
        'emerg',
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    73
        'alert', 
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    74
        'crit', 
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    75
        'err',
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    76
        'warning',
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    77
        'notice',
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    78
        'info', 
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    79
        'debug',
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    80
    )))
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    81
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    82
    FACILITIES = dict(enumerate((
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    83
        'kern',     # 0
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    84
        'user',     # 1
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    85
        'mail',     # 2
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    86
        'daemon',   # 3
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    87
        'auth',     # 4
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    88
        'syslog',   # 5
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    89
        'lpr',      # 6
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    90
        'news',     # 7
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    91
        'uucp',     # 8
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    92
        'cron',     # 9
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    93
        'authpriv', # 10
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    94
        'ftp',      # 11
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    95
        'ntp',      # 12
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    96
        'audit',    # 13
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    97
        'alert',    # 14
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    98
        'clock',    # 15
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
    99
        'local0',   # 16
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   100
        'local1',   # 17
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   101
        'local2',   # 18
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   102
        'local3',   # 19
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   103
        'local4',   # 20
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   104
        'local5',   # 21
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   105
        'local6',   # 22
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   106
        'local7',   # 23
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   107
    )))
70
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   108
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   109
    # default syslogd format
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   110
    SYSLOG_RE = re.compile(
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   111
        # the timestamp+hostname header
67
3324ed10c42f pvl.syslog.parser: missing hostname in 'last message repeated ... times' messages
Tero Marttila <terom@fixme.fi>
parents: 44
diff changeset
   112
        # XXX:  hostname may be missing
3324ed10c42f pvl.syslog.parser: missing hostname in 'last message repeated ... times' messages
Tero Marttila <terom@fixme.fi>
parents: 44
diff changeset
   113
        #       at least in Ubuntu 11.10 syslogd 'last message repeated 2 times'...
69
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   114
            r'(?:<(?P<pri>\d+|(?P<facility>\w+)\.(?P<severity>\w+))>)?'
70
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   115
        +   r'(?P<timestamp>\w{3} [0-9 ][0-9] \d{2}:\d{2}:\d{2}|.+?) '
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   116
        +   r'(?P<hostname>\S+)? '
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   117
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   118
        # the message, including possible tag/pid
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   119
        +   r"(?P<message>(?P<tag>(?P<program>[^:\]]+)(?:\[(?P<pid>\d+)\])?: )?(?P<text>.*))\n?"
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   120
    )
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   121
74
952ee07efd7a pvl.syslog: implement --syslog-facility, implementing proper glob/regexp support in SyslogFilter
Tero Marttila <terom@paivola.fi>
parents: 70
diff changeset
   122
    def __init__ (self, raw=False, facility=None) :
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   123
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   124
            Using given underlying line source.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   125
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   126
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   127
        self.raw = raw
74
952ee07efd7a pvl.syslog: implement --syslog-facility, implementing proper glob/regexp support in SyslogFilter
Tero Marttila <terom@paivola.fi>
parents: 70
diff changeset
   128
        self.facility = facility
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   129
69
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   130
    def parse_pri (self, match) :
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   131
        """
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   132
            Parse pri/facility/severity.
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   133
        """
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   134
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   135
        pri = match.group('pri')
74
952ee07efd7a pvl.syslog: implement --syslog-facility, implementing proper glob/regexp support in SyslogFilter
Tero Marttila <terom@paivola.fi>
parents: 70
diff changeset
   136
        facility = match.group('facility') or self.facility
69
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   137
        severity = match.group('severity')
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   138
        
74
952ee07efd7a pvl.syslog: implement --syslog-facility, implementing proper glob/regexp support in SyslogFilter
Tero Marttila <terom@paivola.fi>
parents: 70
diff changeset
   139
        if pri and pri.isdigit() :
69
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   140
            pri = int(pri)
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   141
            facility, severity = divmod(pri, 8)
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   142
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   143
        return dict(
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   144
            pri         = pri,
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   145
            severity    = self.SEVERITIES.get(severity, severity),
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   146
            facility    = self.FACILITIES.get(facility, facility)
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   147
        )
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   148
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   149
    def parse_timestamp (self, match) :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   150
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   151
            Parse timstamp from line into datetime.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   152
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   153
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   154
        timestamp = match.group('timestamp')
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   155
70
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   156
        # timestamp, in various formats
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   157
        try :
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   158
            return rfc3164(timestamp) or rfc3339(timestamp)
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   159
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   160
        except ValueError as ex:
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   161
            # skip it
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   162
            log.warning("timestamp: %s:", timestamp, exc_info=ex)
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   163
            return None
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   164
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   165
    def parse_prog (self, match) :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   166
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   167
            Parse prog from line.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   168
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   169
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   170
        prog = match.group('program')
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   171
44
977442ccb72d pvl.syslog: split out SyslogFilter
Tero Marttila <terom@fixme.fi>
parents: 43
diff changeset
   172
        if prog :
977442ccb72d pvl.syslog: split out SyslogFilter
Tero Marttila <terom@fixme.fi>
parents: 43
diff changeset
   173
            return prog
977442ccb72d pvl.syslog: split out SyslogFilter
Tero Marttila <terom@fixme.fi>
parents: 43
diff changeset
   174
        else :
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   175
            # no tag
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   176
            return None
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   177
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   178
    def parse (self, line) :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   179
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   180
            Parse given input line into SyslogMessage.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   181
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   182
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   183
        # ignore whitespace
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   184
        line = line.strip()
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   185
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   186
        # timestamp?
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   187
        if self.raw :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   188
            # from defaults
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   189
            return dict(
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   190
                timestamp   = datetime.datetime.now(), # XXX: None?
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   191
                host        = None,
44
977442ccb72d pvl.syslog: split out SyslogFilter
Tero Marttila <terom@fixme.fi>
parents: 43
diff changeset
   192
                prog        = None,
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   193
                pid         = None,
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   194
                msg         = line,
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   195
            )
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   196
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   197
        else :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   198
            # parse
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   199
            match = self.SYSLOG_RE.match(line)
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   200
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   201
            if not match :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   202
                log.warn("Unparseable syslog message: %r", line)
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   203
                return
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   204
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   205
            # parse
69
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   206
            item = dict(
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   207
                timestamp   = self.parse_timestamp(match),
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   208
                host        = match.group('hostname'),
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   209
                prog        = self.parse_prog(match),
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   210
                pid         = match.group('pid'),
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   211
                msg         = match.group('text'),
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   212
            )
70
c8ec745a2aaa pvl.syslog.parser: rfc3339 timestamp support
Tero Marttila <terom@fixme.fi>
parents: 69
diff changeset
   213
           
69
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   214
            # facility/severity prefix?
74
952ee07efd7a pvl.syslog: implement --syslog-facility, implementing proper glob/regexp support in SyslogFilter
Tero Marttila <terom@paivola.fi>
parents: 70
diff changeset
   215
            item.update(self.parse_pri(match))
69
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   216
9da998198936 pvl.syslog.parser: implement support for optional <PRI>
Tero Marttila <terom@fixme.fi>
parents: 67
diff changeset
   217
            return item
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   218
    
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   219
    def process (self, lines) :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   220
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   221
            Yield SyslogMessages from given series of lines.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   222
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   223
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   224
        for line in lines :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   225
            item = self.parse(line)
44
977442ccb72d pvl.syslog: split out SyslogFilter
Tero Marttila <terom@fixme.fi>
parents: 43
diff changeset
   226
74
952ee07efd7a pvl.syslog: implement --syslog-facility, implementing proper glob/regexp support in SyslogFilter
Tero Marttila <terom@paivola.fi>
parents: 70
diff changeset
   227
            log.debug("%s", item)
952ee07efd7a pvl.syslog: implement --syslog-facility, implementing proper glob/regexp support in SyslogFilter
Tero Marttila <terom@paivola.fi>
parents: 70
diff changeset
   228
44
977442ccb72d pvl.syslog: split out SyslogFilter
Tero Marttila <terom@fixme.fi>
parents: 43
diff changeset
   229
            if item :
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   230
                yield item
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   231
43
9d13b101beab pvl.syslog: implement pvl.syslog.args.apply -> SyslogSource as in pvl.verkko-dhcp
Tero Marttila <terom@fixme.fi>
parents: 31
diff changeset
   232
    __call__ = process
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   233