1

import datetime, time

     2

import re

     3

     4

import logging; log = logging.getLogger('pvl.syslog.parser')

     5

     6

RFC3339_RE = re.compile(r'(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(\.\d+)?(Z|[+-]\d{2}:\d{2})?')

     7

RFC3339_FMT = '%Y-%m-%dT%H:%M:%S'

     8

     9

def rfc3339 (timestamp) :

    10

    """

    11

        RFC3339 timestamps as used in some syslog implementations.

    12

    13

        Returns a datetime in some random timezone, possibly localtime.

    14

    """

    15

    16

    match = RFC3339_RE.match(timestamp)

    17

    18

    if not match :

    19

        return None

    20

    21

    # parts

    22

    dt = datetime.datetime.strptime(match.group(1), RFC3339_FMT)

    23

    tz = match.group(2)

    24

    25

    # TODO: timezone?

    26

    return dt

    27

    28

    if not tz :

    29

        # XXX: localtime

    30

        return dt

    31

    32

    elif tz == 'Z' :

    33

        # UTC

    34

        pass

    35

    36

    elif tz[0] in '+-' :

    37

        hours, minutes = tz[1:].split(':')

    38

        td = datetime.timedelta(hours=int(hours), minutes=int(minutes))

    39

    40

        if tz[0] == '-' :

    41

            dt += td

    42

        if tz[0] == '+' :

    43

            dt -= td

    44

    else :

    45

        raise ValueError("Invalid timezone offset: %s" % timestamp)

    46

    47

    # XXX: UTC

    48

    return dt

    49

    50

RFC3164_RE = re.compile(r'\w{3} [0-9 ][0-9] \d{2}:\d{2}:\d{2}')

    51

RFC3164_FMT = '%b %d %H:%M:%S'

    52

RFC3164_PRE = '%Y ' # add missing year, assuming current

    53

    54

def rfc3164 (timestamp) :

    55

    """

    56

        Traditional BSD Syslog timestamps.

    57

    58

        Returns a datetime assumed to be in localtime.

    59

    """

    60

    61

    if not RFC3164_RE.match(timestamp) :

    62

        return

    63

    64

    return datetime.datetime.strptime(time.strftime(RFC3164_PRE) + timestamp, RFC3164_PRE + RFC3164_FMT)

    65

    66

class SyslogParser (object) :

    67

    """

    68

        Parse syslog lines in text format, as used in logfiles/fifos.

    69

    """

    70

    71

    SEVERITIES = dict(enumerate((

    72

        'emerg',

    73

        'alert', 

    74

        'crit', 

    75

        'err',

    76

        'warning',

    77

        'notice',

    78

        'info', 

    79

        'debug',

    80

    )))

    81

    82

    FACILITIES = dict(enumerate((

    83

        'kern',     # 0

    84

        'user',     # 1

    85

        'mail',     # 2

    86

        'daemon',   # 3

    87

        'auth',     # 4

    88

        'syslog',   # 5

    89

        'lpr',      # 6

    90

        'news',     # 7

    91

        'uucp',     # 8

    92

        'cron',     # 9

    93

        'authpriv', # 10

    94

        'ftp',      # 11

    95

        'ntp',      # 12

    96

        'audit',    # 13

    97

        'alert',    # 14

    98

        'clock',    # 15

    99

        'local0',   # 16

   100

        'local1',   # 17

   101

        'local2',   # 18

   102

        'local3',   # 19

   103

        'local4',   # 20

   104

        'local5',   # 21

   105

        'local6',   # 22

   106

        'local7',   # 23

   107

    )))

   108

   109

    # default syslogd format

   110

    SYSLOG_RE = re.compile(

   111

        # the timestamp+hostname header

   112

        # XXX:  hostname may be missing

   113

        #       at least in Ubuntu 11.10 syslogd 'last message repeated 2 times'...

   114

            r'(?:<(?P<pri>\d+|(?P<facility>\w+)\.(?P<severity>\w+))>)?'

   115

        +   r'(?P<timestamp>\w{3} [0-9 ][0-9] \d{2}:\d{2}:\d{2}|.+?) '

   116

        +   r'(?P<hostname>\S+)? '

   117

   118

        # the message, including possible tag/pid

   119

        +   r"(?P<message>(?P<tag>(?P<program>[^:\]]+)(?:\[(?P<pid>\d+)\])?: )?(?P<text>.*))\n?"

   120

    )

   121

   123

        """

   124

            Using given underlying line source.

   125

        """

   126

   127

        self.raw = raw

   129

   130

    def parse_pri (self, match) :

   131

        """

   132

            Parse pri/facility/severity.

   133

        """

   134

   135

        pri = match.group('pri')

   137

        severity = match.group('severity')

   138

   140

            pri = int(pri)

   141

            facility, severity = divmod(pri, 8)

   142

   143

        return dict(

   144

            pri         = pri,

   145

            severity    = self.SEVERITIES.get(severity, severity),

   146

            facility    = self.FACILITIES.get(facility, facility)

   147

        )

   148

   149

    def parse_timestamp (self, match) :

   150

        """

   151

            Parse timstamp from line into datetime.

   152

        """

   153

   154

        timestamp = match.group('timestamp')

   155

   156

        # timestamp, in various formats

   157

        try :

   158

            return rfc3164(timestamp) or rfc3339(timestamp)

   159

   160

        except ValueError as ex:

   161

            # skip it

   162

            log.warning("timestamp: %s:", timestamp, exc_info=ex)

   163

            return None

   164

   165

    def parse_prog (self, match) :

   166

        """

   167

            Parse prog from line.

   168

        """

   169

   170

        prog = match.group('program')

   171

   172

        if prog :

   173

            return prog

   174

        else :

   175

            # no tag

   176

            return None

   177

   178

    def parse (self, line) :

   179

        """

   180

            Parse given input line into SyslogMessage.

   181

        """

   182

   183

        # ignore whitespace

   184

        line = line.strip()

   185

   186

        # timestamp?

   187

        if self.raw :

   188

            # from defaults

   189

            return dict(

   190

                timestamp   = datetime.datetime.now(), # XXX: None?

   191

                host        = None,

   192

                prog        = None,

   193

                pid         = None,

   194

                msg         = line,

   195

            )

   196

   197

        else :

   198

            # parse

   199

            match = self.SYSLOG_RE.match(line)

   200

   201

            if not match :

   202

                log.warn("Unparseable syslog message: %r", line)

   203

                return

   204

   205

            # parse

   206

            item = dict(

   207

                timestamp   = self.parse_timestamp(match),

   208

                host        = match.group('hostname'),

   209

                prog        = self.parse_prog(match),

   210

                pid         = match.group('pid'),

   211

                msg         = match.group('text'),

   212

            )

   213

   214

            # facility/severity prefix?

   216

   217

            return item

   218

   219

    def process (self, lines) :

   220

        """

   221

            Yield SyslogMessages from given series of lines.

   222

        """

   223

   224

        for line in lines :

   225

            item = self.parse(line)

   226

   229

            if item :

   230

                yield item

   231

   232

    __call__ = process

   233