1

"""

     2

    Parse log data into log_events

     3

"""

     4

     5

import re

     6

import datetime

     7

     8

from log_line import LogTypes, LogLine

     9

    10

class LogParseError (Exception) :

    11

    """

    12

        Parsing some line failed

    13

    """

    14

    15

    def __init__ (self, line, offset, message) :

    16

        super(LogParseError, self).__init__("%r@%s: %s" % (line, offset, message))

    17

    18

class LogParser (object) :

    19

    """

    20

        Abstract interface

    21

    """

    22

    23

    def __init__ (self, tz, timestamp_fmt="%H:%M:%S") :

    24

        """

    25

            Setup the parser to use the given format for line timestamps, which are of the given timezone

    26

        """

    27

    28

        self.tz = tz

    29

        self.timestamp_fmt = timestamp_fmt

    30

    31

    def parse_lines (self, channel, lines, date=None, starting_offset=None) :

    32

        """

    33

            Parse the given (iterable) lines of unicode text into a LogEvent, no trailing newline.

    34

    35

            Channel is the LogChannel that these lines belong to.

    36

    37

            Offset is the starting offset, and may be None to not use it.

    38

    39

            Giving date lets the parser build full timestamps, otherwise, unless line timestamps have full date

    40

            information, event timestamps will have a date component of 1900/1/1.

    41

        """

    42

    43

        abstract

    44

    45

class IrssiParser (LogParser) :

    46

    """

    47

        A parser for irssi logfiles

    48

    """

    49

    50

    # subexpression parts

    51

    _TS = r'(?P<timestamp>\S+)'

    52

    _NICK = r'(?P<nickname>.+?)'

    53

    _NICK2 = r'(?P<nickname2>.+?)'

    54

    _TARGET = r'(?P<target>.+?)'

    55

    _CHAN = r'(?P<channel>.+?)'

    56

    _CHAN2 = r'(?P<channel2>.+?)'

    57

    _USERHOST = r'(?P<username>.*?)@(?P<hostname>.*?)'

    58

    _MSG = r'(?P<message>.*)'

    59

    _SRV1 = r'(?P<server1>.+?)'

    60

    _SRV2 = r'(?P<server2>.+?)'

    61

    62

    # regular expressions for matching lines, by type

    63

    TYPE_EXPRS = (

    64

        (   LogTypes.LOG_OPEN,      r'--- Log opened (?P<datetime>.+)'                              ),

    65

        (   LogTypes.LOG_CLOSE,     r'--- Log closed (?P<datetime>.+)'                              ),

    66

        (   LogTypes.MSG,           _TS + r' <(?P<flags>.)' + _NICK + '> ' + _MSG                   ),

    67

        (   LogTypes.NOTICE,        _TS + r' -' + _NICK + ':' + _CHAN + '- ' + _MSG                 ),

    68

        (   LogTypes.ACTION,        _TS + r'  \* ' + _NICK + ' ' + _MSG                             ),

    69

        (   LogTypes.JOIN,          _TS + r' -!- ' + _NICK + ' \[' + _USERHOST + '\] has joined ' + _CHAN                               ), 

    70

        (   LogTypes.PART,          _TS + r' -!- ' + _NICK + ' \[' + _USERHOST + '\] has left ' + _CHAN + ' \[(?P<message>.*?)\]'       ),

    71

        (   LogTypes.KICK,          _TS + r' -!- ' + _TARGET + ' was kicked from ' + _CHAN + ' by ' + _NICK + ' \[(?P<message>.*?)\]'   ),

    72

        # XXX: use hostname instead of nickname for ServerMode

    73

        (   LogTypes.MODE,          _TS + r' -!- (mode|ServerMode)/' + _CHAN + ' \[(?P<mode>.+?)\] by (?P<nickname>\S+)'                ),

    74

        (   LogTypes.NICK,          _TS + r' -!- ' + _NICK + ' is now known as (?P<target>\S+)'                                         ),

    75

        (   LogTypes.QUIT,          _TS + r' -!- ' + _NICK + ' \[' + _USERHOST + '\] has quit \[(?P<message>.*?)\]'                     ),

    76

        (   LogTypes.TOPIC,         _TS + r' -!- (' + _NICK + ' changed the topic of ' + _CHAN + ' to: (?P<topic>.*)|Topic unset by ' + _NICK2 + ' on ' + _CHAN2 + ')'    ),

    77

    78

        (   LogTypes.SELF_NOTICE,   _TS + r' \[notice\(' + _CHAN + '\)\] ' + _MSG                   ),

    79

        (   LogTypes.SELF_NICK,     _TS + r' -!- You\'re now known as (?P<target>\S+)'              ),

    80

    81

        (   LogTypes.NETSPLIT_START,    _TS + r' -!- Netsplit ' + _SRV1 + ' <-> ' + _SRV2 + ' quits: (?P<nick_list>[^(]+)( \(\+(?P<count>\d+) more,\S+\))?'),

    82

        (   LogTypes.NETSPLIT_END,      _TS + r' -!- Netsplit over, joins: (?P<nick_list>[^(]+)( \(\+(?P<count>\d+) more\))?'              ),

    83

    84

        (   'DAY_CHANGED',          r'--- Day changed (?P<date>.+)'                                 ),

    85

    )

    86

    87

    # precompile

    88

    TYPE_REGEXES = [(type, re.compile(expr)) for type, expr in TYPE_EXPRS]

    89

    90

    def parse_line (self, channel, line, date, offset=None) :

    91

        """

    92

            Parse a single line, and return the resulting LogLine, or None, to ignore the line.

    93

    94

            Uses self.TYPE_REGEXES to do the matching

    95

        """

    96

    97

        # empty line

    98

        if not line :

    99

            return

   100

   101

        # look for match

   102

        match = type = None

   103

   104

        # test each type

   105

        for type, regex in self.TYPE_REGEXES :

   106

            # attempt to match

   107

            match = regex.match(line)

   108

   109

            # found, break

   110

            if match :

   111

                break

   112

   113

        # no match found?

   114

        if not match :

   115

            raise LogParseError(line, offset, "Line did not match any type")

   116

   117

        # match groups

   118

        groups = match.groupdict(None)

   119

   120

        # parse timestamp

   121

        if 'datetime' in groups :

   122

            # parse datetime using default asctime() format

   123

            dt = datetime.datetime.strptime(groups['datetime'], '%a %b %d %H:%M:%S %Y')

   124

   125

        elif 'timestamp' in groups :

   126

            # parse timestamp into naive datetime

   127

            dt = datetime.datetime.strptime(groups['timestamp'], self.timestamp_fmt)

   128

   129

            # override date?

   130

            if date :

   131

                dt = dt.replace(year=date.year, month=date.month, day=date.day)

   132

   133

        elif 'date' in groups :

   134

            # parse date-only datetime

   135

            dt = datetime.datetime.strptime(groups['date'], '%a %b %d %Y')

   136

   137

        else :

   138

            # no timestamp !?

   139

            raise LogParseError(line, offset, "No timestamp")

   140

   141

        # now localize with timezone

   142

        dtz = self.tz.localize(dt)

   143

   144

        # channel, currently unused

   145

        channel_name = (groups.get('channel') or groups.get('channel2'))

   146

   147

        # source

   148

        if 'server1' in groups :

   149

            source = (None, None, groups.get('server1'), None)

   150

   151

        else :

   152

            source = (groups.get('nickname') or groups.get('nickname2'), groups.get('username'), groups.get('hostname'), groups.get('flags'))

   153

   154

        # target

   155

        if 'server2' in groups :

   156

            target = groups.get('server2')

   157

   158

        else :

   159

            target = groups.get('target')

   160

   161

        # data

   162

        if 'message' in groups :

   163

            data = groups['message']

   164

   165

        elif 'mode' in groups :

   166

            data = groups['mode']

   167

   168

        elif 'topic' in groups :

   169

            data = groups['topic']

   170

   171

        elif 'nick_list' in groups :

   172

            # split into components

   173

            list = groups['nick_list'].split(', ')

   174

   175

            # additional count?

   176

            if 'count' in groups and groups['count'] :

   177

                list.append('+%d' % int(groups['count']))

   178

   179

            # join

   180

            data = ' '.join(list)

   181

   182

        else :

   183

            data = None

   184

   185

        # custom types?

   186

        if type == 'DAY_CHANGED' :

   187

            # new date

   188

            date = dtz

   189

   190

        # build+return (date, LogLine)

   191

        return date, LogLine(channel, offset, type, dtz, source, target, data)

   192

   193

    def parse_lines (self, channel, lines, date=None, starting_offset=None) :

   194

        """

   195

            Parse the given lines, yielding LogEvents. 

   196

        """

   197

   198

        for offset, line in enumerate(lines) :

   199

            # offset?

   200

            if starting_offset :

   201

                offset = starting_offset + offset

   202

   203

            else :

   204

                offset = None

   205

   206

            # try and parse

   207

            try :

   208

                # update date as needed

   209

                date, line = self.parse_line(channel, line, date, offset)

   210

   211

            # passthrough LogParseError's

   212

            except LogParseError :

   213

                raise

   214

   215

            # wrap other errors as LogParseError

   216

            except Exception, e :

   217

                raise LogParseError(line, offset, "Parsing line failed: %s" % e)

   218

   219

            else :

   220

                # yield unless None

   221

                if line :

   222

                    yield line

   223

   224