logwatcher.py
author Tero Marttila <terom@paivola.fi>
Fri, 21 Mar 2008 02:37:33 +0200
changeset 11 2d33d62cd8f8
parent 10 687b797f709c
child 15 e31e38d654b6
permissions -rw-r--r--
fix utmp.py, it's a proper module, doesn't tightloop and OOM, etc

committer: Tero Marttila <terom@paivola.fi>
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     1
from twisted.internet import protocol, reactor
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     2
from twisted.python import log
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     3
import sys, re
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     4
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     5
import api
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     6
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     7
class TailProcessProtocol (protocol.ProcessProtocol) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     8
    def __init__ (self, module, name, filters) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     9
        self.module = module
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    10
        self.name = name
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    11
        self.filters = filters
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    12
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    13
        self.buf = ""
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    14
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    15
    def errReceived (self, data) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    16
        self.module.error("tail for %s: %s" % (self.name, data))
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    17
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    18
    def outReceived (self, data) :
4
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    19
        data = self.buf + data
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    20
        
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    21
        while "\n" in data :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    22
            line, data = data.split("\n", 1)
4
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    23
            
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    24
            log.msg("Matching line `%s'..." % line)
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    25
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    26
            for filter in self.filters :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    27
                out = filter.test(line)
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    28
4
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    29
                if out :
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    30
                    log.msg("\t%s: %s" % (filter.event_type, out))
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    31
                    self.module.sendEvent(filter.event_type, out)
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    32
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    33
        self.buf = data
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    34
4
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    35
    def processEnded (self, reason) :
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    36
        msg = "tail process for %s quit: %s" % (self.name, reason.getErrorMessage())
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    37
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    38
        log.err(msg)
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    39
        self.module.error(msg)
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    40
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    41
class Filter (object) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    42
    def __init__ (self, regexp, event_type) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    43
        self.regexp = re.compile(regexp)
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    44
        self.event_type = event_type
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    45
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    46
    def test (self, line) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    47
        match = self.regexp.search(line)
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    48
        
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    49
        if match :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    50
            return self._filter(match)
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    51
        
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    52
    def _filter (self, match) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    53
        return match.string
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    54
6
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    55
class AutoFilter (Filter) :
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    56
    # your event type here, as a string
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    57
    EVENT = None
4
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
    58
6
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    59
    # your regexp here, with named matchgroups
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    60
    REGEXP = None
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    61
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    62
    # your output format, with named interpolation params
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    63
    OUTPUT = None
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    64
    
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    65
    def __init__ (self) :
6
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    66
        super(AutoFilter, self).__init__(self.REGEXP, self.EVENT)
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    67
    
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    68
    def _filter (self, match) :
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    69
        return self.OUTPUT % match.groupdict()
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    70
6
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    71
class SudoFilter (AutoFilter) :
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    72
    EVENT   = "sudo"
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    73
    REGEXP  = "sudo:\s*(?P<username>\S+) : TTY=(?P<tty>\S+) ; PWD=(?P<pwd>.+?) ; USER=(?P<target_user>\S+) ; COMMAND=(?P<command>.*)"
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    74
    OUTPUT  = "%(username)s:%(tty)s - %(pwd)s - `%(command)s` as %(target_user)s"
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    75
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    76
class SSHFilter (AutoFilter) :
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    77
    EVENT   = "ssh"
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    78
    REGEXP  = "(?P<success>Accepted|Failed) password for (?P<username>\S+) from (?P<ip>\S+) port (?P<port>\S+) (?P<proto>\S+)"
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    79
    OUTPUT  = "%(success)s login for %(username)s from %(ip)s:%(port)s proto %(proto)s"
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    80
10
687b797f709c fix ExampleModule -> LogWatchModule
Tero Marttila <terom@paivola.fi>
parents: 6
diff changeset
    81
class LogWatchModule (api.Module) :
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    82
    name = "logs"
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    83
    version = 0x0001
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    84
    
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    85
    event_types = [
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    86
        "error",
6
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    87
        "sudo",
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    88
        "ssh",
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    89
    ]
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    90
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    91
    log_files = (
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    92
        ("auth.log", "/var/log/auth.log", (
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    93
            SudoFilter(),
6
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
    94
            SSHFilter(),
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    95
        )),
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    96
    )
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    97
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    98
    log_objs = None
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    99
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   100
    def handleConnect (self) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   101
        log.msg("Spawning tail processes...")
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   102
        
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   103
        self.log_objs = dict()
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   104
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   105
        for name, file, filters in self.log_files :
4
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
   106
            log.msg("\t%s - %s..." % (name, file))
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   107
4
34d7897bd0f5 logwatcher works
Tero Marttila <terom@paivola.fi>
parents: 3
diff changeset
   108
            p = self.log_objs[name] = TailProcessProtocol(self, name, filters)
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   109
6
614161f85d9b some cleanup, bugfixes, commands for the irc bot, shared-secret for the API
Tero Marttila <terom@paivola.fi>
parents: 4
diff changeset
   110
            reactor.spawnProcess(p, "/usr/bin/tail", ["tail", "-n0", "--follow=name", file])
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   111
    
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   112
    def error (self, msg) :
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   113
        self.sendEvent("error", msg)
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   114
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   115
if __name__ == '__main__' :
10
687b797f709c fix ExampleModule -> LogWatchModule
Tero Marttila <terom@paivola.fi>
parents: 6
diff changeset
   116
    LogWatchModule().run()
3
5ab150c4a328 it works \o/ + the start of logwatcher.py
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   117