pvl/syslog/tail.py
author Tero Marttila <terom@fixme.fi>
Fri, 04 Jan 2013 18:43:18 +0200
changeset 49 30c615bf7751
parent 46 0bdbbda4cdea
child 53 685fd90bc610
permissions -rw-r--r--
pvl.syslog.tail: split Tail vs TailFile
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     1
"""
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
     2
    Iterate over input lines in files. 
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
     3
    
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
     4
    Can be read up to eof, on blocking inputs, or polled with a timeout (see: pvl.syslog.syslog.SyslogSource)
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     5
"""
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     6
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     7
import os
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
     8
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
     9
import logging; log = logging.getLogger('pvl.syslog.tail')
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    10
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    11
class Tail (object) :
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    12
    """
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    13
        Follow a file-like object, reading lines until EOF.
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    14
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    15
        Works with python file objects that buffer readlines() when using e.g. `tail -f ... | python -u ...`.
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    16
    """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    17
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    18
    def __init__ (self, file) :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    19
        self.file = file
46
0bdbbda4cdea pvl.syslog: have --syslog-tail skip to end of file
Tero Marttila <terom@fixme.fi>
parents: 31
diff changeset
    20
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    21
    def readline (self) :
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    22
        """
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    23
            Reads a line from the file, without trailing \n
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    24
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    25
            Raises EOF on end-of-file.
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    26
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    27
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    28
        line = self.file.readline()
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    29
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    30
        if not line :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    31
            raise EOFError()
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    32
        else :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    33
            return line.rstrip('\n')
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    34
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    35
    def readlines (self) :
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    36
        """
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    37
            Reads any available lines from the file.
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    38
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    39
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    40
        while True :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    41
            try :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    42
                line = self.readline()
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    43
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    44
            except EOFError :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    45
                log.debug("EOF")
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    46
                break
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    47
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    48
            else :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    49
                yield line
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    50
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    51
    def skip (self) :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    52
        """
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    53
            Skip any available lines.
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    54
        """
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    55
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    56
        for line in self.readlines() :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    57
            pass
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    58
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    59
    __iter__ = readlines
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    60
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    61
class TailFile (Tail) :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    62
    """
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    63
        Follow a file on the filesystem, reading lines until EOF, and re-opening if replaced.
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    64
    """
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    65
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    66
    def __init__ (self, path) :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    67
        self.path = path
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    68
        self.reopen()
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    69
    
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    70
    def stat (cls) :
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    71
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    72
            Return a key identifying the file at our path.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    73
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    74
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    75
        st = os.stat(self.path)
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    76
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    77
        return st.st_dev, st.st_ino
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    78
    
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    79
    def open (cls) :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    80
        return open(self.path, 'r')
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    81
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    82
    def reopen (cls) :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    83
        self.file = self.open()
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    84
        self._stat = self.stat()
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    85
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    86
    def changed (self) :
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    87
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    88
            Has the underlying file changed?
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    89
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    90
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
    91
        return self.stat() != self._stat
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    92
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    93
    def readlines (self, eof_mark=False) :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    94
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    95
            Reads any available lines from the file.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    96
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    97
            Reopens the file on EOF if the underlying file changed.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    98
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
    99
                eof_mark:   yields a special None line when this happens.
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   100
        """
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   101
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   102
        while True :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   103
            try :
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   104
                line = self.readline()
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   105
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   106
            except EOFError :
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   107
                if self.changed() :
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   108
                    log.debug("EOF: reopen")
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   109
                    self.reopen()
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   110
                    
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   111
                    if eof_mark :
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   112
                        yield None # special token
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   113
                    
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   114
                    # keep going
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   115
                    continue
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   116
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   117
                else :
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   118
                    log.debug("EOF")
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   119
                    break # wait
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   120
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   121
            else :
49
30c615bf7751 pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents: 46
diff changeset
   122
                yield line
31
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   123
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   124
    __iter__ = readlines
3e6d0feb115c pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff changeset
   125