author | Tero Marttila <terom@fixme.fi> |
Fri, 04 Jan 2013 18:43:18 +0200 | |
changeset 49 | 30c615bf7751 |
parent 46 | 0bdbbda4cdea |
child 53 | 685fd90bc610 |
permissions | -rw-r--r-- |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
1 |
""" |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
2 |
Iterate over input lines in files. |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
3 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
4 |
Can be read up to eof, on blocking inputs, or polled with a timeout (see: pvl.syslog.syslog.SyslogSource) |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
5 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
6 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
7 |
import os |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
8 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
9 |
import logging; log = logging.getLogger('pvl.syslog.tail') |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
10 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
11 |
class Tail (object) : |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
12 |
""" |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
13 |
Follow a file-like object, reading lines until EOF. |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
14 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
15 |
Works with python file objects that buffer readlines() when using e.g. `tail -f ... | python -u ...`. |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
16 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
17 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
18 |
def __init__ (self, file) : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
19 |
self.file = file |
46
0bdbbda4cdea
pvl.syslog: have --syslog-tail skip to end of file
Tero Marttila <terom@fixme.fi>
parents:
31
diff
changeset
|
20 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
21 |
def readline (self) : |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
22 |
""" |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
23 |
Reads a line from the file, without trailing \n |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
24 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
25 |
Raises EOF on end-of-file. |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
26 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
27 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
28 |
line = self.file.readline() |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
29 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
30 |
if not line : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
31 |
raise EOFError() |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
32 |
else : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
33 |
return line.rstrip('\n') |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
34 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
35 |
def readlines (self) : |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
36 |
""" |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
37 |
Reads any available lines from the file. |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
38 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
39 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
40 |
while True : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
41 |
try : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
42 |
line = self.readline() |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
43 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
44 |
except EOFError : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
45 |
log.debug("EOF") |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
46 |
break |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
47 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
48 |
else : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
49 |
yield line |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
50 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
51 |
def skip (self) : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
52 |
""" |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
53 |
Skip any available lines. |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
54 |
""" |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
55 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
56 |
for line in self.readlines() : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
57 |
pass |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
58 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
59 |
__iter__ = readlines |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
60 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
61 |
class TailFile (Tail) : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
62 |
""" |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
63 |
Follow a file on the filesystem, reading lines until EOF, and re-opening if replaced. |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
64 |
""" |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
65 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
66 |
def __init__ (self, path) : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
67 |
self.path = path |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
68 |
self.reopen() |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
69 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
70 |
def stat (cls) : |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
71 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
72 |
Return a key identifying the file at our path. |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
73 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
74 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
75 |
st = os.stat(self.path) |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
76 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
77 |
return st.st_dev, st.st_ino |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
78 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
79 |
def open (cls) : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
80 |
return open(self.path, 'r') |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
81 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
82 |
def reopen (cls) : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
83 |
self.file = self.open() |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
84 |
self._stat = self.stat() |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
85 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
86 |
def changed (self) : |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
87 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
88 |
Has the underlying file changed? |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
89 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
90 |
|
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
91 |
return self.stat() != self._stat |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
92 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
93 |
def readlines (self, eof_mark=False) : |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
94 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
95 |
Reads any available lines from the file. |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
96 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
97 |
Reopens the file on EOF if the underlying file changed. |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
98 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
99 |
eof_mark: yields a special None line when this happens. |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
100 |
""" |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
101 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
102 |
while True : |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
103 |
try : |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
104 |
line = self.readline() |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
105 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
106 |
except EOFError : |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
107 |
if self.changed() : |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
108 |
log.debug("EOF: reopen") |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
109 |
self.reopen() |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
110 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
111 |
if eof_mark : |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
112 |
yield None # special token |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
113 |
|
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
114 |
# keep going |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
115 |
continue |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
116 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
117 |
else : |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
118 |
log.debug("EOF") |
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
119 |
break # wait |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
120 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
121 |
else : |
49
30c615bf7751
pvl.syslog.tail: split Tail vs TailFile
Tero Marttila <terom@fixme.fi>
parents:
46
diff
changeset
|
122 |
yield line |
31
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
123 |
|
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
124 |
__iter__ = readlines |
3e6d0feb115c
pvl.syslog: import from pvl-collectd
Tero Marttila <terom@paivola.fi>
parents:
diff
changeset
|
125 |