log_source.py
changeset 48 7858b7b8ffe3
parent 46 185504387370
child 50 f13cf27a360b
--- a/log_source.py	Sun Feb 08 03:31:22 2009 +0200
+++ b/log_source.py	Sun Feb 08 04:41:00 2009 +0200
@@ -37,7 +37,7 @@
         self.sep = sep
 
         # open
-        self.file = codecs.open(path, 'r', charset)
+        self.file = open(path, 'rb')
     
     def __iter__ (self) :
         """
@@ -62,8 +62,8 @@
         self.file.seek(0, os.SEEK_END)
 
         # read offset
-        # XXX; why -2 ?
-        size = offset = self.file.tell() - 2
+        # XXX: hack -1 to get rid of trailing newline
+        size = offset = self.file.tell() - 1
 
         # use this blocksize
         BLOCKSIZE = 1024
@@ -72,26 +72,27 @@
         buf = ''
 
         # read a block at a time, backwards
-        while  count > 0 and offset >= 0:
-            # update offset back one block
-            offset -= BLOCKSIZE
+        while len(lines) < count and offset > 0:
+            # calc new offset + size
+            if offset > BLOCKSIZE :
+                # full block
+                offset -= BLOCKSIZE
+                read_size = BLOCKSIZE
 
-            # normalize to zero
-            if offset < 0 :
+            else :
+                # partial block
+                read_size = offset
                 offset = 0
 
             # seek to offset
             self.file.seek(offset)
 
-            # add the new block to our buffer
-            read_buf = self.file.read(BLOCKSIZE)
+            # read the data we want
+            read_buf = self.file.read(read_size)
+            read_len = len(read_buf)
 
-            # XXX: trim off extra...
-            if len(read_buf) > BLOCKSIZE :
-                read_buf = read_buf[:BLOCKSIZE]
-
-            # make sure we got the right amount of data
-            assert len(read_buf) == BLOCKSIZE, "read(%d) @ %d/%d -> %d" % (BLOCKSIZE, offset, size, len(read_buf))
+            # sanity check
+            assert read_len == read_size
 
             # add in our previous buf
             buf = read_buf + buf
@@ -102,11 +103,12 @@
             # keep the first one as our buffer, as it's incomplete
             buf = buf_lines[0]
 
-            # add up to count lines to our lines buffer
+            # prepend up to count lines from the end to our lines buffer
             lines = buf_lines[-min(count, len(buf_lines) - 1):] + lines
-
-            # update count
-            count -= (len(buf_lines) - 1)
+        
+        # decode
+        # XXX: better queue implementation, plz
+        lines = [line.decode(self.charset) for line in lines]
 
         # return the line list
         return lines
@@ -192,9 +194,12 @@
 
         # only read up to 100 files or so
         MAX_FILES = 100
+
+        # read the lines into here
+        lines = []
         
         # loop until done
-        while count > 0 :
+        while len(lines) < count :
             logfile = None
 
             try :
@@ -215,17 +220,10 @@
                 else :
                     # skip to next day
                     continue
+            
+            # read the lines
+            lines = logfile.get_latest(count) + lines
+        
+        # return the lines
+        return lines
 
-            # yield lines
-            for line in logfile.get_latest(count) :
-                # yield while we still need to, otherwise, stop
-                if count > 0 :
-                    # decrement
-                    count -= 1
- 
-                    yield line
-            
-                else :
-                    break
-
-