1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, itertools

     6

import os, errno

     7

import pytz

     8

     9

class LogSource (object) :

    10

    """

    11

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    12

    """

    13

    14

    def get_latest (self, count) :

    15

        """

    16

            Yield the latest events, up to `count` of them.

    17

        """

    18

    19

        abstract

    20

    21

    def get_date (self, dt) :

    22

        """

    23

            Get logs for the given date (as a datetime)

    24

        """

    25

    26

        abstract

    27

    28

class LogFile (LogSource) :

    29

    """

    30

        A file containing LogEvents

    31

    """

    32

    33

    def __init__ (self, path, parser, start_date=None, charset='utf-8', sep='\n') :

    34

        """

    35

            Open the file at the given path, which contains data with the given charset, as lines separated by the

    36

            given separator. Lines are parsed using the given parser, using the given date as an initial date, see

    37

            LogParser for more info. XXX: currently we assume start_date also for the end of the file

    38

        """

    39

    40

        # store

    41

        self.path = path

    42

        self.parser = parser

    43

        self.start_date = start_date

    44

        self.charset = charset

    45

        self.sep = sep

    46

    47

        # open

    48

        self.file = open(path, 'rb')

    49

    50

    def __iter__ (self) :

    51

        """

    52

            Yields a series of unicode lines, as read from the top of the file

    53

        """

    54

    55

        # seek to beginning

    56

        self.file.seek(0)

    57

    58

        # iterate over lines, decoding them as well

    59

        return (line.decode(self.charset) for line in self.file)

    60

    61

    def read_full (self) :

    62

        """

    63

            Reads all LogLines

    64

        """

    65

    66

        # just use our __iter__

    67

        return self.parser.parse_lines(self, self.start_date)

    68

    69

    def read_from (self, dt) :

    70

        """

    71

            Reads all LogLines from the given naive timestamp onwards

    72

        """

    73

    74

        # start reading at beginning

    75

        events = self.read_full()

    76

    77

        # skip unwanted events

    78

        for event in events :

    79

            if event.timestamp < dt :

    80

                continue

    81

    82

            else :

    83

                # include this line as well

    84

                yield event

    85

                break

    86

    87

        # yield the rest as-is

    88

        for event in events :

    89

            yield event

    90

    91

    def read_until (self, dt) :

    92

        """

    93

            Reads all LogLines up until the given naive timestamp

    94

        """

    95

    96

        # start reading events at the beginning

    97

        events = self.read_full()

    98

    99

        # yield events until we hit the given timestamp

   100

        for event in events :

   101

            if event.timestamp <= dt :

   102

                yield event

   103

   104

            else :

   105

                break

   106

   107

        # ignore the rest

   108

        return

   109

   110

    def _read_blocks_reverse (self, blocksize=1024) :

   111

        """

   112

            Yields blocks of file data in reverse order, starting at the end of the file

   113

        """

   114

   115

        # seek to end of file

   116

        self.file.seek(0, os.SEEK_END)

   117

   118

        # read offset

   119

        # XXX: hack -1 to get rid of trailing newline

   120

        size = offset = self.file.tell() - 1

   121

   122

        # do not try to read past the beginning of the file

   123

        while offset > 0:

   124

            # calc new offset + size

   125

            if offset > blocksize :

   126

                # full block

   127

                offset -= blocksize

   128

                read_size = blocksize

   129

   130

            else :

   131

                # partial block

   132

                read_size = offset

   133

                offset = 0

   134

   135

            # seek to offset

   136

            self.file.seek(offset)

   137

   138

            # read the data we want

   139

            block = self.file.read(read_size)

   140

   141

            # sanity check

   142

            assert len(block) == read_size

   143

   144

            # yield 

   145

            yield block

   146

   147

    def _read_lines_reverse (self) :

   148

        """

   149

            Yields decoded lines from the end of the file, in reverse order.

   150

        """

   151

   152

        # partial lines

   153

        buf = ''

   154

   155

        # read from end of file, a block at a time

   156

        for block in self._read_blocks_reverse() :

   157

            # add in our previous buf

   158

            buf = block + buf

   159

   160

            # split up lines

   161

            lines = buf.split(self.sep)

   162

   163

            # keep the first one as our buffer, as it's incomplete

   164

            buf = lines[0]

   165

   166

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   167

            # XXX: use something like islice, this has to build a slice object

   168

            for line in lines[:0:-1] :

   169

                yield line.decode(self.charset)

   170

   171

    def get_latest (self, count) :

   172

        """

   173

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   174

        """

   175

   176

        # the list of lines

   177

        lines = []

   178

   179

        # start reading lines into lines

   180

        for line in self._read_lines_reverse() :

   181

            # append

   182

            lines.append(line)

   183

   184

            # done?

   185

            if len(lines) >= count :

   186

                break

   187

   188

        # decode in reverse order, using our starting date....

   189

        # XXX: use lines[::-1] or reversed?

   190

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   191

        return self.parser.parse_lines(reversed(lines), self.start_date)

   192

   193

class LogDirectory (LogSource) :

   194

    """

   195

        A directory containing a series of timestamped LogFiles

   196

    """

   197

   198

    def __init__ (self, path, tz, parser, charset='utf-8', filename_fmt='%Y-%m-%d') :

   199

        """

   200

            Load the logfiles at the given path.

   201

   202

            The files contain data in the given charset, and are named according the the date in the given timezone and

   203

            date format, and will be parsed using the given parser.

   204

        """

   205

   206

        # store

   207

        self.path = path

   208

        self.tz = tz

   209

        self.parser = parser

   210

        self.charset = charset

   211

        self.filename_fmt = filename_fmt

   212

   213

    def _get_logfile_datetime (self, dt) :

   214

        """

   215

            Get the logfile corresponding to the given datetime

   216

        """

   217

   218

        # convert to target timezone

   219

        dtz = dt.astimezone(self.tz)

   220

   221

        # convert to date and use that

   222

        return self._get_logfile_date(dtz.date())

   223

   224

    def _get_logfile_date (self, d) :

   225

        """

   226

            Get the logfile corresponding to the given naive date in our timezone

   227

        """

   228

   229

        # format filename

   230

        filename = d.strftime(self.filename_fmt)

   231

   232

        # build path

   233

        path = os.path.join(self.path, filename)

   234

   235

        # return the LogFile

   236

        return LogFile(path, self.parser, d, self.charset)

   237

   238

    def _iter_date_reverse (self, dt=None) :

   239

        """

   240

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   241

            given *datetime*, or the the current date, if none given

   242

        """

   243

   244

        # default to now

   245

        if not dt :

   246

            dt = datetime.datetime.now(pytz.utc)

   247

   248

        # convert to target timezone

   249

        dtz = dt.astimezone(self.tz)

   250

   251

        # our timedelta

   252

        ONE_DAY = datetime.timedelta(1)

   253

   254

        # iterate unto infinity

   255

        while True :

   256

            # yield

   257

            yield dtz.date()

   258

   259

            # one day sdrawkcab

   260

            dtz -= ONE_DAY

   261

   262

    def get_latest (self, count) :

   263

        """

   264

            Uses _iter_backwards + _get_logfile_date to read the yield the given lines from as many logfiles as needed

   265

        """

   266

   267

        # iterate backwards from now

   268

        day_iter = self._iter_date_reverse()

   269

   270

        # number of files read

   271

        files = 0

   272

   273

        # only read up to 100 files or so

   274

        MAX_FILES = 100

   275

   276

        # read the events into here

   277

        lines = []

   278

   279

        # loop until done

   280

        while len(lines) < count :

   281

            logfile = None

   282

   283

            try :

   284

                # get next logfile

   285

                files += 1

   286

   287

                # open

   288

                logfile = self._get_logfile_date(day_iter.next())

   289

   290

            except IOError, e :

   291

                # skip nonexistant days if we haven't found any logs yet

   292

                if e.errno != errno.ENOENT :

   293

                    raise

   294

   295

                if files > MAX_FILES :

   296

                    raise Exception("No recent logfiles found")

   297

   298

                else :

   299

                    # skip to next day

   300

                    continue

   301

   302

            # read the events

   303

            # XXX: use a queue

   304

            lines = list(logfile.get_latest(count)) + lines

   305

   306

        # return the events

   307

        return lines

   308

   309

    def get_date (self, dt) :

   310

        """

   311

            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime

   312

            differs from our native datetime, this may involve lines from more than one logfile.

   313

        """

   314

   315

        # begin/end of 24h period, in target timezone

   316

        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)

   317

        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)

   318

   319

        # as dates

   320

        d_begin = dtz_begin.date() 

   321

        d_end = dtz_end.date()

   322

   323

        # if they're the same, just pull the full log for that date

   324

        if d_begin == d_end :

   325

            return self._get_logfile_date(d_begin).read_full()

   326

   327

        # otherwise, we need to pull two partial logs

   328

        else :

   329

            # open both of them

   330

            f_begin = self._get_logfile_date(d_begin)

   331

            f_end = self._get_logfile_date(d_end)

   332

   333

            # chain together the two sources

   334

            return itertools.chain(f_begin.read_from(dtz_begin), f_end.read_until(dtz_end))

   335