1

"""

     2

    A source of IRC log files

     3

"""

     4

     5

import datetime, calendar, itertools

     6

import os, errno

     7

import pytz

     8

     9

class LogSource (object) :

    10

    """

    11

        A collection of IRC logs for a specific target in some format. Provides the possibility to read specific events

    12

    """

    13

    14

    def get_latest (self, count) :

    15

        """

    16

            Yield the latest events, up to `count` of them.

    17

        """

    18

    19

        abstract

    20

    21

    def get_date (self, dt) :

    22

        """

    23

            Get logs for the given date (as a datetime)

    24

        """

    25

    26

        abstract

    27

    28

    def get_month_days (self, dt) :

    29

        """

    30

            Get a set of dates, telling which days in the given month (as a datetime) have logs available

    31

        """

    32

    33

        abstract

    42

class LogFile (object) :

    43

    """

    44

        A file containing LogEvents

    45

    46

        XXX: modify to implement LogSource?

    47

    """

    48

    49

    def __init__ (self, path, parser, start_date=None, charset='utf-8', sep='\n') :

    50

        """

    51

            Open the file at the given path, which contains data with the given charset, as lines separated by the

    52

            given separator. Lines are parsed using the given parser, using the given date as an initial date, see

    53

            LogParser for more info. XXX: currently we assume start_date also for the end of the file

    54

        """

    55

    56

        # store

    57

        self.path = path

    58

        self.parser = parser

    59

        self.start_date = start_date

    60

        self.charset = charset

    61

        self.sep = sep

    62

    63

        # open

    64

        self.file = open(path, 'rb')

    65

    66

    def __iter__ (self) :

    67

        """

    68

            Yields a series of unicode lines, as read from the top of the file

    69

        """

    70

    71

        # seek to beginning

    72

        self.file.seek(0)

    73

    74

        # iterate over lines, decoding them as well

    75

        return (line.decode(self.charset) for line in self.file)

    76

    77

    def read_full (self) :

    78

        """

    79

            Reads all LogLines

    80

        """

    81

    82

        # just use our __iter__

    83

        return self.parser.parse_lines(self, self.start_date)

    84

    85

    def read_from (self, dt) :

    86

        """

    87

            Reads all LogLines from the given naive timestamp onwards

    88

        """

    89

    90

        # start reading at beginning

    91

        events = self.read_full()

    92

    93

        # skip unwanted events

    94

        for event in events :

    95

            if event.timestamp < dt :

    96

                continue

    97

    98

            else :

    99

                # include this line as well

   100

                yield event

   101

                break

   102

   103

        # yield the rest as-is

   104

        for event in events :

   105

            yield event

   106

   107

    def read_until (self, dt) :

   108

        """

   109

            Reads all LogLines up until the given naive timestamp

   110

        """

   111

   112

        # start reading events at the beginning

   113

        events = self.read_full()

   114

   115

        # yield events until we hit the given timestamp

   116

        for event in events :

   117

            if event.timestamp <= dt :

   118

                yield event

   119

   120

            else :

   121

                break

   122

   123

        # ignore the rest

   124

        return

   125

   126

    def _read_blocks_reverse (self, blocksize=1024) :

   127

        """

   128

            Yields blocks of file data in reverse order, starting at the end of the file

   129

        """

   130

   131

        # seek to end of file

   132

        self.file.seek(0, os.SEEK_END)

   133

   134

        # read offset

   135

        # XXX: hack -1 to get rid of trailing newline

   136

        size = offset = self.file.tell() - 1

   137

   138

        # do not try to read past the beginning of the file

   139

        while offset > 0:

   140

            # calc new offset + size

   141

            if offset > blocksize :

   142

                # full block

   143

                offset -= blocksize

   144

                read_size = blocksize

   145

   146

            else :

   147

                # partial block

   148

                read_size = offset

   149

                offset = 0

   150

   151

            # seek to offset

   152

            self.file.seek(offset)

   153

   154

            # read the data we want

   155

            block = self.file.read(read_size)

   156

   157

            # sanity check

   158

            assert len(block) == read_size

   159

   160

            # yield 

   161

            yield block

   162

   163

    def _read_lines_reverse (self) :

   164

        """

   165

            Yields decoded lines from the end of the file, in reverse order.

   166

        """

   167

   168

        # partial lines

   169

        buf = ''

   170

   171

        # read from end of file, a block at a time

   172

        for block in self._read_blocks_reverse() :

   173

            # add in our previous buf

   174

            buf = block + buf

   175

   176

            # split up lines

   177

            lines = buf.split(self.sep)

   178

   179

            # keep the first one as our buffer, as it's incomplete

   180

            buf = lines[0]

   181

   182

            # yield the rest a line at a time in reverse order... this looks weird, but that's how slicing works :)

   183

            # XXX: use something like islice, this has to build a slice object

   184

            for line in lines[:0:-1] :

   185

                yield line.decode(self.charset)

   186

   187

    def read_latest (self, count) :

   188

        """

   189

            Returns up to count events, from the end of the file, or less, if the file doesn't contain that many lines.

   190

        """

   191

   192

        # the list of lines

   193

        lines = []

   194

   195

        # start reading lines into lines

   196

        for line in self._read_lines_reverse() :

   197

            # append

   198

            lines.append(line)

   199

   200

            # done?

   201

            if len(lines) >= count :

   202

                break

   203

   204

        # decode in reverse order, using our starting date....

   205

        # XXX: use lines[::-1] or reversed?

   206

        # XXX: it may make more sense to parse in reverse order, using 'self.end_date' or something like that

   207

        return self.parser.parse_lines(reversed(lines), self.start_date)

   208

   209

class LogDirectory (LogSource) :

   210

    """

   211

        A directory containing a series of timestamped LogFiles

   212

    """

   213

   214

    def __init__ (self, path, tz, parser, charset='utf-8', filename_fmt='%Y-%m-%d') :

   215

        """

   216

            Load the logfiles at the given path.

   217

   218

            The files contain data in the given charset, and are named according the the date in the given timezone and

   219

            date format, and will be parsed using the given parser.

   220

        """

   221

   222

        # store

   223

        self.path = path

   224

        self.tz = tz

   225

        self.parser = parser

   226

        self.charset = charset

   227

        self.filename_fmt = filename_fmt

   228

   229

    def _get_logfile_datetime (self, dt) :

   230

        """

   231

            Get the logfile corresponding to the given datetime

   232

        """

   233

   234

        # convert to target timezone

   235

        dtz = dt.astimezone(self.tz)

   236

   237

        # convert to date and use that

   238

        return self._get_logfile_date(dtz.date())

   239

   240

    def _get_logfile_date (self, d, load=True) :

   241

        """

   242

            Get the logfile corresponding to the given naive date in our timezone. If load is False, only test for the

   243

            presence of the logfile, do not actually open it.

   244

   245

            Returns None if the logfile does not exist.

   246

        """

   247

   248

        # format filename

   249

        filename = d.strftime(self.filename_fmt)

   250

   251

        # build path

   252

        path = os.path.join(self.path, filename)

   253

   254

        try :

   255

            if load :

   256

                # open+return the LogFile

   257

                return LogFile(path, self.parser, d, self.charset)

   258

   259

            else :

   260

                # test

   261

                return os.path.exists(path)

   262

   263

        # XXX: move to LogFile

   264

        except IOError, e :

   265

            # return None for missing files

   266

            if e.errno == errno.ENOENT :

   267

                return None

   268

   269

            else :

   270

                raise

   271

   272

    def _iter_date_reverse (self, dt=None) :

   273

        """

   274

            Yields an infinite series of naive date objects in our timezone, iterating backwards in time starting at the

   275

            given *datetime*, or the the current date, if none given

   276

        """

   277

   278

        # default to now

   279

        if not dt :

   280

            dt = datetime.datetime.now(pytz.utc)

   281

   282

        # convert to target timezone

   283

        dtz = dt.astimezone(self.tz)

   284

   285

        # our timedelta

   286

        ONE_DAY = datetime.timedelta(1)

   287

   288

        # iterate unto infinity

   289

        while True :

   290

            # yield

   291

            yield dtz.date()

   292

   293

            # one day sdrawkcab

   294

            dtz -= ONE_DAY

   295

   297

        """

   302

        """

   303

   306

   314

            logfile = None

   315

   318

   320

            if not logfile :

   323

                    raise Exception("No recent logfiles found")

   324

   325

                else :

   326

                    # skip to next day

   327

                    continue

   328

   348

            # read the events

   349

            # XXX: use a queue

   350

            lines = list(logfile.read_latest(count)) + lines

   351

   352

        # return the events

   353

        return lines

   354

   355

    def get_date (self, dt) :

   356

        """

   357

            A 'day' is considered to be a 24-hour period from 00:00:00 23:59:59. If the timezone of the given datetime

   358

            differs from our native datetime, this may involve lines from more than one logfile.

   359

        """

   360

   361

        # begin/end of 24h period, in target timezone

   362

        dtz_begin = dt.replace(hour=0, minute=0, second=0).astimezone(self.tz)

   363

        dtz_end = dt.replace(hour=23, minute=59, second=59, microsecond=999999).astimezone(self.tz)

   364

   365

        # as dates

   366

        d_begin = dtz_begin.date() 

   367

        d_end = dtz_end.date()

   368

   369

        # if they're the same, just pull the full log for that date

   370

        if d_begin == d_end :

   371

            return self._get_logfile_date(d_begin).read_full()

   372

   373

        # otherwise, we need to pull two partial logs

   374

        else :

   375

            # open both of them

   376

            f_begin = self._get_logfile_date(d_begin)

   377

            f_end = self._get_logfile_date(d_end)

   378

   379

            # chain together the two sources

   380

            return itertools.chain(

   381

                f_begin.read_from(dtz_begin), 

   382

                f_end.read_until(dtz_end) if f_end else []

   383

            )

   384

   385

    def get_month_days (self, month) :

   386

        """

   387

            Returns a set of dates for which logfiles are available in the given datetime's month

   388

        """

   389

   390

        # the set of days

   391

        days = set()

   392

   393

        # iterate over month's days using Calendar

   394

        for date in calendar.Calendar().itermonthdates(month.year, month.month) :

   395

            # convert date to target datetime

   396

            dtz = month.tzinfo.localize(datetime.datetime.combine(date, datetime.time(0))).astimezone(self.tz)

   397

   398

            # date in our target timezone

   399

            log_date = dtz.date()

   400

   401

            # test for it

   402

            if self._get_logfile_date(log_date, load=False) :

   403

                # add to set

   404

                days.add(date)

   405

   406

        # return set

   407

        return days

   408