1

"""

     2

    Full-text searching of logs

     3

"""

     4

     5

import datetime, calendar, pytz

     6

import os.path

     7

     8

import HyperEstraier as hype

     9

    11

    12

class LogSearchError (Exception) :

    13

    """

    14

        General search error

    15

    """

    16

    17

    pass

    18

    19

class NoResultsFound (LogSearchError) :

    20

    """

    21

        No results found

    22

    """

    23

    24

    pass

    25

    26

class LogSearchIndex (object) :

    27

    """

    28

        An index on the logs for a group of channels.

    29

    30

        This uses Hyper Estraier to handle searching, whereby each log line is a document (yes, I have a powerful server).

    31

    32

        These log documents have the following attributes:

    42

    44

    """

    45

    46

    def __init__ (self, channels, path, mode='r') :

    47

        """

    48

            Open the database at the given path, with the given mode:

    49

                r       - read-only

    50

                w       - write, create if not exists

    51

                a       - write, error if not exists

    52

                c       - write, create, error if exists

    53

                *       - write, create, truncate if exists

    54

    55

            Channels is the ChannelList.

    56

        """

    57

    58

        # store

    59

        self.channels = channels

    60

        self.path = path

    61

        self.mode = mode

    62

    63

        # check it does not already exist?

    64

        if mode in 'c' and os.path.exists(path) :

    65

            raise LogSearchError("Index already exists: %s" % (path, ))

    66

    67

        # mapping of { mode -> flags }

    68

        mode_to_flag = {

    69

            'r':    hype.Database.DBREADER,

    70

            'w':    hype.Database.DBWRITER | hype.Database.DBCREAT,

    71

            'a':    hype.Database.DBWRITER,

    72

            'c':    hype.Database.DBWRITER | hype.Database.DBCREAT,

    73

            '*':    hype.Database.DBWRITER | hype.Database.DBCREAT | hype.Database.DBTRUNC,

    74

        }

    75

    76

        # look up flags

    77

        flags = mode_to_flag[mode]

    78

    79

        # make instance

    80

        self.db = hype.Database()

    81

    82

        # open

    83

        if not self.db.open(path, flags) :

    84

            raise Exception("Index open failed: %s, mode=%s, flags=%#06x: %s" % (path, mode, flags, self.db.err_msg(self.db.error())))

    85

    86

    def insert (self, channel, lines) :

    87

        """

    88

            Adds a sequence of LogLines from the given LogChannel to the index, and return the number of added items

    89

        """

    90

    91

        # validate the LogChannel

    92

        assert channel.name

    93

    94

        count = 0

    95

    96

        # iterate

    97

        for line in lines :

    98

            # validate the LogLine

    99

            assert line.offset

   100

            assert line.timestamp

   101

   102

            # create new document

   103

            doc = hype.Document()

   104

   105

            # line date

   106

            date = line.timestamp.date()

   107

   108

            # ensure that it's not 1900

   109

            assert date.year != 1900

   110

   111

            # add URI

   112

            doc.add_attr('@uri',        "%s/%s/%d" % (channel.id, date.strftime('%Y-%m-%d'), line.offset))

   113

   114

            # add channel id

   115

            doc.add_attr('channel',     channel.id)

   116

   117

            # add type

   118

            doc.add_attr('type',        str(line.type))

   119

   120

            # add UTC timestamp

   122

   123

            # add source attribute?

   124

            if line.source :

   125

                source_nickname, source_username, source_hostname, source_chanflags = line.source

   126

   132

   138

   146

            # add data

   147

            if line.data :

   148

                doc.add_text(line.data.encode('utf8'))

   149

   150

            # put, "clean up dispensable regions of the overwritten document"

   151

            if not self.db.put_doc(doc, hype.Database.PDCLEAN) :

   152

                raise Exeception("Index put_doc failed")

   153

   154

            # count

   155

            count += 1

   156

   157

        # return

   158

        return count

   159

   160

    def search_cond (self, cond) :

   161

        """

   162

            Search using a raw hype.Condition. Raises NoResultsFound if there aren't any results

   163

        """

   164

   165

        # execute search, unused 'flags' arg stays zero

   166

        results = self.db.search(cond, 0)

   167

   168

        # no results?

   169

        if not results :

   170

            raise NoResultsFound()

   171

   172

        # iterate over the document IDs

   173

        for doc_id in results :

   174

            # load document, this throws an exception...

   175

            # option constants are hype.Database.GDNOATTR/GDNOTEXT

   176

            doc = self.db.get_doc(doc_id, 0)

   177

   178

            # load the attributes/text

   179

            channel         = self.channels.lookup(doc.attr('channel'))

   180

            type            = int(doc.attr('type'))

   190

            message         = doc.cat_texts().decode('utf8')

   191

   192

            # build+yield to as LogLine

   194

   196

        """

   197

            Search with flexible parameters

   198

   199

                options     - bitmask of hype.Condition.*

   200

                channel     - LogChannel object

   202

                phrase      - the search query phrase

   203

                order       - order attribute expression

   204

                max         - number of results to return

   205

                skip        - number of results to skip

   206

        """

   207

   208

        # build condition

   209

        cond = hype.Condition()

   210

   211

        if options :

   212

            # set options

   213

            cond.set_options(options)

   214

   215

        if channel :

   216

            # add channel attribute

   218

   224

        if phrase :

   225

            # add phrase

   226

            cond.set_phrase(phrase)

   227

   228

        if order :

   229

            # set order

   230

            cond.set_order(order)

   231

   232

        if max :

   233

            # set max

   234

            cond.set_max(max)

   235

   236

        if skip :

   237

            # set skip

   238

            cond.set_skip(skip)

   239

   240

        # execute

   241

        return self.search_cond(cond)

   242

   243

    def search_simple (self, channel, query, count=None, offset=None) :

   244

        """

   245

            Search for lines from the given channel for the given simple query

   246

        """

   247

   248

        # use search(), backwards

   249

        results = list(self.search(

   250

            # simplified phrase

   251

            options     = hype.Condition.SIMPLE,

   252

   253

            # specific channel

   254

            channel     = channel,

   255

   256

            # given phrase

   257

            phrase      = query,

   258

   261

   262

            # count/offset

   263

            max         = count,

   264

            skip        = offset,

   265

        ))

   266

   267

        # reverse

   268

        return reversed(results)

   269