utils.py
author Tero Marttila <terom@fixme.fi>
Wed, 11 Feb 2009 03:32:21 +0200
changeset 98 8c6e36849f9a
parent 95 ebdbda3dd5d0
child 115 751e3fcd11d2
permissions -rw-r--r--
implement tempfile use for scripts/search-index autoload, so that it can resume aborted sessions
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     1
"""
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     2
    Miscellaneous things
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     3
"""
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     4
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
     5
import datetime, calendar, pytz
95
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
     6
import os, errno
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     7
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
     8
from qmsk.web.urltree import URLType
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
     9
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    10
class URLChannelName (URLType) :
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    11
    """
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    12
        Handle LogChannel names in URLs. Deals with instances of LogChannel
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    13
    """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    14
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    15
    def __init__ (self, channels) :
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    16
        """
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    17
            Use the given { name -> LogChannel } dict
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    18
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    19
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    20
        self.channels = channels
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    21
    
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    22
    def parse (self, chan_name) :
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    23
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    24
            chan_name -> LogChannel
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    25
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    26
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    27
        return self.channels[chan_name]
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    28
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    29
    def build (self, chan) :
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    30
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    31
            LogChannel -> chan_name
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    32
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    33
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    34
        return chan.id
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    35
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    36
class URLDateType (URLType) :
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    37
    """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    38
        Handle dates in URLs as naive datetime objects (with indeterminate time info)
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    39
    """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    40
73
5a7188bf2894 split defined configuration constants into config, and implement search result pagination
Tero Marttila <terom@fixme.fi>
parents: 72
diff changeset
    41
    def __init__ (self, date_fmt) :
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    42
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    43
            Format/parse dates using the given format
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    44
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    45
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    46
        self.date_fmt = date_fmt
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    47
    
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    48
    def parse (self, date_str) :
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    49
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    50
            date_str -> naive datetime.datetime
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    51
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    52
        
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    53
        return datetime.datetime.strptime(date_str, self.date_fmt)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    54
    
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    55
    def build (self, date) :
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    56
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    57
            datetime.date -> date_str
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    58
        """
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    59
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    60
        return date.strftime(self.date_fmt)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    61
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    62
class URLTimestampType (URLType) :
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    63
    """
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    64
        Handles an integer UNIX timestamp as an UTC datetime
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    65
    """
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    66
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    67
    def parse (self, timestamp_str) :
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    68
        """
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    69
            timestamp_str -> pytz.utc datetime.datetime
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    70
        """
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    71
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    72
        return from_utc_timestamp(int(timestamp_str))
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    73
    
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    74
    def build (self, dtz) :
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    75
        """
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    76
            pytz.utc datetime.datetime -> timestamp_str
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    77
        """
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    78
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    79
        return str(to_utc_timestamp(dtz))
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    80
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    81
def from_utc_timestamp (timestamp) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    82
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    83
        Converts a UNIX timestamp into a datetime.datetime
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    84
    """
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    85
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    86
    return datetime.datetime.utcfromtimestamp(timestamp).replace(tzinfo=pytz.utc)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    87
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    88
def to_utc_timestamp (dt) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    89
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    90
        Converts a datetime.datetime into a UNIX timestamp
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    91
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    92
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    93
    return calendar.timegm(dt.utctimetuple())
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    94
95
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    95
def mtime (path, ignore_missing=False) :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    96
    """
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    97
        Gets the mtime for the given path as an UTC datetime, or None, if the file doesn't exist and ignore_missing
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    98
    """
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
    99
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   100
    try :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   101
        # stat
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   102
        st = os.stat(path)
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   103
    
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   104
    # trap IOError
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   105
    except os.error, e :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   106
        # ENOENT?
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   107
        if ignore_missing and e.errno == errno.ENOENT :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   108
            return None
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   109
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   110
        else :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   111
            raise
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   112
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   113
    else :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   114
        # decode
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   115
        return from_utc_timestamp(st.st_mtime)
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   116