utils.py
author Tero Marttila <terom@fixme.fi>
Thu, 12 Feb 2009 00:31:34 +0200
changeset 116 81da986f6ed5
parent 115 751e3fcd11d2
child 117 f0b4097f5781
permissions -rw-r--r--
fix wrong timezone for channel_date
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     1
"""
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     2
    Miscellaneous things
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     3
"""
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     4
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
     5
import datetime, calendar, pytz
95
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
     6
import os, errno
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
     7
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
     8
from qmsk.web.urltree import URLType
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
     9
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    10
class URLChannelName (URLType) :
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    11
    """
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    12
        Handle LogChannel names in URLs. Deals with instances of LogChannel
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    13
    """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    14
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    15
    def __init__ (self, channels) :
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    16
        """
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    17
            Use the given { name -> LogChannel } dict
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    18
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    19
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    20
        self.channels = channels
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    21
    
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    22
    def parse (self, chan_name) :
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    23
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    24
            chan_name -> LogChannel
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    25
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    26
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    27
        return self.channels[chan_name]
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    28
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    29
    def build (self, chan) :
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    30
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    31
            LogChannel -> chan_name
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    32
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    33
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    34
        return chan.id
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    35
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    36
class URLDateType (URLType) :
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    37
    """
115
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    38
        Handle dates in URLs as shortened UTC datetime timestamps
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    39
    """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    40
115
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    41
    # percision = hour
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    42
    SCALE = 60 * 60
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    43
73
5a7188bf2894 split defined configuration constants into config, and implement search result pagination
Tero Marttila <terom@fixme.fi>
parents: 72
diff changeset
    44
    def __init__ (self, date_fmt) :
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    45
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    46
            Format/parse dates using the given format
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    47
        """
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    48
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    49
        self.date_fmt = date_fmt
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    50
    
115
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    51
    def parse (self, sts_str) :
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    52
        """
115
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    53
            short_timestamp_str -> datetime.datetime
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    54
        """
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    55
        
115
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    56
        # scale -> from_utc + return
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    57
        return from_utc_timestamp(int(sts_str) * self.SCALE)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    58
    
115
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    59
    def build (self, dtz) :
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    60
        """
115
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    61
            datetime.datetime -> short_timestamp_str
51
07ca28f3a9f2 use improved URLConfig/URLType
Tero Marttila <terom@fixme.fi>
parents: 50
diff changeset
    62
        """
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    63
115
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    64
        # force it to be interpreted as UTC
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    65
        dt_utc = dtz.replace(tzinfo=pytz.utc)
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    66
        
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    67
        # scale the UTC timestamp
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    68
        sts = to_utc_timestamp(dt_utc) / self.SCALE
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    69
        
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    70
        # str
751e3fcd11d2 have dates in URLs be partial timestamps - fix datetime-timezone-comparison mess
Tero Marttila <terom@fixme.fi>
parents: 95
diff changeset
    71
        return str(sts)
50
f13cf27a360b implement more LogSource features (logs for date, cleanup last_logs), implement irssi parser, formatter, other misc. stuff
Tero Marttila <terom@fixme.fi>
parents:
diff changeset
    72
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    73
class URLTimestampType (URLType) :
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    74
    """
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    75
        Handles an integer UNIX timestamp as an UTC datetime
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    76
    """
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    77
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    78
    def parse (self, timestamp_str) :
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    79
        """
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    80
            timestamp_str -> pytz.utc datetime.datetime
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    81
        """
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    82
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    83
        return from_utc_timestamp(int(timestamp_str))
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    84
    
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    85
    def build (self, dtz) :
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    86
        """
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    87
            pytz.utc datetime.datetime -> timestamp_str
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    88
        """
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    89
        
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    90
        return str(to_utc_timestamp(dtz))
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    91
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    92
def from_utc_timestamp (timestamp) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    93
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    94
        Converts a UNIX timestamp into a datetime.datetime
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    95
    """
72
5ade0288f2ec implement line-links as UTC timestamps
Tero Marttila <terom@fixme.fi>
parents: 53
diff changeset
    96
89
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    97
    return datetime.datetime.utcfromtimestamp(timestamp).replace(tzinfo=pytz.utc)
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    98
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
    99
def to_utc_timestamp (dt) :
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   100
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   101
        Converts a datetime.datetime into a UNIX timestamp
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   102
    """
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   103
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   104
    return calendar.timegm(dt.utctimetuple())
2dc6de43f317 add utils.to/from_utc_timestamp functions, fix LogSearchIndex to store all LogLine attributes, add list() method to get LogLines for a given date, and improve scripts/search-index
Tero Marttila <terom@fixme.fi>
parents: 73
diff changeset
   105
95
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   106
def mtime (path, ignore_missing=False) :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   107
    """
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   108
        Gets the mtime for the given path as an UTC datetime, or None, if the file doesn't exist and ignore_missing
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   109
    """
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   110
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   111
    try :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   112
        # stat
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   113
        st = os.stat(path)
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   114
    
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   115
    # trap IOError
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   116
    except os.error, e :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   117
        # ENOENT?
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   118
        if ignore_missing and e.errno == errno.ENOENT :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   119
            return None
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   120
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   121
        else :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   122
            raise
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   123
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   124
    else :
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   125
        # decode
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   126
        return from_utc_timestamp(st.st_mtime)
ebdbda3dd5d0 implement utils.mtime
Tero Marttila <terom@fixme.fi>
parents: 89
diff changeset
   127