name : datedetectortestcase.py # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: t -*- # vi: set ft=python sts=4 ts=4 sw=4 noet : # This file is part of Fail2Ban. # # Fail2Ban is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # Fail2Ban is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with Fail2Ban; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. # Author: Cyril Jaquier # __author__ = "Cyril Jaquier" __copyright__ = "Copyright (c) 2004 Cyril Jaquier" __license__ = "GPL" import unittest import time import datetime from ..server.datedetector import DateDetector from ..server import datedetector from ..server.datetemplate import DatePatternRegex, DateTemplate from .utils import setUpMyTime, tearDownMyTime, LogCaptureTestCase from ..helpers import getLogger logSys = getLogger("fail2ban") class DateDetectorTest(LogCaptureTestCase): def setUp(self): """Call before every test case.""" LogCaptureTestCase.setUp(self) setUpMyTime() self.__datedetector = None def tearDown(self): """Call after every test case.""" LogCaptureTestCase.tearDown(self) tearDownMyTime() @property def datedetector(self): if self.__datedetector is None: self.__datedetector = DateDetector() self.__datedetector.addDefaultTemplate() return self.__datedetector def testGetEpochTime(self): self.__datedetector = DateDetector() self.__datedetector.appendTemplate('EPOCH') # correct epoch time, using all variants: for dateUnix in (1138049999, 32535244799): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): date = date % dateUnix log = date + " [sshd] error: PAM: Authentication failure" datelog = self.datedetector.getTime(log) self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,)) ( datelog, matchlog ) = datelog self.assertEqual(int(datelog), dateUnix) self.assertIn(matchlog.group(1), (str(dateUnix), str(dateUnix)+'.555')) # wrong, no epoch time (< 10 digits, more as 11 digits, begin/end of word) : for dateUnix in ('123456789', '9999999999999999', '1138049999A', 'A1138049999'): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): date = date % dateUnix log = date + " [sshd] error: PAM: Authentication failure" datelog = self.datedetector.getTime(log) self.assertFalse(datelog) def testGetEpochMsTime(self): self.__datedetector = DateDetector() self.__datedetector.appendTemplate('LEPOCH') # correct short/long epoch time, using all variants: for fact in (1, 1000, 1000000): for dateUnix in (1138049999, 32535244799): for date in ("%s", "[%s]", "[%s]", "audit(%s:101)"): dateLong = dateUnix * fact date = date % dateLong log = date + " [sshd] error: PAM: Authentication failure" datelog = self.datedetector.getTime(log) self.assertTrue(datelog, "Parse epoch time for %s failed" % (date,)) ( datelog, matchlog ) = datelog self.assertEqual(int(datelog), dateUnix) self.assertEqual(matchlog.group(1), str(dateLong)) # wrong, no epoch time (< 10 digits, more as 17 digits, begin/end of word) : for dateUnix in ('123456789', '999999999999999999', '1138049999A', 'A1138049999'): for date in ("%s", "[%s]", "[%s.555]", "audit(%s.555:101)"): date = date % dateUnix log = date + " [sshd] error: PAM: Authentication failure" datelog = self.datedetector.getTime(log) self.assertFalse(datelog) def testGetEpochPattern(self): self.__datedetector = DateDetector() self.__datedetector.appendTemplate(r'(?<=\|\s){LEPOCH}(?=\s\|)') # correct short/long epoch time, using all variants: for fact in (1, 1000, 1000000): for dateUnix in (1138049999, 32535244799): dateLong = dateUnix * fact log = "auth-error | %s | invalid password" % dateLong datelog = self.datedetector.getTime(log) self.assertTrue(datelog, "Parse epoch time failed: %r" % (log,)) ( datelog, matchlog ) = datelog self.assertEqual(int(datelog), dateUnix) self.assertEqual(matchlog.group(1), str(dateLong)) # wrong epoch time format (does not match pattern): for log in ("test%s123", "test-right | %stest", "test%s | test-left"): log = log % dateLong datelog = self.datedetector.getTime(log) self.assertFalse(datelog) def testGetTime(self): log = "Jan 23 21:59:59 [sshd] error: PAM: Authentication failure" dateUnix = 1106513999.0 # yoh: testing only up to 6 elements, since the day of the week # is not correctly determined atm, since year is not present # in the log entry. Since this doesn't effect the operation # of fail2ban -- we just ignore incorrect day of the week ( datelog, matchlog ) = self.datedetector.getTime(log) self.assertEqual(datelog, dateUnix) self.assertEqual(matchlog.group(1), 'Jan 23 21:59:59') def testDefaultTimeZone(self): # use special date-pattern (with %Exz), because %z currently does not supported # zone abbreviations except Z|UTC|GMT. dd = DateDetector() dd.appendTemplate('^%ExY-%Exm-%Exd %H:%M:%S(?: ?%Exz)?') dt = datetime.datetime logdt = "2017-01-23 15:00:00" dtUTC = dt(2017, 1, 23, 15, 0) for tz, log, desired in ( # no TZ in input-string: ('UTC+0300', logdt, dt(2017, 1, 23, 12, 0)), # so in UTC, it was noon ('UTC', logdt, dtUTC), # UTC ('UTC-0430', logdt, dt(2017, 1, 23, 19, 30)), ('GMT+12', logdt, dt(2017, 1, 23, 3, 0)), (None, logdt, dt(2017, 1, 23, 14, 0)), # default CET in our test-framework # CET: ('CET', logdt, dt(2017, 1, 23, 14, 0)), ('+0100', logdt, dt(2017, 1, 23, 14, 0)), ('CEST-01', logdt, dt(2017, 1, 23, 14, 0)), # CEST: ('CEST', logdt, dt(2017, 1, 23, 13, 0)), ('+0200', logdt, dt(2017, 1, 23, 13, 0)), ('CET+01', logdt, dt(2017, 1, 23, 13, 0)), ('CET+0100', logdt, dt(2017, 1, 23, 13, 0)), # check offset in minutes: ('CET+0130', logdt, dt(2017, 1, 23, 12, 30)), # TZ in input-string have precedence: ('UTC+0300', logdt+' GMT', dtUTC), # GMT wins ('UTC', logdt+' GMT', dtUTC), # GMT wins ('UTC-0430', logdt+' GMT', dtUTC), # GMT wins (None, logdt+' GMT', dtUTC), # GMT wins ('UTC', logdt+' -1045', dt(2017, 1, 24, 1, 45)), # -1045 wins (None, logdt+' -10:45', dt(2017, 1, 24, 1, 45)), # -1045 wins ('UTC', logdt+' +0945', dt(2017, 1, 23, 5, 15)), # +0945 wins (None, logdt+' +09:45', dt(2017, 1, 23, 5, 15)), # +0945 wins ('UTC+0300', logdt+' Z', dtUTC), # Z wins (UTC) ('GMT+12', logdt+' CET', dt(2017, 1, 23, 14, 0)), # CET wins ('GMT+12', logdt+' CEST', dt(2017, 1, 23, 13, 0)), # CEST wins ('GMT+12', logdt+' CET+0130', dt(2017, 1, 23, 12, 30)), # CET+0130 wins ): logSys.debug('== test %r with TZ %r', log, tz) dd.default_tz=tz; datelog, _ = dd.getTime(log) val = dt.utcfromtimestamp(datelog) self.assertEqual(val, desired, "wrong offset %r != %r by %r with default TZ %r (%r)" % (val, desired, log, tz, dd.default_tz)) self.assertRaises(ValueError, setattr, dd, 'default_tz', 'WRONG-TZ') dd.default_tz = None def testVariousTimes(self): """Test detection of various common date/time formats f2b should understand """ dateUnix = 1106513999.0 # anchored - matching expression (pattern) is anchored # bound - pattern can be tested using word boundary (e.g. False if contains in front some optional part) # sdate - date string used in test log-line # rdate - if specified, the result match, which differs from sdate for anchored, bound, sdate, rdate in ( (False, True, "Jan 23 21:59:59", None), (False, False, "Sun Jan 23 21:59:59 2005", None), (False, False, "Sun Jan 23 21:59:59", None), (False, False, "Sun Jan 23 2005 21:59:59", None), (False, True, "2005/01/23 21:59:59", None), (False, True, "2005.01.23 21:59:59", None), (False, True, "23/01/2005 21:59:59", None), (False, True, "23/01/05 21:59:59", None), (False, True, "23/Jan/2005:21:59:59", None), (False, True, "23/Jan/2005:21:59:59 +0100", None), (False, True, "01/23/2005:21:59:59", None), (False, True, "2005-01-23 21:59:59", None), (False, True, "2005-01-23 21:59:59,000", None), # proftpd (False, True, "23-Jan-2005 21:59:59", None), (False, True, "23-Jan-2005 21:59:59.02", None), (False, True, "23-Jan-2005 21:59:59 +0100", None), (False, True, "23-01-2005 21:59:59", None), (True, True, "1106513999", None), # Portsetry (False, True, "01-23-2005 21:59:59.252", None), # reported on f2b, causes Feb29 fix to break (False, False, "@4000000041f4104f00000000", None), # TAI64N (False, True, "2005-01-23T20:59:59.252Z", None), #ISO 8601 (UTC) (False, True, "2005-01-23T15:59:59-05:00", None), #ISO 8601 with TZ (False, True, "2005-01-23 21:59:59", None), #ISO 8601 no TZ, assume local (False, True, "20050123T215959", None), #Short ISO with T (False, True, "20050123 215959", None), #Short ISO with space (True, True, "<01/23/05@21:59:59>", None), (False, True, "050123 21:59:59", None), # MySQL (True, True, "Jan-23-05 21:59:59", None), # ASSP like (False, True, "Jan 23, 2005 9:59:59 PM", None), # Apache Tomcat (True, True, "1106513999", None), # Regular epoch (True, True, "1106513999.000", None), # Regular epoch with millisec (True, True, "[1106513999.000]", "1106513999.000"), # epoch squared (brackets are not in match) (False, True, "audit(1106513999.000:987)", "1106513999.000"), # SELinux (True, True, "no date line", None), # no date in string ): if rdate is None and sdate != "no date line": rdate = sdate logSys.debug('== test %r', (anchored, bound, sdate, rdate)) for should_match, prefix in ( (rdate is not None, ""), (not anchored, "bogus-prefix "), (False, "word-boundary") ): log = prefix + sdate + "[sshd] error: PAM: Authentication failure" # if not allowed boundary test: if not bound and prefix == "word-boundary": continue logSys.debug(' -- test %-5s for %r', should_match, log) # with getTime: logtime = self.datedetector.getTime(log) if should_match: self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s by prefix %r, anchored: %r, log: %s" % ( sdate, prefix, anchored, log)) ( logUnix, logMatch ) = logtime self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s: by prefix %r \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix)) self.assertEqual(logMatch.group(1), rdate) else: self.assertEqual(logtime, None, "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) # with getTime(matchTime) - this combination used in filter: (timeMatch, template) = matchTime = self.datedetector.matchTime(log) logtime = self.datedetector.getTime(log, matchTime) logSys.debug(' -- found - %r', template.name if timeMatch else False) if should_match: self.assertNotEqual(logtime, None, "getTime retrieved nothing: failure for %s by prefix %r, anchored: %r, log: %s" % ( sdate, prefix, anchored, log)) ( logUnix, logMatch ) = logtime self.assertEqual(logUnix, dateUnix, "getTime comparison failure for %s by prefix %r: \"%s\" is not \"%s\"" % (sdate, prefix, logUnix, dateUnix)) self.assertEqual(logMatch.group(1), rdate) else: self.assertEqual(logtime, None, "getTime should have not matched for %r by prefix %r Got: %s" % (sdate, prefix, logtime)) logSys.debug(' -- OK') def testAllUniqueTemplateNames(self): self.assertRaises(ValueError, self.datedetector.appendTemplate, self.datedetector.templates[0]) def testFullYearMatch_gh130(self): # see https://github.com/fail2ban/fail2ban/pull/130 # yoh: unfortunately this test is not really effective to reproduce the # situation but left in place to assure consistent behavior mu = time.mktime(datetime.datetime(2012, 10, 11, 2, 37, 17).timetuple()) logdate = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') self.assertNotEqual(logdate, None) ( logTime, logMatch ) = logdate self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') # confuse it with year being at the end for i in range(10): ( logTime, logMatch ) = self.datedetector.getTime('11/10/2012 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '11/10/2012 02:37:17') # and now back to the original ( logTime, logMatch ) = self.datedetector.getTime('2012/10/11 02:37:17 [error] 18434#0') self.assertEqual(logTime, mu) self.assertEqual(logMatch.group(1), '2012/10/11 02:37:17') def testDateTemplate(self): t = DateTemplate() t.setRegex('^a{3,5}b?c*$') self.assertEqual(t.regex, '^(a{3,5}b?c*)$') self.assertRaises(Exception, t.getDate, '') self.assertEqual(t.matchDate('aaaac').group(1), 'aaaac') ## no word boundaries left and right: t = DatePatternRegex() t.pattern = '(?iu)**time:%ExY%Exm%ExdT%ExH%ExM%ExS**' # ** was removed from end-regex: self.assertFalse('**' in t.regex) # match date: dt = 'TIME:20050102T010203' self.assertEqual(t.matchDate('X' + dt + 'X').group(1), dt) self.assertEqual(t.matchDate(dt).group(1), dt) # wrong year (for exact %ExY): dt = 'TIME:50050102T010203' self.assertFalse(t.matchDate(dt)) ## start boundary left and word boundary right (automatically if not **): t = DatePatternRegex() t.pattern = '{^LN-BEG}time:%ExY%Exm%ExdT%ExH%ExM%ExS' self.assertTrue('^' in t.regex) # try match date: dt = 'time:20050102T010203' self.assertFalse(t.matchDate('X' + dt)) self.assertFalse(t.matchDate(dt + 'X')) self.assertEqual(t.matchDate('##' + dt + '...').group(1), dt) self.assertEqual(t.matchDate(dt).group(1), dt) # case sensitive: dt = 'TIME:20050102T010203' self.assertFalse(t.matchDate(dt)) ## auto-switching "ignore case" and "unicode" t = DatePatternRegex() t.pattern = '^%Y %b %d' self.assertTrue('(?iu)' in t.regex) dt = '2005 jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt) dt = '2005 Jun 03'; self.assertEqual(t.matchDate(dt).group(1), dt) dt = '2005 JUN 03'; self.assertEqual(t.matchDate(dt).group(1), dt) def testNotAnchoredCollision(self): # try for patterns with and without word boundaries: for dp in (r'%H:%M:%S', r'{UNB}%H:%M:%S'): dd = DateDetector() dd.appendTemplate(dp) # boundary of timestamp changes right and left (and time is left and right in line): for fmt in ('%s test', '%8s test', 'test %s', 'test %8s'): for dt in ( '00:01:02', '00:01:2', '00:1:2', '0:1:2', '00:1:2', '00:01:2', '00:01:02', '0:1:2', '00:01:02', ): t = dd.getTime(fmt % dt) self.assertEqual((t[0], t[1].group()), (1123970462.0, dt)) def testAmbiguousInOrderedTemplates(self): dd = self.datedetector for (debit, line, cnt) in ( # shortest distance to datetime should win: ("030324 0:03:59", "some free text 030324 0:03:59 -- 2003-03-07 17:05:01 ...", 1), # some free text with datetime: ("2003-03-07 17:05:01", "some free text 2003-03-07 17:05:01 test ...", 15), # distance collision detection (date from foreign input should not be found): ("030324 0:04:00", "server mysqld[1000]: 030324 0:04:00 [Warning] Access denied ..." " foreign-input just some free text 2003-03-07 17:05:01 test", 10), # distance collision detection (first date should be found): ("Sep 16 21:30:26", "server mysqld[1020]: Sep 16 21:30:26 server mysqld: 030916 21:30:26 [Warning] Access denied", 15), # just to test sorting: ("2005-10-07 06:09:42", "server mysqld[5906]: 2005-10-07 06:09:42 5907 [Warning] Access denied", 20), ("2005-10-08T15:26:18.237955", "server mysqld[5906]: 2005-10-08T15:26:18.237955 6 [Note] Access denied", 20), # date format changed again: ("051009 10:05:30", "server mysqld[1000]: 051009 10:05:30 [Warning] Access denied ...", 50), ): logSys.debug('== test: %r', (debit, line, cnt)) for i in range(cnt): logSys.debug('Line: %s', line) match, template = dd.matchTime(line) self.assertTrue(match) self.assertEqual(match.group(1), debit) def testLowLevelLogging(self): # test coverage for the deep (heavy) debug messages: try: self.__old_eff_level = datedetector.logLevel if datedetector.logLevel < logSys.getEffectiveLevel()+1: datedetector.logLevel = logSys.getEffectiveLevel()+1 dd = self.datedetector i = 0 for (line, cnt) in ( ("server mysqld[5906]: 2005-10-07 06:09:%02i 5907 [Warning] Access denied", 2), ("server mysqld[5906]: 051007 06:10:%02i 5907 [Warning] Access denied", 5), ("server mysqld[5906]: 2005-10-07 06:09:%02i 5907 [Warning] Access denied", 10), ): for i in range(i, i+cnt+1): logSys.debug('== test: %r', (line % i, cnt)) match, template = dd.matchTime(line % i) self.assertTrue(match) finally: datedetector.logLevel = self.__old_eff_level def testWrongTemplate(self): t = DatePatternRegex('(%ExY%Exm%Exd') # lazy compiling used, so try match: self.assertRaises(Exception, t.matchDate, '(20050101') self.assertLogged("Compile %r failed" % t.name) # abstract: t = DateTemplate() self.assertRaises(Exception, t.getDate, 'no date line') iso8601 = DatePatternRegex(r"%Y-%m-%d[T ]%H:%M:%S(?:\.%f)?%z") class CustomDateFormatsTest(unittest.TestCase): def setUp(self): """Call before every test case.""" unittest.TestCase.setUp(self) setUpMyTime() def tearDown(self): """Call after every test case.""" unittest.TestCase.tearDown(self) tearDownMyTime() def testIso8601(self): date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00Z")[0]) self.assertEqual( date, datetime.datetime(2007, 1, 25, 12, 0)) self.assertRaises(TypeError, iso8601.getDate, None) self.assertRaises(TypeError, iso8601.getDate, date) self.assertEqual(iso8601.getDate(""), None) self.assertEqual(iso8601.getDate("Z"), None) self.assertEqual(iso8601.getDate("2007-01-01T120:00:00Z"), None) self.assertEqual(iso8601.getDate("2007-13-01T12:00:00Z"), None) date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00+0400")[0]) self.assertEqual( date, datetime.datetime(2007, 1, 25, 8, 0)) date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00+04:00")[0]) self.assertEqual( date, datetime.datetime(2007, 1, 25, 8, 0)) date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00-0400")[0]) self.assertEqual( date, datetime.datetime(2007, 1, 25, 16, 0)) date = datetime.datetime.utcfromtimestamp( iso8601.getDate("2007-01-25T12:00:00-04")[0]) self.assertEqual( date, datetime.datetime(2007, 1, 25, 16, 0)) def testAmbiguousDatePattern(self): defDD = DateDetector() defDD.addDefaultTemplate() for (matched, dp, line) in ( # positive case: ('Jan 23 21:59:59', None, 'Test failure Jan 23 21:59:59 for 192.0.2.1'), # ambiguous "unbound" patterns (missed): (False, None, 'Test failure TestJan 23 21:59:59.011 2015 for 192.0.2.1'), (False, None, 'Test failure Jan 23 21:59:59123456789 for 192.0.2.1'), # ambiguous "no optional year" patterns (matched): ('Aug 8 11:25:50', None, 'Aug 8 11:25:50 20030f2329b8 Authentication failed from 192.0.2.1'), ('Aug 8 11:25:50', None, '[Aug 8 11:25:50] 20030f2329b8 Authentication failed from 192.0.2.1'), ('Aug 8 11:25:50 2014', None, 'Aug 8 11:25:50 2014 20030f2329b8 Authentication failed from 192.0.2.1'), # direct specified patterns: ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y$', '192.0.2.1 at 20:00:00 01.02.2003'), ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '192.0.2.1[20:00:00 01.02.2003]'), ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), ('[20:00:00 01.02.2003]', r'\[%H:%M:%S %d.%m.%Y\]$', '192.0.2.1[20:00:00 01.02.2003]'), ('[20:00:00 01.02.2003]', r'^\[%H:%M:%S %d.%m.%Y\]', '[20:00:00 01.02.2003]192.0.2.1'), ('[17/Jun/2011 17:00:45]', r'^\[%d/%b/%Y %H:%M:%S\]', '[17/Jun/2011 17:00:45] Attempt, IP address 192.0.2.1'), ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt [17/Jun/2011 17:00:45] IP address 192.0.2.1'), ('[17/Jun/2011 17:00:45]', r'\[%d/%b/%Y %H:%M:%S\]', 'Attempt IP address 192.0.2.1, date: [17/Jun/2011 17:00:45]'), # direct specified patterns (begin/end, missed): (False, r'%H:%M:%S %d.%m.%Y', '192.0.2.1x20:00:00 01.02.2003'), (False, r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003x192.0.2.1'), # direct specified unbound patterns (no begin/end boundary): ('20:00:00 01.02.2003', r'**%H:%M:%S %d.%m.%Y**', '192.0.2.1x20:00:00 01.02.2003'), ('20:00:00 01.02.2003', r'**%H:%M:%S %d.%m.%Y**', '20:00:00 01.02.2003x192.0.2.1'), # pattern enclosed with stars (in comparison to example above): ('*20:00:00 01.02.2003*', r'\**%H:%M:%S %d.%m.%Y\**', 'test*20:00:00 01.02.2003*test'), # direct specified patterns (begin/end, matched): ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '192.0.2.1 20:00:00 01.02.2003'), ('20:00:00 01.02.2003', r'%H:%M:%S %d.%m.%Y', '20:00:00 01.02.2003 192.0.2.1'), # wrong year in 1st date, so failed by convert using not precise year (filter used last known date), # in the 2nd and 3th tests (with precise year) it should find correct the 2nd date: (None, r'%Y-%Exm-%Exd %ExH:%ExM:%ExS', "0000-12-30 00:00:00 - 2003-12-30 00:00:00"), ('2003-12-30 00:00:00', r'%ExY-%Exm-%Exd %ExH:%ExM:%ExS', "0000-12-30 00:00:00 - 2003-12-30 00:00:00"), ('2003-12-30 00:00:00', None, "0000-12-30 00:00:00 - 2003-12-30 00:00:00"), # wrong date recognized short month/day (unbounded date pattern without separator between parts), # in the 2nd and 3th tests (with precise month and day) it should find correct the 2nd date: ('200333 010203', r'%Y%m%d %H%M%S', "text:200333 010203 | date:20031230 010203"), ('20031230 010203', r'%ExY%Exm%Exd %ExH%ExM%ExS', "text:200333 010203 | date:20031230 010203"), ('20031230 010203', None, "text:200333 010203 | date:20031230 010203"), # Explicit bound in start of the line using {^LN-BEG} key, # (negative) in the 1st case without line begin boundary - wrong date may be found, # (positive) in the 2nd case with line begin boundary - unexpected date / log line (not found) # (positive) and in 3th case with line begin boundary - find the correct date ("20030101 000000", "%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), (None, "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "00001230 010203 - 20030101 000000"), ("20031230 010203", "{^LN-BEG}%ExY%Exm%Exd %ExH%ExM%ExS", "20031230 010203 - 20030101 000000"), # Explicit bound in start of the line using {^LN-BEG} key, # up to 2 non-alphanumeric chars front, ** - no word boundary on the right ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "2003123001020320030101000000"), ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "#2003123001020320030101000000"), ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS**", "##2003123001020320030101000000"), ("20031230010203", "{^LN-BEG}%ExY%Exm%Exd%ExH%ExM%ExS", "[20031230010203]20030101000000"), # UTC/GMT time zone offset (with %z and %Z): (1072746123.0 - 3600, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03] server ..."), (1072746123.0 - 3600, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 UTC] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03 UTC] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 Z] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %z)?", "[2003-12-30 01:02:03 +0000] server ..."), (1072746123.0, "{^LN-BEG}%ExY-%Exm-%Exd %ExH:%ExM:%ExS(?: %Z)?", "[2003-12-30 01:02:03 Z] server ..."), ): logSys.debug('== test: %r', (matched, dp, line)) if dp is None: dd = defDD else: dd = DateDetector() dd.appendTemplate(dp) date = dd.getTime(line) if matched: self.assertTrue(date) if isinstance(matched, str): self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(matched, date[0]) else: self.assertEqual(date, None) def testVariousFormatSpecs(self): for (matched, dp, line) in ( # cover %B (full-month-name) and %I (as 12 == 0): (1106438399.0, "^%B %Exd %I:%ExM:%ExS**", 'January 23 12:59:59'), # cover %U (week of year starts on sunday) and %A (weekday): (985208399.0, "^%y %U %A %ExH:%ExM:%ExS**", '01 11 Wednesday 21:59:59'), # cover %W (week of year starts on monday) and %A (weekday): (984603599.0, "^%y %W %A %ExH:%ExM:%ExS**", '01 11 Wednesday 21:59:59'), # cover %W (week of year starts on monday) and %w (weekday, 0 - sunday): (984949199.0, "^%y %W %w %ExH:%ExM:%ExS**", '01 11 0 21:59:59'), # cover %W (week of year starts on monday) and %w (weekday, 6 - saturday): (984862799.0, "^%y %W %w %ExH:%ExM:%ExS**", '01 11 6 21:59:59'), # cover time only, current date, in test cases now == 14 Aug 2005 12:00 -> back to yesterday (13 Aug): (1123963199.0, "^%ExH:%ExM:%ExS**", '21:59:59'), # cover time only, current date, in test cases now == 14 Aug 2005 12:00 -> today (14 Aug): (1123970401.0, "^%ExH:%ExM:%ExS**", '00:00:01'), # cover date with current year, in test cases now == Aug 2005 -> back to last year (Sep 2004): (1094068799.0, "^%m/%d %ExH:%ExM:%ExS**", '09/01 21:59:59'), # no time (only date) in pattern, assume local 00:00:00 for H:M:S : (1093989600.0, "^%Y-%m-%d**", '2004-09-01'), (1093996800.0, "^%Y-%m-%d%z**", '2004-09-01Z'), ): logSys.debug('== test: %r', (matched, dp, line)) dd = DateDetector() dd.appendTemplate(dp) date = dd.getTime(line) if matched: self.assertTrue(date) if isinstance(matched, str): # pragma: no cover self.assertEqual(matched, date[1].group(1)) else: self.assertEqual(matched, date[0]) else: # pragma: no cover self.assertEqual(date, None) # def testDefaultTempate(self): # self.__datedetector.setDefaultRegex("^\S{3}\s{1,2}\d{1,2} \d{2}:\d{2}:\d{2}") # self.__datedetector.setDefaultPattern("%b %d %H:%M:%S") # # log = "Jan 23 21:59:59 [sshd] error: PAM: Authentication failure" # date = [2005, 1, 23, 21, 59, 59, 1, 23, -1] # dateUnix = 1106513999.0 # # self.assertEqual(self.__datedetector.getTime(log), date) # self.assertEqual(self.__datedetector.getUnixTime(log), dateUnix)