Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import lzma 

2import gzip 

3import bz2 

4from datetime import datetime 

5 

6 

7class Recorder(object): 

8 """Support events recording to rotating compressed files. 

9  

10 By default lzma compression is selected (a.k.a. 'xz' files) 

11  

12 The default time_fmt used as timestamp is '%Y-%m-%d %H:%M:%S.%f' 

13 allowing microseconds precission 

14  

15 The default grouping criteria is records that share the same day. 

16 This is made by rotate_by = '%Y%m%d', so all records that share the 

17 same day when are saved belongs to the same record file. 

18  

19 If you want to record some keys in the same file just use ``aliases`` 

20 parameter to convert keys before duming record. 

21  

22 """ 

23 INFO = { 

24 'xz': lzma, 

25 'gz': gzip, 

26 'bz2': bz2, 

27 } 

28 

29 

30 def __init__(self, compression='xz', 

31 time_fmt='%Y-%m-%d %H:%M:%S.%f', 

32 rotate_by='%Y%m%d', 

33 aliases=None, *args, **kw): 

34 self.compression = compression 

35 self.time_fmt = time_fmt 

36 self.rotate_by = rotate_by 

37 self.aliases = aliases or dict() 

38 

39 self._output = dict() 

40 self._last_flush = dict() 

41 

42 def record(self, key, *event): 

43 # write market data file 

44 t = datetime.utcnow() 

45 now = t.strftime(self.time_fmt) 

46 today = t.strftime(self.rotate_by) 

47 outputs = self._output.get(key) 

48 if not outputs: 

49 outputs = self._output[key] = dict() 

50 

51 output = outputs.get(today) 

52 if not output: 

53 # rotate dump file 

54 for k in list(outputs.keys()): 

55 output.pop(k).close() 

56 

57 # set the new one 

58 output = outputs[today] = self.INFO[self.compression].\ 

59 open(f"{key}.{today}.{self.compression}", 'a') 

60 

61 line = [str(k) for k in event] 

62 line = '.'.join(line) + '\n' 

63 output.write(bytes(line, 'utf-8')) 

64 

65 last_flush = self._last_flush.setdefault(key, t) 

66 if (t - last_flush).seconds > 120: 

67 output._fp.flush() 

68 output.close() # will be reopen next write attempt 

69 outputs.pop(today) 

70 self._last_flush[key] = t 

71 

72 def close(self): 

73 for _, keys in self._output.items(): 

74 for k, f in keys.items(): 

75 print(f"Closing dump file: {f._fp.name}") 

76 f._fp.flush() 

77 f.close() 

78