Coverage for sacred/sacred/observers/queue.py: 84%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

69 statements  

1from collections import namedtuple 

2from queue import Queue 

3from sacred.observers.base import RunObserver 

4from sacred.utils import IntervalTimer 

5import traceback 

6import logging 

7 

8logger = logging.getLogger(__name__) 

9 

10WrappedEvent = namedtuple("WrappedEvent", "name args kwargs") 

11 

12 

13class QueueObserver(RunObserver): 

14 """Wraps any observer and puts processing of events in the background. 

15 

16 If the covered observer fails to process an event, the queue observer 

17 will retry until it works. This is useful for observers that rely on 

18 external services like databases that might become temporarily 

19 unavailable. 

20 """ 

21 

22 def __init__( 

23 self, 

24 covered_observer: RunObserver, 

25 interval: float = 20.0, 

26 retry_interval: float = 10.0, 

27 ): 

28 """Initialize QueueObserver. 

29 

30 Parameters 

31 ---------- 

32 covered_observer 

33 The real observer that is being wrapped. 

34 interval 

35 The interval in seconds at which the background thread is woken up to process new events. 

36 retry_interval 

37 The interval in seconds to wait if an event failed to be processed. 

38 """ 

39 self._covered_observer = covered_observer 

40 self._retry_interval = retry_interval 

41 self._interval = interval 

42 self._queue = None 

43 self._worker = None 

44 self._stop_worker_event = None 

45 logger.debug("just testing") 

46 

47 def queued_event(self, *args, **kwargs): 

48 self._queue.put(WrappedEvent("queued_event", args, kwargs)) 

49 

50 def started_event(self, *args, **kwargs): 

51 self._queue = Queue() 

52 self._stop_worker_event, self._worker = IntervalTimer.create( 

53 self._run, interval=self._interval 

54 ) 

55 self._worker.start() 

56 

57 # Putting the started event on the queue makes no sense 

58 # as it is required for initialization of the covered observer. 

59 return self._covered_observer.started_event(*args, **kwargs) 

60 

61 def heartbeat_event(self, *args, **kwargs): 

62 self._queue.put(WrappedEvent("heartbeat_event", args, kwargs)) 

63 

64 def completed_event(self, *args, **kwargs): 

65 self._queue.put(WrappedEvent("completed_event", args, kwargs)) 

66 self.join() 

67 

68 def interrupted_event(self, *args, **kwargs): 

69 self._queue.put(WrappedEvent("interrupted_event", args, kwargs)) 

70 self.join() 

71 

72 def failed_event(self, *args, **kwargs): 

73 self._queue.put(WrappedEvent("failed_event", args, kwargs)) 

74 self.join() 

75 

76 def resource_event(self, *args, **kwargs): 

77 self._queue.put(WrappedEvent("resource_event", args, kwargs)) 

78 

79 def artifact_event(self, *args, **kwargs): 

80 self._queue.put(WrappedEvent("artifact_event", args, kwargs)) 

81 

82 def log_metrics(self, metrics_by_name, info): 

83 for metric_name, metric_values in metrics_by_name.items(): 

84 self._queue.put( 

85 WrappedEvent("log_metrics", [metric_name, metric_values, info], {}) 

86 ) 

87 

88 def _run(self): 

89 """Empty the queue every interval.""" 

90 while not self._queue.empty(): 

91 try: 

92 event = self._queue.get() 

93 except IndexError: 

94 # Currently there is no event on the queue so 

95 # just go back to sleep. 

96 pass 

97 else: 

98 try: 

99 method = getattr(self._covered_observer, event.name) 

100 except NameError: 

101 # The covered observer does not implement an event handler 

102 # for the event, so just discard the message. 

103 self._queue.task_done() 

104 else: 

105 while True: 

106 try: 

107 method(*event.args, **event.kwargs) 

108 except: 

109 # Something went wrong during the processing of 

110 # the event so wait for some time and 

111 # then try again. 

112 logger.debug( 

113 "Error while processing event. Trying again.\n{}".format( 

114 traceback.format_exc() 

115 ) 

116 ) 

117 # logging.debug(f"""Error while processing event. Trying again. 

118 # {traceback.format_exc()}""") 

119 

120 self._stop_worker_event.wait(self._retry_interval) 

121 continue 

122 else: 

123 self._queue.task_done() 

124 break 

125 

126 def join(self): 

127 if self._queue is not None: 

128 self._queue.join() 

129 self._stop_worker_event.set() 

130 self._worker.join(timeout=10) 

131 

132 def __getattr__(self, item): 

133 return getattr(self._covered_observer, item) 

134 

135 def __eq__(self, other): 

136 return self._covered_observer == other