Coverage for /home/ubuntu/Documents/Research/mut_p6/sacred/sacred/observers/queue.py: 33%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from collections import namedtuple
2from queue import Queue
3from sacred.observers.base import RunObserver
4from sacred.utils import IntervalTimer
5import traceback
6import logging
8logger = logging.getLogger(__name__)
10WrappedEvent = namedtuple("WrappedEvent", "name args kwargs")
13class QueueObserver(RunObserver):
14 """Wraps any observer and puts processing of events in the background.
16 If the covered observer fails to process an event, the queue observer
17 will retry until it works. This is useful for observers that rely on
18 external services like databases that might become temporarily
19 unavailable.
20 """
22 def __init__(
23 self,
24 covered_observer: RunObserver,
25 interval: float = 20.0,
26 retry_interval: float = 10.0,
27 ):
28 """Initialize QueueObserver.
30 Parameters
31 ----------
32 covered_observer
33 The real observer that is being wrapped.
34 interval
35 The interval in seconds at which the background thread is woken up to process new events.
36 retry_interval
37 The interval in seconds to wait if an event failed to be processed.
38 """
39 self._covered_observer = covered_observer
40 self._retry_interval = retry_interval
41 self._interval = interval
42 self._queue = None
43 self._worker = None
44 self._stop_worker_event = None
45 logger.debug("just testing")
47 def queued_event(self, *args, **kwargs):
48 self._queue.put(WrappedEvent("queued_event", args, kwargs))
50 def started_event(self, *args, **kwargs):
51 self._queue = Queue()
52 self._stop_worker_event, self._worker = IntervalTimer.create(
53 self._run, interval=self._interval
54 )
55 self._worker.start()
57 # Putting the started event on the queue makes no sense
58 # as it is required for initialization of the covered observer.
59 return self._covered_observer.started_event(*args, **kwargs)
61 def heartbeat_event(self, *args, **kwargs):
62 self._queue.put(WrappedEvent("heartbeat_event", args, kwargs))
64 def completed_event(self, *args, **kwargs):
65 self._queue.put(WrappedEvent("completed_event", args, kwargs))
66 self.join()
68 def interrupted_event(self, *args, **kwargs):
69 self._queue.put(WrappedEvent("interrupted_event", args, kwargs))
70 self.join()
72 def failed_event(self, *args, **kwargs):
73 self._queue.put(WrappedEvent("failed_event", args, kwargs))
74 self.join()
76 def resource_event(self, *args, **kwargs):
77 self._queue.put(WrappedEvent("resource_event", args, kwargs))
79 def artifact_event(self, *args, **kwargs):
80 self._queue.put(WrappedEvent("artifact_event", args, kwargs))
82 def log_metrics(self, metrics_by_name, info):
83 for metric_name, metric_values in metrics_by_name.items():
84 self._queue.put(
85 WrappedEvent("log_metrics", [metric_name, metric_values, info], {})
86 )
88 def _run(self):
89 """Empty the queue every interval."""
90 while not self._queue.empty():
91 try:
92 event = self._queue.get()
93 except IndexError:
94 # Currently there is no event on the queue so
95 # just go back to sleep.
96 pass
97 else:
98 try:
99 method = getattr(self._covered_observer, event.name)
100 except NameError:
101 # The covered observer does not implement an event handler
102 # for the event, so just discard the message.
103 self._queue.task_done()
104 else:
105 while True:
106 try:
107 method(*event.args, **event.kwargs)
108 except:
109 # Something went wrong during the processing of
110 # the event so wait for some time and
111 # then try again.
112 logger.debug(
113 "Error while processing event. Trying again.\n{}".format(
114 traceback.format_exc()
115 )
116 )
117 # logging.debug(f"""Error while processing event. Trying again.
118 # {traceback.format_exc()}""")
120 self._stop_worker_event.wait(self._retry_interval)
121 continue
122 else:
123 self._queue.task_done()
124 break
126 def join(self):
127 if self._queue is not None:
128 self._queue.join()
129 self._stop_worker_event.set()
130 self._worker.join(timeout=10)
132 def __getattr__(self, item):
133 return getattr(self._covered_observer, item)
135 def __eq__(self, other):
136 return self._covered_observer == other