Coverage for /home/ubuntu/Documents/Research/mut_p6/sacred/sacred/metrics_logger.py: 93%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2# coding=utf-8
3import datetime
4import sacred.optional as opt
6from queue import Queue, Empty
9class MetricsLogger:
10 """MetricsLogger collects metrics measured during experiments.
12 MetricsLogger is the (only) part of the Metrics API.
13 An instance of the class should be created for the Run class, such that the
14 log_scalar_metric method is accessible from running experiments using
15 _run.metrics.log_scalar_metric.
16 """
18 def __init__(self):
19 # Create a message queue that remembers
20 # calls of the log_scalar_metric
21 self._logged_metrics = Queue()
22 self._metric_step_counter = {}
23 """Remembers the last number of each metric."""
25 def log_scalar_metric(self, metric_name, value, step=None):
26 """
27 Add a new measurement.
29 The measurement will be processed by the MongoDB observer
30 during a heartbeat event.
31 Other observers are not yet supported.
33 :param metric_name: The name of the metric, e.g. training.loss.
34 :param value: The measured value.
35 :param step: The step number (integer), e.g. the iteration number
36 If not specified, an internal counter for each metric
37 is used, incremented by one.
38 """
39 if opt.has_numpy:
40 np = opt.np
41 if isinstance(value, np.generic):
42 value = value.item()
43 if isinstance(step, np.generic):
44 step = step.item()
45 if step is None:
46 step = self._metric_step_counter.get(metric_name, -1) + 1
47 self._logged_metrics.put(
48 ScalarMetricLogEntry(metric_name, step, datetime.datetime.utcnow(), value)
49 )
50 self._metric_step_counter[metric_name] = step
52 def get_last_metrics(self):
53 """Read all measurement events since last call of the method.
55 :return List[ScalarMetricLogEntry]
56 """
57 read_up_to = self._logged_metrics.qsize()
58 messages = []
59 for i in range(read_up_to):
60 try:
61 messages.append(self._logged_metrics.get_nowait())
62 except Empty:
63 pass
64 return messages
67class ScalarMetricLogEntry:
68 """Container for measurements of scalar metrics.
70 There is exactly one ScalarMetricLogEntry per logged scalar metric value.
71 """
73 def __init__(self, name, step, timestamp, value):
74 self.name = name
75 self.step = step
76 self.timestamp = timestamp
77 self.value = value
80def linearize_metrics(logged_metrics):
81 """
82 Group metrics by name.
84 Takes a list of individual measurements, possibly belonging
85 to different metrics and groups them by name.
87 :param logged_metrics: A list of ScalarMetricLogEntries
88 :return: Measured values grouped by the metric name:
89 {"metric_name1": {"steps": [0,1,2], "values": [4, 5, 6],
90 "timestamps": [datetime, datetime, datetime]},
91 "metric_name2": {...}}
92 """
93 metrics_by_name = {}
94 for metric_entry in logged_metrics:
95 if metric_entry.name not in metrics_by_name:
96 metrics_by_name[metric_entry.name] = {
97 "steps": [],
98 "values": [],
99 "timestamps": [],
100 "name": metric_entry.name,
101 }
102 metrics_by_name[metric_entry.name]["steps"].append(metric_entry.step)
103 metrics_by_name[metric_entry.name]["values"].append(metric_entry.value)
104 metrics_by_name[metric_entry.name]["timestamps"].append(metric_entry.timestamp)
105 return metrics_by_name