Coverage for sacred/sacred/metrics_logger.py: 23%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

43 statements  

1#!/usr/bin/env python 

2# coding=utf-8 

3import datetime 

4import sacred.optional as opt 

5 

6from queue import Queue, Empty 

7 

8 

9class MetricsLogger: 

10 """MetricsLogger collects metrics measured during experiments. 

11 

12 MetricsLogger is the (only) part of the Metrics API. 

13 An instance of the class should be created for the Run class, such that the 

14 log_scalar_metric method is accessible from running experiments using 

15 _run.metrics.log_scalar_metric. 

16 """ 

17 

18 def __init__(self): 

19 # Create a message queue that remembers 

20 # calls of the log_scalar_metric 

21 self._logged_metrics = Queue() 

22 self._metric_step_counter = {} 

23 """Remembers the last number of each metric.""" 

24 

25 def log_scalar_metric(self, metric_name, value, step=None): 

26 """ 

27 Add a new measurement. 

28 

29 The measurement will be processed by the MongoDB observer 

30 during a heartbeat event. 

31 Other observers are not yet supported. 

32 

33 :param metric_name: The name of the metric, e.g. training.loss. 

34 :param value: The measured value. 

35 :param step: The step number (integer), e.g. the iteration number 

36 If not specified, an internal counter for each metric 

37 is used, incremented by one. 

38 """ 

39 if opt.has_numpy: 

40 np = opt.np 

41 if isinstance(value, np.generic): 

42 value = value.item() 

43 if isinstance(step, np.generic): 

44 step = step.item() 

45 if step is None: 

46 step = self._metric_step_counter.get(metric_name, -1) + 1 

47 self._logged_metrics.put( 

48 ScalarMetricLogEntry(metric_name, step, datetime.datetime.utcnow(), value) 

49 ) 

50 self._metric_step_counter[metric_name] = step 

51 

52 def get_last_metrics(self): 

53 """Read all measurement events since last call of the method. 

54 

55 :return List[ScalarMetricLogEntry] 

56 """ 

57 read_up_to = self._logged_metrics.qsize() 

58 messages = [] 

59 for i in range(read_up_to): 

60 try: 

61 messages.append(self._logged_metrics.get_nowait()) 

62 except Empty: 

63 pass 

64 return messages 

65 

66 

67class ScalarMetricLogEntry: 

68 """Container for measurements of scalar metrics. 

69 

70 There is exactly one ScalarMetricLogEntry per logged scalar metric value. 

71 """ 

72 

73 def __init__(self, name, step, timestamp, value): 

74 self.name = name 

75 self.step = step 

76 self.timestamp = timestamp 

77 self.value = value 

78 

79 

80def linearize_metrics(logged_metrics): 

81 """ 

82 Group metrics by name. 

83 

84 Takes a list of individual measurements, possibly belonging 

85 to different metrics and groups them by name. 

86 

87 :param logged_metrics: A list of ScalarMetricLogEntries 

88 :return: Measured values grouped by the metric name: 

89 {"metric_name1": {"steps": [0,1,2], "values": [4, 5, 6], 

90 "timestamps": [datetime, datetime, datetime]}, 

91 "metric_name2": {...}} 

92 """ 

93 metrics_by_name = {} 

94 for metric_entry in logged_metrics: 

95 if metric_entry.name not in metrics_by_name: 

96 metrics_by_name[metric_entry.name] = { 

97 "steps": [], 

98 "values": [], 

99 "timestamps": [], 

100 "name": metric_entry.name, 

101 } 

102 metrics_by_name[metric_entry.name]["steps"].append(metric_entry.step) 

103 metrics_by_name[metric_entry.name]["values"].append(metric_entry.value) 

104 metrics_by_name[metric_entry.name]["timestamps"].append(metric_entry.timestamp) 

105 return metrics_by_name