Coverage for /home/ubuntu/Documents/Research/mut_p6/sacred/sacred/observers/sql.py: 29%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

92 statements  

1#!/usr/bin/env python 

2# coding=utf-8 

3 

4import json 

5from threading import Lock 

6import warnings 

7 

8from sacred.commandline_options import cli_option 

9from sacred.observers.base import RunObserver 

10from sacred.serializer import flatten 

11 

12DEFAULT_SQL_PRIORITY = 40 

13 

14 

15# ############################# Observer #################################### # 

16 

17 

18class SqlObserver(RunObserver): 

19 @classmethod 

20 def create(cls, url, echo=False, priority=DEFAULT_SQL_PRIORITY): 

21 warnings.warn( 

22 "SqlObserver.create(...) is deprecated. Please use" 

23 " SqlObserver(...) instead.", 

24 DeprecationWarning, 

25 ) 

26 return cls(url, echo, priority) 

27 

28 def __init__(self, url, echo=False, priority=DEFAULT_SQL_PRIORITY): 

29 from sqlalchemy.orm import sessionmaker, scoped_session 

30 import sqlalchemy as sa 

31 

32 engine = sa.create_engine(url, echo=echo) 

33 session_factory = sessionmaker(bind=engine) 

34 # make session thread-local to avoid problems with sqlite (see #275) 

35 session = scoped_session(session_factory) 

36 self.engine = engine 

37 self.session = session 

38 self.priority = priority 

39 self.run = None 

40 self.lock = Lock() 

41 

42 @classmethod 

43 def create_from(cls, engine, session, priority=DEFAULT_SQL_PRIORITY): 

44 """Instantiate a SqlObserver with an existing engine and session.""" 

45 self = cls.__new__(cls) # skip __init__ call 

46 self.engine = engine 

47 self.session = session 

48 self.priority = priority 

49 self.run = None 

50 self.lock = Lock() 

51 return self 

52 

53 def started_event( 

54 self, ex_info, command, host_info, start_time, config, meta_info, _id 

55 ): 

56 return self._add_event( 

57 ex_info, 

58 command, 

59 host_info, 

60 config, 

61 meta_info, 

62 _id, 

63 "RUNNING", 

64 start_time=start_time, 

65 ) 

66 

67 def queued_event( 

68 self, ex_info, command, host_info, queue_time, config, meta_info, _id 

69 ): 

70 return self._add_event( 

71 ex_info, command, host_info, config, meta_info, _id, "QUEUED" 

72 ) 

73 

74 def _add_event( 

75 self, ex_info, command, host_info, config, meta_info, _id, status, **kwargs 

76 ): 

77 from .sql_bases import Base, Experiment, Host, Run 

78 

79 Base.metadata.create_all(self.engine) 

80 sql_exp = Experiment.get_or_create(ex_info, self.session) 

81 sql_host = Host.get_or_create(host_info, self.session) 

82 if _id is None: 

83 i = self.session.query(Run).order_by(Run.id.desc()).first() 

84 _id = 0 if i is None else i.id + 1 

85 

86 self.run = Run( 

87 run_id=str(_id), 

88 config=json.dumps(flatten(config)), 

89 command=command, 

90 priority=meta_info.get("priority", 0), 

91 comment=meta_info.get("comment", ""), 

92 experiment=sql_exp, 

93 host=sql_host, 

94 status=status, 

95 **kwargs, 

96 ) 

97 self.session.add(self.run) 

98 self.save() 

99 return _id or self.run.run_id 

100 

101 def heartbeat_event(self, info, captured_out, beat_time, result): 

102 self.run.info = json.dumps(flatten(info)) 

103 self.run.captured_out = captured_out 

104 self.run.heartbeat = beat_time 

105 self.run.result = result 

106 self.save() 

107 

108 def completed_event(self, stop_time, result): 

109 self.run.stop_time = stop_time 

110 self.run.result = result 

111 self.run.status = "COMPLETED" 

112 self.save() 

113 

114 def interrupted_event(self, interrupt_time, status): 

115 self.run.stop_time = interrupt_time 

116 self.run.status = status 

117 self.save() 

118 

119 def failed_event(self, fail_time, fail_trace): 

120 self.run.stop_time = fail_time 

121 self.run.fail_trace = "\n".join(fail_trace) 

122 self.run.status = "FAILED" 

123 self.save() 

124 

125 def resource_event(self, filename): 

126 from .sql_bases import Resource 

127 

128 res = Resource.get_or_create(filename, self.session) 

129 self.run.resources.append(res) 

130 self.save() 

131 

132 def artifact_event(self, name, filename, metadata=None, content_type=None): 

133 from .sql_bases import Artifact 

134 

135 a = Artifact.create(name, filename) 

136 self.run.artifacts.append(a) 

137 self.save() 

138 

139 def save(self): 

140 with self.lock: 

141 self.session.commit() 

142 

143 def query(self, _id): 

144 from .sql_bases import Run 

145 

146 run = self.session.query(Run).filter_by(id=_id).first() 

147 return run.to_json() 

148 

149 def __eq__(self, other): 

150 if isinstance(other, SqlObserver): 

151 # fixme: this will probably fail to detect two equivalent engines 

152 return self.engine == other.engine and self.session == other.session 

153 return False 

154 

155 

156# ######################## Commandline Option ############################### # 

157 

158 

159@cli_option("-s", "--sql") 

160def sql_option(args, run): 

161 """Add a SQL Observer to the experiment. 

162 

163 The typical form is: dialect://username:password@host:port/database 

164 """ 

165 run.observers.append(SqlObserver(args))