Coverage for /home/ubuntu/Documents/Research/mut_p6/sacred/sacred/observers/sql.py: 29%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python
2# coding=utf-8
4import json
5from threading import Lock
6import warnings
8from sacred.commandline_options import cli_option
9from sacred.observers.base import RunObserver
10from sacred.serializer import flatten
12DEFAULT_SQL_PRIORITY = 40
15# ############################# Observer #################################### #
18class SqlObserver(RunObserver):
19 @classmethod
20 def create(cls, url, echo=False, priority=DEFAULT_SQL_PRIORITY):
21 warnings.warn(
22 "SqlObserver.create(...) is deprecated. Please use"
23 " SqlObserver(...) instead.",
24 DeprecationWarning,
25 )
26 return cls(url, echo, priority)
28 def __init__(self, url, echo=False, priority=DEFAULT_SQL_PRIORITY):
29 from sqlalchemy.orm import sessionmaker, scoped_session
30 import sqlalchemy as sa
32 engine = sa.create_engine(url, echo=echo)
33 session_factory = sessionmaker(bind=engine)
34 # make session thread-local to avoid problems with sqlite (see #275)
35 session = scoped_session(session_factory)
36 self.engine = engine
37 self.session = session
38 self.priority = priority
39 self.run = None
40 self.lock = Lock()
42 @classmethod
43 def create_from(cls, engine, session, priority=DEFAULT_SQL_PRIORITY):
44 """Instantiate a SqlObserver with an existing engine and session."""
45 self = cls.__new__(cls) # skip __init__ call
46 self.engine = engine
47 self.session = session
48 self.priority = priority
49 self.run = None
50 self.lock = Lock()
51 return self
53 def started_event(
54 self, ex_info, command, host_info, start_time, config, meta_info, _id
55 ):
56 return self._add_event(
57 ex_info,
58 command,
59 host_info,
60 config,
61 meta_info,
62 _id,
63 "RUNNING",
64 start_time=start_time,
65 )
67 def queued_event(
68 self, ex_info, command, host_info, queue_time, config, meta_info, _id
69 ):
70 return self._add_event(
71 ex_info, command, host_info, config, meta_info, _id, "QUEUED"
72 )
74 def _add_event(
75 self, ex_info, command, host_info, config, meta_info, _id, status, **kwargs
76 ):
77 from .sql_bases import Base, Experiment, Host, Run
79 Base.metadata.create_all(self.engine)
80 sql_exp = Experiment.get_or_create(ex_info, self.session)
81 sql_host = Host.get_or_create(host_info, self.session)
82 if _id is None:
83 i = self.session.query(Run).order_by(Run.id.desc()).first()
84 _id = 0 if i is None else i.id + 1
86 self.run = Run(
87 run_id=str(_id),
88 config=json.dumps(flatten(config)),
89 command=command,
90 priority=meta_info.get("priority", 0),
91 comment=meta_info.get("comment", ""),
92 experiment=sql_exp,
93 host=sql_host,
94 status=status,
95 **kwargs,
96 )
97 self.session.add(self.run)
98 self.save()
99 return _id or self.run.run_id
101 def heartbeat_event(self, info, captured_out, beat_time, result):
102 self.run.info = json.dumps(flatten(info))
103 self.run.captured_out = captured_out
104 self.run.heartbeat = beat_time
105 self.run.result = result
106 self.save()
108 def completed_event(self, stop_time, result):
109 self.run.stop_time = stop_time
110 self.run.result = result
111 self.run.status = "COMPLETED"
112 self.save()
114 def interrupted_event(self, interrupt_time, status):
115 self.run.stop_time = interrupt_time
116 self.run.status = status
117 self.save()
119 def failed_event(self, fail_time, fail_trace):
120 self.run.stop_time = fail_time
121 self.run.fail_trace = "\n".join(fail_trace)
122 self.run.status = "FAILED"
123 self.save()
125 def resource_event(self, filename):
126 from .sql_bases import Resource
128 res = Resource.get_or_create(filename, self.session)
129 self.run.resources.append(res)
130 self.save()
132 def artifact_event(self, name, filename, metadata=None, content_type=None):
133 from .sql_bases import Artifact
135 a = Artifact.create(name, filename)
136 self.run.artifacts.append(a)
137 self.save()
139 def save(self):
140 with self.lock:
141 self.session.commit()
143 def query(self, _id):
144 from .sql_bases import Run
146 run = self.session.query(Run).filter_by(id=_id).first()
147 return run.to_json()
149 def __eq__(self, other):
150 if isinstance(other, SqlObserver):
151 # fixme: this will probably fail to detect two equivalent engines
152 return self.engine == other.engine and self.session == other.session
153 return False
156# ######################## Commandline Option ############################### #
159@cli_option("-s", "--sql")
160def sql_option(args, run):
161 """Add a SQL Observer to the experiment.
163 The typical form is: dialect://username:password@host:port/database
164 """
165 run.observers.append(SqlObserver(args))