Coverage for sacred/sacred/config/config_scope.py: 16%

Shortcuts on this page

r m x   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

146 statements  

1#!/usr/bin/env python 

2# coding=utf-8 

3 

4import ast 

5import inspect 

6import io 

7import re 

8from tokenize import tokenize, TokenError, COMMENT 

9from copy import copy 

10 

11from sacred import SETTINGS 

12from sacred.config.config_summary import ConfigSummary 

13from sacred.config.utils import dogmatize, normalize_or_die, recursive_fill_in 

14from sacred.config.signature import get_argspec 

15from sacred.utils import ConfigError 

16 

17 

18class ConfigScope: 

19 def __init__(self, func): 

20 self.args, vararg_name, kw_wildcard, _, kwargs = get_argspec(func) 

21 assert vararg_name is None, "*args not allowed for ConfigScope functions" 

22 assert kw_wildcard is None, "**kwargs not allowed for ConfigScope functions" 

23 assert not kwargs, "default values are not allowed for ConfigScope functions" 

24 

25 self._func = func 

26 self._body_code = get_function_body_code(func) 

27 self._var_docs = get_config_comments(func) 

28 self.__doc__ = self._func.__doc__ 

29 

30 def __call__(self, fixed=None, preset=None, fallback=None): 

31 """ 

32 Evaluate this ConfigScope. 

33 

34 This will evaluate the function body and fill the relevant local 

35 variables into entries into keys in this dictionary. 

36 

37 :param fixed: Dictionary of entries that should stay fixed during the 

38 evaluation. All of them will be part of the final config. 

39 :type fixed: dict 

40 :param preset: Dictionary of preset values that will be available 

41 during the evaluation (if they are declared in the 

42 function argument list). All of them will be part of the 

43 final config. 

44 :type preset: dict 

45 :param fallback: Dictionary of fallback values that will be available 

46 during the evaluation (if they are declared in the 

47 function argument list). They will NOT be part of the 

48 final config. 

49 :type fallback: dict 

50 :return: self 

51 :rtype: ConfigScope 

52 """ 

53 cfg_locals = dogmatize(fixed or {}) 

54 fallback = fallback or {} 

55 preset = preset or {} 

56 fallback_view = {} 

57 

58 available_entries = set(preset.keys()) | set(fallback.keys()) 

59 

60 for arg in self.args: 

61 if arg not in available_entries: 

62 raise KeyError( 

63 "'{}' not in preset for ConfigScope. " 

64 "Available options are: {}".format(arg, available_entries) 

65 ) 

66 if arg in preset: 

67 cfg_locals[arg] = preset[arg] 

68 else: # arg in fallback 

69 fallback_view[arg] = fallback[arg] 

70 

71 cfg_locals.fallback = fallback_view 

72 

73 with ConfigError.track(cfg_locals): 

74 eval(self._body_code, copy(self._func.__globals__), cfg_locals) 

75 

76 added = cfg_locals.revelation() 

77 config_summary = ConfigSummary( 

78 added, 

79 cfg_locals.modified, 

80 cfg_locals.typechanges, 

81 cfg_locals.fallback_writes, 

82 docs=self._var_docs, 

83 ) 

84 # fill in the unused presets 

85 recursive_fill_in(cfg_locals, preset) 

86 

87 for key, value in cfg_locals.items(): 

88 try: 

89 config_summary[key] = normalize_or_die(value) 

90 except ValueError: 

91 pass 

92 return config_summary 

93 

94 

95def get_function_body(func): 

96 func_code_lines, start_idx = inspect.getsourcelines(func) 

97 func_code = "".join(func_code_lines) 

98 arg = "(?:[a-zA-Z_][a-zA-Z0-9_]*)" 

99 arguments = r"{0}(?:\s*,\s*{0})*,?".format(arg) 

100 func_def = re.compile( 

101 r"^[ \t]*def[ \t]*{}[ \t]*\(\s*({})?\s*\)[ \t]*:[ \t]*(?:#[^\n]*)?\n".format( 

102 func.__name__, arguments 

103 ), 

104 flags=re.MULTILINE, 

105 ) 

106 defs = list(re.finditer(func_def, func_code)) 

107 assert defs 

108 line_offset = start_idx + func_code[: defs[0].end()].count("\n") - 1 

109 func_body = func_code[defs[0].end() :] 

110 return func_body, line_offset 

111 

112 

113def is_empty_or_comment(line): 

114 sline = line.strip() 

115 return sline == "" or sline.startswith("#") 

116 

117 

118def iscomment(line): 

119 return line.strip().startswith("#") 

120 

121 

122def dedent_line(line, indent): 

123 for i, (line_sym, indent_sym) in enumerate(zip(line, indent)): 

124 if line_sym != indent_sym: 

125 start = i 

126 break 

127 else: 

128 start = len(indent) 

129 return line[start:] 

130 

131 

132def dedent_function_body(body): 

133 lines = body.split("\n") 

134 # find indentation by first line 

135 indent = "" 

136 for line in lines: 

137 if is_empty_or_comment(line): 

138 continue 

139 else: 

140 indent = re.match(r"^\s*", line).group() 

141 break 

142 

143 out_lines = [dedent_line(line, indent) for line in lines] 

144 return "\n".join(out_lines) 

145 

146 

147def get_function_body_code(func): 

148 filename = inspect.getfile(func) 

149 func_body, line_offset = get_function_body(func) 

150 body_source = dedent_function_body(func_body) 

151 try: 

152 body_code = compile(body_source, filename, "exec", ast.PyCF_ONLY_AST) 

153 body_code = ast.increment_lineno(body_code, n=line_offset) 

154 body_code = compile(body_code, filename, "exec") 

155 except SyntaxError as e: 

156 if e.args[0] == "'return' outside function": 

157 filename, lineno, _, statement = e.args[1] 

158 raise SyntaxError( 

159 "No return statements allowed in ConfigScopes\n" 

160 "('{}' in File \"{}\", line {})".format( 

161 statement.strip(), filename, lineno 

162 ) 

163 ) 

164 elif e.args[0] == "'yield' outside function": 

165 filename, lineno, _, statement = e.args[1] 

166 raise SyntaxError( 

167 "No yield statements allowed in ConfigScopes\n" 

168 "('{}' in File \"{}\", line {})".format( 

169 statement.strip(), filename, lineno 

170 ) 

171 ) 

172 else: 

173 raise 

174 return body_code 

175 

176 

177def is_ignored(line): 

178 for pattern in SETTINGS.CONFIG.IGNORED_COMMENTS: 

179 if re.match(pattern, line) is not None: 

180 return True 

181 return False 

182 

183 

184def find_doc_for(ast_entry, body_lines): 

185 lineno = ast_entry.lineno - 1 

186 line_io = io.BytesIO(body_lines[lineno].encode()) 

187 try: 

188 tokens = tokenize(line_io.readline) or [] 

189 line_comments = [token.string for token in tokens if token.type == COMMENT] 

190 

191 if line_comments: 

192 formatted_lcs = [line[1:].strip() for line in line_comments] 

193 filtered_lcs = [line for line in formatted_lcs if not is_ignored(line)] 

194 if filtered_lcs: 

195 return filtered_lcs[0] 

196 except TokenError: 

197 pass 

198 

199 lineno -= 1 

200 while lineno >= 0: 

201 if iscomment(body_lines[lineno]): 

202 comment = body_lines[lineno].strip("# ") 

203 if not is_ignored(comment): 

204 return comment 

205 if not body_lines[lineno].strip() == "": 

206 return None 

207 lineno -= 1 

208 return None 

209 

210 

211def add_doc(target, variables, body_lines): 

212 if isinstance(target, ast.Name): 

213 # if it is a variable name add it to the doc 

214 name = target.id 

215 if name not in variables: 

216 doc = find_doc_for(target, body_lines) 

217 if doc is not None: 

218 variables[name] = doc 

219 elif isinstance(target, ast.Tuple): 

220 # if it is a tuple then iterate the elements 

221 # this can happen like this: 

222 # a, b = 1, 2 

223 for e in target.elts: 

224 add_doc(e, variables, body_lines) 

225 

226 

227def get_config_comments(func): 

228 filename = inspect.getfile(func) 

229 func_body, line_offset = get_function_body(func) 

230 body_source = dedent_function_body(func_body) 

231 body_code = compile(body_source, filename, "exec", ast.PyCF_ONLY_AST) 

232 body_lines = body_source.split("\n") 

233 

234 variables = {"seed": "the random seed for this experiment"} 

235 

236 for ast_root in body_code.body: 

237 for ast_entry in [ast_root] + list(ast.iter_child_nodes(ast_root)): 

238 if isinstance(ast_entry, ast.Assign): 

239 # we found an assignment statement 

240 # go through all targets of the assignment 

241 # usually a single entry, but can be more for statements like: 

242 # a = b = 5 

243 for t in ast_entry.targets: 

244 add_doc(t, variables, body_lines) 

245 

246 return variables