qubx.pyenv

1 """ 2 Support for user scripting, including 3 - output injection, multi-re-direction and rotating logs 4 - well-defined global namespace 5 - persistent additions to sys.path 6 - persistent two-stage startup script 7 - throwing KeyboardInterrupt into "interruptible" threads 8 9 The official global scope is module qubx.global_namespace, populated from global_names.py to avoid circular imports 10 - To set a global variable: env.globals['x'] = 1 OR qubx.global_namespace.x = 1 11 - To run a user one-liner: env.exec_cmd(string) 12 - To run a user script: env.exec_script(string) 13 - To run a file: env.exec_file(path, raise_exceptions=True) 14 - To evaluate, eval(code_string, env.globals) 15 The exec_* environments eat exceptions and report them via env.On*Exception callbacks (except exec_file with default raise_exceptions=True). 16 17 Copyright 2007-2014 Research Foundation State University of New York 18 This file is part of QUB Express. 19 20 QUB Express is free software; you can redistribute it and/or modify 21 it under the terms of the GNU General Public License as published by 22 the Free Software Foundation, either version 3 of the License, or 23 (at your option) any later version. 24 25 QUB Express is distributed in the hope that it will be useful, 26 but WITHOUT ANY WARRANTY; without even the implied warranty of 27 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 GNU General Public License for more details. 29 30 You should have received a copy of the GNU General Public License, 31 named LICENSE.txt, in the QUB Express program directory. If not, see 32 <http://www.gnu.org/licenses/>. 33 34 """ 35 36 import collections 37 import ctypes 38 import datetime 39 import gc 40 import inspect 41 import linecache 42 import os 43 import platform 44 import re 45 import shutil 46 import sys 47 import threading 48 import time 49 import traceback 50 import zipfile 51 import qubx.util_types 52 import qubx.global_namespace 53 54 from itertools import izip_longest 55 56 import __main__ 57 58 59 env = None 60 61 62 # We provide OnOutput callback, which could do potentially non-thread-safe 63 # things like write to the gui. Typically in this situation there is a 64 # way to request execution on the gui thread, e.g. gobject.idle_add. 65 # For simpler programs, we can call OnOutput directly, in-thread:

66 -def call_now(func, *args, **kw):

67 func(*args, **kw)

68

69 -def call_no_delay(ms, func, *args, **kw):

70 func(*args, **kw)

71 72

73 -def bind_scriptable_base(bind_f, expr, callback, *args, **kw):

74 bound = bind_f(callback, *args, **kw) 75 def act(*args, **kw): 76 env.OnScriptable(expr) 77 return bound(*args, **kw)

78 return act 79

80 -def bind_scriptable(expr, callback, *args, **kw):

81 """Wrapper for L{qubx.util_types.bind} which emits pyenv.OnScriptable(expr) prior to each callback.""" 82 return bind_scriptable_base(qubx.util_types.bind, expr, callback, *args, **kw)

83

84 -def bind_with_args_scriptable(expr, callback, *args, **kw):

85 """Wrapper for L{qubx.util_types.bind} which emits pyenv.OnScriptable(expr) prior to each callback.""" 86 return bind_scriptable_base(qubx.util_types.bind_with_args, expr, callback, *args, **kw)

87

88 -def bind_with_args_before_scriptable(expr, callback, *args, **kw):

89 """Wrapper for L{qubx.util_types.bind} which emits pyenv.OnScriptable(expr) prior to each callback.""" 90 return bind_scriptable_base(qubx.util_types.bind_with_args_before, expr, callback, *args, **kw)

91 92

93 -class PythonEnvironment(object):

94 """ 95 Python environment management for user scripting. 96 Don't construct this directly; call Init() and use global "env" 97 98 @ivar path: list of additions to sys.path 99 @ivar script: string to execute on startup/reload 100 @ivar init_script_dir: folder for startup scripts 101 @ivar OnOutput: L{WeakEvent}(string) when anything is printed to stdout or stderr 102 @ivar OnSetAlt: L{WeakEvent}(keystring, expr) 103 @ivar OnScriptable: L{WeakEvent}(expr) when any part of the program does something reproducible 104 @ivar OnPlayScript: L{WeakEvent}(ScriptModule) 105 @ivar OnStopScript: L{WeakEvent}(ScriptModule) 106 @ivar OnPauseScript: L{WeakEvent}(ScriptModule) 107 @ivar OnResumeScript: L{WeakEvent}(ScriptModule) 108 @ivar OnScriptLine: L{WeakEvent}(ScriptModule, lineno, line) 109 @ivar OnCompileException: L{WeakEvent}(ScriptModule, typ, val, tb) 110 @ivar OnScriptException: L{WeakEvent}(ScriptModule, typ, val, tb) 111 """

112 - def __init__(self, folder, call_later=call_now, call_delayed=call_no_delay, MainLoop=None, process_events=lambda: None, app_path=None):

113 global env 114 env = self 115 self.folder = folder 116 self.app_path = folder if (app_path is None) else app_path 117 self.call_later = call_later 118 self.call_delayed = call_delayed 119 self.MainLoop = MainLoop 120 self.process_events = process_events 121 self.rotate_logs() 122 self.log = open(os.path.join(folder, 'output.log'), 'w') 123 self.OnOutput = qubx.util_types.WeakEvent("PythonEnvironment.OnOutput") # (str) 124 self.OnSetAlt = qubx.util_types.WeakEvent() 125 self.OnScriptable = qubx.util_types.WeakEvent() 126 self.OnPlayScript = qubx.util_types.WeakEvent() 127 self.OnStopScript = qubx.util_types.WeakEvent() 128 self.OnPauseScript = qubx.util_types.WeakEvent() 129 self.OnResumeScript = qubx.util_types.WeakEvent() 130 self.OnScriptLine = qubx.util_types.WeakEvent() 131 self.OnCompileException = qubx.util_types.WeakEvent() 132 self.OnScriptException = qubx.util_types.WeakEvent() 133 systemType = platform.system() 134 self.global_modifier = 'Command' if (systemType == 'Darwin') else 'Alt' 135 self.silence = False 136 if 'actual_stdout' in __main__.__dict__: 137 self.stdout = __main__.actual_stdout # sys.stdout 138 self.stderr = __main__.actual_stderr # sys.stderr 139 else: 140 self.stdout = sys.stdout 141 self.stderr = sys.stderr 142 sys.stdout = self 143 sys.stderr = self 144 self.__have_console = not ((systemType == 'Darwin') or ('windows' in systemType.lower())) 145 self.globals = qubx.global_namespace.__dict__ 146 self._path = [] 147 path = [] 148 self.exec_script = ScriptModule("<script>", self.globals) 149 self.exec_altkey = ScriptModule('<altkey>', self.globals) 150 self.exec_cmd = ScriptModule("<command>", self.globals) 151 for module in (self.exec_script, self.exec_altkey, self.exec_cmd): 152 self.track_module_in_tasks(module) 153 module.OnCompileException += self.OnCompileException 154 module.OnException += self.OnScriptException 155 self.__r_on_comp_exc = self.__on_exc ### ? 156 self.__r_on_exc = self.__on_exc 157 self.OnCompileException += self.__r_on_comp_exc 158 self.OnScriptException += self.__r_on_exc 159 160 # now that we're safe from circular module imports, populate the global namespace: 161 try: 162 self.exec_file(os.path.join(self.app_path, 'global_names.py'), as_module='qubx.global_namespace') 163 except: 164 traceback.print_stack() 165 traceback.print_exc() 166 167 self.script = """ 168 169 def after_initialize(): 170 pass # this happens after everything is set up and the InitScripts have been exec'd 171 172 """ 173 self.init_script_dir = os.path.join(folder, 'InitScripts') 174 try: 175 prefs = open(os.path.join(folder, 'env.py'), 'r') 176 try: 177 path = eval(prefs.readline()) 178 except: 179 pass 180 self.script = prefs.read() 181 except: 182 pass 183 self.path = path 184 try: 185 self.init_script_dir = open(os.path.join(folder, 'init_dir.txt'), 'r').readline().strip() 186 except: 187 pass 188 if not os.path.exists(self.init_script_dir): 189 try: 190 os.makedirs(self.init_script_dir) 191 open(os.path.join(self.init_script_dir, 'README.txt'), 'w').write(""" 192 Python scripts in this folder are run automatically on startup (extension .py only), in alphabetical order. 193 They are run in the global namespace, so please don't use "from whatever import *" 194 """) 195 except: 196 traceback.print_exc() 197 script_path = os.path.join(self.folder, 'Scripts') 198 if not os.path.exists(script_path): 199 os.makedirs(script_path) 200 self.reload() 201 self.altmap = collections.defaultdict(lambda: "") 202 try: 203 keypat = re.compile(r"(.):\s+(.*)") 204 for line in open(os.path.join(folder, 'altmap.txt'), 'r'): 205 match = keypat.match(line) 206 if match: 207 self.altmap[match.group(1)] = match.group(2) 208 except: 209 pass 210 self.__serial_gc = 0

211 - def track_module_in_tasks(self, module, untrack=False):

212 if untrack: 213 module.OnPlay -= self.OnPlayScript 214 module.OnStop -= self.OnStopScript 215 module.OnPause -= self.OnPauseScript 216 module.OnResume -= self.OnResumeScript 217 module.OnLine -= self.OnScriptLine 218 else: 219 module.OnPlay += self.OnPlayScript 220 module.OnStop += self.OnStopScript 221 module.OnPause += self.OnPauseScript 222 module.OnResume += self.OnResumeScript 223 module.OnLine += self.OnScriptLine

224

225 - def save_altmap(self):

226 try: 227 f = open(os.path.join(self.folder, 'altmap.txt'), 'w') 228 for k in sorted(self.altmap.keys()): 229 f.write("%s: %s\n" % (k, self.altmap[k])) 230 except: 231 traceback.print_exc()

232 - def run_alt(self, k):

233 expr = self.altmap[k] 234 if os.path.exists(expr): 235 self.OnScriptable('qubx.pyenv.env.exec_file(%s)' % repr(expr)) 236 self.exec_file(expr) 237 else: 238 self.OnScriptable(expr) 239 self.exec_altkey(expr)

240 - def set_alt(self, k, expr):

241 self.altmap[k] = expr 242 self.OnSetAlt(k, expr)

243 - def gc_collect_on_idle(self):

244 self.__serial_gc += 1 245 self.call_later(self.__gc_collect, self.__serial_gc)

246 - def __gc_collect(self, serial):

247 if serial == self.__serial_gc: 248 gc.collect()

249 - def set_path(self, path):

250 for p in self._path: 251 sys.path.remove(p) 252 self._path = path 253 for p in self._path: 254 sys.path.append(p)

255 path = property(lambda s: s._path, set_path)

256 - def reload(self):

257 """executes the startup script""" 258 self.exec_script(self.script)

259 - def eval_str(self, expr, eat_exceptions=True):

260 try: 261 return eval(expr, self.globals) 262 except: 263 if eat_exceptions: 264 traceback.print_stack() 265 traceback.print_exc() 266 print 'evaulating expr:',expr 267 else: 268 raise

269 - def exec_file(self, fname, raise_exceptions=True, as_module=None):

270 if fname == '-': 271 fi = sys.stdin 272 fn = '<stdin>' 273 else: 274 fi = open(fname, 'r') 275 fn = fname 276 try: 277 module = ScriptModule(as_module or fn, self.globals) 278 module.OnPlay += self.OnPlayScript 279 module.OnStop += self.OnStopScript 280 module.OnPause += self.OnPauseScript 281 module.OnResume += self.OnResumeScript 282 module.OnLine += self.OnScriptLine 283 if not raise_exceptions: 284 module.OnCompileException += self.OnCompileException 285 module.OnScriptException += self.OnScriptException 286 287 text = fi.read() 288 module(text, raise_exceptions=raise_exceptions) 289 except: 290 if raise_exceptions: 291 raise 292 else: 293 traceback.print_stack() 294 traceback.print_exc()

295 - def exec_init_scripts(self):

296 """executes each .py file in app_path/InitScripts then user init_script_dir, in sorted() order, with the global (__main__) namespace.""" 297 for init_dir in (os.path.join(self.app_path, 'InitScripts'), self.init_script_dir): 298 try: 299 for base, dirs, files in os.walk(init_dir): 300 dirs[:] = [] # don't recurse 301 sort_alphabetical(files) 302 for fname in files: 303 if fname[-3:].lower() == '.py': 304 self.exec_file(os.path.join(base, fname)) 305 except: 306 traceback.print_stack() 307 traceback.print_exc()

308 - def after_initialize(self):

309 """calls exec_init_scripts, then global after_initialize(). call this when your program is totally running.""" 310 self.exec_init_scripts() 311 try: 312 self.globals['after_initialize']() 313 except: 314 traceback.print_stack() 315 traceback.print_exc()

316 - def save(self):

317 """writes init_script_dir, path and script to disk""" 318 try: 319 prefs = open(os.path.join(self.folder, 'env.py'), 'w') 320 prefs.write('%s\n'%self.path) 321 prefs.write(self.script) 322 if (not self.script) or (self.script[-1] != '\n'): 323 prefs.write('\n') 324 except: 325 traceback.print_stack() 326 traceback.print_exc() 327 try: 328 open(os.path.join(self.folder, 'init_dir.txt'), 'w').write("%s\n" % self.init_script_dir) 329 except: 330 traceback.print_stack() 331 traceback.print_exc()

332 - def write(self, x):

333 """Aappends x to sys.stdout and the log-file, and passes x to any OnOutput listeners.""" 334 if self.silence: return 335 self.log.write(x) 336 if self.__have_console: 337 # suppress program files\qub express\dist\express.exe.log 338 # prevent mac hang on stdout.write() 339 self.stdout.write(x) 340 self.call_later(self.OnOutput, x)

341 - def flush(self):

342 self.log.flush() 343 self.stdout.flush()

344 - def rotate_logs(self):

345 """moves the older log files along toward oblivion; called automatically on construction.""" 346 MAXLOG = 5 347 base = os.path.join(self.folder, 'output.log') 348 name = lambda i: "%s.%i" % (base, i) 349 names = [name(i) for i in reversed(xrange(MAXLOG))] + [base] 350 for i in xrange(len(names)-1): 351 try: 352 if os.path.exists(names[i]): 353 os.remove(names[i]) 354 if os.path.exists(names[i+1]): 355 os.rename(names[i+1], names[i]) 356 except: 357 traceback.print_stack() 358 traceback.print_exc()

359 - def scriptable_if_matching(self, expr, pairs):

360 diff = [] 361 for name, obj in pairs: 362 if obj != self.eval_str(name): 363 diff.append(name) 364 if diff: 365 self.OnScriptable("# Disabled due to missing (background?) object%s: %s\n# %s" % 366 ((len(diff)>1) and "s" or "", 367 ', '.join(diff), 368 expr)) 369 else: 370 self.OnScriptable(expr)

371 - def __on_exc(self, module, typ, val, tb):

372 traceback.print_exception(typ, val, tb)

373 374

375 -def parse_version_str(vs):

376 components = [] 377 while vs: 378 match = re.search(r"""(\d+)([0-9.]*)""", vs) 379 if match: 380 components.append(int(match.group(1))) 381 vs = match.group(2) 382 else: 383 vs = "" 384 return tuple(components)

385

386 -def read_version(text, symbol='VERSION'):

387 match = re.search(r"""%s[^'"]*['"]([0-9.]+)""" % symbol, text) 388 if match: 389 return parse_version_str(match.group(1)) 390 return (0, 0, 0)

391

392 -def read_plugin_version(path):

393 init_text = "" 394 if os.path.isdir(path): 395 splits = qubx.util_types.path_splits(path) 396 init_path = os.path.join(path, 'qubx_plugin_init.py') 397 if os.path.exists(init_path): 398 init_text = open(init_path, 'r').read() 399 else: 400 zip = zipfile.ZipFile(path, 'r') 401 contents = zip.infolist() 402 for item in contents: 403 splits = qubx.util_types.path_splits(item.filename) 404 if splits[-1] == 'qubx_plugin_init.py': 405 init_text = zip.read(item.filename) 406 break 407 return read_version(init_text)

408

409 -def plugin_version_cmp(v1, v2):

410 """Returns -1 if v1 < v2, 0 if v1 == v2, 1 if v1 > v2.""" 411 for x1, x2 in izip_longest(v1, v2, fillvalue=0): 412 if x1 < x2: 413 return -1 414 elif x1 == x2: 415 pass 416 else: 417 return 1 418 return 0

419

420 -class PluginManager(object):

421 - def __init__(self):

422 self.OnUpdate = qubx.util_types.WeakEvent() # (Plugins, index) 423 self.OnRemoving = qubx.util_types.WeakEvent() # (Plugins, index) 424 self.paths = [] 425 self.fnames = [] 426 self.names = [] 427 self.abouts = [] 428 self.imports = [] 429 self.finis = []

430 - def initialize(self, sys_path, user_path):

431 self.sys_path = sys_path 432 self.user_path = user_path 433 try: 434 if not os.path.exists(user_path): 435 os.makedirs(user_path) 436 except: 437 pass 438 plugins = collections.defaultdict(lambda: qubx.util_types.Anon(zip=collections.defaultdict(lambda: qubx.util_types.Anon()), 439 dir=collections.defaultdict(lambda: qubx.util_types.Anon()))) 440 for is_sys, x_path in enumerate([user_path, sys_path]): 441 for base, dirs, files in os.walk(x_path): 442 for fi in files: 443 name, ext = os.path.splitext(fi) 444 if ext.lower() == '.zip': 445 plugins[name].zip[bool(is_sys)].path = os.path.join(base, fi) 446 plugins[name].zip[bool(is_sys)].version = read_plugin_version(os.path.join(base, fi)) 447 for d in dirs: 448 if d[0] != '.': 449 plugins[d].dir[bool(is_sys)].path = os.path.join(base, d) 450 plugins[d].dir[bool(is_sys)].version = read_plugin_version(os.path.join(base, d)) 451 dirs[:] = [] 452 def pick_plugin(name, choices): 453 for is_sys in (False, True): # prefer per-user stuff 454 if choices.dir[is_sys].path: # prefer folder unless zip is newer 455 if choices.zip[is_sys].path and (plugin_version_cmp(choices.dir[is_sys].version, choices.zip[is_sys].version) > 0): 456 return choices.zip[is_sys].path 457 return choices.dir[is_sys].path 458 if choices.zip[is_sys].path: 459 return choices.zip[is_sys].path

460 plugins = [pick_plugin(k, plugins[k]) for k in sorted(plugins.keys())] 461 for path in plugins: 462 self.add(path)

463 - def add(self, path):

464 print 'Loading plugin:',os.path.splitext(os.path.split(path)[1])[0],path 465 if os.path.splitext(os.path.split(path)[1])[0] in [os.path.split(p)[1] for p in self.paths]: 466 print 'ignoring duplicate plugin:',path 467 return 468 ix = len(self.paths) 469 self.paths.append(path) 470 self.fnames.append('') 471 self.names.append('Error') 472 self.abouts.append('') 473 self.imports.append([]) 474 self.finis.append('') 475 messages = [] 476 try: 477 if os.path.isdir(path): 478 splits = qubx.util_types.path_splits(path) 479 if not splits[-1]: 480 del splits[-1] 481 fname = splits[-1] 482 folder = path 483 else: 484 zip = zipfile.ZipFile(path, 'r') 485 contents = zip.infolist() 486 fname = qubx.util_types.path_splits(contents[0].filename)[0] 487 folder = os.path.join(self.user_path, fname) 488 if not os.path.exists(folder): 489 os.makedirs(folder) 490 for item in contents: 491 dest = os.path.join(self.user_path, item.filename) 492 splits = qubx.util_types.path_splits(item.filename) 493 if not splits[-1]: 494 if not os.path.exists(dest): 495 os.makedirs(dest) 496 else: 497 open(dest, 'wb').write( zip.read(item.filename) ) 498 self.paths[ix] = folder 499 self.fnames[ix] = fname 500 try: 501 plugname = open(os.path.join(folder, 'qubx_plugin_name.txt'), 'r').read().strip() 502 except: 503 plugname = folder 504 # do imports 505 sys.path.append(folder) 506 imports_path = os.path.join(folder, 'qubx_plugin_imports.txt') 507 if os.path.exists(imports_path): 508 for line in open(imports_path, 'r'): 509 package = line.strip() 510 if package: 511 self.imports[ix].append(package) 512 try: 513 try: 514 eval('%s.__name__' % package, env.globals) 515 exec('reload(%s)' % package, env.globals) 516 except (AttributeError, NameError): 517 exec('import %s' % package, env.globals) 518 except: 519 messages.append(traceback.format_exception(*sys.exc_info())) 520 # run init 521 init_path = os.path.join(folder, 'qubx_plugin_init.py') 522 if os.path.exists(init_path): 523 try: 524 env.exec_file(init_path, raise_exceptions=True) 525 except: 526 messages.append(traceback.format_exception(*sys.exc_info())) 527 # replace about from file 528 about_path = os.path.join(folder, 'qubx_plugin_about.txt') 529 if os.path.exists(init_path): 530 self.abouts[ix] = open(about_path, 'r').read() 531 # store fini 532 fini_path = os.path.join(folder, 'qubx_plugin_fini.py') 533 if os.path.exists(fini_path): 534 self.finis[ix] = open(fini_path, 'r').read() 535 # name for success 536 self.names[ix] = plugname 537 except: 538 messages.append(traceback.format_exception(*sys.exc_info())) 539 if messages: 540 self.abouts[ix] = self.abouts[ix] + '\n'.join(['']+['\n'.join(m) for m in messages]) 541 self.OnUpdate(self, ix)

542 - def reload(self, ix):

543 messages = [] 544 try: 545 folder = self.paths[ix] 546 # finalize 547 if self.finis[ix]: 548 try: 549 module = ScriptModule(os.path.join(folder, 'qubx_plugin_fini.py'), env.globals) 550 module(self.finis[ix], raise_exceptions=True) # stored in case the file has changed 551 except: 552 traceback.print_stack() 553 messages.append(traceback.format_stack()) 554 traceback.print_exc() 555 messages.append(traceback.format_exception(*sys.exc_info())) 556 # reload packages 557 for package in self.imports[ix]: 558 try: 559 exec('reload(%s)' % package, env.globals) 560 except: 561 traceback.print_stack() 562 messages.append(traceback.format_stack()) 563 traceback.print_exc() 564 messages.append(traceback.format_exception(*sys.exc_info())) 565 try: 566 exec('import %s' % package, env.globals) 567 except: 568 traceback.print_stack() 569 messages.append(traceback.format_stack()) 570 traceback.print_exc() 571 messages.append(traceback.format_exception(*sys.exc_info())) 572 # run init 573 init_path = os.path.join(folder, 'qubx_plugin_init.py') 574 if os.path.exists(init_path): 575 try: 576 exec(open(init_path, 'r').read(), env.globals) 577 except: 578 traceback.print_stack() 579 messages.append(traceback.format_stack()) 580 traceback.print_exc() 581 messages.append(traceback.format_exception(*sys.exc_info())) 582 # store fini 583 fini_path = os.path.join(folder, 'qubx_plugin_fini.py') 584 if os.path.exists(fini_path): 585 self.finis[ix] = open(fini_path, 'r').read() 586 # replace about from file 587 about_path = os.path.join(folder, 'qubx_plugin_about.txt') 588 if os.path.exists(init_path): 589 self.abouts[ix] = open(about_path, 'r').read() 590 except: 591 traceback.print_stack() 592 messages.append(traceback.format_stack()) 593 traceback.print_exc() 594 messages.append(traceback.format_exception(*sys.exc_info())) 595 if messages: 596 self.abouts[ix] = self.abouts[ix] + '\n'.join(['']+['\n'.join(m) for m in messages]) 597 self.OnUpdate(self, ix)

598 - def remove(self, ix):

599 self.OnRemoving(self, ix) 600 folder = self.paths[ix] 601 # run finalize 602 if self.finis[ix]: 603 try: 604 module = ScriptModule(os.path.join(folder, 'qubx_plugin_fini.py'), env.globals) 605 module(self.finis[ix], raise_exceptions=True) # stored in case the file has changed 606 except: 607 traceback.print_stack() 608 traceback.print_exc() 609 sys.path.remove(folder) 610 # delete path (recurs) 611 try: 612 shutil.rmtree(folder) 613 except: 614 traceback.print_stack() 615 traceback.print_exc() 616 del self.paths[ix] 617 del self.fnames[ix] 618 del self.names[ix] 619 del self.abouts[ix] 620 del self.imports[ix] 621 del self.finis[ix]

622

623 -def sort_alphabetical(lst):

624 lst.sort(lambda x, y: cmp(x.lower(),y.lower()))

625 626

627 -class DeferredAction(object):

628 """Calls a func(*args, **kw) after a short delay; 629 if multiple expressions are received from one source within the delay time, only the final one is passed through; 630 if the source changes, previous func is called immediately. 631 """

632 - def __init__(self, delay_ms=500):

633 self.delay_ms = 500 634 self.source = None 635 self.time = None 636 self.func = None 637 self.undoStack = None 638 self.args = None 639 self.kw = None

640 - def __call__(self, source, func, undoStack=None, *args, **kw):

641 if self.source and (source != self.source): 642 self.emit() 643 self.source = source 644 waiting = bool(self.time) 645 self.time = datetime.datetime.now() 646 self.func = func 647 self.undoStack = undoStack 648 self.args = args 649 self.kw = kw 650 if not waiting: 651 env.call_delayed(self.delay_ms, self.__on_timeout)

652 - def __on_timeout(self):

653 if not self.time: return # already handled 654 delta = datetime.datetime.now() - self.time 655 ms = 1000 * delta.seconds + delta.microseconds / 1000 656 if ms > (self.delay_ms*4)/5: 657 self.emit() 658 else: 659 env.call_delayed(int(round(self.delay_ms-ms)), self.__on_timeout)

660 - def emit(self):

661 if self.undoStack: 662 self.undoStack.seal_undo('actions') 663 if self.func: 664 self.func(*self.args, **self.kw) 665 self.source = self.time = self.func = self.args = self.kw = None

666 667

668 -class DeferredScriptable(DeferredAction):

669 """Passes through expressions to qubx.pyenv.env.OnScriptable after a short delay; 670 if multiple expressions are received from one source within the delay time, only the final one is passed through. 671 """

672 - def __call__(self, source, expr, if_matching=[], undoStack=None):

673 self.expr = expr 674 self.if_matching = if_matching 675 DeferredAction.__call__(self, source, lambda: self.expr and env.scriptable_if_matching(self.expr, self.if_matching), undoStack)

676

677 -class DeferredScriptableScroll(DeferredScriptable):

678 - def __init__(self, delay_ms=500):

679 DeferredScriptable.__init__(self, delay_ms) 680 self.__sum_offset = 0

681 - def __call__(self, source, template, offset, fine=False, coarse=False, if_matching=[], undoStack=None):

682 """template e.g. 'stuff = scrolled_float(stuff, %s, lo=0.0)""" 683 src = (source, fine, coarse) 684 if src != self.source: 685 self.emit() 686 self.template = template 687 self.fine = fine 688 self.coarse = coarse 689 self.__sum_offset = 0 690 self.__sum_offset += offset 691 self.rewrite_expr() 692 DeferredScriptable.__call__(self, src, self.expr, if_matching, undoStack)

693 - def rewrite_expr(self):

694 if self.__sum_offset: 695 self.expr = self.template % ("offset=%s, fine=%s, coarse=%s" % (repr(self.__sum_offset), repr(bool(self.fine)), repr(bool(self.coarse)))) 696 else: 697 self.expr = ""

698 699

700 -class ScriptRecursionError(Exception):

701 pass

702

703 -class ScriptModule(object):

704 - def __init__(self, filename, globals, process_events=True):

705 self.filename = os.path.split(filename)[1] 706 self.global_name = globals['__name__'] 707 self.globals = globals # FakeModuleDict(base=globals) 708 self.process_events = process_events 709 #self.globals.set_in_module('__file__', self.filename) 710 #self.globals.set_in_module('__name__', self.name) 711 self.OnPlay = qubx.util_types.WeakEvent() 712 self.OnPause = qubx.util_types.WeakEvent() 713 self.OnResume = qubx.util_types.WeakEvent() 714 self.OnStop = qubx.util_types.WeakEvent() 715 self.OnLine = qubx.util_types.WeakEvent() 716 self.OnCompileException = qubx.util_types.WeakEvent() 717 self.OnException = qubx.util_types.WeakEvent() 718 self.code = None 719 self.script = None 720 self.running = False 721 self.rq_pause = False 722 self(None) # safely init self.*

723 - def __nonzero__(self):

724 return bool(self.script)

725 - def preload(self, expr, raise_exceptions=True):

726 self.script = expr 727 self.lines = re.split(r"[\r\n]", expr) if expr else [] 728 self.lineno = -1 729 try: 730 self.code = compile(expr, self.filename, 'exec') 731 except: 732 self.script = None 733 self.lines = [] 734 self.rq_pause = False 735 self.lineno = -1 736 if raise_exceptions: 737 raise 738 else: 739 self.OnCompileException(self, *sys.exc_info()) 740 return False 741 return True

742 - def __call__(self, expr=None, locals=None, raise_exceptions=False):

743 if self.running: 744 raise ScriptRecursionError("Can't recursively execute ScriptModule(%s)" % repr(self.filename)) 745 self.stopped = False 746 self.paused = False 747 self.resume = self.resume_none 748 if ((not (expr is None)) or not self.code) and ((not expr) or (not self.preload(expr, raise_exceptions))): 749 self.rq_pause = False 750 return 751 self.prev_trace = sys.gettrace() 752 self.running = True 753 self.OnPlay(self) 754 sys.settrace(self.on_trace) 755 run_exc = None 756 loc = self.globals if (locals is None) else locals 757 try: 758 exec(self.code, self.globals, loc) 759 except: 760 if raise_exceptions: 761 run_exc = sys.exc_info() 762 else: 763 self.OnException(self, *sys.exc_info()) 764 sys.settrace(self.prev_trace) 765 self.lines = [] 766 self.rq_pause = False 767 self.running = False 768 self.OnStop(self) 769 self.lineno = -1 770 if not (run_exc is None): 771 typ, val, tb = run_exc 772 raise typ, val, tb

773 - def resume_none(self):

774 pass

775 - def script_pause(self):

776 """Called from within a script; doesn't return until someone calls resume().""" 777 if self.paused or not self.running: 778 return 779 recursive_main = env.MainLoop() 780 self.paused = True 781 self.resume = recursive_main.quit 782 self.OnPause(self) 783 try: 784 recursive_main.run() 785 finally: 786 self.paused = False 787 self.resume = self.resume_none 788 self.OnResume(self)

789 - def pause(self):

790 """Called from outside a script; pauses via the trace function.""" 791 self.rq_pause = True

792 - def step(self):

793 """Moves forward one line in the script.""" 794 if not (self.running and self.paused): 795 return 796 self.rq_pause = True 797 self.resume()

798 - def stop(self):

799 self.stopped = True 800 self.resume()

801 - def on_trace(self, frame, event, arg):

802 if hasattr(frame, 'f_code'): 803 filename = frame.f_code.co_filename 804 else: 805 filename = '<unknown>' 806 if '__name__' in frame.f_globals: 807 name = frame.f_globals["__name__"] 808 else: # py2exe doesn't replicate on main module: 809 name = '__main__' 810 if filename == self.filename: 811 if (event == 'line'): 812 lineno = frame.f_lineno 813 self.lineno = lineno 814 line = self.lines[lineno-1] if (lineno <= len(self.lines)) else "" 815 if env.globals['DEBUG']: 816 env.stdout.write("%s:%s: %s\n" % (filename, lineno, line.rstrip())) 817 self.OnLine(self, lineno, line) 818 if self.stopped: 819 raise KeyboardInterrupt() 820 if self.rq_pause: 821 self.rq_pause = False 822 self.script_pause() 823 if self.process_events: 824 env.process_events() 825 else: 826 return None 827 return self.on_trace

828 829

830 -def call_async_wait(func, *args, **kw):

831 """Calls func, returns a delayed result via call_later 832 833 func must be written to return a result via a callback on the main event loop, e.g. 834 >>> gobject.idle_add(receiver, results, ...) 835 It will be called 836 >>> func(receiver, *args, **kw) 837 then a recursive main loop will run until receiver is called (be careful this is not forever). 838 Receiver's args are returned as a tuple. 839 """ 840 result = [None] 841 recursive_main = qubx.pyenv.env.MainLoop() 842 def receive(*tup): 843 result[0] = tup 844 recursive_main.quit()

845 func(receive, *args, **kw) 846 recursive_main.run() 847 return result[0] 848 849

850 -def doze(seconds):

851 """Runs a recursive main loop for a number of seconds, then returns.""" 852 recursive_main = env.MainLoop() 853 env.call_delayed(int(round(seconds*1000)), recursive_main.quit) 854 recursive_main.run()

855 856

857 -class DeferredSideEffects(object):

858 """Context manager, for use in a script, which disables the trace function -- 859 faster execution because it doesn't wait for side-effects to complete after each line -- 860 but dangerous if the side effects need to happen before the script continues. 861 """

862 - def __enter__(self):

863 self.__prev_trace = sys.gettrace() 864 sys.settrace(None)

865 - def __exit__(self, type, value, tback):

866 env.process_events() 867 sys.settrace(self.__prev_trace)

868 869 870 featured_pat = re.compile(r"\.([^.']+)'") 871

872 -def featured_property_name(cls):

873 match = featured_pat.search(str(cls)) 874 if match: 875 return '_%s__explore_featured' % match.group(1) 876 return ""

877

878 -def generate_featured(cls, obj=None, recursive=True):

879 if not ((obj is None) or hasattr(obj, '__dict__')): 880 return 881 if not has_featured(cls): 882 return 883 pname = featured_property_name(cls) 884 psrc = cls if (obj is None) else obj 885 plist = getattr(psrc, pname) if hasattr(psrc, pname) else [] 886 if plist: 887 yield plist 888 if recursive: 889 for b in cls.__bases__: 890 for ff in generate_featured(b, obj): 891 yield ff

892

893 -def has_featured_impl(cls):

894 pname = featured_property_name(cls) 895 if hasattr(cls, pname): 896 return True 897 for b in cls.__bases__: 898 if has_featured_impl(b): 899 return True 900 return False

901 has_featured = qubx.util_types.memoize(has_featured_impl) 902

903 -def get_name_map_featured(obj, objname=""):

904 featured = {} 905 for plist in generate_featured(obj.__class__, obj): 906 for k in plist: 907 try: 908 featured[k] = getattr(obj, k) 909 except AttributeError: 910 name = str(obj) if (not (objname)) else objname 911 print 'Missing featured attribute "%s" of %s' % (k, name) 912 if featured and hasattr(obj, 'INSTANCES'): 913 featured['INSTANCES'] = obj.INSTANCES 914 return featured

915 916 917 918 functional_class_str = set(("<type 'function'>", "<type 'instancemethod'>", "<type 'builtin_function_or_method'>", 919 "<type 'numpy.ufunc'>"))

920 -def is_functional_class(class_str):

921 return class_str in functional_class_str

922 923

924 -def mark_instances(in_modules=[qubx], max_depth=7):

925 """To help bridge the gap between the class hierarchy and the working global namespace, 926 creates a defaultdict(list) INSTANCES in each sub-module of the listed modules, with global names of instances. 927 928 @param module: e.g. qubx 929 @param max_depth: how many dots (or dict/list lookups) to descend through the global hierarchy 930 """ 931 # traverse sub-modules, marking leaves with .exemplars = [] 932 modules = [] 933 def setup(module): 934 modules.append(module) 935 module.INSTANCES = collections.defaultdict(list) 936 module.INSTANCES_AS_BASE = collections.defaultdict(list) 937 try: 938 for name, member in inspect.getmembers(module, inspect.ismodule): 939 if not (member in modules): 940 setup(member) 941 except ImportError: # e.g. _winreg under linux (scipy optionals) 942 pass

943 for module in in_modules: 944 setup(module) 945 # traverse globals with BFS; ignore str,number,functional,numpy,scipy,gtk 946 visited = {} 947 ignore_types = set((int, float, long, bool, str, type)) 948 def consider(obj, name, next_row): 949 obj_id = id(obj) 950 if obj_id in visited: 951 return 952 try: 953 c = obj.__class__ 954 if c in ignore_types: 955 return 956 else: 957 cs = str(c) 958 if ('numpy' in cs) or ('scipy' in cs) or ('gtk.' in cs) or is_functional_class(cs) or ('GProps' in cs): 959 return 960 except AttributeError: 961 return 962 next_row.append((obj, name)) 963 visited[obj_id] = True 964 def mark(cls, name, instance=True, bases=True, topcls=None): 965 try: 966 module = eval(cls.__module__, env.globals) 967 if module in modules: 968 if instance: 969 module.INSTANCES[format_classname(cls)].append(name) 970 if not (topcls is None): 971 module.INSTANCES_AS_BASE[format_classname(cls)].append((name, str(topcls))) 972 if bases: 973 for base in cls.__bases__: 974 if base != object: 975 top = cls if (topcls is None) else topcls 976 mark(base, name, False, True, top) 977 except: 978 pass # either module isn't reachable from qubx.global_namespace, or not hasattr(obj, "__module__") 979 def traverse(row, depth): 980 next_row = [] 981 for obj, name in row: 982 if isinstance(obj, dict): 983 for k in obj: 984 consider(obj[k], "%s[%s]"%(name, repr(k)), next_row) 985 elif isinstance(obj, tuple): 986 for i, sub in enumerate(obj): 987 consider(sub, "%s[%i]"%(name, i), next_row) 988 elif isinstance(obj, list): 989 if len(obj): # probably all same type, so just do one 990 consider(obj[0], "%s[0]"%name, next_row) 991 else: 992 subs = get_name_map_featured(obj, name) 993 if subs: 994 subs = subs.iteritems() 995 else: 996 try: 997 subs = inspect.getmembers(obj) 998 except AttributeError: # __abstractmethods__ bug fixed in Python 3 999 pass 1000 except ImportError: # e.g. _winreg under linux (scipy optionals) 1001 pass 1002 for k,sub in subs: 1003 if -1 == k.find("__"): 1004 consider(sub, name and ("%s.%s" % (name, k)) or k, next_row) 1005 mark(obj.__class__, name) 1006 row[:] = [] 1007 if depth > 0: 1008 traverse(next_row, depth-1) 1009 traverse([(qubx.global_namespace, "")], max_depth) 1010 for module in modules: 1011 if module.INSTANCES: 1012 for k,v in module.INSTANCES.iteritems(): 1013 v.sort() 1014 else: 1015 del module.INSTANCES 1016 if module.INSTANCES_AS_BASE: 1017 for k,v in module.INSTANCES_AS_BASE.iteritems(): 1018 v.sort() 1019 else: 1020 del module.INSTANCES_AS_BASE 1021 1022 classname_pat = re.compile(r".*\.([^.']+)'")

1023 -def format_classname(cls):

1024 match = classname_pat.search(repr(cls)) 1025 if match: 1026 return match.group(1) 1027 return ""

1028 1029 1030 PYTHON_HELP = """ 1031 basic operators: +, -, *, / 1032 exponents: pow(base, exp) or base**exp 1033 1034 standard math: pow, exp, log, log10, ceil, floor, fabs, cos, sin, tan, acos, asin, atan, atan2, ... 1035 Type e.g. "help(atan2)" at the ">>>" prompt for details. 1036 1037 Functions from numpy and scipy are also available, and you can define custom functions in 1038 Admin -> Scripts -> Environment. 1039 1040 Enter whole numbers with a decimal point (e.g. 3. or 3.0, not 3). a/b uses integer division if both a and b are integers, so (2/3)==0 but (2.0/3.0)==0.66666666667. 1041 """ 1042

1043 -def Init(folder, call_later=call_now, call_delayed=call_no_delay, MainLoop=None, process_events=lambda: None, app_path=None):

1044 """ 1045 Sets up the global variable "env" (class PythonEnvironment) 1046 1047 @param folder: path where settings and logs are saved 1048 @param call_later: e.g. gobject.idle_add if your OnOutput callback calls on gtk 1049 @param call_delayed: e.g. gobject.timeout_add if your OnOutput callback calls on gtk 1050 @param MainLoop: e.g. gobject.MainLoop or act-alike; req. for call_async_wait and doze only 1051 @param process_events: func() which processes any outstanding events before returning 1052 @param app_path: location of global_names.py 1053 """ 1054 global env, Plugins 1055 if not os.path.exists(folder): 1056 os.makedirs(folder) 1057 env = PythonEnvironment(folder, call_later, call_delayed, MainLoop, process_events, app_path) 1058 Plugins = PluginManager()

1059

Source Code for Module qubx.pyenv