qubx.table

1 """ 2 Classes to represent tabular information. 3 - L{Table} defines the interface 4 - L{SimpleTable} lets you define fields and store information 5 - L{ObjectTable} presents a collection of existing objects 6 - L{TableUndo} helps interface with L{qubx.undo}. 7 8 All tables have fields 'Index' (maintained automatically as the row number) 9 and 'Group' (default 0, meant for future classification purposes, see QUB:Select). 10 11 Copyright 2008-2015 Research Foundation State University of New York 12 This file is part of QUB Express. 13 14 QUB Express is free software; you can redistribute it and/or modify 15 it under the terms of the GNU General Public License as published by 16 the Free Software Foundation, either version 3 of the License, or 17 (at your option) any later version. 18 19 QUB Express is distributed in the hope that it will be useful, 20 but WITHOUT ANY WARRANTY; without even the implied warranty of 21 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 22 GNU General Public License for more details. 23 24 You should have received a copy of the GNU General Public License, 25 named LICENSE.txt, in the QUB Express program directory. If not, see 26 <http://www.gnu.org/licenses/>. 27 28 """ 29 30 import collections 31 import cStringIO 32 from itertools import * 33 from math import * 34 import numpy 35 from qubx.util_types import * 36 from qubx.accept import * 37 import qubx.notebook 38 import qubx.pyenv 39 import traceback 40 41 MAX_FIELDS = 256 42 MAX_BOOLS = 16 43 44 COPY_ROWS_ALL, COPY_ROWS_GROUP, COPY_ROWS_CRITERIA, COPY_ROWS_CHECKED = (0, 1, 2, 3) 45

46 -def get_none(obj):

47 pass

48 -def set_none(obj, val):

49 pass

50

51 -def run_now(f, *args, **kw):

52 f(*args, **kw)

53 54 # change this variable at program init to something like gobject.idle_add 55 # (done automatically when you import qubx.tableGTK) 56 run_later = run_now 57

58 -class TableRow(Anon):

59 """Represents the data in one row of a L{Table}, with .field_name access, and a dict of fields. 60 61 @ivar fields: dict: field_name -> value 62 """

63 - def __init__(self, fields, safenames=False):

64 Anon.__init__(self, **fields) 65 self.__dict__['fields'] = fields 66 if safenames: 67 for k, v in fields.items(): 68 safek = SafeName(k) 69 if ((k != safek) and (not (safek in self.__dict__))): 70 self.__dict__[safek] = v

71 - def __setattr__(self, name, value):

72 raise Exception('TableRow is read-only.')

73 74

75 -class Table(object):

76 """Base class for info that can be represented as a table. 77 78 @ivar label: the table's name 79 @ivar global_name: how to locate self in global namespace, if possible 80 @ivar sortable: (def False) whether the user should be able to sort by column 81 @ivar size: the number of rows 82 @ivar fields: list of field names 83 @ivar fields_independent: list of field names marked independent e.g. 'Index' 84 @ivar default: dict: field name -> default value 85 @ivar accept: dict: field name -> conversion_function(string)->value; can raise Exception on invalid input 86 @ivar format: dict: field name -> format_function(value)->string 87 @ivar units: dict: field name -> units_string 88 @ivar choices: dict: field name -> (list of values to choose among) or None 89 @ivar checked: list[row index] of bool, should row be included in batch actions 90 @ivar user_can_remove: list[row index] of bool, can remove via gui; def True 91 @ivar mean: {field_name : f(group=None) -> mean value of field for all segments or group index} 92 @ivar std: {field_name : f(group=None) -> std deviation of field for all segments or group index} 93 @ivar median: {field_name : f(group=None) -> median value of field for all segments or group index} 94 @ivar mode: {field_name : f(group=None) -> most common value of field for all segments or group index} 95 @ivar groups_occupied: f(include_zero=False) -> sorted list of 'Group' values, no duplicates, possibly no 0 96 @ivar rows_in_group: f(group) -> list of row indices with Group==group 97 @ivar count_in_group: f(group) -> number of rows with Group==group 98 @ivar field_stats: {field_name : [mean, std, median, mode]} 99 @ivar group_stats: [groups_occupied, rows_in_group, count_in_group] 100 @ivar all_stats: field_stats + group_stats 101 @ivar OnAddField: L{WeakEvent}(name) called when a field is added 102 @ivar OnRemovingField: L{WeakEvent}(name) called when a field is being removed 103 @ivar OnInsert: L{WeakEvent}(index, undoing) called when a row is added (read-only please) 104 @ivar OnAfterInsert: L{WeakEvent}(index, undoing) called after insertion and all OnInsert callbacks (.set(...) if you like) 105 @ivar OnRemoving: L{WeakEvent}(index, undoing) called when a row is about to be removed 106 @ivar OnRemoved: L{WeakEvent}(index, undoing) called when a row was just removed 107 @ivar OnSet: L{WeakEvent}(index, field_name, val, prev, undoing) called when one field of a row is changed 108 @ivar OnSelect: L{WeakEvent}(index, field_name or None, sender or None) called when a row is highlighted 109 @ivar OnDoubleClick: L{WeakEvent}(index, field_name or None) called (e.g. by TableView) when a row is double-clicked 110 @ivar OnChecked: L{WeakEvent}(index, checked) called on set_checked 111 """ 112 __slots__ = ['__ref', 'label', '__global_name', 'sortable', 'fields', 'fields_independent', 'default', 'accept', 'format', 113 'units', 'choices', 'OnAddField', 'OnRemovingField', 'OnInsert', 'OnAfterInsert', 'OnRemoving', 'OnRemoved', 114 'OnSet', 'OnSelect', 'OnDoubleClick', 'OnChecked', 'max_fields', 'max_bools', 'checked', 'user_can_remove', 115 'mean', 'std', 'median', 'mode', 'groups_occupied', 'rows_in_group', 'count_in_group', 'field_stats', 116 'group_stats', 'all_stats', 'notebook', 'notebook_items', 'custom']

117 - def __init__(self, sortable=False):

118 self.__ref = Reffer() 119 self.label = '' 120 self.__global_name = '' 121 self.sortable = sortable 122 self.fields = ['Index', 'Group'] 123 self.fields_independent = ['Index'] 124 self.default = {'Group' : 0} 125 self.accept = {'Index' : acceptNothing, 'Group' : acceptIntGreaterThanOrEqualTo(0)} 126 self.format = {'Index' : str, 'Group' : str} 127 self.units = {'Index' : "", 'Group' : ""} 128 self.choices = {'Index' : None, 'Group' : None} 129 self.OnAddField = WeakEvent() # (name) 130 self.OnRemovingField = WeakEvent() 131 self.OnInsert = WeakEvent() # (i, undoing) 132 self.OnAfterInsert = WeakEvent() # (i, undoing) 133 self.OnRemoving = WeakEvent() # (i, undoing) 134 self.OnRemoved = WeakEvent() # (i, undoing) 135 self.OnSet = WeakEvent() # (i, field_name, val, prev, undoing) 136 self.OnSelect = WeakEvent() # (i, field_name or None, sender or None) 137 self.OnDoubleClick = WeakEvent() 138 self.OnChecked = WeakEvent() 139 self.max_fields = MAX_FIELDS 140 self.max_bools = MAX_BOOLS 141 self.checked = [] 142 self.user_can_remove = [] 143 self.mean = {} 144 self.std = {} 145 self.median = {} 146 self.mode = {} 147 self.groups_occupied = memoize(self.__groups_occupied) 148 self.rows_in_group = memoize(self.__rows_in_group) 149 self.count_in_group = memoize(self.__count_in_group) 150 self.field_stats = collections.defaultdict(lambda: []) 151 self.group_stats = [self.groups_occupied, self.rows_in_group, self.count_in_group] 152 self.all_stats = [x for x in self.group_stats] 153 self.add_field_to_stats('Index') 154 self.add_field_to_stats('Group') 155 self.OnInsert += self.__ref(self.__onInsert) 156 self.OnRemoved += self.__ref(self.__onRemoved) 157 self.notebook = {} 158 self.notebook_items = [] 159 self.add_notebook_item('Default', qubx.notebook.NbTable('Table', self.global_name and ('%s.notebook["Default"]'%self.global_name) or '', 160 self.nb_get_caption, 161 self.nb_get_shape, self.nb_get_headers, 162 self.nb_get_row, self.nb_get_col, self.nb_get_col_format, 163 self.nb_get_type)) 164 self.custom = Anon() # for dynamic nonsense like select_charts' .fitall

165

166 - def dispose(self):

167 """Releases references to improve chances of garbage collection.""" 168 self.notebook.clear() 169 del self.notebook_items[:] 170 self.__ref.clear()

171 - def set_global_name(self, x):

172 self.__global_name = x 173 for k,v in self.notebook.iteritems(): 174 v.global_name = '%s.notebook[%s]' % (x, repr(k))

175 global_name = property(lambda self: self.__global_name, lambda self, x: self.set_global_name(x)) 176 size = property(lambda self: self.get_size())

177 - def __len__(self):

178 return self.get_size()

179 - def __getitem__(self, key):

180 if hasattr(key, '__iter__'): 181 r,f = key 182 return self.get(r, f) 183 else: 184 return self.get_row(key)

185 - def __setitem__(self, key, val):

186 if hasattr(key, '__iter__'): 187 r,f = key 188 return self.set(r, f, val) 189 else: 190 for f,v in val.iteritems(): 191 self.set(key, f, v)

192 - def __iter__(self):

193 return (self.get_row(i) for i in xrange(self.size))

194 - def __str__(self):

195 if not self.size: return "" 196 def format_item(item): 197 if isinstance(item, float): 198 return '%.6g' % item 199 else: 200 return str(item)

201 def format_row(items): 202 return ' '.join(['%10s'%format_item(item) for item in items])

203 return '\n'.join([' '.join([('%10s' % s) for s in self.fields])] + 204 [format_row([self.get(r, field) for field in self.fields]) for r in xrange(self.size)])

205 - def to_text(self, separator='\t'):

206 if not self.size: return "" 207 def format_item(item): 208 field, val = item 209 if isinstance(val, float): 210 return '%.12g' % val 211 else: 212 return self.format[field](val) # str(item)

213 def format_row(items): 214 return separator.join([format_item(item) for item in items]) 215 return '\n'.join([separator.join(self.fields)] + 216 [format_row([(field, self.get(r, field)) for field in self.fields]) for r in xrange(self.size)])

217 - def from_text(self, txt, keep_fields=False):

218 headers, types, columns = read_table_text(txt) 219 if (not columns) or (not columns[0]): return False 220 self.clear() 221 self.max_fields = max(self.max_fields, 2*len(headers)) 222 for i,h in enumerate(headers): 223 if not h: 224 headers[i] = default_column_name(i) 225 if not (keep_fields is True): 226 for field in reversed(self.fields): 227 if field in ('Index', 'Group'): 228 continue 229 if keep_fields and (field in keep_fields): 230 continue 231 self.remove_field(field) 232 for h,t in izip(headers, types): 233 if not (h in self.fields): 234 self.add_field(h, t(), t, (t == float) and '%.6g' or str, '') 235 for items in izip(*columns): 236 self.append(dict(izip(headers, items))) 237 return True

238 - def to_json(self):

239 return [JsonAnon(row.fields) for row in self.all_rows()]

240 - def from_json(self, json):

241 self.clear() 242 if json: 243 for row in json: 244 self.append(row)

245 - def append(self, entry):

246 """equivalent to table.insert(table.size, entry)""" 247 self.insert(self.size, entry)

248 - def insert(self, i, entry, undoing=False):

249 raise Exception('abstract method needs definition')

250 - def remove(self, i, undoing=False):

251 raise Exception('abstract method needs definition')

252 - def get(self, i, field_name):

253 raise Exception('abstract method needs definition')

254 - def set(self, i, field_name, val, undoing=False):

255 raise Exception('abstract method needs definition')

256 - def select(self, i, field_name=None, sender=None):

257 """Triggers OnSelect. 258 259 This mechanism is provided so that, if there are two or more views on a table, you 260 can select an item in one and it will be selected in the other(s). A view calls select with 261 sender=self so it can ignore its own selection activity. 262 """ 263 if qubx.pyenv.env.globals['DEBUG']: 264 print 'select\t%d\t%s\t%s\t%s' % (i, self.label, field_name, sender) 265 traceback.print_stack() 266 print 267 run_later(self.OnSelect, i, field_name, sender)

268 - def clear(self):

269 """Removes all rows.""" 270 for i in reversed(xrange(self.size)): 271 self.remove(i)

272 - def get_row(self, i, safenames=False):

273 """Returns a the i'th L{TableRow}.""" 274 fields = {} 275 for f in self.fields: 276 fields[f] = self.get(i, f) 277 return TableRow(fields, safenames)

278 - def all_rows(self):

279 """Returns a list of L{TableRow}.""" 280 all = [] 281 for i in xrange(self.size): 282 all.append(self.get_row(i)) 283 return all

284 - def index(self, name, raise_error=True):

285 """Returns the index of the first row with "Name"==name.""" 286 for i in xrange(self.size): 287 if self.get(i, 'Name') == name: 288 return i 289 if raise_error: 290 raise KeyError(name) 291 else: 292 return -1

293 - def __onInsert(self, i, undoing=False):

294 self.checked.insert(i, False) 295 self.user_can_remove.insert(i, True)

296 - def __onRemoved(self, i, undoing=False):

297 del self.checked[i] 298 del self.user_can_remove[i]

299 - def set_checked(self, i, checked=True):

300 x = bool(checked) 301 if x == self.checked[i]: 302 return 303 self.checked[i] = x 304 self.OnChecked(i, x)

305 - def get_by_name(self, name, field):

306 """Returns the value of field, in the first row with "Name"=name; or raises KeyError.""" 307 return self.get(self.index(name), field)

308 - def get_row_by_name(self, name):

309 """Returns a L{TableRow} for the first row with "Name"=name; or raises KeyError.""" 310 return self.get_row(self.index(name))

311 - def rows_meeting_criteria(self, expr, row_count=None):

312 if not expr: return [] 313 locals = OverridingDict() 314 locals['field_mean'] = lambda name: self.mean[name]() 315 locals['group_mean'] = lambda name, group: self.mean[name](group) 316 locals['field_std'] = lambda name: self.std[name]() 317 locals['group_std'] = lambda name, group: self.std[name](group) 318 locals['field_median'] = lambda name: self.median[name]() 319 locals['group_median'] = lambda name, group: self.median[name](group) 320 locals['field_mode'] = lambda name: self.mode[name]() 321 locals['group_mode'] = lambda name, group: self.mode[name](group) 322 def test_row(row): 323 locals['row'] = row.__dict__ 324 locals.base = row.__dict__ 325 return eval(expr, qubx.pyenv.env.globals, locals)

326 return [i for i in xrange(row_count or self.size) if test_row(self.get_row(i, safenames=True))]

327 - def copy_rows_from(self, table, copy_rows_type=COPY_ROWS_ALL, group=1, inverse=False, criteria="", on_copy=None):

328 if copy_rows_type == COPY_ROWS_ALL: 329 row_ixs = xrange(table.size) 330 elif copy_rows_type == COPY_ROWS_GROUP: 331 if inverse: 332 inv = lambda x: not x 333 else: 334 inv = lambda x: x 335 row_ixs = [i for i in xrange(table.size) if inv(group == table[i, 'Group'])] 336 elif copy_rows_type == COPY_ROWS_CRITERIA: 337 row_ixs = table.rows_meeting_criteria(criteria) 338 elif copy_rows_type == COPY_ROWS_CHECKED: 339 row_ixs = [i for i in xrange(table.size) if table.checked[i]] 340 for i in row_ixs: 341 row = table.get_row(i).__dict__ 342 self.append( dict((f, row[f]) for f in table.fields) ) 343 if on_copy: 344 on_copy(table, i, self, self.size-1)

345 - def check_rows_if(self, copy_rows_type=COPY_ROWS_ALL, group=1, inverse=False, criteria="", on_copy=None):

346 if copy_rows_type == COPY_ROWS_ALL: 347 row_ixs = xrange(self.size) 348 elif copy_rows_type == COPY_ROWS_GROUP: 349 if inverse: 350 inv = lambda x: not x 351 else: 352 inv = lambda x: x 353 row_ixs = [i for i in xrange(self.size) if inv(group == self[i, 'Group'])] 354 elif copy_rows_type == COPY_ROWS_CRITERIA: 355 row_ixs = self.rows_meeting_criteria(criteria) 356 elif copy_rows_type == COPY_ROWS_CHECKED: 357 row_ixs = [i for i in xrange(self.size) if self.checked[i]] 358 for i in row_ixs: 359 self.set_checked(i)

360 - def calc_expr_values(self, expr, indices=None):

361 if indices is None: 362 ixs = xrange(self.size) 363 else: 364 ixs = indices 365 if not expr: 366 return [0.0 for i in ixs] 367 locals = OverridingDict() 368 def calc_row(row): 369 locals['row'] = row.__dict__ 370 locals.base = row.__dict__ 371 return eval(expr, qubx.pyenv.env.globals, locals)

372 return [calc_row(self.get_row(r, safenames=True)) for r in ixs] 373 # these stats are memoized to avoid duplicate and unused computations; reset them as needed (e.g. set, insert)

374 - def add_field_to_stats(self, field_name):

375 mean = self.mean[field_name] = memoize(bind_with_args(self.__mean, field_name)) 376 std = self.std[field_name] = memoize(bind_with_args(self.__std, field_name)) 377 median = self.median[field_name] = memoize(bind_with_args(self.__median, field_name)) 378 mode = self.mode[field_name] = memoize(bind_with_args(self.__mode, field_name)) 379 for lst in (self.field_stats[field_name], self.all_stats): 380 lst.extend([mean, std, median, mode])

381 - def remove_field_from_stats(self, field_name):

382 for lst in (self.field_stats[field_name], self.all_stats): 383 for stat in (self.mean, self.std, self.median, self.mode): 384 lst.remove(stat[field_name]) 385 for stat in (self.mean, self.std, self.median, self.mode): 386 del stat[field_name]

387 - def __mean(self, field_name, group=None):

388 tot = 0.0 389 cnt = 0 390 for i in self.rows_in_group(group): 391 try: 392 tot += self.get(i, field_name) 393 cnt += 1 394 except: 395 pass 396 return cnt and (tot / cnt) or 0.0

397 - def __std(self, field_name, group=None):

398 tot = 0.0 399 cnt = 0 400 mean = self.mean[field_name](group) 401 for i in self.rows_in_group(group): 402 try: 403 tot += (mean - self.get(i, field_name))**2 404 cnt += 1 405 except: 406 pass 407 return cnt and sqrt(tot / cnt) or 0.0

408 - def __median(self, field_name, group=None):

409 try: 410 vals = sorted([self.get(i, field_name) for i in self.rows_in_group(group)]) 411 if len(vals) % 2: 412 return vals[len(vals)/2] 413 elif len(vals): 414 mid = len(vals)/2 415 return (vals[mid] + vals[mid-1]) / 2.0 416 else: 417 return None 418 except: 419 return None

420 - def __mode(self, field_name, group=None):

421 counter = collections.defaultdict(lambda: 0) 422 for i in self.rows_in_group(group): 423 counter[self.get(i, field_name)] += 1 424 counted = collections.defaultdict(lambda: []) 425 maxcnt = 0 426 for val, cnt in counter.iteritems(): 427 counted[cnt].append(val) 428 if cnt > maxcnt: 429 maxcnt = cnt 430 modes = counted[maxcnt] 431 if len(modes) > 1: 432 return sum(modes) / len(modes) 433 elif modes: 434 return modes[0] 435 else: 436 return None

437 - def __groups_occupied(self, include_zero=False):

438 groups = [self.get(i, 'Group') for i in xrange(self.size)] 439 return sorted(list(set([g for g in groups if include_zero or (g > 0)])))

440 - def __rows_in_group(self, group):

441 if group is None: 442 return range(self.size) 443 return [i for i in xrange(self.size) if (self.get(i, 'Group') == group)]

444 - def __count_in_group(self, group):

445 return len(self.rows_in_group(group))

446

447 - def nb_get_caption(self):

448 return self.label

449 - def nb_get_shape(self):

450 if not (self.size and self.fields): 451 return 0, 0 452 return self.size, len(self.fields)

453 - def nb_get_headers(self):

454 return self.fields

455 - def nb_get_row(self, r):

456 return [self.get(r, f) for f in self.fields]

457 - def nb_get_col(self, c):

458 field = self.fields[c] 459 return [self.get(r, field) for r in xrange(self.size)]

460 - def nb_get_col_format(self, c):

461 return self.format[self.fields[c]]

462 - def nb_get_type(self, fields=None):

463 check_fields = self.fields if (fields is None) else fields 464 try: 465 for f in check_fields: 466 float(self.get(0, f)) 467 return float 468 except: 469 return str

470 - def add_notebook_item(self, name, item):

471 self.notebook[name] = item 472 self.notebook_items[:] = [self.notebook[k] for k in sorted(self.notebook)]

473 474 475

476 -class TableUndo(object):

477 """Binds a L{Table} to a L{qubx.undo.UndoStack}. 478 479 Simply create a TableUndo object, and all edits are pushed to the undo stack. 480 You still have to seal_undo to mark each restore point (that way several edits 481 can be sealed as a single undo). 482 483 To disassociate the table and the undoStack, call .dispose(). 484 """ 485 __slots__ = ['table', 'undoStack', 'ref']

486 - def __init__(self, table, undoStack):

487 self.table = table 488 self.undoStack = undoStack 489 self.ref = Reffer() 490 self.table.OnInsert += self.ref(self.onInsert), 'TableUndo<%s>.onInsert' % table.label 491 self.table.OnRemoving += self.ref(self.onRemoving), 'TableUndo<%s>.onRemoving' % table.label 492 self.table.OnSet += self.ref(self.onSet), 'TableUndo<%s>.onSet' % table.label

493 - def dispose(self):

494 """Disassociates the table from the undoStack. No more undos are pushed.""" 495 self.table.OnInsert -= self.ref(self.onInsert) 496 self.table.OnRemoving -= self.ref(self.onRemoving) 497 self.table.OnSet -= self.ref(self.onSet)

498 - def onInsert(self, i, undoing):

499 if undoing: return 500 entry = {} 501 for f in self.table.fields: 502 entry[f] = self.table.get(i, f) 503 self.undoStack.push_undo(lambda: self.table.remove(i, True), 504 lambda: self.table.insert(i, entry, True))

505 - def onRemoving(self, i, undoing):

506 if undoing: return 507 entry = {} 508 for f in self.table.fields: 509 entry[f] = self.table.get(i, f) 510 self.undoStack.push_undo(lambda: self.table.insert(i, entry, True), 511 lambda: self.table.remove(i, True))

512 - def onSet(self, i, field, val, prev, undoing):

513 if undoing: return 514 self.undoStack.push_undo(lambda: self.table.set(i, field, prev, True), 515 lambda: self.table.set(i, field, val, True))

516 517

518 -def auto_format(x):

519 try: 520 return '%.3g' % x 521 except: 522 return ''

523

524 -class SimpleTable(Table):

525 """Implements L{Table} by storing all rows and fields internally. 526 527 @ivar entries: list of entries; each entry is a dict from field_name to value 528 """ 529 __slots__ = ['auto_add_fields', 'auto_default', 'auto_accept', 'auto_format', 'entries']

530 - def __init__(self, label, auto_add_fields=False, auto_default=0.0, auto_accept=acceptEval(), auto_format=auto_format, global_name="", sortable=False):

531 Table.__init__(self, sortable=sortable) 532 self.label = label 533 self.global_name = global_name 534 self.auto_add_fields = auto_add_fields 535 self.auto_default = auto_default 536 self.auto_accept = auto_accept 537 self.auto_format = acceptFormat(auto_format) 538 self.entries = [] # of {field_name : value}

539 - def get_size(self):

540 return len(self.entries)

541 - def add_field(self, name, default, accept, format, units, choices=None, independent=False):

542 """Adds a column. 543 544 @param name: column label 545 @param default: default value 546 @param accept: f(str) -> value; can raise Exception on invalid input 547 @param format: f(value) -> str, or a printf-style format string (e.g. "%.6g") 548 @param units: string 549 @param choices: a list of acceptable values, or None to just use the accept function 550 @param independent: True if the field can be used as the independent variable 551 """ 552 if len(self.fields) == self.max_fields: 553 return false 554 self.add_field_to_stats(name) 555 self.fields.append(name) 556 self.default[name] = default 557 self.accept[name] = accept 558 self.format[name] = acceptFormat(format) 559 self.units[name] = units 560 self.choices[name] = choices 561 for e in self.entries: 562 if not (name in e): 563 e[name] = default 564 if independent: 565 self.fields_independent.append(name) 566 self.OnAddField(name) 567 return true

568 - def remove_field(self, name):

569 if name in self.fields: 570 self.OnRemovingField(name) 571 self.fields.remove(name) 572 self.remove_field_from_stats(name) 573 if name in self.fields_independent: 574 self.fields_independent.remove(name)

575 - def insert(self, i, entry, undoing=False):

576 i = min(self.size, max(0, i)) 577 d = {'Index' : i} 578 for f in self.fields[1:]: 579 if entry.has_key(f): 580 d[f] = entry[f] 581 else: 582 d[f] = self.default[f] 583 self.entries.insert(i, d) 584 for stat in self.all_stats: 585 stat.reset() 586 self.OnInsert(i, undoing) 587 for j in xrange(i+1, self.size): 588 self.set(j, 'Index', j) 589 self.OnAfterInsert(i, undoing) 590 if self.auto_add_fields: 591 for k in sorted(entry.keys()): 592 if (k != 'fields') and not (k in self.fields): 593 self.set(i, k, entry[k])

594 - def remove(self, i, undoing=False, renumber=True):

595 if not (0 <= i < self.size): 596 return 597 self.OnRemoving(i, undoing) 598 self.entries[i:i+1] = [] 599 for stat in self.all_stats: 600 stat.reset() 601 self.OnRemoved(i, undoing) 602 if renumber: 603 for j in xrange(i, self.size): 604 self.set(j, 'Index', j)

605 - def get(self, i, field_name):

606 return self.entries[i][field_name]

607 - def set(self, i, field_name, val, undoing=False):

608 if not (field_name in self.fields): 609 if self.auto_add_fields: 610 if not self.add_field(field_name, self.auto_default, self.auto_accept, self.auto_format, ''): 611 return 612 else: 613 raise KeyError('%s is not a field of %s' % (field_name, self.label)) 614 entry = self.entries[i] 615 prev = entry[field_name] 616 entry[field_name] = val 617 if field_name == 'Group': 618 for stat in self.group_stats: 619 stat.reset() 620 for stat in self.field_stats[field_name]: 621 stat.reset() 622 self.OnSet(i, field_name, val, prev, undoing)

623 - def get_row(self, i, safenames=False):

624 """Returns a the i'th L{TableRow}.""" 625 return TableRow(self.entries[i], safenames)

626 - def clone(self):

627 clone = SimpleTable(self.label, self.auto_add_fields, self.auto_default, self.auto_accept, self.auto_format, self.global_name, self.sortable) 628 for field in self.fields: 629 if not (field in ['Index', 'Group']): 630 clone.add_field(field, self.default[field], self.accept[field], self.format[field], 631 self.units[field], self.choices[field]) 632 for i in xrange(self.size): 633 clone.append(self.get_row(i).__dict__) 634 return clone

635 636

637 -class ObjectTable(Table):

638 """Implements L{Table} as a list of pre-existing objects. You define a field with get and set functions on an object.""" 639 __slots__ = ['__ref', 'entries', 'getter', 'setter', '__setting_field']

640 - def __init__(self, label, global_name=""):

641 Table.__init__(self) 642 self.__ref = Reffer() 643 self.label = label 644 self.global_name = global_name 645 self.entries = [] 646 self.getter = {} 647 self.setter = {} 648 self.__setting_field = False

649 - def get_size(self):

650 return len(self.entries)

651 - def add_field(self, name, default, accept, format, units, choices=None, get=get_none, set=set_none, independent=False):

652 """Adds a column. 653 654 @param name: column label 655 @param default: default value 656 @param accept: f(str) -> value; can raise Exception on invalid input 657 @param format: f(value) -> str, or a printf-style format string (e.g. "%.6g") 658 @param units: string 659 @param choices: a list of acceptable values, or None to just use the accept function 660 @param get: f(object) -> value 661 @param set: f(object, value) 662 """ 663 self.add_field_to_stats(name) 664 self.fields[-1:-1] = [name] 665 self.default[name] = default 666 self.accept[name] = accept 667 self.format[name] = acceptFormat(format) 668 self.units[name] = units 669 self.choices[name] = choices 670 self.getter[name] = get 671 self.setter[name] = set 672 if independent: 673 self.fields_independent.append(name) 674 self.OnAddField(name)

675 - def insert(self, i, obj, undoing=False):

676 i = min(self.size, max(0, i)) 677 self.entries.insert(i, obj) 678 try: 679 obj.group 680 except: 681 obj.group = 0 682 for stat in self.all_stats: 683 stat.reset() 684 self.OnInsert(i, undoing) 685 for j in xrange(i+1, self.size): 686 self.OnSet(j, 'Index', j, j-1, undoing) 687 obj.OnChangeField += self.__ref(self._onChangeField) 688 self.OnAfterInsert(i, undoing)

689 - def remove(self, i, undoing=False):

690 if not (0 <= i < self.size): 691 return 692 self.entries[i].OnChangeField -= self.__ref(self._onChangeField) 693 self.OnRemoving(i, undoing) 694 self.entries[i:i+1] = [] 695 for stat in self.all_stats: 696 stat.reset() 697 self.OnRemoved(i, undoing) 698 for j in xrange(i, self.size): 699 self.OnSet(j, 'Index', j, j+1, undoing)

700 - def get(self, i, field_name):

701 obj = self.entries[i] 702 if field_name == 'Index': 703 return i 704 elif field_name == 'Group': 705 return obj.group 706 else: 707 return self.getter[field_name](obj)

708 - def set(self, i, field_name, val, undoing=False):

709 self.__setting_field = True 710 prev = self.get(i, field_name) 711 obj = self.entries[i] 712 if field_name == 'Group': 713 obj.group = val 714 for stat in chain(self.group_stats, self.field_stats[field_name]): 715 stat.reset() 716 self.OnSet(i, 'Group', val, prev, undoing) 717 elif self.setter[field_name]: 718 try: 719 self.setter[field_name](obj, self.accept[field_name](val)) 720 for stat in self.field_stats[field_name]: 721 stat.reset() 722 self.OnSet(i, field_name, val, prev, undoing) 723 except FieldAcceptException: 724 pass 725 self.__setting_field = False

726 - def _onChangeField(self, obj, field_name):

727 if self.__setting_field: return 728 i = self.entries.index(obj) 729 val = self.get(i, field_name) 730 for stat in self.field_stats[field_name]: 731 stat.reset() 732 self.OnSet(i, field_name, val, val, False)

733 734

735 -class TextStats(object):

736 __slots__ = ['tokenize', 'convert', 'debug', 'min_fields', 'max_fields', 'numbers', 'headers', 'skip', 737 'col_count', 'signal_count']

738 - def __init__(self, tokenize, convert, debug=False):

739 self.tokenize = tokenize 740 self.convert = convert 741 self.debug = debug 742 self.min_fields = None 743 self.max_fields = 0 744 self.numbers = 0 745 self.headers = [] 746 self.skip = 0

747 - def score(self):

748 self.col_count = self.max_fields 749 self.signal_count = self.col_count 750 return (self.max_fields - (self.min_fields or 0)) + self.numbers + int(bool(self.headers))

751 - def finish(self):

752 self.col_count = self.max_fields 753 self.signal_count = self.col_count

754 - def add(self, line):

755 fields = self.tokenize(line) 756 if len(fields) and fields[0]: 757 if (self.min_fields is None) or (self.min_fields > len(fields)): 758 self.min_fields = len(fields) 759 self.max_fields = max(self.max_fields, len(fields)) 760 if self.debug: 761 print fields 762 anything = False 763 anynum = 0 764 nonempty = [x for x in [xx.strip() for xx in fields] if x] 765 for x in nonempty: 766 anything = True 767 try: 768 self.convert(x) 769 anynum += 1 770 except ValueError: 771 pass 772 if not anything: 773 if (not self.numbers) and (not self.headers): 774 #print 'skipe',line 775 self.skip += 1 776 elif (anynum < len(nonempty)) and (self.numbers == 0): 777 if not self.headers: 778 #print 'skipp',line 779 self.headers = fields 780 self.skip += 1 781 self.numbers += anynum 782 elif not self.numbers: 783 #print 'skipf',fields,line 784 self.skip += 1

785 786

787 -def float_decimal_comma(s):

788 return float(s.replace(',', '.'))

789

790 -def read_table_text(txt):

791 """Returns headers, types, columns.""" 792 tok_tab = lambda s: s.split('\t') 793 re_comma = re.compile(r", *") 794 tok_comma = lambda s: re_comma.split(s) 795 formats = [TextStats(tok_tab, float), TextStats(tok_tab, float_decimal_comma), 796 TextStats(tok_comma, float), TextStats(tok_comma, float_decimal_comma)] 797 lines = txt.split('\n') 798 #print lines 799 for line in lines[:min(len(lines), 10)]: 800 for format in formats: 801 format.add(line) 802 if not [format for format in formats if format.numbers]: 803 return [], [], [] 804 scores = numpy.array([format.score() for format in formats]) 805 format = formats[numpy.argmax(scores)] 806 iline = 0 807 tokenize = format.tokenize 808 convert = format.convert 809 810 headers = format.headers 811 while len(headers) < format.max_fields: 812 headers.append("") 813 columns = [[] for x in headers] 814 typmap = collections.defaultdict(lambda: float) 815 typmap['Index'] = typmap['Group'] = int 816 types = [typmap[x] for x in headers] # [int for x in headers] 817 for line in lines[format.skip:]: 818 tokens = [t.strip() for t in tokenize(line)] 819 if not any(tokens): 820 continue 821 while len(tokens) < len(columns): 822 tokens.append('') 823 for i, tok in enumerate(tokens): 824 if not tok.strip(): 825 columns[i].append(types[i]()) # zero 826 else: 827 try: 828 num = convert(tok) 829 #if (types[i] != str) and (num != int(num)): 830 # types[i] = float 831 columns[i].append(convert(tok)) 832 except ValueError: 833 types[i] = str 834 columns[i].append(tok) 835 columns = [ [typ(val) for val in col] for col,typ in izip(columns, types) ] 836 return headers, types, columns

837

838 -def default_column_name(i):

839 carry = i / 26 840 ones = i % 26 841 toBase26 = lambda d: chr(ord('A')+d) 842 if carry: 843 return toBase26(carry-1)+toBase26(ones) 844 else: 845 return toBase26(ones)

846

Source Code for Module qubx.table