qubx.tree

1 """Classes for reading and writing all QuB binary files. 2 3 A QuB Tree is a hierarchy of Nodes. Each node may have a name and some text or numeric data. 4 Specific QuB file types, e.g. QMF (qub model files), expect specific heirarchies and data types. 5 6 History: 7 8 In the late 1990s Dr. Lorin Milescu defined the binary layout and used it for QuB data files (QDF). 9 He wrote a class in Delphi pascal called "QFS" to read and modify open files on disk. 10 11 By the early 2000s we were collaborating on a DLL interface between his GUI, written in Delphi, and 12 older algorithms written in C. We found ourselves spending a lot of time defining struct- and pointer-based 13 containers for all the varieties of information shared between GUI and DLL. At the same time, 14 folks were asking us for new extensible formats for models and idealized data. Lorin naturally proposed 15 QFS. I was feeling some pressure to use human-readable ASCII text, so I proposed a file that could 16 be stored as binary or text, depending whether efficiency or transparency was foremost. Then I went 17 ahead and implemented it in C/C++, with bindings to Delphi. I never really got the text part right, but 18 no-one really cared. 19 20 So now we had two implementations: QFS, and "qubtree" or "native". We had some idea of changing 21 everything over to the newer one, but it would have meant a ton of extra work re-implementing the core of QuB. 22 When I made the Delphi bindings, I had our DLL quagmire in mind. qubtree.dll took care of memory management 23 and reference counting so you could pass trees willy-nilly between the EXE and DLL without headaches. And now 24 that everything boiled down to one or two ref-counted qubtree pointers, we weren't constantly breaking binary 25 compatibility between DLL and EXE. Similarly, I added Python bindings and rudimentary callback support, so we 26 could do a lot of stuff across the three languages without too much intricate boilerplate. 27 28 This unit can interface to the DLL/.so, and it introduces a third implementation -- in Python, using numpy and mmap. 29 This 'numpy' flavor behaves exactly like the 'native' one, except: 30 - it doesn't even try to ReadString or ReadText; like I said, always broken anyway 31 - it ignores lock/unlock since python's single-threaded for ref-counting purposes 32 - you can't edit string data one character at a time, due to a limitation in module mmap 33 - you can get direct access to numeric data as numpy.array: node.storage.data 34 - you can resize a node with empty data, by specifying a default type 35 36 37 Flavors: 38 39 By default this module still uses the 'native' flavor, so there are no surprises for existing programs. 40 If you don't have the compiled _qubtree library, it falls back on the 'numpy' flavor. Of course, 41 you must have numpy, available separately. If you have both, you can switch the default for new nodes: 42 43 >>> qubx.tree.CHOOSE_FLAVOR('numpy') 44 45 and even convert a tree between the two flavors: 46 47 >>> as_numpy = qubx.tree.AS_FLAVOR(native_node, 'numpy') 48 49 50 Limitations: 51 52 32 bits, except when signed/unsigned are treated inconsistently; then it's more like 31. 53 (max safe file size 2GB - 1) 54 55 Supported by no-one else 56 57 Not very efficient when you have lots of nodes. 58 59 60 U{Older qubtree documentation<http://www.qub.buffalo.edu/qubdoc>} 61 62 63 Copyright 2007-2014 Research Foundation State University of New York 64 This file is part of QUB Express. 65 66 QUB Express is free software; you can redistribute it and/or modify 67 it under the terms of the GNU General Public License as published by 68 the Free Software Foundation, either version 3 of the License, or 69 (at your option) any later version. 70 71 QUB Express is distributed in the hope that it will be useful, 72 but WITHOUT ANY WARRANTY; without even the implied warranty of 73 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 74 GNU General Public License for more details. 75 76 You should have received a copy of the GNU General Public License, 77 named LICENSE.txt, in the QUB Express program directory. If not, see 78 <http://www.gnu.org/licenses/>. 79 80 """ 81 82 import struct 83 import array 84 import weakref 85 import mmap 86 import cStringIO as StringIO 87 import operator 88 import time 89 import traceback 90 import os 91 from itertools import izip 92 import collections 93 import sys 94 95 FLAVOR_DEFAULT = 'native' 96 FLAVORS_AVAIL = [] 97 FLAVOR = FLAVOR_DEFAULT 98 99 have_lib = False 100 try: 101 import _qubtree 102 have_lib = True 103 except: 104 try: 105 import qubx.tree_native as _qubtree 106 have_lib = True 107 PNode = _qubtree.PNode 108 except: 109 traceback.print_exc() 110 pass 111 if have_lib: 112 #print 'have native qubtree' 113 FLAVORS_AVAIL.append('native') 114 Node = _qubtree.Node 115 NullNode = _qubtree.NullNode 116 Open = _qubtree.Open 117 ReadBytes = _qubtree.ReadBytes 118 119 try: 120 import numpy 121 have_numpy = True 122 FLAVORS_AVAIL.append('numpy') 123 except: 124 have_numpy = False 125

126 -def CHOOSE_FLAVOR(flav):

127 """Selects the default flavor for new Nodes, either 'native' (default) or 'numpy'.""" 128 global Node, NullNode, Open, ReadBytes, FLAVOR 129 if flav == 'native': 130 Node = _qubtree.Node 131 NullNode = _qubtree.NullNode 132 Open = _qubtree.Open 133 ReadBytes = _qubtree.ReadBytes 134 elif flav == 'numpy': 135 Node = Node_numpy 136 NullNode = NullNode_numpy 137 Open = Open_numpy 138 ReadBytes = ReadBytes_numpy 139 FLAVOR = flav

140

141 -def AS_FLAVOR(node, flav, copyAnyway=False):

142 """Returns node in the requested flavor. If it is already, we can return the original (default) or copyAnyway.""" 143 if have_lib and ((0 <= str(type(node)).find('QUBTree')) or (0 <= str(type(node)).find('native.Node'))): 144 if flav == 'native': 145 if copyAnyway: 146 return node.clone() 147 else: 148 return node 149 elif flav == 'numpy': 150 if node.isNull: 151 return NullNode_numpy() 152 else: 153 return ReadBytes_numpy(node.getBytes()) 154 elif have_numpy and isinstance(node, NullNode_numpy): 155 if flav == 'numpy': 156 return node 157 elif flav == 'native': 158 return _qubtree.NullNode() 159 elif have_numpy and isinstance(node, Node_numpy): 160 if flav == 'numpy': 161 if copyAnyway: 162 return node.clone() 163 else: 164 return node 165 elif flav == 'native': 166 return _qubtree.ReadBytes(node.getBytes())

167 168 # node.data.type constants 169 170 QTR_TYPE_EMPTY, \ 171 QTR_TYPE_UNKNOWN, \ 172 QTR_TYPE_POINTER, \ 173 QTR_TYPE_STRING, \ 174 QTR_TYPE_UCHAR, \ 175 QTR_TYPE_CHAR, \ 176 QTR_TYPE_USHORT, \ 177 QTR_TYPE_SHORT, \ 178 QTR_TYPE_UINT, \ 179 QTR_TYPE_INT, \ 180 QTR_TYPE_ULONG, \ 181 QTR_TYPE_LONG, \ 182 QTR_TYPE_FLOAT, \ 183 QTR_TYPE_DOUBLE, \ 184 QTR_TYPE_LDOUBLE = range(15) 185 TYPES = (QTR_TYPE_EMPTY, 186 QTR_TYPE_UNKNOWN, 187 QTR_TYPE_POINTER, 188 QTR_TYPE_STRING, 189 QTR_TYPE_UCHAR, 190 QTR_TYPE_CHAR, 191 QTR_TYPE_USHORT, 192 QTR_TYPE_SHORT, 193 QTR_TYPE_UINT, 194 QTR_TYPE_INT, 195 QTR_TYPE_ULONG, 196 QTR_TYPE_LONG, 197 QTR_TYPE_FLOAT, 198 QTR_TYPE_DOUBLE, 199 QTR_TYPE_LDOUBLE) 200 201 # for human consumption: 202 TYPENAMES = {QTR_TYPE_EMPTY: "empty", 203 QTR_TYPE_UNKNOWN: "unknown", 204 QTR_TYPE_POINTER : "pointer", 205 QTR_TYPE_STRING : "string", 206 QTR_TYPE_UCHAR : "unsigned 8-bit int", 207 QTR_TYPE_CHAR : "signed 8-bit int", 208 QTR_TYPE_USHORT : "unsigned 16-bit int", 209 QTR_TYPE_SHORT : "signed 16-bit int", 210 QTR_TYPE_UINT : "unsigned int", 211 QTR_TYPE_INT : "int", 212 QTR_TYPE_ULONG : "unsigned 64-bit int", 213 QTR_TYPE_LONG : "signed 64-bit int", 214 QTR_TYPE_FLOAT : "single-precision float", 215 QTR_TYPE_DOUBLE : "float", 216 QTR_TYPE_LDOUBLE : "extended-precision float" 217 } 218 219 # for resizing: 220 TYPE_DEFAULTS = (None, None, None, ' ', 0, 0, 0, 0, 0, 0, 0, 0, 0.0, 0.0, 0.0) 221 222 # size of one element, in bytes 223 SIZE_OF_TYPE = (0, 0, 4, 1, 1, 1, 2, 2, 4, 4, 8, 8, 4, 8, 10) 224 225 # iteration, since it's too late to build in as a method 226

227 -def children(tree, name=None):

228 """Yields all children of tree, matching name if specified.""" 229 if name is None: 230 child = tree.child 231 else: 232 child = tree.find(name) 233 while not child.isNull: 234 yield child 235 if name is None: 236 child = child.sibling 237 else: 238 child = child.nextSameName()

239

240 -def tree_to_dict(tree, is_root=True):

241 d = dict( (child.name, tree_to_dict(child, False)) for child in children(tree) ) 242 data = treedata_to_dict(tree.data) 243 if not (data is None): 244 d['__data'] = data 245 if is_root: 246 d['__root_name'] = tree.name 247 return d 248 elif is_root or not tree.child.isNull: 249 return d 250 elif '__data' in d: 251 return d['__data'] 252 else: 253 return None

254

255 -def treedata_to_dict(data):

256 if not data: 257 return None 258 if data.type == QTR_TYPE_STRING: 259 return str(data) 260 if data.count == 1: 261 return data[0] 262 else: 263 return data[:]

264

265 -def dict_to_tree(d, is_root=True, name=None):

266 nm = d['__root_name'] if is_root else name 267 tree = Node(nm) 268 if not isinstance(d, dict): 269 tree.data = d 270 else: 271 if '__data' in d: 272 tree.data = d['__data'] 273 child = NullNode() 274 exclude = set(['__root_name', '__data']) 275 for k in sorted(d.keys()): 276 if not (k in exclude): 277 child = tree.insert(dict_to_tree(d[k], False, k), child) 278 return tree

279 280

281 -def node_data_or_def(node, default, index=None):

282 """Returns the numeric or string data of a L{qubx.tree.Node_numpy}, or C{default} if the node has no data. 283 284 Suitable for reading preferences that might be missing. 285 """ 286 if not (index is None): 287 if node.data.count > index: 288 return node.data[index] 289 else: 290 if node.data.type == QTR_TYPE_STRING: 291 return str(node.data) 292 elif node.data.count: 293 return node.data[0] 294 return default

295 296

297 -def node_data_or_set_def(parent, child_name, default):

298 """Returns parent[child_name].data[0] or default; sets default in tree if missing.""" 299 node = parent[child_name] 300 if node.data: 301 return node.data[0] 302 else: 303 node.data = default 304 return default

305 306 307 308 # reading raw nodes: 309 310 framedef = "<BBBBiiiiiiBBBB" 311 framelen = struct.calcsize(framedef) 312 frame_dataoff = 12 313 314 QTR_FLAG_PRELOAD = 0x2 315 QTR_FLAG_DATA_IN_NODE = 0x4 316 QTR_MAGIC = "QUB_(;-)_QFS" 317

318 -class dtree(object):

319 """A light intermediary between a file and a reader of qubtree nodes, for debugging.""" 320

321 - def __init__(self, fi):

322 """Sets up a dtree reader given an open file fi.""" 323 self.fi = fi

324 - def rawframe(self, off):

325 """Returns the tuple of all fields of a qubtree node, located at offset off, except the name. 326 327 @return: (flags, a, b, type, size, count, dataPos, childPos, siblingPos, c, d, e, f, nameLen) 328 where a thru f are reserved (ignored) bytes. 329 """ 330 self.fi.seek(off) 331 dat = self.fi.read(framelen) 332 fdef = framedef 333 while fdef: 334 try: 335 return struct.unpack(fdef, dat) 336 except: 337 traceback.print_exc() 338 return tuple()

339 - def frame(self, off):

340 """Returns the tuple of all fields of a qubtree.Node including the name, with flags upacked. 341 342 @return: (preload, data_in_node, type, size, count, dataPos, childPos, siblingPos, name) 343 """ 344 raw = list(self.rawframe(off)) 345 raw = raw + [None] * (14-len(raw)) 346 flags, a, b, type, size, count, dPos, cPos, sPos, c, d, e, f, nameLen = raw 347 preload = bool(flags and (flags & 2)) 348 dataIn = bool(flags and (flags & 4)) 349 if not (type is None): 350 type = [None, None, None, "s", "B", 'b', 'H', 'h', 'I', 'i', 'L', 'l', 'f', 'd', None][type] 351 name = None 352 if nameLen == 0: 353 name = "" 354 elif nameLen: 355 try: 356 self.fi.seek(off+framelen) 357 name = self.fi.read(nameLen) 358 except: 359 pass 360 return (preload, dataIn, type, size, count, dPos, cPos, sPos, name)

361 - def hier(self, off=12, indent=""):

362 """Prints the tree of frames rooted at offset off, with indentation to show hierarchy.""" 363 frame = self.frame(off) 364 print indent,'%d:'%off,frame 365 if frame[6]: 366 self.hier(frame[6], indent+' ') 367 if frame[7]: 368 self.hier(frame[7], indent)

369 - def data(self, off, count, format):

370 """Prints the array of count number of items at offset off, with type given as a format string such as in module struct.""" 371 self.fi.seek(off) 372 arr = array.array(format) 373 arr.fromfile(self.fi, count) 374 return arr

375 376 377 # numpy flavor: 378 379 if have_numpy: 380 381 # mapping between numpy and qubtree types: 382 TYPES_NUMPY = [None, 383 None, 384 numpy.dtype('<u4'), 385 numpy.dtype('<u1'), 386 numpy.dtype('<u1'), 387 numpy.dtype('<i1'), 388 numpy.dtype('<u2'), 389 numpy.dtype('<i2'), 390 numpy.dtype('<u4'), 391 numpy.dtype('<i4'), 392 numpy.dtype('<u8'), 393 numpy.dtype('<i8'), 394 numpy.dtype('<f4'), 395 numpy.dtype('<f8'), 396 numpy.dtype('complex128')] # 'float80'] 397 398 399 TYPE_OF_NUMPY = collections.defaultdict(lambda:QTR_TYPE_DOUBLE) 400 for i in xrange(QTR_TYPE_STRING, len(TYPES_NUMPY)): 401 TYPE_OF_NUMPY[str(TYPES_NUMPY[i])] = i 402 TYPE_OF_NUMPY['bool'] = QTR_TYPE_INT 403 404 MAX_QUICK_READ_BYTES = 256*256*8 # 512KiB 405 MAX_QUICK_WRITE_BYTES = 10 * 256*256*16 # 10MiB 406 QTR_INITIAL_CAP = MAX_QUICK_READ_BYTES 407 ZERO_BATCH = 256*256*4 408

409 - class Truncation(Exception):

410 - def __init__(self, offset):

411 Exception.__init__(self, 'Unexpected EOF in node at offset %i' % offset) 412 self.offset = offset

413 414 # node.data when node.isNull:

415 - class NullData(object):

416 """Represents the data of a null node, which is empty and will stay empty."""

417 - def __repr__(self):

418 return '<QUBTree NULL data>'

419 - def __str__(self):

420 return ''

421 - def __len__(self):

422 return 0

423 - def __getitem__(self, key):

424 raise IndexError()

425 - def __setitem__(self, key, val):

426 raise IndexError()

427 - def get_node(self):

428 return NullNode_numpy_instance

429 node = property(get_node, doc="the null node whose data this is.")

430 - def set_none(self, x):

431 pass

432 preload = property(lambda self: True, set_none) 433 count = property(__len__) 434 rows = property(lambda self: 0) 435 cols = property(lambda self: 0) 436 type = property(lambda self: QTR_TYPE_EMPTY) 437 size = property(lambda self: 0) 438 loaded = property(lambda self: (-1, -1)) 439 loadedRows = property(lambda self: (-1, -1))

440 - def setup(self, type, rows, cols):

441 pass

442 - def resize(self, nr):

443 pass

444 - def clear(self):

445 pass

446 - def loadRows(self, first, last, do_read=True):

447 pass

448 - def unloadRows(self, do_write=True):

449 pass

450 - def row(self, i):

451 raise IndexError()

452 - def col(self, j):

453 raise IndexError()

454

455 - class NullNode_numpy(object):

456 """Represents the absence of a qubtree node. 457 458 Has all the properties and methods, so you can e.g. ask blithely if it has any data or children (no). 459 """ 460 __slots__ = ['__data', '__weakref__']

461 - def __init__(self):

462 self.__data = NullData()

463 - def __repr__(self):

464 return '<QUBTree Node : NULL>'

465 - def __str__(self):

466 return '\n'

467 - def __cmp__(self, other):

468 try: 469 if other.isNull: 470 return 0 471 else: 472 return -1 473 except KeyboardInterrupt: 474 raise 475 except: 476 return -1

477 - def __nonzero__(self):

478 return False # a null node is Falsy

479 - def __len__(self):

480 return 0

481 - def __getitem__(self, key):

482 return self

483 - def set_none(self, x):

484 pass

485 name = property(lambda self: '', set_none) 486 data = property(lambda self: self.__data, set_none) 487 lineComment = property(lambda self: '', set_none) 488 isNull = property(lambda self: True) 489 child = property(lambda self: self) 490 sibling = property(lambda self: self) 491 parent = property(lambda self: self) 492 path = property(lambda self: '') 493 modified = property(lambda self: False) 494 root = property(lambda self: self)

495 - def clone(self, deep=True):

496 return self

497 - def saveAs(self, path):

498 return False

499 - def saveAsText(self, path):

500 return False

501 - def save(self):

502 return False

503 - def close(self):

504 return False

505 - def getBytes(self):

506 return ''

507 - def list(self):

508 return []

509 - def find(self, name):

510 return self

511 - def next(self, name):

512 return self

513 - def nextSameName(self):

514 return self

515 - def append(self, childOrName):

516 return self

517 - def appendClone(self, other, deep=True):

518 return self

519 - def insert(self, child, after=None):

520 return self

521 - def insertClone(self, other, deep=True, after=None):

522 return self

523 - def remove(self, child):

524 pass

525 - def setChanged(self):

526 pass

527 - def lock(self, timeoutMS=None):

528 pass

529 - def unlock(self):

530 pass

531 532 # from here on out we try to use this instance instead of constructing a flock of identical Nulls. 533 NullNode_numpy_instance = NullNode_numpy() 534 535 # generic data wrappers 536 # These three (Data, DataRow, DataCol) help read and manipulate node data, with 537 # the same API as the 'native' flavor. 538 # The real work happens in the storage and Node classes below.

539 - class Data(object):

540 __slots__ = ['__node'] 541 """A handler for the data in one qubtree node. Don't construct one for yourself; use node.data"""

542 - def __init__(self, node):

543 """Constructs a data handler for a qubtree node. Don't call this directly.""" 544 self.__node = weakref.ref(node)

545 - def __repr__(self):

546 return '<QUBTree data for %s>' % str(self.node).strip()

547 - def __str__(self):

548 if self.node.isNull: 549 return '' 550 elif self.type == QTR_TYPE_STRING: 551 if self.node.storage.data[0] == '\0': 552 return '' 553 else: 554 return str(self.node.storage.data[:]) 555 elif self.cols > 1 and self.rows > 1: 556 return '(%s)' % '\\\n'.join([' '.join([x for x in self.row(r)]) for r in xrange(self.rows)]) 557 elif self.count: 558 return ' '.join(str(x) for x in self) 559 else: 560 return ''

561 - def __len__(self):

562 return (not self.node.isNull) and self.node.storage.dataCount or 0

563 - def __getitem__(self, key):

564 s = self.node.storage 565 n = s.dataCount 566 if isinstance(key, slice): 567 return [s.get_data_item(i) for i in xrange(*key.indices(n))] 568 elif -n <= key < 0: 569 return s.get_data_item(n+key) 570 elif 0 <= key < n: 571 return s.get_data_item(key) 572 else: 573 raise IndexError()

574 - def __setitem__(self, key, val):

575 s = self.node.storage 576 n = s.dataCount 577 if isinstance(key, slice): 578 for i, v in izip(xrange(*key.indices(n)), val): 579 s.set_data_item(i, v) 580 elif -n <= key < 0: 581 return s.set_data_item(n+key, val) 582 elif 0 <= key < n: 583 return s.set_data_item(key, val) 584 else: 585 raise IndexError()

586 - def get_node(self):

587 node = self.__node() 588 if not node: 589 node = NullNode_numpy_instance 590 self.__node = weakref.ref(node) 591 return node

592 node = property(get_node, doc="the qubtree node whose data this is") 593 preload = property(lambda self: self.node.storage.preload, lambda self, x: self.node.storage.set_preload(x), 594 doc="whether all data is loaded into memory on file open") 595 count = property(__len__, doc="total data elements (rows * cols)") 596 rows = property(lambda self: self.count and self.node.storage.rows, doc="number of rows") 597 cols = property(lambda self: self.count and self.node.storage.cols, doc="number of columns") 598 type = property(lambda self: self.count and self.node.storage.dataType, doc="type of data elements, in QTR_TYPE_*") 599 size = property(lambda self: self.count and self.node.storage.dataSize, doc="size of one data element in bytes") 600 loaded = property(lambda self: self.node.storage.loaded, doc="(first, last) element loaded into memory") 601 loadedRows = property(lambda self: self.node.storage.loadedRows, doc="(first, last) row loaded into memory")

602 - def setup(self, type, rows, cols):

603 """Discards any data and sets up space for rows*cols of the specified type.""" 604 self.node.storage.setup_data(type, rows, cols)

605 - def resize(self, nr, type_if_none=QTR_TYPE_DOUBLE):

606 """Changes the number of rows allocated.""" 607 self.node.storage.resize_data(nr, type_if_none)

608 - def clear(self):

609 """Discards any data.""" 610 self.node.storage.clear_data()

611 - def loadRows(self, first=-2, last=-2, do_read=True):

612 """Loads the rows from first up to and including last into memory. By default loads all. 613 614 In this numpy flavor, all data is always loaded (memory mapped), and loadRows just adjusts .loadedRows.""" 615 if first == -2: 616 if self.rows: 617 f = 0 618 else: 619 f = -1 620 else: 621 f = first 622 if last == -2: 623 if self.count: 624 l = self.rows - 1 625 else: 626 l = -1 627 else: 628 l = last 629 self.node.storage.load_rows(f, l, do_read)

630 - def unloadRows(self, do_write=True):

631 """Unloads any loaded data. 632 633 In this numpy flavor, all data is always loaded (memory mapped), and unloadRows just adjusts .loadedRows.""" 634 self.node.storage.unload_rows(do_write)

635 - def read_rows(self, first=None, last=None):

636 """Returns a copy of the data in rows first..last""" 637 f = 0 if (first is None) else first 638 l = (self.rows-1) if (last is None) else last 639 return self.node.storage.read_rows(f, l)

640 - def write_rows(self, first, last, data):

641 """Replaces the data in rows first..last with the given data.""" 642 self.node.storage.write_rows(first, last, data)

643 - def get_rows(self, first, last):

644 """Returns a slice into the first..last rows of data. If possible, it will be a direct, mutable reference. 645 This may not be possible with some string data.""" 646 return self.node.storage.get_rows(first, last)

647 - def row(self, i):

648 """Returns an object that can manipulate the data in row i, and acts like a list.""" 649 if 0 <= i < self.rows: 650 return DataRow(self, i) 651 raise IndexError()

652 - def col(self, j):

653 """Returns an object that can manipulate the data in column j, and acts like a list.""" 654 if 0 <= self.cols: 655 return DataCol(self, j) 656 raise IndexError()

657

658 - class DataRow(object):

659 """A handler for the data in one row of a qubtree node. Don't construct one for yourself; use node.data.row(i)""" 660 __slots__ = ['data', 'r']

661 - def __init__(self, data, r):

662 """Don't call this directly.""" 663 self.data = data 664 self.r = r

665 - def __repr__(self):

666 return '<QUBTree data for %s, row %i>' % (self.data.node.name, self.r)

667 - def __str__(self):

668 return ' '.join([x for x in self])

669 - def __len__(self):

670 return self.data.cols

671 - def __getitem__(self, key):

672 r = self.r 673 d = self.data 674 l = d.cols 675 s = r * l 676 if isinstance(key, slice): 677 return [d[s+i] for i in xrange(*key.indices(l))] 678 elif -l <= key < 0: 679 return d[s+l+key] 680 elif 0 <= key < l: 681 return d[s+key] 682 else: 683 raise IndexError()

684 - def __setitem__(self, key, val):

685 r = self.r 686 d = self.data 687 l = d.cols 688 s = r * l 689 if isinstance(key, slice): 690 for i,v in izip(xrange(*key.indices(l)), val): 691 d[s+i] = v 692 elif -l <= key < 0: 693 d[s+l+key] = val 694 elif 0 <= key < l: 695 d[s+key] = val 696 else: 697 raise IndexError()

698

699 - class DataCol(object):

700 """A handler for the data in one column of a qubtree node. Don't construct one for yourself; use node.data.col(j)""" 701 __slots__ = ['data', 'c']

702 - def __init__(self, data, c):

703 """Don't call this directly.""" 704 self.data = data 705 self.c = c

706 - def __repr__(self):

707 return '<QUBTree data for %s, col %i>' % (self.data.node.name, self.c)

708 - def __str__(self):

709 return ' '.join([x for x in self])

710 - def __len__(self):

711 return self.data.rows

712 - def __getitem__(self, key):

713 c = self.c 714 d = self.data 715 l = d.rows 716 nc = d.cols 717 if isinstance(key, slice): 718 return [d[i*nc+c] for i in xrange(*key.indices(l))] 719 elif -l <= key < 0: 720 return d[(l+key)*nc+c] 721 elif 0 <= key < l: 722 return d[key*nc+c] 723 else: 724 raise IndexError()

725 - def __setitem__(self, key, val):

726 c = self.c 727 d = self.data 728 l = d.rows 729 nc = d.cols 730 if isinstance(key, slice): 731 for i,v in izip(xrange(*key.indices(l)), val): 732 d[i*nc+c] = v 733 elif -l <= key < 0: 734 d[(l+key)*nc+c] = val 735 elif 0 <= key < l: 736 d[key*nc+c] = val 737 else: 738 raise IndexError()

739 740 # clean vs modified: 741 # - in the native flavor's C code, these are the flags "changed" vs "child_changed" 742 # - node.storage.clean tells if the node is saved on disk, and completely up to date. It's true only: 743 # + in an open file 744 # + after Opening 745 # + after saving 746 # + before changing this node's data, child, or sibling 747 # - actually, it may be fraudulently True if e.g. the child's name was changed 748 # + name change usually means the node has to move 749 # + this node doesn't get the new child address until the "pre-save" linkup 750 # + pre-save happens automatically, right before saving, and when checking node.modified 751 # - node.modified tells if this node or any descendent is not clean. 752 # - "storage" doesn't have access to the Node object or its parent, so it signals "newly unclean" by a callback event 753 # - the callback is weak-referenced to avoid circular reference 754 # - if there is no callback provided, it calls this function:

755 - def ignore_dirtied(storage):

756 pass

757 758 # abstract storage

759 - class NodeStorage(object):

760 """Abstract base class for managing a concrete representation of one qubtree node. 761 762 Subclasses NodeInFile and NodeInMem are instantiated as needed by class Node.""" 763 __slots__ = ['__clean', 'OnDirtied', '__dirtied'] 764 clean = property(lambda self: self.__clean, lambda self, x: self.set_clean(x), 765 doc="whether this node has an up-to-date representation on disk.") 766 OnDirtied = property(lambda self: self.get_ondirtied(), lambda self, x: self.set_ondirtied(x), 767 "client-provided f(storage) is called when clean becomes False.") 768 name = property(lambda self: self.get_name(), lambda self, x: self.set_name(x), doc="the node's name") 769 preload = property(lambda self: self.get_preload(), lambda self, x: self.set_preload(x), 770 doc="whether to load all data elements on file open") 771 dataCount = property(lambda self: self.get_dataCount(), doc="total number of data elements; rows*cols") 772 dataSize = property(lambda self: self.get_dataSize(), doc="size of one data element, in bytes") 773 dataType = property(lambda self: self.get_dataType(), doc="type of data, in QTR_TYPE_*") 774 dataPos = property(lambda self: self.get_dataPos(), doc="position of data in a file, in bytes") 775 childPos = property(lambda self: self.get_childPos(), lambda self, x: self.set_childPos(x), 776 doc="position of child node in file, in bytes, or 0") 777 siblingPos = property(lambda self: self.get_siblingPos(), lambda self, x: self.set_siblingPos(x), 778 doc="position of sibling node in file, in bytes, or 0") 779 rows = property(lambda self: self.get_rowcount(), doc="number of rows of data") 780 cols = property(lambda self: self.get_colcount(), doc="number of columns of data") 781 loaded = property(lambda self: self.get_loaded(), doc="indices of (first, last) loaded rows") 782 loadedRows = property(lambda self: self.get_loadedRows(), doc="indices of (first, last) loaded data elements") 783 data = property(lambda self: self.get_data(), doc="the actual data, mutable if possible. Depending on data type and storage type, can be an mmap.mmap, a slice of an mmap.mmap, a numpy.array, a numpy array onto a slice of an mmap.mmap, or a plain string.") 784 file = property(lambda self: self.get_file(), doc="the NodeFile containing this node, or None") 785 offset = property(lambda self: self.get_offset(), doc="the location of this node (frame) in file, in bytes")

786 - def __init__(self):

787 self.__clean = False # dirty unless it's on disk and up-to-date 788 self.OnDirtied = None

789 - def set_clean(self, x):

790 """Called by subclass methods when they make a change to the node structure from what's on disk.""" 791 if x == self.__clean: return 792 self.__clean = x 793 if not x: 794 self.OnDirtied(self)

795 - def get_ondirtied(self): # the OnDirtied callback, for when it flips from clean to unclean

796 return self.__dirtied() or ignore_dirtied

797 - def set_ondirtied(self, x):

798 if x: 799 self.__dirtied = weakref.ref(x) 800 else: 801 self.__dirtied = weakref.ref(ignore_dirtied)

802 - def get_data_item(self, i):

803 pass

804 - def set_data_item(self, i, x):

805 pass

806 - def get_dataPos(self):

807 return 0 # position in bytes of the start of data, or 0 if none; only relevant in a file

808 - def get_childPos(self):

809 return 0 # position in bytes of the child's node structure; only relevant in a file

810 - def set_childPos(self, x):

811 pass

812 - def get_siblingPos(self):

813 return 0 # position in bytes of the sibling's node structure; only relevant in a file

814 - def set_siblingPos(self, x):

815 pass

816 - def setup_data(self, type, rows, cols):

817 pass # allocate storage, and put an array, mmap-slice or string in self.data

818 - def resize_data(self, nr, type_if_none):

819 pass # change number of rows

820 - def clear_data(self):

821 pass

822 - def map_data(self):

823 pass # in file only: reconnect numpy -> mmap

824 - def set_string_data(self, s):

825 pass # special bypass since you can't first allocate then slice-assign string data

826 - def load_rows(self, first, last, do_read):

827 pass # or since we'll have them all loaded anyway, just change self.loaded*

828 - def unload_rows(self, do_write):

829 pass # make self.loaded* = (-1, -1)

830 - def read_rows(self, first, last):

831 pass # return a copy of the first..last rows

832 - def write_rows(self, first, last, arr):

833 pass # copy arr into the first..last rows

834 - def get_rows(self, first, last):

835 pass # return a slice into the actual rows if possible, or a copy

836 - def get_file(self):

837 return None # NodeFile, if on disk

838 - def get_offset(self):

839 return 0 # position in bytes of the node structure; only relevant in a file

840 - def read(self):

841 pass # if on disk, read node structure and set up self.data

842 - def write(self):

843 pass # if on disk, write node structure

844 - def measure(self):

845 """Returns the number of bytes required for this node and its data (not children etc).""" 846 return framelen + len(self.name) \ 847 + ((not can_data_in_node(self.dataType, self.dataCount)) and 1 or 0) * \ 848 (SIZE_OF_TYPE[self.dataType]*self.rows*self.cols + \ 849 ((self.dataType == QTR_TYPE_STRING) and 1 or 0))

850

851 - class NodeInMem(NodeStorage):

852 """Manages the attributes and data of a Node that's not in a file.""" 853 __slots__ = ['__name', '__dataType', '__dataSize', '__dataCount', '__rows', '__cols', '__preload', 854 '__loadedFirst', '__loadedLast', '__data']

855 - def __init__(self, init):

856 """Instantiated by Node with either a NodeStorage template to copy, or the name for a fresh new one. 857 858 Basically it uses self.__whatever for the attributes, and keeps data in either numpy.array or mmap.mmap (for strings).""" 859 NodeStorage.__init__(self) 860 if isinstance(init, NodeStorage): 861 self.__name = init.name 862 self.__dataType = init.dataType 863 self.__dataSize = init.dataSize 864 self.__dataCount = init.dataCount 865 self.__rows = init.rows 866 self.__cols = init.cols 867 self.__preload = init.preload 868 bounds = init.loadedRows 869 self.__loadedFirst = bounds[0] 870 self.__loadedLast = bounds[1] 871 self.__data = None 872 if self.__dataCount: 873 other_data = init.get_rows(0, init.rows-1) 874 if init.dataType == QTR_TYPE_STRING: 875 self.__data = bytearray(len(other_data)) # mmap.mmap(-1, len(other_data), access=mmap.ACCESS_WRITE) 876 self.__data[:] = other_data 877 else: 878 self.__data = other_data.copy() 879 else: 880 self.__dataType = QTR_TYPE_EMPTY 881 elif isinstance(init, str): 882 self.__name = init 883 self.__dataType = QTR_TYPE_EMPTY 884 self.__dataSize = 0 885 self.__dataCount = 0 886 self.__rows = 0 887 self.__cols = 0 888 self.__preload = True 889 self.__loadedFirst = -1 890 self.__loadedLast = -1 891 self.__data = None

892 #def __del__(self): 893 # if self.__dataType == QTR_TYPE_STRING: 894 # self.__data.close()

895 - def get_name(self):

896 return self.__name

897 - def set_name(self, x):

898 self.__name = x

899 - def get_preload(self):

900 return self.__preload

901 - def set_preload(self, x):

902 self.__preload = x

903 - def get_data(self):

904 return self.__data

905 - def get_dataCount(self):

906 return self.__dataCount

907 - def get_dataSize(self):

908 return self.__dataSize

909 - def get_dataType(self):

910 return self.__dataType

911 - def get_rowcount(self):

912 return self.__rows

913 - def get_colcount(self):

914 return self.__cols

915 - def get_loaded(self):

916 if self.__loadedFirst < 0: 917 return (-1, -1) 918 else: 919 return (self.__loadedFirst * self.cols, (self.__loadedLast + 1) * self.cols - 1)

920 - def get_loadedRows(self):

921 return (self.__loadedFirst, self.__loadedLast)

922 - def get_data_item(self, i):

923 if self.__dataType == QTR_TYPE_STRING: 924 return self.__data[i] 925 else: 926 return self.__data.item(i/self.__cols, i%self.__cols)

927 - def set_data_item(self, i, x):

928 if self.__dataType == QTR_TYPE_STRING: 929 self.__data[i] = x 930 else: 931 self.__data.itemset(i/self.__cols, i%self.__cols, x)

932 - def setup_data(self, type, rows, cols):

933 if (rows <= 0) or (cols <= 0): 934 type = QTR_TYPE_EMPTY 935 rows, cols = 0, 0 936 self.__dataType = type 937 self.__dataCount = rows * cols 938 self.__rows = rows 939 self.__cols = cols 940 self.__dataSize = SIZE_OF_TYPE[type] 941 if type == QTR_TYPE_STRING: 942 self.__data = bytearray(rows*cols) # mmap.mmap(-1, rows*cols, access=mmap.ACCESS_WRITE) 943 else: 944 self.__data = numpy.array([TYPE_DEFAULTS[type]], dtype=TYPES_NUMPY[type]) 945 self.__data.resize((rows, cols)) 946 if self.__preload: 947 self.__loadedFirst = min(0, self.__dataCount-1) 948 self.__loadedLast = self.__rows - 1 949 else: 950 self.__loadedFirst = self.__loadedLast = -1

951 - def resize_data(self, nr, type_if_none):

952 if self.__dataCount == 0: 953 self.setup_data(type_if_none, nr, 1) 954 else: 955 if self.__dataType == QTR_TYPE_STRING: 956 d2 = bytearray(nr*self.cols) 957 d2[:min(nr, self.__rows)*self.cols] = self.__data[:min(nr, self.__rows)*self.cols] 958 self.__data = d2 959 else: 960 self.__data.resize((nr, self.cols), refcheck=0) 961 self.__rows = nr 962 self.__dataCount = nr * self.__cols 963 self.__adjust_load()

964 - def __adjust_load(self):

965 # in case data was resized shorter than the bound(s) 966 nr = self.__rows 967 if self.__dataCount: 968 self.__loadedFirst = min(nr-1, self.__loadedFirst) 969 self.__loadedLast = min(nr-1, self.__loadedLast) 970 else: 971 self.__loadedFirst = self.__loadedLast = -1

972 - def clear_data(self):

973 self.setup_data(QTR_TYPE_EMPTY, 0, 0)

974 - def set_string_data(self, s):

975 l = len(s) 976 self.setup_data(QTR_TYPE_STRING, l, 1) 977 self.data[:] = s

978 - def load_rows(self, first, last, do_read):

979 self.__loadedFirst = first 980 self.__loadedLast = last

981 - def unload_rows(self, do_write):

982 self.__loadedFirst = self.__loadedLast = -1

983 - def read_rows(self, first, last):

984 if self.__dataType == QTR_TYPE_STRING: 985 return str(self__data[first:last+1]) 986 else: 987 return self.__data[first:last+1].copy()

988 - def write_rows(self, first, last, arr): # flat array

989 if (self.cols > 1 and isinstance(arr, list)) or (arr.shape[1] != self.cols): 990 raise ValueError() 991 if len(arr) != (last - first + 1)*self.cols: 992 raise ValueError() 993 self.__data[first:last+1] = arr

994 - def get_rows(self, first, last):

995 return self.__data[first:last+1]

996

997 - class NodeFile(object):

998 """Manages a memory-mapped qubtree file, either on disk or an anonymous buffer."""

999 - def __init__(self, path=None, buf=None, capacity=QTR_INITIAL_CAP, write=True):

1000 """ 1001 Opens a file for qubtree use:: 1002 1003 if path: 1004 opens the file, creating if write and not exists, and memory-maps it 1005 elif buf: 1006 copies buf into a new anonymous memory map 1007 else: 1008 sets up a new anonymous memory map with capacity in bytes 1009 """ 1010 self.path = path 1011 self.buf = buf 1012 self.write = write 1013 self.mmap = None 1014 self.fileno = 0 1015 self.size = 0 1016 self.cap = 0 1017 if path: 1018 exists = os.path.exists(path) 1019 size = 0 1020 if exists: 1021 #st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime = os.stat(path) 1022 size = os.stat(path)[6] 1023 if write: 1024 if not exists: 1025 size = len(QTR_MAGIC) 1026 cap = capacity 1027 open(path, 'wb').write(QTR_MAGIC) 1028 self.fileno = os.open(path, os.O_RDWR | os.O_CREAT) 1029 os.lseek(self.fileno, 0, os.SEEK_END) 1030 nzero = cap - size 1031 while nzero: 1032 batch = min(ZERO_BATCH, nzero) 1033 os.write(self.fileno, numpy.zeros(shape=(batch,), dtype='int8')) 1034 nzero -= batch 1035 else: 1036 self.fileno = os.open(path, os.O_RDWR) 1037 cap = size 1038 self.mmap = mmap.mmap(self.fileno, cap, access=mmap.ACCESS_WRITE) 1039 elif size: 1040 cap = size 1041 self.fileno = os.open(path, os.O_RDONLY) 1042 self.mmap = mmap.mmap(self.fileno, cap, access=mmap.ACCESS_READ) 1043 if self.mmap is None: 1044 size = cap = 0 1045 self.size = size 1046 self.cap = cap 1047 if self.mmap: 1048 return 1049 if buf and (len(buf) >= len(QTR_MAGIC)): 1050 self.size = len(buf) 1051 self.cap = max(capacity, self.size) 1052 self.mmap = mmap.mmap(-1, self.cap, access=mmap.ACCESS_WRITE) 1053 self.mmap[:len(buf)] = buf 1054 else: 1055 self.size = len(QTR_MAGIC) 1056 self.cap = max(capacity, self.size) 1057 self.mmap = mmap.mmap(-1, self.cap, access=mmap.ACCESS_WRITE) 1058 self.mmap[:len(QTR_MAGIC)] = QTR_MAGIC

1059 - def close(self):

1060 if self.mmap: 1061 self.mmap.close() 1062 self.mmap = None 1063 if self.fileno: 1064 print 'closing',self.path 1065 os.close(self.fileno) 1066 self.fileno = None

1067 - def flush(self):

1068 """Flushes the memory map.""" 1069 if self.mmap: 1070 self.mmap.flush()

1071 - def alloc(self, block_size):

1072 # doesn't track empty spaces, just expands the file. 1073 offset = self.size 1074 oldcap = self.cap 1075 while (offset + block_size) > self.cap: 1076 self.cap *= 2 1077 if self.cap != oldcap: 1078 if self.fileno: 1079 self.mmap.close() 1080 os.lseek(self.fileno, 0, os.SEEK_END) 1081 os.write(self.fileno, "\0"*(self.cap - oldcap)) 1082 self.mmap = mmap.mmap(self.fileno, self.cap, access=mmap.ACCESS_WRITE) 1083 else: 1084 newmap = mmap.mmap(-1, self.cap, access=mmap.ACCESS_WRITE) 1085 newmap[:self.size] = self.mmap[:self.size] 1086 self.mmap.close() 1087 self.mmap = newmap 1088 ### TODO: notify or mark (serial?) that storage.data has become invalid 1089 self.size += block_size 1090 return offset

1091

1092 - def can_data_in_node(type, count):

1093 "Returns True if the data will fit inside the 4 byte dataPos field.""" 1094 # delphi qub can't handle strings in-node 1095 if type == QTR_TYPE_STRING: 1096 return False 1097 return 4 >= (SIZE_OF_TYPE[type] * (count + ((type == QTR_TYPE_STRING) and 1 or 0)))

1098

1099 - class NodeInFile(NodeStorage):

1100 """Manages the attributes and data of a Node inside a NodeFile.""" 1101 __slots__ = ['__file', '__offset', '__data', '__dataType', '__dataSize', '__dataCount', '__rows', '__cols', '__preload', 1102 '__loadedFirst', '__loadedLast', '__dataPos', '__childPos', '__siblingPos', '__data_in_node', '__name']

1103 - def __init__(self, file, offset=0, template=None, name=""):

1104 """ 1105 Instantiated by Node to either manage an existing node or create a new one within file:: 1106 1107 if offset: 1108 reads in the node structure at offset bytes and set up access to its data 1109 elif template: 1110 creates a new node and copy its attributes and data from NodeStorage template 1111 else: 1112 creates a new empty node with name 1113 """ 1114 NodeStorage.__init__(self) 1115 self.__file = file 1116 if offset: 1117 self.__offset = offset 1118 self.read() 1119 return 1120 if not (template is None): 1121 name = name or template.name 1122 self.__dataType = template.dataType 1123 self.__dataSize = template.dataSize 1124 self.__dataCount = template.dataCount 1125 self.__rows = template.rows 1126 self.__cols = template.cols 1127 self.__preload = template.preload 1128 bounds = template.loadedRows 1129 self.__loadedFirst = bounds[0] 1130 self.__loadedLast = bounds[1] 1131 else: 1132 name = name 1133 self.__dataType = QTR_TYPE_EMPTY 1134 self.__dataSize = 0 1135 self.__dataCount = 0 1136 self.__rows = 0 1137 self.__cols = 0 1138 self.__preload = True 1139 self.__loadedFirst = -1 1140 self.__loadedLast = -1 1141 self.__dataPos = 0 1142 self.__childPos = 0 1143 self.__siblingPos = 0 1144 self.__data_in_node = False 1145 self.__name = '_@_no_@_name' 1146 self.set_name(name) # also sets __offset 1147 self.map_data() 1148 if (not (template is None)) and template.dataCount: 1149 if self.__dataType == QTR_TYPE_STRING: 1150 sdata = str(template.data[:]) 1151 self.__file.mmap[self.__dataPos:self.__dataPos+len(sdata)] = sdata 1152 self.__data = sdata 1153 else: 1154 self.__data[:] = template.data

1155 - def get_file(self):

1156 return self.__file

1157 - def get_offset(self):

1158 return self.__offset

1159 - def get_name(self):

1160 return self.__name

1161 - def set_name(self, x):

1162 """Changes the node's name. If it's longer than the old one, the node will have to be reallocated to make room.""" 1163 if self.__name == x: return 1164 self.__name = x 1165 namelen = len(x) 1166 self.__offset = self.__file.alloc(framelen+namelen) 1167 if namelen: 1168 self.__file.mmap[self.__offset+framelen:self.__offset+framelen+namelen] = self.__name 1169 self.clean = False

1170 - def get_preload(self):

1171 return self.__preload

1172 - def set_preload(self, x):

1173 if self.__preload == x: return 1174 self.__preload = x 1175 self.clean = False

1176 - def get_data(self):

1177 return self.__data

1178 - def get_dataCount(self):

1179 return self.__dataCount

1180 - def get_dataSize(self):

1181 return self.__dataSize

1182 - def get_dataType(self):

1183 return self.__dataType

1184 - def get_dataPos(self):

1185 return self.__dataPos

1186 - def get_childPos(self):

1187 return self.__childPos

1188 - def set_childPos(self, x):

1189 if self.__childPos == x: return 1190 self.__childPos = x 1191 self.clean = False

1192 - def get_siblingPos(self):

1193 return self.__siblingPos

1194 - def set_siblingPos(self, x):

1195 if self.__siblingPos == x: return 1196 self.__siblingPos = x 1197 self.clean = False

1198 - def get_rowcount(self):

1199 return self.__rows

1200 - def get_colcount(self):

1201 return self.__cols

1202 - def get_loaded(self):

1203 if self.__loadedFirst < 0: 1204 return (-1, -1) 1205 else: 1206 return (self.__loadedFirst * self.cols, (self.__loadedLast + 1) * self.cols - 1)

1207 - def get_loadedRows(self):

1208 return (self.__loadedFirst, self.__loadedLast)

1209 - def get_data_item(self, i):

1210 if self.__dataType == QTR_TYPE_STRING: 1211 return self.__data[i] 1212 else: 1213 return self.__data.item(i/self.__cols, i%self.__cols)

1214 - def set_data_item(self, i, x):

1215 if self.__dataType == QTR_TYPE_STRING: 1216 self.__file.mmap[self.__dataPos+i] = x 1217 else: 1218 self.__data.itemset(i/self.__cols, i%self.__cols, x)

1219 - def map_data(self, read_only=False):

1220 # sets up self.__data according to type, shape, position. 1221 # if dataPos is 0 but count > 0, allocates space in self.file 1222 if self.__dataCount: 1223 if not self.__dataPos: 1224 if not read_only: 1225 self.__data_in_node = can_data_in_node(self.dataType, self.dataCount) 1226 if self.__data_in_node: 1227 self.__dataPos = self.__offset + frame_dataoff 1228 else: 1229 self.__dataPos = self.__file.alloc(self.__rows*self.__cols*SIZE_OF_TYPE[self.__dataType] + 1230 ((self.__dataType == QTR_TYPE_STRING) and 1 or 0)) 1231 if self.__dataType == QTR_TYPE_STRING: 1232 self.__data = self.__file.mmap[self.__dataPos:self.__dataPos+self.__dataCount] 1233 else: 1234 self.__data = numpy.frombuffer(self.__file.mmap, TYPES_NUMPY[self.__dataType], self.__rows * self.__cols, self.__dataPos) 1235 self.__data = self.__data.reshape((self.__rows, self.__cols)) 1236 else: 1237 self.__dataPos = 0

1238 - def setup_data(self, type, rows, cols):

1239 oldbytes = SIZE_OF_TYPE[self.dataType] * self.dataCount + ((self.dataType == QTR_TYPE_STRING) and 1 or 0) 1240 newbytes = SIZE_OF_TYPE[type] * rows * cols + ((type == QTR_TYPE_STRING) and 1 or 0) 1241 if (rows <= 0) or (cols <= 0): 1242 type = QTR_TYPE_EMPTY 1243 rows, cols = 0, 0 1244 self.__dataType = type 1245 self.__dataCount = rows * cols 1246 self.__rows = rows 1247 self.__cols = cols 1248 self.__dataSize = SIZE_OF_TYPE[type] 1249 if newbytes > oldbytes: # can't reuse 1250 self.__dataPos = 0 1251 self.map_data() 1252 if type == QTR_TYPE_STRING: 1253 self.__file.mmap[self.dataPos+self.dataCount] = '\0' 1254 self.clean = False 1255 if self.__preload: 1256 self.__loadedFirst = min(0, self.__dataCount-1) 1257 self.__loadedLast = self.__rows - 1 1258 else: 1259 self.__loadedFirst = self.__loadedLast = -1

1260 - def resize_data(self, nr, type_if_none):

1261 old_nr = self.rows 1262 if self.__dataCount == 0: 1263 self.setup_data(type_if_none, nr, 1) 1264 elif nr <= old_nr: 1265 self.__rows = nr 1266 self.__dataCount = nr * self.__cols 1267 self.clean = False 1268 self.map_data() 1269 else: 1270 if self.__dataType == QTR_TYPE_STRING: 1271 old_data = self.data[:] 1272 self.setup_data(self.dataType, nr, self.cols) 1273 self.__file.mmap[self.dataPos:self.dataPos+old_nr] = old_data 1274 else: 1275 old_data = self.data.copy() 1276 self.setup_data(self.dataType, nr, self.cols) 1277 self.data[:old_nr] = old_data 1278 self.__adjust_load()

1279 - def __adjust_load(self):

1280 # in case data was resized shorter than the bound(s) 1281 nr = self.__rows 1282 if self.__dataCount: 1283 self.__loadedFirst = min(nr-1, self.__loadedFirst) 1284 self.__loadedLast = min(nr-1, self.__loadedLast) 1285 else: 1286 self.__loadedFirst = self.__loadedLast = -1

1287 - def clear_data(self):

1288 self.setup_data(QTR_TYPE_EMPTY, 0, 0)

1289 - def set_string_data(self, s):

1290 l = len(s) 1291 self.setup_data(QTR_TYPE_STRING, l, 1) 1292 self.__file.mmap[self.dataPos:self.dataPos+l] = s 1293 self.map_data() # recopy data since it's just a copy since mmap slices aren't live

1294 - def load_rows(self, first, last, do_read):

1295 self.__loadedFirst = first 1296 self.__loadedLast = last

1297 - def unload_rows(self, do_write):

1298 self.__loadedFirst = self.__loadedLast = -1

1299 - def read_rows(self, first, last):

1300 if self.__dataType == QTR_TYPE_STRING: 1301 return self.__data[first:last+1] 1302 else: 1303 return self.__data[first:last+1].copy()

1304 - def write_rows(self, first, last, arr):

1305 if (self.cols > 1 and isinstance(arr, list)) or (arr.shape[1] != self.cols): 1306 raise ValueError() 1307 1308 if len(arr) != (last - first + 1)*self.cols: 1309 raise ValueError() 1310 if self.__dataType == QTR_TYPE_STRING: 1311 self.__file.mmap[self.dataPos+first:self.dataPos+last+1] = arr 1312 else: 1313 self.__data[first:last+1] = arr

1314 - def get_rows(self, first, last):

1315 return self.__data[first:last+1]

1316 - def read(self):

1317 if (self.__offset+framelen) > len(self.__file.mmap): 1318 print 'truncated or missing node at offset %i' % self.__offset 1319 raise Truncation(self.__offset) 1320 flags, a, b, type, size, count, dPos, cPos, sPos, c, d, e, f, nameLen \ 1321 = struct.unpack(framedef, self.__file.mmap[self.__offset:self.__offset+framelen]) 1322 # print flags, a, b, type, size, count, dPos, cPos, sPos, c, d, e, f, nameLen 1323 if (self.__offset+framelen+nameLen) > len(self.__file.mmap): 1324 print 'truncated or missing node name at offset %i' % (self.__offset+framelen) 1325 raise Truncation(self.__offset) 1326 self.__name = self.__file.mmap[self.__offset+framelen:self.__offset+framelen+nameLen] 1327 # numpy doesn't work with 80-bit floats anyway, but even if it did, 1328 # we'd have to check dataSize to see if they're mis-saved 64-bit floats (my bug, qub express, 2010) 1329 self.__dataType = min(type, QTR_TYPE_LDOUBLE-1) 1330 # some old files are mis-written as QTR_TYPE_LONG for 4-byte data: 1331 if (type == QTR_TYPE_LONG) and (size == 4): 1332 type = self.__dataType = QTR_TYPE_INT 1333 # on disk, "size" is the number of bytes in a row, and 1334 # "count" is the number of rows 1335 self.__dataSize = SIZE_OF_TYPE[type] 1336 self.__rows = count 1337 self.__cols = self.__dataSize and (size / self.__dataSize) or 0 1338 self.__dataCount = self.__rows * self.__cols 1339 self.__childPos = cPos 1340 self.__siblingPos = sPos 1341 self.__preload = bool(flags and (flags & 2)) 1342 if self.__preload and self.__dataCount: 1343 self.__loadedFirst = 0 1344 self.__loadedLast = self.__rows - 1 1345 else: 1346 self.__loadedFirst = self.__loadedLast = -1 1347 1348 1349 self.__data_in_node = bool(flags and (flags & 4)) 1350 if self.__data_in_node: 1351 self.__dataPos = 0 1352 else: 1353 self.__dataPos = dPos 1354 # clean up QFS string data that has extra null embedded 1355 if type == QTR_TYPE_STRING: 1356 if ((self.__data_in_node and (self.__file.mmap[self.__offset+frame_dataoff+count-1] == '\0')) or 1357 (self.__file.mmap[self.__dataPos+count-1] == '\0')): 1358 self.__rows -= 1 1359 self.__dataCount -= 1 1360 self.map_data(read_only=True) 1361 self.clean = True

1362 - def write(self):

1363 dataPos = self.dataPos 1364 if self.__data_in_node: 1365 dataPos = struct.unpack("i", self.__file.mmap[self.__offset+frame_dataoff:self.__offset+frame_dataoff+4])[0] 1366 a = b = c = d = e = f = 0 1367 flags = (self.preload and 2 or 0) + (self.__data_in_node and 4 or 0) 1368 #print framedef, flags, a, b, self.dataType, self.dataSize*self.cols, self.rows, \ 1369 # dataPos, self.childPos, self.siblingPos, c, d, e, f, len(self.__name) 1370 correction = 0 1371 if self.dataType == QTR_TYPE_STRING: 1372 correction = 1 # delphi qub expects the null terminator to be counted as a character (row) 1373 self.__file.mmap[self.__offset:self.__offset+framelen] = \ 1374 struct.pack(framedef, flags, a, b, self.dataType, self.dataSize*self.cols, self.rows+correction, 1375 dataPos, self.childPos, self.siblingPos, c, d, e, f, min(255, len(self.__name))) 1376 self.clean = True

1377 1378

1379 - class Dirtee(object):

1380 __slots__ = ['dirty', 'on_set', 'on_clear']

1381 - def __init__(self, init=False):

1382 self.dirty = init 1383 self.on_set = self._on_set # capture ref 1384 self.on_clear = self._on_clear

1385 - def dispose(self):

1386 self.on_set = self.on_clear = None

1387 - def _on_set(self, sender):

1388 self.dirty = True

1389 - def _on_clear(self, sender):

1390 self.dirty = False

1391 1392 # top-level Node object

1393 - class Node_numpy(object):

1394 """Represents the (sub-)tree under a particular node in a qub tree, in an open file or in memory. 1395 1396 This is the 'numpy' flavor of qubx.tree.Node 1397 """ 1398 __slots__ = ['__data', 'storage', '__parent', '__child', '__sibling', '__dirtee', '__weakref__']

1399 - def __init__(self, name="", storage=None):

1400 """Creates a new node, in memory, with name. 1401 1402 Internal use only: Manages existing NodeStorage.""" 1403 self.__data = Data(self) 1404 self.storage = storage or NodeInMem(name) 1405 self.__parent = weakref.ref(NullNode_numpy_instance) 1406 self.__child = NullNode_numpy_instance 1407 self.__sibling = NullNode_numpy_instance 1408 self.__dirtee = Dirtee(storage and (not storage.clean)) 1409 # self.__ref_onDirtied = self.__dirtee.on_set # self.__onDirtied 1410 self.storage.OnDirtied = self.__dirtee.on_set # self.ref_onDirtied

1411 - def __del__(self):

1412 # set child nodes free as independent in-memory trees, and close any file without saving 1413 self.loosen_shared_children() 1414 self.storage.OnDirtied = None 1415 self.__dirtee.dispose() 1416 self.__child = self.__sibling = self.__data = None 1417 if self.storage.file: 1418 self.storage.file.close()

1419 - def loosen_shared_children(self):

1420 to_remove = [] 1421 for child in children(self): 1422 if sys.getrefcount(child) > 4: 1423 to_remove.append(child) 1424 else: 1425 child.loosen_shared_children() 1426 for child in to_remove: 1427 self.remove(child)

1428 # print 'loading into memory shared subtree:', child.name

1429 - def __repr__(self):

1430 return '<QUBTree Node : %s>' % self.name

1431 - def __str__(self):

1432 buf = StringIO.StringIO() 1433 self.__re_str(buf) 1434 return buf.getvalue()

1435 - def __re_str(self, fi, indent=''):

1436 fi.write(indent) 1437 # skip text flags since we give up on reading 1438 fi.write(self.name) 1439 plusindent = len(self.name) 1440 if self.data.count: 1441 fi.write(' =') 1442 plusindent += 2 1443 self.__str_data(fi, indent + plusindent*' ') 1444 elif self.find('QTR_LINE_COMMENT'): 1445 fi.write('#\\') 1446 fi.write(self.lineComment) 1447 fi.write('\n') 1448 1449 c = self.child 1450 if c.name == 'QTR_LINE_COMMENT': 1451 c = c.sibling 1452 if c.isNull: 1453 return 1454 1455 chindent = indent + '\t' 1456 fi.write(indent) 1457 fi.write('{\n') 1458 while c: 1459 if c.name != 'QTR_LINE_COMMENT': 1460 nskip = 1 1461 while nskip: 1462 nskip = self.__str_datas(c, fi, chindent) 1463 for i in xrange(nskip): 1464 c = c.sibling 1465 if c and (c.name != 'QTR_LINE_COMMENT'): 1466 c.__re_str(fi, chindent) 1467 c = c.sibling 1468 fi.write(indent) 1469 fi.write('}\n')

1470 - def __str_data(self, fi, indent):

1471 if self.data.type == QTR_TYPE_STRING: 1472 if self.storage.data.find('\r'): 1473 sep = '\r\n' 1474 elif self.storage.data.find('\n'): 1475 sep = '\n' 1476 else: 1477 sep = None 1478 if sep: 1479 lines = str(self.storage.data).split(sep) 1480 fi.write(lines[0]) 1481 for i in xrange(1, len(lines)): 1482 fi.write(' \\\n') 1483 fi.write(lines[i]) 1484 else: 1485 fi.write(str(self.storage.data)) 1486 if self.lineComment: 1487 fi.write('#\\') 1488 fi.write(self.lineComment) 1489 elif self.data.count: 1490 if self.data.cols > 1: 1491 fi.write('(') 1492 for r in xrange(self.data.rows): 1493 row = self.data.row(r) 1494 for c in xrange(self.data.cols): 1495 fi.write('\t') 1496 fi.write(str(row[c])) 1497 if r < (self.data.rows - 1): 1498 fi.write('\n') 1499 fi.write(indent) 1500 fi.write(' )') 1501 else: 1502 multi = self.data.count > 10 1503 if multi: 1504 fi.write('(') 1505 for r in xrange(0, self.data.count, 10): 1506 for i in xrange(min(10, self.data.count-r)): 1507 if i: 1508 fi.write('\t') 1509 fi.write(str(self.data[r+i])) 1510 if (r+10) < self.data.count: 1511 fi.write('\n') 1512 fi.write(indent) 1513 if multi: 1514 fi.write(')')

1515 - def __str_datas(self, cf, fi, indent):

1516 arrlen = 0 1517 narr = 0 1518 cl = cf 1519 while cl: 1520 if (not cl.name) or (cl.name.find(' ') >= 0) or cl.child or (cl.data.cols > 1) or (cl.data.type in [QTR_TYPE_STRING, QTR_TYPE_POINTER, QTR_TYPE_UNKNOWN]): 1521 break 1522 if not arrlen: 1523 if cl.data.count < 8: 1524 break 1525 arrlen = cl.data.count 1526 elif arrlen != cl.data.count: 1527 break 1528 narr += 1 1529 cl = cl.sibling 1530 if narr > 1: 1531 names = [] 1532 seqs = [] 1533 c = cf 1534 while c != cl: 1535 names.append(c.name) 1536 seqs.append(c.storage.data[:,0]) 1537 c = c.sibling 1538 fi.write(indent) 1539 fi.write('(') 1540 for n in names: 1541 fi.write('\t') 1542 fi.write(n) 1543 for j in xrange(arrlen): 1544 fi.write('\n') 1545 fi.write(indent) 1546 for s in seqs: 1547 fi.write('\t') 1548 fi.write(str(s[j])) 1549 fi.write(' )\n') 1550 return narr 1551 return 0

1552 - def print_summary(self, indent=''):

1553 print indent, self.name, 1554 if self.data.type == QTR_TYPE_STRING: 1555 print '=',str(self.data) 1556 elif 0 < self.data.count <= 10: 1557 print '=',str(self.data) 1558 elif self.data.count: 1559 print ':', '%d x %d' % (self.data.rows, self.data.cols) 1560 else: 1561 print 1562 if self.child: 1563 print indent,'{' 1564 for c in children(self): 1565 c.print_summary(indent+'\t') 1566 print indent,'}'

1567 - def __nonzero__(self):

1568 return True # a non-null node is Truthy

1569 - def __len__(self):

1570 """Returns the number of child nodes.""" 1571 count = 0 1572 child = self.child 1573 while not child.isNull: 1574 child = child.sibling 1575 count += 1 1576 return count

1577 - def __getitem__(self, key):

1578 """Returns a child node, by number or by name. If by name, and None, returns a newly-appended Node.""" 1579 if isinstance(key, int): 1580 child = self.child 1581 while key and not child.isNull: 1582 child = child.sibling 1583 key -= 1 1584 return child 1585 else: 1586 name = str(key) 1587 child = self.child 1588 prior = NullNode_numpy_instance 1589 while not child.isNull: 1590 if child.name == name: 1591 return child 1592 prior = child 1593 child = child.sibling 1594 return self.insert(name, prior)

1595 #def __onDirtied(self, sender): 1596 # """When storage flips from clean to unclean, sets __modified on self and all parents up to the root.""" 1597 # self.name,'dirtied' 1598 # p = self 1599 # while p: 1600 # p.__modified = True 1601 # p = p.parent 1602 name = property(lambda self: self.storage.name, lambda self, x: self.set_name(x), doc="the label") 1603 data = property(lambda self: self.__data, lambda self, x: self.set_data(x), 1604 doc=""" 1605 read: the L{Data} object managing this node's data 1606 1607 write: replaces this node's data with the contents of a string, 1608 numpy.array (len(shape) <= 2), 1609 [list of] list of numbers, 1610 another Node.data, or None.""") 1611 lineComment = property(lambda self: self.get_lineComment(), lambda self, x: self.set_lineComment(x), 1612 doc="read/write the string data of the child named QTR_LINE_COMMENT") 1613 isNull = property(lambda self: False) 1614 child = property(lambda self: self.__child, doc="first child Node, or the NullNode") 1615 sibling = property(lambda self: self.__sibling, doc="sibling Node, or the NullNode") 1616 parent = property(lambda self: self.get_parent(), doc="parent Node, or the NullNode") 1617 path = property(lambda self: self.get_path(), doc="path of the open file containing this node, or ''") 1618 modified = property(lambda self: self.get_modified(), doc="whether this or any child... has changed since Open or save(as)") 1619 root = property(lambda self: self.get_root(), doc="the head of the tree, with no parent; possibly self.")

1620 - def set_name(self, x):

1621 if (not self.storage.file) or (self != self.root): 1622 self.storage.name = x

1623 - def set_data(self, x):

1624 if operator.isNumberType(x) and not operator.isSequenceType(x): 1625 x = numpy.array([x]) 1626 if x.dtype == numpy.int64: 1627 x = numpy.array(x, dtype=numpy.int32) 1628 if x is None: 1629 self.data.setup(QTR_TYPE_EMPTY, 0, 0) 1630 elif isinstance(x, Data): 1631 # copy from other Node 1632 if x.type == QTR_TYPE_STRING: 1633 self.storage.set_string_data(str(x)) 1634 else: 1635 self.data.setup(x.type, x.rows, x.cols) 1636 self.data[:] = x[:] 1637 elif isinstance(x, numpy.ndarray): 1638 if any(sh == 0 for sh in x.shape): 1639 self.data.clear() 1640 elif len(x.shape) == 1: 1641 self.data.setup(TYPE_OF_NUMPY[str(x.dtype)], x.shape[0], 1) 1642 self.storage.data[:,0] = x 1643 else: 1644 self.data.setup(TYPE_OF_NUMPY[str(x.dtype)], x.shape[0], x.shape[1]) 1645 if self.storage.data.shape != x.shape: 1646 print self.storage.data.shape, x.shape, x 1647 traceback.print_stack() 1648 self.storage.data[:] = x 1649 elif (not isinstance(x, str)) \ 1650 and operator.isSequenceType(x) and all(operator.isNumberType(z) for z in x): 1651 # 1d list/tuple 1652 if len(x): 1653 type = all(isinstance(k, int) or isinstance(k, numpy.int32) for k in x) and QTR_TYPE_INT or QTR_TYPE_DOUBLE 1654 self.data.setup(type, len(x), 1) 1655 self.storage.data[:,0] = x 1656 else: 1657 self.data.clear() 1658 elif (not isinstance(x, str)) \ 1659 and operator.isSequenceType(x) \ 1660 and all(operator.isSequenceType(z) and all(operator.isNumberType(zz) for zz in z) for z in x): 1661 # 2d list/tuple 1662 type = all(all(isinstance(k, int) for k in z) for z in x) and QTR_TYPE_INT or QTR_TYPE_DOUBLE 1663 self.data.setup(type, len(x), max(len(z) for z in x)) 1664 for i,z in enumerate(x): 1665 self.data.row(i)[:len(z)] = z 1666 else: # string or unknown written as string 1667 self.storage.set_string_data(str(x))

1668 - def get_lineComment(self):

1669 return str(self.find("QTR_LINE_COMMENT").data)

1670 - def set_lineComment(self, x):

1671 if x: 1672 self['QTR_LINE_COMMENT'].data = x 1673 else: 1674 self.remove(self['QTR_LINE_COMMENT'])

1675 - def get_parent(self):

1676 return self.__parent() or NullNode_numpy_instance

1677 - def get_path(self):

1678 return self.storage.file and self.storage.file.path or ''

1679 - def get_root(self):

1680 root = self 1681 while not root.parent.isNull: 1682 root = root.parent 1683 return root

1684 - def clone(self, deep=True, file=None):

1685 """Returns a copy of the (sub-)tree rooted here, copied into file, or else in-memory. If not deep, skips the children.""" 1686 if file: 1687 new_stor = NodeInFile(file, template=self.storage) 1688 else: 1689 new_stor = NodeInMem(self.storage) 1690 mini_me = Node_numpy(storage=new_stor) 1691 if deep: 1692 last = NullNode_numpy_instance 1693 for c in children(self): 1694 last = mini_me.insert(c.clone(file=file), last) 1695 return mini_me

1696 - def __re_store(self, file=None):

1697 # recursively switches the storage between file(s) and/or memory 1698 if file: 1699 self.storage = NodeInFile(file, template=self.storage) 1700 else: 1701 self.storage = NodeInMem(self.storage) 1702 #self.__ref_onDirtied = self.__onDirtied 1703 self.storage.OnDirtied = self.__dirtee.on_set # self.__ref_onDirtied 1704 self.__dirtee.dirty = not self.storage.clean 1705 for c in children(self): 1706 c.__re_store(file)

1707 - def get_modified(self):

1708 """Returns whether anything's different from on disk. Adjusts child and sibling positions first in case anything's moved.""" 1709 return self.__pre_save()

1710 - def saveAs(self, path, as_copy=False):

1711 """Creates a new file at path, moves the (sub-)tree rooted here into the file, and save()s. This node becomes the root. 1712 1713 @param as_copy: (def False) if True: may save a copy without re-parenting anything (faster) if overall size <= MAX_QUICK_WRITE_BYTES 1714 """ 1715 size = len(QTR_MAGIC) + self.__re_measure() 1716 if as_copy: 1717 if size <= MAX_QUICK_WRITE_BYTES: 1718 try: 1719 open(path, 'wb').write(self.getBytes()) 1720 return True 1721 except: 1722 traceback.print_exc() 1723 if (not self.storage.file) or (path != self.storage.file.path): 1724 if os.path.exists(path): 1725 os.remove(path) 1726 file = NodeFile(path, capacity=size, write=True) 1727 self.__re_store(file) 1728 return self.save()

1729 - def saveAsText(self, path):

1730 """Writes a text representation of the (sub-)tree rooted here, to path on disk.""" 1731 open(path, 'w').write(str(self))

1732 - def save(self):

1733 """When this node is the root of an open file (self.storage.file != None), writes any modifications to disk.""" 1734 if self != self.root: 1735 raise Exception("Attempt to save non-root node.") 1736 if not self.storage.file: 1737 raise Exception('save() without saveAs()') 1738 1739 if self.__pre_save(): # harmonize child, sibling pos 1740 self.__re_save() # write dirty nodes 1741 self.storage.file.flush() 1742 self.re_map_data() 1743 return True

1744 - def __pre_save(self):

1745 """Recursively refreshes the child and sibling positions, in case they have moved, 1746 so nodes are accurately marked unclean and modified.""" 1747 mod = False 1748 for c in children(self): 1749 mod = c.__pre_save() or mod 1750 self.storage.childPos = self.child and self.child.storage.offset or 0 1751 self.storage.siblingPos = self.sibling and self.sibling.storage.offset or 0 1752 return mod or self.__dirtee.dirty

1753 - def __re_save(self):

1754 """Recursively writes any modifications to disk.""" 1755 if not self.storage.clean: 1756 self.storage.write() 1757 for c in children(self): 1758 c.__re_save() 1759 self.__dirtee.on_clear(self)

1760 - def re_read(self):

1761 """Recursively reads storage and instantiates child and sibling nodes, during Open.""" 1762 st = self.storage 1763 if not st.file: return 1764 try: 1765 st.read() 1766 except Truncation, tr: 1767 traceback.print_exc() 1768 if self.root != self: 1769 print 'Unexpected EOF reading node at offset %s; truncating' % tr.offset 1770 self.parent.remove(self) 1771 else: 1772 raise 1773 if st.childPos and not self.child: 1774 self.__child = Node_numpy(storage=NodeInFile(st.file, offset=st.childPos)) 1775 self.__child.__parent = weakref.ref(self) 1776 if st.siblingPos and not self.sibling: 1777 self.__sibling = Node_numpy(storage=NodeInFile(st.file, offset=st.siblingPos)) 1778 self.__sibling.__parent = weakref.ref(self.parent) 1779 for c in children(self): 1780 c.re_read() 1781 self.__dirtee.on_clear(self)

1782 - def close(self):

1783 """If this node is the root of an open file, moves it into memory and close the file.""" 1784 if not self.storage.file: return True 1785 if self != self.root: 1786 raise Exception("Attempt to close non-root node.") 1787 file = self.storage.file 1788 self.__re_store() 1789 file.close() 1790 return True

1791 - def getBytes(self):

1792 """Returns the binary representation of the (sub-)tree rooted here, just as it would be on disk, as a string.""" 1793 file = NodeFile(capacity=len(QTR_MAGIC)+self.__re_measure()) # no path: anonymous mem map 1794 dupe = self.clone(file=file) 1795 dupe.save() 1796 return str(file.mmap[:])

1797 - def __re_measure(self):

1798 """Returns the number of bytes needed to store the (sub-)tree rooted here.""" 1799 return self.storage.measure() + sum(c.__re_measure() for c in children(self))

1800 - def re_map_data(self):

1801 self.storage.map_data() 1802 for child in children(self): 1803 child.re_map_data()

1804 - def list(self):

1805 """Returns the names of all child Nodes.""" 1806 return [c.name for c in children(self)]

1807 - def find(self, name):

1808 """Returns the first child with name, or the NullNode.""" 1809 child = self.child 1810 while not child.isNull: 1811 if child.name == name: 1812 return child 1813 child = child.sibling 1814 return child

1815 - def next(self, name):

1816 """Returns the next sibling with name, or the NullNode.""" 1817 child = self.sibling 1818 while not child.isNull: 1819 if child.name == name: 1820 return child 1821 child = child.sibling 1822 return child

1823 - def nextSameName(self):

1824 """Returns the next sibling with name == self.name, or the NullNode.""" 1825 return self.next(self.name)

1826 - def append(self, childOrName):

1827 """inserts after the last child node and returns it.""" 1828 last = self.child 1829 while not last.sibling.isNull: 1830 last = last.sibling 1831 return self.insert(childOrName, last)

1832 - def insert(self, childOrName, after=None):

1833 """insert(str, after): inserts a new child node after the given child or first, with the given name, or '', and returns it 1834 append(Node, after): inserts an existing node after the given child or first, first removing it from its former parent if any, and returns it""" 1835 if childOrName == self: return self 1836 if isinstance(childOrName, NullNode_numpy): return childOrName 1837 if isinstance(childOrName, Node_numpy): 1838 c = childOrName 1839 if not c.parent.isNull: 1840 c.parent.remove(c, False) 1841 elif c.storage.file and (c.storage.file != self.storage.file): # root of its own open file 1842 c.close() 1843 if c.storage.file != self.storage.file: 1844 c.__re_store(self.storage.file) 1845 c.__parent = weakref.ref(self) 1846 if after: 1847 c.__sibling = after.sibling 1848 after.__sibling = c 1849 else: 1850 c.__sibling = self.child 1851 self.__child = c 1852 return c 1853 else: 1854 return self.insert(Node_numpy(name=str(childOrName)), after)

1855 - def appendClone(self, other, deep=True):

1856 """Inserts a copy of other, after the last child, and returns it. If not deep, skip other's children.""" 1857 return self.append( other.clone(deep, self.storage.file) )

1858 - def insertClone(self, other, deep=True, after=None):

1859 """Inserts a copy of other, after the given child or first, and returns it. If not deep, skip other's children.""" 1860 return self.insert( other.clone(deep, self.storage.file), after )

1861 - def remove(self, child, re_store=True):

1862 """Removes the given child, making it its own root. 1863 1864 Internal use only: re_store=False: when we remove it from a file-bound tree only to re-insert it somewhere else in the same tree.""" 1865 if not child: return 1866 1867 if child == self.child: 1868 self.__child = child.sibling 1869 child.__parent = weakref.ref(NullNode_numpy_instance) 1870 if self.storage.file and re_store: 1871 child.__re_store() 1872 return 1873 1874 prior = self.child 1875 while not prior.isNull: 1876 if prior.sibling == child: 1877 prior.__sibling = child.sibling 1878 child.__parent = weakref.ref(NullNode_numpy_instance) 1879 if self.storage.file and re_store: 1880 child.__re_store() 1881 return 1882 prior = prior.sibling

1883 - def setChanged(self):

1884 """artifact from when data edits didn't go memmap straight to disk; had to be marked changed; does nothing.""" 1885 pass

1886 - def lock(self, timeoutMS=None):

1887 """the native flavor has a mutex, mainly to protect the ref-count. This flavor does nothing.""" 1888 pass

1889 - def unlock(self):

1890 """the native flavor has a mutex, mainly to protect the ref-count. This flavor does nothing.""" 1891 pass

1892 1893

1894 - class NodeFile_NoMap(object):

1895 """Manages a non-memory-mapped qubtree file on disk. Read-only. 1896 1897 Solves the (mainly 32-bit) problem of opening files that don't fit in address space, by 1898 leaving non-preload data on disk until node.data.read_rows(). 1899 """

1900 - def __init__(self, path, write=True):

1901 """ 1902 Opens a file for qubtree use. 1903 1904 """ 1905 self.path = path 1906 self.write = write 1907 self.file = None 1908 self.size = 0 1909 try: 1910 exists = os.path.exists(path) 1911 size = 0 1912 if exists: 1913 #st_mode, st_ino, st_dev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime = os.stat(path) 1914 size = os.stat(path)[6] 1915 self.file = open(path, 'rb') 1916 self.size = size 1917 except KeyboardInterrupt: 1918 raise 1919 except: 1920 traceback.print_exc()

1921 - def close(self):

1922 if self.file: 1923 print 'closing',self.path 1924 self.file = None

1925 1926

1927 - class NodeInFile_NoMap(NodeStorage):

1928 """Manages the attributes and data of a Node inside a NodeFile_NoMap. Read-only. 1929 1930 Solves the (mainly 32-bit) problem of opening files that don't fit in address space, by 1931 leaving non-preload data on disk until node.data.read_rows(). 1932 """ 1933 __slots__ = ['__file', '__offset', '__name', '__preload', '__data', '__dataCount', '__dataSize', '__dataType', '__dataPos', 1934 '__childPos', '__siblingPos', '__rows', '__cols', '__loadedFirst', '__loadedLast', '__data_in_node']

1935 - def __init__(self, file, offset):

1936 """ 1937 Instantiated by Node to either manage an existing node or create a new one within file: 1938 1939 reads in the node structure at offset bytes and set up access to its data. 1940 """ 1941 NodeStorage.__init__(self) 1942 self.__file = file 1943 self.__offset = offset 1944 self.read()

1945 - def get_file(self):

1946 return self.__file

1947 - def get_offset(self):

1948 return self.__offset

1949 - def get_name(self):

1950 return self.__name

1951 - def set_name(self, x):

1952 raise Exception('read only')

1953 - def get_preload(self):

1954 return self.__preload

1955 - def set_preload(self, x):

1956 raise Exception('read only')

1957 - def get_data(self):

1958 return self.__data

1959 - def get_dataCount(self):

1960 return self.__dataCount

1961 - def get_dataSize(self):

1962 return self.__dataSize

1963 - def get_dataType(self):

1964 return self.__dataType

1965 - def get_dataPos(self):

1966 return self.__dataPos

1967 - def get_childPos(self):

1968 return self.__childPos

1969 - def set_childPos(self, x):

1970 raise Exception('read only')

1971 - def get_siblingPos(self):

1972 return self.__siblingPos

1973 - def set_siblingPos(self, x):

1974 raise Exception('read only')

1975 - def get_rowcount(self):

1976 return self.__rows

1977 - def get_colcount(self):

1978 return self.__cols

1979 - def get_loaded(self):

1980 if self.__loadedFirst < 0: 1981 return (-1, -1) 1982 else: 1983 return (self.__loadedFirst * self.cols, (self.__loadedLast + 1) * self.cols - 1)

1984 - def get_loadedRows(self):

1985 return (self.__loadedFirst, self.__loadedLast)

1986 - def get_data_item(self, i):

1987 if self.__loadedFirst < 0: 1988 raise Exception('use node.data.read()') 1989 if self.__dataType == QTR_TYPE_STRING: 1990 return self.__data[i] 1991 else: 1992 return self.__data.item(i/self.__cols, i%self.__cols)

1993 - def set_data_item(self, i, x):

1994 if self.__loadedFirst < 0: 1995 raise Exception('use node.data.read()') 1996 if self.__dataType == QTR_TYPE_STRING: 1997 self.__file.mmap[self.__dataPos+i] = x 1998 else: 1999 self.__data.itemset(i/self.__cols, i%self.__cols, x)

2000 - def map_data(self):

2001 # sets up self.__data according to type, shape, position. 2002 if self.__dataCount: 2003 if self.__data_in_node: 2004 self.__dataPos = self.__offset + frame_dataoff 2005 if self.__preload: 2006 self.__file.file.seek(self.__dataPos) 2007 if self.__dataType == QTR_TYPE_STRING: 2008 self.__data = self.__file.file.read(self.__dataSize * self.__dataCount) 2009 else: 2010 self.__data = numpy.fromfile(self.__file.file, TYPES_NUMPY[self.__dataType], self.__rows * self.__cols, '') 2011 self.__data = self.__data.reshape((self.__rows, self.__cols)) 2012 else: 2013 self.__data = None 2014 else: 2015 self.__dataPos = 0

2016 - def setup_data(self, type, rows, cols):

2017 raise Exception('read only')

2018 - def resize_data(self, nr, type_if_none):

2019 raise Exception('read only')

2020 - def clear_data(self):

2021 raise Exception('read only')

2022 - def load_rows(self, first, last, do_read):

2023 raise Exception('use node.data.read()')

2024 - def unload_rows(self, do_write):

2025 raise Exception('use node.data.read()')

2026 - def read_rows(self, first, last):

2027 if self.__data is None: 2028 self.__file.file.seek(self.__dataPos+self.__dataSize*self.cols*first) 2029 if self.__dataType == QTR_TYPE_STRING: 2030 return self.__file.file.read((last - first + 1) * self.__dataSize*self.cols) 2031 else: 2032 arr = numpy.fromfile(self.__file.file, TYPES_NUMPY[self.__dataType], (last - first + 1) * self.__cols, '') 2033 return arr.reshape(((last - first + 1), self.__cols)) 2034 else: 2035 if self.__dataType == QTR_TYPE_STRING: 2036 return self.__data[first:last+1] 2037 else: 2038 return self.__data[first:last+1].copy()

2039 - def write_rows(self, first, last, arr):

2040 raise Exception('read only')

2041 - def get_rows(self, first, last):

2042 return self.__data[first:last+1]

2043 - def read(self):

2044 if (self.__offset+framelen) > self.__file.size: 2045 print 'truncated or missing node at offset %i' % self.__offset 2046 raise Truncation(self.__offset) 2047 fi = self.__file.file 2048 fi.seek(self.__offset) 2049 dat = fi.read(framelen) 2050 flags, a, b, type, size, count, dPos, cPos, sPos, c, d, e, f, nameLen \ 2051 = struct.unpack(framedef, dat) 2052 # print flags, a, b, type, size, count, dPos, cPos, sPos, c, d, e, f, nameLen 2053 if (self.__offset+framelen+nameLen) > self.__file.size: 2054 print 'truncated or missing node name at offset %i' % (self.__offset+framelen) 2055 raise Truncation(self.__offset) 2056 fi.seek(self.__offset+framelen) 2057 self.__name = fi.read(nameLen) 2058 # numpy doesn't work with 80-bit floats anyway, but even if it did, 2059 # we'd have to check dataSize to see if they're mis-saved 64-bit floats (my bug, qub express, 2010) 2060 self.__dataType = min(type, QTR_TYPE_LDOUBLE-1) 2061 # some old files are mis-written as QTR_TYPE_LONG for 4-byte data: 2062 if (type == QTR_TYPE_LONG) and (size == 4): 2063 type = self.__dataType = QTR_TYPE_INT 2064 # on disk, "size" is the number of bytes in a row, and 2065 # "count" is the number of rows 2066 self.__dataSize = SIZE_OF_TYPE[type] 2067 self.__rows = count 2068 self.__cols = self.__dataSize and (size / self.__dataSize) or 0 2069 self.__dataCount = self.__rows * self.__cols 2070 self.__childPos = cPos 2071 self.__siblingPos = sPos 2072 self.__preload = bool(flags and (flags & 2)) 2073 if self.__preload and self.__dataCount: 2074 self.__loadedFirst = 0 2075 self.__loadedLast = self.__rows - 1 2076 else: 2077 self.__loadedFirst = self.__loadedLast = -1 2078 2079 self.__data_in_node = bool(flags and (flags & 4)) 2080 if self.__data_in_node: 2081 self.__dataPos = 0 2082 else: 2083 self.__dataPos = dPos 2084 # clean up QFS string data that has extra null embedded 2085 if type == QTR_TYPE_STRING: 2086 if self.__data_in_node: 2087 fi.seek(self.__offset+frame_dataoff+count-1) 2088 else: 2089 fi.seek(self.__dataPos+count-1) 2090 if fi.read(1) == '\0': 2091 self.__rows -= 1 2092 self.__dataCount -= 1 2093 self.map_data() 2094 self.clean = True

2095 - def write(self):

2096 raise Exception('read only')

2097 2098 2099 # top-level Node object

2100 - class Node_numpy_NoMap(object):

2101 """Represents the (sub-)tree under a particular node in a qub tree. Read-only. 2102 2103 Solves the (mainly 32-bit) problem of opening files that don't fit in address space, by 2104 leaving non-preload data on disk until node.data.read_rows(). 2105 2106 Instantiated by Open_numpy when memory runs out. 2107 """ 2108 __slots__ = ['__data', 'storage', '__parent', '__child', '__sibling', '__dirtee', '__weakref__']

2109 - def __init__(self, storage):

2110 """Creates a new node, in memory, with name. 2111 2112 Internal use only: Manages existing NodeStorage.""" 2113 self.__data = Data(self) 2114 self.storage = storage 2115 self.__parent = weakref.ref(NullNode_numpy_instance) 2116 self.__child = NullNode_numpy_instance 2117 self.__sibling = NullNode_numpy_instance 2118 self.__dirtee = Dirtee(storage and (not storage.clean)) 2119 # self.__ref_onDirtied = self.__dirtee.on_set # self.__onDirtied 2120 self.storage.OnDirtied = self.__dirtee.on_set # self.ref_onDirtied

2121 - def __del__(self):

2122 self.storage.OnDirtied = None 2123 self.__dirtee.dispose() 2124 self.storage.file.close()

2125 - def __repr__(self):

2126 return '<QUBTree Node : %s>' % self.name

2127 - def __str__(self):

2128 buf = StringIO.StringIO() 2129 self.__re_str(buf) 2130 return buf.getvalue()

2131 - def __re_str(self, fi, indent=''):

2132 fi.write(indent) 2133 # skip text flags since we give up on reading 2134 fi.write(self.name) 2135 plusindent = len(self.name) 2136 if self.data.count: 2137 fi.write(' =') 2138 plusindent += 2 2139 self.__str_data(fi, indent + plusindent*' ') 2140 elif self.find('QTR_LINE_COMMENT'): 2141 fi.write('#\\') 2142 fi.write(self.lineComment) 2143 fi.write('\n') 2144 2145 c = self.child 2146 if c.name == 'QTR_LINE_COMMENT': 2147 c = c.sibling 2148 if c.isNull: 2149 return 2150 2151 chindent = indent + '\t' 2152 fi.write(indent) 2153 fi.write('{\n') 2154 while c: 2155 if c.name != 'QTR_LINE_COMMENT': 2156 nskip = 1 2157 while nskip: 2158 nskip = self.__str_datas(c, fi, chindent) 2159 for i in xrange(nskip): 2160 c = c.sibling 2161 if c and (c.name != 'QTR_LINE_COMMENT'): 2162 c.__re_str(fi, chindent) 2163 c = c.sibling 2164 fi.write(indent) 2165 fi.write('}\n')

2166 - def __str_data(self, fi, indent):

2167 if self.storage.data is None: 2168 return 2169 if self.data.type == QTR_TYPE_STRING: 2170 if self.storage.data.find('\r'): 2171 sep = '\r\n' 2172 elif self.storage.data.find('\n'): 2173 sep = '\n' 2174 else: 2175 sep = None 2176 if sep: 2177 lines = str(self.storage.data).split(sep) 2178 fi.write(lines[0]) 2179 for i in xrange(1, len(lines)): 2180 fi.write(' \\\n') 2181 fi.write(lines[i]) 2182 else: 2183 fi.write(str(self.storage.data)) 2184 if self.lineComment: 2185 fi.write('#\\') 2186 fi.write(self.lineComment) 2187 elif self.data.count: 2188 if self.data.cols > 1: 2189 fi.write('(') 2190 for r in xrange(self.data.rows): 2191 row = self.data.row(r) 2192 for c in xrange(self.data.cols): 2193 fi.write('\t') 2194 fi.write(str(row[c])) 2195 if r < (self.data.rows - 1): 2196 fi.write('\n') 2197 fi.write(indent) 2198 fi.write(' )') 2199 else: 2200 multi = self.data.count > 10 2201 if multi: 2202 fi.write('(') 2203 for r in xrange(0, self.data.count, 10): 2204 for i in xrange(min(10, self.data.count-r)): 2205 if i: 2206 fi.write('\t') 2207 fi.write(str(self.data[r+i])) 2208 if (r+10) < self.data.count: 2209 fi.write('\n') 2210 fi.write(indent) 2211 if multi: 2212 fi.write(')')

2213 - def __str_datas(self, cf, fi, indent):

2214 arrlen = 0 2215 narr = 0 2216 cl = cf 2217 while cl: 2218 if (not cl.name) or (cl.name.find(' ') >= 0) or cl.child or (cl.data.cols > 1) or (cl.data.type in [QTR_TYPE_STRING, QTR_TYPE_POINTER, QTR_TYPE_UNKNOWN]): 2219 break 2220 if not arrlen: 2221 if cl.data.count < 8: 2222 break 2223 arrlen = cl.data.count 2224 elif arrlen != cl.data.count: 2225 break 2226 narr += 1 2227 cl = cl.sibling 2228 if narr > 1: 2229 names = [] 2230 seqs = [] 2231 c = cf 2232 while c != cl: 2233 names.append(c.name) 2234 seqs.append(c.storage.data[:,0]) 2235 c = c.sibling 2236 fi.write(indent) 2237 fi.write('(') 2238 for n in names: 2239 fi.write('\t') 2240 fi.write(n) 2241 for j in xrange(arrlen): 2242 fi.write('\n') 2243 fi.write(indent) 2244 for s in seqs: 2245 fi.write('\t') 2246 fi.write(str(s[j])) 2247 fi.write(' )\n') 2248 return narr 2249 return 0

2250 - def print_summary(self, indent=''):

2251 print indent, self.name, 2252 if self.data.type == QTR_TYPE_STRING: 2253 print '=',str(self.data) 2254 elif 0 < self.data.count <= 10: 2255 print '=',str(self.data) 2256 elif self.data.count: 2257 print ':', '%d x %d' % (self.data.rows, self.data.cols) 2258 else: 2259 print 2260 if self.child: 2261 print indent,'{' 2262 for c in children(self): 2263 c.print_summary(indent+'\t') 2264 print indent,'}'

2265 - def __nonzero__(self):

2266 return True # a non-null node is Truthy

2267 - def __len__(self):

2268 """Returns the number of child nodes.""" 2269 count = 0 2270 child = self.child 2271 while not child.isNull: 2272 child = child.sibling 2273 count += 1 2274 return count

2275 - def __getitem__(self, key):

2276 """Returns a child node, by number or by name. If by name, and None, returns a newly-appended Node.""" 2277 if isinstance(key, int): 2278 child = self.child 2279 while key: 2280 child = child.sibling 2281 key -= 1 2282 return child 2283 else: 2284 name = str(key) 2285 child = self.child 2286 prior = NullNode_numpy_instance 2287 while not child.isNull: 2288 if child.name == name: 2289 return child 2290 prior = child 2291 child = child.sibling 2292 return self.insert(name, prior)

2293 name = property(lambda self: self.storage.name, lambda self, x: self.set_name(x), doc="the label") 2294 data = property(lambda self: self.__data, lambda self, x: self.set_data(x), 2295 doc="""read: the L{Data} object managing this node's data.""") 2296 lineComment = property(lambda self: self.get_lineComment(), lambda self, x: self.set_lineComment(x), 2297 doc="read/write the string data of the child named QTR_LINE_COMMENT") 2298 isNull = property(lambda self: False) 2299 child = property(lambda self: self.__child, doc="first child Node, or the NullNode") 2300 sibling = property(lambda self: self.__sibling, doc="sibling Node, or the NullNode") 2301 parent = property(lambda self: self.get_parent(), doc="parent Node, or the NullNode") 2302 path = property(lambda self: self.get_path(), doc="path of the open file containing this node, or ''") 2303 modified = property(lambda self: self.get_modified(), doc="whether this or any child... has changed since Open or save(as)") 2304 root = property(lambda self: self.get_root(), doc="the head of the tree, with no parent; possibly self.")

2305 - def set_name(self, x):

2306 if (not self.storage.file) or (self != self.root): 2307 self.storage.name = x

2308 - def set_data(self, x):

2309 raise Exception('read only')

2310 - def get_lineComment(self):

2311 return str(self.find("QTR_LINE_COMMENT").data)

2312 - def set_lineComment(self, x):

2313 pass

2314 - def get_parent(self):

2315 return self.__parent() or NullNode_numpy_instance

2316 - def get_path(self):

2317 return self.storage.file and self.storage.file.path or ''

2318 - def get_root(self):

2319 root = self 2320 while not root.parent.isNull: 2321 root = root.parent 2322 return root

2323 - def clone(self, deep=True, file=None):

2324 """Returns a copy of the (sub-)tree rooted here, copied into file, or else in-memory. If not deep, skips the children.""" 2325 if file: 2326 new_stor = NodeInFile(file, template=self.storage) 2327 else: 2328 new_stor = NodeInMem(self.storage) 2329 mini_me = Node_numpy(storage=new_stor) 2330 if deep: 2331 last = NullNode_numpy_instance 2332 for c in children(self): 2333 last = mini_me.insert(c.clone(file=file), last) 2334 return mini_me

2335 - def get_modified(self):

2336 """Returns whether anything's different from on disk. Adjusts child and sibling positions first in case anything's moved.""" 2337 return False

2338 - def saveAs(self, path, as_copy=False):

2339 """Creates a new file at path, moves the (sub-)tree rooted here into the file, and save()s. This node becomes the root. 2340 2341 @param as_copy: (def False) if True: may save a copy without re-parenting anything (faster) if overall size <= MAX_QUICK_WRITE_BYTES 2342 """ 2343 return self.save()

2344 - def saveAsText(self, path):

2345 """Writes a text representation of the (sub-)tree rooted here, to path on disk.""" 2346 open(path, 'w').write(str(self))

2347 - def save(self):

2348 print 'ignoring save of Node_numpy_NoMap' 2349 return True

2350 - def re_read(self):

2351 """Recursively reads storage and instantiates child and sibling nodes, during Open.""" 2352 st = self.storage 2353 if not st.file: return 2354 try: 2355 st.read() 2356 except Truncation, tr: 2357 traceback.print_exc() 2358 if self.root != self: 2359 print 'Unexpected EOF reading node at offset %s; truncating' % tr.offset 2360 self.parent.remove(self) 2361 else: 2362 raise 2363 if st.childPos and not self.child: 2364 self.__child = Node_numpy_NoMap(storage=NodeInFile_NoMap(st.file, offset=st.childPos)) 2365 self.__child.__parent = weakref.ref(self) 2366 if st.siblingPos and not self.sibling: 2367 self.__sibling = Node_numpy_NoMap(storage=NodeInFile_NoMap(st.file, offset=st.siblingPos)) 2368 self.__sibling.__parent = weakref.ref(self.parent) 2369 for c in children(self): 2370 c.re_read() 2371 self.__dirtee.on_clear(self)

2372 - def close(self):

2373 print 'ignoring close of Node_numpy_NoMap' 2374 return True

2375 - def getBytes(self):

2376 """Returns the binary representation of the (sub-)tree rooted here, just as it would be on disk, as a string.""" 2377 file = NodeFile(capacity=len(QTR_MAGIC)+self.__re_measure()) # no path: anonymous mem map 2378 dupe = self.clone(file=file) 2379 dupe.save() 2380 return str(file.mmap[:])

2381 - def __re_measure(self):

2382 """Returns the number of bytes needed to store the (sub-)tree rooted here.""" 2383 return self.storage.measure() + sum(c.__re_measure() for c in children(self))

2384 - def re_map_data(self):

2385 pass

2386 - def list(self):

2387 """Returns the names of all child Nodes.""" 2388 return [c.name for c in children(self)]

2389 - def find(self, name):

2390 """Returns the first child with name, or the NullNode.""" 2391 child = self.child 2392 while not child.isNull: 2393 if child.name == name: 2394 return child 2395 child = child.sibling 2396 return child

2397 - def next(self, name):

2398 """Returns the next sibling with name, or the NullNode.""" 2399 child = self.sibling 2400 while not child.isNull: 2401 if child.name == name: 2402 return child 2403 child = child.sibling 2404 return child

2405 - def nextSameName(self):

2406 """Returns the next sibling with name == self.name, or the NullNode.""" 2407 return self.next(self.name)

2408 - def append(self, childOrName):

2409 raise Exception('read only')

2410 - def insert(self, childOrName, after=None):

2411 raise Exception('read only')

2412 - def appendClone(self, other, deep=True):

2413 raise Exception('read only')

2414 - def insertClone(self, other, deep=True, after=None):

2415 raise Exception('read only')

2416 - def remove(self, child, re_store=True):

2417 raise Exception('read only')

2418 - def setChanged(self):

2419 """artifact from when data edits didn't go memmap straight to disk; had to be marked changed; does nothing.""" 2420 pass

2421 - def lock(self, timeoutMS=None):

2422 """the native flavor has a mutex, mainly to protect the ref-count. This flavor does nothing.""" 2423 pass

2424 - def unlock(self):

2425 """the native flavor has a mutex, mainly to protect the ref-count. This flavor does nothing.""" 2426 pass

2427 2428

2429 - def Open_numpy(path, readOnly=False):

2430 """Returns the root Node of the qubtree file at path, or the NullNode. The file remains open until close()d or unref'd. 2431 Small read-only trees may not be kept open. 2432 2433 This is the 'numpy' flavor of qubx.tree.Open 2434 """ 2435 if readOnly and os.path.exists(path): 2436 size = os.stat(path)[6] 2437 if size <= MAX_QUICK_READ_BYTES: 2438 return ReadBytes_numpy(open(path, 'rb').read()) 2439 try: 2440 file = NodeFile(path=path, write=not readOnly) 2441 if file.mmap is None: 2442 return NullNode() 2443 elif file.size == len(QTR_MAGIC): 2444 if readOnly: 2445 return NullNode() 2446 else: 2447 root = Node_numpy(storage=NodeInFile(file)) 2448 else: 2449 root = Node_numpy(storage=NodeInFile(file, offset=len(QTR_MAGIC))) 2450 root.re_read() 2451 return root 2452 except Exception, e: 2453 print 'while reading %s' % path 2454 traceback.print_exc() 2455 msg = traceback.format_exc().lower() 2456 try: # if e.args and (('memory' in msg) or ('storage' in msg)): 2457 print 'trying alternate read' 2458 return _qubtree.Open(path, readOnly) 2459 except: 2460 traceback.print_exc() 2461 try: # if e.args and (('memory' in msg) or ('storage' in msg)): 2462 print 'trying alternate read 2' 2463 file = NodeFile_NoMap(path) 2464 if (not file) or (file.size <= len(QTR_MAGIC)): 2465 return NullNode() 2466 root = Node_numpy_NoMap(NodeInFile_NoMap(file, offset=len(QTR_MAGIC))) 2467 root.re_read() 2468 return root 2469 except: 2470 traceback.print_exc() 2471 return NullNode()

2472

2473 - def ReadBytes_numpy(buf):

2474 """Returns the root node of the qubtree copied from the binary in buf. 2475 2476 This is the 'numpy' flavor of qubx.tree.ReadBytes 2477 """ 2478 try: 2479 file = NodeFile(buf=buf, write=False) 2480 if file.size == len(QTR_MAGIC): 2481 root = Node_numpy(storage=NodeInFile(file)) 2482 else: 2483 root = Node_numpy(storage=NodeInFile(file, offset=len(QTR_MAGIC))) 2484 root.re_read() 2485 root.close() # copy into own mem 2486 return root 2487 except: 2488 print 'while reading buffer...' 2489 traceback.print_exc() 2490 return NullNode()

2491 2492 # omitted since the originals never worked completely; were never really used. 2493 # def ReadText(path): 2494 # ## 2495 # pass 2496 # 2497 # def ReadString(s): 2498 # ## 2499 # pass 2500 2501 if not have_lib: 2502 # this is the default flavor only if the 'native' flavor is missing 2503 Node = Node_numpy 2504 NullNode = NullNode_numpy 2505 Open = Open_numpy 2506 ReadBytes = ReadBytes_numpy 2507 FLAVOR = 'numpy' 2508 2509 2510 2511 if __name__ == '__main__': # tests for the 'numpy' flavor (node.storage is not a field in 'native') 2512 CHOOSE_FLAVOR('numpy') 2513

2514 - def presume(condition, comment=''):

2515 if not condition: 2516 raise Exception(comment)

2517

2518 - def bin_compare(lbl, apath, bpath):

2519 adat = open(apath, 'rb').read() 2520 bdat = open(bpath, 'rb').read() 2521 if len(adat) != len(bdat): 2522 print lbl,"Length diff: A:%d B:%d" % (len(adat), len(bdat)) 2523 for i in xrange(min(len(adat), len(bdat))): 2524 if adat[i] != bdat[i]: 2525 print lbl,'%i:\t%s\t%s' % (i, adat[i], bdat[i])

2526

2527 - def build_links(root):

2528 presume(root == root) 2529 child = root['child'] 2530 presume(child) 2531 presume(child.parent) 2532 presume(child.parent == root) 2533 left = root.insert('left') 2534 presume(root.child == left) 2535 presume(left.sibling == child) 2536 right = root.insert(Node('right'), child) 2537 presume(child.sibling == right) 2538 presume(right.parent == root) 2539 root.insertClone(root, True, left).name = 'clone' 2540 presume(root['clone'].sibling == child) 2541 root.appendClone(root, True).name = 'clown' 2542 presume(root.find('clown')) 2543 presume(root.clone().getBytes() == root.getBytes()) 2544 root.append('child') # another 2545 child = root.child 2546 presume(child.name == 'left') 2547 child = child.sibling 2548 presume(child.name == 'clone') 2549 child = child.sibling 2550 presume(child.name == 'child') 2551 presume(child.next('clown').name == 'clown') 2552 child = child.nextSameName() 2553 presume(child.name == 'child') 2554 presume(not child.sibling) 2555 presume(child.sibling.isNull) 2556 root.remove(child) 2557 presume(not child.parent) 2558 presume(tuple(root.list()) == ('left', 'clone', 'child', 'right', 'clown')) 2559 presume(len(root.list()) == len(root))

2560

2561 - def build_data(root):

2562 if not root: return 2563 root['str'].data = 'foobar' 2564 presume(root['str'].data.type == QTR_TYPE_STRING) 2565 presume(str(root['str'].data) == 'foobar') 2566 root['str2'].data = root['str'].data 2567 presume(str(root['str2'].data) == 'foobar') 2568 root['str2'].data = None 2569 presume(root['str2'].data.count == 0) 2570 root.remove(root['str2']) 2571 root['float'].data.setup(QTR_TYPE_FLOAT, 1, 1) 2572 root['float'].data[0] = 4.5 2573 presume(root['float'].data[0] == 4.5) 2574 presume(str(root['float'].storage.data.dtype) == 'float32') 2575 root['double'].data = [3, 4.0, 5] 2576 presume(root['double'].data.type == QTR_TYPE_DOUBLE) 2577 presume(root['double'].data[2] == 5.0) 2578 root['1d'].data = (1, 2, 3) 2579 presume(root['1d'].data[0] == 1) 2580 root['1d'].data = [1.0, 2.0] 2581 presume(root['1d'].data.type == QTR_TYPE_DOUBLE) 2582 presume(root['1d'].data.rows == 2) 2583 presume(root['1d'].data.rows == root['1d'].data.count) 2584 presume(root['1d'].data[1] == 2.0) 2585 root['2d'].data = ((1,2,3),(4,5,6)) 2586 presume(1 < root['2d'].data.rows < root['2d'].data.cols < 4) 2587 presume(root['2d'].data.row(1)[0] == 4) 2588 presume(root['2d'].data.type == QTR_TYPE_INT) 2589 presume(root['2d'].storage.data[0,1] == 2) 2590 root['2d2'].data = root['2d'].data 2591 open('/tmp/qtr_2d', 'wb').write(root['2d'].getBytes()) 2592 _2dc = root['2d2'].clone() 2593 _2dc.name = '2d' 2594 open('/tmp/qtr_2d2', 'wb').write(_2dc.getBytes()) 2595 bin_compare('2d vs 2d2', '/tmp/qtr_2d', '/tmp/qtr_2d2') 2596 root['2d'].data.resize(5) 2597 root['2d2'].data.resize(1) 2598 presume(root['2d'].data.count == 15) 2599 presume(root['2d2'].data.count == 3) 2600 presume(tuple(root['2d'].data[:3]) == (1,2,3)) 2601 root['2d'].data[12:15] = (6, 6, 6) 2602 presume(tuple(root['2d'].storage.data[4,:]) == (6, 6, 6)) 2603 root['2d'].data.col(2)[2:4] = (7, 8) 2604 presume(tuple(root['2d'].storage.data[2:4,2]) == (7, 8)) 2605 root['clr'].data = 'foobar' 2606 presume(str(root['clr'].data) == 'foobar') 2607 root['clr'].data.clear() 2608 presume(root['clr'].data.type == QTR_TYPE_EMPTY) 2609 presume(root.data.node == root)

2610 2611 2612 presume(NullNode() == NullNode()) 2613 2614 root = Node('root') 2615 build_links(root) 2616 build_data(root) 2617 open('/tmp/qtr_gb_root', 'wb').write(root.getBytes()) 2618 assert(not root.path) 2619 root.saveAs('/tmp/qtr_sa_root') 2620 assert(root.path == '/tmp/qtr_sa_root') 2621 root.close() 2622 assert(root.path == '') 2623 froot = Node('root') 2624 froot.saveAs('/tmp/qtr_inf_root') 2625 build_links(froot) 2626 build_data(froot) 2627 froot.save() 2628 froot.close() 2629 assert(root.getBytes() == froot.getBytes()) 2630 2631 del root 2632 del froot 2633 2634 bin_compare('getbytes vs saveas', '/tmp/qtr_gb_root', '/tmp/qtr_sa_root') 2635 # these next two should differ because of the removal and the renaming: 2636 #bin_compare('getbytes vs buildin', '/tmp/qtr_gb_root', '/tmp/qtr_inf_root') 2637 #bin_compare('saveas vs buildin', '/tmp/qtr_sa_root', '/tmp/qtr_inf_root') 2638 # but these rebuild it compact, then test Open/ro and ReadBytes: 2639 f = Open('/tmp/qtr_inf_root', True) 2640 presume(f) 2641 presume(not f.modified) 2642 open('/tmp/qtr_infb_root', 'wb').write(f.getBytes()) 2643 del f 2644 bin_compare('getbytes vs buildin/Open/getbytes', '/tmp/qtr_gb_root', '/tmp/qtr_infb_root') 2645 f = ReadBytes(open('/tmp/qtr_inf_root', 'rb').read()) 2646 presume(f) 2647 open('/tmp/qtr_infbb_root', 'wb').write(f.getBytes()) 2648 del f 2649 bin_compare('saveas vs buildin/ReadBytes/getbytes', '/tmp/qtr_gb_root', '/tmp/qtr_infb_root') 2650 2651 # open, edit, save, close, reopen, check 2652 f = Open('/tmp/qtr_gb_root') 2653 f.remove(f['left']) 2654 f['child'].data = 4.2 2655 f['int'].data = 21 2656 presume(f.modified) 2657 f.save() 2658 f.close() 2659 f = Open('/tmp/qtr_gb_root', True) 2660 presume(not ('left' in f.list())) 2661 presume(f['child'].data[0] == 4.2) 2662 presume(f['int'].data[0] == 21) 2663 del f 2664 2665 2666 # not done on purpose: 2667 # ReadString, ReadText, .lock(), .unlock(), mutable characters in string data in open file. 2668 2669 2670 2671 2672 # test inaccessible files and locations; error handling 2673 # re-group methods and classes 2674 # what isn't explained/well? 2675

Source Code for Module qubx.tree