qubx.tree_native

1 """ 2 3 Wrapper for qubtree shared library (gui mainly uses numpy variant from qubx.tree); 4 this is the 'native' flavor, unless you have the obsolete _qubtree extension library. 5 6 Copyright 2007-2014 Research Foundation State University of New York 7 This file is part of QUB Express. 8 9 QUB Express is free software; you can redistribute it and/or modify 10 it under the terms of the GNU General Public License as published by 11 the Free Software Foundation, either version 3 of the License, or 12 (at your option) any later version. 13 14 QUB Express is distributed in the hope that it will be useful, 15 but WITHOUT ANY WARRANTY; without even the implied warranty of 16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 17 GNU General Public License for more details. 18 19 You should have received a copy of the GNU General Public License, 20 named LICENSE.txt, in the QUB Express program directory. If not, see 21 <http://www.gnu.org/licenses/>. 22 23 """ 24 25 26 27 from qubx.fast.fast_utils import * 28 from ctypes import * 29 import collections 30 import linecache 31 import numpy 32 import sys 33 import weakref 34 35 try: 36 qubtree = cdll.LoadLibrary('qubtree.dll') 37 except: 38 # RTLD_GLOBAL makes symbols available to subsequent ctypes LoadLibrary 39 try: 40 qubtree = ctypes.CDLL('libqubtree.so', mode=ctypes.RTLD_GLOBAL) 41 except OSError: 42 qubtree = ctypes.CDLL('@executable_path/../Frameworks/libqubtree.so', mode=ctypes.RTLD_GLOBAL) 43 44 45 QTR_TYPE_EMPTY, \ 46 QTR_TYPE_UNKNOWN, \ 47 QTR_TYPE_POINTER, \ 48 QTR_TYPE_STRING, \ 49 QTR_TYPE_UCHAR, \ 50 QTR_TYPE_CHAR, \ 51 QTR_TYPE_USHORT, \ 52 QTR_TYPE_SHORT, \ 53 QTR_TYPE_UINT, \ 54 QTR_TYPE_INT, \ 55 QTR_TYPE_ULONG, \ 56 QTR_TYPE_LONG, \ 57 QTR_TYPE_FLOAT, \ 58 QTR_TYPE_DOUBLE, \ 59 QTR_TYPE_LDOUBLE = range(15) 60 TYPES = (QTR_TYPE_EMPTY, 61 QTR_TYPE_UNKNOWN, 62 QTR_TYPE_POINTER, 63 QTR_TYPE_STRING, 64 QTR_TYPE_UCHAR, 65 QTR_TYPE_CHAR, 66 QTR_TYPE_USHORT, 67 QTR_TYPE_SHORT, 68 QTR_TYPE_UINT, 69 QTR_TYPE_INT, 70 QTR_TYPE_ULONG, 71 QTR_TYPE_LONG, 72 QTR_TYPE_FLOAT, 73 QTR_TYPE_DOUBLE, 74 QTR_TYPE_LDOUBLE) 75 76 # for human consumption: 77 TYPENAMES = {QTR_TYPE_EMPTY: "empty", 78 QTR_TYPE_UNKNOWN: "unknown", 79 QTR_TYPE_POINTER : "pointer", 80 QTR_TYPE_STRING : "string", 81 QTR_TYPE_UCHAR : "unsigned 8-bit int", 82 QTR_TYPE_CHAR : "signed 8-bit int", 83 QTR_TYPE_USHORT : "unsigned 16-bit int", 84 QTR_TYPE_SHORT : "signed 16-bit int", 85 QTR_TYPE_UINT : "unsigned int", 86 QTR_TYPE_INT : "int", 87 QTR_TYPE_ULONG : "unsigned 64-bit int", 88 QTR_TYPE_LONG : "signed 64-bit int", 89 QTR_TYPE_FLOAT : "single-precision float", 90 QTR_TYPE_DOUBLE : "float", 91 QTR_TYPE_LDOUBLE : "extended-precision float" 92 } 93 94 TYPES_NUMPY = [None, 95 None, 96 numpy.dtype('<u4'), 97 numpy.dtype('<u1'), 98 numpy.dtype('<u1'), 99 numpy.dtype('<i1'), 100 numpy.dtype('<u2'), 101 numpy.dtype('<i2'), 102 numpy.dtype('<u4'), 103 numpy.dtype('<i4'), 104 numpy.dtype('<u8'), 105 numpy.dtype('<i8'), 106 numpy.dtype('<f4'), 107 numpy.dtype('<f8'), 108 numpy.dtype('complex128')] # 'float80'] 109 110 TYPES_CTYPES = [None, 111 c_uint8, 112 c_uint32, 113 c_char, 114 c_uint8, 115 c_int8, 116 c_uint16, 117 c_int16, 118 c_uint32, 119 c_int32, 120 c_uint64, 121 c_int64, 122 c_float, 123 c_double, 124 c_double] # 'float80'] 125 126 127 TYPE_OF_NUMPY = collections.defaultdict(lambda:QTR_TYPE_DOUBLE) 128 for i in xrange(QTR_TYPE_STRING, len(TYPES_NUMPY)): 129 TYPE_OF_NUMPY[str(TYPES_NUMPY[i])] = i 130 TYPE_OF_NUMPY['bool'] = QTR_TYPE_INT 131 132 # size of one element, in bytes 133 SIZE_OF_TYPE = (0, 0, 4, 1, 1, 1, 2, 2, 4, 4, 8, 8, 4, 8, 10) 134 135 QTR_FLAG_PRELOAD = 0x2 136 QTR_FLAG_DATA_IN_NODE = 0x4 137 QTR_FLAG_CHANGED = 0x1000 138 QTR_FLAG_CHILDCHANGED = 0x2000 139

140 -class NodeStruct(Structure):

141 _pack_ = 1

142 PNode = POINTER(NodeStruct) 143 NodeStruct._fields_ = [('flags', c_short), 144 ('flags_rsv', c_uint8), 145 ('dataType', c_uint8), 146 ('dataSize', c_int), 147 ('dataCount', c_int), 148 ('dataPos', c_int), 149 ('childPos', c_int), 150 ('siblingPos', c_int), 151 ('reserved', 7*c_char), 152 ('nameLen', c_uint8), 153 154 ('refs', c_int), 155 ('name', c_void_p), 156 ('parent', PNode), 157 ('child', PNode), 158 ('sibling', PNode), 159 ('loadStart', c_int), 160 ('loadEnd', c_int), 161 ('data', c_void_p), 162 ('file', c_void_p), 163 ('fileStart', c_int), 164 ('mutex', c_void_p), 165 ('readers', c_int), 166 ('root', PNode), 167 ('dataMem', c_void_p)] 168

169 -def ploc(p):

170 return 0 if ((p is None) or not bool(p)) else addressof(p.contents)

171

172 -class Node(object):

173 - def __init__(self, name="", impl=None):

174 if not (impl is None): 175 if impl: # this could be more indirect than you think, e.g. impl=some_node.child is the child field INSIDE some_node 176 # so it will not stay the same if some_node.child changes! 177 # make a distinct memory location just in case. 178 my_impl = PNode() 179 my_impl.contents = impl.contents 180 else: 181 my_impl = impl 182 object.__setattr__(self, 'impl', my_impl or cast(0, PNode)) # bypass self.__setattr__ which reads self.impl 183 if my_impl: 184 qubtree.QTR_INCREF(my_impl) 185 else: 186 object.__setattr__(self, 'impl', qubtree.QTR_Create(name))

187 - def __del__(self):

188 if ploc(self.impl): 189 qubtree.QTR_DECREF(self.impl)

190 - def __cmp__(self, other):

191 if isinstance(other, Node): 192 return cmp(ploc(self.impl), ploc(other.impl)) 193 else: 194 return cmp(ploc(self.impl), id(other))

195 - def __nonzero__(self):

196 return bool(ploc(self.impl))

197 - def __repr__(self):

198 name = str(qubtree.QTR_Name(self.impl)) if ploc(self.impl) else "NULL" 199 return "<QUBTree Node : %s>" % name

200 - def __str__(self):

201 result_ptr = qubtree.QTR_ToCString(self.impl) 202 result = cast(result_ptr, c_char_p).value 203 qubtree.QTR_FreeDataCString(result_ptr) 204 return result

205 - def __getattr__(self, key):

206 if key == 'storage': 207 return Storage(self.impl) 208 if key == 'data': 209 return Data(self.impl) 210 if not ploc(self.impl): 211 if key in ('parent', 'child', 'sibling'): 212 return NullNode() 213 if key == 'isNull': 214 return True 215 if key in ('name', 'path'): 216 return "" 217 if key in ('refs', 'modified'): 218 return 0 219 else: 220 if key == 'name': 221 return str(qubtree.QTR_Name(self.impl)) 222 if key == 'parent': 223 return Node(impl=self.impl[0].parent) 224 if key == 'child': 225 return Node(impl=self.impl[0].child) 226 if key == 'sibling': 227 return Node(impl=self.impl[0].sibling) 228 if key == 'isNull': 229 return False 230 if key == 'refs': 231 return self.impl[0].refs 232 if key == 'path': 233 path = qubtree.QTR_FilePath(self.impl) 234 if path is None: 235 return "" 236 return str(path) 237 if key == 'modified': 238 return bool(qubtree.QTR_Flag(self.impl, QTR_FLAG_CHANGED) or qubtree.QTR_Flag(self.impl, QTR_FLAG_CHILDCHANGED)) 239 raise AttributeError("QUBTree Node object has no attribute '%s'" % key)

240 - def __setattr__(self, key, val):

241 if not ploc(self.impl): 242 return # null, don't care 243 if key == 'name': 244 qubtree.QTR_SetName(self.impl, str(val)) 245 return 246 if key == 'data': 247 d = self.data 248 if val is None: 249 d.clear() 250 return 251 if isinstance(val, Data): 252 qubtree.QTR_ImitateData(self.impl, val.impl) 253 return 254 if not isinstance(val, str): 255 try: 256 d.setup(TYPE_OF_NUMPY[str(val.dtype)], *val.shape) 257 if len(val.shape) == 0: 258 d.get_rows(0, 0)[0,0] = val 259 elif len(val.shape) == 1: 260 d.get_rows(0, val.shape[0]-1)[:,0] = val 261 else: 262 d.get_rows(0, val.shape[0]-1)[:] = val 263 return 264 except AttributeError: # val is not numpy 265 pass 266 except (IndexError, ValueError): # wrong dimensions 267 pass 268 if hasattr(val, '__iter__'): 269 try: 270 all_int = True 271 for v in val: 272 if not isinstance(v, int): 273 all_int = False 274 break 275 if all_int: 276 d.setup(QTR_TYPE_INT, len(val), 1) 277 d[:] = val 278 return 279 else: 280 fvals = [float(v) for v in val] 281 d.setup(QTR_TYPE_DOUBLE, len(fvals), 1) 282 d[:] = fvals 283 return 284 except TypeError: # val is not a num. sequence 285 pass 286 try: 287 if isinstance(val, int): 288 d.setup(QTR_TYPE_INT, 1, 1) 289 d[0] = val 290 return 291 fval = float(val) 292 d.setup(QTR_TYPE_DOUBLE, 1, 1) 293 d[0] = fval 294 return 295 except TypeError: # val is not numeric 296 pass 297 val = str(val) 298 n = len(val) 299 d.setup(QTR_TYPE_STRING, n, 1) 300 if n: 301 d[:n] = val 302 return 303 raise AttributeError("QUBTree Node object has no (settable) attribute '%s'" % key)

304 - def __len__(self):

305 count = 0 306 if ploc(self.impl): 307 child = self.impl[0].child 308 while bool(child): 309 count += 1 310 child = child[0].sibling 311 return count

312 - def __getitem__(self, key):

313 try: 314 if int(key+1) == (key+1): # is numeric 315 ix = int(key) 316 at = 0 317 ch = self.impl[0].child 318 while bool(ch) and (at < ix): 319 ch = ch[0].sibling 320 at += 1 321 return Node(impl=ch) 322 except TypeError: 323 pass 324 child = self.find(key) 325 if child.isNull: 326 child = self.insert(str(key)) 327 return child

328 - def save(self):

329 if ploc(self.impl): 330 return 0 <= qubtree.QTR_Save(self.impl) 331 return False

332 - def saveAs(self, path, as_copy=False):

333 if ploc(self.impl): 334 if as_copy: 335 try: 336 open(path, 'wb').write(self.getBytes()) 337 return True 338 except: 339 traceback.print_exc() 340 if 0 <= qubtree.QTR_SaveAs(self.impl, path): 341 return self.save() 342 return False

343 - def close(self):

344 if ploc(self.impl): 345 qubtree.QTR_Close(self.impl) 346 return True

347 - def clone(self, deep=True):

348 if ploc(self.impl): 349 cl = Node(impl=qubtree.QTR_Clone(self.impl, int(deep))) 350 qubtree.QTR_DECREF(cl.impl) 351 return cl 352 return self

353 - def list(self):

354 names = [] 355 if ploc(self.impl): 356 ch = self.impl[0].child 357 while bool(ch): 358 names.append(str(qubtree.QTR_Name(ch))) 359 ch = ch[0].sibling 360 return names

361 - def find(self, name):

362 if ploc(self.impl): 363 name_interned = qubtree.QTR_LookupName(str(name)) 364 ch = self.impl[0].child 365 while bool(ch): 366 if name_interned == ch[0].name: 367 return Node(impl=ch) 368 ch = ch[0].sibling 369 return NullNode()

370 - def next(self, name=None):

371 if not ploc(self.impl): 372 return NullNode() 373 if name is None: 374 return Node(impl=self.impl[0].sibling) 375 name_interned = qubtree.QTR_LookupName(name) 376 nx = self.impl[0].sibling 377 while ploc(nx): 378 if name_interned == nx[0].name: 379 return Node(impl=nx) 380 nx = nx[0].sibling 381 return NullNode()

382 - def nextSameName(self):

383 if not self.impl: 384 return NullNode() 385 name_interned = self.impl[0].name 386 nx = self.impl[0].sibling 387 while bool(nx): 388 if name_interned == nx[0].name: 389 return Node(impl=nx) 390 nx = nx[0].sibling 391 return NullNode()

392 - def append(self, child_or_name):

393 if not ploc(self.impl): 394 return self 395 try: 396 if not ploc(child_or_name.impl): # raises AttributeError e.g. if it's a name string 397 return child_or_name # don't append null 398 child = child_or_name 399 except AttributeError: 400 # must be a name 401 child = Node(child_or_name) 402 after = self.impl[0].child 403 while ploc(after) and bool(after[0].sibling): 404 after = after[0].sibling 405 qubtree.QTR_InsertChild(self.impl, after, child.impl) 406 return child

407 - def appendClone(self, node, deep=True):

408 if (not ploc(self.impl)) or (not ploc(node.impl)): 409 return NullNode() 410 after = self.impl[0].child 411 while ploc(after) and ploc(after[0].sibling): 412 after = after[0].sibling 413 clone = Node(impl=qubtree.QTR_InsertClone(self.impl, after, node.impl, int(deep))) 414 qubtree.QTR_DECREF(clone.impl) 415 return clone

416 - def insert(self, child_or_name, after=None):

417 if not ploc(self.impl): 418 return NullNode() 419 try: 420 if not ploc(child_or_name.impl): # raises AttributeError e.g. if it's a name string 421 return child_or_name # don't insert null 422 child = child_or_name 423 except AttributeError: 424 # must be a name 425 child = Node(child_or_name) 426 after_impl = None if (after is None) else after.impl 427 qubtree.QTR_InsertChild(self.impl, after_impl, child.impl) 428 return child

429 - def insertClone(self, node, deep=True, after=None):

430 if (not ploc(self.impl)) or (not ploc(node.impl)): 431 return NullNode() 432 after_impl = None if (after is None) else after.impl 433 clone = Node(impl=qubtree.QTR_InsertClone(self.impl, after_impl, node.impl, int(deep))) 434 qubtree.QTR_DECREF(clone.impl) 435 return clone

436 - def remove(self, child, after=None):

437 if (not ploc(self.impl)) or (not ploc(child.impl)): 438 return 439 prev = self.impl[0].child 440 if ploc(child.impl) == ploc(prev): 441 qubtree.QTR_RemoveChild(self.impl, None, child.impl) 442 else: 443 while ploc(prev) and (ploc(prev[0].sibling) != ploc(child.impl)): 444 prev = prev[0].sibling 445 if ploc(prev): 446 qubtree.QTR_RemoveChild(self.impl, prev, child.impl)

447 - def lock(self, timeout_ms=None):

448 if ploc(self.impl): 449 return qubtree.QTR_Lock(self.impl, timeout_ms) 450 return True

451 - def unlock(self):

452 if ploc(self.impl): 453 qubtree.QTR_Unlock(self.impl)

454 - def getBytes(self):

455 size = qubtree.QTR_ToBufferSize(self.impl) 456 if size <= 0: 457 return "" 458 buf = create_string_buffer(size) 459 qubtree.QTR_ToBuffer(self.impl, buf) 460 return buf.raw

461 462

463 -def NullNode():

464 return Node(impl=0)

465

466 -def Open(path, readOnly=False):

467 result = Node(impl=qubtree.QTR_Open(path, readOnly)) 468 if ploc(result.impl): 469 qubtree.QTR_DECREF(result.impl) 470 return result

471

472 -def ReadBytes(bytes):

473 result = Node(impl=qubtree.QTR_FromBuffer(bytes)) 474 if ploc(result.impl): 475 qubtree.QTR_DECREF(result.impl) 476 return result

477 478

479 -class Data(object):

480 - def __init__(self, impl):

481 object.__setattr__(self, 'impl', impl or cast(0, PNode)) # bypass self.__setattr__ which reads self.impl 482 #self.__dict__['impl'] = impl or cast(0, PNode) # bypass self.__setattr__ which reads self.impl 483 if impl: 484 qubtree.QTR_INCREF(impl)

485 - def __del__(self):

486 if ploc(self.impl): 487 qubtree.QTR_DECREF(self.impl)

488 - def __cmp__(self, other):

489 if isinstance(other, Data): 490 return cmp(ploc(self.impl), ploc(other.impl)) 491 else: 492 return cmp(ploc(self.impl), id(other))

493 - def __nonzero__(self):

494 return bool(ploc(self.impl) and self.impl[0].dataCount)

495 - def __repr__(self):

496 if ploc(self.impl): 497 name = str(qubtree.QTR_Name(self.impl)) 498 nr = self.impl[0].dataCount 499 nc = qubtree.QTR_NumCol(self.impl) 500 dtype = TYPENAMES[self.impl[0].dataType] 501 return "<QUBTree Node data for %s: %ix%i of %s>" % (name, nr, nc, dtype) 502 return "<QUBTree null node data>"

503 - def __str__(self):

504 result_ptr = qubtree.QTR_DataToCString(self.impl) 505 result = cast(result_ptr, c_char_p).value 506 qubtree.QTR_FreeDataCString(result_ptr) 507 return result

508 - def __getattr__(self, key):

509 if key == 'node': 510 return Node(impl=self.impl) 511 if not ploc(self.impl): 512 if key in ('type', 'size', 'count', 'rows', 'cols', 'preload'): 513 return 0 514 if key in ('loaded', 'loadedRows'): 515 return (-1, -1) 516 else: 517 o = self.impl[0] 518 if key == 'type': 519 return o.dataType 520 if key == 'size': 521 return o.dataSize 522 if key == 'count': 523 return o.dataCount * qubtree.QTR_NumCol(self.impl) 524 if key == 'rows': 525 return o.dataCount 526 if key == 'cols': 527 return qubtree.QTR_NumCol(self.impl) 528 if key == 'preload': 529 return qubtree.QTR_Flag(self.impl, QTR_FLAG_PRELOAD) 530 if key == 'loaded': 531 nc = qubtree.QTR_NumCol(self.impl) 532 return (o.loadStart * nc, (o.loadEnd+1) * nc - 1) 533 if key == 'loadedRows': 534 return (o.loadStart, o.loadEnd) 535 raise AttributeError("QUBTree Data object has no (settable) attribute '%s'" % key)

536 - def __setattr__(self, key, val):

537 if not ploc(self.impl): 538 return # null, don't care 539 if key == 'preload': 540 qubtree.QTR_SetFlag(self.impl, QTR_FLAG_PRELOAD, int(val)) 541 return 542 raise AttributeError("QUBTree Node object has no (settable) attribute '%s'" % key)

543 - def __len__(self):

544 return ploc(self.impl) and self.impl[0].dataCount

545 - def __getitem__(self, key):

546 lf, ll = self.loaded 547 if lf < 0: 548 raise IndexError('No data or none loaded') 549 if self.type == QTR_TYPE_STRING: 550 buf = pybuf(cast(self.impl[0].data, c_char_p), ll-lf+1) 551 else: 552 buf = NodeDataAsArray(self.impl).flat 553 if isinstance(key, slice): 554 return [buf[ii-lf] for ii in xrange(*key.indices(lf+len(buf)))] 555 return buf[key-lf]

556 - def __setitem__(self, key, val):

557 lf, ll = self.loaded 558 if lf < 0: 559 raise IndexError('No data or none loaded') 560 if self.type == QTR_TYPE_STRING: 561 buf = pybuf(cast(self.impl[0].data, c_char_p), ll-lf+1) 562 else: 563 buf = NodeDataAsArray(self.impl).reshape(((ll-lf+1),)) 564 if isinstance(key, slice): 565 for i, ii in enumerate(xrange(*key.indices(lf+len(buf)))): 566 try: 567 buf[ii-lf] = val[i] 568 except TypeError: 569 buf[ii-lf] = val 570 else: 571 buf[key-lf] = val

572 - def resize(self, nrow, type_if_none=QTR_TYPE_DOUBLE):

573 if self.count == 0: 574 self.setup(type_if_none) 575 qubtree.QTR_ResizeData(self.impl, nrow)

576 - def clear(self):

577 qubtree.QTR_ClearData(self.impl)

578 - def setup(self, dtype, nr=1, nc=1):

579 qubtree.QTR_SetupNumData(self.impl, dtype, nr, nc)

580 - def loadRows(self, first, last, doRead=True):

581 qubtree.QTR_LoadRows(self.impl, first, last, doRead)

582 - def unloadRows(self, doWrite=True):

583 qubtree.QTR_UnloadRows(self.impl, doWrite)

584 - def read_rows(self, first, last):

585 if self.type == QTR_TYPE_STRING: 586 buf = create_string_buffer(last - first + 2) 587 buf[len(buf)-1] = chr(0) 588 qubtree.QTR_GetRows(self.impl, buf, first, last) 589 return buf.raw 590 buf = numpy.zeros(shape=(last-first+1, self.cols), dtype=TYPES_NUMPY[self.type]) 591 qubtree.QTR_GetRows(self.impl, cdata(buf, POINTER(TYPES_CTYPES[self.type])), first, last) 592 return buf

593 - def write_rows(self, first, last, data):

594 if self.type == QTR_TYPE_STRING: 595 qubtree.QTR_SetRows(self.impl, c_char_p(data), first, last) 596 else: 597 qubtree.QTR_SetRows(self.impl, cdata(data.astype(TYPES_NUMPY[self.type]), TYPES_CTYPES[self.type]), first, last)

598 - def get_rows(self, first, last):

599 lf, ll = self.loadedRows 600 return NodeDataAsArray(self.impl)[first-lf:last+1-lf,:]

601 - def row(self, i):

602 return DataRow(self.impl, i)

603 - def col(self, j):

604 return DataCol(self.impl, j)

605 606

607 -class DataRow(object):

608 - def __init__(self, impl, r):

609 self.impl = impl 610 self.r = r 611 if ploc(impl): 612 qubtree.QTR_INCREF(impl)

613 - def __del__(self):

614 if ploc(self.impl): 615 qubtree.QTR_DECREF(self.impl)

616 - def __cmp__(self, other):

617 if isinstance(other, DataRow): 618 return cmp((ploc(self.impl), self.r), 619 (ploc(other.impl), other.r)) 620 else: 621 return cmp(ploc(self.impl), id(other))

622 - def __repr__(self):

623 if not ploc(self.impl): 624 return "<QUBTree row data of null node>" 625 return "<QUBTree '%s' row %i>" % (str(qubtree.QTR_Name(self.impl)), self.r)

626 - def __str__(self):

627 if not ploc(self.impl): 628 return "" 629 if not (self.impl[0].loadStart <= self.r < self.impl[0].loadEnd): 630 return "" 631 return '\t'.join(str(x) for x in NodeDataAsArray(self.impl)[self.r,:].flatten())

632 - def __len__(self):

633 return qubtree.QTR_NumCol(self.impl) if ploc(self.impl) else 0

634 - def __getitem__(self, key):

635 lf, ll = self.impl[0].loadStart, self.impl[0].loadEnd 636 if lf < 0: 637 raise IndexError('No data or none loaded') 638 if self.impl[0].dataType == QTR_TYPE_STRING: 639 buf = pybuf(cast(self.impl[0].data, c_char_p), ll-lf+1)[self.r-lf:] 640 else: 641 buf = NodeDataAsArray(self.impl)[self.r-lf,:] 642 if isinstance(key, slice): 643 return [buf[ii-lf] for ii in xrange(*key.indices(len(self)))] 644 return buf[key-lf]

645 - def __setitem__(self, key, val):

646 lf, ll = self.impl[0].loadStart, self.impl[0].loadEnd 647 if lf < 0: 648 raise IndexError('No data or none loaded') 649 if self.impl[0].dataType == QTR_TYPE_STRING: 650 buf = pybuf(cast(self.impl[0].data, c_char_p), ll-lf+1)[self.r-lf:] 651 else: 652 buf = NodeDataAsArray(self.impl)[self.r-lf,:].flat 653 if isinstance(key, slice): 654 for i, ii in enumerate(xrange(*key.indices(len(self)))): 655 try: 656 buf[ii-lf] = val[i] 657 except TypeError: 658 buf[ii-lf] = val 659 else: 660 buf[key-lf] = val

661 662

663 -class DataCol(object):

664 - def __init__(self, impl, c):

665 self.impl = impl 666 self.c = c 667 if ploc(impl): 668 qubtree.QTR_INCREF(impl)

669 - def __del__(self):

670 if ploc(self.impl): 671 qubtree.QTR_DECREF(self.impl)

672 - def __cmp__(self, other):

673 if isinstance(other, DataCol): 674 return cmp((ploc(self.impl), self.c), 675 (ploc(other.impl), other.c)) 676 else: 677 return cmp(ploc(self.impl), id(other))

678 - def __repr__(self):

679 if not ploc(self.impl): 680 return "<QUBTree column data of null node>" 681 return "<QUBTree '%s' column %i>" % (str(qubtree.QTR_Name(self.impl)), self.c)

682 - def __str__(self):

683 if (not ploc(self.impl)) or (self.impl[0].loadStart < 0): 684 return "" 685 return '\t'.join(str(x) for x in NodeDataAsArray(self.impl)[:,self.c].flatten())

686 - def __len__(self):

687 return self.impl[0].dataCount if ploc(self.impl) else 0

688 - def __getitem__(self, key):

689 lf, ll = self.impl[0].loadStart, self.impl[0].loadEnd 690 if lf < 0: 691 raise IndexError('No data or none loaded') 692 if self.impl[0].dataType == QTR_TYPE_STRING: 693 buf = pybuf(cast(self.impl[0].data, c_char_p), ll-lf+1) 694 else: 695 buf = NodeDataAsArray(self.impl)[:,self.c] 696 if isinstance(key, slice): 697 return [buf[ii-lf] for ii in xrange(*key.indices(len(self)))] 698 return buf[key-lf]

699 - def __setitem__(self, key, val):

700 lf, ll = self.impl[0].loadStart, self.impl[0].loadEnd 701 if lf < 0: 702 raise IndexError('No data or none loaded') 703 if self.impl[0].dataType == QTR_TYPE_STRING: 704 buf = pybuf(cast(self.impl[0].data, c_char_p), ll-lf+1) 705 else: 706 buf = NodeDataAsArray(self.impl)[:,self.c] 707 if isinstance(key, slice): 708 for i, ii in enumerate(xrange(*key.indices(len(self)))): 709 try: 710 buf[ii-lf] = val[i] 711 except TypeError: 712 buf[ii-lf] = val 713 else: 714 buf[key-lf] = val

715 716

717 -class Storage(object):

718 # for numpy-flavor compat. in qubx.data_types, node.storage.data gives direct access

719 - def __init__(self, impl):

720 self.impl = impl or cast(0, PNode) 721 if impl: 722 qubtree.QTR_INCREF(impl)

723 - def __del__(self):

724 if ploc(self.impl): 725 qubtree.QTR_DECREF(self.impl)

726 data = property(lambda self: self.get_data())

727 - def get_data(self):

728 """Imitates Node_numpy.storage.data as a direct view on node data.""" 729 if (not ploc(self.impl)) or (not self.impl[0].dataType) or (self.impl[0].loadStart < 0): 730 return None # make it clearly unavailable when not loaded 731 if self.impl[0].dataType == QTR_TYPE_STRING: 732 return pybuf(cast(self.impl[0].data, c_char_p), self.impl[0].loadEnd - self.impl[0].loadStart + 1) 733 return NodeDataAsArray(self.impl)

734 735

736 -def NodeDataAsArray(impl):

737 "Returns a node's data as a 2-d numpy.array." 738 if (not ploc(impl)) or (not impl[0].dataType) or (impl[0].loadStart < 0): 739 return numpy.array([]) 740 rec = impl[0] 741 nr = rec.loadEnd - rec.loadStart + 1 # impl[0].dataCount 742 nc = qubtree.QTR_NumCol(impl) 743 dtype = TYPES_NUMPY[rec.dataType] 744 ptr = cast(rec.data, POINTER(TYPES_CTYPES[rec.dataType])) 745 return numpy.frombuffer(pybuf(ptr, nr*rec.dataSize), dtype=dtype, count=nr*nc).reshape((nr, nc))

746 747 748 qubtree.QTR_LookupName.argtypes = (c_char_p,) 749 qubtree.QTR_LookupName.restype = c_void_p 750 qubtree.QTR_INCREF.argtypes = (PNode,) 751 qubtree.QTR_INCREF.restype = c_int 752 qubtree.QTR_DECREF.argtypes = (PNode,) 753 qubtree.QTR_DECREF.restype = c_int 754 qubtree.QTR_Lock.argtypes = (PNode, c_int) 755 qubtree.QTR_Lock.restype = c_int 756 qubtree.QTR_Unlock.argtypes = (PNode,) 757 qubtree.QTR_Unlock.restype = None 758 qubtree.QTR_Create.argtypes = (c_char_p,) 759 qubtree.QTR_Create.restype = PNode 760 qubtree.QTR_Clone.argtypes = (PNode, c_int) 761 qubtree.QTR_Clone.restype = PNode 762 qubtree.QTR_Open.argtypes = (c_char_p, c_int) 763 qubtree.QTR_Open.restype = PNode 764 qubtree.QTR_Close.argtypes = (PNode,) 765 qubtree.QTR_Close.restype = None 766 qubtree.QTR_Save.argtypes = (PNode,) 767 qubtree.QTR_Save.restype = c_int 768 qubtree.QTR_SaveAs.argtypes = (PNode, c_char_p) 769 qubtree.QTR_SaveAs.restype = c_int 770 qubtree.QTR_SaveAsTemp.argtypes = (PNode,) 771 qubtree.QTR_SaveAsTemp.restype = c_int 772 qubtree.QTR_CreateChild.argtypes = (PNode, PNode, c_char_p) 773 qubtree.QTR_CreateChild.restype = PNode 774 qubtree.QTR_InsertClone.argtypes = (PNode, PNode, PNode, c_int) 775 qubtree.QTR_InsertClone.restype = PNode 776 qubtree.QTR_InsertChild.argtypes = (PNode, PNode, PNode) 777 qubtree.QTR_InsertChild.restype = None 778 qubtree.QTR_RemoveChild.argtypes = (PNode, PNode, PNode) 779 qubtree.QTR_RemoveChild.restype = None 780 qubtree.QTR_Flag.argtypes = (PNode, c_int) 781 qubtree.QTR_Flag.restype = c_int 782 qubtree.QTR_SetFlag.argtypes = (PNode, c_int, c_int) 783 qubtree.QTR_SetFlag.restype = None 784 qubtree.QTR_Changed.argtypes = (PNode,) 785 qubtree.QTR_Changed.restype = None 786 qubtree.QTR_ChildChanged.argtypes = (PNode,) 787 qubtree.QTR_ChildChanged.restype = None 788 qubtree.QTR_Name.argtypes = (PNode,) 789 qubtree.QTR_Name.restype = c_char_p 790 qubtree.QTR_SetName.argtypes = (PNode, c_char_p) 791 qubtree.QTR_SetName.restype = None 792 qubtree.QTR_IsInFile.argtypes = (PNode,) 793 qubtree.QTR_IsInFile.restype = c_int 794 qubtree.QTR_FilePath.argtypes = (PNode,) 795 qubtree.QTR_FilePath.restype = c_char_p 796 qubtree.QTR_SetupData.argtypes = (PNode, c_int, c_int, c_int) 797 qubtree.QTR_SetupData.restype = c_int 798 qubtree.QTR_SetupNumData.argtypes = (PNode, c_int, c_int, c_int) 799 qubtree.QTR_SetupNumData.restype = c_int 800 qubtree.QTR_SetupStringData.argtypes = (PNode, c_int) 801 qubtree.QTR_SetupStringData.restype = c_int 802 qubtree.QTR_ResizeData.argtypes = (PNode, c_int) 803 qubtree.QTR_ResizeData.restype = c_int 804 qubtree.QTR_ClearData.argtypes = (PNode,) 805 qubtree.QTR_ClearData.restype = c_int 806 qubtree.QTR_NumCol.argtypes = (PNode,) 807 qubtree.QTR_NumCol.restype = c_int 808 qubtree.QTR_LoadRows.argtypes = (PNode, c_int, c_int, c_int) 809 qubtree.QTR_LoadRows.restype = c_int 810 qubtree.QTR_UnloadRows.argtypes = (PNode, c_int) 811 qubtree.QTR_UnloadRows.restype = c_int 812 qubtree.QTR_GetRows.argtypes = (PNode, c_void_p, c_int, c_int) 813 qubtree.QTR_GetRows.restype = c_int 814 qubtree.QTR_SetRows.argtypes = (PNode, c_void_p, c_int, c_int) 815 qubtree.QTR_SetRows.restype = c_int 816 qubtree.QTR_ToBufferSize.argtypes = (PNode,) 817 qubtree.QTR_ToBufferSize.restype = c_int 818 qubtree.QTR_ToBuffer.argtypes = (PNode, c_char_p) 819 qubtree.QTR_ToBuffer.restype = c_int 820 qubtree.QTR_FromBuffer.argtypes = (c_char_p,) 821 qubtree.QTR_FromBuffer.restype = PNode 822 823 qubtree.QTR_DataToCString.argtypes = (PNode,) 824 qubtree.QTR_DataToCString.restype = c_void_p 825 qubtree.QTR_ToCString.argtypes = (PNode,) 826 qubtree.QTR_ToCString.restype = c_void_p 827 qubtree.QTR_FreeDataCString.argtypes = (c_void_p,) 828 qubtree.QTR_FreeDataCString.restype = None 829 qubtree.QTR_FromString.argtypes = (c_char_p,) 830 qubtree.QTR_FromString.restype = PNode 831 832 qubtree.QTR_ImitateData.argtypes = (PNode, PNode) 833 qubtree.QTR_ImitateData.restype = None 834 835 836 if __name__ == '__main__': 837

838 - def traceit(frame, event, arg):

839 if event == "line": 840 lineno = frame.f_lineno 841 filename = frame.f_globals["__file__"] 842 if filename == "<stdin>": 843 filename = "<stdin> " 844 if (filename.endswith(".pyc") or 845 filename.endswith(".pyo")): 846 filename = filename[:-1] 847 name = frame.f_globals["__name__"] 848 line = linecache.getline(filename, lineno) 849 sys.stdout.write("%s:%s: %s\n" % (name, lineno, line.rstrip())) 850 try: 851 print (" -- r2d -- %s"%r2d) 852 print (" -- r2d -- %s"%r2d.impl) 853 print (" -- r2d -- %s"%ploc(r2d.impl)) 854 except NameError: 855 pass 856 except: 857 traceback.print_exc() 858 sys.stdout.flush() 859 return traceit

860 #sys.settrace(traceit) 861

862 - def presume(condition, comment=''):

863 if not condition: 864 raise Exception(comment)

865

866 - def bin_compare(lbl, apath, bpath):

867 adat = open(apath, 'rb').read() 868 bdat = open(bpath, 'rb').read() 869 if len(adat) != len(bdat): 870 print lbl,"Length diff: A:%d B:%d" % (len(adat), len(bdat)) 871 for i in xrange(min(len(adat), len(bdat))): 872 if adat[i] != bdat[i]: 873 print lbl,'%i:\t%s\t%s' % (i, adat[i], bdat[i])

874

875 - def build_links(root):

876 presume(root == root) 877 child = root['child'] 878 presume(child) 879 presume(child.parent) 880 presume(child.parent == root) 881 left = root.insert('left') 882 presume(root.child == left) 883 presume(left.sibling == child) 884 right = root.insert(Node('right'), child) 885 presume(child.sibling == right) 886 presume(right.parent == root) 887 root.insert(root.clone(True), left).name = 'clone' 888 presume(root['clone'].sibling == child) 889 root.append(root.clone(True)).name = 'clown' 890 presume(root.find('clown')) 891 presume(root.clone().getBytes() == root.getBytes()) 892 root.append('child') # another 893 child = root.child 894 presume(child.name == 'left') 895 child = child.sibling 896 presume(child.name == 'clone') 897 child = child.sibling 898 presume(child.name == 'child') 899 presume(child.next('clown').name == 'clown') 900 child = child.nextSameName() 901 presume(child.name == 'child') 902 presume(not child.sibling) 903 presume(child.sibling.isNull) 904 root.remove(child) 905 presume(not child.parent) 906 presume(tuple(root.list()) == ('left', 'clone', 'child', 'right', 'clown')) 907 presume(len(root.list()) == len(root))

908

909 - def build_data(root):

910 global r2d 911 if not root: return 912 root['str'].data = 'foobar' 913 presume(root['str'].data.type == QTR_TYPE_STRING) 914 presume(str(root['str'].data) == 'foobar') 915 root['str2'].data = str(root['str'].data) # won't copy without str() 916 presume(str(root['str2'].data) == 'foobar') 917 root['str2'].data = None 918 presume(root['str2'].data.count == 0) 919 root.remove(root['str2']) 920 root['float'].data.setup(QTR_TYPE_FLOAT, 1, 1) 921 root['float'].data[0] = 4.5 922 presume(root['float'].data[0] == 4.5) 923 presume(str(root['float'].storage.data.dtype) == 'float32') 924 root['double'].data = [3, 4.0, 5] 925 presume(root['double'].data.type == QTR_TYPE_DOUBLE) 926 presume(root['double'].data[2] == 5.0) 927 root['1d'].data = (1, 2, 3) 928 presume(root['1d'].data[0] == 1) 929 root['1d'].data = [1.0, 2.0] 930 presume(root['1d'].data.type == QTR_TYPE_DOUBLE) 931 presume(root['1d'].data.rows == 2) 932 presume(root['1d'].data.rows == root['1d'].data.count) 933 presume(root['1d'].data[1] == 2.0) 934 root['2d'].data = numpy.matrix('1 2 3; 4 5 6', dtype='int32') # can't handle nested arrays: ((1,2,3),(4,5,6)) 935 presume(1 < root['2d'].data.rows < root['2d'].data.cols < 4) 936 presume(root['2d'].data.row(1)[0] == 4) 937 presume(root['2d'].data.type == QTR_TYPE_INT) 938 presume(root['2d'].data[1] == 2) 939 r2d = root['2d'].data 940 root['2d2'].data = r2d 941 open('/tmp/qtr_2d', 'wb').write(root['2d'].getBytes()) 942 _2dc = root['2d2'].clone() 943 _2dc.name = '2d' 944 open('/tmp/qtr_2d2', 'wb').write(_2dc.getBytes()) 945 bin_compare('2d vs 2d2', '/tmp/qtr_2d', '/tmp/qtr_2d2') 946 root['2d'].data.resize(5) 947 root['2d2'].data.resize(1) 948 presume(root['2d'].data.count == 15) 949 presume(root['2d2'].data.count == 3) 950 presume(tuple(root['2d'].data[:3]) == (1,2,3)) 951 root['2d'].data[7:12] = 0 952 953 root['2d'].data[12:15] = (6, 6, 6) 954 presume(tuple(root['2d'].data[12:15]) == (6, 6, 6)) 955 root['2d'].data.col(2)[2:4] = (7, 8) 956 presume(tuple(root['2d'].data.col(2)[2:4]) == (7, 8)) 957 root['clr'].data = 'foobar' 958 presume(str(root['clr'].data) == 'foobar') 959 root['clr'].data.clear() 960 presume(root['clr'].data.type == QTR_TYPE_EMPTY) 961 presume(root.data.node == root)

962 963 print 'Testing qubx.tree_native...' 964 965 presume(NullNode() == NullNode()) 966 967 root = Node('root') 968 print 'links...' 969 build_links(root) 970 print 'data...' 971 build_data(root) 972 print 'write...' 973 open('/tmp/qtr_gb_root', 'wb').write(root.getBytes()) 974 print repr(root.path) 975 assert(not root.path) 976 root.saveAs('/tmp/qtr_sa_root') 977 assert(root.path == '/tmp/qtr_sa_root') 978 root.close() 979 assert(root.path == '') 980 print 'froot...' 981 froot = Node('root') 982 froot.saveAs('/tmp/qtr_inf_root') 983 build_links(froot) 984 build_data(froot) 985 froot.save() 986 froot.close() 987 assert(root.getBytes() == froot.getBytes()) 988 989 del root 990 del froot 991 992 bin_compare('getbytes vs saveas', '/tmp/qtr_gb_root', '/tmp/qtr_sa_root') 993 # these next two should differ because of the removal and the renaming: 994 #bin_compare('getbytes vs buildin', '/tmp/qtr_gb_root', '/tmp/qtr_inf_root') 995 #bin_compare('saveas vs buildin', '/tmp/qtr_sa_root', '/tmp/qtr_inf_root') 996 # but these rebuild it compact, then test Open/ro and ReadBytes: 997 f = Open('/tmp/qtr_inf_root', True) 998 presume(f) 999 presume(not f.modified) 1000 open('/tmp/qtr_infb_root', 'wb').write(f.getBytes()) 1001 del f 1002 bin_compare('getbytes vs buildin/Open/getbytes', '/tmp/qtr_gb_root', '/tmp/qtr_infb_root') 1003 f = ReadBytes(open('/tmp/qtr_inf_root', 'rb').read()) 1004 presume(f) 1005 open('/tmp/qtr_infbb_root', 'wb').write(f.getBytes()) 1006 del f 1007 bin_compare('saveas vs buildin/ReadBytes/getbytes', '/tmp/qtr_gb_root', '/tmp/qtr_infb_root') 1008 1009 # open, edit, save, close, reopen, check 1010 f = Open('/tmp/qtr_gb_root') 1011 f.remove(f['left']) 1012 f['child'].data = 4.2 1013 f['int'].data = 21 1014 presume(f.modified) 1015 f.save() 1016 f.close() 1017 f = Open('/tmp/qtr_gb_root', True) 1018 presume(not ('left' in f.list())) 1019 presume(f['child'].data[0] == 4.2) 1020 presume(f['int'].data[0] == 21) 1021 del f 1022

Source Code for Module qubx.tree_native