qubx.data

1 """ 2 Reads Pulse and Patchmaster files. 3 4 Copyright 2008-2013 Research Foundation State University of New York 5 This file is part of QUB Express. 6 7 QUB Express is free software; you can redistribute it and/or modify 8 it under the terms of the GNU General Public License as published by 9 the Free Software Foundation, either version 3 of the License, or 10 (at your option) any later version. 11 12 QUB Express is distributed in the hope that it will be useful, 13 but WITHOUT ANY WARRANTY; without even the implied warranty of 14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 GNU General Public License for more details. 16 17 You should have received a copy of the GNU General Public License, 18 named LICENSE.txt, in the QUB Express program directory. If not, see 19 <http://www.gnu.org/licenses/>. 20 21 """ 22 23 import struct 24 import sys 25 26 from qubx.util_types import Anon, memoize 27 from qubx.data_types import * 28 from qubx.fast.fast_utils import * 29 from ctypes import * 30 31 c_uint_p = POINTER(c_uint) 32 c_short_p = POINTER(c_short) 33 34 TREE_MAGIC = 0x054726565 35 DAT2_MAGIC = 0x044415432 36 37 SizeByte = 1 38 SizeChar = 1 39 SizeEnum = 1 40 SizeBoolean = 1 41 SizeInt16 = 2 42 SizeCard16 = 2 43 SizeSet16 = 2 44 SizeInt32 = 4 45 SizeCard32 = 4 46 SizeReal = 4 47 SizeLongReal = 8 48 49 String8Size = 8 50 String32Size = 32 51 String80Size = 80 52 String400Size = 400 53 54 ParamDescrSize = 24 55 SwUserParamNo = 4 56 SeUserParamNo = 4 57 GrStimParams = 10 58 59 SizeStateVersion = 8 60 SizeSerialNumber = 8 61 SizeCalibDate = 16 62 63 64 RootLevel = 0 65 GroupLevel = 1 66 SeriesLevel = 2 67 SweepLevel = 3 68 TraceLevel = 4 69 70 71 LittleEndianBit = 0 72 IsLeak = 1 73 IsVirtual = 2 74 IsImon = 3 75 IsVmon = 4 76 Clip = 5 77 78 79 # DataKind -> meaning of bits: 80 # - LittleEndianBit => byte sequence 81 # "PowerPC Mac" = cleared 82 # "Windows and Intel Mac" = set 83 # - IsLeak 84 # set if trace is a leak trace 85 # - IsVirtual 86 # set if trace is a virtual trace 87 # - IsImon 88 # -> set if trace was from Imon ADC 89 # -> it flags a trace to be used to 90 # compute LockIn traces from 91 # -> limited to "main" traces, not "leaks"! 92 # - IsVmon 93 # -> set if trace was from Vmon ADC 94 # - Clip 95 # -> set if amplifier of trace was clipping 96 97 # RecordingModeType = 98 ( InOut, 99 OnCell, 100 OutOut, 101 WholeCell, 102 CClamp, 103 VClamp, 104 NoMode ) = range(7) 105 106 # DataFormatType = 107 ( int16, 108 int32, 109 real32, 110 real64 ) = range(4) 111 112 BothEndian = [ Anon(), Anon() ] 113 114 for iS, Sbase in enumerate([LittleEndianStructure, BigEndianStructure]): 115 # pulse v9:

116 - class S(Sbase):

117 - def __str__(self):

118 lines = (["%s:" % self.__class__] 119 + ["%20s:\t%s" % (field, self.__getattribute__(field)) for field, typ in self._fields_]) 120 return "\n".join(lines)

121

122 - class BundleItem(S):

123 _pack_ = 1 124 _fields_ = [('oStart', c_int), 125 ('oLength', c_int), 126 ('oExtension', c_char*8)]

127

128 - class BundleHeader(S):

129 _pack_ = 1 130 _fields_ = [('oSignature', c_char*8), 131 ('oVersion', c_char*32), 132 ('oTime', c_double), 133 ('oItems', c_int), 134 ('oIsLittleEndian', c_char), 135 ('oReserved', c_char*11), 136 ('oBundleItems', 12*(c_char*16))]

137 # ('oBundleItems', 12*BundleItem)] does not support other endian 138 BothEndian[iS].BundleHeader = BundleHeader 139 140

141 - class UserParamDescrType(S):

142 _pack_ = 1 143 _fields_ = [('name', c_char*String32Size), 144 ('aUnit', c_char*String8Size)]

145

146 - class TraceRecord(S):

147 _pack_ = 1 148 _fields_ = [('mark', c_int), 149 ('label', c_char*String32Size), 150 ('traceCount', c_int), 151 ('data', c_int), 152 ('dataPoints', c_int), 153 ('internalSolution', c_int), 154 ('averageCount', c_int), 155 ('leakCount', c_int), 156 ('leakTraces', c_int), 157 ('dataKind', c_short), 158 ('useXStart', c_byte), 159 ('filler1', c_byte), 160 ('recordingMode', c_byte), 161 ('amplIndex', c_byte), 162 ('dataFormat', c_byte), 163 ('dataAbscissa', c_byte), 164 ('dataScaler', c_double), 165 ('timeOffset', c_double), 166 ('zeroData', c_double), 167 ('yUnit', c_char*String8Size), 168 ('xInterval', c_double), 169 ('xStart', c_double), 170 ('xUnit', c_char*String8Size), 171 ('yRange', c_double), 172 ('yOffset', c_double), 173 ('bandwidth', c_double), 174 ('pipetteResistance', c_double), 175 ('cellPotential', c_double), 176 ('sealResistance', c_double), 177 ('cSlow', c_double), 178 ('gSeries', c_double), 179 ('rsValue', c_double), 180 ('gLeak', c_double), 181 ('mConductance', c_double), 182 ('linkDAChannel', c_int), 183 ('validYrange', c_byte), 184 ('adcMode', c_byte), 185 ('adcChannel', c_short), 186 ('yMin', c_double), 187 ('yMax', c_double), 188 ('sourceChannel', c_int), 189 ('externalSolution', c_int), 190 ('CM', c_double), 191 ('GM', c_double), 192 ('phase', c_double), 193 ('dataCRC', c_int), 194 ('CRC', c_int), 195 ('GS', c_double), 196 ('selfChannel', c_int), 197 ('interleaveSize', c_int), 198 ('interleaveSkip', c_int), 199 ('imageIndex', c_int), 200 ('markers', c_double*10), 201 ('SECM_X', c_double), 202 ('SECM_Y', c_double), 203 ('SECM_Z', c_double)]

204

205 - class SweepRecord(S):

206 _pack_ = 1 207 _fields_ = [('mark', c_int), 208 ('label', c_char*String32Size), 209 ('auxDataFileOffset', c_int), 210 ('stimCount', c_int), 211 ('sweepCount', c_int), 212 ('time', c_double), 213 ('timer', c_double), 214 ('userParams', c_double*4), 215 ('temperature', c_double), 216 ('oldIntSol', c_int), 217 ('oldExtSol', c_int), 218 ('digitalIn', c_short), 219 ('sweepKind', c_short), 220 ('digitalOut', c_short), 221 ('filler1', c_short), 222 ('markers', c_double*4), 223 ('filler2', c_int), 224 ('CRC', c_int)]

225

226 - class SeriesRecord(S):

227 _pack_ = 1 228 _fields_ = [('mark', c_int), 229 ('label', c_char*String32Size), 230 ('comment', c_char*String80Size), 231 ('seriesCount', c_int), 232 ('numberSweeps', c_int), 233 ('amplStateOffset', c_int), 234 ('amplStateSeries', c_int), 235 ('seriesType', c_byte), 236 ('useXStart', c_byte), 237 ('filler1', c_byte), 238 ('filler2', c_byte), 239 ('time', c_double), 240 ('pageWidth', c_double), 241 # ('userParamDescr', UserParamDescrType*4), 242 ('UP0_0_name', c_char*String32Size), 243 ('UP0_0_aUnit', c_char*String8Size), 244 ('UP0_1_name', c_char*String32Size), 245 ('UP0_1_aUnit', c_char*String8Size), 246 ('UP0_2_name', c_char*String32Size), 247 ('UP0_2_aUnit', c_char*String8Size), 248 ('UP0_3_name', c_char*String32Size), 249 ('UP0_3_aUnit', c_char*String8Size), 250 251 ('filler3', c_char*32), 252 ('userParams1', c_double*4), 253 ('lockInParams', 96*c_char), # see Pulsed.de 254 ('amplifierState', 400*c_char), 255 ('username', c_char*String80Size), 256 # ('userParamDescr1', UserParamDescrType*4), 257 ('UP1_0_name', c_char*String32Size), 258 ('UP1_0_aUnit', c_char*String8Size), 259 ('UP1_1_name', c_char*String32Size), 260 ('UP1_1_aUnit', c_char*String8Size), 261 ('UP1_2_name', c_char*String32Size), 262 ('UP1_2_aUnit', c_char*String8Size), 263 ('UP1_3_name', c_char*String32Size), 264 ('UP1_3_aUnit', c_char*String8Size), 265 ('filler4', c_int), 266 ('CRC', c_int), 267 ('userParams2', c_double*4), 268 # ('userParamDescr2', UserParamDescrType*4), 269 ('UP2_0_name', c_char*String32Size), 270 ('UP2_0_aUnit', c_char*String8Size), 271 ('UP2_1_name', c_char*String32Size), 272 ('UP2_1_aUnit', c_char*String8Size), 273 ('UP2_2_name', c_char*String32Size), 274 ('UP2_2_aUnit', c_char*String8Size), 275 ('UP2_3_name', c_char*String32Size), 276 ('UP2_3_aUnit', c_char*String8Size), 277 ('scanParams', 96*c_char)]

278

279 - class GroupRecord(S):

280 _pack_ = 1 281 _fields_ = [('mark', c_int), 282 ('label', c_char*String32Size), 283 ('text', c_char*String80Size), 284 ('experimentNumber', c_int), 285 ('groupCount', c_int), 286 ('CRC', c_int)]

287

288 - class RootRecord(S):

289 _pack_ = 1 290 _fields_ = [('version', c_int), 291 ('mark', c_int), 292 ('versionName', c_char*String32Size), 293 ('auxFileName', c_char*String80Size), 294 ('rootText', c_char*String400Size), 295 ('startTime', c_double), 296 ('maxSamples', c_int), 297 ('CRC', c_int), 298 ('features', c_short), 299 ('filler1', c_short), 300 ('filler2', c_int)]

301 302 BothEndian[iS].LevelRecords9 = [RootRecord, GroupRecord, SeriesRecord, SweepRecord, TraceRecord] 303 304 305 # PGF: 306 307 # meaning of these "Info" variables: 308 # indices of longreal array "InfoLReals" for LockIn: 309 310 iLCycle = 0 # (* length of one LockIn Cycle *) 311 iLAmplitude = 1 # (* LockIn Amplitude *) 312 iLVReversal = 2 # (* LockIn Reversal *) 313 314 # (* indices of INT32 array: *) 315 iLSkip = 0 #; (* number of LockIn cycles to skip *) 316 iMaxCycles = 1 #; (* maximal number of cycles in photo-channel *) 317 iSegmentNo = 2 #; (* number of segments per cycle in photo-channel *) 318 319 # (* indices of CHAR array: *) 320 iAdapt = 0 #; (* adapt to maximal length in photo-channel *) 321 322 323 # CompressionMode : Specifies how to the data 324 # -> meaning of bits: 325 # bit 0 (CompReal) -> high = store as real 326 # low = store as int16 327 # bit 1 (CompMean) -> high = use mean 328 # low = use single sample 329 # bit 2 (CompFilter) -> high = use digital filter 330 331 # StimToDacID : Specifies how to convert the Segment "Voltage" to 332 # the actual voltage sent to the DAC 333 # -> meaning of bits: 334 # bit 0 (UseStimScale) -> use StimScale 335 # bit 1 (UseRelative) -> relative to Vmemb 336 # bit 2 (UseFileTemplate) -> use file template 337 # bit 3 (UseForLockIn) -> Info-record has 338 # LockIn parameters 339 # bit 4 (UseForWavelength) 340 # bit 5 (UseScaling) 341 # bit 6 (UseForChirp) 342 # bit 7 (UseForImaging) 343 # bit 15 (UseReserved) 344 345 # SegmentClass = 346 ( SegmentConstant, 347 SegmentRamp, 348 SegmentContinuous, 349 SegmentConstSine, 350 SegmentSquarewave, 351 SegmentChirpwave ) = range(6) 352 353 # IncrementModeType = 354 ( ModeInc, 355 ModeDec, 356 ModeIncInterleaved, 357 ModeDecInterleaved, 358 ModeAlternate, 359 ModeLogInc, 360 ModeLogDec, 361 ModeLogIncInterleaved, 362 ModeLogDecInterleaved, 363 ModeLogAlternate ) = range(10) 364 365 # ExtTriggerType = 366 ( TrigNone, 367 TrigSeries, 368 TrigSweep, 369 TrigSweepNoLeak ) = range(4) 370 371 # AmplModeType = 372 ( AnyAmplMode, 373 VCAmplMode, 374 CCAmplMode, 375 IDensityMode ) = range(4) 376 377 # AutoRangingType = 378 ( AutoRangingOff, 379 AutoRangingPeak, 380 AutoRangingMean, 381 AutoRangingRelSeg ) = range(4) 382 383 # AdcType = 384 ( AdcOff, 385 Analog, 386 Digitals, 387 Digital, 388 AdcVirtual ) = range(5) 389 390 # LeakStoreType = 391 ( LNone, 392 LStoreAvg, 393 LStoreEach, 394 LNoStore ) = range(4) 395 396 # LeakHoldType = 397 ( Labs, 398 Lrel, 399 LabsLH, 400 LrelLH ) = range(4) 401 402

403 - class StimSegmentRecord(S):

404 _pack_ = 1 405 _fields_ = [('seMark', c_int), 406 ('seClass', c_byte), 407 ('seDoStore', c_byte), 408 ('seVoltageIncMode', c_byte), 409 ('seDurationIncMode', c_byte), 410 ('seVoltage', c_double), 411 ('seVoltageSource', c_int), 412 ('seDeltaVFactor', c_double), 413 ('seDeltaVIncrement', c_double), 414 ('seDuration', c_double), 415 ('seDurationSource', c_int), 416 ('seDeltaTFactor', c_double), 417 ('seDeltaTIncrement', c_double), 418 ('seFiller1', c_int), 419 ('seCRC', c_uint), 420 ('seScanRate', c_double)]

421

422 - class StimChannelRecord(S):

423 _pack_ = 1 424 _fields_ = [('chMark', c_int), 425 ('chLinkedChannel', c_int), 426 ('chCompressionFactor', c_int), 427 ('chYUnit', c_char*8), 428 ('chAdcChannel', c_short), 429 ('chAdcMode', c_byte), 430 ('chDoWrite', c_byte), 431 ('stLeakStore', c_byte), 432 ('chAmplMode', c_byte), 433 ('chOwnSegTime', c_byte), 434 ('chSetLastSegVMemb', c_byte), 435 ('chDacDhannel', c_short), 436 ('chDacMode', c_byte), 437 ('chFiller1', c_byte), 438 ('chRelevantXSegment', c_int), 439 ('chRelevantYSegment', c_int), 440 ('chDacUnit', c_char*8), 441 ('chHolding', c_double), 442 ('chLeakHolding', c_double), 443 ('chLeakSize', c_double), 444 ('chLeakHoldMode', c_byte), 445 ('chLeakAlternate', c_byte), 446 ('chAltLeakAveraging', c_byte), 447 ('chLeakPulseOn', c_byte), 448 ('chStimToDacID', c_ushort), 449 ('chCompressionMode', c_ushort), 450 ('chCompressionSkip', c_int), 451 ('chDacBit', c_short), 452 ('chHasSinewaves', c_byte), 453 ('chBreakMode', c_byte), 454 ('chZeroSeg', c_int), 455 ('chFiller2', c_int), 456 ('chInfoLReal', c_double*8), 457 ('chInfoLInt', c_int*8), 458 ('chInfoIChar', c_char*8), 459 ('chDacOffset', c_double), 460 ('chAdcOffset', c_double), 461 ('chTraceMathFormat', c_byte), 462 ('chHasChirp', c_byte), 463 ('chFiller3', c_byte*30), 464 ('chCompressionOffset', c_int), 465 ('chPhotoMode', c_int), 466 ('chBreakLevel', c_double), 467 ('chTraceMath', c_char*128), 468 ('chOldCRC', c_uint), 469 ('chFiller4', c_int), 470 ('chCRC', c_uint)]

471

472 - class StimulationRecord(S):

473 _pack_ = 1 474 _fields_ = [('stMark', c_int), 475 ('stEntryName', c_char*32), 476 ('stFileName', c_char*32), 477 ('stAnalName', c_char*32), 478 ('stDataStartSegment', c_int), 479 ('stDataStartTime', c_double), 480 ('stSampleInterval', c_double), 481 ('stSweepInterval', c_double), 482 ('stLeakDelay', c_double), 483 ('stFilterFactor', c_double), 484 ('stNumberSweeps', c_int), 485 ('stNumberLeaks', c_int), 486 ('stNumberAverages', c_int), 487 ('stActualAdcChannels', c_int), 488 ('stActualDacChannels', c_int), 489 ('stExtTrigger', c_byte), 490 ('stNoStartWait', c_byte), 491 ('stUseScanRates', c_byte), 492 ('stNoContAq', c_byte), 493 ('stHasSinewaves', c_byte), 494 ('stOldStartMacKind', c_byte), 495 ('stOldEndMacKind', c_byte), 496 ('stAutoRange', c_byte), 497 ('stBreakNext', c_byte), 498 ('stIsExpanded', c_byte), 499 ('stLeakCompMode', c_byte), 500 ('stHasChirp', c_byte), 501 ('stOldStartMacro', c_char*32), 502 ('stOldEndMacro', c_char*32), 503 ('stIsGapFree', c_byte), 504 ('sHandledExternally', c_byte), 505 ('stFiller1', c_byte), 506 ('stFiller2', c_byte), 507 ('stCRC', c_uint), 508 ('stTag', c_char*32)]

509

510 - class StimRootRecord(S):

511 _pack_ = 1 512 _fields_ = [('roVersion', c_int), 513 ('roMark', c_int), 514 ('roVersionName', c_char*32), 515 ('roMaxSamples', c_int), 516 ('roFiller1', c_int), 517 ('roParams', c_double*10), 518 ('roParamText', c_char*320), # (c_char*32)*10 519 ('roReserved', c_char*128), 520 ('roFiller2', c_int), 521 ('roCRC', c_uint)]

522 523 BothEndian[iS].StimRecords9 = [StimRootRecord, StimulationRecord, StimChannelRecord, StimSegmentRecord] 524 525 # shared?:

526 - class Header(S):

527 _pack_ = 1 528 _fields_ = [('magicNumber', c_int), 529 ('levels', c_int), 530 ('levelSize', c_int*10)]

531 BothEndian[iS].Header = Header 532 ActualHeaderSize = lambda levels: (2+levels)*sizeof(c_int) 533

534 - class LevelFooter(S):

535 _pack_ = 1 536 _fields_ = [('childCount', c_int)]

537 BothEndian[iS].LevelFooter = LevelFooter 538 539 540 # V7

541 - class SweepRecord(S):

542 _pack_ = 1 543 _fields_ = [('time', c_double), # UNIX (seconds) + Rest 544 ('stimCount', c_int), # relevant entry in StimTree 545 ('sweepCount', c_int), # .. at time of acquisition 546 ('averageCount', c_int), # number of on-line averages 547 ('leak', c_byte), # TRUE, if Record has a leaksweep 548 ('secondTrace', c_byte), # TRUE, if Record has a 2. trace 549 ('lbl', c_char*14), 550 ('dataPoints', c_int), # Number of data points in RAM 551 ('data', c_int), # Offset in raw data file 552 ('dataPointer', c_int), # Raw data ADDRESS in memory 553 # EPC 7/9 settings 554 ('dataFactor1', c_double), # Amperes/ADC-Unit 555 ('dataFactor2', c_double), # Volts/ADC-Unit 556 ('cSlow', c_double), # Capacitance, Farad 557 ('gSeries', c_double), # Series conductance, Siemens 558 ('rsValue', c_double), # Series resistance setting, Ohms 559 ('rsFraction', c_double), # % RsCompensation 560 # Results of pre-analysis 561 ('zeroCurrent', c_double), # Amperes 562 ('onlineYResult', c_double), # Param. determined by last anal. 563 ('onlineXResult', c_double), # Param. determined by last anal. 564 565 ('totalPoints', c_int), # Total data points in file 566 ('offset', c_int), # Offset of loaded data 567 568 ('sweepKind', c_short), # meaning of bits: 569 # ExtendedBit(1) => extended fields defined 570 # FuraBit(2) => Fura.Active 571 # LockInBit(4) => LockIn.Active 572 # DataFormatBit(8) => "DataFormat" field valid 573 # LittleEndianBit(16) => byte sequence / "DOS" = set 574 # ScanRateBit(32) => set: use scan-rate! 575 576 ('furaPoints', c_int), # Number of data points in RAM 577 ('furaData', c_int), # Offset in raw data file 578 ('furaPointer', c_int), # Raw data ADDRESS in memory 579 ('onlineYResult2', c_double),# Param. determined by last anal. 580 ('onlineXResult2', c_double),# Param. determined by last anal. 581 582 ('dispFactor1', c_double), # reserved by PULSE 583 ('dispFactor2', c_double), # reserved by PULSE 584 585 # Since we introduced the new "DataFormat" field in a field 586 # which before was without importance, we are interpreting it 587 # ONLY if "SweepKind" has ExtendedBit and DataFormatBit set! 588 # Thus, the procedure "StimIO.InitializeSweep" scans 589 # the "*.pul" tree upon loading and enforces that check. 590 ('dataFormat', c_byte), # 0:int16, 1:int32, 2:float32, 3:float64 591 ('dataAbscissa', c_byte), 592 # DataAbscissaType = ( Time, 593 # Amplitude, LnAmplitude, LogAmplitude, 594 # Frequency, LnFrequency, LogFrequency ); 595 596 ('timer', c_double), 597 ('spares', c_char*10)]

598

599 - class SeriesRecord(S):

600 _pack_ = 1 601 _fields_ = [('time', c_double), # seconds 602 # Patch parameters, usually obtained before series 603 ('bandwidth', c_double), # Hertz 604 ('pipettePotential', c_double), # Volts 605 ('cellPotential', c_double), # Volts 606 ('pipetteResistance', c_double), # Ohms 607 ('sealResistance', c_double), # Ohms 608 ('backgroundNoise', c_double), # Amperes, RMS 609 ('temperature', c_double), # Degrees C 610 ('pipettePressure', c_double), # in cm H20 611 ('userParam1Value', c_double), 612 ('userParam1Name', c_char*14), 613 ('userParam1Unit', c_char*2), 614 ('userParam2Value', c_double), 615 ('userParam2Name', c_char*14), 616 ('userParam2Unit', c_char*2), 617 ('recordingMode', c_byte), 618 # RecordingModeType = ( InOut, 619 # OnCell, 620 # OutOut, 621 # WholeCell, 622 # PCClamp, 623 # VClamp, 624 # NoRec, 625 # TestInt, 626 # TestExt ); 627 ('filler1', c_byte), 628 ('comment', c_char*80), 629 ('epc9State', c_byte*104), 630 ('internalSolution', c_int), 631 ('externalSolution', c_int), # solutions according to solution code as stored in '*.sol' 632 ('extraYUnit1', c_char*2), 633 ('extraYUnit2', c_char*2), 634 ('dispYUnit1', c_char*4), # reserved by PULSE 635 ('dispYUnit2', c_char*4), # reserved by PULSE 636 ('furaK', c_double), 637 ('furaMin', c_double), 638 ('furaMax', c_double), 639 ('lockInExtPhase', c_double), 640 ('timer', c_double), 641 # Reserved for extensions 642 ('extraLongReal', c_double*4)]

643

644 - class GroupRecord(S):

645 """ A GroupRecord describes a group of series, such as patch and 646 whole cell currents obtained simultanuously, or groups of series 647 obtained in sequence under different sets of conditions 648 """ 649 _pack_ = 1 650 _fields_ = [('label', c_char*14), 651 ('text', c_char*80), 652 ('experimentNumber', c_int), 653 ('extraLongReal', c_double)]

654

655 - class RootRecord(S):

656 _pack_ = 1 657 _fields_ = [('version', c_short), 658 ('versionName', c_char*14), 659 ('fileName', c_char*14), 660 ('comments', c_char*400), 661 ('startTime', c_double)]

662 663 BothEndian[iS].LevelRecords7 = [RootRecord, GroupRecord, SeriesRecord, SweepRecord, TraceRecord] 664

665 - class StimSegmentRecord(S):

666 _pack_ = 1 667 _fields_ = [('cls', c_byte), # constant, ramp, or condit. 668 ('isHolding', c_byte), # TRUE if Voltage is to be set to holding 669 ('voltage', c_double), 670 ('duration', c_double), 671 ('deltaVFactor', c_double), 672 ('deltaVIncrement', c_double), 673 ('deltaTFactor', c_double), 674 ('deltaTIncrement', c_double)]

675 676 #TYPE SegmentClass = ( SegmentConstant, 677 # SegmentRamp, 678 # SegmentConditioning, 679 # SegmentContinuous, 680 # SegmentConstSine, 681 # SegmentRampSine ); 682 683 #TYPE IncrementModeType = ( ModeInc, 684 # ModeDec, 685 # ModeIncInterleaved, 686 # ModeDecInterleaved, 687 # ModeAlternate ); 688 689 #TYPE GUpdateType = ( NoGUpdate, 690 # SwGSlow, 691 # SwGFast, 692 # SwGBoth, 693 # SeGSlow, 694 # SeGFast, 695 # SeGBoth ); 696 697 #TYPE WriteModeType = ( WriteEnabled, 698 # WriteDisabled, 699 # NoWriteNoShow, 700 # WriteButNoShow ); 701 702 #TYPE ExtTriggerType = ( TrigNone, TrigSeries, TrigSweep ); 703 704 #TYPE PostVmembType = ( VmembSweepIncr, 705 # VmembValue, 706 # VmembSeriesIncr ); 707 708 #TYPE LockInType = ( Loff, Lnormal, Lpwlinear ); 709 710 #TYPE AmplModeType = ( AllAmplModes, VCAmplMode, CCAmplMode ); 711

712 - class StimulationRecord(S):

713 _pack_ = 1 714 _fields_ = [('fileName', c_char*14), # Source File 715 ('entryName', c_char*14), # Identifier 716 ('sampleInterval', c_double), # Seconds 717 ('filterFactor', c_double), # oversampling factor 718 ('sweepInterval', c_double), # Repetition-interval 719 ('numberSweeps', c_int), # Number of sweeps in Series 720 ('numberRepeats', c_int), # Number of Seq. repeats 721 ('repeatWait', c_int), # Wait between repeats 722 ('linkedSequence', c_char*14), 723 ('linkedWait', c_double), # Wait between linked sequences 724 725 ('leakCount', c_int), # number of leak sweeps 726 ('leakSize', c_double), # rel. amplitude of leak pulse 727 # e.g. 0.25 for P/4 protocol 728 ('leakHolding', c_double), 729 ('leakAlternate', c_byte), # norm. or alt. leak protoc. 730 ('altLeakAveraging', c_byte), # norm. or alt. leak while aver. 731 ('leakDelay', c_double), # Seconds 732 ('trig1Segment', c_short), 733 ('trig1Time', c_double), # Seconds, within trig.Segment 734 ('trig1Length', c_double), # Length of Trig.Pulse, sec 735 ('trig1Amplitude', c_double), # Amplitude of Trig.Pulse 736 ('trig1Dac', c_short), 737 ('trig2Segment', c_short), 738 ('trig2Time', c_double), 739 ('trig2Length', c_double), 740 ('trig2Amplitude', c_double), 741 ('trig2Dac', c_short), 742 ('trig3Segment', c_short), 743 ('trig3Time', c_double), 744 ('trig3Length', c_double), 745 ('trig3Amplitude', c_double), 746 ('trig3Dac', c_short), 747 ('numberOfTriggers', c_short), # Number actually used; 0<x<4 748 ('relevantXSegment', c_short), # usually that which changes 749 ('relevantYSegment', c_short), # usually that which changes 750 751 ('writeMode', c_byte), 752 ('incrementMode', c_byte), 753 754 ('totalSweepLength', c_int), # Total sweeplength including a possible continuous segment, 755 # in units of sample intervals. 756 757 ('maxSweepLength', c_int), # max length of sweep to be shown. 758 # For a pulse without a continuous segment it is identical 759 # to the max sweeplength from trig $1, 760 # in units of sample intervals 761 762 ('inputChannels', c_short), # num. input channels. Default 763 ('gUpdate', c_byte), # make C-slow/fast bef. each pulse 764 # only for EPC9 765 ('relAbsPot', c_byte), # absolute or relativ potentials 766 ('hasContinuous', c_byte), 767 ('logIncrement', c_byte), 768 ('stimDac', c_short), 769 ('adc1', c_short), 770 ('adc2', c_short), 771 ('yUnit1', c_char*2), 772 ('yUnit2', c_char*2), 773 ('VmembIncrement', c_float), 774 ('extTrigger', c_byte), 775 ('fileTemplate', c_byte), 776 ('stimKind', c_short), 777 # meaning of bits: 778 # 0 => extended fields defined 779 # 1 => LockIn.Active 780 # 2 => Fura.Active 781 # 15 => set: use scan-rate! 782 783 ('lockInCycle', c_double), 784 ('lockInAmplitude', c_double), 785 ('furaOn', c_byte), 786 ('VmembMode', c_byte), 787 ('furaTotLength', c_double), 788 ('furaDelay', c_double), 789 ('furaLength1', c_double), 790 ('furaLength2', c_double), 791 ('furaWaveLength0', c_double), 792 ('furaWaveLength1', c_double), 793 ('furaWaveLength2', c_double), 794 ('furaRepeats', c_short), 795 ('lockInSkip', c_int), 796 ('lockInVReversal', c_double), 797 ('lockInMode', c_byte), 798 ('lockInShow', c_byte), 799 ('configMacro', c_char*16), 800 ('endMacro', c_char*16), 801 ('amplModeKind', c_byte), 802 ('filler1', c_byte)]

803

804 - class StimRootRecord(S):

805 _pack_ = 1 806 _fields_ = [('version', c_short)]

807 808 BothEndian[iS].StimRecords7 = [StimRootRecord, StimulationRecord, StimSegmentRecord] 809 810 811

812 -def ReadStruct(rectype, pulmap, offset):

813 size = sizeof(rectype) 814 source = pulmap 815 remain = len(pulmap) - offset 816 if 0 < remain < size: 817 source = pulmap[offset:] + "\0"*(size-remain) 818 offset = 0 819 return rectype.from_buffer_copy(source, offset)

820

821 -class TreeLevel(list):

822 - def __init__(self, rec):

823 list.__init__(self) 824 self.rec = rec

825

826 -def LoadOneLevel(pulmap, header, LevelRecords, LevelFooter, offset, level):

827 rec = ReadStruct(LevelRecords[level], pulmap, offset) 828 result = TreeLevel(rec) 829 offset += header.levelSize[level] 830 footer = ReadStruct(LevelFooter, pulmap, offset) 831 offset += sizeof(LevelFooter) 832 for i in xrange(footer.childCount): 833 child, offset = LoadOneLevel(pulmap, header, LevelRecords, LevelFooter, offset, level+1) 834 result.append(child) 835 return result, offset

836

837 -def LoadTree(path, GetLevels=lambda header, endian: None):

838 fileno = pulmap = None 839 try: 840 size = os.path.exists(path) and os.stat(path)[6] or 0 841 if size: 842 fileno = os.open(path, os.O_RDONLY) 843 pulmap = mmap.mmap(fileno, size, access=mmap.ACCESS_READ) 844 if mmap is None: 845 size = 0 846 if not size: 847 raise Exception("Can't open %s" % path) 848 dat_header = None 849 pul_offset = 0 850 pgf_offset = 0 851 if pulmap[:4] == 'DAT2': 852 dat_header = BothEndian[0].BundleHeader.from_buffer_copy(pulmap, 0) 853 elif pulmap[:4] == '2TAD': 854 dat_header = BothEndian[1].BundleHeader.from_buffer_copy(pulmap, 0) 855 if not (dat_header is None): 856 for i in xrange(dat_header.oItems): 857 if dat_header.oBundleItems[i][8:12].lower() == '.pul': 858 pul_offset = struct.unpack('i', dat_header.oBundleItems[i][:4])[0] 859 elif dat_header.oBundleItems[i][8:12].lower() == '.pgf': 860 pgf_offset = struct.unpack('i', dat_header.oBundleItems[i][:4])[0] 861 endian = BothEndian[0] 862 header = endian.Header.from_buffer_copy(pulmap, pul_offset) 863 if header.magicNumber != TREE_MAGIC: 864 endian = BothEndian[1] 865 header = endian.Header.from_buffer_copy(pulmap, pul_offset) 866 if header.magicNumber != TREE_MAGIC: 867 raise Exception('Bad magic') 868 levelRecords = GetLevels(header, endian) 869 offset = pul_offset + ActualHeaderSize(header.levels) 870 root, offset = LoadOneLevel(pulmap, header, levelRecords, endian.LevelFooter, offset, 0) 871 pgf_root = None 872 if False: # pgf_offset: 873 pgf_header = endian.Header.from_buffer_copy(pulmap, pgf_offset) 874 offset = pgf_offset + ActualHeaderSize(pgf_header.levels) 875 pgf_root, pgf_offset = LoadOneLevel(pulmap, pgf_header, endian.StimRecords9, endian.LevelFooter, offset, 0) 876 print pgf_header 877 for i in xrange(pgf_header.levels): 878 print pgf_header.levelSize[i], 879 print 880 print pgf_root.rec 881 for stim in pgf_root: 882 print stim.rec 883 for chan in stim: 884 print chan.rec 885 for sgm in chan: 886 print sgm.rec 887 print ' /chan' 888 print ' /stim' 889 return header, endian, root 890 finally: 891 if pulmap: 892 pulmap.close() 893 del pulmap 894 if fileno: 895 os.close(fileno)

896 897

898 -def LoadPUL(path):

899 def GetLevels(header, endian): 900 if header.levels == 5: 901 return endian.LevelRecords9 902 else: 903 return endian.LevelRecords7

904 return LoadTree(path, GetLevels) 905

906 -def LoadStim(path):

907 return LoadTree(path, lambda header, endian: endian.StimRecords7)

908

909 -def adjust_units(units, scale):

910 if units == 'A': 911 return 'pA', 1e12*scale 912 elif units == 'V': 913 return 'mV', 1e3*scale 914 else: 915 return units, scale 916

917 -class QubData_PUL_Signal(QubData_Analog):

918 - def __init__(self, data):

919 """ 920 @param data: L{QubData_PUL} 921 """ 922 QubData_Analog.__init__(self) 923 self.data = data

924 - def read(self, first, last, latency=0, skip=1):

925 ifirst, ilast = [self.data.segmentation.index_at(x) for x in (first, last)] 926 if ifirst != ilast: 927 raise Exception("PUL: can't read() from multiple segments at once") 928 segfirst = self.data.segmentation.segments[ifirst][0] 929 seglast = self.data.segmentation.segments[ifirst][1] 930 return read_with_latency(self.read_in_seg, first, last, latency, skip, ifirst, segfirst, seglast)

931 932

933 -class QubData_PUL_Analog(QubData_PUL_Signal):

934 - def __init__(self, data, channel):

935 """ 936 @param data: L{QubData_PUL} 937 @param channel: integer 0-based signal index 938 """ 939 QubData_PUL_Signal.__init__(self, data) 940 self.channel = channel

941 - def read_in_seg(self, iseg, first, last, skip):

942 samples = numpy.zeros(dtype='float32', shape=(last-first+1,)) 943 try: 944 sweep = self.data.sweeps[iseg][self.channel] 945 except IndexError: 946 if qubx.global_namespace.DEBUG: 947 print 'missing sweep: segment %i, signal %i, points %i-%i' % (iseg, self.channel, first, last) 948 return samples 949 start = 0 950 for ichunk, f, l in self.data.sweep_segm[iseg, self.channel].split(first, last): 951 n = l-f+1 952 samples[start:start+n] = sweep[ichunk][f:l+1] 953 samples = samples[::skip] 954 samples *= self.data.trace_scale[self.channel] 955 scale = self.data.signals.get(self.channel, 'Scale') 956 if scale != 1.0: 957 samples *= scale 958 offset = self.data.signals.get(self.channel, 'Offset') 959 if offset != 0.0: 960 samples += offset 961 return samples

962

963 -class QubData_PUL(QubData):

964 """One open Patchmaster file. 965 966 """

967 - def __init__(self, path, progressf):

968 QubData.__init__(self) 969 self.path = path 970 971 base, ext = os.path.splitext(path) 972 sess = qubx.tree.Open(base+'.qsf', True) 973 sess.close() # release file, keep in mem 974 975 self.sampling = ((sess.find('Sampling') and sess['Sampling'].data[0]) 976 or 1.0e-4) 977 978 header, endian, self.root = LoadPUL(path) 979 datapath = os.path.splitext(path)[0]+'.dat' 980 size = os.path.exists(datapath) and os.stat(datapath)[6] or 0 981 if size: 982 self.fileno = os.open(datapath, os.O_RDONLY) 983 self.mmap = mmap.mmap(self.fileno, size, access=mmap.ACCESS_READ) 984 if mmap is None: 985 os.close(self.fileno) 986 self.fileno = size = 0 987 if not size: 988 raise Exception("Can't open %s" % datapath) 989 990 self.sweeps = [] 991 self.sweep_segm = {} 992 self.trace_scale = [] 993 994 if header.levels == 5: 995 sweep0 = self.root[0][0][0] 996 self.sampling = sweep0[0].rec.xInterval 997 for c, trace in enumerate(sweep0): 998 units, scale = adjust_units(trace.rec.yUnit, trace.rec.dataScaler) 999 self.signals.append({'Name' : trace.rec.label, 1000 'Units' : units}) 1001 self.trace_scale.append(scale) 1002 self.set_analog(c, QubData_PUL_Analog(self, c)) 1003 1004 if sweep0[0].rec.dataKind & 1: 1005 typ = '<' 1006 else: 1007 typ = '>' 1008 typ += {0 : 'i2', 1009 1 : 'i4', 1010 2 : 'f4', 1011 3 : 'f8'}[sweep0[0].rec.dataFormat] 1012 dataSize = int(typ[2]) 1013 last = -1 1014 i = 0 1015 nSeries = 0 1016 iSeg = 0 1017 #hdrlog = open(os.path.splitext(path)[0]+'_headers.txt', 'w') 1018 for group in self.root: 1019 #hdrlog.write("%s\n" % group.rec) 1020 for series in group: 1021 #hdrlog.write("%s\n" % series.rec) 1022 nSeries += 1 1023 series_list = self.lists.show_list(series.rec.label or ("Series %i" % nSeries)) 1024 for sweep in series: 1025 #hdrlog.write("%s\n" % sweep.rec) 1026 seglen = sweep[0].rec.dataPoints 1027 first = last + 1 1028 last = first + seglen - 1 1029 self.segmentation.add_seg(first, last, first*self.sampling*1e3) 1030 series_list.insert_selection(first, last, sweep.rec.label) 1031 #for trace in sweep: 1032 # hdrlog.write("%s\n" % trace.rec) 1033 #hdrlog.write('---\n') 1034 arrays = [] 1035 self.sweeps.append(arrays) 1036 for itrace, trace in enumerate(sweep): 1037 offset = trace.rec.data 1038 self.sweep_segm[iSeg, itrace] = trace_segm = Segmentation() 1039 if trace.rec.interleaveSize: 1040 ilSize = sweep[0].rec.interleaveSize 1041 ilSkip = sweep[0].rec.interleaveSkip 1042 chunks = [] 1043 remain = seglen 1044 chunkoff = offset 1045 segm_f = 0 1046 while remain: 1047 chunkn = min(remain, ilSize/dataSize) 1048 chunks.append(numpy.frombuffer(self.mmap, typ, chunkn, chunkoff)) 1049 chunkoff += ilSize + ilSkip # docs are inconsistent whether to increment by ilSize 1050 remain -= chunkn 1051 trace_segm.add_seg(segm_f, segm_f+chunkn-1, 0.0) 1052 segm_f += chunkn 1053 arrays.append(chunks) 1054 else: 1055 arrays.append([numpy.frombuffer(self.mmap, typ, seglen, offset)]) 1056 trace_segm.add_seg(0, seglen-1, 0.0) 1057 iSeg += 1 1058 #hdrlog.write(' ---- end series -----\n') 1059 #hdrlog.write('----- end group -----\n') 1060 elif header.levels == 4: # v7 1061 try: 1062 stimHeader, stimEndian, stimRoot = LoadStim(os.path.splitext(path)[0]+'.pgf') 1063 self.sampling = stimRoot[0].rec.sampleInterval 1064 except: 1065 self.sampling = RequestSampling(self.sampling) 1066 1067 hasLeak = hasSecond = False 1068 for group in self.root: 1069 for series in group: 1070 if series: 1071 if series[0].rec.leak: 1072 hasLeak = True 1073 if series[0].rec.secondTrace: 1074 hasSecond = (series.rec.extraYUnit2, series[0].rec.dataFactor2) 1075 signals = [Anon(units = self.root[0][0].rec.extraYUnit1, 1076 scale = self.root[0][0][0].rec.dataFactor1)] 1077 if hasSecond: 1078 signals.append(Anon(units = hasSecond[0], scale = hasSecond[1], second = True)) 1079 if hasLeak: 1080 signals.append(signals[0].clone(leak = True)) 1081 1082 for c, signal in enumerate(signals): 1083 units, scale = adjust_units(signal.units, signal.scale) 1084 self.signals.append({'Name' : signal.leak and 'Leak' or (signal.second and 'Ch 1' or 'Ch 0'), 1085 'Units' : units}) 1086 self.trace_scale.append(scale) 1087 self.set_analog(c, QubData_PUL_Analog(self, c)) 1088 1089 sweep0 = self.root[0][0][0].rec 1090 dataFormat = (sweep0.sweepKind & 8) and sweep0.dataFormat or 0 1091 if sweep0.sweepKind & 16: 1092 typ = '<' 1093 else: 1094 typ = '>' 1095 typ += {0 : 'i2', 1096 1 : 'i4', 1097 2 : 'f4', 1098 3 : 'f8'}[dataFormat] 1099 dataSize = int(typ[2]) 1100 last = -1 1101 i = 0 1102 iSeg = 0 1103 shared_zeros = memoize(lambda count: numpy.zeros(dtype='float32', shape=(count,))) 1104 for group in self.root: 1105 for series in group: 1106 for sweep in series: 1107 seglen = sweep.rec.totalPoints 1108 first = last + 1 1109 last = first + seglen - 1 1110 self.segmentation.add_seg(first, last, first*self.sampling*1e3) 1111 offLeak = sweep.rec.data + (sweep.rec.leak and (seglen*dataSize) or 0) 1112 offSecond = offLeak + seglen*dataSize 1113 arrays = [] 1114 for c, signal in enumerate(signals): 1115 self.sweep_segm[iSeg, c] = trace_segm = Segmentation() 1116 if signal.leak: 1117 if sweep.rec.leak: 1118 offset = offLeak 1119 else: 1120 offset = -1 1121 elif signal.second: 1122 if sweep.rec.secondTrace: 1123 offset = offSecond 1124 else: 1125 offset = -1 1126 else: 1127 offset = sweep.rec.data 1128 if offset >= 0: 1129 arrays.append([numpy.frombuffer(self.mmap, typ, seglen, offset)]) 1130 else: 1131 arrays.append([shared_zeros(seglen)]) 1132 self.sweeps.append(arrays) 1133 trace_segm.add_seg(0, seglen-1, 0.0) 1134 iSeg += 1 1135 1136 # override QuB's integer scaling values? copied from abf, why there? 1137 for chan in qubx.tree.children(sess.find('DataChannels'), 'Channel'): 1138 chan['Scaling'].data = 1.0 1139 chan['Offset'].data = 0.0 1140 1141 self.read_session(sess, progressf) 1142 1143 if not self.lists.lists: 1144 last = -1 1145 if header.levels == 5: 1146 for g, group in enumerate(self.root): 1147 for s, series in enumerate(group): 1148 lbl = ('%i: ' % g) if (len(self.root) > 1) else '' 1149 lbl = '%sSeries %i' % (lbl, s+1) 1150 if series.rec.label: 1151 lbl = '%s: %s' % (lbl, series.rec.label) 1152 list = self.lists.show_list(lbl) 1153 for sweep in series: 1154 seglen = sweep[0].rec.dataPoints 1155 first = last + 1 1156 last = first + seglen - 1 1157 list.insert_selection(first, last, sweep.rec.label) 1158 elif header.levels == 4: 1159 for group in self.root: 1160 for s, series in enumerate(group): 1161 lbl = ('%i: ' % g) if (len(self.root) > 1) else '' 1162 lbl = '%sSeries %i' % (lbl, s+1) 1163 list = self.lists.show_list(lbl) 1164 for sweep in series: 1165 seglen = sweep.rec.totalPoints 1166 first = last + 1 1167 last = first + seglen - 1 1168 list.insert_selection(first, last, sweep.rec.lbl)

1169

1170 - def __del__(self):

1171 if not (self.mmap is None): 1172 self.mmap.close() 1173 if self.fileno: 1174 os.close(self.fileno)

1175 1176 1177 fallback_reader = None 1178

1179 -def QubData_PUL_Open(path, progressf):

1180 try: 1181 return QubData_PUL(path, progressf) 1182 except: 1183 if fallback_reader: 1184 return fallback_reader(path, progressf) 1185 else: 1186 raise

1187 1188 fallback_reader = SetReader('.dat', 'DAT files', QubData_PUL_Open) # makes trouble if you reload(qubx.data_pulse) 1189 SetReader('.pul', 'Patchmaster/Pulse files', QubData_PUL_Open) 1190 1191

1192 -def print_levels(root, indent=""):

1193 print_struct(root.rec, indent) 1194 for child in root: 1195 print_levels(child, indent+" ")

1196 1197 if __name__ == '__main__': 1198 header, endian, root = LoadPUL(sys.argv[1]) 1199 print_struct(header) 1200 print_levels(root) 1201

Source Code for Module qubx.data_pulse