1 """Curve fitting tool acting on data window.
2
3 Copyright 2008-2014 Research Foundation State University of New York
4 This file is part of QUB Express.
5
6 QUB Express is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
10
11 QUB Express is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
15
16 You should have received a copy of the GNU General Public License,
17 named LICENSE.txt, in the QUB Express program directory. If not, see
18 <http://www.gnu.org/licenses/>.
19
20 """
21
22 from __future__ import with_statement
23
24 import os
25 import sys
26 import traceback
27 import numpy
28 import gobject
29 import cairo
30 import gtk
31 import qubx.fast.data
32 import qubx.GTK
33 import qubx.pyenv
34 import qubx.fast.data
35 import qubx.fit_space
36 import qubx.faces
37 import qubx.notebook
38 import qubx.notebookGTK
39 import qubx.settings
40 import qubx.task
41 import qubx.dataGTK
42
43 from gtk import gdk, keysyms
44 from itertools import izip, count, chain
45 from qubx.util_types import *
46 from qubx.GTK import build_menuitem
47 from qubx.accept import acceptFloatGreaterThan
48 from qubx.toolspace import ColorInfo
49 from qubx.tree import node_data_or_set_def
50
51 MAX_FIT_POINTS = 1000000
52
53 COLOR_DATAFIT_NUMBER = ('data_fit.number', (1, 1, 1, 1))
54 ColorInfo[COLOR_DATAFIT_NUMBER[0]].label = 'Data curvefit numbers'
55 COLOR_DATAFIT_BUTTON = ('data_fit.button', (.9, .35, .35, 1))
56 ColorInfo[COLOR_DATAFIT_BUTTON[0]].label = 'Data curvefit buttons'
57
59 """L{qubx.toolspace.LayerSet} to add least-squares curve fitting to the L{qubx.dataGTK.QubDataView_Hi}.
60
61 """
62 - def __init__(self, label='CurveFit', global_name=""):
63 qubx.fit_space.FitControls.__init__(self, label=label, cLayer=qubx.toolspace.LAYER_BG, cText=qubx.toolspace.LAYER_FG,
64 cNumber=COLOR_DATAFIT_NUMBER, cButton=COLOR_DATAFIT_BUTTON, global_name=global_name)
65 self.__ref = Reffer()
66 cat = qubx.settings.SettingsMgr[label]
67 self.profile = cat.active
68 cat.OnSet = self.__ref(self.__onUpdateProfile)
69 self.__relX = node_data_or_set_def(self.profile, 'RelX', 1)
70 self.__resample = bool(node_data_or_set_def(self.profile, 'Resample', 0))
71 self.__resampleStd = node_data_or_set_def(self.profile, 'ResampleStd', 0.1)
72 self.__nb_next_stats = False
73 self.__source_segs = self.__source_bounds = self.__source_view = None
74 self.__list = self.__list_ix = None
75 self.__list_priority = False
76 self.__resume_list_fit = False
77 self.QubX = qubx.pyenv.env.globals['QubX']
78 self.QubX.OnQuit += self.__ref(lambda: self.dispose())
79 self.datasrc = FitDataSource(self.QubX)
80 self.datasrc.use_list = False
81 self.datasrc.use_together = bool(node_data_or_set_def(self.profile, 'use_together', self.datasrc.use_together))
82 self.datasrc.OnChangeFile += self.__ref(self.__onChangeDataFile)
83 self.datasrc.OnChange += self.__ref(self.__onChangeData)
84 self.datasrc.OnListSelect += self.__ref(self.__onListSelect)
85 self.robot.OnExpr += self.__ref(self.__onExpr)
86 self.robot.OnParam += self.__ref(self.__onParam)
87 self.robot.OnStats += self.__ref(self.__onStats)
88 self.robot.OnException += self.__ref(self.__onRobotException)
89 self.robot.OnInterrupt += self.__ref(self.__onRobotInterrupt)
90 self.robot.OnIteration += self.__ref(self.__onIteration)
91 self.robot.OnStartFit += self.__ref(self.__onStartFit)
92 self.robot.OnEndFit += self.__ref(self.__onEndFit)
93 self.OnClickFit += self.__ref(self.__onClickFit)
94 self.layNotebook = qubx.toolspace.Layer(x=28, y=1+qubx.fit_space.LINE_EMS, w=2, h=2, cBG=qubx.toolspace.COLOR_CLEAR)
95 self.subNotebook = qubx.notebookGTK.SubLayer_Notebook(x=0, y=0, w=2, h=2)
96 self.layNotebook.add_sublayer(self.subNotebook)
97 self.layers.append(self.layNotebook)
98 self.nbParams = qubx.notebook.NbDynText('Parameters', 'QubX.Data.controls_fit.nbParams', self.nb_get_param_text)
99 self.nbParams.std_err_est = None
100 self.subNotebook.items.append(self.nbParams)
101 self.nbPicture = qubx.notebook.NbItems('Data image', 'QubX.Data.controls_fit.nbPicture')
102 self.subNotebook.items.append(self.nbPicture)
103 self.nbPictureNO = qubx.notebook.NbItems('Data image, no overlays', 'QubX.Data.controls_fit.nbPictureNO')
104 self.subNotebook.items.append(self.nbPictureNO)
105 self.nbChart = qubx.notebook.NbItems('Data table', 'QubX.Data.controls_fit.nbChart')
106 self.subNotebook.items.append(self.nbChart)
107 qubx.notebook.Notebook.register_auto('DataFit.Parameters', 'Data Fit Parameters, on Fit', True)
108 qubx.notebook.Notebook.register_auto('DataFit.Results', 'Data Fit results, on Fit', True)
109 qubx.notebook.Notebook.register_auto('DataFit.Chart', 'Data chart, on Fit', False)
110 qubx.notebook.Notebook.register_auto('DataFit.Picture', 'Data image, on Fit', True)
111 qubx.notebook.Notebook.register_auto('DataFit.PictureNO', 'Data image, no overlays, on Fit', False)
112
113 self.layBack = qubx.toolspace.Layer(x=1, y=-4*qubx.fit_space.LINE_EMS,
114 w=4+qubx.fit_space.LINE_EMS, h=qubx.fit_space.LINE_EMS,
115 cBG=qubx.toolspace.COLOR_CLEAR)
116 self.subBack = qubx.toolspace.SubLayer_Arrow(color=self.cButton, angle=pi,
117 x=0, y=0, w=qubx.fit_space.LINE_EMS, h=qubx.fit_space.LINE_EMS,
118 action=self.__ref(self.__onClickBack))
119 self.subBack.tooltip = 'Return to navigation controls'
120 self.layBack.add_sublayer(self.subBack)
121 self.layBack.add_sublayer(qubx.toolspace.SubLayer_Label('Back',
122 x=qubx.fit_space.LINE_EMS, y=0, w=4, h=qubx.fit_space.LINE_EMS,
123 action=self.__ref(self.__onClickBack)))
124 self.layers.append(self.layBack)
125
126 self.layData = qubx.toolspace.Layer(x=0, y=-2*qubx.fit_space.LINE_EMS-.5,
127 w=-.1, h=qubx.fit_space.LINE_EMS+.5,
128 cBG=self.cLayer)
129 self.layers.append(self.layData)
130 self.layData.add_sublayer(qubx.toolspace.SubLayer_Label('Data:', x=0, y=0.25, w=7, h=qubx.fit_space.LINE_EMS))
131 self.subUseOnscreen = qubx.toolspace.SubLayer_Check(x=8, y=0.25, w=10, h=qubx.fit_space.LINE_EMS,
132 color=self.cButton, caption='onscreen')
133 self.subUseOnscreen.tooltip = "Fit onscreen portion of current segment"
134 self.subUseOnscreen.set_active(not self.datasrc.use_list)
135 self.subUseOnscreen.OnToggle += self.__ref(self.__onToggleUseOnscreen)
136 self.layData.add_sublayer(self.subUseOnscreen)
137 self.subUseList = qubx.toolspace.SubLayer_Check(x=18, y=0.25, w=10, h=qubx.fit_space.LINE_EMS,
138 color=self.cButton, caption='List')
139 self.subUseList.tooltip = "Fit all selections in List"
140 self.subUseList.set_active(self.datasrc.use_list)
141 self.subUseList.OnToggle += self.__ref(self.__onToggleUseList)
142 self.layData.add_sublayer(self.subUseList)
143 self.subUseTogether = qubx.toolspace.SubLayer_Check(x=26, y=0.25, w=10, h=qubx.fit_space.LINE_EMS,
144 color=self.cButton, caption='together')
145 self.subUseTogether.tooltip = "Fit all selections at once, or else one-by-one"
146 self.subUseTogether.set_active(self.datasrc.use_together)
147 self.subUseTogether.OnToggle += self.__ref(self.__onToggleUseTogether)
148 if self.datasrc.use_list:
149 self.layData.add_sublayer(self.subUseTogether)
150
151
152 self.subBackSeg = qubx.toolspace.SubLayer_Label('-', x=-24, w=1, h=qubx.fit_space.LINE_EMS,
153 action=self.__ref(self.__onClickBackSeg))
154 self.subBackSeg.tooltip = 'Back one segment'
155 self.layData.add_sublayer(self.subBackSeg)
156 self.subBackScreen = qubx.toolspace.SubLayer_Label('<', x=-22.5, w=1, h=qubx.fit_space.LINE_EMS,
157 action=self.__ref(self.__onClickBackScreen))
158 self.subBackScreen.tooltip = 'Back one screenful'
159 self.layData.add_sublayer(self.subBackScreen)
160 self.subFwdScreen = qubx.toolspace.SubLayer_Label('>', x=-19.5, w=1, h=qubx.fit_space.LINE_EMS,
161 action=self.__ref(self.__onClickFwdScreen))
162 self.subFwdScreen.tooltip = 'Forward one screenful'
163 self.layData.add_sublayer(self.subFwdScreen)
164 self.subFwdSeg = qubx.toolspace.SubLayer_Label('+', x=-18, w=1, h=qubx.fit_space.LINE_EMS,
165 action=self.__ref(self.__onClickFwdSeg))
166 self.subFwdSeg.tooltip = 'Forward one segment'
167 self.layData.add_sublayer(self.subFwdSeg)
168 self.subGrab = qubx.toolspace.SubLayer_Label('?', x=-16, w=1, h=qubx.fit_space.LINE_EMS,
169 action=self.__ref(self.__onClickGrab))
170 self.subGrab.tooltip = 'Rescale data to screen'
171 self.layData.add_sublayer(self.subGrab)
172
173
174 self.layBottom.remove_sublayer(self.subN)
175 self.subN.x = -9
176 self.layData.add_sublayer(self.subN)
177
178
179 self.subRelX = qubx.toolspace.SubLayer_Check(x=5, y=0, w=23, h=qubx.fit_space.LINE_EMS,
180 color=self.cButton,
181 caption='x coordinate starts at 0')
182 self.subRelX.set_active(self.__relX)
183 self.subRelX.OnToggle += self.__ref(self.__onToggleRelX)
184 self.layBottom.add_sublayer(self.subRelX)
185
186 self.subResample = qubx.toolspace.SubLayer_Check(x=28, y=0, w=17, h=qubx.fit_space.LINE_EMS,
187 color=self.cButton, caption='Resample by delta y=')
188 self.subResample.set_active(self.__resample)
189 self.subResample.OnToggle += self.__ref(self.__onToggleResample)
190 self.layBottom.add_sublayer(self.subResample)
191 self.subDelta = qubx.toolspace.SubLayer_Label('%.3g'%self.__resampleStd, -1, 0, x=46, y=0,
192 w=8, h=qubx.fit_space.LINE_EMS, color=self.cText,
193 action=self.__ref(self.__onClickDelta))
194 self.layBottom.add_sublayer(self.subDelta)
195
196 self.subRestoreFit = qubx.toolspace.SubLayer_Label('Restore last fit', 0, 0, x=-33, y=0, w=14, h=qubx.fit_space.LINE_EMS, color=self.cButton,
197 action=self.__ref(self.__onClickRestoreFit))
198 self.__showing_restore = False
199
200 gobject.idle_add(self.__onChangeDataFile)
201 gobject.idle_add(self.__onChangeData)
202 self.__serial_data = 0
203 self.__serial_fit = 0
204 self.__delayed_curve = False
205 self.__xx = numpy.array([])
206 self.__ff = numpy.array([])
207 self.__ll = numpy.array([])
208 self.__vvv = []
215 relX = property(lambda self: self.__relX, lambda self, x: self.set_relX(x))
222 resample = property(lambda self: self.__resample, lambda self, x: self.set_resample(x))
224 if x != self.__resampleStd:
225 self.__resampleStd = x
226 self.profile['ResampleStd'].data = x
227 self.subDelta.label = '%.3g' % x
228 self.__onChangeData()
229 resampleStd = property(lambda self: self.__resampleStd, lambda self, x: self.set_resampleStd(x))
237 use_list = property(lambda self: self.datasrc.use_list, lambda self, x: self.set_use_list(x))
247 if use_segments:
248 self.layData.add_sublayer(self.subUseTogether)
249 else:
250 self.layData.remove_sublayer(self.subUseTogether)
256 use_together = property(lambda self: self.datasrc.use_together, lambda self, x: self.set_use_together(x))
270 self.__stopped = True
271 qubx.pyenv.env.OnScriptable('QubX.Data.view.request_fit_controls(False)')
272 gobject.idle_add(self.datasrc.data.request_fit_controls, False)
298 if self.__delayed_curve:
299 self.__delayed_curve = False
300 self.__onChangeData()
302 have_fit = not ((self.__list is None) or (self.__list_ix is None) or (self.__list.fits[self.__list_ix] is None))
303 if self.__showing_restore and not have_fit:
304 self.layBottom.remove_sublayer(self.subRestoreFit)
305 self.__showing_restore = False
306 elif have_fit and not self.__showing_restore:
307 self.layBottom.add_sublayer(self.subRestoreFit)
308 self.__showing_restore = True
310 self.profileCat.setProperties(self.__list.fits[self.__list_ix])
332 traceback.print_exception(typ, val, trace)
334 self.__stopped = True
335 - def __onExpr(self, curve_name, expr, params, param_vals, lo, hi, can_fit):
341 - def __onParam(self, index, name, value, lo, hi, can_fit):
350 self.__serial_fit += 1
351 if not (param_vals is None):
352 self.__param_vals = param_vals
353 gobject.idle_add(self.__update_fit, self.__param_vals, self.__serial_fit)
377 - def fit(self, wait=True, receiver=None):
384 self.__stopped = False
386 self.__nb_next_stats = True
387 gobject.idle_add(self.update_fit)
389 self.__list_receiver = receiver
390 qubx.fit_space.FitControls.fit(self, wait=False)
392 self.__list_ix = ix
393 self.__list_priority = True
399
400
401
402
403
404
406 if self.__list_ix < (self.__list.size - 1):
407 self.__resume_list_fit = True
408 self.__zoom_to_list(self.__list_ix + 1)
409 elif self.__list_receiver:
410 gobject.idle_add(self.__list_receiver)
411 self.__list_receiver = None
412 - def __onStats(self, correlation, is_pseudo, std_err_est, ssr, r2, runs_prob):
413 self.update_fit()
414 if self.__nb_next_stats:
415 self.__nb_next_stats = False
416 if not self.__stopped and self.datasrc.use_list and not self.datasrc.use_together:
417 gobject.idle_add(self.fit_next_segment)
418 n = len(self.__xx)
419 segm = ((self.datasrc.use_list and not self.datasrc.use_together)
420 and ("Selection %i\n" % self.__list_ix)
421 or "")
422 resamp = self.__resample and ("(resampled by delta = %.3g"%self.__resampleStd) or ""
423 qubx.notebook.Notebook.send(qubx.notebook.NbText("""CurveFit finished.
424 %(segm)s\tN: %(n)i %(resamp)s
425 \tSSR: %(ssr).6g
426 \tR^2: %(r2).6g
427 \tWald-Wolfowitz runs probability: %(runs_prob).6g
428 \tCorrelation:
429 %(correlation)s
430 """ % locals()), auto_id='DataFit.Results')
431 self.nbParams.std_err_est = std_err_est
432 qubx.notebook.Notebook.send(self.nbParams, auto_id='DataFit.Parameters')
433 qubx.notebook.Notebook.send(self.__source_view.hires.nbChart, auto_id='DataFit.Chart')
434 qubx.notebook.Notebook.send(self.__source_view.hires.nbPicture, auto_id='DataFit.Picture')
435 qubx.notebook.Notebook.send(self.__source_view.hires.nbPictureNO, auto_id='DataFit.PictureNO')
436 self.nbParams.std_err_est = None
437 if not (self.__list_ix is None):
438
439 tb = self.__list
440 si = self.__list_ix
441 tb.set(si, "fit N", n)
442 tb.set(si, "fit SSR", ssr)
443 tb.set(si, "fit SSR per N", n and (ssr/n) or 0.0)
444 tb.set(si, "fit R2", r2)
445 tb.set(si, "fit runs prob", runs_prob)
446 for i, name in enumerate(self.__param_names):
447 tb.set(si, "fit %s" % name, self.__param_vals[i])
448 tb.set(si, "fit %s err" % name, std_err_est[i])
449
450 preset = qubx.tree.Node('Curve')
451 preset.appendClone(self.profile['Name'])
452 preset.appendClone(self.profile['Eqn'])
453 preset.appendClone(self.profile['Weight'])
454 preset.appendClone(self.profile['Params'])
455
456 self.__list.fits[self.__list_ix] = preset
463 if (not self.QubX.Data.view) or (not self.space): return
464 self.__fit_fails = 0
465 self.__serial_data += 1
466
467 if self.__delayed_curve:
468 if len(self.__xx):
469 self.robot.do(self.robot_null_data, self.__serial_data)
470 else:
471 self.robot.do(self.robot_read_data, self.__serial_data)
473 if serial < self.__serial_data: return
474 with self.robot.main_hold:
475 self.__source_view = self.QubX.Data.view
476 if not self.__resume_list_fit:
477 self.__list = self.QubX.Data.file.list
478 if (self.__list_priority is True) and not (self.__list_ix is None):
479 f, l = self.__list[self.__list_ix, 'From'], self.__list[self.__list_ix, 'To']
480 segs = self.__source_segs = self.QubX.Data.view.get_segmentation_indexed(f, l, self.QubX.DataSource.signal)
481 vsegs = [self.QubX.Data.view.get_segmentation_indexed(f, l, signal=i) for i in xrange(segs[0].file.signals.size)]
482 self.__list_priority = 'done'
483 elif self.__list_priority is False:
484 segs = self.__source_segs = self.datasrc.get_segmentation()
485 vsegs = [self.datasrc.get_segmentation(signal=i) for i in xrange(segs[0].file.signals.size)]
486 self.__list_ix = None
487 if len(segs) == 1:
488 f, l = segs[0].f, segs[0].l
489 lst = segs[0].file.list
490 sels = lst.selections_in(f, l)
491 for sel in sels:
492 if (abs(sel[0] - f) < 3) and (abs(sel[1] - l) < 3):
493 self.__list = lst
494 self.__list_ix = sel[3]
495 break
496 else:
497 self.__list_priority = False
498 return
499 bounds = self.__source_bounds = []
500 firsts = []
501 lasts = []
502 xxs = []
503 vvvs = []
504 total_n = 0
505 bounds_at = 0
506 for i in xrange(segs[0].file.signals.size):
507 vvvs.append([])
508 for iseg, seg in enumerate(segs):
509 seg_n = 0
510 vseg = [vsegg[iseg] for vsegg in vsegs]
511 for c, chunk in enumerate(seg.chunks):
512 if chunk.included:
513 chunk.n = min(chunk.n, MAX_FIT_POINTS-total_n)
514 if chunk.n <= 0:
515 break
516 chunk.l = chunk.f + chunk.n - 1
517 xx = numpy.arange(chunk.f-seg.f, chunk.l-seg.f+1, dtype='float32')
518 xx *= seg.sampling
519 if not self.__relX:
520 xx += 1e-3 * (seg.start - seg.file.segments[0, 'Start'])
521 vvv = [vseg[i].chunks[c].get_samples().samples for i in xrange(len(vvvs))]
522 if self.__resample:
523 means, stds, ff, ll, closest = qubx.fast.data.adaptive_resample(vvv[seg.signal], self.__resampleStd)
524 xx = numpy.array(xx[closest], dtype='float32', copy=True)
525 vvv = [numpy.array(v[closest], dtype='float32', copy=True) for v in vvv]
526 vvv[seg.signal] = means
527 ff += chunk.f
528 ll += chunk.f
529 else:
530 ff = ll = numpy.arange(chunk.f, chunk.l+1, dtype='int32')
531 firsts.append(ff)
532 lasts.append(ll)
533 xxs.append(xx)
534 for i, v in enumerate(vvv):
535 vvvs[i].append(v)
536 total_n += len(xx)
537 seg_n += len(xx)
538 bounds.append( (bounds_at, bounds_at+seg_n-1) )
539 bounds_at += seg_n
540 v_names = [segs[0].file.signals.get(i, 'Name') for i in xrange(segs[0].file.signals.size)]
541 self.__ff = numpy.hstack(firsts)
542 self.__ll = numpy.hstack(lasts)
543 self.__xx = numpy.hstack(xxs)
544 self.__vvv = [numpy.hstack(vvv) for vvv in vvvs]
545 self.robot.robot_set_data(self.__xx, self.__vvv[seg.signal], self.__vvv, v_names)
546 if self.__resume_list_fit:
547 self.__resume_list_fit = False
548 gobject.idle_add(self.robot.fit, False)
549 else:
550 gobject.idle_add(self.__update_restore_fit)
552 if serial < self.__serial_data: return
553 self.__ff = numpy.array([])
554 self.__ll = numpy.array([])
555 self.__xx = numpy.array([])
556 self.__vvv = []
557 self.robot.robot_set_data(self.__xx, self.__xx, self.__vvv, [])
558
559
561 """Provides sampled and data to the CurveFit specification.
562
563 @ivar use_list: True to fit selections from the List
564 @ivar use_together: False to fit selections one by one
565 @ivar signal: zero-based index of active signal
566 @ivar OnChangeFile: L{WeakEvent}()
567 @ivar OnChange: L{WeakEvent}() on any change including file
568 """
583 if x == self.__use_list: return
584 self.__use_list = x
585 self.OnChange()
586 use_list = property(lambda self: self.__use_list, lambda self, x: self.set_use_list(x))
588 if x == self.__use_together: return
589 self.__use_together = x
590 if self.__use_list:
591 self.OnChange()
592 use_together = property(lambda self: self.__use_together, lambda self, x: self.set_use_together(x))
608 data = property(lambda self: self.__data, lambda self, x: self.set_data(x))
613 if self.__use_list and self.__use_together: return
614 self.OnChange()
632 - def gen_samples(self, chunks, main_hold=None, signal=None, maxlen=(1<<20), get_excluded=False):
633 """Yields a L{SourceChunk} with fields .samples and .sampling for each in chunks.
634
635 @param chunks: list of L{SourceChunk}
636 @param main_hold: L{qubx.task.GTK_Main_Hold} if in a worker thread, or None in the main thread
637 @param signal: signal index, or None to use the chunk.signal
638 @param maxlen: chunks longer than this will be yielded in pieces
639 @param get_excluded: if True, yields sampled chunks even when included==False (if False, yields orig chunk when excluded).
640 """
641 return generate_chunk_samples(chunks, main_hold, signal, maxlen, get_excluded)
642