mvpa.clfs.transerror

1 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 2 # vi: set ft=python sts=4 ts=4 sw=4 et: 3 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 4 # 5 # See COPYING file distributed along with the PyMVPA package for the 6 # copyright and license terms. 7 # 8 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 9 """Utility class to compute the transfer error of classifiers.""" 10 11 __docformat__ = 'restructuredtext' 12 13 import mvpa.support.copy as copy 14 15 import numpy as N 16 17 from StringIO import StringIO 18 from math import log10, ceil 19 20 from mvpa.base import externals 21 22 from mvpa.misc.errorfx import meanPowerFx, rootMeanPowerFx, RMSErrorFx, \ 23 CorrErrorFx, CorrErrorPFx, RelativeRMSErrorFx, MeanMismatchErrorFx, \ 24 AUCErrorFx 25 from mvpa.base import warning 26 from mvpa.misc.state import StateVariable, ClassWithCollections 27 from mvpa.base.dochelpers import enhancedDocString, table2string 28 from mvpa.clfs.stats import autoNullDist 29 30 if __debug__: 31 from mvpa.base import debug 32 33 if externals.exists('scipy'): 34 from scipy.stats.stats import nanmean 35 else: 36 from mvpa.clfs.stats import nanmean

37 38 -def _p2(x, prec=2):

39 """Helper to print depending on the type nicely. For some 40 reason %.2g for 100 prints exponential form which is ugly 41 """ 42 if isinstance(x, int): 43 return "%d" % x 44 elif isinstance(x, float): 45 s = ("%%.%df" % prec % x).rstrip('0').rstrip('.').lstrip() 46 if s == '': 47 s = '0' 48 return s 49 else: 50 return "%s" % x

51

52 53 54 -class SummaryStatistics(object):

55 """Basic class to collect targets/predictions and report summary statistics 56 57 It takes care about collecting the sets, which are just tuples 58 (targets, predictions, values). While 'computing' the matrix, all 59 sets are considered together. Children of the class are 60 responsible for computation and display. 61 """ 62 63 _STATS_DESCRIPTION = ( 64 ('# of sets', 65 'number of target/prediction sets which were provided', 66 None), ) 67 68

69 - def __init__(self, targets=None, predictions=None, values=None, sets=None):

70 """Initialize SummaryStatistics 71 72 targets or predictions cannot be provided alone (ie targets 73 without predictions) 74 75 :Parameters: 76 targets 77 Optional set of targets 78 predictions 79 Optional set of predictions 80 values 81 Optional set of values (which served for prediction) 82 sets 83 Optional list of sets 84 """ 85 self._computed = False 86 """Flag either it was computed for a given set of data""" 87 88 self.__sets = (sets, [])[int(sets is None)] 89 """Datasets (target, prediction) to compute confusion matrix on""" 90 91 self._stats = {} 92 """Dictionary to keep statistics. Initialized here to please pylint""" 93 94 if not targets is None or not predictions is None: 95 if not targets is None and not predictions is None: 96 self.add(targets=targets, predictions=predictions, 97 values=values) 98 else: 99 raise ValueError, \ 100 "Please provide none or both targets and predictions"

101 102

103 - def add(self, targets, predictions, values=None):

104 """Add new results to the set of known results""" 105 if len(targets) != len(predictions): 106 raise ValueError, \ 107 "Targets[%d] and predictions[%d]" % (len(targets), 108 len(predictions)) + \ 109 " have different number of samples" 110 111 if values is not None and len(targets) != len(values): 112 raise ValueError, \ 113 "Targets[%d] and values[%d]" % (len(targets), 114 len(values)) + \ 115 " have different number of samples" 116 117 # enforce labels in predictions to be of the same datatype as in 118 # targets, since otherwise we are getting doubles for unknown at a 119 # given moment labels 120 nonetype = type(None) 121 for i in xrange(len(targets)): 122 t1, t2 = type(targets[i]), type(predictions[i]) 123 # if there were no prediction made - leave None, otherwise 124 # convert to appropriate type 125 if t1 != t2 and t2 != nonetype: 126 #warning("Obtained target %s and prediction %s are of " % 127 # (t1, t2) + "different datatypes.") 128 if isinstance(predictions, tuple): 129 predictions = list(predictions) 130 predictions[i] = t1(predictions[i]) 131 132 if values is not None: 133 # assure that we have a copy, or otherwise further in-place 134 # modifications might screw things up (some classifiers share 135 # values and spit out results) 136 values = copy.deepcopy(values) 137 138 self.__sets.append( (targets, predictions, values) ) 139 self._computed = False

140 141

142 - def asstring(self, short=False, header=True, summary=True, 143 description=False):

144 """'Pretty print' the matrix 145 146 :Parameters: 147 short : bool 148 if True, ignores the rest of the parameters and provides consise 149 1 line summary 150 header : bool 151 print header of the table 152 summary : bool 153 print summary (accuracy) 154 description : bool 155 print verbose description of presented statistics 156 """ 157 raise NotImplementedError

158 159

160 - def __str__(self):

161 """String summary over the `SummaryStatistics` 162 163 It would print description of the summary statistics if 'CM' 164 debug target is active 165 """ 166 if __debug__: 167 description = ('CM' in debug.active) 168 else: 169 description = False 170 return self.asstring(short=False, header=True, summary=True, 171 description=description)

172 173

174 - def __iadd__(self, other):

175 """Add the sets from `other` s `SummaryStatistics` to current one 176 """ 177 #print "adding ", other, " to ", self 178 # need to do shallow copy, or otherwise smth like "cm += cm" 179 # would loop forever and exhaust memory eventually 180 othersets = copy.copy(other.__sets) 181 for set in othersets: 182 self.add(*set)#[0], set[1]) 183 return self

184 185

186 - def __add__(self, other):

187 """Add two `SummaryStatistics`s 188 """ 189 result = copy.copy(self) 190 result += other 191 return result

192 193

194 - def compute(self):

195 """Actually compute the confusion matrix based on all the sets""" 196 if self._computed: 197 return 198 199 self._compute() 200 self._computed = True

201 202

203 - def _compute(self):

204 """Compute basic statistics 205 """ 206 self._stats = {'# of sets' : len(self.sets)}

207 208 209 @property

210 - def summaries(self):

211 """Return a list of separate summaries per each stored set""" 212 return [ self.__class__(sets=[x]) for x in self.sets ]

213 214 215 @property

216 - def error(self):

217 raise NotImplementedError

218 219 220 @property

221 - def stats(self):

222 self.compute() 223 return self._stats

224 225

226 - def reset(self):

227 """Cleans summary -- all data/sets are wiped out 228 """ 229 self.__sets = [] 230 self._computed = False

231 232 233 sets = property(lambda self:self.__sets)

234

235 236 -class ROCCurve(object):

237 """Generic class for ROC curve computation and plotting 238 """ 239

240 - def __init__(self, labels, sets=None):

241 """ 242 :Parameters: 243 labels : list 244 labels which were used (in order of values if multiclass, 245 or 1 per class for binary problems (e.g. in SMLR)) 246 sets : list of tuples 247 list of sets for the analysis 248 """ 249 self._labels = labels 250 self._sets = sets 251 self.__computed = False

252 253

254 - def _compute(self):

255 """Lazy computation if needed 256 """ 257 if self.__computed: 258 return 259 # local bindings 260 labels = self._labels 261 Nlabels = len(labels) 262 sets = self._sets 263 264 # Handle degenerate cases politely 265 if Nlabels < 2: 266 warning("ROC was asked to be evaluated on data with %i" 267 " labels which is a degenerate case.") 268 self._ROCs = [] 269 self._aucs = [] 270 return 271 272 # take sets which have values in the shape we can handle 273 def _checkValues(set_): 274 """Check if values are 'acceptable'""" 275 if len(set_)<3: return False 276 x = set_[2] 277 # TODO: OPT: need optimization 278 if (x is None) or len(x) == 0: return False # undefined 279 for v in x: 280 try: 281 if Nlabels <= 2 and N.isscalar(v): 282 continue 283 if (isinstance(v, dict) or # not dict for pairs 284 ((Nlabels>=2) and len(v)!=Nlabels) # 1 per each label for multiclass 285 ): return False 286 except Exception, e: 287 # Something else which is not supported, like 288 # in shogun interface we don't yet extract values per each label or 289 # in pairs in the case of built-in multiclass 290 if __debug__: 291 debug('ROC', "Exception %s while checking " 292 "either %s are valid labels" % (str(e), x)) 293 return False 294 return True

295 296 sets_wv = filter(_checkValues, sets) 297 # check if all had values, if not -- complain 298 Nsets_wv = len(sets_wv) 299 if Nsets_wv > 0 and len(sets) != Nsets_wv: 300 warning("Only %d sets have values assigned from %d sets. " 301 "ROC estimates might be incorrect." % 302 (Nsets_wv, len(sets))) 303 # bring all values to the same 'shape': 304 # 1 value per each label. In binary classifier, if only a single 305 # value is provided, add '0' for 0th label 'value'... it should 306 # work taking drunk Yarik logic ;-) 307 # yoh: apparently it caused problems whenever we had just a single 308 # unique label in the sets. Introduced handling for 309 # NLabels == 1 310 for iset,s in enumerate(sets_wv): 311 # we will do inplace modification, thus go by index 312 values = s[2] 313 # we would need it to be a list to reassign element with a list 314 if isinstance(values, N.ndarray) and len(values.shape)==1: 315 # XXX ??? so we are going away from inplace modifications? 316 values = list(values) 317 rangev = None 318 for i in xrange(len(values)): 319 v = values[i] 320 if N.isscalar(v): 321 if Nlabels == 1: 322 # ensure the right dimensionality 323 values[i] = N.array(v, ndmin=2) 324 elif Nlabels == 2: 325 def last_el(x): 326 """Helper function. Returns x if x is scalar, and 327 last element if x is not (ie list/tuple)""" 328 if N.isscalar(x): return x 329 else: return x[-1]

330 if rangev is None: 331 # we need to figure out min/max values 332 # to invert for the 0th label 333 values_ = [last_el(x) for x in values] 334 rangev = N.min(values_) + N.max(values_) 335 values[i] = [rangev - v, v] 336 else: 337 raise ValueError, \ 338 "Cannot have a single 'value' for multiclass" \ 339 " classification. Got %s" % (v) 340 elif len(v) != Nlabels: 341 raise ValueError, \ 342 "Got %d values whenever there is %d labels" % \ 343 (len(v), Nlabels) 344 # reassign possibly adjusted values 345 sets_wv[iset] = (s[0], s[1], N.asarray(values)) 346 347 348 # we need to estimate ROC per each label 349 # XXX order of labels might not correspond to the one among 'values' 350 # which were used to make a decision... check 351 ROCs, aucs = [], [] # 1 per label 352 for i,label in enumerate(labels): 353 aucs_pl = [] 354 ROCs_pl = [] 355 for s in sets_wv: 356 targets_pl = (N.asanyarray(s[0]) == label).astype(int) 357 # XXX we might unify naming between AUC/ROC 358 ROC = AUCErrorFx() 359 aucs_pl += [ROC([N.asanyarray(x)[i] for x in s[2]], targets_pl)] 360 ROCs_pl.append(ROC) 361 if len(aucs_pl)>0: 362 ROCs += [ROCs_pl] 363 aucs += [nanmean(aucs_pl)] 364 #aucs += [N.mean(aucs_pl)] 365 366 # store results within the object 367 self._ROCs = ROCs 368 self._aucs = aucs 369 self.__computed = True 370 371 372 @property

373 - def aucs(self):

374 """Compute and return set of AUC values 1 per label 375 """ 376 self._compute() 377 return self._aucs

378 379 380 @property

381 - def ROCs(self):

382 self._compute() 383 return self._ROCs

384 385

386 - def plot(self, label_index=0):

387 """ 388 389 TODO: make it friendly to labels given by values? 390 should we also treat labels_map? 391 """ 392 externals.exists("pylab", raiseException=True) 393 import pylab as P 394 395 self._compute() 396 397 labels = self._labels 398 # select only ROCs for the given label 399 ROCs = self.ROCs[label_index] 400 401 fig = P.gcf() 402 ax = P.gca() 403 404 P.plot([0, 1], [0, 1], 'k:') 405 406 for ROC in ROCs: 407 P.plot(ROC.fp, ROC.tp, linewidth=1) 408 409 P.axis((0.0, 1.0, 0.0, 1.0)) 410 P.axis('scaled') 411 P.title('Label %s. Mean AUC=%.2f' % (label_index, self.aucs[label_index])) 412 413 P.xlabel('False positive rate') 414 P.ylabel('True positive rate')

415

416 417 -class ConfusionMatrix(SummaryStatistics):

418 """Class to contain information and display confusion matrix. 419 420 Implementation of the `SummaryStatistics` in the case of 421 classification problem. Actual computation of confusion matrix is 422 delayed until all data is acquired (to figure out complete set of 423 labels). If testing data doesn't have a complete set of labels, 424 but you like to include all labels, provide them as a parameter to 425 the constructor. 426 427 Confusion matrix provides a set of performance statistics (use 428 asstring(description=True) for the description of abbreviations), 429 as well ROC curve (http://en.wikipedia.org/wiki/ROC_curve) 430 plotting and analysis (AUC) in the limited set of problems: 431 binary, multiclass 1-vs-all. 432 """ 433 434 _STATS_DESCRIPTION = ( 435 ('TP', 'true positive (AKA hit)', None), 436 ('TN', 'true negative (AKA correct rejection)', None), 437 ('FP', 'false positive (AKA false alarm, Type I error)', None), 438 ('FN', 'false negative (AKA miss, Type II error)', None), 439 ('TPR', 'true positive rate (AKA hit rate, recall, sensitivity)', 440 'TPR = TP / P = TP / (TP + FN)'), 441 ('FPR', 'false positive rate (AKA false alarm rate, fall-out)', 442 'FPR = FP / N = FP / (FP + TN)'), 443 ('ACC', 'accuracy', 'ACC = (TP + TN) / (P + N)'), 444 ('SPC', 'specificity', 'SPC = TN / (FP + TN) = 1 - FPR'), 445 ('PPV', 'positive predictive value (AKA precision)', 446 'PPV = TP / (TP + FP)'), 447 ('NPV', 'negative predictive value', 'NPV = TN / (TN + FN)'), 448 ('FDR', 'false discovery rate', 'FDR = FP / (FP + TP)'), 449 ('MCC', "Matthews Correlation Coefficient", 450 "MCC = (TP*TN - FP*FN)/sqrt(P N P' N')"), 451 ('AUC', "Area under (AUC) curve", None), 452 ) + SummaryStatistics._STATS_DESCRIPTION 453 454

455 - def __init__(self, labels=None, labels_map=None, **kwargs):

456 """Initialize ConfusionMatrix with optional list of `labels` 457 458 :Parameters: 459 labels : list 460 Optional set of labels to include in the matrix 461 labels_map : None or dict 462 Dictionary from original dataset to show mapping into 463 numerical labels 464 targets 465 Optional set of targets 466 predictions 467 Optional set of predictions 468 """ 469 470 SummaryStatistics.__init__(self, **kwargs) 471 472 if labels == None: 473 labels = [] 474 self.__labels = labels 475 """List of known labels""" 476 self.__labels_map = labels_map 477 """Mapping from original into given labels""" 478 self.__matrix = None 479 """Resultant confusion matrix"""

480 481 482 # XXX might want to remove since summaries does the same, just without 483 # supplying labels 484 @property

485 - def matrices(self):

486 """Return a list of separate confusion matrix per each stored set""" 487 return [ self.__class__(labels=self.labels, 488 labels_map=self.labels_map, 489 sets=[x]) for x in self.sets]

490 491

492 - def _compute(self):

493 """Actually compute the confusion matrix based on all the sets""" 494 495 super(ConfusionMatrix, self)._compute() 496 497 if __debug__: 498 if not self.__matrix is None: 499 debug("LAZY", 500 "Have to recompute %s#%s" \ 501 % (self.__class__.__name__, id(self))) 502 503 504 # TODO: BinaryClassifier might spit out a list of predictions for each 505 # value need to handle it... for now just keep original labels 506 try: 507 # figure out what labels we have 508 labels = \ 509 list(reduce(lambda x, y: x.union(set(y[0]).union(set(y[1]))), 510 self.sets, 511 set(self.__labels))) 512 except: 513 labels = self.__labels 514 515 # Check labels_map if it was provided if it covers all the labels 516 labels_map = self.__labels_map 517 if labels_map is not None: 518 labels_set = set(labels) 519 map_labels_set = set(labels_map.values()) 520 521 if not map_labels_set.issuperset(labels_set): 522 warning("Provided labels_map %s is not coherent with labels " 523 "provided to ConfusionMatrix. No reverse mapping " 524 "will be provided" % labels_map) 525 labels_map = None 526 527 # Create reverse map 528 labels_map_rev = None 529 if labels_map is not None: 530 labels_map_rev = {} 531 for k,v in labels_map.iteritems(): 532 v_mapping = labels_map_rev.get(v, []) 533 v_mapping.append(k) 534 labels_map_rev[v] = v_mapping 535 self.__labels_map_rev = labels_map_rev 536 537 labels.sort() 538 self.__labels = labels # store the recomputed labels 539 540 Nlabels, Nsets = len(labels), len(self.sets) 541 542 if __debug__: 543 debug("CM", "Got labels %s" % labels) 544 545 # Create a matrix for all votes 546 mat_all = N.zeros( (Nsets, Nlabels, Nlabels), dtype=int ) 547 548 # create total number of samples of each label counts 549 # just for convinience I guess since it can always be 550 # computed from mat_all 551 counts_all = N.zeros( (Nsets, Nlabels) ) 552 553 # reverse mapping from label into index in the list of labels 554 rev_map = dict([ (x[1], x[0]) for x in enumerate(labels)]) 555 for iset, set_ in enumerate(self.sets): 556 for t,p in zip(*set_[:2]): 557 mat_all[iset, rev_map[p], rev_map[t]] += 1 558 559 560 # for now simply compute a sum of votes across different sets 561 # we might do something more sophisticated later on, and this setup 562 # should easily allow it 563 self.__matrix = N.sum(mat_all, axis=0) 564 self.__Nsamples = N.sum(self.__matrix, axis=0) 565 self.__Ncorrect = sum(N.diag(self.__matrix)) 566 567 TP = N.diag(self.__matrix) 568 offdiag = self.__matrix - N.diag(TP) 569 stats = { 570 '# of labels' : Nlabels, 571 'TP' : TP, 572 'FP' : N.sum(offdiag, axis=1), 573 'FN' : N.sum(offdiag, axis=0)} 574 575 stats['CORR'] = N.sum(TP) 576 stats['TN'] = stats['CORR'] - stats['TP'] 577 stats['P'] = stats['TP'] + stats['FN'] 578 stats['N'] = N.sum(stats['P']) - stats['P'] 579 stats["P'"] = stats['TP'] + stats['FP'] 580 stats["N'"] = stats['TN'] + stats['FN'] 581 stats['TPR'] = stats['TP'] / (1.0*stats['P']) 582 # reset nans in TPRs to 0s whenever there is no entries 583 # for those labels among the targets 584 stats['TPR'][stats['P'] == 0] = 0 585 stats['PPV'] = stats['TP'] / (1.0*stats["P'"]) 586 stats['NPV'] = stats['TN'] / (1.0*stats["N'"]) 587 stats['FDR'] = stats['FP'] / (1.0*stats["P'"]) 588 stats['SPC'] = (stats['TN']) / (1.0*stats['FP'] + stats['TN']) 589 590 MCC_denom = N.sqrt(1.0*stats['P']*stats['N']*stats["P'"]*stats["N'"]) 591 nz = MCC_denom!=0.0 592 stats['MCC'] = N.zeros(stats['TP'].shape) 593 stats['MCC'][nz] = \ 594 (stats['TP'] * stats['TN'] - stats['FP'] * stats['FN'])[nz] \ 595 / MCC_denom[nz] 596 597 stats['ACC'] = N.sum(TP)/(1.0*N.sum(stats['P'])) 598 stats['ACC%'] = stats['ACC'] * 100.0 599 600 # 601 # ROC computation if available 602 ROC = ROCCurve(labels=labels, sets=self.sets) 603 aucs = ROC.aucs 604 if len(aucs)>0: 605 stats['AUC'] = aucs 606 if len(aucs) != Nlabels: 607 raise RuntimeError, \ 608 "We must got a AUC per label. Got %d instead of %d" % \ 609 (len(aucs), Nlabels) 610 self.ROC = ROC 611 else: 612 # we don't want to provide ROC if it is bogus 613 stats['AUC'] = [N.nan] * Nlabels 614 self.ROC = None 615 616 617 # compute mean stats 618 for k,v in stats.items(): 619 stats['mean(%s)' % k] = N.mean(v) 620 621 self._stats.update(stats)

622 623

624 - def asstring(self, short=False, header=True, summary=True, 625 description=False):

626 """'Pretty print' the matrix 627 628 :Parameters: 629 short : bool 630 if True, ignores the rest of the parameters and provides consise 631 1 line summary 632 header : bool 633 print header of the table 634 summary : bool 635 print summary (accuracy) 636 description : bool 637 print verbose description of presented statistics 638 """ 639 if len(self.sets) == 0: 640 return "Empty" 641 642 self.compute() 643 644 # some shortcuts 645 labels = self.__labels 646 labels_map_rev = self.__labels_map_rev 647 matrix = self.__matrix 648 649 labels_rev = [] 650 if labels_map_rev is not None: 651 labels_rev = [','.join([str(x) for x in labels_map_rev[l]]) 652 for l in labels] 653 654 out = StringIO() 655 # numbers of different entries 656 Nlabels = len(labels) 657 Nsamples = self.__Nsamples.astype(int) 658 659 stats = self._stats 660 if short: 661 return "%(# of sets)d sets %(# of labels)d labels " \ 662 " ACC:%(ACC).2f" \ 663 % stats 664 665 Ndigitsmax = int(ceil(log10(max(Nsamples)))) 666 Nlabelsmax = max( [len(str(x)) for x in labels] ) 667 668 # length of a single label/value 669 L = max(Ndigitsmax+2, Nlabelsmax) #, len("100.00%")) 670 res = "" 671 672 stats_perpredict = ["P'", "N'", 'FP', 'FN', 'PPV', 'NPV', 'TPR', 673 'SPC', 'FDR', 'MCC'] 674 # print AUC only if ROC was computed 675 if self.ROC is not None: stats_perpredict += [ 'AUC' ] 676 stats_pertarget = ['P', 'N', 'TP', 'TN'] 677 stats_summary = ['ACC', 'ACC%', '# of sets'] 678 679 680 #prefixlen = Nlabelsmax + 2 + Ndigitsmax + 1 681 prefixlen = Nlabelsmax + 1 682 pref = ' '*(prefixlen) # empty prefix 683 684 if matrix.shape != (Nlabels, Nlabels): 685 raise ValueError, \ 686 "Number of labels %d doesn't correspond the size" + \ 687 " of a confusion matrix %s" % (Nlabels, matrix.shape) 688 689 # list of lists of what is printed 690 printed = [] 691 underscores = [" %s" % ("-" * L)] * Nlabels 692 if header: 693 # labels 694 printed.append(['@l----------. '] + labels_rev) 695 printed.append(['@lpredictions\\targets'] + labels) 696 # underscores 697 printed.append(['@l `------'] \ 698 + underscores + stats_perpredict) 699 700 # matrix itself 701 for i, line in enumerate(matrix): 702 l = labels[i] 703 if labels_rev != []: 704 l = '@r%10s / %s' % (labels_rev[i], l) 705 printed.append( 706 [l] + 707 [ str(x) for x in line ] + 708 [ _p2(stats[x][i]) for x in stats_perpredict]) 709 710 if summary: 711 ## Various alternative schemes ;-) 712 # printed.append([''] + underscores) 713 # printed.append(['@lPer target \ Means:'] + underscores + \ 714 # [_p2(x) for x in mean_stats]) 715 # printed.append(['Means:'] + [''] * len(labels) 716 # + [_p2(x) for x in mean_stats]) 717 printed.append(['@lPer target:'] + underscores) 718 for stat in stats_pertarget: 719 printed.append([stat] + [ 720 _p2(stats[stat][i]) for i in xrange(Nlabels)]) 721 722 # compute mean stats 723 # XXX refactor to expose them in stats as well, as 724 # mean(FCC) 725 mean_stats = N.mean(N.array([stats[k] for k in stats_perpredict]), 726 axis=1) 727 printed.append(['@lSummary \ Means:'] + underscores 728 + [_p2(stats['mean(%s)' % x]) 729 for x in stats_perpredict]) 730 731 for stat in stats_summary: 732 printed.append([stat] + [_p2(stats[stat])]) 733 734 table2string(printed, out) 735 736 if description: 737 out.write("\nStatistics computed in 1-vs-rest fashion per each " \ 738 "target.\n") 739 out.write("Abbreviations (for details see " \ 740 "http://en.wikipedia.org/wiki/ROC_curve):\n") 741 for d, val, eq in self._STATS_DESCRIPTION: 742 out.write(" %-3s: %s\n" % (d, val)) 743 if eq is not None: 744 out.write(" " + eq + "\n") 745 746 #out.write("%s" % printed) 747 result = out.getvalue() 748 out.close() 749 return result

750 751

752 - def plot(self, labels=None, numbers=False, origin='upper', 753 numbers_alpha=None, xlabels_vertical=True, numbers_kwargs={}, 754 **kwargs):

755 """Provide presentation of confusion matrix in image 756 757 :Parameters: 758 labels : list of int or basestring 759 Optionally provided labels guarantee the order of 760 presentation. Also value of None places empty column/row, 761 thus provides visual groupping of labels (Thanks Ingo) 762 numbers : bool 763 Place values inside of confusion matrix elements 764 numbers_alpha : None or float 765 Controls textual output of numbers. If None -- all numbers 766 are plotted in the same intensity. If some float -- it controls 767 alpha level -- higher value would give higher contrast. (good 768 value is 2) 769 origin : basestring 770 Which left corner diagonal should start 771 xlabels_vertical : bool 772 Either to plot xlabels vertical (benefitial if number of labels 773 is large) 774 numbers_kwargs : dict 775 Additional keyword parameters to be added to numbers (if numbers 776 is True) 777 **kwargs 778 Additional arguments given to imshow (\eg me cmap) 779 780 :Returns: 781 (fig, im, cb) -- figure, imshow, colorbar 782 """ 783 784 externals.exists("pylab", raiseException=True) 785 import pylab as P 786 787 self.compute() 788 labels_order = labels 789 790 # some shortcuts 791 labels = self.__labels 792 labels_map = self.__labels_map 793 labels_map_rev = self.__labels_map_rev 794 matrix = self.__matrix 795 796 # craft original mapping from a label into index in the matrix 797 labels_indexes = dict([(x,i) for i,x in enumerate(labels)]) 798 799 labels_rev = [] 800 if labels_map_rev is not None: 801 labels_rev = [','.join([str(x) for x in labels_map_rev[l]]) 802 for l in labels] 803 labels_map_full = dict(zip(labels_rev, labels)) 804 805 if labels_order is not None: 806 labels_order_filtered = filter(lambda x:x is not None, labels_order) 807 labels_order_filtered_set = set(labels_order_filtered) 808 # Verify if all labels provided in labels 809 if set(labels) == labels_order_filtered_set: 810 # We were provided numerical (most probably) set 811 labels_plot = labels_order 812 elif len(labels_rev) \ 813 and set(labels_rev) == labels_order_filtered_set: 814 # not clear if right whenever there were multiple labels 815 # mapped into the same 816 labels_plot = [] 817 for l in labels_order: 818 v = None 819 if l is not None: v = labels_map_full[l] 820 labels_plot += [v] 821 else: 822 raise ValueError, \ 823 "Provided labels %s do not match set of known " \ 824 "original labels (%s) or mapped labels (%s)" % \ 825 (labels_order, labels, labels_rev) 826 else: 827 labels_plot = labels 828 829 # where we have Nones? 830 isempty = N.array([l is None for l in labels_plot]) 831 non_empty = N.where(N.logical_not(isempty))[0] 832 # numbers of different entries 833 NlabelsNN = len(non_empty) 834 Nlabels = len(labels_plot) 835 836 if matrix.shape != (NlabelsNN, NlabelsNN): 837 raise ValueError, \ 838 "Number of labels %d doesn't correspond the size" + \ 839 " of a confusion matrix %s" % (NlabelsNN, matrix.shape) 840 841 confusionmatrix = N.zeros((Nlabels, Nlabels)) 842 mask = confusionmatrix.copy() 843 ticks = [] 844 tick_labels = [] 845 # populate in a silly way 846 reordered_indexes = [labels_indexes[i] for i in labels_plot 847 if i is not None] 848 for i, l in enumerate(labels_plot): 849 if l is not None: 850 j = labels_indexes[l] 851 confusionmatrix[i, non_empty] = matrix[j, reordered_indexes] 852 confusionmatrix[non_empty, i] = matrix[reordered_indexes, j] 853 ticks += [i + 0.5] 854 if labels_map_rev is not None: 855 tick_labels += ['/'.join(labels_map_rev[l])] 856 else: 857 tick_labels += [str(l)] 858 else: 859 mask[i, :] = mask[:, i] = 1 860 861 confusionmatrix = N.ma.MaskedArray(confusionmatrix, mask=mask) 862 863 # turn off automatic update if interactive 864 if P.matplotlib.get_backend() == 'TkAgg': 865 P.ioff() 866 867 fig = P.gcf() 868 ax = P.gca() 869 ax.axis('off') 870 871 # some customization depending on the origin 872 xticks_position, yticks, ybottom = { 873 'upper': ('top', [Nlabels-x for x in ticks], 0.1), 874 'lower': ('bottom', ticks, 0.2) 875 }[origin] 876 877 878 # Plot 879 axi = fig.add_axes([0.15, ybottom, 0.7, 0.7]) 880 im = axi.imshow(confusionmatrix, interpolation="nearest", origin=origin, 881 aspect='equal', extent=(0, Nlabels, 0, Nlabels), 882 **kwargs) 883 884 # plot numbers 885 if numbers: 886 numbers_kwargs_ = {'fontsize': 10, 887 'horizontalalignment': 'center', 888 'verticalalignment': 'center'} 889 maxv = float(N.max(confusionmatrix)) 890 colors = [im.to_rgba(0), im.to_rgba(maxv)] 891 for i,j in zip(*N.logical_not(mask).nonzero()): 892 v = confusionmatrix[j, i] 893 # scale alpha non-linearly 894 if numbers_alpha is None: 895 alpha = 1.0 896 else: 897 # scale according to value 898 alpha = 1 - N.array(1 - v / maxv) ** numbers_alpha 899 y = {'lower':j, 'upper':Nlabels-j-1}[origin] 900 numbers_kwargs_['color'] = colors[int(v<maxv/2)] 901 numbers_kwargs_.update(numbers_kwargs) 902 P.text(i+0.5, y+0.5, '%d' % v, alpha=alpha, **numbers_kwargs_) 903 904 maxv = N.max(confusionmatrix) 905 boundaries = N.linspace(0, maxv, N.min((maxv, 10)), True) 906 907 # Label axes 908 P.xlabel("targets") 909 P.ylabel("predictions") 910 911 P.setp(axi, xticks=ticks, yticks=yticks, 912 xticklabels=tick_labels, yticklabels=tick_labels) 913 914 axi.xaxis.set_ticks_position(xticks_position) 915 axi.xaxis.set_label_position(xticks_position) 916 917 if xlabels_vertical: 918 P.setp(P.getp(axi, 'xticklabels'), rotation='vertical') 919 920 axcb = fig.add_axes([0.8, ybottom, 0.02, 0.7]) 921 cb = P.colorbar(im, cax=axcb, format='%d', ticks = boundaries) 922 923 if P.matplotlib.get_backend() == 'TkAgg': 924 P.ion() 925 P.draw() 926 # Store it primarily for testing 927 self._plotted_confusionmatrix = confusionmatrix 928 return fig, im, cb

929 930 931 @property

932 - def error(self):

933 self.compute() 934 return 1.0-self.__Ncorrect*1.0/sum(self.__Nsamples)

935 936 937 @property

938 - def labels(self):

939 self.compute() 940 return self.__labels

941 942

943 - def getLabels_map(self):

944 return self.__labels_map

945 946

947 - def setLabels_map(self, val):

948 if val is None or isinstance(val, dict): 949 self.__labels_map = val 950 else: 951 raise ValueError, "Cannot set labels_map to %s" % val 952 # reset it just in case 953 self.__labels_map_rev = None 954 self._computed = False

955 956 957 @property

958 - def matrix(self):

959 self.compute() 960 return self.__matrix

961 962 963 @property

964 - def percentCorrect(self):

965 self.compute() 966 return 100.0*self.__Ncorrect/sum(self.__Nsamples)

967 968 labels_map = property(fget=getLabels_map, fset=setLabels_map)

969

970 971 -class RegressionStatistics(SummaryStatistics):

972 """Class to contain information and display on regression results. 973 974 """ 975 976 _STATS_DESCRIPTION = ( 977 ('CCe', 'Error based on correlation coefficient', 978 '1 - corr_coef'), 979 ('CCp', 'Correlation coefficient (p-value)', None), 980 ('RMSE', 'Root mean squared error', None), 981 ('STD', 'Standard deviation', None), 982 ('RMP', 'Root mean power (compare to RMSE of results)', 983 'sqrt(mean( data**2 ))'), 984 ) + SummaryStatistics._STATS_DESCRIPTION 985 986

987 - def __init__(self, **kwargs):

988 """Initialize RegressionStatistics 989 990 :Parameters: 991 targets 992 Optional set of targets 993 predictions 994 Optional set of predictions 995 """ 996 997 SummaryStatistics.__init__(self, **kwargs)

998 999

1000 - def _compute(self):

1001 """Actually compute the confusion matrix based on all the sets""" 1002 1003 super(RegressionStatistics, self)._compute() 1004 sets = self.sets 1005 Nsets = len(sets) 1006 1007 stats = {} 1008 1009 funcs = { 1010 'RMP_t': lambda p,t:rootMeanPowerFx(t), 1011 'STD_t': lambda p,t:N.std(t), 1012 'RMP_p': lambda p,t:rootMeanPowerFx(p), 1013 'STD_p': lambda p,t:N.std(p), 1014 'CCe': CorrErrorFx(), 1015 'CCp': CorrErrorPFx(), 1016 'RMSE': RMSErrorFx(), 1017 'RMSE/RMP_t': RelativeRMSErrorFx() 1018 } 1019 1020 for funcname, func in funcs.iteritems(): 1021 funcname_all = funcname + '_all' 1022 stats[funcname_all] = [] 1023 for i, (targets, predictions, values) in enumerate(sets): 1024 stats[funcname_all] += [func(predictions, targets)] 1025 stats[funcname_all] = N.array(stats[funcname_all]) 1026 stats[funcname] = N.mean(stats[funcname_all]) 1027 stats[funcname+'_std'] = N.std(stats[funcname_all]) 1028 stats[funcname+'_max'] = N.max(stats[funcname_all]) 1029 stats[funcname+'_min'] = N.min(stats[funcname_all]) 1030 1031 # create ``summary`` statistics, since some per-set statistics 1032 # might be uncomputable if a set contains just a single number 1033 # (like in the case of correlation coefficient) 1034 targets, predictions = [], [] 1035 for i, (targets_, predictions_, values_) in enumerate(sets): 1036 targets += list(targets_) 1037 predictions += list(predictions_) 1038 1039 for funcname, func in funcs.iteritems(): 1040 funcname_all = 'Summary ' + funcname 1041 stats[funcname_all] = func(predictions, targets) 1042 1043 self._stats.update(stats)

1044 1045

1046 - def plot(self, 1047 plot=True, plot_stats=True, 1048 splot=True 1049 #labels=None, numbers=False, origin='upper', 1050 #numbers_alpha=None, xlabels_vertical=True, 1051 #numbers_kwargs={}, 1052 #**kwargs 1053 ):

1054 """Provide presentation of regression performance in image 1055 1056 :Parameters: 1057 plot : bool 1058 Plot regular plot of values (targets/predictions) 1059 plot_stats : bool 1060 Print basic statistics in the title 1061 splot : bool 1062 Plot scatter plot 1063 1064 :Returns: 1065 (fig, im, cb) -- figure, imshow, colorbar 1066 """ 1067 externals.exists("pylab", raiseException=True) 1068 import pylab as P 1069 1070 self.compute() 1071 # total number of plots 1072 nplots = plot + splot 1073 1074 # turn off automatic update if interactive 1075 if P.matplotlib.get_backend() == 'TkAgg': 1076 P.ioff() 1077 1078 fig = P.gcf() 1079 P.clf() 1080 sps = [] # subplots 1081 1082 nplot = 0 1083 if plot: 1084 nplot += 1 1085 sps.append(P.subplot(nplots, 1, nplot)) 1086 xstart = 0 1087 lines = [] 1088 for s in self.sets: 1089 nsamples = len(s[0]) 1090 xend = xstart+nsamples 1091 xs = xrange(xstart, xend) 1092 lines += [P.plot(xs, s[0], 'b')] 1093 lines += [P.plot(xs, s[1], 'r')] 1094 # vertical line 1095 P.plot([xend, xend], [N.min(s[0]), N.max(s[0])], 'k--') 1096 xstart = xend 1097 if len(lines)>1: 1098 P.legend(lines[:2], ('Target', 'Prediction')) 1099 if plot_stats: 1100 P.title(self.asstring(short='very')) 1101 1102 if splot: 1103 nplot += 1 1104 sps.append(P.subplot(nplots, 1, nplot)) 1105 for s in self.sets: 1106 P.plot(s[0], s[1], 'o', 1107 markeredgewidth=0.2, 1108 markersize=2) 1109 P.gca().set_aspect('equal') 1110 1111 if P.matplotlib.get_backend() == 'TkAgg': 1112 P.ion() 1113 P.draw() 1114 1115 return fig, sps

1116

1117 - def asstring(self, short=False, header=True, summary=True, 1118 description=False):

1119 """'Pretty print' the statistics""" 1120 1121 if len(self.sets) == 0: 1122 return "Empty" 1123 1124 self.compute() 1125 1126 stats = self.stats 1127 1128 if short: 1129 if short == 'very': 1130 # " RMSE/RMP_t:%(RMSE/RMP_t).2f" \ 1131 return "%(# of sets)d sets CCe=%(CCe).2f p=%(CCp).2g" \ 1132 " RMSE:%(RMSE).2f" \ 1133 " Summary (stacked data): " \ 1134 "CCe=%(Summary CCe).2f p=%(Summary CCp).2g" \ 1135 % stats 1136 else: 1137 return "%(# of sets)d sets CCe=%(CCe).2f+-%(CCe_std).3f" \ 1138 " RMSE=%(RMSE).2f+-%(RMSE_std).3f" \ 1139 " RMSE/RMP_t=%(RMSE/RMP_t).2f+-%(RMSE/RMP_t_std).3f" \ 1140 % stats 1141 1142 stats_data = ['RMP_t', 'STD_t', 'RMP_p', 'STD_p'] 1143 # CCp needs tune up of format so excluded 1144 stats_ = ['CCe', 'RMSE', 'RMSE/RMP_t'] 1145 stats_summary = ['# of sets'] 1146 1147 out = StringIO() 1148 1149 printed = [] 1150 if header: 1151 # labels 1152 printed.append(['Statistics', 'Mean', 'Std', 'Min', 'Max']) 1153 # underscores 1154 printed.append(['----------', '-----', '-----', '-----', '-----']) 1155 1156 def print_stats(printed, stats_): 1157 # Statistics itself 1158 for stat in stats_: 1159 s = [stat] 1160 for suffix in ['', '_std', '_min', '_max']: 1161 s += [ _p2(stats[stat+suffix], 3) ] 1162 printed.append(s)

1163 1164 printed.append(["Data: "]) 1165 print_stats(printed, stats_data) 1166 printed.append(["Results: "]) 1167 print_stats(printed, stats_) 1168 printed.append(["Summary: "]) 1169 printed.append(["CCe", _p2(stats['Summary CCe']), "", "p=", '%g' % stats['Summary CCp']]) 1170 printed.append(["RMSE", _p2(stats['Summary RMSE'])]) 1171 printed.append(["RMSE/RMP_t", _p2(stats['Summary RMSE/RMP_t'])]) 1172 1173 if summary: 1174 for stat in stats_summary: 1175 printed.append([stat] + [_p2(stats[stat])]) 1176 1177 table2string(printed, out) 1178 1179 if description: 1180 out.write("\nDescription of printed statistics.\n" 1181 " Suffixes: _t - targets, _p - predictions\n") 1182 1183 for d, val, eq in self._STATS_DESCRIPTION: 1184 out.write(" %-3s: %s\n" % (d, val)) 1185 if eq is not None: 1186 out.write(" " + eq + "\n") 1187 1188 result = out.getvalue() 1189 out.close() 1190 return result

1191 1192 1193 @property

1194 - def error(self):

1195 self.compute() 1196 return self.stats['RMSE']

1197

1198 1199 1200 -class ClassifierError(ClassWithCollections):

1201 """Compute (or return) some error of a (trained) classifier on a dataset. 1202 """ 1203 1204 confusion = StateVariable(enabled=False) 1205 """TODO Think that labels might be also symbolic thus can't directly 1206 be indicies of the array 1207 """ 1208 1209 training_confusion = StateVariable(enabled=False, 1210 doc="Proxy training_confusion from underlying classifier.") 1211 1212

1213 - def __init__(self, clf, labels=None, train=True, **kwargs):

1214 """Initialization. 1215 1216 :Parameters: 1217 clf : Classifier 1218 Either trained or untrained classifier 1219 labels : list 1220 if provided, should be a set of labels to add on top of the 1221 ones present in testdata 1222 train : bool 1223 unless train=False, classifier gets trained if 1224 trainingdata provided to __call__ 1225 """ 1226 ClassWithCollections.__init__(self, **kwargs) 1227 self.__clf = clf 1228 1229 self._labels = labels 1230 """Labels to add on top to existing in testing data""" 1231 1232 self.__train = train 1233 """Either to train classifier if trainingdata is provided"""

1234 1235 1236 __doc__ = enhancedDocString('ClassifierError', locals(), ClassWithCollections) 1237 1238

1239 - def __copy__(self):

1240 """TODO: think... may be we need to copy self.clf""" 1241 out = ClassifierError.__new__(TransferError) 1242 ClassifierError.__init__(out, self.clf) 1243 return out

1244 1245

1246 - def _precall(self, testdataset, trainingdataset=None):

1247 """Generic part which trains the classifier if necessary 1248 """ 1249 if not trainingdataset is None: 1250 if self.__train: 1251 # XXX can be pretty annoying if triggered inside an algorithm 1252 # where it cannot be switched of, but retraining might be 1253 # intended or at least not avoidable. 1254 # Additonally isTrained docs say: 1255 # MUST BE USED WITH CARE IF EVER 1256 # 1257 # switching it off for now 1258 #if self.__clf.isTrained(trainingdataset): 1259 # warning('It seems that classifier %s was already trained' % 1260 # self.__clf + ' on dataset %s. Please inspect' \ 1261 # % trainingdataset) 1262 if self.states.isEnabled('training_confusion'): 1263 self.__clf.states._changeTemporarily( 1264 enable_states=['training_confusion']) 1265 self.__clf.train(trainingdataset) 1266 if self.states.isEnabled('training_confusion'): 1267 self.training_confusion = self.__clf.training_confusion 1268 self.__clf.states._resetEnabledTemporarily() 1269 1270 if self.__clf.states.isEnabled('trained_labels') and \ 1271 not testdataset is None: 1272 newlabels = set(testdataset.uniquelabels) \ 1273 - set(self.__clf.trained_labels) 1274 if len(newlabels)>0: 1275 warning("Classifier %s wasn't trained to classify labels %s" % 1276 (`self.__clf`, `newlabels`) + 1277 " present in testing dataset. Make sure that you have" + 1278 " not mixed order/names of the arguments anywhere")

1279 1280 ### Here checking for if it was trained... might be a cause of trouble 1281 # XXX disabled since it is unreliable.. just rely on explicit 1282 # self.__train 1283 # if not self.__clf.isTrained(trainingdataset): 1284 # self.__clf.train(trainingdataset) 1285 # elif __debug__: 1286 # debug('CERR', 1287 # 'Not training classifier %s since it was ' % `self.__clf` 1288 # + ' already trained on dataset %s' % `trainingdataset`) 1289 1290

1291 - def _call(self, testdataset, trainingdataset=None):

1292 raise NotImplementedError

1293 1294

1295 - def _postcall(self, testdataset, trainingdataset=None, error=None):

1296 pass

1297 1298

1299 - def __call__(self, testdataset, trainingdataset=None):

1300 """Compute the transfer error for a certain test dataset. 1301 1302 If `trainingdataset` is not `None` the classifier is trained using the 1303 provided dataset before computing the transfer error. Otherwise the 1304 classifier is used in it's current state to make the predictions on 1305 the test dataset. 1306 1307 Returns a scalar value of the transfer error. 1308 """ 1309 self._precall(testdataset, trainingdataset) 1310 error = self._call(testdataset, trainingdataset) 1311 self._postcall(testdataset, trainingdataset, error) 1312 if __debug__: 1313 debug('CERR', 'Classifier error on %s: %.2f' 1314 % (testdataset, error)) 1315 return error

1316 1317

1318 - def untrain(self):

1319 """Untrain the *Error which relies on the classifier 1320 """ 1321 self.clf.untrain()

1322 1323 1324 @property

1325 - def clf(self):

1326 return self.__clf

1327 1328 1329 @property

1330 - def labels(self):

1331 return self._labels

1332

1333 1334 1335 -class TransferError(ClassifierError):

1336 """Compute the transfer error of a (trained) classifier on a dataset. 1337 1338 The actual error value is computed using a customizable error function. 1339 Optionally the classifier can be trained by passing an additional 1340 training dataset to the __call__() method. 1341 """ 1342 1343 null_prob = StateVariable(enabled=True, 1344 doc="Stores the probability of an error result under " 1345 "the NULL hypothesis") 1346 samples_error = StateVariable(enabled=False, 1347 doc="Per sample errors computed by invoking the " 1348 "error function for each sample individually. " 1349 "Errors are available in a dictionary with each " 1350 "samples origid as key.") 1351

1352 - def __init__(self, clf, errorfx=MeanMismatchErrorFx(), labels=None, 1353 null_dist=None, **kwargs):

1354 """Initialization. 1355 1356 :Parameters: 1357 clf : Classifier 1358 Either trained or untrained classifier 1359 errorfx 1360 Functor that computes a scalar error value from the vectors of 1361 desired and predicted values (e.g. subclass of `ErrorFunction`) 1362 labels : list 1363 if provided, should be a set of labels to add on top of the 1364 ones present in testdata 1365 null_dist : instance of distribution estimator 1366 """ 1367 ClassifierError.__init__(self, clf, labels, **kwargs) 1368 self.__errorfx = errorfx 1369 self.__null_dist = autoNullDist(null_dist)

1370 1371 1372 __doc__ = enhancedDocString('TransferError', locals(), ClassifierError) 1373 1374

1375 - def __copy__(self):

1376 """Performs deepcopying of the classifier.""" 1377 # TODO -- use ClassifierError.__copy__ 1378 out = TransferError.__new__(TransferError) 1379 TransferError.__init__(out, self.clf.clone(), self.errorfx, self._labels) 1380 1381 return out

1382 1383 # XXX: TODO: unify naming? test/train or with ing both

1384 - def _call(self, testdataset, trainingdataset=None):

1385 """Compute the transfer error for a certain test dataset. 1386 1387 If `trainingdataset` is not `None` the classifier is trained using the 1388 provided dataset before computing the transfer error. Otherwise the 1389 classifier is used in it's current state to make the predictions on 1390 the test dataset. 1391 1392 Returns a scalar value of the transfer error. 1393 """ 1394 # OPT: local binding 1395 clf = self.clf 1396 if testdataset is None: 1397 # We cannot do anythin, but we can try to figure out WTF and 1398 # warn the user accordingly in some usecases 1399 import traceback as tb 1400 filenames = [x[0] for x in tb.extract_stack(limit=100)] 1401 rfe_matches = [f for f in filenames if f.endswith('/rfe.py')] 1402 cv_matches = [f for f in filenames if 1403 f.endswith('cvtranserror.py')] 1404 msg = "" 1405 if len(rfe_matches) > 0 and len(cv_matches): 1406 msg = " It is possible that you used RFE with stopping " \ 1407 "criterion based on the TransferError and directly" \ 1408 " from CrossValidatedTransferError, such approach" \ 1409 " would require exposing testing dataset " \ 1410 " to the classifier which might heavily bias " \ 1411 " generalization performance estimate. If you are " \ 1412 " sure to use it that way, create CVTE with " \ 1413 " parameter expose_testdataset=True" 1414 raise ValueError, "Transfer error call obtained None " \ 1415 "as a dataset for testing.%s" % msg 1416 predictions = clf.predict(testdataset.samples) 1417 1418 # compute confusion matrix 1419 # Should it migrate into ClassifierError.__postcall? 1420 # -> Probably not because other childs could estimate it 1421 # not from test/train datasets explicitely, see 1422 # `ConfusionBasedError`, where confusion is simply 1423 # bound to classifiers confusion matrix 1424 states = self.states 1425 if states.isEnabled('confusion'): 1426 confusion = clf._summaryClass( 1427 #labels=self.labels, 1428 targets=testdataset.labels, 1429 predictions=predictions, 1430 values=clf.states.get('values', None)) 1431 try: 1432 confusion.labels_map = testdataset.labels_map 1433 except: 1434 pass 1435 states.confusion = confusion 1436 1437 if states.isEnabled('samples_error'): 1438 samples_error = [] 1439 for i, p in enumerate(predictions): 1440 samples_error.append(self.__errorfx([p], testdataset.labels[i:i+1])) 1441 1442 states.samples_error = dict(zip(testdataset.origids, samples_error)) 1443 1444 # compute error from desired and predicted values 1445 error = self.__errorfx(predictions, testdataset.labels) 1446 1447 return error

1448 1449

1450 - def _postcall(self, vdata, wdata=None, error=None):

1451 """ 1452 """ 1453 # estimate the NULL distribution when functor and training data is 1454 # given 1455 if not self.__null_dist is None and not wdata is None: 1456 # we need a matching transfer error instances (e.g. same error 1457 # function), but we have to disable the estimation of the null 1458 # distribution in that child to prevent infinite looping. 1459 null_terr = copy.copy(self) 1460 null_terr.__null_dist = None 1461 self.__null_dist.fit(null_terr, wdata, vdata) 1462 1463 1464 # get probability of error under NULL hypothesis if available 1465 if not error is None and not self.__null_dist is None: 1466 self.null_prob = self.__null_dist.p(error)

1467 1468 1469 @property

1470 - def errorfx(self): return self.__errorfx

1471 1472 @property

1473 - def null_dist(self): return self.__null_dist

1474

1475 1476 1477 -class ConfusionBasedError(ClassifierError):

1478 """For a given classifier report an error based on internally 1479 computed error measure (given by some `ConfusionMatrix` stored in 1480 some state variable of `Classifier`). 1481 1482 This way we can perform feature selection taking as the error 1483 criterion either learning error, or transfer to splits error in 1484 the case of SplitClassifier 1485 """ 1486

1487 - def __init__(self, clf, labels=None, confusion_state="training_confusion", 1488 **kwargs):

1489 """Initialization. 1490 1491 :Parameters: 1492 clf : Classifier 1493 Either trained or untrained classifier 1494 confusion_state 1495 Id of the state variable which stores `ConfusionMatrix` 1496 labels : list 1497 if provided, should be a set of labels to add on top of the 1498 ones present in testdata 1499 """ 1500 ClassifierError.__init__(self, clf, labels, **kwargs) 1501 1502 self.__confusion_state = confusion_state 1503 """What state to extract from""" 1504 1505 if not clf.states.isKnown(confusion_state): 1506 raise ValueError, \ 1507 "State variable %s is not defined for classifier %s" % \ 1508 (confusion_state, `clf`) 1509 if not clf.states.isEnabled(confusion_state): 1510 if __debug__: 1511 debug('CERR', "Forcing state %s to be enabled for %s" % 1512 (confusion_state, `clf`)) 1513 clf.states.enable(confusion_state)

1514 1515 1516 __doc__ = enhancedDocString('ConfusionBasedError', locals(), 1517 ClassifierError) 1518 1519

1520 - def _call(self, testdata, trainingdata=None):

1521 """Extract transfer error. Nor testdata, neither trainingdata is used 1522 """ 1523 confusion = self.clf.states[self.__confusion_state].value 1524 self.confusion = confusion 1525 return confusion.error

1526

Source Code for Module mvpa.clfs.transerror