Source Code for Module mvpa.tests.test_svm

  1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  # vi: set ft=python sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Unit tests for SVM classifier""" 
 10   
 11  from mvpa.datasets.splitters import NFoldSplitter 
 12  from mvpa.clfs.meta import ProxyClassifier 
 13  from mvpa.clfs.transerror import TransferError 
 14  from mvpa.algorithms.cvtranserror import CrossValidatedTransferError 
 15   
 16  from tests_warehouse import pureMultivariateSignal 
 17  from tests_warehouse import * 
 18  from tests_warehouse_clfs import * 
 19   
 20 -class SVMTests(unittest.TestCase): 
 21   
 22  #    @sweepargs(nl_clf=clfswh['non-linear', 'svm'] ) 
 23  #    @sweepargs(nl_clf=clfswh['non-linear', 'svm'] ) 
 24 -    def testMultivariate(self): 
 25          mv_perf = [] 
 26          mv_lin_perf = [] 
 27          uv_perf = [] 
 28   
 29          l_clf = clfswh['linear', 'svm'][0] 
 30          nl_clf = clfswh['non-linear', 'svm'][0] 
 31   
 32          #orig_keys = nl_clf.param._params.keys() 
 33          #nl_param_orig = nl_clf.param._params.copy() 
 34   
 35          # l_clf = LinearNuSVMC() 
 36   
 37          # XXX ??? not sure what below meant and it is obsolete if 
 38          # using SG... commenting out for now 
 39          # for some reason order is not preserved thus dictionaries are not 
 40          # the same any longer -- lets compare values 
 41          #self.failUnlessEqual([nl_clf.param._params[k] for k in orig_keys], 
 42          #                     [nl_param_orig[k] for k in orig_keys], 
 43          #   msg="New instance mustn't override values in previously created") 
 44          ## and keys separately 
 45          #self.failUnlessEqual(set(nl_clf.param._params.keys()), 
 46          #                     set(orig_keys), 
 47          #   msg="New instance doesn't change set of parameters in original") 
 48   
 49          # We must be able to deepcopy not yet trained SVMs now 
 50          import mvpa.support.copy as copy 
 51          try: 
 52              nl_clf.untrain() 
 53              nl_clf_copy = copy.deepcopy(nl_clf) 
 54          except: 
 55              self.fail(msg="Failed to deepcopy not-yet trained SVM %s" % nl_clf) 
 56   
 57          for i in xrange(20): 
 58              train = pureMultivariateSignal( 20, 3 ) 
 59              test = pureMultivariateSignal( 20, 3 ) 
 60   
 61              # use non-linear CLF on 2d data 
 62              nl_clf.train(train) 
 63              p_mv = nl_clf.predict(test.samples) 
 64              mv_perf.append(N.mean(p_mv==test.labels)) 
 65   
 66              # use linear CLF on 2d data 
 67              l_clf.train(train) 
 68              p_lin_mv = l_clf.predict(test.samples) 
 69              mv_lin_perf.append(N.mean(p_lin_mv==test.labels)) 
 70   
 71              # use non-linear CLF on 1d data 
 72              nl_clf.train(train.selectFeatures([0])) 
 73              p_uv = nl_clf.predict(test.selectFeatures([0]).samples) 
 74              uv_perf.append(N.mean(p_uv==test.labels)) 
 75   
 76          mean_mv_perf = N.mean(mv_perf) 
 77          mean_mv_lin_perf = N.mean(mv_lin_perf) 
 78          mean_uv_perf = N.mean(uv_perf) 
 79   
 80          # non-linear CLF has to be close to perfect 
 81          self.failUnless( mean_mv_perf > 0.9 ) 
 82          # linear CLF cannot learn this problem! 
 83          self.failUnless( mean_mv_perf > mean_mv_lin_perf ) 
 84          # univariate has insufficient information 
 85          self.failUnless( mean_uv_perf < mean_mv_perf ) 
 86   
 87   
 88      # XXX for now works only with linear... think it through -- should 
 89      # work non-linear, shouldn't it? 
 90      # now all non-linear have C>0 thus skipped anyways 
 91   
 92      # TODO: For some reason libsvm's weight assignment has no effect 
 93      # as well -- need to be fixed :-/ 
 94      @sweepargs(clf=clfswh['svm', 'sg', '!regression', '!gnpp', '!meta']) 
 95 -    def testCperClass(self, clf): 
 96          try: 
 97              if clf.C > 0: 
 98                  # skip those with fixed C 
 99                  return 
100          except: 
101              # classifier has no C 
102              return 
103   
104          if clf.C < -5: 
105              # too soft margin helps to fight disbalance, thus skip 
106              # it in testing 
107              return 
108          #print clf 
109          ds = datasets['uni2small'].copy() 
110          ds__ = datasets['uni2small'].copy() 
111          # 
112          # ballanced set 
113          # Lets add a bit of noise to drive classifier nuts. same 
114          # should be done for disballanced set 
115          ds__.samples = ds__.samples + 0.5 * N.random.normal(size=(ds__.samples.shape)) 
116          # 
117          # disballanced set 
118          # lets overpopulate label 0 
119          times = 10 
120          ds_ = ds.selectSamples(range(ds.nsamples) + range(ds.nsamples/2) * times) 
121          ds_.samples = ds_.samples + 0.7 * N.random.normal(size=(ds_.samples.shape)) 
122          spl = ds_.samplesperlabel 
123          #print ds_.labels, ds_.chunks 
124   
125          cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(), 
126                                            enable_states='confusion') 
127          e = cve(ds__) 
128          if cfg.getboolean('tests', 'labile', default='yes'): 
129              # without disballance we should already have some hits 
130              self.failUnless(cve.confusion.stats["P'"][1] > 0) 
131   
132          e = cve(ds_) 
133          if cfg.getboolean('tests', 'labile', default='yes'): 
134              self.failUnless(cve.confusion.stats["P'"][1] < 5, 
135                              msg="With disballance we should have almost no " 
136                              "hits. Got %f" % cve.confusion.stats["P'"][1]) 
137              #print "D:", cve.confusion.stats["P'"][1], cve.confusion.stats['MCC'][1] 
138   
139          # Set '1 C per label' 
140          # recreate cvte since previous might have operated on copies 
141          cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(), 
142                                            enable_states='confusion') 
143          oldC = clf.C 
144          ratio = N.sqrt(float(spl[0])/spl[1]) 
145          clf.C = (-1/ratio, -1*ratio) 
146          try: 
147              e_ = cve(ds_) 
148              # reassign C 
149              clf.C = oldC 
150          except: 
151              clf.C = oldC 
152              raise 
153          #print "B:", cve.confusion.stats["P'"][1], cve.confusion.stats['MCC'][1] 
154          if cfg.getboolean('tests', 'labile', default='yes'): 
155              # Finally test if we get any 'hit' for minor category. In the 
156              # classifier, which has way to 'ballance' should be non-0 
157              self.failUnless(cve.confusion.stats["P'"][1] > 0) 
158   
159   
160 -    def testSillyness(self): 
161          """Test if we raise exceptions on incorrect specifications 
162          """ 
163   
164          if externals.exists('libsvm') or externals.exists('shogun'): 
165              self.failUnlessRaises(TypeError, SVM,  C=1.0, nu=2.3) 
166   
167          if externals.exists('libsvm'): 
168              self.failUnlessRaises(TypeError, libsvm.SVM,  C=1.0, nu=2.3) 
169              self.failUnlessRaises(TypeError, LinearNuSVMC, C=2.3) 
170              self.failUnlessRaises(TypeError, LinearCSVMC, nu=2.3) 
171   
172          if externals.exists('shogun'): 
173              self.failUnlessRaises(TypeError, sg.SVM, C=10, kernel_type='RBF', 
174                                    coef0=3) 
175   
176 -def suite(): 
177      return unittest.makeSuite(SVMTests) 
178   
179   
180  if __name__ == '__main__': 
181      import runner 
182