mvpa.clfs.sg.svm

124 """Support Vector Machine Classifier(s) based on Shogun 125 126 This is a simple base interface 127 """ 128 129 num_threads = Parameter(1, 130 min=1, 131 doc='Number of threads to utilize') 132 133 # NOTE: gamma is width in SG notation for RBF(Gaussian) 134 _KERNELS = {} 135 if externals.exists('shogun', raiseException=True): 136 _KERNELS = { "linear": (shogun.Kernel.LinearKernel, 137 ('scale',), LinearSVMWeights), 138 "rbf" : (shogun.Kernel.GaussianKernel, 139 ('gamma',), None), 140 "rbfshift": (shogun.Kernel.GaussianShiftKernel, 141 ('gamma', 'max_shift', 'shift_step'), None), 142 "sigmoid": (shogun.Kernel.SigmoidKernel, 143 ('cache_size', 'gamma', 'coef0'), None), 144 } 145 146 _KNOWN_PARAMS = [ 'epsilon' ] 147 _KNOWN_KERNEL_PARAMS = [ ] 148 149 _clf_internals = _SVM._clf_internals + [ 'sg', 'retrainable' ] 150 151 if externals.exists('sg ge 0.6.4'): 152 _KERNELS['linear'] = (shogun.Kernel.LinearKernel, (), LinearSVMWeights) 153 154 # Some words of wisdom from shogun author: 155 # XXX remove after proper comments added to implementations 156 """ 157 If you'd like to train linear SVMs use SGD or OCAS. These are (I am 158 serious) the fastest linear SVM-solvers to date. (OCAS cannot do SVMs 159 with standard additive bias, but will L2 reqularize it - though it 160 should not matter much in practice (although it will give slightly 161 different solutions)). Note that SGD has no stopping criterion (you 162 simply have to specify the number of iterations) and that OCAS has a 163 different stopping condition than svmlight for example which may be more 164 tight and more loose depending on the problem - I sugeest 1e-2 or 1e-3 165 for epsilon. 166 167 If you would like to train kernel SVMs use libsvm/gpdt/svmlight - 168 depending on the problem one is faster than the other (hard to say when, 169 I *think* when your dataset is very unbalanced chunking methods like 170 svmlight/gpdt are better), for smaller problems definitely libsvm. 171 172 If you use string kernels then gpdt/svmlight have a special 'linadd' 173 speedup for this (requires sg 0.6.2 - there was some inefficiency in the 174 code for python-modular before that). This is effective for big datasets 175 and (I trained on 10 million strings based on this). 176 177 And yes currently we only implemented parallel training for svmlight, 178 however all SVMs can be evaluated in parallel. 179 """ 180 _KNOWN_IMPLEMENTATIONS = {} 181 if externals.exists('shogun', raiseException=True): 182 _KNOWN_IMPLEMENTATIONS = { 183 "libsvm" : (shogun.Classifier.LibSVM, ('C',), 184 ('multiclass', 'binary'), 185 "LIBSVM's C-SVM (L2 soft-margin SVM)"), 186 "gmnp" : (shogun.Classifier.GMNPSVM, ('C',), 187 ('multiclass', 'binary'), 188 "Generalized Nearest Point Problem SVM"), 189 # XXX should have been GPDT, shogun has it fixed since some version 190 "gpbt" : (shogun.Classifier.GPBTSVM, ('C',), ('binary',), 191 "Gradient Projection Decomposition Technique for " \ 192 "large-scale SVM problems"), 193 "gnpp" : (shogun.Classifier.GNPPSVM, ('C',), ('binary',), 194 "Generalized Nearest Point Problem SVM"), 195 196 ## TODO: Needs sparse features... 197 # "svmlin" : (shogun.Classifier.SVMLin, ''), 198 # "liblinear" : (shogun.Classifier.LibLinear, ''), 199 # "subgradient" : (shogun.Classifier.SubGradientSVM, ''), 200 ## good 2-class linear SVMs 201 # "ocas" : (shogun.Classifier.SVMOcas, ''), 202 # "sgd" : ( shogun.Classifier.SVMSGD, ''), 203 204 # regressions 205 "libsvr": (shogun.Regression.LibSVR, ('C', 'tube_epsilon',), 206 ('regression',), 207 "LIBSVM's epsilon-SVR"), 208 } 209 210

211 - def __init__(self, 212 kernel_type='linear', 213 **kwargs):

214 """Interface class to Shogun's classifiers and regressions. 215 216 Default implementation is 'libsvm'. 217 """ 218 219 svm_impl = kwargs.get('svm_impl', 'libsvm').lower() 220 kwargs['svm_impl'] = svm_impl 221 222 # init base class 223 _SVM.__init__(self, kernel_type=kernel_type, **kwargs) 224 225 self.__svm = None 226 """Holds the trained svm.""" 227 self.__svm_apply = None 228 """Compatibility convenience to bind to the classify/apply method 229 of __svm""" 230 # Need to store original data... 231 # TODO: keep 1 of them -- just __traindata or __traindataset 232 # For now it is needed for computing sensitivities 233 self.__traindataset = None 234 235 # internal SG swig proxies 236 self.__traindata = None 237 self.__kernel = None 238 self.__kernel_test = None 239 self.__testdata = None

240 241

242 - def __condition_kernel(self, kernel):

243 # XXX I thought that it is needed only for retrainable classifier, 244 # but then krr gets confused, and svrlight needs it to provide 245 # meaningful results even without 'retraining' 246 if self._svm_impl in ['svrlight', 'lightsvm']: 247 try: 248 kernel.set_precompute_matrix(True, True) 249 except Exception, e: 250 # N/A in shogun 0.9.1... TODO: RF 251 if __debug__: 252 debug('SG_', "Failed call to set_precompute_matrix for %s: %s" 253 % (self, e))

254 255

256 - def _train(self, dataset):

257 """Train SVM 258 """ 259 # XXX watchout 260 # self.untrain() 261 newkernel, newsvm = False, False 262 # local bindings for faster lookup 263 retrainable = self.params.retrainable 264 265 if retrainable: 266 _changedData = self._changedData 267 268 # LABELS 269 ul = None 270 self.__traindataset = dataset 271 272 273 # OK -- we have to map labels since 274 # binary ones expect -1/+1 275 # Multiclass expect labels starting with 0, otherwise they puke 276 # when ran from ipython... yikes 277 if __debug__: 278 debug("SG_", "Creating labels instance") 279 280 if 'regression' in self._clf_internals: 281 labels_ = N.asarray(dataset.labels, dtype='double') 282 else: 283 ul = dataset.uniquelabels 284 ul.sort() 285 286 if len(ul) == 2: 287 # assure that we have -1/+1 288 _labels_dict = {ul[0]:-1.0, ul[1]:+1.0} 289 elif len(ul) < 2: 290 raise FailedToTrainError, \ 291 "We do not have 1-class SVM brought into SG yet" 292 else: 293 # can't use plain enumerate since we need them swapped 294 _labels_dict = dict([ (ul[i], i) for i in range(len(ul))]) 295 296 # reverse labels dict for back mapping in _predict 297 _labels_dict_rev = dict([(x[1], x[0]) 298 for x in _labels_dict.items()]) 299 300 # bind to instance as well 301 self._labels_dict = _labels_dict 302 self._labels_dict_rev = _labels_dict_rev 303 304 # Map labels 305 # 306 # TODO: top level classifier should take care about labels 307 # mapping if that is needed 308 if __debug__: 309 debug("SG__", "Mapping labels using dict %s" % _labels_dict) 310 labels_ = N.asarray([ _labels_dict[x] for x in dataset.labels ], dtype='double') 311 312 labels = shogun.Features.Labels(labels_) 313 _setdebug(labels, 'Labels') 314 315 316 # KERNEL 317 if not retrainable or _changedData['traindata'] or _changedData['kernel_params']: 318 # If needed compute or just collect arguments for SVM and for 319 # the kernel 320 kargs = [] 321 for arg in self._KERNELS[self._kernel_type_literal][1]: 322 value = self.kernel_params[arg].value 323 # XXX Unify damn automagic gamma value 324 if arg == 'gamma' and value == 0.0: 325 value = self._getDefaultGamma(dataset) 326 kargs += [value] 327 328 if retrainable and __debug__: 329 if _changedData['traindata']: 330 debug("SG", 331 "Re-Creating kernel since training data has changed") 332 333 if _changedData['kernel_params']: 334 debug("SG", 335 "Re-Creating kernel since params %s has changed" % 336 _changedData['kernel_params']) 337 338 # create training data 339 if __debug__: debug("SG_", "Converting input data for shogun") 340 self.__traindata = _tosg(dataset.samples) 341 342 if __debug__: 343 debug("SG", "Creating kernel instance of %s giving arguments %s" % 344 (`self._kernel_type`, kargs)) 345 346 self.__kernel = kernel = \ 347 self._kernel_type(self.__traindata, self.__traindata, 348 *kargs) 349 350 if externals.exists('sg ge 0.6.4'): 351 kernel.set_normalizer(shogun.Kernel.IdentityKernelNormalizer()) 352 353 newkernel = True 354 self.kernel_params.reset() # mark them as not-changed 355 _setdebug(kernel, 'Kernels') 356 357 self.__condition_kernel(kernel) 358 if retrainable: 359 if __debug__: 360 debug("SG_", "Resetting test kernel for retrainable SVM") 361 self.__kernel_test = None 362 self.__kernel_args = kargs 363 364 # TODO -- handle _changedData['params'] correctly, ie without recreating 365 # whole SVM 366 Cs = None 367 if not retrainable or self.__svm is None or _changedData['params']: 368 # SVM 369 if self.params.isKnown('C'): 370 C = self.params.C 371 if not operator.isSequenceType(C): 372 # we were not given a tuple for balancing between classes 373 C = [C] 374 375 Cs = list(C[:]) # copy 376 for i in xrange(len(Cs)): 377 if Cs[i]<0: 378 Cs[i] = self._getDefaultC(dataset.samples)*abs(Cs[i]) 379 if __debug__: 380 debug("SG_", "Default C for %s was computed to be %s" % 381 (C[i], Cs[i])) 382 383 # XXX do not jump over the head and leave it up to the user 384 # ie do not rescale automagically by the number of samples 385 #if len(Cs) == 2 and not ('regression' in self._clf_internals) and len(ul) == 2: 386 # # we were given two Cs 387 # if N.max(C) < 0 and N.min(C) < 0: 388 # # and both are requested to be 'scaled' TODO : 389 # # provide proper 'features' to the parameters, 390 # # so we could specify explicitely if to scale 391 # # them by the number of samples here 392 # nl = [N.sum(labels_ == _labels_dict[l]) for l in ul] 393 # ratio = N.sqrt(float(nl[1]) / nl[0]) 394 # #ratio = (float(nl[1]) / nl[0]) 395 # Cs[0] *= ratio 396 # Cs[1] /= ratio 397 # if __debug__: 398 # debug("SG_", "Rescaled Cs to %s to accomodate the " 399 # "difference in number of training samples" % 400 # Cs) 401 402 # Choose appropriate implementation 403 svm_impl_class = self.__get_implementation(ul) 404 405 if __debug__: 406 debug("SG", "Creating SVM instance of %s" % `svm_impl_class`) 407 408 if self._svm_impl in ['libsvr', 'svrlight']: 409 # for regressions constructor a bit different 410 self.__svm = svm_impl_class(Cs[0], self.params.epsilon, self.__kernel, labels) 411 elif self._svm_impl in ['krr']: 412 self.__svm = svm_impl_class(self.params.tau, self.__kernel, labels) 413 else: 414 self.__svm = svm_impl_class(Cs[0], self.__kernel, labels) 415 self.__svm.set_epsilon(self.params.epsilon) 416 417 # To stay compatible with versions across API changes in sg 1.0.0 418 self.__svm_apply = hasattr(self.__svm, 'apply') \ 419 and self.__svm.apply \ 420 or self.__svm.classify # the last one for old API 421 422 # Set shrinking 423 if self.params.isKnown('shrinking'): 424 shrinking = self.params.shrinking 425 if __debug__: 426 debug("SG_", "Setting shrinking to %s" % shrinking) 427 self.__svm.set_shrinking_enabled(shrinking) 428 429 if Cs is not None and len(Cs) == 2: 430 if __debug__: 431 debug("SG_", "Since multiple Cs are provided: %s, assign them" % Cs) 432 self.__svm.set_C(Cs[0], Cs[1]) 433 434 self.params.reset() # mark them as not-changed 435 newsvm = True 436 _setdebug(self.__svm, 'SVM') 437 # Set optimization parameters 438 if self.params.isKnown('tube_epsilon') and \ 439 hasattr(self.__svm, 'set_tube_epsilon'): 440 self.__svm.set_tube_epsilon(self.params.tube_epsilon) 441 self.__svm.parallel.set_num_threads(self.params.num_threads) 442 else: 443 if __debug__: 444 debug("SG_", "SVM instance is not re-created") 445 if _changedData['labels']: # labels were changed 446 if __debug__: debug("SG__", "Assigning new labels") 447 self.__svm.set_labels(labels) 448 if newkernel: # kernel was replaced 449 if __debug__: debug("SG__", "Assigning new kernel") 450 self.__svm.set_kernel(self.__kernel) 451 assert(_changedData['params'] is False) # we should never get here 452 453 if retrainable: 454 # we must assign it only if it is retrainable 455 self.states.retrained = not newsvm or not newkernel 456 457 # Train 458 if __debug__ and 'SG' in debug.active: 459 if not self.regression: 460 lstr = " with labels %s" % dataset.uniquelabels 461 else: 462 lstr = "" 463 debug("SG", "%sTraining %s on data%s" % 464 (("","Re-")[retrainable and self.states.retrained], 465 self, lstr)) 466 467 self.__svm.train() 468 469 if __debug__: 470 debug("SG_", "Done training SG_SVM %s" % self._kernel_type) 471 472 # Report on training 473 if (__debug__ and 'SG__' in debug.active) or \ 474 self.states.isEnabled('training_confusion'): 475 trained_labels = self.__svm_apply().get_labels() 476 else: 477 trained_labels = None 478 479 if __debug__ and "SG__" in debug.active: 480 debug("SG__", "Original labels: %s, Trained labels: %s" % 481 (dataset.labels, trained_labels)) 482 483 # Assign training confusion right away here since we are ready 484 # to do so. 485 # XXX TODO use some other state variable like 'trained_labels' and 486 # use it within base Classifier._posttrain to assign predictions 487 # instead of duplicating code here 488 # XXX For now it can be done only for regressions since labels need to 489 # be remapped and that becomes even worse if we use regression 490 # as a classifier so mapping happens upstairs 491 if self.regression and self.states.isEnabled('training_confusion'): 492 self.states.training_confusion = self._summaryClass( 493 targets=dataset.labels, 494 predictions=trained_labels)

495

496 - def _predict(self, data):

497 """Predict values for the data 498 """ 499 500 retrainable = self.params.retrainable 501 502 if retrainable: 503 changed_testdata = self._changedData['testdata'] or \ 504 self.__kernel_test is None 505 506 if not retrainable or changed_testdata: 507 testdata = _tosg(data) 508 509 if not retrainable: 510 if __debug__: 511 debug("SG__", 512 "Initializing SVMs kernel of %s with training/testing samples" 513 % self) 514 # We can just reuse kernel used for training 515 self.__kernel.init(self.__traindata, testdata) 516 self.__condition_kernel(self.__kernel) 517 else: 518 if changed_testdata: 519 if __debug__: 520 debug("SG__", 521 "Re-creating testing kernel of %s giving " 522 "arguments %s" % 523 (`self._kernel_type`, self.__kernel_args)) 524 kernel_test = self._kernel_type(self.__traindata, testdata, 525 *self.__kernel_args) 526 _setdebug(kernel_test, 'Kernels') 527 528 custk_args = ([self.__traindata, testdata], [])[ 529 int(externals.exists('sg ge 0.6.4'))] 530 if __debug__: 531 debug("SG__", 532 "Re-creating custom testing kernel giving " 533 "arguments %s" % (str(custk_args))) 534 kernel_test_custom = shogun.Kernel.CustomKernel(*custk_args) 535 536 _setdebug(kernel_test_custom, 'Kernels') 537 self.__kernel_test = kernel_test_custom 538 self.__kernel_test.set_full_kernel_matrix_from_full( 539 kernel_test.get_kernel_matrix()) 540 elif __debug__: 541 debug("SG__", "Re-using testing kernel") 542 543 assert(self.__kernel_test is not None) 544 self.__svm.set_kernel(self.__kernel_test) 545 546 if __debug__: 547 debug("SG_", "Classifying testing data") 548 549 # doesn't do any good imho although on unittests helps tiny bit... hm 550 #self.__svm.init_kernel_optimization() 551 values_ = self.__svm_apply() 552 if values_ is None: 553 raise RuntimeError, "We got empty list of values from %s" % self 554 555 values = values_.get_labels() 556 557 if retrainable: 558 # we must assign it only if it is retrainable 559 self.states.repredicted = repredicted = not changed_testdata 560 if __debug__: 561 debug("SG__", "Re-assigning learing kernel. Repredicted is %s" 562 % repredicted) 563 # return back original kernel 564 self.__svm.set_kernel(self.__kernel) 565 566 if __debug__: 567 debug("SG__", "Got values %s" % values) 568 569 if ('regression' in self._clf_internals): 570 predictions = values 571 else: 572 # local bindings 573 _labels_dict = self._labels_dict 574 _labels_dict_rev = self._labels_dict_rev 575 576 if len(_labels_dict) == 2: 577 predictions = 1.0 - 2*N.signbit(values) 578 else: 579 predictions = values 580 581 # assure that we have the same type 582 label_type = type(_labels_dict.values()[0]) 583 584 # remap labels back adjusting their type 585 predictions = [_labels_dict_rev[label_type(x)] 586 for x in predictions] 587 588 if __debug__: 589 debug("SG__", "Tuned predictions %s" % predictions) 590 591 # store state variable 592 # TODO: extract values properly for multiclass SVMs -- 593 # ie 1 value per label or pairs for all 1-vs-1 classifications 594 self.values = values 595 596 ## to avoid leaks with not yet properly fixed shogun 597 if not retrainable: 598 try: 599 testdata.free_features() 600 except: 601 pass 602 603 return predictions

604 605

606 - def untrain(self):

607 super(SVM, self).untrain() 608 if not self.params.retrainable: 609 if __debug__: 610 debug("SG__", "Untraining %(clf)s and destroying sg's SVM", 611 msgargs={'clf':self}) 612 613 # to avoid leaks with not yet properly fixed shogun 614 # XXX make it nice... now it is just stable ;-) 615 if True: # not self.__traindata is None: 616 if True: 617 # try: 618 if self.__kernel is not None: 619 del self.__kernel 620 self.__kernel = None 621 622 if self.__kernel_test is not None: 623 del self.__kernel_test 624 self.__kernel_test = None 625 626 if self.__svm is not None: 627 del self.__svm 628 self.__svm = None 629 self.__svm_apply = None 630 631 if self.__traindata is not None: 632 # Let in for easy demonstration of the memory leak in shogun 633 #for i in xrange(10): 634 # debug("SG__", "cachesize pre free features %s" % 635 # (self.__svm.get_kernel().get_cache_size())) 636 self.__traindata.free_features() 637 del self.__traindata 638 self.__traindata = None 639 640 self.__traindataset = None 641 642 643 #except: 644 # pass 645 646 if __debug__: 647 debug("SG__", 648 "Done untraining %(self)s and destroying sg's SVM", 649 msgargs=locals()) 650 elif __debug__: 651 debug("SG__", "Not untraining %(self)s since it is retrainable", 652 msgargs=locals())

653 654

655 - def __get_implementation(self, ul):

656 if 'regression' in self._clf_internals or len(ul) == 2: 657 svm_impl_class = SVM._KNOWN_IMPLEMENTATIONS[self._svm_impl][0] 658 else: 659 if self._svm_impl == 'libsvm': 660 svm_impl_class = shogun.Classifier.LibSVMMultiClass 661 elif self._svm_impl == 'gmnp': 662 svm_impl_class = shogun.Classifier.GMNPSVM 663 else: 664 raise RuntimeError, \ 665 "Shogun: Implementation %s doesn't handle multiclass " \ 666 "data. Got labels %s. Use some other classifier" % \ 667 (self._svm_impl, self.__traindataset.uniquelabels) 668 if __debug__: 669 debug("SG_", "Using %s for multiclass data of %s" % 670 (svm_impl_class, self._svm_impl)) 671 672 return svm_impl_class

673 674 675 svm = property(fget=lambda self: self.__svm) 676 """Access to the SVM model.""" 677 678 traindataset = property(fget=lambda self: self.__traindataset) 679 """Dataset which was used for training 680 681 TODO -- might better become state variable I guess"""

Source Code for Module mvpa.clfs.sg.svm