mvpa.base.externals

1 # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 2 # vi: set ft=python sts=4 ts=4 sw=4 et: 3 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 4 # 5 # See COPYING file distributed along with the PyMVPA package for the 6 # copyright and license terms. 7 # 8 ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 9 """Helper to verify presence of external libraries and modules 10 """ 11 12 __docformat__ = 'restructuredtext' 13 import os 14 15 from mvpa.base import warning 16 from mvpa import cfg 17 from mvpa.misc.support import SmartVersion 18 19 if __debug__: 20 from mvpa.base import debug 21

22 -class _VersionsChecker(dict):

23 """Helper class to check the versions of the available externals 24 """

25 - def __getitem__(self, key):

26 if not self.has_key(key): 27 exists(key, force=True, raiseException=True) 28 return super(_VersionsChecker, self).__getitem__(key)

29 30 versions = _VersionsChecker() 31 """Versions of available externals, as tuples 32 """ 33 34

35 -def __check_scipy():

36 """Check if scipy is present an if it is -- store its version 37 """ 38 import warnings 39 exists('numpy', raiseException='always') 40 # To don't allow any crappy warning to sneak in 41 warnings.simplefilter('ignore', DeprecationWarning) 42 try: 43 import scipy as sp 44 except: 45 warnings.simplefilter('default', DeprecationWarning) 46 raise 47 warnings.simplefilter('default', DeprecationWarning) 48 # Infiltrate warnings if necessary 49 numpy_ver = versions['numpy'] 50 scipy_ver = versions['scipy'] = SmartVersion(sp.__version__) 51 # There is way too much deprecation warnings spit out onto the 52 # user. Lets assume that they should be fixed by scipy 0.7.0 time 53 if scipy_ver >= "0.6.0" and scipy_ver < "0.7.0" \ 54 and numpy_ver > "1.1.0": 55 import warnings 56 if not __debug__ or (__debug__ and not 'PY' in debug.active): 57 if __debug__: 58 debug('EXT', "Setting up filters for numpy DeprecationWarnings") 59 filter_lines = [ 60 ('NumpyTest will be removed in the next release.*', 61 DeprecationWarning), 62 ('PyArray_FromDims: use PyArray_SimpleNew.', 63 DeprecationWarning), 64 ('PyArray_FromDimsAndDataAndDescr: use PyArray_NewFromDescr.', 65 DeprecationWarning), 66 # Trick re.match, since in warnings absent re.DOTALL in re.compile 67 ('[\na-z \t0-9]*The original semantics of histogram is scheduled to be.*' 68 '[\na-z \t0-9]*', Warning) ] 69 for f, w in filter_lines: 70 warnings.filterwarnings('ignore', f, w)

71 72

73 -def __check_numpy():

74 """Check if numpy is present (it must be) an if it is -- store its version 75 """ 76 import numpy as N 77 versions['numpy'] = SmartVersion(N.__version__)

78 79

80 -def __check_pywt(features=None):

81 """Check for available functionality within pywt 82 83 :Parameters: 84 features : list of basestring 85 List of known features to check such as 'wp reconstruct', 86 'wp reconstruct fixed' 87 """ 88 import pywt 89 import numpy as N 90 data = N.array([ 0.57316901, 0.65292526, 0.75266733, 0.67020084, 0.46505364, 91 0.76478331, 0.33034164, 0.49165547, 0.32979941, 0.09696717, 92 0.72552711, 0.4138999 , 0.54460628, 0.786351 , 0.50096306, 93 0.72436454, 0.2193098 , -0.0135051 , 0.34283984, 0.65596245, 94 0.49598417, 0.39935064, 0.26370727, 0.05572373, 0.40194438, 95 0.47004551, 0.60327258, 0.25628266, 0.32964893, 0.24009889,]) 96 mode = 'per' 97 wp = pywt.WaveletPacket(data, 'sym2', mode) 98 wp2 = pywt.WaveletPacket(data=None, wavelet='sym2', mode=mode) 99 try: 100 for node in wp.get_level(2): wp2[node.path] = node.data 101 except: 102 raise ImportError, \ 103 "Failed to reconstruct WP by specifying data in the layer" 104 105 if 'wp reconstruct fixed' in features: 106 rec = wp2.reconstruct() 107 if N.linalg.norm(rec[:len(data)] - data) > 1e-3: 108 raise ImportError, \ 109 "Failed to reconstruct WP correctly" 110 return True

111 112

113 -def __check_libsvm_verbosity_control():

114 """Check for available verbose control functionality 115 """ 116 import mvpa.clfs.libsvmc._svmc as _svmc 117 try: 118 _svmc.svm_set_verbosity(0) 119 except: 120 raise ImportError, "Provided version of libsvm has no way to control " \ 121 "its level of verbosity"

122

123 -def __check_shogun(bottom_version, custom_versions=[]):

124 """Check if version of shogun is high enough (or custom known) to 125 be enabled in the testsuite 126 127 :Parameters: 128 bottom_version : int 129 Bottom version which must be satisfied 130 custom_versions : list of int 131 Arbitrary list of versions which could got patched for 132 a specific issue 133 """ 134 import shogun.Classifier as __sc 135 ver = __sc.Version_get_version_revision() 136 if (ver in custom_versions) or (ver >= bottom_version): 137 return True 138 else: 139 raise ImportError, 'Version %s is smaller than needed %s' % \ 140 (ver, bottom_version)

141 142

143 -def __check_weave():

144 """Apparently presence of scipy is not sufficient since some 145 versions experience problems. E.g. in Sep,Oct 2008 lenny's weave 146 failed to work. May be some other converter could work (? See 147 http://lists.debian.org/debian-devel/2008/08/msg00730.html for a 148 similar report. 149 150 Following simple snippet checks compilation of the basic code using 151 weave 152 """ 153 from scipy import weave 154 from scipy.weave import converters, build_tools 155 import numpy as N 156 # to shut weave up 157 import sys 158 # we can't rely on weave at all at the restoring argv. On etch box 159 # restore_sys_argv() is apparently is insufficient 160 oargv = sys.argv[:] 161 ostdout = sys.stdout 162 if not( __debug__ and 'EXT_' in debug.active): 163 from StringIO import StringIO 164 sys.stdout = StringIO() 165 # *nix specific solution to shut weave up. 166 # Some users must complain and someone 167 # needs to fix this to become more generic. 168 cargs = [">/dev/null", "2>&1"] 169 else: 170 cargs = [] 171 fmsg = None 172 try: 173 data = N.array([1,2,3]) 174 counter = weave.inline("data[0]=fabs(-1);", ['data'], 175 type_converters=converters.blitz, 176 verbose=0, 177 extra_compile_args=cargs, 178 compiler = 'gcc') 179 except Exception, e: 180 fmsg = "Failed to build simple weave sample." \ 181 " Exception was %s" % str(e) 182 183 sys.stdout = ostdout 184 # needed to fix sweave which might "forget" to restore sysv 185 # build_tools.restore_sys_argv() 186 sys.argv = oargv 187 if fmsg is not None: 188 raise ImportError, fmsg 189 else: 190 return "Everything is cool"

191 192

193 -def __check_atlas_family(family):

194 # XXX I guess pylint will dislike it a lot 195 from mvpa.atlases.warehouse import KNOWN_ATLAS_FAMILIES 196 names, pathpattern = KNOWN_ATLAS_FAMILIES[family] 197 filename = pathpattern % {'name':names[0]} 198 if not os.path.exists(filename): 199 raise ImportError, "Cannot find file %s for atlas family %s" \ 200 % (filename, family) 201 pass

202 203

204 -def __check_stablerdist():

205 import scipy.stats 206 import numpy as N 207 ## Unfortunately 0.7.0 hasn't fixed the issue so no chance but to do 208 ## a proper numerical test here 209 try: 210 scipy.stats.rdist(1.32, 0, 1).cdf(-1.0 + N.finfo(float).eps) 211 # Actually previous test is insufficient for 0.6, so enabling 212 # elderly test on top 213 # ATM all known implementations which implement custom cdf for 214 # rdist are misbehaving, so there should be no _cdf 215 if '_cdf' in scipy.stats.distributions.rdist_gen.__dict__.keys(): 216 raise ImportError, \ 217 "scipy.stats carries misbehaving rdist distribution" 218 except ZeroDivisionError: 219 raise RuntimeError, "RDist in scipy is still unstable on the boundaries"

220 221

222 -def __check_rv_discrete_ppf():

223 """Unfortunately 0.6.0-12 of scipy pukes on simple ppf 224 """ 225 import scipy.stats 226 try: 227 bdist = scipy.stats.binom(100, 0.5) 228 bdist.ppf(0.9) 229 except TypeError: 230 raise RuntimeError, "pmf is broken in discrete dists of scipy.stats"

231 232

233 -def __check_in_ipython():

234 # figure out if ran within IPython 235 if '__IPYTHON__' in globals()['__builtins__']: 236 return 237 raise RuntimeError, "Not running in IPython session"

238

239 -def __check_openopt():

240 try: 241 import openopt as _ 242 return 243 except ImportError: 244 pass 245 import scikits.openopt as _ 246 return

247

248 -def __check_matplotlib():

249 """Check for presence of matplotlib and set backend if requested.""" 250 import matplotlib 251 backend = cfg.get('matplotlib', 'backend') 252 if backend: 253 matplotlib.use(backend)

254

255 -def __check_pylab():

256 """Check if matplotlib is there and then pylab""" 257 exists('matplotlib', raiseException='always') 258 import pylab as P

259

260 -def __check_pylab_plottable():

261 """Simple check either we can plot anything using pylab. 262 263 Primary use in unittests 264 """ 265 try: 266 exists('pylab', raiseException='always') 267 import pylab as P 268 fig = P.figure() 269 P.plot([1,2], [1,2]) 270 P.close(fig) 271 except: 272 raise RuntimeError, "Cannot plot in pylab" 273 return True

274 275

276 -def __check_griddata():

277 """griddata might be independent module or part of mlab 278 """ 279 280 try: 281 from griddata import griddata as __ 282 return True 283 except ImportError: 284 if __debug__: 285 debug('EXT_', 'No python-griddata available') 286 287 from matplotlib.mlab import griddata as __ 288 return True

289 290

291 -def __check_reportlab():

292 import reportlab as rl 293 versions['reportlab'] = SmartVersion(rl.Version)

294 295

296 -def __check_rpy():

297 """Check either rpy is available and also set it for the sane execution 298 """ 299 #import rpy_options 300 #rpy_options.set_options(VERBOSE=False, SETUP_READ_CONSOLE=False) # SETUP_WRITE_CONSOLE=False) 301 #rpy_options.set_options(VERBOSE=False, SETUP_WRITE_CONSOLE=False) # SETUP_WRITE_CONSOLE=False) 302 # if not cfg.get('rpy', 'read_console', default=False): 303 # print "no read" 304 # rpy_options.set_options(SETUP_READ_CONSOLE=False) 305 # if not cfg.get('rpy', 'write_console', default=False): 306 # print "no write" 307 # rpy_options.set_options(SETUP_WRITE_CONSOLE=False) 308 import rpy 309 if not cfg.getboolean('rpy', 'interactive', default=True) \ 310 and (rpy.get_rpy_input() is rpy.rpy_io.rpy_input): 311 if __debug__: 312 debug('EXT_', "RPy: providing dummy callback for input to return '1'") 313 def input1(*args): return "1" # which is "1: abort (with core dump, if enabled)" 314 rpy.set_rpy_input(input1)

315 316 317 # contains list of available (optional) external classifier extensions 318 _KNOWN = {'libsvm':'import mvpa.clfs.libsvmc._svm as __; x=__.convert2SVMNode', 319 'libsvm verbosity control':'__check_libsvm_verbosity_control();', 320 'nifti':'from nifti import NiftiImage as __', 321 'nifti ge 0.20090205.1': 322 'from nifti.clib import detachDataFromImage as __', 323 'ctypes':'import ctypes as __', 324 'shogun':'import shogun as __', 325 'shogun.krr': 'import shogun.Regression as __; x=__.KRR', 326 'shogun.mpd': 'import shogun.Classifier as __; x=__.MPDSVM', 327 'shogun.lightsvm': 'import shogun.Classifier as __; x=__.SVMLight', 328 'shogun.svrlight': 'from shogun.Regression import SVRLight as __', 329 'numpy': "__check_numpy()", 330 'scipy': "__check_scipy()", 331 'good scipy.stats.rdist': "__check_stablerdist()", 332 'good scipy.stats.rv_discrete.ppf': "__check_rv_discrete_ppf()", 333 'weave': "__check_weave()", 334 'pywt': "import pywt as __", 335 'pywt wp reconstruct': "__check_pywt(['wp reconstruct'])", 336 'pywt wp reconstruct fixed': "__check_pywt(['wp reconstruct fixed'])", 337 'rpy': "__check_rpy()", 338 'lars': "exists('rpy', raiseException='always'); import rpy; rpy.r.library('lars')", 339 'elasticnet': "exists('rpy', raiseException='always'); import rpy; rpy.r.library('elasticnet')", 340 'glmnet': "exists('rpy', raiseException='always'); import rpy; rpy.r.library('glmnet')", 341 'matplotlib': "__check_matplotlib()", 342 'pylab': "__check_pylab()", 343 'pylab plottable': "__check_pylab_plottable()", 344 'openopt': "__check_openopt()", 345 'mdp': "import mdp as __", 346 'mdp ge 2.4': "from mdp.nodes import LLENode as __", 347 'sg_fixedcachesize': "__check_shogun(3043, [2456])", 348 # 3318 corresponds to release 0.6.4 349 'sg ge 0.6.4': "__check_shogun(3318)", 350 'hcluster': "import hcluster as __", 351 'griddata': "__check_griddata()", 352 'cPickle': "import cPickle as __", 353 'gzip': "import gzip as __", 354 'lxml': "from lxml import objectify as __", 355 'atlas_pymvpa': "__check_atlas_family('pymvpa')", 356 'atlas_fsl': "__check_atlas_family('fsl')", 357 'running ipython env': "__check_in_ipython()", 358 'reportlab': "__check_reportlab()", 359 'nose': "import nose as __", 360 } 361 362

363 -def exists(dep, force=False, raiseException=False, issueWarning=None):

364 """ 365 Test whether a known dependency is installed on the system. 366 367 This method allows us to test for individual dependencies without 368 testing all known dependencies. It also ensures that we only test 369 for a dependency once. 370 371 :Parameters: 372 dep : string or list of string 373 The dependency key(s) to test. 374 force : boolean 375 Whether to force the test even if it has already been 376 performed. 377 raiseException : boolean or 'always' 378 Whether to raise RuntimeError if dependency is missing. 379 If True, it is still conditioned on the global setting 380 MVPA_EXTERNALS_RAISE_EXCEPTION, while would raise exception 381 if missing despite the configuration if 'always'. 382 issueWarning : string or None or True 383 If string, warning with given message would be thrown. 384 If True, standard message would be used for the warning 385 text. 386 """ 387 # if we are provided with a list of deps - go through all of them 388 if isinstance(dep, list) or isinstance(dep, tuple): 389 results = [ exists(dep_, force, raiseException) for dep_ in dep ] 390 return bool(reduce(lambda x,y: x and y, results, True)) 391 392 # where to look in cfg 393 cfgid = 'have ' + dep 394 395 # pre-handle raiseException according to the global settings and local argument 396 if isinstance(raiseException, str): 397 if raiseException.lower() == 'always': 398 raiseException = True 399 else: 400 raise ValueError("Unknown value of raiseException=%s. " 401 "Must be bool or 'always'" % raiseException) 402 else: # must be bool conditioned on the global settings 403 raiseException = raiseException \ 404 and cfg.getboolean('externals', 'raise exception', True) 405 406 # prevent unnecessarry testing 407 if cfg.has_option('externals', cfgid) \ 408 and not cfg.getboolean('externals', 'retest', default='no') \ 409 and not force: 410 if __debug__: 411 debug('EXT', "Skip retesting for '%s'." % dep) 412 413 # check whether an exception should be raised, even though the external 414 # was already tested previously 415 if not cfg.getboolean('externals', cfgid) and raiseException: 416 raise RuntimeError, "Required external '%s' was not found" % dep 417 return cfg.getboolean('externals', cfgid) 418 419 420 # determine availability of external (non-cached) 421 422 # default to 'not found' 423 result = False 424 425 if not _KNOWN.has_key(dep): 426 raise ValueError, "%s is not a known dependency key." % (dep) 427 else: 428 # try and load the specific dependency 429 if __debug__: 430 debug('EXT', "Checking for the presence of %s" % dep) 431 432 # Exceptions which are silently caught while running tests for externals 433 _caught_exceptions = [ImportError, AttributeError, RuntimeError] 434 435 # check whether RPy is involved and catch its excpetions as well. 436 # however, try to determine whether this is really necessary, as 437 # importing RPy also involved starting a full-blown R session, which can 438 # take seconds and therefore is quite nasty... 439 if dep.count('rpy') or _KNOWN[dep].count('rpy'): 440 try: 441 if dep == 'rpy': 442 __check_rpy() # needed to be run to adjust options first 443 else: 444 if exists('rpy'): 445 # otherwise no need to add anything -- test 446 # would fail since rpy isn't available 447 from rpy import RException 448 _caught_exceptions += [RException] 449 except: 450 pass 451 452 453 estr = '' 454 try: 455 exec _KNOWN[dep] 456 result = True 457 except tuple(_caught_exceptions), e: 458 estr = ". Caught exception was: " + str(e) 459 460 if __debug__: 461 debug('EXT', "Presence of %s is%s verified%s" % 462 (dep, {True:'', False:' NOT'}[result], estr)) 463 464 if not result: 465 if raiseException: 466 raise RuntimeError, "Required external '%s' was not found" % dep 467 if issueWarning is not None \ 468 and cfg.getboolean('externals', 'issue warning', True): 469 if issueWarning is True: 470 warning("Required external '%s' was not found" % dep) 471 else: 472 warning(issueWarning) 473 474 475 # store result in config manager 476 if not cfg.has_section('externals'): 477 cfg.add_section('externals') 478 if result: 479 cfg.set('externals', 'have ' + dep, 'yes') 480 else: 481 cfg.set('externals', 'have ' + dep, 'no') 482 483 return result

484 485

486 -def testAllDependencies(force=False):

487 """ 488 Test for all known dependencies. 489 490 :Parameters: 491 force : boolean 492 Whether to force the test even if it has already been 493 performed. 494 495 """ 496 # loop over all known dependencies 497 for dep in _KNOWN: 498 if not exists(dep, force): 499 warning("%s is not available." % dep) 500 501 if __debug__: 502 debug('EXT', 'The following optional externals are present: %s' \ 503 % [ k[5:] for k in cfg.options('externals') 504 if k.startswith('have') \ 505 and cfg.getboolean('externals', k) == True ])

506

Source Code for Module mvpa.base.externals