Source Code for Package mvpa.base

  1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  # vi: set ft=python sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Base functionality of PyMVPA 
 10   
 11  Module Organization 
 12  =================== 
 13   
 14  mvpa.base module contains various modules which are used through out 
 15  PyMVPA code, and are generic building blocks 
 16   
 17  .. packagetree:: 
 18     :style: UML 
 19   
 20  :group Basic: externals, config, verbosity, dochelpers 
 21  """ 
 22   
 23  __docformat__ = 'restructuredtext' 
 24   
 25   
 26  import sys 
 27  from mvpa.base.config import ConfigManager 
 28  from mvpa.base.verbosity import LevelLogger, OnceLogger 
 29   
 30  # 
 31  # Setup verbose and debug outputs 
 32  # 
 33 -class _SingletonType(type): 
 34      """Simple singleton implementation adjusted from 
 35      http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/412551 
 36      """ 
 37 -    def __init__(mcs, *args): 
 38          type.__init__(mcs, *args) 
 39          mcs._instances = {} 
 40   
 41 -    def __call__(mcs, sid, instance, *args): 
 42          if not sid in mcs._instances: 
 43              mcs._instances[sid] = instance 
 44          return mcs._instances[sid] 
 45   
 46 -class __Singleton: 
 47      """To ensure single instance of a class instantiation (object) 
 48   
 49      """ 
 50   
 51      __metaclass__ = _SingletonType 
 52 -    def __init__(self, *args): 
 53          pass 
 54      # Provided __call__ just to make silly pylint happy 
 55 -    def __call__(self): 
 56          raise NotImplementedError 
 57   
 58  # 
 59  # As the very first step: Setup configuration registry instance and 
 60  # read all configuration settings from files and env variables 
 61  # 
 62  cfg = __Singleton('cfg', ConfigManager()) 
 63   
 64  verbose = __Singleton("verbose", LevelLogger( 
 65      handlers = cfg.get('verbose', 'output', default='stdout').split(','))) 
 66   
 67  # Not supported/explained/used by now since verbose(0, is to print errors 
 68  #error = __Singleton("error", LevelLogger( 
 69  #    handlers=environ.get('MVPA_ERROR_OUTPUT', 'stderr').split(','))) 
 70   
 71  # Levels for verbose 
 72  # 0 -- nothing besides errors 
 73  # 1 -- high level stuff -- top level operation or file operations 
 74  # 2 -- cmdline handling 
 75  # 3 -- 
 76  # 4 -- computation/algorithm relevant thingies 
 77   
 78  # Helper for errors printing 
 79 -def error(msg, critical=True): 
 80      """Helper function to output errors in a consistent way. 
 81   
 82      :Parameters: 
 83        msg : string 
 84          Actual error message (will be prefixed with ERROR:) 
 85        critical : bool 
 86          If critical error -- exit with 
 87      """ 
 88      verbose(0, "ERROR: " + msg) 
 89      if critical: 
 90          raise sys.exit(1) 
 91   
 92  # Lets check if environment can tell us smth 
 93  if cfg.has_option('general', 'verbose'): 
 94      verbose.level = cfg.getint('general', 'verbose') 
 95   
 96   
 97 -class WarningLog(OnceLogger): 
 98      """Logging class of messsages to be printed just once per each message 
 99   
100      """ 
101   
102 -    def __init__(self, btlevels=10, btdefault=False, 
103                   maxcount=1, *args, **kwargs): 
104          """Define Warning logger. 
105   
106          It is defined by 
107            btlevels : int 
108              how many levels of backtrack to print to give a hint on WTF 
109            btdefault : bool 
110              if to print backtrace for all warnings at all 
111            maxcount : int 
112              how many times to print each warning 
113          """ 
114          OnceLogger.__init__(self, *args, **kwargs) 
115          self.__btlevels = btlevels 
116          self.__btdefault = btdefault 
117          self.__maxcount = maxcount 
118          self.__explanation_seen = False 
119   
120   
121 -    def __call__(self, msg, bt=None): 
122          import traceback 
123          if bt is None: 
124              bt = self.__btdefault 
125          tb = traceback.extract_stack(limit=2) 
126          msgid = repr(tb[-2])         # take parent as the source of ID 
127          fullmsg = "WARNING: %s" % msg 
128          if not self.__explanation_seen: 
129              self.__explanation_seen = True 
130              fullmsg += "\n * Please note: warnings are "  + \ 
131                    "printed only once, but underlying problem might " + \ 
132                    "occur many times *" 
133          if bt and self.__btlevels > 0: 
134              fullmsg += "Top-most backtrace:\n" 
135              fullmsg += reduce(lambda x, y: x + "\t%s:%d in %s where '%s'\n" % \ 
136                                y, 
137                                traceback.extract_stack(limit=self.__btlevels), 
138                                "") 
139   
140          OnceLogger.__call__(self, msgid, fullmsg, self.__maxcount) 
141   
142   
143 -    def _setMaxCount(self, value): 
144          """Set maxcount for the warning""" 
145          self.__maxcount = value 
146   
147      maxcount = property(fget=lambda x:x.__maxcount, fset=_setMaxCount) 
148   
149  # XXX what is 'bt'? Maybe more verbose name? 
150  if cfg.has_option('warnings', 'bt'): 
151      warnings_btlevels = cfg.getint('warnings', 'bt') 
152      warnings_bt = True 
153  else: 
154      warnings_btlevels = 10 
155      warnings_bt = False 
156   
157  if cfg.has_option('warnings', 'count'): 
158      warnings_maxcount = cfg.getint('warnings', 'count') 
159  else: 
160      warnings_maxcount = 1 
161   
162  warning = WarningLog( 
163      handlers={ 
164          False: cfg.get('warnings', 'output', default='stdout').split(','), 
165          True: []}[cfg.getboolean('warnings', 'suppress', default=False)], 
166      btlevels=warnings_btlevels, 
167      btdefault=warnings_bt, 
168      maxcount=warnings_maxcount 
169      ) 
170   
171   
172  if __debug__: 
173      from mvpa.base.verbosity import DebugLogger 
174      # NOTE: all calls to debug must be preconditioned with 
175      # if __debug__: 
176   
177      debug = __Singleton("debug", DebugLogger( 
178          handlers=cfg.get('debug', 'output', default='stdout').split(','))) 
179   
180      # set some debugging matricses to report 
181      # debug.registerMetric('vmem') 
182   
183      # List agreed sets for debug 
184      debug.register('PY',   "No suppression of various warnings (numpy, scipy) etc.") 
185      debug.register('DBG',  "Debug output itself") 
186      debug.register('DOCH', "Doc helpers") 
187      debug.register('INIT', "Just sequence of inits") 
188      debug.register('RANDOM', "Random number generation") 
189      debug.register('EXT',  "External dependencies") 
190      debug.register('EXT_', "External dependencies (verbose)") 
191      debug.register('TEST', "Debug unittests") 
192      debug.register('MODULE_IN_REPR', "Include module path in __repr__") 
193      debug.register('ID_IN_REPR', "Include id in __repr__") 
194      debug.register('CMDLINE', "Handling of command line parameters") 
195   
196      debug.register('DG',   "Data generators") 
197      debug.register('LAZY', "Miscelaneous 'lazy' evaluations") 
198      debug.register('LOOP', "Support's loop construct") 
199      debug.register('PLR',  "PLR call") 
200      debug.register('SLC',  "Searchlight call") 
201      debug.register('SA',   "Sensitivity analyzers") 
202      debug.register('SOM',  "Self-organizing-maps (SOM)") 
203      debug.register('IRELIEF', "Various I-RELIEFs") 
204      debug.register('SA_',  "Sensitivity analyzers (verbose)") 
205      debug.register('PSA',  "Perturbation analyzer call") 
206      debug.register('RFEC', "Recursive Feature Elimination call") 
207      debug.register('RFEC_', "Recursive Feature Elimination call (verbose)") 
208      debug.register('IFSC', "Incremental Feature Search call") 
209      debug.register('DS',   "*Dataset") 
210      debug.register('DS_NIFTI', "NiftiDataset(s)") 
211      debug.register('DS_',  "*Dataset (verbose)") 
212      debug.register('DS_ID',   "ID Datasets") 
213      debug.register('DS_STATS',"Datasets statistics") 
214      debug.register('SPL',   "*Splitter") 
215   
216      debug.register('TRAN',  "Transformers") 
217      debug.register('TRAN_', "Transformers (verbose)") 
218   
219      # CHECKs 
220      debug.register('CHECK_DS_SELECT', 
221                     "Check in dataset.select() for sorted and unique indexes") 
222      debug.register('CHECK_DS_SORTED', "Check in datasets for sorted") 
223      debug.register('CHECK_IDS_SORTED', 
224                     "Check for ids being sorted in mappers") 
225      debug.register('CHECK_TRAINED', 
226                     "Checking in checking if clf was trained on given dataset") 
227      debug.register('CHECK_RETRAIN', "Checking in retraining/retesting") 
228      debug.register('CHECK_STABILITY', "Checking for numerical stability") 
229      debug.register('ENFORCE_STATES_ENABLED', "Forcing all states to be enabled") 
230   
231      debug.register('MAP',   "*Mapper") 
232      debug.register('MAP_',  "*Mapper (verbose)") 
233   
234      debug.register('COL',  "Generic Collectable") 
235      debug.register('UATTR', "Attributes with unique") 
236      debug.register('ST',   "State") 
237      debug.register('STV',  "State Variable") 
238      debug.register('COLR', "Collector for states and classifier parameters") 
239      debug.register('ES',   "Element selectors") 
240   
241      debug.register('CLF',    "Base Classifiers") 
242      debug.register('CLF_',   "Base Classifiers (verbose)") 
243      #debug.register('CLF_TB', 
244      #    "Report traceback in train/predict. Helps to resolve WTF calls it") 
245      debug.register('CLFBST', "BoostClassifier") 
246      #debug.register('CLFBST_TB', "BoostClassifier traceback") 
247      debug.register('CLFBIN', "BinaryClassifier") 
248      debug.register('CLFTREE', "TreeClassifier") 
249      debug.register('CLFMC',  "MulticlassClassifier") 
250      debug.register('CLFSPL', "SplitClassifier") 
251      debug.register('CLFSPL_',"SplitClassifier (verbose)") 
252      debug.register('CLFFS',  "FeatureSelectionClassifier") 
253      debug.register('CLFFS_', "FeatureSelectionClassifier (verbose)") 
254   
255      debug.register('STAT',   "Statistics estimates") 
256      debug.register('STAT_',  "Statistics estimates (verbose)") 
257      debug.register('STAT__', "Statistics estimates (very verbose)") 
258      debug.register('STATMC', "Progress in Monte-Carlo estimation") 
259   
260      debug.register('FS',     "FeatureSelections") 
261      debug.register('FS_',    "FeatureSelections (verbose)") 
262      debug.register('FSPL',   "FeatureSelectionPipeline") 
263   
264      debug.register('SVM',    "SVM") 
265      debug.register('SVM_',   "SVM (verbose)") 
266      debug.register('LIBSVM', "Internal libsvm output") 
267   
268      debug.register('SMLR',    "SMLR") 
269      debug.register('SMLR_',   "SMLR verbose") 
270   
271      debug.register('LARS',    "LARS") 
272      debug.register('LARS_',   "LARS (verbose)") 
273   
274      debug.register('ENET',    "ENET") 
275      debug.register('ENET_',   "ENET (verbose)") 
276   
277      debug.register('GLMNET',    "GLMNET") 
278      debug.register('GLMNET_',   "GLMNET (verbose)") 
279   
280      debug.register('GNB',     "GNB - Gaussian Naive Bayes") 
281   
282      debug.register('GPR',     "GPR") 
283      debug.register('GPR_WEIGHTS', "Track progress of GPRWeights computation") 
284      debug.register('KERNEL',  "Kernels module") 
285      debug.register('MOD_SEL', "Model Selector (also makes openopt's iprint=0)") 
286      debug.register('OPENOPT', "OpenOpt toolbox verbose (iprint=1)") 
287   
288      debug.register('SG',  "PyMVPA SG wrapping") 
289      debug.register('SG_', "PyMVPA SG wrapping verbose") 
290      debug.register('SG__', "PyMVPA SG wrapping debug") 
291      debug.register('SG_SVM', "Internal shogun debug output for SVM itself") 
292      debug.register('SG_FEATURES', "Internal shogun debug output for features") 
293      debug.register('SG_LABELS', "Internal shogun debug output for labels") 
294      debug.register('SG_KERNELS', "Internal shogun debug output for kernels") 
295      debug.register('SG_PROGRESS', 
296                     "Internal shogun progress bar during computation") 
297   
298      debug.register('IOH',    "IO Helpers") 
299      debug.register('IO_HAM', "Hamster") 
300      debug.register('CM',   "Confusion matrix computation") 
301      debug.register('ROC',  "ROC analysis") 
302      debug.register('CROSSC', "Cross-validation call") 
303      debug.register('CERR', "Various ClassifierErrors") 
304   
305      debug.register('ATL',    "Atlases") 
306      debug.register('ATL_',   "Atlases (verbose)") 
307      debug.register('ATL__',  "Atlases (very verbose)") 
308   
309      debug.register('REP',    "Reports") 
310      debug.register('REP_',   "Reports (verbose)") 
311   
312      # Lets check if environment can tell us smth 
313      if cfg.has_option('general', 'debug'): 
314          debug.setActiveFromString(cfg.get('general', 'debug')) 
315   
316      # Lets check if environment can tell us smth 
317      if cfg.has_option('debug', 'metrics'): 
318          debug.registerMetric(cfg.get('debug', 'metrics').split(",")) 
319   
320   
321   
322  if __debug__: 
323      debug('INIT', 'mvpa.base end') 
324