Source Code for Module app.autoranking.check

  1  ''' 
  2  Created on 07 Mar 2012 
  3   
  4  @author: Eleftherios Avramidis 
  5  ''' 
  6   
  7  from suite import AutorankingSuite 
  8  import argparse  
  9  import sys 
 10  import os 
 11  import logging 
 12   
 13  """ 
 14  Gathers the results from the app folders created with python Experiment Suite 
 15  """ 
 16           
 17       
 18   
 19   
 20       
 21 -def retrieve_results(mysuite, path, reps = [0]): 
 22      """ 
 23      Parallelizes and aligned the vectors provided by the Experiment Suite API 
 24      @param mysuite: instance of the Experiment Suite or subclass 
 25      @type mysuite: L{ExperimentSuite} 
 26      @param path: the directory that will be searched for results. It needs to have the structure suported by Experiment Suite 
 27      @type path: str  
 28      @param reps: a list or a range of repetition values that need to be checked. For simple experiments only one repetition exists, so the default value is 0 
 29      @type reps: [int, ...] 
 30      @return: A list of with one tuple per app. Each tuple contains a dictionary of the app parameters and a a dictionary of the result values 
 31      @rtype: [({parameter_name:parameter_value},{result_name:result_value}), ...]  
 32      The key of each entry is a tuple containing (attribute_set,classifier_name,discretization,filterscorediff) and  
 33      its value is a list of (float) values, respectively to their names entered in the list 'required_feature_names' 
 34      """ 
 35       
 36      results = [] 
 37      exps = mysuite.get_exps(path=path) 
 38      logging.info("Found %s experiments", len(exps))     
 39      logging.debug("exps: %s", exps) 
 40      result_names = set() 
 41       
 42      #browse app directories one by one 
 43      for exp in exps: 
 44          for rep in reps: 
 45              #get a dictionary with all result values for the current app 
 46              values = mysuite.get_value(exp, rep, 'all') 
 47              if values: 
 48                  params = mysuite.get_params(exp) 
 49                  params["app"] = os.path.basename(os.path.dirname(exp)) 
 50                  results.append((params, values)) 
 51      logging.info("found %s experiments", len(results)) 
 52       
 53      return results 
 54   
 55                   
 56   
 57           
 58 -def get_tabfile(results, preferred_params=[], preferred_scores=[], display_header=True, delimiter="\t"): 
 59   
 60       
 61      #get a list with the result names 
 62      param_keys = set() 
 63      score_keys = set() 
 64      for params, values in results: 
 65          param_keys.update(params.keys()) 
 66          score_keys.update(values.keys()) 
 67       
 68  #    keys_toremove = set([param_key for param_key in param_keys if param_key.startswith("attset_")]) 
 69  #    keys_toremove.add('iterations') 
 70  #    keys_toremove.add('repetitions') 
 71  #    keys_toremove.add('path') 
 72  #    keys_toremove.add('test_set') 
 73  #    keys_toremove.add('hidden_attributes') 
 74  #     
 75       
 76  #    param_keys = param_keys - keys_toremove      
 77       
 78      if not preferred_params: 
 79          preferred_params = param_keys 
 80   
 81      if not preferred_scores: 
 82          preferred_scores = score_keys 
 83       
 84      preferred_scores = preferred_scores 
 85   
 86      if display_header: 
 87          print delimiter.join(preferred_params) + delimiter + delimiter.join(preferred_scores) 
 88       
 89      for params, values in results: 
 90   
 91          #retain only the preferred params 
 92          params = [str(params.setdefault(param_name,"None")) for param_name in preferred_params] 
 93           
 94           
 95          onlyvalues = [] 
 96           
 97          #collect a list with the values 
 98          for key in preferred_scores: 
 99              #extract from array, if possible 
100              try: 
101                  onlyvalues.append(str(values[key][0])) 
102              except: 
103                  try: 
104                      onlyvalues.append(str(values[key])) 
105                  except KeyError: 
106                      onlyvalues.append('') 
107                       
108          print delimiter.join(params) + delimiter + delimiter.join(onlyvalues) 
109               
110               
111  if __name__ == "__main__": 
112       
113       
114       
115      #dev example  
116      #python2.7 check.py --path /home/Eleftherios Avramidis/taraxu_data/selection-mechanism/emnlp/app/4b --reps 0 --config config/autoranking.suite.bernux.cfg --params app classifier att mode ties include_references  > test.csv 
117      parser = argparse.ArgumentParser(description='Process some integers.') 
118      parser.add_argument('--path', nargs=1, 
119                     help='the path were experiments will be found') 
120      parser.add_argument('--reps', type = int, nargs='*',                    
121                     help='list of repetition ids to keep (default: 0)') 
122       
123      parser.add_argument('--params', nargs='*',                    
124                     help='Names of parameters to be displayed (default: all)') 
125       
126      parser.add_argument('--metrics', nargs='*',                    
127                     help='Names of metrics to be displayed (default: all)') 
128       
129      parser.add_argument('--config', nargs=1,                    
130                     help='the configuration file to be checked') 
131                      
132      parser.add_argument('--logging', nargs="?", default = None, 
133                     help='should logging be performed, if set to True writes the debug level to debug.log, ') 
134   
135   
136   
137      args = parser.parse_args(sys.argv[1:]) 
138       
139      if args.logging: 
140          logging.basicConfig(filename='debug.log',level=getattr(logging, args.logging.upper())) 
141       
142      sys.argv = [sys.argv[0]] 
143      sys.argv.extend(["--config", args.config]) 
144      mysuite = AutorankingSuite() 
145       
146      path = args.path[0] 
147      reps = args.reps 
148      preferred_params = args.params 
149      preferred_scores = args.metrics 
150       
151      results = retrieve_results(mysuite, path, reps) 
152      get_tabfile(results, preferred_params, preferred_scores) 
153