Source Code for Module helpers.UpdateTableData

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: UpdateTableData.py 7146 2010-06-29 15:10:06Z EckhardSutorius $ 
  4  """ 
  5     Update database tables with new data from a file for given date range. 
  6   
  7     @author: E. Sutorius 
  8     @org:    WFAU, IfA, University of Edinburgh 
  9   
 10     @newfield contributors: Contributors, Contributors (Alphabetical Order) 
 11     @contributors: R.S. Collins 
 12   
 13     @todo: Needs to be upgraded to become a CuSession to save curation history. 
 14  """ 
 15  #------------------------------------------------------------------------------ 
 16  import dircache as dc 
 17  import os 
 18   
 19  from   wsatools.CLI                 import CLI 
 20  import wsatools.CSV                     as csv 
 21  from   wsatools.DbConnect.DbSession import DbSession 
 22  from   wsatools.Logger              import ForLoopMonitor, Logger 
 23  #------------------------------------------------------------------------------ 
 24   
 25 -def makelist(indir, preflist, verslist, startdate, enddate): 
 26      # sort out prefix 
 27      tmpfilelist = [elem for elem in dc.listdir(indir) 
 28                          for pref in preflist if elem.startswith(pref)] 
 29      # sort out versionnr 
 30      filelist = [elem for elem in tmpfilelist 
 31                       for vers in verslist 
 32                       if elem[:elem.find(".list")].endswith('_'+vers)] 
 33      datedlist = [] 
 34      for entry in filelist: 
 35          filestart = int(entry[entry.find("_20")+1:entry.find("_20")+9]) 
 36          fileend = int(entry[entry.rfind("_20")+1:entry.rfind("_20")+9]) 
 37          if startdate == enddate == filestart == fileend: 
 38              return [entry] 
 39          elif startdate <= fileend and enddate >= filestart: 
 40              datedlist.append(entry) 
 41      return datedlist 
 42   
 43  #------------------------------------------------------------------------------ 
 44   
 45 -def updateDb(archive, table, filePath, format): 
 46      try: 
 47          Logger.addMessage("Updating table...") 
 48          updates = csv.File(filePath) 
 49          progress = ForLoopMonitor(updates) 
 50          for values in updates: 
 51              archive.update(table, zip(format, values), 
 52                where="%s LIKE '%%%s' AND %s=%s" % 
 53                      (format[0], values[0], format[1], values[1])) 
 54              progress.testForOutput() 
 55          archive.commitTransaction() 
 56   
 57      except Exception, details: 
 58          Logger.addExceptionMessage(details) 
 59          archive.rollbackTransaction() 
 60          print("rolled back!") 
 61      else: 
 62          print("updated successfully!") 
 63   
 64  #------------------------------------------------------------------------------ 
 65  # Entry point for script. 
 66   
 67  # Allow module to be imported as well as executed from the command line 
 68  if __name__ == '__main__': 
 69      # Define additional command-line interface options 
 70      CLI.progArgs += [ 
 71          CLI.Argument('startdate', '20051213', isValOK=CLI.isDateOK), 
 72          CLI.Argument('enddate', '20051224', isValOK=CLI.isDateOK)] 
 73      CLI.progOpts += [ 
 74          CLI.Option('f', 'format', 
 75              'a csv list of the attributes in the given files', 'LIST', 
 76              'fileName,multiframeID,darkID,confID,flatID,frinID,skyID,maskID'), 
 77          CLI.Option('i', 'indir', 
 78              "the directory where the files 'fileprefix*' are", 'DIR', 
 79              'prodoutput'), 
 80          CLI.Option('l', 'list', 
 81              'a list of file prefixes which are to be ingested', 'LIST', 
 82              'wrongprod'), 
 83          CLI.Option('v', 'version', 
 84              "a list of versions to update", 'LIST', '1,2', 
 85              isValOK=lambda x: all(y.replace('.', '').isdigit() 
 86                                    for y in csv.values(x)))] 
 87   
 88      cli = CLI("UpdateTableData", '$Revision: 7146 $', __doc__) 
 89      Logger.isVerbose = False 
 90      Logger.addMessage(cli.getProgDetails()) 
 91   
 92      startdate = int(cli.getArg('startdate')) 
 93      enddate = int(cli.getArg('enddate')) 
 94   
 95      format = csv.values(cli.getOpt('format')) 
 96      filepref = csv.values(cli.getOpt('list')) 
 97      versions = csv.values(cli.getOpt('version')) 
 98      indir = cli.getOpt('indir') 
 99      if not os.path.exists(indir): 
100          raise SystemExit("No dir %s" % indir) 
101   
102      table = "Multiframe" 
103   
104      for filename in makelist(indir, filepref, versions, startdate, enddate): 
105          filePathName = os.path.join(indir, filename) 
106          updateDb(DbSession(cli=cli), table, filePathName, format) 
107   
108  #------------------------------------------------------------------------------ 
109  # Change log: 
110  # 
111  # 11-Apr-2006, ETWS: Original version. 
112  #  8-May-2006, ETWS: Added input directory 
113  # 22-Feb-2007,  RSC: Updated to reflect move of loadServerHost() constant from 
114  #                    DbConstants to SystemConstants 
115  # 30-May-2007,  RSC: Upgraded to use DbSession and CSV modules, removed 
116  #                    dependence on obsoleted methods. 
117  #  3-Dec-2007,  RSC: Upgraded to use CLI and updated for new DbSession i/f. 
118