Source Code for Module helpers.UpdateOSAFixedData

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: UpdateOSAFixedData.py 9972 2013-07-30 15:07:36Z EckhardSutorius $ 
  4  """ 
  5     Script to fix illum-fixed OSA FITS files that have been released by CASU 
  6     after catalogue ingests have been completed: 
  7        1. Move the old pixel file to a deprecated dir and deprecate MF, MFD 
  8           in DB. 
  9        2. Write a transfer script and transfer the new *_fix.cats and the 
 10           associated pixel files. 
 11   
 12     @author: E.T.W. Sutorius 
 13     @org:    WFAU, IfA, University of Edinburgh 
 14  """ 
 15  #------------------------------------------------------------------------------ 
 16  from   collections  import defaultdict, namedtuple 
 17  import mx.ODBC.unixODBC as odbc 
 18  import os 
 19  import string 
 20  import time 
 21   
 22  from   wsatools.CLI                 import CLI 
 23  from   wsatools.DbConnect.DbSession import DbSession, Join 
 24  from   wsatools.File                import File 
 25  import wsatools.FitsUtils               as fits 
 26  from   wsatools.Logger              import Logger 
 27  from   wsatools.SystemConstants     import DepCodes 
 28  import wsatools.Utilities               as utils 
 29  #------------------------------------------------------------------------------ 
 30   
 31 -class UpdateOSAFixedData(object): 
 32      """ 
 33      """ 
 34      FileNames = namedtuple("FileNames", "filename fixcat") 
 35       
 36 -    def __init__(self, cli): 
 37          """ 
 38          Initialise class from command-line options. 
 39   
 40          @param cli: Command-line arguments. 
 41          @type  cli: CLI 
 42   
 43          """ 
 44          self.logFileName = cli.getArg("xferlog") 
 45          self.archive = DbSession(cli=cli, autoCommit=True) 
 46          self.sysc = self.archive.sysc 
 47          self.isTestRun = cli.getOpt("test") 
 48   
 49      #-------------------------------------------------------------------------- 
 50   
 51 -    def createFileDateDict(self, fileList): 
 52          """Create a dictionary from the given list of files""" 
 53          fileDateDict = defaultdict(list) 
 54          for fileName in fileList: 
 55              fitsFile = File(fileName) 
 56              fileDateDict[fitsFile.subdir].append(fitsFile.name) 
 57              del fitsFile 
 58          return fileDateDict 
 59   
 60      #-------------------------------------------------------------------------- 
 61   
 62 -    def getDetectionTable(self, multiframeID): 
 63          """ Get the detection table name for the given multiframeID 
 64          """ 
 65          detName = self.archive.query( 
 66              "tableName", 
 67              Join(["programmeTable", "programmeFrame"], ["programmeID"]), 
 68              " AND ".join(["multiframeID=%d" % multiframeID, 
 69                            "tableName like '%Detection'"]))  
 70          return detName[0] + "Raw" 
 71       
 72      #-------------------------------------------------------------------------- 
 73   
 74      @staticmethod 
 75 -    def readLogfile(fileName): 
 76          """Read fixed cat names. 
 77          """ 
 78          logFile = File(fileName) 
 79          logFile.ropen() 
 80          fileList = logFile.readlines(commentChar='#') 
 81          logFile.close() 
 82          return fileList 
 83   
 84      #-------------------------------------------------------------------------- 
 85   
 86 -    def readMfIDs(self, fileNameList): 
 87          """ Read filename,multiframeIDs from the given database. 
 88          """ 
 89          Logger.addMessage("Getting MultiframeIDs...") 
 90          mfIDDict = defaultdict(int) 
 91          detMfIDDict = defaultdict(list) 
 92          for entry in sorted(fileNameList): 
 93              for fN, mfID in self.archive.query( 
 94                    "fileName, multiframeID", "Multiframe", 
 95                    "filename like '%" + entry.filename + "%'"): 
 96                  mfIDDict[fN] = mfID 
 97                  # get detection table 
 98                  detTable = self.getDetectionTable(mfID) 
 99                  detMfIDDict[detTable].append(mfID) 
100                  detMfIDDict["Multiframe"].append(mfID) 
101                  detMfIDDict["MultiframeDetector"].append(mfID) 
102          return mfIDDict, detMfIDDict 
103   
104      #-------------------------------------------------------------------------- 
105   
106 -    def updateDeprecations(self, mfidDict): 
107          """ Update the given table's deprecation codes. 
108          """ 
109          for table in sorted(mfidDict): 
110              numRows = 0 
111              mfIdList = mfidDict[table] 
112              Logger.addMessage("Updating deprecated flag for table %s..." % \ 
113                                table) 
114              entryList = "deprecated = deprecated + %d" % DepCodes.reprocCASU 
115              whereStr = " AND ".join([ 
116                  "multiframeID in (%s)" % ','.join(map(str, mfIdList)), 
117                  "deprecated < %s" % DepCodes.reprocCASU]) 
118               
119              if self.isTestRun: 
120                  Logger.addMessage("UPDATE %s SET %s WHERE %s" % ( 
121                      table, entryList, whereStr)) 
122              else: 
123                  numRows += self.archive.update(table, entryList, whereStr) 
124              Logger.addMessage("%s rows affected." % numRows) 
125   
126      #-------------------------------------------------------------------------- 
127   
128 -    def updateFileNames(self, mfID, filePath): 
129          """ Update filename entries in Multiframe and FlatFileLookup 
130          """ 
131          Logger.addMessage("Updating mfID %d: %s" % (mfID, filePath)) 
132          entryList = "fileName = '%s'" % filePath 
133          whereStr = "multiframeID = %d" % mfID 
134          for table in ["Multiframe", "FlatFileLookup"]: 
135              if self.isTestRun: 
136                  Logger.addMessage("UPDATE %s SET %s WHERE %s" % ( 
137                          table, entryList, whereStr)) 
138              else: 
139                  numRows = self.archive.update(table, entryList, whereStr) 
140   
141      #-------------------------------------------------------------------------- 
142   
143 -    def run(self): 
144          """  
145          """ 
146          fixedCats = self.readLogfile(self.logFileName) 
147          fileNameList = [self.FileNames( 
148              x.replace(self.sysc.fixcatSuffix + self.sysc.catType, 
149                        self.sysc.mefType), x) for x in fixedCats] 
150          mfidDict, detMfIDDict = self.readMfIDs(fileNameList) 
151   
152          # update deprecations in DB tables 
153          self.updateDeprecations(detMfIDDict) 
154   
155          xferSet = set() 
156          for entry in fileNameList: 
157              deprPath = entry.filename.replace(self.sysc.fitsDir, 
158                                                self.sysc.deprecatedDir) 
159              utils.ensureDirExist(os.path.dirname(deprPath)) 
160   
161              # move old files into deprecated dir 
162              if self.isTestRun: 
163                  print entry.filename,"=>",deprPath 
164              else: 
165                  os.rename(entry.filename, deprPath) 
166   
167              # update filenames in DB 
168              self.updateFileNames(mfidDict[entry.filename], deprPath) 
169   
170              # write xfer log 
171              fPath, fName = os.path.split(entry.filename) 
172              dateStr, versStr = os.path.basename(fPath).partition("_v")[::2] 
173              xferSet.add(os.path.join(dateStr, fName)) 
174              xferSet.add(os.path.join(dateStr, os.path.basename(entry.fixcat))) 
175               
176          # create transfer log file 
177          xferLog = File(os.path.join( 
178              self.sysc.curationCodePath(), "invocations", "cu0", 
179              "xferfix_%s.log" % utils.makeTimeStamp().replace(' ','_'))) 
180          xferLog.wopen() 
181          for entry in sorted(xferSet): 
182              xferLog.writetheline(entry) 
183          xferLog.close() 
184          Logger.addMessage("TransferLog written to %s" % xferLog.name) 
185          Logger.addMessage("Please run: ./DataBuilder.py -l %s -v %s -x 1 %s" %( 
186              xferLog.base, versStr, self.archive.database)) 
187   
188  #------------------------------------------------------------------------------ 
189  # Entry point for script. 
190   
191  # Allow module to be imported as well as executed from the command line 
192  if __name__ == '__main__': 
193      # Define command-line interface settings for UpdateFitsFileNames 
194      CLI.progArgs += [CLI.Argument("xferlog", "xfer.log", True)] 
195      CLI.progOpts += [] 
196      cli = CLI(UpdateOSAFixedData, "$Revision: 9972 $") 
197      Logger.addMessage(cli.getProgDetails()) 
198   
199      UpdateOSAFixedData(cli).run() 
200