Package helpers :: Module FixFitsFiles
[hide private]

Source Code for Module helpers.FixFitsFiles

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: FixFitsFiles.py 10044 2013-09-06 12:50:23Z EckhardSutorius $ 
  4  """ 
  5     Script to fix problematic FITS files: 
  6        1. Reads multiframeIDs from the database for a list of files which may've 
  7           been ingested before and write them into the files. 
  8        2. Fixes any (attribute, value) pair for a list of FITS files. 
  9        3. Fixes FITS files and DB entries for JPEs VISTA non-survey. 
 10   
 11     @author: E.T.W. Sutorius 
 12     @org:    WFAU, IfA, University of Edinburgh 
 13  """ 
 14  #------------------------------------------------------------------------------ 
 15  from   collections  import defaultdict 
 16  import mx.ODBC.unixODBC as odbc 
 17  import os 
 18  import string 
 19  import time 
 20   
 21  from   wsatools.CLI                 import CLI 
 22  from   wsatools.DbConnect.DbSession import DbSession 
 23  from   wsatools.File                import File 
 24  import wsatools.FitsUtils               as fits 
 25  from   wsatools.Logger              import Logger 
 26  #------------------------------------------------------------------------------ 
 27   
28 -class FixFitsFiles(object):
29 """Script to fix multiframe IDs in FITS files with data from the database. 30 """ 31 defVals = {"DETECTID": {2:"RSC:H2:60", 3:"RSC:H2:63", 4:"RSC:H2:76", 32 5:"RSC:H2:41", 33 "comment": "Serial number of detector array"}, 34 "DCOLUMNS": {2:2048, 3:2048, 4:2048, 5:2048, 35 "comment":"[pixel] Number of detector columns"}, 36 "CAMNUM": {2:1, 3:2, 4:3, 5:4, 37 "comment":"Number of WFCAM camera (1, 2, 3 or 4)"}} 38
39 - def __init__(self, cli):
40 """ 41 Initialise class from command-line options. 42 43 @param cli: Command-line arguments. 44 @type cli: CLI 45 46 """ 47 self.updateTime = cli.getOpt("update") 48 self.updateFileTime = cli.getOpt("timestamp") 49 if cli.getOpt("fixkey"): 50 self.updateKey, self.updateValue = \ 51 cli.getOpt("fixkey").partition(',')[::2] 52 if not self.updateValue: 53 Logger.addMessage("<ERROR> No key/value given!") 54 raise SystemExit 55 else: 56 self.updateKey = self.updateValue = None 57 58 self.fixJPEns = cli.getOpt("fixjpens") 59 self.fixrecal = cli.getOpt("fixrecal") 60 if self.fixJPEns: 61 self.updateKey, self.updateValue = [ 62 "HIERARCH ESO OBS PROG ID", "284.C-5034(A)"] 63 self.logFileName = cli.getArg("xferlog") 64 self.archive = DbSession(cli=cli, autoCommit=True)
65 66 #-------------------------------------------------------------------------- 67
68 - def readMfIDs(self, dateVersStrList):
69 """ Read filename,multiframeIDs from the given database. 70 """ 71 mfIDDict = defaultdict() 72 for dateVersStr in sorted(dateVersStrList): 73 Logger.addMessage("Getting data for %s..." % dateVersStr) 74 for fN, mfID, fTS in self.archive.query( 75 "fileName, multiframeID, fileTimeStamp", "Multiframe", 76 "filename like '%" + dateVersStr + "%'"): 77 fileName = \ 78 fN.replace(self.archive.sysc.pixelServerHostName, '') 79 80 mfIDDict[fileName] = (mfID, fTS) 81 82 return mfIDDict
83 84 #-------------------------------------------------------------------------- 85
86 - def createFileDateDict(self, fileList):
87 """Create a dictionary from the given list of files""" 88 fileDateDict = defaultdict(list) 89 for fileName in fileList: 90 fitsFile = File(fileName) 91 fileDateDict[fitsFile.subdir].append(fitsFile.name) 92 del fitsFile 93 return fileDateDict
94 95 #-------------------------------------------------------------------------- 96
97 - def updateTimeStamp(self, multiframeID, timeStamp):
98 numRows = self.archive.update("Multiframe", 99 "fileTimeStamp=%s" % timeStamp, 100 "multiframeID=%s" % multiframeID) 101 Logger.addMessage("%s file updated in Multiframe." % numRows)
102 103 #-------------------------------------------------------------------------- 104
105 - def updateJPEns(self, mfIDDict):
106 """Update tables for JPE's non-survey.""" 107 # Multiframe 108 numRows = 0 109 for fileName in mfIDDict: 110 numRows += self.archive.update( 111 "Multiframe", 112 "project='%s'" % self.updateValue, 113 "multiframeID=%d" % mfIDDict[fileName][0]) 114 Logger.addMessage("%d rows updated in Multiframe." % numRows) 115 116 # MultiframeEsoKeys 117 numRows = 0 118 for fileName in mfIDDict: 119 numRows += self.archive.update( 120 "MultiframeEsoKeys", 121 "obsProgID='%s'" % self.updateValue, 122 "multiframeID=%d" % mfIDDict[fileName][0]) 123 Logger.addMessage("%d rows updated in MultiframeEsoKeys." % numRows) 124 125 # ProgrammeFrame 126 numRows = 0 127 progID = self.archive.query("programmeID", "Programme", 128 "dfsIDString='%s'" % self.updateValue)[0] 129 mfIDList = [] 130 for fileName in mfIDDict: 131 numRows += self.archive.update( 132 "ProgrammeFrame", 133 "programmeID=%d" % progID, 134 "multiframeID=%d" % mfIDDict[fileName][0]) 135 mfIDList.append(mfIDDict[fileName][0]) 136 137 Logger.addMessage("%d rows updated in ProgrammeFrame." % numRows) 138 139 # d284c5034aDetection* 140 identity = string.maketrans('', '') 141 prefix = self.updateValue.translate(identity, ".-()").lower() 142 for suffix in ["Raw", "Photometry", "Astrometry"]: 143 Logger.addMessage("Copying from calDetection%s into d%sDetection%s" 144 " ..." % (suffix, prefix, suffix)) 145 numRowsCopied = 0 146 numRowsDeleted = 0 147 for fileName in mfIDDict: 148 numRows = 0 149 try: 150 numRows = self.archive.copyIntoTable( 151 destinationTable="d%sDetection%s" % (prefix, suffix), 152 sourceTable="calDetection%s" % suffix, 153 columns='*', 154 where="multiframeID=%d" % mfIDDict[fileName][0]) 155 srcTable = "calDetection%s" % suffix 156 if numRows == 0 and \ 157 self.archive.sysc.tileSuffix in fileName and \ 158 not self.archive.sysc.confSuffix in fileName: 159 numRows = self.archive.copyIntoTable( 160 destinationTable="d%sDetection%s" % (prefix, suffix), 161 sourceTable="maintDetection%s" % suffix, 162 columns='*', 163 where="multiframeID=%d" % mfIDDict[fileName][0]) 164 srcTable = "maintDetection%s" % suffix 165 if numRows == 0: 166 Logger.addMessage("No data available for %d: %s" % ( 167 mfIDDict[fileName][0], fileName)) 168 else: 169 numRowsCopied += numRows 170 except odbc.IntegrityError: 171 Logger.addMessage("Data already exists in d%sDetection%s " 172 "for %d" % (prefix, suffix, mfIDDict[fileName][0])) 173 except: 174 print fileName, mfIDDict[fileName][0] 175 raise 176 else: 177 if numRows > 0: 178 numRowsDeleted += self.archive.delete( 179 srcTable, 180 whereStr="multiframeID=%d" % mfIDDict[fileName][0]) 181 182 Logger.addMessage("%d rows copied into d%sDetection%s" % ( 183 numRowsCopied, prefix, suffix)) 184 Logger.addMessage("%d rows deleted from calDetection%s" % ( 185 numRowsDeleted, suffix))
186 187 #-------------------------------------------------------------------------- 188
189 - def run(self):
190 """ 191 """ 192 mfIDDict = defaultdict() 193 if self.fixJPEns: 194 for fileName, mfID in self.archive.query( 195 "fileName, multiframeID", "Multiframe", 196 " AND ".join(["raBase BETWEEN 10.25 and 11", 197 "decBase BETWEEN -61 and -58", 198 "vistaRunNo BETWEEN 137 and 880", 199 "utDate = '2010-02-03'"])): 200 201 mfIDDict[fileName] = (mfID, 0) 202 203 fileList = mfIDDict.keys() 204 205 else: 206 logFile = File(self.logFileName) 207 logFile.ropen() 208 fileList = logFile.readlines(commentChar='#') 209 logFile.close() 210 211 # read file names from log file 212 fileDateDict = self.createFileDateDict(fileList) 213 214 if self.fixJPEns: 215 # update data in DB 216 self.updateJPEns(mfIDDict) 217 else: 218 # read data from DB 219 if self.updateKey: 220 for dateVersStr in sorted(fileDateDict): 221 for fileName in fileDateDict[dateVersStr]: 222 mfIDDict[fileName] = (0, 0) 223 else: 224 mfIDDict = self.readMfIDs(fileDateDict.keys()) 225 226 num = 0 227 timefmt = "'%y%m%d%H%M%S'" 228 for dateVersStr in sorted(fileDateDict): 229 for fileName in fileDateDict[dateVersStr]: 230 if mfIDDict.has_key(fileName): 231 num += 1 232 # get file creation date 233 dirName = os.path.split(os.path.dirname(fileName))[1][2:8] 234 modtime = time.strptime(time.ctime(os.stat( 235 fileName).st_mtime)) 236 wsaTimestamp = dirName + time.strftime(timefmt, modtime)[1:-1] 237 238 mfID, dbTimeStamp = mfIDDict[fileName] 239 Logger.addMessage("#%4d %s: %d, %s" % (num, fileName, mfID, 240 wsaTimestamp)) 241 242 if self.updateKey: 243 if not self.archive.isTrialRun: 244 # open file in update mode 245 hduList = fits.open(fileName, "update") 246 for i, hdu in enumerate(hduList): 247 try: 248 if self.updateValue == "check": 249 if i > 0 and self.updateKey in FixFitsFiles.defVals and not hdu.header.has_key(self.updateKey): 250 updateValue = FixFitsFiles.defVals[ 251 self.updateKey][i + 1] 252 updateComment = FixFitsFiles.defVals[ 253 self.updateKey]["comment"] 254 Logger.addMessage( 255 "Updating %s in ext %d with '%s'" % 256 (self.updateKey, i, updateValue)) 257 fits.writeToFitsHdu( 258 hduList, (i,), self.updateKey, 259 updateValue, updateComment, 260 redoing=True) 261 else: 262 card = hdu.header.ascardlist()[ 263 self.updateKey] 264 if card.value != self.updateValue: 265 Logger.addMessage( 266 "Updating %s in ext %d with '%s'" % 267 (card.key, i, self.updateValue)) 268 fits.writeToFitsHdu( 269 hduList, (i,), self.updateKey, 270 self.updateValue, card.comment, 271 redoing=True) 272 else: 273 Logger.addMessage( 274 "%s in ext %d already has value '%s'" % 275 (card.key, i, self.updateValue)) 276 except KeyError: 277 Logger.addMessage( 278 "%s not found in ext %d" % 279 (self.updateKey, i)) 280 281 # close file 282 hduList.close() 283 else: 284 Logger.addMessage("TestRun, otherwise update %s " 285 "here." % self.updateKey) 286 287 elif int(wsaTimestamp) != int(dbTimeStamp): 288 Logger.addMessage( 289 "<WARNING> Timestamps differ: file: %s <=>db: %s" % \ 290 (wsaTimestamp, dbTimeStamp)) 291 if self.updateTime: 292 self.updateTimeStamp(mfID, wsaTimestamp) 293 else: 294 if not self.archive.isTrialRun: 295 # open file in update mode 296 hduList = fits.open(fileName, "update") 297 # write wsaTimestamp 298 extNum = (1 if "_tl" in fileName else 0) 299 fits.writeToFitsHdu( 300 hduList, (extNum,), 301 "%s_TIME" % self.archive.sysc.loadDatabase, 302 wsaTimestamp, "%s time stamp" % \ 303 self.archive.sysc.loadDatabase, redoing=True) 304 305 # write MultiframeID 306 fits.writeToFitsHdu( 307 hduList, (extNum,), 308 "%s_MFID" % self.archive.sysc.loadDatabase, 309 mfID, "%s Multiframe ID" % \ 310 self.archive.sysc.loadDatabase, redoing=True) 311 312 # close file 313 hduList.close() 314 315 # if file was modified, set the modification date back 316 # to the original one 317 if self.updateFileTime: 318 timetpl = time.strptime(str(dbTimeStamp)[-12:], 319 '%y%m%d%H%M%S') 320 else: 321 timetpl = time.strptime(wsaTimestamp[len(dirName):], 322 '%y%m%d%H%M%S') 323 filetime = time.strptime(time.ctime(os.stat( 324 fileName).st_mtime)) 325 fileTimestamp = time.strftime(timefmt, filetime)[1:-1] 326 if fileTimestamp != time.strftime(timefmt, timetpl)[1:-1]: 327 modtime = time.mktime(timetpl) 328 os.utime(fileName, (time.time(), modtime)) 329 Logger.addMessage("<INFO> Timestamp reset to DB value.")
330 331 #------------------------------------------------------------------------------ 332 # Entry point for script. 333 334 # Allow module to be imported as well as executed from the command line 335 if __name__ == '__main__': 336 # Define command-line interface settings for UpdateFitsFileNames 337 CLI.progArgs += [CLI.Argument("xferlog", "xfer.log", True)] 338 CLI.progOpts += [ 339 CLI.Option('U', 'update', 340 "update the timestamp in the database"), 341 CLI.Option('T', 'timestamp', 342 "update the file's timestamp according to value in the DB."), 343 CLI.Option('K', 'fixkey', 344 "update the file's fitskey according to given key,value pair." 345 " If key is one of " + ', '.join(FixFitsFiles.defVals.keys()) + 346 " and value=check, these keys are fixed with default values.", 347 "STR", ''), 348 CLI.Option('J', 'fixjpens', 349 "fix the FITS files and the database entries for JPEs VISTA" 350 " non-survey (no xferlog used)."), 351 CLI.Option('R', 'fixrecal', 352 "fix FITS files that have been recalibrated using CU8 and " 353 " have some missing header keywords")] 354 355 cli = CLI(FixFitsFiles, "$Revision: 10044 $") 356 Logger.addMessage(cli.getProgDetails()) 357 if not cli.getOpt("fixjpens") and not cli.getArg("xferlog") and not cli.getOpt('fixrecal'): 358 Logger.addMessage("Wrong set of options and arguments.") 359 elif cli.getOpt("fixjpens") and \ 360 "VSA" not in cli.getArg("database").partition('.')[2]: 361 Logger.addMessage("JPE's files can only be fixed in a VISTA DB.") 362 else: 363 FixFitsFiles(cli).run() 364 365 #------------------------------------------------------------------------------ 366 # Change log: 367 # 368 # 2-Dec-2005, ETWS: first version; 369 # included restoring of original file timestamp 370 # 10-Jul-2006, RSC: Replaced string module functions with str() equivalents as 371 # the string module with be obsoleted in Python 3.0 372 # 22-Feb-2007, RSC: Updated to reflect move of loadServerHost() constant from 373 # DbConstants to SystemConstants 374 # 04-May-2007, ETWS: Included <date><version> variable to target 375 # files correctly. 376