1
2
3
4 """
5 Script to fix problematic FITS files:
6 1. Reads multiframeIDs from the database for a list of files which may've
7 been ingested before and write them into the files.
8 2. Fixes any (attribute, value) pair for a list of FITS files.
9 3. Fixes FITS files and DB entries for JPEs VISTA non-survey.
10
11 @author: E.T.W. Sutorius
12 @org: WFAU, IfA, University of Edinburgh
13 """
14
15 from collections import defaultdict
16 import mx.ODBC.unixODBC as odbc
17 import os
18 import string
19 import time
20
21 from wsatools.CLI import CLI
22 from wsatools.DbConnect.DbSession import DbSession
23 from wsatools.File import File
24 import wsatools.FitsUtils as fits
25 from wsatools.Logger import Logger
26
27
29 """Script to fix multiframe IDs in FITS files with data from the database.
30 """
31 defVals = {"DETECTID": {2:"RSC:H2:60", 3:"RSC:H2:63", 4:"RSC:H2:76",
32 5:"RSC:H2:41",
33 "comment": "Serial number of detector array"},
34 "DCOLUMNS": {2:2048, 3:2048, 4:2048, 5:2048,
35 "comment":"[pixel] Number of detector columns"},
36 "CAMNUM": {2:1, 3:2, 4:3, 5:4,
37 "comment":"Number of WFCAM camera (1, 2, 3 or 4)"}}
38
40 """
41 Initialise class from command-line options.
42
43 @param cli: Command-line arguments.
44 @type cli: CLI
45
46 """
47 self.updateTime = cli.getOpt("update")
48 self.updateFileTime = cli.getOpt("timestamp")
49 if cli.getOpt("fixkey"):
50 self.updateKey, self.updateValue = \
51 cli.getOpt("fixkey").partition(',')[::2]
52 if not self.updateValue:
53 Logger.addMessage("<ERROR> No key/value given!")
54 raise SystemExit
55 else:
56 self.updateKey = self.updateValue = None
57
58 self.fixJPEns = cli.getOpt("fixjpens")
59 self.fixrecal = cli.getOpt("fixrecal")
60 if self.fixJPEns:
61 self.updateKey, self.updateValue = [
62 "HIERARCH ESO OBS PROG ID", "284.C-5034(A)"]
63 self.logFileName = cli.getArg("xferlog")
64 self.archive = DbSession(cli=cli, autoCommit=True)
65
66
67
83
84
85
87 """Create a dictionary from the given list of files"""
88 fileDateDict = defaultdict(list)
89 for fileName in fileList:
90 fitsFile = File(fileName)
91 fileDateDict[fitsFile.subdir].append(fitsFile.name)
92 del fitsFile
93 return fileDateDict
94
95
96
102
103
104
106 """Update tables for JPE's non-survey."""
107
108 numRows = 0
109 for fileName in mfIDDict:
110 numRows += self.archive.update(
111 "Multiframe",
112 "project='%s'" % self.updateValue,
113 "multiframeID=%d" % mfIDDict[fileName][0])
114 Logger.addMessage("%d rows updated in Multiframe." % numRows)
115
116
117 numRows = 0
118 for fileName in mfIDDict:
119 numRows += self.archive.update(
120 "MultiframeEsoKeys",
121 "obsProgID='%s'" % self.updateValue,
122 "multiframeID=%d" % mfIDDict[fileName][0])
123 Logger.addMessage("%d rows updated in MultiframeEsoKeys." % numRows)
124
125
126 numRows = 0
127 progID = self.archive.query("programmeID", "Programme",
128 "dfsIDString='%s'" % self.updateValue)[0]
129 mfIDList = []
130 for fileName in mfIDDict:
131 numRows += self.archive.update(
132 "ProgrammeFrame",
133 "programmeID=%d" % progID,
134 "multiframeID=%d" % mfIDDict[fileName][0])
135 mfIDList.append(mfIDDict[fileName][0])
136
137 Logger.addMessage("%d rows updated in ProgrammeFrame." % numRows)
138
139
140 identity = string.maketrans('', '')
141 prefix = self.updateValue.translate(identity, ".-()").lower()
142 for suffix in ["Raw", "Photometry", "Astrometry"]:
143 Logger.addMessage("Copying from calDetection%s into d%sDetection%s"
144 " ..." % (suffix, prefix, suffix))
145 numRowsCopied = 0
146 numRowsDeleted = 0
147 for fileName in mfIDDict:
148 numRows = 0
149 try:
150 numRows = self.archive.copyIntoTable(
151 destinationTable="d%sDetection%s" % (prefix, suffix),
152 sourceTable="calDetection%s" % suffix,
153 columns='*',
154 where="multiframeID=%d" % mfIDDict[fileName][0])
155 srcTable = "calDetection%s" % suffix
156 if numRows == 0 and \
157 self.archive.sysc.tileSuffix in fileName and \
158 not self.archive.sysc.confSuffix in fileName:
159 numRows = self.archive.copyIntoTable(
160 destinationTable="d%sDetection%s" % (prefix, suffix),
161 sourceTable="maintDetection%s" % suffix,
162 columns='*',
163 where="multiframeID=%d" % mfIDDict[fileName][0])
164 srcTable = "maintDetection%s" % suffix
165 if numRows == 0:
166 Logger.addMessage("No data available for %d: %s" % (
167 mfIDDict[fileName][0], fileName))
168 else:
169 numRowsCopied += numRows
170 except odbc.IntegrityError:
171 Logger.addMessage("Data already exists in d%sDetection%s "
172 "for %d" % (prefix, suffix, mfIDDict[fileName][0]))
173 except:
174 print fileName, mfIDDict[fileName][0]
175 raise
176 else:
177 if numRows > 0:
178 numRowsDeleted += self.archive.delete(
179 srcTable,
180 whereStr="multiframeID=%d" % mfIDDict[fileName][0])
181
182 Logger.addMessage("%d rows copied into d%sDetection%s" % (
183 numRowsCopied, prefix, suffix))
184 Logger.addMessage("%d rows deleted from calDetection%s" % (
185 numRowsDeleted, suffix))
186
187
188
190 """
191 """
192 mfIDDict = defaultdict()
193 if self.fixJPEns:
194 for fileName, mfID in self.archive.query(
195 "fileName, multiframeID", "Multiframe",
196 " AND ".join(["raBase BETWEEN 10.25 and 11",
197 "decBase BETWEEN -61 and -58",
198 "vistaRunNo BETWEEN 137 and 880",
199 "utDate = '2010-02-03'"])):
200
201 mfIDDict[fileName] = (mfID, 0)
202
203 fileList = mfIDDict.keys()
204
205 else:
206 logFile = File(self.logFileName)
207 logFile.ropen()
208 fileList = logFile.readlines(commentChar='#')
209 logFile.close()
210
211
212 fileDateDict = self.createFileDateDict(fileList)
213
214 if self.fixJPEns:
215
216 self.updateJPEns(mfIDDict)
217 else:
218
219 if self.updateKey:
220 for dateVersStr in sorted(fileDateDict):
221 for fileName in fileDateDict[dateVersStr]:
222 mfIDDict[fileName] = (0, 0)
223 else:
224 mfIDDict = self.readMfIDs(fileDateDict.keys())
225
226 num = 0
227 timefmt = "'%y%m%d%H%M%S'"
228 for dateVersStr in sorted(fileDateDict):
229 for fileName in fileDateDict[dateVersStr]:
230 if mfIDDict.has_key(fileName):
231 num += 1
232
233 dirName = os.path.split(os.path.dirname(fileName))[1][2:8]
234 modtime = time.strptime(time.ctime(os.stat(
235 fileName).st_mtime))
236 wsaTimestamp = dirName + time.strftime(timefmt, modtime)[1:-1]
237
238 mfID, dbTimeStamp = mfIDDict[fileName]
239 Logger.addMessage("#%4d %s: %d, %s" % (num, fileName, mfID,
240 wsaTimestamp))
241
242 if self.updateKey:
243 if not self.archive.isTrialRun:
244
245 hduList = fits.open(fileName, "update")
246 for i, hdu in enumerate(hduList):
247 try:
248 if self.updateValue == "check":
249 if i > 0 and self.updateKey in FixFitsFiles.defVals and not hdu.header.has_key(self.updateKey):
250 updateValue = FixFitsFiles.defVals[
251 self.updateKey][i + 1]
252 updateComment = FixFitsFiles.defVals[
253 self.updateKey]["comment"]
254 Logger.addMessage(
255 "Updating %s in ext %d with '%s'" %
256 (self.updateKey, i, updateValue))
257 fits.writeToFitsHdu(
258 hduList, (i,), self.updateKey,
259 updateValue, updateComment,
260 redoing=True)
261 else:
262 card = hdu.header.ascardlist()[
263 self.updateKey]
264 if card.value != self.updateValue:
265 Logger.addMessage(
266 "Updating %s in ext %d with '%s'" %
267 (card.key, i, self.updateValue))
268 fits.writeToFitsHdu(
269 hduList, (i,), self.updateKey,
270 self.updateValue, card.comment,
271 redoing=True)
272 else:
273 Logger.addMessage(
274 "%s in ext %d already has value '%s'" %
275 (card.key, i, self.updateValue))
276 except KeyError:
277 Logger.addMessage(
278 "%s not found in ext %d" %
279 (self.updateKey, i))
280
281
282 hduList.close()
283 else:
284 Logger.addMessage("TestRun, otherwise update %s "
285 "here." % self.updateKey)
286
287 elif int(wsaTimestamp) != int(dbTimeStamp):
288 Logger.addMessage(
289 "<WARNING> Timestamps differ: file: %s <=>db: %s" % \
290 (wsaTimestamp, dbTimeStamp))
291 if self.updateTime:
292 self.updateTimeStamp(mfID, wsaTimestamp)
293 else:
294 if not self.archive.isTrialRun:
295
296 hduList = fits.open(fileName, "update")
297
298 extNum = (1 if "_tl" in fileName else 0)
299 fits.writeToFitsHdu(
300 hduList, (extNum,),
301 "%s_TIME" % self.archive.sysc.loadDatabase,
302 wsaTimestamp, "%s time stamp" % \
303 self.archive.sysc.loadDatabase, redoing=True)
304
305
306 fits.writeToFitsHdu(
307 hduList, (extNum,),
308 "%s_MFID" % self.archive.sysc.loadDatabase,
309 mfID, "%s Multiframe ID" % \
310 self.archive.sysc.loadDatabase, redoing=True)
311
312
313 hduList.close()
314
315
316
317 if self.updateFileTime:
318 timetpl = time.strptime(str(dbTimeStamp)[-12:],
319 '%y%m%d%H%M%S')
320 else:
321 timetpl = time.strptime(wsaTimestamp[len(dirName):],
322 '%y%m%d%H%M%S')
323 filetime = time.strptime(time.ctime(os.stat(
324 fileName).st_mtime))
325 fileTimestamp = time.strftime(timefmt, filetime)[1:-1]
326 if fileTimestamp != time.strftime(timefmt, timetpl)[1:-1]:
327 modtime = time.mktime(timetpl)
328 os.utime(fileName, (time.time(), modtime))
329 Logger.addMessage("<INFO> Timestamp reset to DB value.")
330
331
332
333
334
335 if __name__ == '__main__':
336
337 CLI.progArgs += [CLI.Argument("xferlog", "xfer.log", True)]
338 CLI.progOpts += [
339 CLI.Option('U', 'update',
340 "update the timestamp in the database"),
341 CLI.Option('T', 'timestamp',
342 "update the file's timestamp according to value in the DB."),
343 CLI.Option('K', 'fixkey',
344 "update the file's fitskey according to given key,value pair."
345 " If key is one of " + ', '.join(FixFitsFiles.defVals.keys()) +
346 " and value=check, these keys are fixed with default values.",
347 "STR", ''),
348 CLI.Option('J', 'fixjpens',
349 "fix the FITS files and the database entries for JPEs VISTA"
350 " non-survey (no xferlog used)."),
351 CLI.Option('R', 'fixrecal',
352 "fix FITS files that have been recalibrated using CU8 and "
353 " have some missing header keywords")]
354
355 cli = CLI(FixFitsFiles, "$Revision: 10044 $")
356 Logger.addMessage(cli.getProgDetails())
357 if not cli.getOpt("fixjpens") and not cli.getArg("xferlog") and not cli.getOpt('fixrecal'):
358 Logger.addMessage("Wrong set of options and arguments.")
359 elif cli.getOpt("fixjpens") and \
360 "VSA" not in cli.getArg("database").partition('.')[2]:
361 Logger.addMessage("JPE's files can only be fixed in a VISTA DB.")
362 else:
363 FixFitsFiles(cli).run()
364
365
366
367
368
369
370
371
372
373
374
375
376