1
2
3
4 """
5 Creates a basic Multiframe SQL Schema file from all fits files in a given
6 directory.
7
8 @author: E.T.W. Sutorius
9 @org: WFAU, IfA, University of Edinburgh
10 """
11
12 from collections import defaultdict
13 import dircache
14 import inspect
15 import os
16 import math
17 import traceback
18
19 from wsatools.CLI import CLI
20 import wsatools.DbConnect.DbConstants as dbc
21 from wsatools.DbConnect.DbSession import DbSession
22 from wsatools.File import File, PickleFile
23 import wsatools.FitsUtils as fits
24 from wsatools.DbConnect.IngCuSession import IngCuSession
25 from wsatools.Logger import Logger
26 from wsatools.SystemConstants import SystemConstants
27 import wsatools.Utilities as utils
28
29
31 """
32 """
33
34
35
36
37 outDir = "parsed_fits_keys"
38 attrDescriptionFileName = "OSA_FitsKeyDescription.txt"
39 fileTypeLookUp = {"stack":"_st", "sky":"sky_" , "cat":"_cat",
40 "conf":"_conf", "dark":"dark_"}
41 typeDefaults = {
42 'bit': "%u" % dbc.tinyIntDefault(), 'int': "%d" % dbc.intDefault(),
43 'bigint': "%d" % dbc.intDefault(), 'real': "%e" % dbc.realDefault(),
44 'float': "%e" % dbc.floatDefault(),
45 'datetime': repr(dbc.dateTimeDefault()),
46 'binary': "%u" % dbc.tinyIntDefault(),
47 'smallint': "%d" % dbc.smallIntDefault(),
48 'tinyint': "%u" % dbc.tinyIntDefault(),
49 'varchar': repr(dbc.charDefault())}
50
51 nullDefaults = {
52 'bit': "%u" % dbc.tinyIntDefault(), 'int': "%d" % dbc.intDefault(),
53 'bigint': "%d" % dbc.intDefault(), 'real': "%e" % dbc.realDefault(),
54 'float': "%e" % dbc.floatDefault(), 'datetime': "12-31-9999",
55 'binary': "%u" % dbc.tinyIntDefault(),
56 'smallint': "%d" % dbc.smallIntDefault(),
57 'tinyint': "%u" % dbc.tinyIntDefault(), 'varchar': dbc.charDefault()}
58
59
60 omitAttr = ("tfields", "tform", "ttype", "tunit", "symbol", "prov",
61 "checksum", "dataChecksum", "dataMD5", "datasum", "zdatasum",
62 "zhecksum")
63
64 noShow = set(['HISTORY', 'COMMENT'])
65 translatAttr = {"bitpix": "bitsPerPix", "bscale": "dataScale",
66 "bzero": "dataZero", "instrume": "instrument",
67 "ra": "raBase", "dec": "decBase"}
68
69 osaAttrSet = set()
70 esoAttrSet = set()
71
72 allHeaders = {
73 "Schema": [
74 "--" + '='*77,
75 "--",
76 "-- $Id: CreateFitsAttributeList.py 9344 2012-08-03 11:28:27Z EckhardSutorius $", "--",
77 "-- Database schema file for multiframe images, programme and filter",
78 "-- data in the VST Atlas Science Archive. Contains sql scripts to create",
79 "-- the relevant tables. NB: Tables are ordered by foreign key dependencies.", "--",
80 "-- Original author: Nigel Hambly, WFAU, IfA, University of Edinburgh,",
81 "-- based on WSA_MultiframeSchema.sql",
82 "--",
83 "--" + '='*77],
84 "Multiframe": {"top": [
85 "",
86 "CREATE TABLE Multiframe(",
87 "--" + '-'*77,
88 "--/H Contains details of all multiframes stored in the archive.",
89 "--",
90 "--/T Required constraints: primary key is (multiframeID)",
91 "--/T (filterID) references Filter(filterID)",
92 "--" + '-'*77,
93 "multiframeID bigint not null default -99999999, --/D UID of the multiframe (assigned sequentially by the archive ingest process) --/C ID_FRAME",
94 "vstAtlasRunNo int not null default -99999999, --/D Original VST Atlas run number (from filename) --/C REFER_CODE --/Q fileName",
95 "creationDate datetime not null default '31-Dec-9999', --/D File creation date (YYYY-MM-DD HH:MM:SS) --/U MM-DD-YYYY --/C TIME_DATE --/K DATE --/N 12-31-9999",
96 "frameType varchar(64) not null default 'NONE', --/D The type of multiframe (eg. stack|tile|mosaic|difference|calibration|interleaved etc). --/C meta.code.class --/N normal --/I A multiframe can have a combination of different types. --/G Multiframe::frameType",
97 "cuEventID int not null default -99999999, --/D UID of curation event giving rise to this record --/C REFER_CODE",
98 "julianDayNum int not null default -99999999, --/D the Julian Day number of the VST night --/U Julian days --/C TIME_DATE --/Q utDate --/R 2453280,None",
99 "fileTimeStamp bigint not null default -99999999, --/D Time stamp digits (from the original CASU directory name and file time stamp) for enforcing uniqueness --/C ??",
100 "filterID tinyint not null default 0, --/D UID of combined filter (assigned in OSA: 1=Z,2=Y,3=J,4=H,5=Ks) --/C INST_FILTER_CODE --/Q filterName"],
101 "bottom": [
102 "fileName varchar(256) not null default 'NONE', --/D the filename for the multiframe, eg. server:/path/filename.fit --/C ID_FILE --/Q fitsfilename",
103 "catName varchar(256) not null default 'NONE', --/D the filename of the associated catalogue MEF, eg. server:/path/filename.fits --/C ID_CATALOG --/Q fitsfilename",
104 "versionNum real not null default -0.9999995e9, --/D a version number for this frame (if available) --/C VERSNUM --/N -0.9999995e9 --/G Multiframe::versionNum",
105 "-- from Detector ext HDU: more sensible if these are here:",
106 "darkID bigint not null default -99999999, --/D UID of library calibration dark frame --/C ID_FRAME --/N -99999999 --/Q darkID --/K IMAGE.DARKCOR",
107
108 "confID bigint not null default -99999999, --/D UID of library calibration confidence frame --/C ID_FRAME --/N -99999999 --/Q confID --/K IMAGE.CIR_CPM",
109 "flatID bigint not null default -99999999, --/D UID of library calibration flatfield frame --/C ID_FRAME --/N -99999999 --/Q flatID --/K IMAGE.FLATSRC",
110 "frinID bigint not null default -99999999, --/D UID of library calibration fringe frame --/C ID_FRAME --/N -99999999 --/Q frinID --/K IMAGE.DEFRINGE.",
111 "biasID bigint not null default -99999999, --/D UID of library bias frame --/C ID_FRAME --/N -99999999 --/Q biasID --/K IMAGE.BIASSRC",
112
113 "frinScale real not null default -0.9999995e9, --/D Fringe frame scale factor --/C ID_FRAME --/N -0.9999995e9 --/Q frinScale --/K IMAGE.DEFRINGE",
114 "-- end Detector ext HDU",
115 "deprecated smallint not null default 0, --/D Code for a current (=0) or deprecated (!=0) multiframe --/C CODE_MISC --/N 0 --/G allTables::deprecated",
116 "newlyIngested tinyint not null default 0, --/D Curation flag for internal use only (0=no, 1=yes) --/C ?? --/N 1",
117 "skyAlgorithm varchar(64) not null default 'NONE', --/D Sky estimation algorithm --/K SKYALGO --/N NONE",
118 "unfilteredID bigint not null default -99999999, --/D UID of original unfiltered frame corresponding to this filtered frame --/C ID_FRAME --/N -99999999",
119 "--\n-- Additional attributes required to be added by ALTER TABLE statements:\n--\n-- ...\n--\n-- Additional attributes end.\n--",
120 "CONSTRAINT pk_Mul_Fram PRIMARY KEY (multiframeID),",
121 "CONSTRAINT unq_Mul_Fram UNIQUE (filename, fileTimeStamp)",
122 ")",
123 "GO"]},
124 "MultiframeDetector": {"top": [
125 "",
126 "CREATE TABLE MultiframeDetector(",
127 "--" + '-'*77,
128 "--/H Contains details of individual detector frames that are part of a multiframe.",
129 "--",
130 "--/T Required constraints: primary key is (multiframeID,extNum)",
131 "--/T (multiframeID) references Multiframe(multiframeID)",
132 "--" + '-'*77,
133
134 "multiframeID bigint not null default -99999999, --/D the UID of the relevant multiframe --/C ID_FRAME",
135 "extNum tinyint not null default 0, --/D the extension number of this frame --/C NUMBER --/Q fitsfile",
136 "cuEventID int not null default -99999999, --/D UID of curation event giving rise to this record --/C REFER_CODE",
137 "compFile varchar(256) not null default 'NONE', --/D Filename of the compressed image of this image frame, eg. server:/path/filename.jpg --/C ID_FILE --/N NONE",
138 "--julianDayNum int not null default -99999999, --/D the Julian Day number of the VST night --/U Julian days --/C TIME_DATE --/K UTDATE --/Q julianDayNum --/R 2453280,None",
139 "--camNum int not null default -99999999, --/D Number of WFCAM camera (1, 2, 3 or 4) --/C ID_PLATE --/K IMAGE.CAMNUM",
140 "--configIndex int not null default -99999999, --/D Configuration index --/C SPECT_INDEX_MISC --/K IMAGE.CNFINDEX --/N -99999999",
141 "deviceID varchar(32) not null default 'NONE', --/D Device UID identifying every IR device --/C ID_PLATE --/K IMAGE.DETECTID",
142 "filterID tinyint not null default 0, --/D UID of combined filter (assigned in OSA: 1=Z,2=Y,3=J,4=H,5=Ks) --/C INST_FILTER_CODE --/K FILTER --/Q filterID --/V Z,Y,J,H,K,H2,1-0S1,Br,BGamma,Blank,1.205nbJ,NBJ",
143 "runID varchar(256) not null default 'NONE', --/D Name of CASU raw data file --/C ID_FILE --/K IMAGE.RUNID --/N NONE",
144 "creationDate datetime not null default '31-Dec-9999', --/D Creation date/time of file --/C ?? --/U MM-DD-YYYY:hh:mm:ss.sss --/K IMAGE.DATE --/N 12-31-9999"],
145 "bottom": [
146 "CONSTRAINT pk_Det_Fram PRIMARY KEY (multiframeID,extNum)",
147 ")",
148 "GO"]},
149 "MultiframeEsoKeys": {"top": [
150 "",
151 "CREATE TABLE MultiframeEsoKeys(",
152 "--" + '-'*77,
153 "--/H Contains details of all hierarchical ESO attributes for the primary HDU.",
154 "--",
155 "--/T Required constraints: primary key is (multiframeID,extNum)",
156 "--/T (multiframeID) references Multiframe(multiframeID)",
157 "--" + '-'*77,
158
159 "multiframeID bigint not null default -99999999, --/D the UID of the relevant multiframe --/C ID_FRAME",
160 "extNum tinyint not null default 0, --/D the extension number of this frame --/C NUMBER --/Q fitsfile",
161 "cuEventID int not null default -99999999, --/D UID of curation event giving rise to this record --/C REFER_CODE"],
162 "bottom": [
163 "CONSTRAINT pk_Prim_Eso_Fram PRIMARY KEY (multiframeID,extNum)",
164 ")",
165 "GO"]},
166 "MultiframeDetectorEsoKeys": {"top": [
167 "",
168 "CREATE TABLE MultiframeDetectorEsoKeys(",
169 "--" + '-'*77,
170 "--/H Contains details of all hierarchical ESO attributes for secondary HDUs.",
171 "--",
172 "--/T Required constraints: primary key is (multiframeID,extNum)",
173 "--/T (multiframeID) references Multiframe(multiframeID)",
174 "--" + '-'*77,
175
176 "multiframeID bigint not null default -99999999, --/D the UID of the relevant multiframe --/C ID_FRAME",
177 "extNum tinyint not null default 0, --/D the extension number of this frame --/C NUMBER --/Q fitsfile",
178 "cuEventID int not null default -99999999, --/D UID of curation event giving rise to this record --/C REFER_CODE"],
179 "bottom": [
180 "CONSTRAINT pk_Det_Eso_Fram PRIMARY KEY (multiframeID,extNum)",
181 ")",
182 "GO"]}
183 }
184
185
186
197 """
198 @param beginDate: First date to process, eg. 20050101.
199 @type beginDate: str
200 @param comment: Descriptive comment as to why curation task is
201 being performed.
202 @type comment: str
203 @param curator: Name of curator.
204 @type curator: str
205 @param database: Name of the database to connect to.
206 @type database: str
207 @param endDate: Last date to process, eg. 20050131.
208 @type endDate: str
209 @param isTrialRun: If True, do not perform database modifications.
210 @type isTrialRun: bool
211 @param readPickled: If True, read pickled FITS data.
212 @type readPickled: bool
213 @param versionStr: Version number of the data.
214 @type versionStr: str
215 @param writeExampleFits: Write an example FITS file to the
216 attribute line.
217 @type writeExampleFits: str
218
219 """
220
221 super(ExportAttributes, self).__init__(cuNum=9999,
222 curator=curator,
223 comment=comment,
224 reqWorkDir=False,
225 keepWorkDir=False,
226 database=database,
227 autoCommit=False,
228 isTrialRun=isTrialRun)
229
230 beginDate, endDate = \
231 self.sysc.obsCal.getDatesFromInput(beginDate, endDate)
232
233 typeTranslation = {"curator":str,
234 "database":str,
235 "beginDate":str,
236 "endDate":str,
237 "versionStr":str,
238 "readPickled":bool,
239 "isTrialRun":bool,
240 "comment":str}
241
242 super(ExportAttributes, self).attributesFromArguments(
243 inspect.getargspec(ExportAttributes.__init__)[0], locals(),
244 types=typeTranslation)
245
246 if not self.versionStr:
247 self.versionStr = str(max(
248 self.sysc.obsCal.maxVersOfDate(self.beginDate),
249 self.sysc.obsCal.maxVersOfDate(self.endDate)))
250
251 utils.ensureDirExist(ExportAttributes.outDir)
252 self.allFileTypes = set(ExportAttributes.fileTypeLookUp.keys()
253 + ["all", "catalogue"])
254 self.attrDict = defaultdict()
255
256
257
259 """Returns the next power of 2 that is greater than or equal to n"""
260 if n == 0:
261 return 2
262 else:
263 return int(2 ** math.ceil(math.log(n, 2)))
264
265
266
268 if num < 0:
269 return -1
270 elif num > 0:
271 return 1
272 else:
273 return num
274
275
276
290
291
292
294 if typeSizeTuple:
295 maxCheck = typeSizeTuple[1]
296 attrSign = typeSizeTuple[2]
297 else:
298 maxCheck = 0
299 attrSign = self.signum(value)
300 if value < 0:
301 attrSign = -1
302
303 if isinstance(value, (int,float)):
304 attrSize = max(maxCheck, value)
305 if str(abs(value)).isdigit():
306 if abs(value) <= 2**8 and abs(value)==value:
307 attrType = 'tinyint'
308 elif abs(value) <= 2**16:
309 attrType = 'smallint'
310 elif abs(value) <= 2**32:
311 attrType = 'int'
312 else:
313 attrType = 'bigint'
314 else:
315 if abs(value) <= 2**32:
316 attrType = 'real'
317 else:
318 attrType = 'float'
319 elif isinstance(value, bool):
320 attrType = 'bit'
321 attrSize = 1
322 elif ':' in value and 'T' in value and '-' in value \
323 and not ('V' in value or '.fits' in value):
324 attrType = 'datetime'
325 attrSize = 8
326 elif isinstance(value, str):
327 attrSize = self.next_power_of_2(max(maxCheck, len(value)))
328 attrSize = (4 if attrSize < 4 else attrSize)
329 attrType = 'varchar'
330 return attrType, attrSize, attrSign
331
332
333
346
347
348
363
364
365
366 - def createAttrLines(self, extensionDict, table, extNum, attrDescriptions,
367 isCatalogue=False, examples={}):
368
369 esoLines = []
370 attrLines = []
371 for attr in sorted(set(extensionDict[extNum].keys()) -
372 ExportAttributes.noShow):
373 atttype = extensionDict[extNum][attr][1][0]
374 atttypeStr = (
375 "%s(%s)" % (atttype, extensionDict[extNum][attr][1][1])
376 if atttype == "varchar" else atttype)
377 typeDefault = '%s not null default %s, ' % \
378 (atttypeStr, ExportAttributes.typeDefaults[atttype])
379 newAttr = self.reFormat(attr)
380 line = ' '.join(
381 [newAttr, ((15-len(newAttr)) * ' ' if len(newAttr)<14
382 else 3 * ' '), typeDefault])
383
384 if attr.upper() in attrDescriptions[table]:
385 line = ' '.join(
386 [line, "--/D %s" % attrDescriptions[table][attr]])
387 else:
388 line = ' '.join(
389 [line, "--/D %s" % extensionDict[extNum][attr][0]])
390
391
392
393
394 keySource = ("BINTABLE." if isCatalogue else "IMAGE.")
395 tmpAttr = ("HIERARCH " + attr.replace('_',' ')
396 if attr.startswith("ESO") else attr)
397 fitsKey = (keySource+tmpAttr
398 if extNum > 0 or attr.startswith("ESO") else attr)
399
400 line = ' '.join([
401 line,
402 "--/K %s" % fitsKey,
403 "--/N %s" % ExportAttributes.nullDefaults[atttype]])
404
405
406 if self.writeExampleFits:
407 if attr in examples:
408 specType = ("catalogue" if isCatalogue else "all")
409 numExpls = len(examples[attr][specType])
410 line = ' '.join([
411 line, "[(%d)<%s>]" % (
412 numExpls,
413 examples[attr][specType][max(0, int(numExpls/2))])])
414 else:
415 line = ' '.join([line, "[(0)<--->]"])
416
417 self.attrDict["%s"%(attr if extNum == 0 else keySource+attr)] = \
418 atttypeStr
419 if not newAttr.startswith(ExportAttributes.omitAttr):
420 if attr.startswith("ESO"):
421 if newAttr not in ExportAttributes.esoAttrSet:
422 esoLines.append(line)
423 ExportAttributes.esoAttrSet.add(newAttr)
424 else:
425 if newAttr not in ExportAttributes.osaAttrSet:
426 attrLines.append(line)
427 ExportAttributes.osaAttrSet.add(newAttr)
428
429 return attrLines, esoLines
430
431
432
433 - def createSqlFiles(self, imageDict, catalogueDict={}, exampleDict={}):
434 """
435 Write the sql Multiframe schema.
436
437 @param imageDict: Dictionary of extension number and attributes,
438 types, and descriptions for image files.
439 Dict[i][attr] = [descr, (atttype, attsize)]
440 @type imageDict: dict
441 @param catalogueDict: Dictionary of extension number and atttributes,
442 types, and descriptions for catalogue files.
443 Dict[i][attr] = [descr, (atttype, attsize)]
444 @type catalogueDict: dict
445 @param exampleDict: Dictionary of attributes and associated filenames.
446 Dict[attr] = filePath
447 @type exampleDict: dict
448
449
450 """
451 self.sqlFile.wopen()
452
453 self.sqlFile.writelines(ExportAttributes.allHeaders["Schema"])
454
455
456 attrDescriptions = self.readAuxFiles()
457 esoTableLines = defaultdict()
458 for extNum, table in enumerate(["Multiframe", "MultiframeDetector"]):
459 self.sqlFile.writelines(ExportAttributes.allHeaders[table]["top"])
460 if extNum == 0:
461 self.sqlFile.writetheline(
462 "--\n-- primary HDU FITS keys go here:\n--")
463 else:
464 self.sqlFile.writetheline(
465 "--\n-- Image extension HDU FITS keys:\n--")
466
467 attrLines, esoLines = \
468 self.createAttrLines(imageDict, table, extNum, attrDescriptions,
469 examples=exampleDict)
470
471 self.sqlFile.writelines(attrLines)
472 if extNum == 0:
473 esoTableLines["MultiframeEsoKeys"] = [
474 "--\n-- Primary Header ESO FITS keys\n--"]
475 esoTableLines["MultiframeEsoKeys"].extend(esoLines)
476 else:
477 esoTableLines["MultiframeDetectorEsoKeys"] = [
478 "--\n-- Secondary Header ESO FITS keys\n--"]
479 esoTableLines["MultiframeDetectorEsoKeys"].extend(esoLines)
480
481
482
483 if extNum == 0:
484 self.sqlFile.writetheline("--\n-- end of PHDU FITS keys.\n--")
485
486 else:
487 if catalogueDict:
488 attrLines, esoLines = self.createAttrLines(catalogueDict,
489 table, extNum, attrDescriptions, True,
490 examples=exampleDict)
491
492 self.sqlFile.writetheline(
493 "--\n-- Binary table extension keywords\n--")
494 self.sqlFile.writelines(attrLines)
495 self.sqlFile.writetheline(
496 "--\n-- Additional attributes required to be added by ALTER TABLE statements:\n--")
497 esoTableLines["MultiframeDetectorEsoKeys"].extend([
498 "--\n-- Catalogue Secondary Header ESO FITS keys\n--"])
499 esoTableLines["MultiframeDetectorEsoKeys"].extend(esoLines)
500
501 self.sqlFile.writelines(
502 ExportAttributes.allHeaders[table]["bottom"])
503
504 for table in ["MultiframeEsoKeys", "MultiframeDetectorEsoKeys"]:
505 self.sqlFile.writelines(ExportAttributes.allHeaders[table]["top"])
506 self.sqlFile.writelines(esoTableLines[table])
507 self.sqlFile.writelines(
508 ExportAttributes.allHeaders[table]["bottom"])
509
510
511
512
513
514
515
516 self.sqlFile.close()
517
518
519
521
522
523 wsaSchemaFile = File(os.path.join(self.sysc.sqlScriptPath,
524 'WSA_MultiframeSchema.sql'))
525 wsaSchemaFile.ropen()
526 lines = wsaSchemaFile.readlines(commentChar='-')
527 wsaSchemaFile.close()
528 self.wsaAttrDict = defaultdict()
529 for line in lines:
530 if line and "not null" in line:
531 words = line.split()
532 self.wsaAttrDict[words[0].lower().replace('_','')] = words[:2]
533
534
535 fitsDirs = fits.FitsList(self.sysc)
536 fitsDirs.createFitsDateDict(self.sysc.availableRaidFileSystem())
537 fitsList = []
538 fileDict = defaultdict()
539 exampleDict = defaultdict(dict)
540
541 beginSem = self.sysc.obsCal.checkDate(self.beginDate)
542 dateList = self.sysc.obsCal.getSemDayList(beginSem)
543 endSem = self.sysc.obsCal.checkDate(self.endDate)
544 dateList.extend(self.sysc.obsCal.getSemDayList(endSem))
545
546
547 dateList = sorted(set(dateList))[dateList.index(self.beginDate):
548 dateList.index(self.endDate)+1]
549 for dateStr in dateList:
550 dateVersStr = '_v'.join([dateStr, self.versionStr])
551 fitsList.extend([
552 os.path.join(fitsDirs.invFitsDateDict[dateVersStr],
553 dateVersStr, x)
554 for x in dircache.listdir(os.path.join(
555 fitsDirs.invFitsDateDict[dateVersStr], dateVersStr))
556 if x.endswith((self.sysc.mefType, self.sysc.catType))])
557
558 for filePath in fitsList:
559 fileName = os.path.basename(filePath)
560 fileType = self.getFileType(fileName)
561 fileDict[fileName] = fileType
562
563
564 ddFile = PickleFile(os.path.join(self.outDir, "dataDict.pkl"))
565 edFile = PickleFile(os.path.join(self.outDir, "exampleDict.pkl"))
566 if self.readPickled:
567 dataDict = list(ddFile.pickleRead())[0]
568 exampleDict = list(edFile.pickleRead())[0]
569 else:
570 dataDict, exampleDict = self.readFitsFiles(fitsList, fileDict)
571
572 ddFile.pickleWrite(dataDict)
573 edFile.pickleWrite(exampleDict)
574
575 expAttr.createSqlFiles(dataDict["all"], dataDict["catalogue"],
576 exampleDict)
577 timestamp = utils.makeTimeStamp().replace(' ','_')
578 Logger.dump(file("CFAL_%s.log" % timestamp, 'w'))
579
580
581
582
584 """Read the FITS files' metadata into a dictionary.
585 """
586 dataDict = defaultdict(dict)
587 for fileType in self.allFileTypes:
588 for i in range(0,2):
589 dataDict[fileType][i] = defaultdict(list)
590 exampleDict = defaultdict(dict)
591
592 for filePath in fitsList:
593 fileDir, fileName = os.path.split(filePath)
594 fileType = fileDict[fileName]
595 dateVersStr = os.path.basename(fileDir)
596 try:
597 print ":: %s :: %s :::: %s" % (dateVersStr, fileName, fileType)
598 for i,hdu in enumerate(fits.open(filePath)):
599 if i in (0,1):
600 cardList = hdu.header.ascardlist()
601 for card in cardList:
602 attr = card.key.replace(' ', '_')
603 descr = card.comment
604
605
606
607
608
609
610
611 if attr:
612 typeSizeSign = (
613 dataDict[fileType][i][attr][1]
614 if attr in dataDict[fileType][i]
615 else None)
616 atttype, attsize, attsign = self.getAttrType(
617 card.value, typeSizeSign)
618 dataDict[fileType][i][attr] = \
619 [descr, (atttype, attsize, attsign)]
620 if "cat" not in fileType:
621 specType = "all"
622 else:
623 specType = "catalogue"
624
625 typeSizeSign = (
626 dataDict[specType][i][attr][1]
627 if attr in dataDict[specType][i]
628 else None)
629 atttype, attsize, attsign = self.getAttrType(
630 card.value, typeSizeSign)
631 dataDict[specType][i][attr] = \
632 [descr, (atttype, attsize, attsign)]
633 if specType not in exampleDict[attr]:
634 exampleDict[attr][specType] = [filePath]
635 else:
636 exampleDict[attr][specType].append(filePath)
637 except Exception as error:
638 eType = filePath
639 Logger.addMessage(repr(traceback.format_exc()))
640 Logger.addExceptionMessage(error, eType)
641
642 return dataDict, exampleDict
643
644
645
647 attrKeyDict = defaultdict(dict)
648 if file.exists():
649 file.ropen()
650 lines = file.readlines(commentChar=('#','--', "CONSTRAINT"),
651 omitEmptyLines=True)
652 file.close()
653 for l in lines:
654 if l.startswith("CREATE TABLE"):
655 tableName = l.rsplit(None, 1)[1][:-1]
656 try:
657 _attrName, attrType, text = l.split(None, 2)
658 fitsKey = text.split(None, 2)[2].partition(
659 ',')[2].strip().rpartition('--/K')[2].strip().partition(
660 '--/')[0].strip()
661 if _attrName == "dribble":
662 print _attrName, attrType,fitsKey
663 except (IndexError, ValueError):
664 attrType, text = None, None
665 else:
666
667
668 if tableName in ["CurrentAstrometry", "ProgrammeFrame"]:
669 tableName = "MultiframeDetector"
670 if len(fitsKey) > 0:
671 attrKeyDict[tableName][fitsKey.rpartition('.')[2]] = (
672 fitsKey.rpartition('.')[0],attrType)
673 return attrKeyDict
674
675
676
678 newKeyDict = self.readAttr(self.sqlFile)
679 dbSqlFile = File(os.path.join(self.sysc.sqlScriptPath,
680 self.sqlFile.base))
681 oldKeyDict = self.readAttr(dbSqlFile)
682 outFile = File(os.path.join(self.outDir, "keycheck"))
683 outFile.wopen()
684 for tableName in sorted(newKeyDict):
685 outFile.writetheline("\n %s => %s" % (dbSqlFile.name, self.sqlFile.name))
686 outFile.writetheline("\n %s Fits Keys: old type => new type" % tableName)
687 outFile.writetheline('=' * (34 + len(tableName)))
688 for key in sorted(set(oldKeyDict[tableName].keys() \
689 + newKeyDict[tableName].keys())):
690 fmt = '{0:45}: {1:20} => {2}'
691 try:
692 if key == "DRIBBLE":
693 print key,oldKeyDict[tableName][key],newKeyDict[tableName][key]
694 if oldKeyDict[tableName][key] != newKeyDict[tableName][key]:
695 outFile.writetheline(
696 fmt.format(key, oldKeyDict[tableName][key],
697 newKeyDict[tableName][key]))
698 except KeyError:
699 if key in oldKeyDict[tableName] \
700 and key not in newKeyDict[tableName]:
701 outFile.writetheline(
702 fmt.format(key, oldKeyDict[tableName][key],
703 "?????"))
704 elif key not in oldKeyDict[tableName] \
705 and key in newKeyDict[tableName]:
706 outFile.writetheline(
707 fmt.format(key, "?????",
708 newKeyDict[tableName][key]))
709 else:
710 outFile.writetheline(fmt.format(key, "?????", "?????"))
711
712
713
714
715
716 if __name__ == "__main__":
717
718 CLI.progOpts += [
719 CLI.Option('b', "begin", "begin date to process, eg. 20110816",
720 "DATE", '20110816'),
721 CLI.Option('e', "end", "end date to process, eg. 20110816",
722 "DATE", '20111231'),
723 CLI.Option('o', "outdir", "output directory for parsed files",
724 "DIR", ExportAttributes.outDir),
725 CLI.Option('v', "version", "version number of the data",
726 "STR", ''),
727 CLI.Option('C', "compare", "compare derived with existing schema"),
728 CLI.Option('E', "examples", "add example fits file names to the "
729 "attribute line"),
730 CLI.Option('P', "pickleddata", "read FITS data from pickled data files")
731 ]
732
733 cli = CLI("CreateFitsAttributeList", "$Revision: 9344 $", __doc__)
734 Logger.isVerbose = False
735 Logger.addMessage(cli.getProgDetails())
736
737 compareSchemas = cli.getOpt('compare')
738 expAttr = ExportAttributes(cli.getOpt("curator"),
739 cli.getArg("database"),
740 cli.getOpt("begin"),
741 cli.getOpt("end"),
742 cli.getOpt("version"),
743 cli.getOpt("pickleddata"),
744 cli.getOpt("examples"),
745 cli.getOpt("test"),
746 cli.getArg("comment"))
747
748 expAttr.outDir = cli.getOpt('outdir')
749 expAttr.sqlFile = File(os.path.join(expAttr.outDir,
750 "OSA_MultiframeSchema.sql"))
751 if compareSchemas and expAttr.sqlFile.exists():
752 expAttr.compareSchemaFiles()
753 else:
754 expAttr.run()
755
756
757