1
2
3
4 """
5 Compares the header contents of the given FITS catalogue file (and the
6 corresponding image file) to the archive schema to spot new keywords.
7
8 @author: R.S. Collins
9 @org: WFAU, IfA, University of Edinburgh
10 """
11
12 from wsatools.CLI import CLI
13 import wsatools.DbConnect.Schema as schema
14 import wsatools.FitsUtils as fits
15 from wsatools.Logger import Logger
16 from wsatools.SystemConstants import SystemConstants
17
18
19
20 if __name__ == '__main__':
21 CLI.progArgs.append(CLI.Argument('cat_file',
22 '/disk04/wsa/ingest/fits/20061223_v1/w20061223_01713_st_cat.fits',
23 isValOK=CLI.assertFileExists))
24 CLI.progOpts.append(CLI.Option('i', 'ignore_dupes',
25 "don't report on keywords duplicated between catalogue and image"))
26 CLI.progOpts.append(CLI.Option('s', 'strict',
27 'consider keys duplicated between primary and image extension headers'))
28 CLI.progOpts.append(CLI.Option('e', 'esokeys',
29 'include ESO keys and tables'))
30 CLI.progOpts.append(CLI.Option('m', 'missing',
31 'print only keys missing from schema'))
32
33 cli = CLI("FitsSchemaCheck", "$Revision: 7245 $", __doc__)
34 Logger.setEchoOn()
35 Logger.addMessage(cli.getProgDetails())
36
37 try:
38 camera = fits.open(cli.getArg('cat_file'))[0].header["INSTRUME"]
39 except KeyError:
40 try:
41 camera = fits.open(cli.getArg('cat_file'))[1].header["INSTRUME"]
42 except KeyError:
43 raise SystemExit("Unrecognised FITS file")
44
45 sysc = SystemConstants(camera)
46
47 if not any([sysc.isVSA(), sysc.isWSA()]):
48 raise SystemExit("Unrecognised FITS file")
49
50 tableList = ["Multiframe", "MultiframeDetector",
51 "CurrentAstrometry", "ProgrammeFrame"]
52 if cli.getOpt('esokeys'):
53 tableList.extend(["MultiframeEsoKeys", "MultiframeDetectorEsoKeys"])
54 Logger.addMessage("Tables used: %s" % ', '.join(tableList))
55
56 tSchema = schema.parseTables(sysc.metadataSchema(), tableList)
57
58
59
60
61 fitsStdKeys = [(0, "HISTORY"), (0, "COMMENT"), (0, "EXTEND"), (0, "SIMPLE"),
62 (1, "HISTORY"), (1, "COMMENT"), (1, "EXTEND"), (1, "SIMPLE")]
63 qTagKeys = [(0, "UTDATE"), (0, "MJD-OBS"), (0, "PROJECT"), (0, "FILTER"),
64 (0, "WSA_MFID"), (0, "WSA_TIME"), (0, "OBSNUM"), (1, "FLATCOR"),
65 (1, "CIR_CPM"), (1, "SKYSUB"), (1, "XTENSION"), (1, "EXTNAME"),
66 (1, "ZNAXIS"), (1, "ZNAXIS1"), (1, "ZNAXIS2"), (1, "NAXIS"),
67 (1, "NAXIS1"), (1, "NAXIS2"), (1, "CRVAL1"), (1, "CRVAL2"),
68 (1, "CD1_1"), (1, "CD1_2"), (1, "CD2_1"), (1, "CD2_2"),
69 (1, "CAMNUM"), (1, "AMSTART"), (1, "AMEND"), (1, "MAGZPT"),
70 (1, "MAGZRR"), (1, "EXTINCT"),
71 (1, ("EXP_TIME" if sysc.isWSA() else "EXPTIME")),
72 (1, "CRVAL1"), (1, "CRVAL2"),(0, "VSA_MFID"), (0, "VSA_TIME")]
73 otherKeys = [(1, "ZIMAGE"), (1, "PROJP3"), (1, "ZTILE1"), (1, "ZVAL1"),
74 (1, "ZCMPTYPE"), (1, "ZBITPIX"), (1, "PROJP1"), (1, "ZNAME1"),
75 (1, "ZTILE2"), (1, "ZNAME2"), (1, "ZSIMPLE"), (1, "ZVAL2"),
76 (1, "ZEXTEND")]
77
78
79 knownKeys = set(fitsStdKeys + qTagKeys + otherKeys)
80
81 if not cli.getOpt('strict'):
82 knownKeys.update(set((ext ^ 1, key) for ext, key in knownKeys))
83
84
85
86 isTile = (True if "_tl" in cli.getArg('cat_file') else False)
87 schemaImgKeys = set((attr.fitsHDU, attr.fitsKeyword.replace('HIERARCH ',''))
88 for table in tSchema
89 for attr in table.columns
90 if attr.fitsKeyword and not attr.isCatFile)
91 schemaCatKeys = set((attr.fitsHDU, attr.fitsKeyword.replace('HIERARCH ',''))
92 for table in tSchema
93 for attr in table.columns
94 if attr.fitsKeyword and attr.isCatFile)
95
96
97 imgFile = fits.open(cli.getArg('cat_file').replace('_cat.fits', '.fit'))
98 imgFile.close()
99 fitsImgKeys = set()
100 for ext in xrange(2):
101 fitsImgKeys.update(set((ext, key) for key, value
102 in imgFile[ext].header.items()
103 if not key.startswith('PROV') and
104 not key.startswith('SYMBOL') and
105 not key.startswith('TTYPE') and
106 not key.startswith('TUNIT') and
107 not key.startswith('TFORM') and
108 (not (key.startswith('ESO') or
109 key.startswith('HIERARCH'))
110 if not cli.getOpt("esokeys")
111 else True)))
112
113 catFile = fits.open(cli.getArg('cat_file'))
114 catFile.close()
115 fitsCatKeys = set()
116 for ext in xrange(2):
117 fitsCatKeys.update(set((ext, key) for key, value
118 in catFile[ext].header.items()
119 if not key.startswith('PROV') and
120 not key.startswith('SYMBOL') and
121 not key.startswith('TTYPE') and
122 not key.startswith('TUNIT') and
123 not key.startswith('TFORM') and
124 (not (key.startswith('ESO') or
125 key.startswith('HIERARCH'))
126 if not cli.getOpt("esokeys")
127 else True)))
128
129 if cli.getOpt("missing"):
130 allImgKeys = set(key for ext, key in (fitsImgKeys - knownKeys))
131
132 allSchemaKeys = set(key for ext, key in
133 schemaImgKeys.union(schemaCatKeys).difference(knownKeys))
134 notInFitsKeys = allSchemaKeys.difference(allImgKeys)
135 notinSchemaKeys = allImgKeys.difference(allSchemaKeys)
136 Logger.addMessage("\n%s keys in Schema but not in FITS file: %s\n" %
137 (len(notInFitsKeys),
138 ', '.join(sorted(notInFitsKeys))))
139 Logger.addMessage("\n%s keys in FITS file but not in Schema: %s\n" %
140 (len(notinSchemaKeys),
141 ', '.join(sorted(notinSchemaKeys))))
142
143
144 extImgKeys = fitsImgKeys - knownKeys - schemaImgKeys - schemaCatKeys
145 unuImgKeys = fitsImgKeys - knownKeys - schemaImgKeys - extImgKeys
146 if not cli.getOpt('strict'):
147 extImgKeys -= set((ext ^ 1, key) for ext, key in schemaImgKeys)
148 pImgKeys = [key for ext, key in extImgKeys if ext is 0]
149 eImgKeys = [key for ext, key in extImgKeys if ext is 1]
150
151 if cli.getOpt('ignore_dupes'):
152 dupeStr = ''
153 else:
154 dupeStr = "\n%s keys not used in image, but taken from catalogue:\n%s"\
155 % (len(unuImgKeys), ', '.join("%s[%s]" % (key, ext)
156 for ext, key in unuImgKeys))
157
158 if not cli.getOpt("missing"):
159 Logger.addMessage("Unused FITS image keywords:\n"
160 "\n%s primary header keys:\n%s\n"
161 "\n%s extension header keys:\n%s%s\n" %
162 (len(pImgKeys), ', '.join(pImgKeys), len(eImgKeys),
163 ', '.join(eImgKeys), dupeStr))
164
165 extCatKeys = fitsCatKeys - knownKeys - schemaCatKeys - schemaImgKeys
166
167
168 if not cli.getOpt('strict'):
169 extCatKeys -= set((ext ^ 1, key) for ext, key in schemaImgKeys)
170 unuCatKeys = fitsCatKeys - knownKeys - schemaCatKeys - extCatKeys
171 pCatKeys = [key for ext, key in extCatKeys if ext is 0]
172 eCatKeys = [key for ext, key in extCatKeys if ext is 1]
173
174 if cli.getOpt('ignore_dupes'):
175 dupeStr = ''
176 else:
177 dupeStr = "\n%s keys not used in catalogue, but taken from image:\n%s"\
178 % (len(unuCatKeys), ', '.join("%s[%s]" % (key, ext)
179 for ext, key in sorted(unuCatKeys)))
180
181 if not cli.getOpt("missing"):
182 Logger.addMessage("Unused FITS catalogue keywords:\n"
183 "\n%s primary header keys:\n%s\n"
184 "\n%s extension header keys:\n%s%s\n" %
185 (len(pCatKeys), ', '.join(sorted(pCatKeys)),
186 len(eCatKeys), ', '.join(sorted(eCatKeys)), dupeStr))
187
188 oldImgKeys = schemaImgKeys - fitsImgKeys
189 pOldImgKeys = [key for ext, key in oldImgKeys if ext is 0]
190 eOldImgKeys = [key for ext, key in oldImgKeys if ext is 1]
191
192 if not cli.getOpt("missing"):
193 Logger.addMessage("Keywords defined in the schema that don't exist in the "
194 "image FITS file:\n\n%s in the primary header:\n%s\n"
195 "\n%s in the extension header:\n%s\n"
196 % (len(pOldImgKeys), ', '.join(sorted(pOldImgKeys)),
197 len(eOldImgKeys), ', '.join(sorted(eOldImgKeys))))
198
199 oldCatKeys = schemaCatKeys - fitsCatKeys
200 pOldCatKeys = [key for ext, key in oldCatKeys if ext is 0]
201 eOldCatKeys = [key for ext, key in oldCatKeys if ext is 1]
202
203 if not cli.getOpt("missing"):
204 Logger.addMessage("Keywords defined in the schema that don't exist in the "
205 "catalogue FITS file:\n\n%s in the primary header:\n%s\n"
206 "\n%s in the extension header:\n%s\n"
207 % (len(pOldCatKeys), ', '.join(sorted(pOldCatKeys)),
208 len(eOldCatKeys), ', '.join(sorted(eOldCatKeys))))
209
210
211
212
213
214