1
2
3
4 """
5 Checks if the number of detections in the database correspond to the number
6 of detections in the catalogue files.
7
8 @author: E. Sutorius
9 @org: WFAU, IfA, University of Edinburgh
10 """
11
12 from collections import defaultdict
13 import getopt
14 import sys
15
16 from wsatools.DataFactory import ProgrammeTable
17 from wsatools.DbConnect.DbSession import DbSession
18
19
21 print("""Usage:
22 CheckDetections.py [-c/--compare detectionSuffixes] [-d/--date dateStr]
23 [-D/--deprecated] [-p/--programme programmeList]
24 [-s/--sum] Adds up all counts of one file.
25
26 Check if the number of detections in the database correspond to the number of
27 detections in the catalogue files.
28
29 -d/--date checks only the given date.
30 -c/--compare compares the given detection split tables,
31 eg. 'Raw,Photometry'.
32 -D/--deprecated checks also deprecated file entries.
33 programmeList is a list of programmes for which the data should be
34 checked. (default: all)""")
35
36
37
38 -def getCounts(archive, detTable, detTableSuffixes, multiframeID):
39 """
40 Get number of detections for given MultiframeID.
41
42 @param archive: Connection to database to query.
43 @type archive: DbSession
44 @param detTable: The detection table name.
45 @type detTable: str
46 @param detTableSuffixes: List of detection table suffixes.
47 @type detTableSuffixes: str
48 @param multiframeID: The multiframe ID.
49 @type multiframeID: int
50
51 @return: Dictionary containing the number of detections per extension.
52 @rtype: dict(int: int)
53
54 """
55 retDict = {}
56 for suffix in detTableSuffixes:
57 retDict[suffix] = {2:0, 3:0, 4:0, 5:0}
58
59 countsPerFrame = dict(archive.queryNumRows(detTable+suffix,
60 whereStr="multiframeID=%s" % multiframeID,
61 groupBy="extNum"))
62
63 retDict[suffix].update(countsPerFrame)
64
65 return retDict
66
67
68
69 -def getFiles(archive, programmeID, dateVersStr='', deprFlag=False):
70 """
71 Get all multiframeIDs and their corresponding catalogue names for
72 the given programmeID.
73
74 @param archive: Connection to database to query.
75 @type archive: DbSession
76 @param programmeID: The programme ID.
77 @type programmeID: int
78 @param dateVersStr: Date and version of search.
79 @type dateVersStr: str
80 @param deprFlag: Flag if also deprecated files should be checked.
81 @type deprFlag: bool
82
83 @return: Dictionary containing the catalogue name for every multiframe ID.
84 @rtype: dict(int: str)
85
86 """
87 where = ''.join(["p.programmeID=", str(programmeID),
88 " and p.multiframeID=m.multiframeID",
89 " and m.catName not like '%empty_catalogue.fits%'",
90 " and f.multiframeID=m.multiframeID"])
91 if dateVersStr:
92 where += " and f.dateVersStr='%s'" % dateVersStr
93 if not deprFlag:
94 where += " and m.deprecated<128"
95
96 qRes = archive.query(
97 selectStr="m.multiframeID, m.catName, m.deprecated",
98 fromStr="Multiframe as m, ProgrammeFrame as p, FlatFileLookUp as f",
99 whereStr=where)
100
101 return dict((m, [p, d]) for m, p, d in qRes)
102
103
104
106 """
107 Get the number of rows (=detections) via the FITS keyword.
108
109 @param archive: Connection to database to query.
110 @type archive: DbSession
111 @param multiframeID: The multiframe ID.
112 @type multiframeID: int
113
114 @return: Dictionary containing number of table rows for each extension.
115 @rtype: dict(int: int)
116
117 """
118 aDict = {2:0, 3:0, 4:0, 5:0}
119 aDict.update(dict(archive.query("extNum, tableRows", "MultiframeDetector",
120 whereStr="multiframeID=%s" % multiframeID)))
121 return aDict
122
123
124
125
126
127 if __name__ == "__main__":
128 try:
129 opts, args = getopt.getopt(sys.argv[1:], "c:d:Dfp:sv:h",
130 ["compare=","date=","deprecated",
131 "filecomp","programme=", "sum",
132 "version=", "help"])
133 except getopt.GetoptError:
134
135 print(sys.argv)
136 usage()
137 exit()
138
139 programmes = []
140 dateVersStr = ''
141 versStr = '1'
142 deprFlag = False
143 summarise = False
144 fileComp = False
145 detTableSuffixes = ["Raw"]
146 for o, a in opts:
147 if o in ("-h", "--help"):
148 usage()
149 exit()
150 if o in ("-c", "--compare"):
151 detTableSuffixes = a.split(',')
152 if o in ("-d","--date"):
153 dateVersStr = a
154 if o in ("-v","--version"):
155 versStr = a
156 if o in ("-p", "--programme"):
157 programmes = a.split(',')
158 if o in ("-s", "-sum"):
159 summarise = True
160 if o in ("-f","--filecomp"):
161 fileComp = True
162 if o in ("-D", "--deprecated"):
163 deprFlag = True
164
165 fileComp = fileComp or len(detTableSuffixes) == 1
166
167 if dateVersStr:
168 dateVersStr += "_v" + versStr
169
170 archive = DbSession()
171 progTable = ProgrammeTable(archive)
172
173
174 if not programmes or 'all' in programmes:
175 progIDs = progTable.getProgIDList()
176 elif 'ukidss' in programmes:
177 progIDs = [progID for progID in progTable.getProgIDList()
178 if 10 < progID <= 105]
179 elif 'ns' in programmes:
180 progIDs = progTable.getProgIDList(onlyNonSurvey=True)
181 else:
182 progIDs = [(int(prog) if prog.isdigit() else
183 progTable.getProgIDfromName(prog.replace('/','')))
184 for prog in programmes]
185
186
187 fileDict = defaultdict(dict)
188 for progID in progIDs:
189 fileDict[progID].update(
190 getFiles(archive, progID, dateVersStr, deprFlag))
191
192
193 detCount = defaultdict(dict)
194 fitsCount = defaultdict(dict)
195 for progID in progIDs:
196 for mfID in fileDict[progID]:
197 detTable = progTable.getDetectionTable(programmeID=progID).replace(
198 "Raw",'')
199 detCount[progTable.getAcronym(progID)][mfID] = \
200 getCounts(archive, detTable, detTableSuffixes, mfID)
201 fitsCount[progTable.getAcronym(progID)][mfID] = \
202 getDetectorInfo(archive, mfID)
203
204
205 fileName = "missingDetections"
206 if programmes:
207 fileName += "_" + '-'.join(programmes)
208 if dateVersStr:
209 fileName += "_" + dateVersStr
210 if deprFlag:
211 fileName += "_depr"
212 fileName += ".dat"
213 checkDataFile = open(fileName, 'w')
214 for progID in progIDs:
215 progName = progTable.getAcronym(progID)
216 line = "getting info for:" + progName
217 print line
218 checkDataFile.write(line + '\n')
219 detTotalSum1 = 0
220 detTotalSum2 = 0
221 for mfID in sorted(fileDict[progID]):
222 if len(detTableSuffixes) == 1:
223 suffix = detTableSuffixes[0]
224 for extNum in detCount[progName][mfID][suffix]:
225 if detCount[progName][mfID][suffix][extNum] - 1 \
226 != fitsCount[progName][mfID][extNum]:
227 detTotalSum1 += fitsCount[progName][mfID][extNum]
228 detTotalSum2 += detCount[progName][mfID][suffix][extNum]
229 line = ''.join([
230 fileDict[progID][mfID][0], '[', str(extNum), ']: "',
231 str(fitsCount[progName][mfID][extNum]),
232 " -> DB: ",
233 str(detCount[progName][mfID][suffix][extNum]),
234 "(",progTable.getAcronym(progID),")"])
235 print line
236 checkDataFile.write(line + '\n')
237 elif len(detTableSuffixes) == 2:
238 suffix1, suffix2 = detTableSuffixes
239 if summarise:
240 detSum1 = 0
241 for extNum in detCount[progName][mfID][suffix1]:
242 detSum1 += detCount[progName][mfID][suffix1][extNum]
243 detSum2 = 0
244 for extNum in detCount[progName][mfID][suffix2]:
245 detSum2 += detCount[progName][mfID][suffix2][extNum]
246 fitsSum = 0
247 extNums = 0
248 for extNum in fitsCount[progName][mfID]:
249 if fitsCount[progName][mfID][extNum] > 0:
250 extNums += 1
251 fitsSum += fitsCount[progName][mfID][extNum]
252 if detSum1 != detSum2 \
253 or (detSum1 - extNums != fitsSum and fileComp):
254 line = ''.join([
255 fileDict[progID][mfID][0], ': ',
256 str(fitsSum),
257 " -> DB(",suffix1[0],"/",suffix2[0],"): ",
258 str(detSum1),'/',str(detSum2),
259 "(", progTable.getAcronym(progID),
260 ", %s " % fileDict[progID][mfID][1] if deprFlag
261 else '', ")"])
262 print line
263 checkDataFile.write(line + '\n')
264 else:
265 for extNum in detCount[progName][mfID][suffix1]:
266 if detCount[progName][mfID][suffix1][extNum] \
267 != detCount[progName][mfID][suffix2][extNum] \
268 or (detCount[progName][mfID][suffix1][extNum]-1
269 != fitsCount[progName][mfID][extNum]
270 and fileComp):
271 line = ''.join([
272 fileDict[progID][mfID][0],
273 '[', str(extNum), ']: ',
274 str(fitsCount[progName][mfID][extNum]),
275 " -> DB(",suffix1[0],"/",suffix2[0],"): ",
276 str(detCount[progName][mfID][suffix1][extNum]),
277 "/",
278 str(detCount[progName][mfID][suffix2][extNum]),
279 "(",progTable.getAcronym(progID),")"])
280 print line
281 checkDataFile.write(line + '\n')
282 line = "Total: Files: %d -> DB: %d" % (detTotalSum1, detTotalSum2)
283 print line
284 checkDataFile.write(line + '\n')
285 checkDataFile.close()
286
287
288
289
290
291