Package invocations :: Package monitoring :: Module CheckDetections
[hide private]

Source Code for Module invocations.monitoring.CheckDetections

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: CheckDetections.py 8551 2011-08-05 11:40:21Z EckhardSutorius $ 
  4  """ 
  5     Checks if the number of detections in the database correspond to the number 
  6     of detections in the catalogue files. 
  7   
  8     @author: E. Sutorius 
  9     @org:    WFAU, IfA, University of Edinburgh 
 10  """ 
 11  #------------------------------------------------------------------------------ 
 12  from   collections import defaultdict 
 13  import getopt 
 14  import sys 
 15   
 16  from   wsatools.DataFactory         import ProgrammeTable 
 17  from   wsatools.DbConnect.DbSession import DbSession 
 18  #------------------------------------------------------------------------------ 
 19   
20 -def usage():
21 print("""Usage: 22 CheckDetections.py [-c/--compare detectionSuffixes] [-d/--date dateStr] 23 [-D/--deprecated] [-p/--programme programmeList] 24 [-s/--sum] Adds up all counts of one file. 25 26 Check if the number of detections in the database correspond to the number of 27 detections in the catalogue files. 28 29 -d/--date checks only the given date. 30 -c/--compare compares the given detection split tables, 31 eg. 'Raw,Photometry'. 32 -D/--deprecated checks also deprecated file entries. 33 programmeList is a list of programmes for which the data should be 34 checked. (default: all)""")
35 36 #------------------------------------------------------------------------------ 37
38 -def getCounts(archive, detTable, detTableSuffixes, multiframeID):
39 """ 40 Get number of detections for given MultiframeID. 41 42 @param archive: Connection to database to query. 43 @type archive: DbSession 44 @param detTable: The detection table name. 45 @type detTable: str 46 @param detTableSuffixes: List of detection table suffixes. 47 @type detTableSuffixes: str 48 @param multiframeID: The multiframe ID. 49 @type multiframeID: int 50 51 @return: Dictionary containing the number of detections per extension. 52 @rtype: dict(int: int) 53 54 """ 55 retDict = {} 56 for suffix in detTableSuffixes: 57 retDict[suffix] = {2:0, 3:0, 4:0, 5:0} 58 59 countsPerFrame = dict(archive.queryNumRows(detTable+suffix, 60 whereStr="multiframeID=%s" % multiframeID, 61 groupBy="extNum")) 62 63 retDict[suffix].update(countsPerFrame) 64 65 return retDict
66 67 #------------------------------------------------------------------------------ 68
69 -def getFiles(archive, programmeID, dateVersStr='', deprFlag=False):
70 """ 71 Get all multiframeIDs and their corresponding catalogue names for 72 the given programmeID. 73 74 @param archive: Connection to database to query. 75 @type archive: DbSession 76 @param programmeID: The programme ID. 77 @type programmeID: int 78 @param dateVersStr: Date and version of search. 79 @type dateVersStr: str 80 @param deprFlag: Flag if also deprecated files should be checked. 81 @type deprFlag: bool 82 83 @return: Dictionary containing the catalogue name for every multiframe ID. 84 @rtype: dict(int: str) 85 86 """ 87 where = ''.join(["p.programmeID=", str(programmeID), 88 " and p.multiframeID=m.multiframeID", 89 " and m.catName not like '%empty_catalogue.fits%'", 90 " and f.multiframeID=m.multiframeID"]) 91 if dateVersStr: 92 where += " and f.dateVersStr='%s'" % dateVersStr 93 if not deprFlag: 94 where += " and m.deprecated<128" 95 96 qRes = archive.query( 97 selectStr="m.multiframeID, m.catName, m.deprecated", 98 fromStr="Multiframe as m, ProgrammeFrame as p, FlatFileLookUp as f", 99 whereStr=where) 100 101 return dict((m, [p, d]) for m, p, d in qRes)
102 103 #------------------------------------------------------------------------------ 104
105 -def getDetectorInfo(archive, multiframeID):
106 """ 107 Get the number of rows (=detections) via the FITS keyword. 108 109 @param archive: Connection to database to query. 110 @type archive: DbSession 111 @param multiframeID: The multiframe ID. 112 @type multiframeID: int 113 114 @return: Dictionary containing number of table rows for each extension. 115 @rtype: dict(int: int) 116 117 """ 118 aDict = {2:0, 3:0, 4:0, 5:0} 119 aDict.update(dict(archive.query("extNum, tableRows", "MultiframeDetector", 120 whereStr="multiframeID=%s" % multiframeID))) 121 return aDict
122 123 #------------------------------------------------------------------------------ 124 # Entry point for script. 125 126 # Allow module to be imported as well as executed from the command line 127 if __name__ == "__main__": 128 try: 129 opts, args = getopt.getopt(sys.argv[1:], "c:d:Dfp:sv:h", 130 ["compare=","date=","deprecated", 131 "filecomp","programme=", "sum", 132 "version=", "help"]) 133 except getopt.GetoptError: 134 # print help information and exit: 135 print(sys.argv) 136 usage() 137 exit() 138 139 programmes = [] 140 dateVersStr = '' 141 versStr = '1' 142 deprFlag = False 143 summarise = False 144 fileComp = False 145 detTableSuffixes = ["Raw"] 146 for o, a in opts: 147 if o in ("-h", "--help"): 148 usage() 149 exit() 150 if o in ("-c", "--compare"): 151 detTableSuffixes = a.split(',') 152 if o in ("-d","--date"): 153 dateVersStr = a 154 if o in ("-v","--version"): 155 versStr = a 156 if o in ("-p", "--programme"): 157 programmes = a.split(',') 158 if o in ("-s", "-sum"): 159 summarise = True 160 if o in ("-f","--filecomp"): 161 fileComp = True 162 if o in ("-D", "--deprecated"): 163 deprFlag = True 164 165 fileComp = fileComp or len(detTableSuffixes) == 1 166 167 if dateVersStr: 168 dateVersStr += "_v" + versStr 169 170 archive = DbSession() 171 progTable = ProgrammeTable(archive) 172 173 # create list of programmeIDs 174 if not programmes or 'all' in programmes: 175 progIDs = progTable.getProgIDList() 176 elif 'ukidss' in programmes: 177 progIDs = [progID for progID in progTable.getProgIDList() 178 if 10 < progID <= 105] 179 elif 'ns' in programmes: 180 progIDs = progTable.getProgIDList(onlyNonSurvey=True) 181 else: 182 progIDs = [(int(prog) if prog.isdigit() else 183 progTable.getProgIDfromName(prog.replace('/',''))) 184 for prog in programmes] 185 186 # get all mfids of files with catalogues for given programme IDs 187 fileDict = defaultdict(dict) 188 for progID in progIDs: 189 fileDict[progID].update( 190 getFiles(archive, progID, dateVersStr, deprFlag)) 191 192 # get detection counts from detection table and from detector keyword 193 detCount = defaultdict(dict) 194 fitsCount = defaultdict(dict) 195 for progID in progIDs: 196 for mfID in fileDict[progID]: 197 detTable = progTable.getDetectionTable(programmeID=progID).replace( 198 "Raw",'') 199 detCount[progTable.getAcronym(progID)][mfID] = \ 200 getCounts(archive, detTable, detTableSuffixes, mfID) 201 fitsCount[progTable.getAcronym(progID)][mfID] = \ 202 getDetectorInfo(archive, mfID) 203 204 # write differences into file 205 fileName = "missingDetections" 206 if programmes: 207 fileName += "_" + '-'.join(programmes) 208 if dateVersStr: 209 fileName += "_" + dateVersStr 210 if deprFlag: 211 fileName += "_depr" 212 fileName += ".dat" 213 checkDataFile = open(fileName, 'w') 214 for progID in progIDs: 215 progName = progTable.getAcronym(progID) 216 line = "getting info for:" + progName 217 print line 218 checkDataFile.write(line + '\n') 219 detTotalSum1 = 0 220 detTotalSum2 = 0 221 for mfID in sorted(fileDict[progID]): 222 if len(detTableSuffixes) == 1: 223 suffix = detTableSuffixes[0] 224 for extNum in detCount[progName][mfID][suffix]: 225 if detCount[progName][mfID][suffix][extNum] - 1 \ 226 != fitsCount[progName][mfID][extNum]: 227 detTotalSum1 += fitsCount[progName][mfID][extNum] 228 detTotalSum2 += detCount[progName][mfID][suffix][extNum] 229 line = ''.join([ 230 fileDict[progID][mfID][0], '[', str(extNum), ']: "', 231 str(fitsCount[progName][mfID][extNum]), 232 " -> DB: ", 233 str(detCount[progName][mfID][suffix][extNum]), 234 "(",progTable.getAcronym(progID),")"]) 235 print line 236 checkDataFile.write(line + '\n') 237 elif len(detTableSuffixes) == 2: 238 suffix1, suffix2 = detTableSuffixes 239 if summarise: 240 detSum1 = 0 241 for extNum in detCount[progName][mfID][suffix1]: 242 detSum1 += detCount[progName][mfID][suffix1][extNum] 243 detSum2 = 0 244 for extNum in detCount[progName][mfID][suffix2]: 245 detSum2 += detCount[progName][mfID][suffix2][extNum] 246 fitsSum = 0 247 extNums = 0 248 for extNum in fitsCount[progName][mfID]: 249 if fitsCount[progName][mfID][extNum] > 0: 250 extNums += 1 251 fitsSum += fitsCount[progName][mfID][extNum] 252 if detSum1 != detSum2 \ 253 or (detSum1 - extNums != fitsSum and fileComp): 254 line = ''.join([ 255 fileDict[progID][mfID][0], ': ', 256 str(fitsSum), 257 " -> DB(",suffix1[0],"/",suffix2[0],"): ", 258 str(detSum1),'/',str(detSum2), 259 "(", progTable.getAcronym(progID), 260 ", %s " % fileDict[progID][mfID][1] if deprFlag 261 else '', ")"]) 262 print line 263 checkDataFile.write(line + '\n') 264 else: 265 for extNum in detCount[progName][mfID][suffix1]: 266 if detCount[progName][mfID][suffix1][extNum] \ 267 != detCount[progName][mfID][suffix2][extNum] \ 268 or (detCount[progName][mfID][suffix1][extNum]-1 269 != fitsCount[progName][mfID][extNum] 270 and fileComp): 271 line = ''.join([ 272 fileDict[progID][mfID][0], 273 '[', str(extNum), ']: ', 274 str(fitsCount[progName][mfID][extNum]), 275 " -> DB(",suffix1[0],"/",suffix2[0],"): ", 276 str(detCount[progName][mfID][suffix1][extNum]), 277 "/", 278 str(detCount[progName][mfID][suffix2][extNum]), 279 "(",progTable.getAcronym(progID),")"]) 280 print line 281 checkDataFile.write(line + '\n') 282 line = "Total: Files: %d -> DB: %d" % (detTotalSum1, detTotalSum2) 283 print line 284 checkDataFile.write(line + '\n') 285 checkDataFile.close() 286 287 #------------------------------------------------------------------------------ 288 # Change log: 289 # 290 # 7-Sep-2007, ETWS: First version. 291