helpers.OverlappingFileFinder

32 """ Finds all frames in a comparison survey that match with a set of 33 frames in a main survey 34 """ 35 mainSurveyData = None 36 compSurveyData = None 37 productType = None 38 archive = None #: Connection to database to query. 39 combFileName = None #: Path to top-level file with provenance info. 40 frameType = None #: Base component frame type. 41 isPawPrint = False #: Is the top-level file a paw-print? 42 normalFrameInfo = None #: ? 43 normalList = None #: ? 44 sysc = None #: Initialised SystemConstants for this archive. 45 isMainDb = False 46 isCompDb = False 47 productsOnly = False 48 49 _frameInfoOfType = None #: Dictionary of frame info referenced by type. 50 _provList = None #: Complete list of component files (ProvFile). 51 _projection = 'ZPN' 52 53 RequiredMosaic = namedtuple('RequiredMosaic', 54 'productID ra dec raExtent decExtent posAngle') 55 56 #-------------------------------------------------------------------------- 57

58 - def __init__(self, mainSurvey, compSurvey, mainData, compData):

59 """ 60 Queries archive to determine full provenance info of given file. 61 62 """ 63 self.mainSurvey = mainSurvey 64 self.compSurvey = compSurvey 65 self.mainSurveyData = mainData 66 self.compSurveyData = compData 67 isDataAvailable = self.checkDataSources() 68 if not isDataAvailable: 69 raise InputError("Missing or incorrect input")

70

71 - def run(self):

72 """ 73 """ 74 # Find mosaic info in mainSurvey 75 mainSurveyPointingData = self.getData(True) 76 77 # Compare comparison survey to main 78 self.getData(False) 79 80 # create grid of points for each main image 81 mainDataGridDict = self.getMainDataGrid(mainSurveyPointingData) 82 83 # get set of compSurvey frames. 84 framesDict = self.checkFrames(mainDataGridDict) 85 outputLines = ["# %s region, %s fileName, %s filter\n" 86 % (self.mainSurvey, self.compSurvey, self.compSurvey)] 87 outputLines += ["%s %s %s\n" % (pointName, os.path.basename(fileName), 88 self.filterDict[fID]) 89 for pointName in framesDict 90 for fileName, fID in framesDict[pointName]] 91 outputFilePath = os.path.join( 92 self.sysc.testOutputPath(), "%s_%s_overlap_%s.dat" 93 % (self.mainSurvey, self.compSurvey, 94 time.strftime("%Y%m%d", time.gmtime()))) 95 96 file(outputFilePath, "w").writelines(outputLines) 97 Logger.addMessage("Written overlap information to %s" % outputFilePath)

98 99 #-------------------------------------------------------------------------- 100

101 - def checkFrames(self, mainDataGridDict):

102 """ 103 """ 104 finalFramesDict = {} 105 for pointName in mainDataGridDict: 106 finalFramesDict[pointName] = getMatchingFrames( 107 mainDataGridDict[pointName], self.halfSpaces) 108 109 return finalFramesDict

110 #-------------------------------------------------------------------------- 111

112 - def getMainDataGrid(self, mainSurveyPointingData):

113 """ 114 """ 115 mainDataGridDict = {} 116 # @TODO: Do corners and posAngll 117 for name, minRa, maxRa, minDec, maxDec in mainSurveyPointingData: 118 119 120 mainDataGridDict[name] = createGrid(minRa, maxRa, minDec, maxDec, 121 self.compScaleSize / 10.) 122 return mainDataGridDict

123

124 - def checkDataSources(self):

125 """ 126 """ 127 mainDataExists = (fileExists(self.mainSurveyData) or 128 self.checkDB(self.mainSurveyData, self.mainSurvey, True)) 129 compDataExists = (fileExists(self.compSurveyData) or 130 self.checkDB(self.compSurveyData, self.compSurvey, False)) 131 return mainDataExists and compDataExists

132 #-------------------------------------------------------------------------- 133

134 - def checkDB(self, dataBase, surveyName, isMain):

135 """ Checks whether survey programme exists on file system... 136 """ 137 isDB = False 138 try: 139 db = DbSession(dataBase, userName="ldservrw") 140 if isMain: 141 progTable = df.ProgrammeTable(db) 142 progID = None 143 for prog in progTable: 144 if prog.getAcronym() == surveyName: 145 progID = prog.getProgIDfromName() 146 if progID: 147 progTable.setCurRow(programmeID=progID) 148 self.isMainDb = True 149 self.mainArchive = db 150 self.sysc = db.sysc 151 self.mainProgramme = progTable 152 else: 153 progTable = df.ProgrammeTable(db) 154 progID = None 155 for ii, detTable in enumerate(progTable['detectiontable']): 156 if detTable.replace('Detection', '').upper() == surveyName.upper(): 157 progID = progTable['programmeid'][ii] 158 if progID: 159 progTable.setCurRow(programmeID=progID) 160 self.isCompDb = True 161 self.compArchive = db 162 self.sysc = db.sysc 163 self.compProgramme = progTable 164 self.filterDict = dict(self.compArchive.query( 165 "filterID,shortName", "Filter")) 166 return True 167 except: 168 print("DB connection failed") 169 return isDB

170 171 #-------------------------------------------------------------------------- 172

173 - def getData(self, isMain):

174 """ 175 """ 176 if isMain: 177 if self.isMainDb: 178 return self.getArchiveData(isMain) 179 else: 180 return self.parseFile(self.mainSurveyData) 181 else: 182 if self.isCompDb: 183 return self.getArchiveData(isMain) 184 else: 185 return self.parseFile(self.compSurveyData)

186 #-------------------------------------------------------------------------- 187

188 - def getArchiveData(self, isMain):

189 """ 190 """ 191 noDeeps = True 192 if not self.productType: 193 self.productType = self.compProgramme.getAttr("sourceProdType") 194 noDeeps = False 195 self.productsOnly = True 196 self._projection = 'TAN' if self.productType == 'tile' else 'ZPN' 197 198 # Get frames 199 frameTypeSel = queries.getFrameSelection(self.productType, 200 noDeeps=noDeeps) 201 if not isMain: 202 self.compScaleSize = min(self.sysc.fieldOfViewTile 203 if self.productType == 'tile' else 204 (self.sysc.instrumentXnPix * self.sysc.instrumentPixScale / 3600., 205 self.sysc.instrumentYnPix * self.sysc.instrumentPixScale / 3600.)) 206 self.halfSpaces = self.getHalfSpaces(frameTypeSel, isMain) 207 else: 208 # get outline points 209 print("Still to do")

210 211 #-------------------------------------------------------------------------- 212

213 - def getHalfSpaces(self, fTypeSel, isMain):

214 """ Calculates half-spaces for each detector of the correct frame type. 215 """ 216 # TODO Get next cuEventID 217 programme = self.mainProgramme if isMain else self.compProgramme 218 archive = self.mainArchive if isMain else self.compArchive 219 programmeID = programme.getAttr("programmeID") 220 221 222 # USE LEFT JOIN with astroInfo table 223 # Setup half-spaces 2 pixels away from dither boundary and detector 224 # boundary 225 # @TODO: update for tiles 226 nCoordInHSDef = 3 227 initPos = [0, 2, 4, 6] 228 productSel = 'AND productID>0' if self.productsOnly else '' 229 hSpaceFrames = archive.query( 230 selectStr="m.fileName,m.filterID,MultiframeDetector.multiframeID," 231 "MultiframeDetector.extNum," 232 "MultiframeDetector.axis1length,MultiframeDetector.axis2length," 233 "MultiframeDetector.detRows,MultiframeDetector.detCols,m.nuStep," 234 "crPixX, crPixY, crValX, crValY, cd11, cd12, cd21, cd22, " 235 "pv21, pv22, pv23%s" 236 % (', pv24, pv25' if self.sysc.isVSA() else 237 ', 0. as pv24, 0. as pv25'), 238 fromStr="ProgrammeFrame as p,CurrentAstrometry as c," 239 "Multiframe as m,MultiframeDetector", 240 whereStr="p.programmeID=%d AND p.multiframeID=m.multiframeID AND " 241 "m.multiframeID=MultiframeDetector.multiframeID AND c.multiframeID=" 242 "MultiframeDetector.multiframeID AND c.extNum=" 243 "MultiframeDetector.extNum AND %s AND " 244 "MultiframeDetector.%s %s" 245 % (programmeID, fTypeSel, 246 DepCodes.selectNonDeprecated, productSel)) 247 halfSpaces = [] 248 for frame in hSpaceFrames: 249 # @@TODO: This condition shouldn't be necessary if selected out, 250 # and then it would be worthwhile having a ForLoopMonitor, 251 # otherwise its inconsistent, and in fact distractingly pointless 252 # when no updates necessary. 253 254 # Corners and mid points of frames 255 detEdges = [(1, 1), (1, int(0.5 * (1 + frame.axis2length))), 256 (1, frame.axis2length), 257 (int(0.5 * (1 + frame.axis1length)), frame.axis2length), 258 (frame.axis1length, frame.axis2length), 259 (frame.axis1length, int(0.5 * (1 + frame.axis2length))), 260 (frame.axis1length, 1), (int(0.5 * (1 + frame.axis1length)), 1)] 261 detEdges.append(detEdges[0]) 262 263 wcs = (frame.crPixX, frame.crPixY, frame.crValX, frame.crValY, 264 frame.cd11, frame.cd12, frame.cd21, frame.cd22, frame.pv21, 265 frame.pv22, frame.pv23, frame.pv24, frame.pv25) 266 values = [frame.fileName, frame.filterID, frame.multiframeID, 267 frame.extNum] 268 269 coords = astro.convPixelToCartSky(detEdges, wcs=wcs, 270 projection=self._projection) 271 values.append(astro.calcHSpaces(coords, initPos, nCoordInHSDef)) 272 # namespace 273 halfSpaces.append(values) 274 return halfSpaces

275 #-------------------------------------------------------------------------- 276

277 - def parseFile(self, fileName):

278 """ 279 """ 280 # @TODO: For now assume ASCII 281 lines = file(fileName).readlines() 282 ltype = None 283 dataList = [] 284 self.additionalConstraintsDict = {} 285 for line in lines: 286 if line[0] == '#': 287 if 'MINMAXRADEC' in line: 288 ltype = 1 289 elif 'NAMECENTRE' in line: 290 ltype = 2 291 elif 'POSANGLE' in line: 292 self.additionalConstraintsDict['POSANGLE'] = float(line.split('=')[1]) 293 elif 'RAEXTENT' in line: 294 self.additionalConstraintsDict['RAEXTENT'] = float(line.split('=')[1]) 295 elif 'DECEXTENT' in line: 296 self.additionalConstraintsDict['DECEXTENT'] = float(line.split('=')[1]) 297 else: 298 dataList.append(self.parseLine(line, ltype)) 299 return dataList

300 #-------------------------------------------------------------------------- 301

302 - def parseLine(self, line, ltype):

303 """ 304 """ 305 if ltype == 1: 306 if len(line.split(' ')) != 5: 307 raise InputError("File has incorrect format") 308 parts = line.split(' ') 309 # @TODO: not elegant 310 return [parts[0], float(parts[1]), float(parts[2]), 311 float(parts[3]), float(parts[4])] 312 elif ltype == 2: 313 name = line.split(' ')[0] 314 id = 0 315 parts = name.split('_') 316 raCen = float(parts[1]) 317 decCen = float(parts[2]) 318 # set up RequiredMosaic-like nametuple 319 #reqMos = OverlappingFileFinder.RequiredMosaic(*[id, raCen, decCen, 320 # self.additionalConstraintsDict['RAEXTENT'], 321 # self.additionalConstraintsDict['DECEXTENT'], 322 # self.additionalConstraintsDict['POSANGLE']]) 323 #corners = astro.calcMosaicProductCorners([reqMos])[id] 324 #print("POINTING: ", name, corners) 325 raMin = raCen - 0.5 * self.additionalConstraintsDict['RAEXTENT'] 326 raMax = raCen + 0.5 * self.additionalConstraintsDict['RAEXTENT'] 327 decMin = decCen - 0.5 * self.additionalConstraintsDict['DECEXTENT'] 328 decMax = decCen + 0.5 * self.additionalConstraintsDict['DECEXTENT'] 329 return [name, raMin, raMax, decMin, decMax]

330 331 #-------------------------------------------------------------------------- 332

333 - def findOverlaps(self):

334 """ 335 """

Source Code for Module helpers.OverlappingFileFinder