wsatools.WfcamsrcInterface

1 #------------------------------------------------------------------------------ 2 #$Id: WfcamsrcInterface.py 8940 2012-01-20 09:30:18Z RossCollins $ 3 """ 4 Wrapper functions to call the wfcamsrc C++ external programs. For ingest of 5 catalogue & image metadata, as well as source merging. 6 7 @author: I.A. Bond 8 @org: WFAU, IfA, University of Edinburgh 9 10 @newfield contributors: Contributors, Contributors (Alphabetical Order) 11 @contributors: R.S. Collins, N.C. Hambly, E. Sutorius 12 """ 13 #------------------------------------------------------------------------------ 14 import os 15 import re 16 17 import wsatools.DbConnect.DbConstants as dbc 18 import wsatools.ExternalProcess as extp 19 from wsatools.Logger import Logger 20 from wsatools.SystemConstants import SystemConstants 21 import wsatools.Utilities as utils 22 #------------------------------------------------------------------------------ 23

24 -def extractMetadata(tableListPath, fileListPath, cuEventID, ingestDirPath, 25 filePrefix, logPath, sysc=SystemConstants("WSA")):

26 """ 27 Wrapper function to run the external program that extracts image metadata 28 from FITS headers and prepares a CSV file for ingestion. 29 30 @param tableListPath: Full path to the file with a list of tables for data 31 to be ingested into, and their schema file name. 32 @type tableListPath: str 33 @param fileListPath: Full path to the file with list of FITS files to 34 extract the metadata from. 35 @type fileListPath: str 36 @param cuEventID: Curation event ID. 37 @type cuEventID: int 38 @param ingestDirPath: Path where the CSV ingest file is to be written. 39 @type ingestDirPath: str 40 @param filePrefix: Common prefix for ingest files. 41 @type filePrefix: str 42 @param logPath: Full path to the log file for this curation event. 43 @type logPath: str 44 @param sysc: Initialised SystemConstants object. 45 @type sysc: SystemConstants 46 47 @return: List of files where metadata could not be extracted and an int if 48 no NaNs detected, otherwise None. 49 @rtype: list(str), int 50 51 """ 52 command = "exmeta -tables %s -files %s -cuEvent %s"\ 53 " -csvpath %s -sqlpath %s -csvprefix %s -archive %s" % ( 54 tableListPath, fileListPath, cuEventID, ingestDirPath, 55 sysc.sqlScriptPath, filePrefix, sysc.loadDatabase) 56 57 # too much output for the gui so, turn the Logger's echo off 58 Logger.setEchoOff() 59 try: 60 print("Log is no longer echoed to terminal, but continues to be " 61 "written to " + logPath) 62 63 return parseOutput(extp.run(command)) 64 finally: 65 Logger.setEchoOn()

66 67 #------------------------------------------------------------------------------ 68

69 -def extractDetections(tableListPath, fileListPath, cuEventID, ingestDirPath, 70 filePrefix, illumName, outTables, nextObjID, 71 sysc=SystemConstants("WSA")):

72 """ 73 Wrapper function to run the external program that extracts detection 74 catalog data from FITS tables into a binary ingest file. 75 76 @param tableListPath: Full path to the file with a list of tables for data 77 to be ingested into, and their schema file name. 78 @type tableListPath: str 79 @param fileListPath: Full path to the file with list of FITS catalogue 80 files to extract the detections from. 81 @type fileListPath: str 82 @param cuEventID: Curation event ID. 83 @type cuEventID: int 84 @param ingestDirPath: Path where the binary ingest file is to be written. 85 @type ingestDirPath: str 86 @param filePrefix: Common prefix for ingest files. 87 @type filePrefix: str 88 @param illumName: The illumination table file used for this data. 89 @type illumName: str 90 @param outTables: Comma separated list of tables to process. 91 @type outTables: str 92 @param nextObjID: Next objectID for this programme, counting negative 93 from -1 onwards. 94 @type nextObjID: int 95 @param sysc: Initialised SystemConstants object. 96 @type sysc: SystemConstants 97 98 @return: List of files where catalogues could not be extracted and the last 99 object ID. 100 @rtype: list(str), int 101 102 """ 103 command = "exnumeric -tables %s -files %s -cuEvent %s -csvpath %s " \ 104 " -sqlpath %s -csvprefix %s -illumname %s -outtables %s " \ 105 "-newobjid %s -archive %s" % ( 106 tableListPath, fileListPath, cuEventID, ingestDirPath, 107 sysc.sqlScriptPath, filePrefix, illumName, outTables, nextObjID, 108 sysc.loadDatabase) 109 110 # too much output for the gui so, turn the Logger's echo off 111 Logger.setEchoOff() 112 try: 113 return parseOutput(extp.run(command)) 114 finally: 115 Logger.setEchoOn()

116 117 #------------------------------------------------------------------------------ 118

119 -def mergeDetections(mergeTablePathName, resultsPathName, sourceTableName, 120 schemaFile, cameoSchemas, epochs, pixScales, cuEventID, 121 firstID, frameSetID, extractor="CASU", 122 frameSetIDName="frameSetID", sourceIDName="sourceID", 123 centralPos=None, minEpochSep=None):

124 """ 125 Wrapper function to mkmerge. The C++ program that merges detections into a 126 source table stored in a binary file for ingest into the database. 127 128 @param mergeTablePathName: Name of the "merge table" pointers file created 129 by the "pairing" C code. 130 @type mergeTablePathName: str 131 @param resultsPathName: Full path to output binary file containing the 132 accumulated merged sources. 133 @type resultsPathName: str 134 @param sourceTableName: Name of source table where results are ingest. 135 @type sourceTableName: str 136 @param schemaFile: Name of .sql script file describing the schema 137 for this programme. 138 @type schemaFile: str 139 @param cameoSchemas: A list of cameo schema files for each passband. 140 Ordered by filterID. 141 @type cameoSchemas: list(str) 142 @param epochs: A list of epochs for each passband's frame. The 143 passband order should be identical to that of 144 the cameoSchemas. 145 @type epochs: list(float) 146 @param pixScales: A list of pixel scales for each passband's 147 frame. The passband order should be identical to 148 that of the cameoSchemas. 149 @type pixScales: list(float) 150 @param cuEventID: Curation use case event ID. 151 @type cuEventID: int 152 @param firstID: The UID to assign to the first new source. 153 @type firstID: int 154 @param frameSetID: The unique identifier of the frame set. 155 @type frameSetID: int 156 @param extractor: Name of source catalogue extraction tool used 157 to create these detections (same format as 158 extractTool parameter in RequiredStack table). 159 @type extractor: str 160 @param frameSetIDName: Name of the column in the source table that 161 identifies the unique ID of the frame set for 162 the source. 163 @type frameSetIDName: str 164 @param sourceIDName: Name of the column in the source table for the 165 unique source ID. 166 @type sourceIDName: str 167 @param centralPos: If proper motion is to be calculated need frame 168 set centre co-ordinates. 169 @type centralPos: tuple(float, float) 170 @param minEpochSep: If proper motion is to be calculated need to 171 know what the minimum epoch separation criterion 172 is in days. 173 @type minEpochSep: float 174 175 """ 176 command = "mkmerge -mergefile %s -out %s -table %s" \ 177 % (mergeTablePathName, resultsPathName, sourceTableName) 178 179 command += " -schema " 180 command += os.path.join(SystemConstants.sqlScriptPath, schemaFile) 181 182 cameoFlag = " -cameo" 183 command += cameoFlag 184 command += utils.joinNested(enumerate(cameoSchemas), cameoFlag, " ") 185 186 epochFlag = " -epoch" 187 command += epochFlag + utils.joinNested(enumerate(epochs), epochFlag, " ") 188 189 pixScaleFlag = " -pixScale" 190 command += pixScaleFlag 191 command += utils.joinNested(enumerate(pixScales), pixScaleFlag, " ") 192 193 command += " -archive %s -cuEventID %s -firstID %s -%s %s -extractor %s " \ 194 "-sourceIDName %s" % (SystemConstants(schemaFile).loadDatabase, 195 cuEventID, firstID, frameSetIDName, frameSetID, 196 extractor.lower(), sourceIDName) 197 198 if centralPos and minEpochSep: 199 command += " -ra %s -dec %s" % centralPos 200 command += " -minEpochSep %s" % minEpochSep 201 202 outputLines = extp.run(command) 203 for line in outputLines: 204 if line.endswith(("singular values", "quality control")): 205 Logger.addMessage("<Info> Frame set ID %s has " % frameSetID 206 + line + ". Its proper motions will not be calculated.") 207 208 Logger.addMessage('\n'.join(outputLines), alwaysLog=False)

209 210 #------------------------------------------------------------------------------ 211

212 -def parseOutput(stdOut):

213 """ 214 Parses the CU3 and 4 stdout to determine the result of the data extraction. 215 216 @param stdOut: Standard output. 217 @type stdOut: list(str) 218 219 @return: List of files where data could not be extracted and the last 220 detection object ID or, for metadata extraction, None if NaNs. 221 @rtype: list(str), int 222 223 """ 224 lastObjID = 0 225 idStr = "<SYSTEM> Last ObjID: " 226 for line in stdOut: 227 Logger.addMessage(line) 228 if line.startswith(idStr): 229 line = line.replace(idStr, '') 230 # @@TODO: Why default to small int here, surely with negative obj 231 # IDs we are best to default to zero? 232 lastObjID = (int(line.split('.')[0]) if '.' in line else 233 dbc.smallIntDefault()) 234 235 errorPhrases = ["range error", "nan value error", "undef value error", 236 "missing key error", "unknown error", "pix and cat mismatch"] 237 238 failedFiles = set() 239 for line in stdOut: 240 match = re.search(r'/disk[0-9][0-9]/wsa/', line) 241 if match and line.startswith("<SYSTEM>") \ 242 and any(phrase in line for phrase in errorPhrases): 243 failedFiles.add(line[match.start():line.rfind(' (')]) 244 elif line.startswith("<ERROR>") and "CurrentAstrometry" in line \ 245 and "NaN value found" in line: 246 lastObjID = None 247 248 return sorted(failedFiles), lastObjID

249 250 #------------------------------------------------------------------------------ 251 # Change log: 252 # 253 # 26-Apr-2004, IAB: Added python docstrings 254 # 24-Jun-2004, IAB: Added more python docstrings 255 # 30-Jul-2004, NCH: Very small change 256 # 2-Aug-2004, NCH: Fixed bugs 257 # 3-Aug-2004, NCH: Added sqlpath argument to exmeta, exnumeric and their 258 # associated Python wrapper functions. 259 # 10-Aug-2004, NCH: Generalised makeFileList such that a catalogue file 260 # is not assumed to exist for every pixel file. 261 # 11-Aug-2004, ETWS: Added 'all' parameter to type in makeFileList 262 # 23-Aug-2004, NCH: Added a wrapper function for the C++ application 263 # "mkmerge" 264 # 30-Aug-2004, NCH: Bug fix in cameo argument string. 265 # 1-Sep-2004, NCH: Changed documentation string quotes to doubles 266 # 1-Mar-2005, ETWS: Replaced types in makeFlieList with suffix check 267 # 2-Mar-2005, NCH: Small change in merged source table specification in 268 # mergeProgrammeDetections to make it work for merged 269 # source remeasurements. 270 # 9-Mar-2005, ETWS: added more flexibility for catalogue files 271 # in makeFileList 272 # 11-Mar-2005, ETWS: Added more flexibility for non-standard named files 273 # in makeFileList 274 # 18-Mar-2005, ETWS: Fixed naming error for non-standard named CASU files 275 # 25-Mar-2005, ETWS: added error log return lists to extractSources 276 # 25-May-2005, ETWS: included objid in parameter list of extractSources 277 # 13-Jun-2005, ETWS: included objid in return parameters of extractMetadata 278 # 5-Jul-2005, ETWS: set compressed image filename to default value 279 # 19-Sep-2005, ETWS: reflect changes due to movement of external process' 280 # output to CuFunctions 281 # resolved circular import 282 # 22-Nov-2005, NCH: Enhanced mergeProgrammeDetections for non-survey schemas. 283 # 26-Jan-2006, ETWS: Included WFAU file products (having wfauPrefix) 284 # 28-Jun-2006, RSC: Added a switch to indicate to mkmerge that the UDS uses 285 # sextractor instead of the CASU source extractor 286 # 2-Aug-2006, RSC: * Added in parseExternalOutput() from ExternalProcess 287 # * Updated to new ExternalProcess.py design. 288 # * Simplified makeFileList() 289 # 2-Aug-2006, RSC: * Renamed to WfcamsrcInterface.py from DataOps.py. 290 # * Move makeFileList() into Utilities.py 291 # 2-May-2007, ETWS: Included WCS NaN error check. 292 # 5-May-2008, ETWS: Finalised processing of detection table subset 293 # in extractSources. 294

Source Code for Module wsatools.WfcamsrcInterface