wsatools.DataFactory

1 #------------------------------------------------------------------------------ 2 #$Id: DataFactory.py 10248 2014-03-17 15:55:01Z RossCollins $ 3 """ 4 Python data structures for database curation information. 5 6 @author: R.S. Collins 7 @org: WFAU, IfA, University of Edinburgh 8 9 @newfield contributors: Contributors, Contributors (Alphabetical Order) 10 @contributors: I.A. Bond, N.C. Hambly, E. Sutorius 11 12 @todo: Move Table and View into a general Database module / package. 13 @todo: Replace with metadata classes? Where a record is a metadata class 14 whose attributes are created on the fly, rather than using a 15 dictionary for referencing attributes. 16 @todo: Usage documentation. 17 @todo: Complement getAttr() with [], but beware of conflicts of use. 18 """ 19 #------------------------------------------------------------------------------ 20 from __future__ import division, print_function 21 from future_builtins import map, zip, filter 22 23 from collections import defaultdict 24 from operator import methodcaller 25 26 import wsatools.DbConnect.DbConstants as dbc 27 from wsatools.DbConnect.DbSession import DbSession, Join 28 import wsatools.Utilities as utils 29 #------------------------------------------------------------------------------ 30

31 -class PassbandList(object):

32 """ Provides a monostate list of passbands for a given programme. 33 """ 34 _isSynoptic = False #: Is the current list a synoptic passband list? 35 _programmeID = None #: Programme ID of the current list. 36 filterIDs = None #: List of filter IDs for current programme. 37 passbands = None #: List of passbands for current programme. 38

39 - def __init__(self, programme, isSynoptic=False):

40 """ 41 Queries database to determine passband list for given programme, unless 42 this has already been done. 43 44 @param programme: Programme table of database to query with the 45 current row set to that of the desired programme. 46 @type programme: ProgrammeTable 47 @param isSynoptic: If True, prepare a list of synoptic table passbands. 48 @type isSynoptic: bool 49 50 """ 51 programmeID = programme.getAttr("programmeID") 52 53 if programmeID == PassbandList._programmeID \ 54 and isSynoptic == PassbandList._isSynoptic: 55 return # Already initialised for this programme 56 57 PassbandList._isSynoptic = isSynoptic 58 PassbandList._programmeID = programmeID 59 60 filterPasses = programme._db.query( 61 selectStr="Filter.filterID, shortName, " 62 + ("1" if isSynoptic else "nPass"), 63 fromStr=Join(["RequiredFilters", "Filter"], ["filterID"]), 64 whereStr="programmeID=%s" % programmeID + 65 (" AND isSynoptic=1" if isSynoptic else ""), 66 orderBy="Filter.filterID") 67 68 PassbandList.filterIDs = [] 69 PassbandList.passbands = [] 70 for filterID, shortName, numEpochs in filterPasses: 71 PassbandList.filterIDs.append(filterID) 72 if numEpochs > 1: 73 PassbandList.passbands += ["%s_%s" % (shortName, obsNum + 1) 74 for obsNum in range(numEpochs)] 75 else: 76 PassbandList.passbands.append(shortName)

77 78 #-------------------------------------------------------------------------- 79

80 - def getBroadbandPassbands(self):

81 """ @return: Broadband passbands in lower case. 82 @rtype: list(str) 83 """ 84 return [passband for passband, filterID 85 in zip(self.getPassbandsLowerCase(), self.filterIDs) 86 if filterID <= 5]

87 88 #-------------------------------------------------------------------------- 89

90 - def getFilterIDsWithMfCols(self):

91 """ @return: Filter IDs with names of the multiframeID columns for each 92 passband in a merge-log. 93 @rtype: list(tuple(int, str)) 94 """ 95 return zip(self.filterIDs, self.getMfCols())

96 97 #-------------------------------------------------------------------------- 98

99 - def getMfCols(self):

100 """ @return: Names of the multiframeID columns for each passband in a 101 merge-log. 102 @rtype: list(str) 103 """ 104 return [passband + "mfID" for passband in self.getPassbandsLowerCase()]

105 106 #-------------------------------------------------------------------------- 107

108 - def getMfdCols(self):

109 """ @return: Names of the multiframeID, extNum column pairs for each 110 passband in a merge-log 111 @rtype: list(tuple(str, str)) 112 """ 113 return [(passband + "mfID", passband + "eNum") 114 for passband in self.getPassbandsLowerCase()]

115 116 #-------------------------------------------------------------------------- 117

118 - def getPassbandsLowerCase(self):

119 """ @return: Passbands in lower case. 120 @rtype: sequence(str) 121 """ 122 return map(methodcaller('lower'), self.passbands)

123 124 #------------------------------------------------------------------------------ 125

126 -class Table(dict):

127 """ 128 Use this class to store the complete contents of a database table in a 129 local Python data object. However, it must be possible to define a unique 130 row of the table with just a single attribute. This obviates the need to 131 constantly query the database, and provides a nice clear Python solution 132 for access to small data sets (e.g. curation tables). Optimised for the 133 scenario where setCurRow is called once, and getAttr many times. Data is 134 stored as a dict(columnName: columnValuesList). 135 136 """ 137 #: A dictionary of the current row's values referenced by column name. 138 _curRow = None 139 #: A DbSession connection to the database. 140 _db = None 141 #: Name of the database table mirrored in this object. 142 name = '' 143 144 #-------------------------------------------------------------------------- 145

146 - def __init__(self, tableName, db=None):

147 """ 148 Accesses the default database and extracts the requested table data. 149 150 @param tableName: Name of database table to capture. 151 @type tableName: str 152 @param db: Optional alternative database connection. 153 @type db: DbSession 154 155 """ 156 self.name = tableName 157 self._db = db or DbSession() 158 colNames = \ 159 [name.lower() for name in self._db.queryColumnNames(self.name)] 160 rows = self._db.query('*', self.name) 161 cols = map(list, zip(*rows)) # Transpose list of rows 162 super(Table, self).__init__(zip(colNames, cols))

163 164 #-------------------------------------------------------------------------- 165

166 - def setCurRow(self, **kwds):

167 """ 168 Sets the current row by finding the row where the specified attribute 169 is equal to the given value. Supply keyword argument as for example: 170 setCurRow(programmeID=101). NB: Keywords must define a unique row 171 (otherwise the row is the first instance of this keyword value). 172 173 """ 174 assert len(kwds) == 1, \ 175 "DataFactory.Table.setCurRow() only takes 1 keyword argument: %s "\ 176 "were given!\n Full command given: DataFactory.Table.setCurRow(%s)"\ 177 % (len(kwds), utils.joinDict(kwds)) 178 179 attrName = kwds.keys()[0] 180 self._setCurRow(attrName, attrValue=kwds[attrName], 181 caller="DataFactory.Table.setCurRow(%s=" % attrName)

182 183 #-------------------------------------------------------------------------- 184

185 - def _setCurRow(self, attrName, attrValue, caller):

186 """ 187 Sets the current row by finding the row where the specified attribute 188 is equal to the given value. NB: This must define a unique row 189 (otherwise the row is the first instance of this keyword value). 190 191 """ 192 try: 193 strCmp = lambda x: str(x[1]).lower() == str(attrValue).lower() 194 rowIndex = \ 195 filter(strCmp, enumerate(self.get(attrName.lower()))).next()[0] 196 self._curRow = dict((colName, value[rowIndex]) 197 for colName, value in self.items()) 198 except TypeError: 199 raise AttributeError("%s%r): table %s does not have a %r column." % 200 (caller, attrValue, self.name, attrName)) 201 except StopIteration: 202 raise ValueError( 203 "Table %s in database %s does not have a row where %s=%r" % 204 (self.name, self._db.database, attrName, attrValue))

205 206 #-------------------------------------------------------------------------- 207

208 - def getAll(self, attrName):

209 """ 210 Returns the column list of all values for given attribute. 211 212 @param attrName: Name of attribute column. 213 @type attrName: str 214 215 @return: List of all attribute values for a given column. 216 @rtype: list 217 218 """ 219 oldRow, self._curRow = self._curRow, None 220 column = self.get(attrName.lower(), []) 221 self._curRow = oldRow 222 return column

223 224 #-------------------------------------------------------------------------- 225

226 - def getAttr(self, attrName, **kwds):

227 # @@TODO: Also supply __getitem__(), but avoid conflict with get(). 228 """ 229 Returns the value of the given attribute in the current row (as set by 230 setCurRow()). If a keyword is supplied then the attribute value is 231 sought for the row where that keyword value is true instead of the 232 current row. 233 234 @param attrName: Name of attribute value to retrieve. 235 @type attrName: str 236 @param kwds: Optional keyword argument list, to specify a 237 particular row other than the current row, e.g. 238 getAttr('title', programmeID=101). NB: Keywords must 239 define a unique row (otherwise the row is the first 240 instance of this keyword value). 241 @type kwds: dict 242 243 @return: Value of the attribute in the current row or the row where the 244 keyword argument value is true. 245 @rtype: scalar 246 247 """ 248 if kwds: 249 oldRow, self._curRow = self._curRow, None 250 self.setCurRow(**kwds) 251 252 value = (self._curRow.get(attrName.lower()) if self._curRow else 253 self.get(attrName.lower())) 254 if kwds: 255 self._curRow = oldRow 256 257 return value

258 259 #-------------------------------------------------------------------------- 260

261 - def updateDetails(self, uidName=None):

262 """ 263 Updates the Table object with the latest database values. The current 264 row will be reset unless an unchanged unique ID can be supplied to 265 index the new current row. 266 267 @param uidName: Column name that contains the unique ID values in the 268 table that will unchanged. 269 @type uidName: str 270 271 """ 272 if uidName and self._curRow: 273 uID = self.getAttr(uidName) 274 Table.__init__(self, self.name, self._db) 275 if not uidName: 276 self._curRow = None 277 elif self._curRow: 278 self._setCurRow(uidName, uID, 279 caller="DataFactory.Table.updateDetails(uidName=")

280 281 #------------------------------------------------------------------------------ 282

283 -class View(dict):

284 """ 285 Use this class to store a selection of contents from any database table in 286 a local Python data object. Every selected column is stored in a dictionary 287 referenced by column name. Iterating through the container provides access 288 to the data in each row. This class provides an object-oriented alternative 289 to working directly of the list of tuples result set from a database query, 290 making code that operates on result sets containing multiple attributes 291 much more clear. 292 293 """

294 - def __init__(self, colHeader, rowList):

295 self._numRows = 0 296 self._rowIndex = None 297 colNames = [name.lower() for name in colHeader] 298 # Transpose frameSetList from a list of rows to a list of columns 299 cols = zip(*rowList) 300 super(View, self).__init__(zip(colNames, cols))

301 302 #-------------------------------------------------------------------------- 303

304 - def __getitem__(self, attr):

305 """ @return: Value of attribute for current row, if iterating, else 306 a list of values for all rows. 307 @rtype: object 308 """ 309 try: 310 # Try to return just attribute value in current row 311 return self.get(attr.lower())[self._rowIndex] 312 except TypeError: 313 # Not inside a loop so return full row-set 314 return self.get(attr.lower())

315 316 #-------------------------------------------------------------------------- 317

318 - def __iter__(self):

319 """ This class can be iterated. """ 320 return self

321 322 #-------------------------------------------------------------------------- 323

324 - def __len__(self):

325 """ @return: Number of rows stored in this container. 326 @rtype: int 327 """ 328 return self._numRows

329 330 #-------------------------------------------------------------------------- 331

332 - def next(self):

333 """ Iterate through rows. """ 334 try: 335 self._rowIndex += 1 336 except TypeError: 337 self._rowIndex = 0 338 if self._rowIndex == self._numRows: 339 self._rowIndex = None # @@TODO: May want to save state? 340 raise StopIteration 341 return self

342 343 #-------------------------------------------------------------------------- 344

345 - def values(self):

346 if self._rowIndex is not None: 347 return [self[key][self._rowIndex] for key in self.keys()] 348 else: 349 return super(View, self).values()

350 351 #------------------------------------------------------------------------------ 352

353 -class CurationTaskTable(Table):

354 # @@TODO: Should live in CuSession 355 """ A specific database table container for the CurationTask table. """ 356

357 - def __init__(self, db, cuNum):

358 """ 359 Accesses the default database and extracts the requested table data 360 for this Curation task. 361 362 @param db: Database connection. 363 @type db: DbSession 364 @param cuNum: Curation task number. 365 @type cuNum: int 366 367 """ 368 super(CurationTaskTable, self).__init__('CurationTask', db) 369 self.setCurRow(cuID=cuNum)

370 371 #-------------------------------------------------------------------------- 372

373 - def getDescription(self):

374 """ @return: Curation task description. 375 @rtype: str 376 """ 377 return self.getAttr('description')

378 379 #-------------------------------------------------------------------------- 380

381 - def updateDetails(self):

382 """ Updates CurationTaskTable object with the latest database values. 383 """ 384 super(CurationTaskTable, self).updateDetails(uidName="cuID")

385 386 #------------------------------------------------------------------------------ 387

388 -class ProgrammeTable(Table):

389 """ A specific database table container for the Programme table. 390 """ 391 #: Dictionary of acronyms for each programme ID. 392 _acronym = None 393 #: Dictionary of monthly-split detection tables for every programme. 394 _detTablesOfProg = None 395 #: Dictionary of programmme IDs given the programme acronym. 396 _progNameToID = None 397 #: Dictionary containing a list of programme IDs for every table. 398 _progIDsForTable = None 399 400 #-------------------------------------------------------------------------- 401

402 - def __init__(self, db=None):

403 """ 404 Accesses the default database, extracts the requested table data, and 405 prepares special access objects: acronym, progNameToID and 406 progIDsForTable. 407 408 @param db: Optional alternative database connection. 409 @type db: DbSession 410 411 """ 412 super(ProgrammeTable, self).__init__('Programme', db) 413 self._initialiseProgramme()

414 415 #-------------------------------------------------------------------------- 416

417 - def _initialiseProgramme(self):

418 """ 419 Accesses the default database, extracts the requested table data, and 420 prepares special access objects: acronym, progNameToID and 421 progIDsForTable. 422 423 """ 424 self._detTablesOfProg = dict() 425 426 # Determine programme acronyms from the detection table names 427 self._acronym = dict((progID, self._parseAcronym(programmeID=progID)) 428 for progID in self.getProgIDList(excludeNew=False)) 429 430 # Distinguish SV data 431 for progID in self._acronym: 432 if progID in [11, 12, 13, 14, 15]: 433 self._acronym[progID] += '_sv' 434 435 self._progNameToID = \ 436 dict((value.lower(), key) for key, value in self._acronym.items()) 437 438 self._progIDsForTable = defaultdict(list) 439 for attrName, values in self.items(): 440 if 'table' in attrName: 441 for index, tableName in enumerate(values): 442 self._progIDsForTable[tableName.lower()] += \ 443 [self.getAll('programmeID')[index]] 444 445 try: # Remove default row if it exists 446 del self._progIDsForTable[dbc.charDefault().lower()] 447 except KeyError: 448 pass

449 450 #-------------------------------------------------------------------------- 451

452 - def _parseAcronym(self, **kwds):

453 """ @return: A case-sensitive acronym for the programme. 454 @rtype: str 455 """ 456 if self.isSatelliteProg(**kwds): 457 return self.getAttr('dfsIDString', **kwds).replace('/', '').lower() 458 459 try: 460 return self.getAttr('detectionTable', **kwds)\ 461 .replace('Detection', '') 462 except AttributeError: 463 return self.getAttr('title', **kwds)

464 465 #-------------------------------------------------------------------------- 466

467 - def getAcronym(self, progID=None):

468 """ 469 Converts the given programme ID into the corresponding programme 470 acronym. If none supplied then the current programme's acronym is 471 returned. 472 473 @param progID: Optional unique identifier for the programme. 474 @type progID: int 475 476 @return: Programme acronym (case-sensitive; meaning lower-case unless 477 given otherwise in detection table name). 478 @rtype: str 479 480 """ 481 return self._acronym.get(progID or self.getAttr('programmeID'))

482 483 #-------------------------------------------------------------------------- 484

485 - def getAstroInfoTable(self, **kwds):

486 """ @returns: AstrometricInfo table name for current programme. 487 @rtype: str 488 """ 489 varTable = self.getAttr('variabilityTable', **kwds) 490 if type(varTable) is str: 491 return varTable.replace('Variability', 'AstrometricInfo') 492 else: 493 return [table.replace('Variability', 'AstrometricInfo') 494 for table in varTable]

495 496 #-------------------------------------------------------------------------- 497

498 - def getAstrometryTable(self, **kwds):

499 """ @returns: DetectionAstrometry table name for current programme. 500 @rtype: str 501 """ 502 detTable = self.getAttr('detectionTable', **kwds) 503 if type(detTable) is str: 504 return detTable + 'Astrometry' 505 else: 506 return [table + 'Astrometry' for table in detTable]

507 508 #-------------------------------------------------------------------------- 509

510 - def getBestAperTable(self, **kwds):

511 """ @returns: AstrometricInfo table name for current programme. 512 @rtype: str 513 """ 514 varTable = self.getAttr('variabilityTable', **kwds) 515 if type(varTable) is str: 516 return varTable.replace('Variability', 'BestAper') 517 else: 518 return [table.replace('Variability', 'BestAper') 519 for table in varTable]

520 521 #-------------------------------------------------------------------------- 522

523 - def getDetectionTable(self, **kwds):

524 """ @returns: DetectionRaw table name for current programme. 525 @rtype: str 526 """ 527 detTable = self.getAttr('detectionTable', **kwds) 528 if type(detTable) is str: 529 return detTable + 'Raw' 530 else: 531 return [table + 'Raw' for table in detTable]

532 533 #-------------------------------------------------------------------------- 534

535 - def getListRemeasTable(self, **kwds):

536 """ @returns: ListRemeasurement table name for current programme. 537 @rtype: str 538 """ 539 return self.getAttr('listRemeasTable', **kwds)

540 541 #-------------------------------------------------------------------------- 542

543 - def getMergeLogTable(self, **kwds):

544 """ @returns: MergeLog table name for current programme. 545 @rtype: str 546 """ 547 return self.getAttr('mergeLogTable', **kwds)

548 549 #-------------------------------------------------------------------------- 550

551 - def _getMfIdRange(self, month):

552 """ @returns: Multiframe ID range, in SQL syntax, of a given month that 553 has a monthly-split detection table. 554 @rtype: list(str) 555 """ 556 mfIdRange = self._db.query("definition", "sys.check_constraints", 557 "name = 'multiframeIDRaw%s'" % month, firstOnly=True) 558 559 mfIDs = set(map(int, utils.expandNumberRange( 560 mfIdRange.replace('(', '').replace(')', '') 561 .replace('[', '').replace(']', '').replace('.', '') 562 .replace(' OR', ',').replace(' AND ', '-') 563 .replace('multiframeID', '').replace('=', '') 564 .replace('<', '').replace('>', '')).split(','))) 565 566 return " AND " + mfIdRange, mfIDs

567 568 #-------------------------------------------------------------------------- 569

570 - def _getMonthlyDetectionTables(self, detTable):

571 """ @returns: Current monthly DetectionRaw table names. 572 @rtype: list(str) 573 """ 574 return sorted(self._db.query("name", "sysobjects", 575 whereStr="xtype = 'U' AND name LIKE '%s%%'" % (detTable + "20")))

576 577 #-------------------------------------------------------------------------- 578

579 - def getMonthlyDetectionTables(self, dateRange=None, deepOnly=False, 580 excludeDeeps=False, mfIDs=None, **kwds):

581 """ @returns: All DetectionRaw table names for current programme, up to 582 and including the last date given in the date range. If a 583 single mfID is given then it returns the single table 584 that contains that mfID, otherwise it returns a list of 585 all tables together with their mfID constraint ranges 586 that contain the list of given mfIDs. 587 @rtype: list(str) or list(tuple(str, str)) or str 588 """ 589 detTable = self.getDetectionTable(**kwds) 590 if type(detTable) is not str: 591 return list(utils.unpackList( 592 self._getMonthlyDetectionTables(detTableName) 593 for detTableName in detTable)) 594 595 detTables = self._detTablesOfProg.get(detTable) 596 if not detTables: 597 detTables = self._getMonthlyDetectionTables(detTable) or [detTable] 598 self._detTablesOfProg[detTable] = detTables 599 600 hasMonthlyTables = len(detTables) > 1 601 # NB: Don't need to worry about start dates for VVV (only) 602 if dateRange and hasMonthlyTables: 603 lastTable = detTable + str(dateRange.end.year) 604 lastTable += str(dateRange.end.month).zfill(2) 605 if lastTable in detTables: 606 lastTableIdx = detTables.index(lastTable) + 1 607 adDeepTables = [] 608 if not excludeDeeps: 609 adDeepTables = [table for table in detTables[lastTableIdx:] 610 if table.endswith('00')] 611 612 detTables = detTables[:lastTableIdx] + adDeepTables 613 614 if deepOnly and hasMonthlyTables: 615 detTables = [table for table in detTables if table.endswith('00')] 616 617 if excludeDeeps and hasMonthlyTables: 618 detTables = \ 619 [table for table in detTables if not table.endswith('00')] 620 621 if mfIDs is None: 622 return detTables 623 624 if not hasMonthlyTables: 625 return list(zip(detTables, [''])) 626 627 mfIDs = set(mfIDs) 628 mfIdRanges = [] 629 for table in detTables: 630 sqlRange, mfIdRange = \ 631 self._getMfIdRange(table.replace(detTable, '')) 632 633 if mfIdRange.intersection(mfIDs): 634 if len(mfIDs) is 1: 635 return table 636 637 mfIdRanges.append((table, sqlRange)) 638 639 return mfIdRanges

640 641 #-------------------------------------------------------------------------- 642

643 - def getName(self, **kwds):

644 """ 645 Simple wrapper for getAttr('title'). 646 647 @return: Full name of the programme. 648 @rtype: str 649 650 """ 651 return self.getAttr('title', **kwds)

652 653 #-------------------------------------------------------------------------- 654

655 - def getOrphanTable(self, **kwds):

656 """ @returns: Orphan table name for current programme. 657 @rtype: str 658 """ 659 varTable = self.getAttr('variabilityTable', **kwds) 660 if type(varTable) is str: 661 return varTable.replace('Variability', 'Orphan') 662 else: 663 return [table.replace('Variability', 'Orphan') 664 for table in varTable]

665 666 #-------------------------------------------------------------------------- 667

668 - def getPhotometryTable(self, **kwds):

669 """ @returns: DetectionPhotometry table name for current programme. 670 @rtype: str 671 """ 672 detTable = self.getAttr('detectionTable', **kwds) 673 if type(detTable) is str: 674 return detTable + 'Photometry' 675 else: 676 return [table + 'Photometry' for table in detTable]

677 678 #-------------------------------------------------------------------------- 679

680 - def getSourceTable(self, **kwds):

681 """ @returns: Source table name for current programme. 682 @rtype: str 683 """ 684 return self.getAttr('sourceTable', **kwds)

685 686 #-------------------------------------------------------------------------- 687

688 - def getSourceRemeasTable(self, **kwds):

689 """ @returns: SourceRemeasurement table name for current programme. 690 @rtype: str 691 """ 692 return self.getAttr('sourceRemeasTable', **kwds)

693 694 #-------------------------------------------------------------------------- 695

696 - def getSynopticBestMatchTable(self, **kwds):

697 """ @returns: BestMatch table name for current programme. 698 @rtype: str 699 """ 700 return self.getAttr('synopticBestMatchTable', **kwds)

701 702 #-------------------------------------------------------------------------- 703

704 - def getSynopticMergeLogTable(self, **kwds):

705 """ @returns: SynopticMergeLog table name for current programme. 706 @rtype: str 707 """ 708 srcTable = self.getAttr('synopticSourceTable', **kwds) 709 if type(srcTable) is str: 710 return srcTable.replace('Source', 'MergeLog') 711 else: 712 return [table.replace('Source', 'MergeLog') for table in srcTable]

713 714 #-------------------------------------------------------------------------- 715

716 - def getSynopticNeighbourTable(self, **kwds):

717 """ @returns: Name of neighbour table that was created for synoptic 718 curation for current programme. This varies depending on 719 whether a synoptic source table has been made. 720 @rtype: str 721 """ 722 bestMatchTableName = self.getAttr('synopticBestMatchTable', **kwds) 723 return bestMatchTableName.replace('BestMatch', '')

724 725 #-------------------------------------------------------------------------- 726

727 - def getSynopticSourceTable(self, **kwds):

728 """ @returns: SynopticSource table name for current programme. 729 @rtype: str 730 """ 731 return self.getAttr('synopticSourceTable', **kwds)

732 733 #-------------------------------------------------------------------------- 734

735 - def getTileSetTable(self, **kwds):

736 """ @returns: TileSet table name for current programme. 737 @rtype: str 738 """ 739 srcTable = self.getAttr('sourceTable', **kwds) 740 if type(srcTable) is str: 741 return srcTable.replace('Source', 'TileSet') 742 else: 743 return [table.replace('Source', 'TileSet') for table in srcTable]

744 745 #-------------------------------------------------------------------------- 746

747 - def getTilePpwTable(self, **kwds):

748 """ @returns: TilePawPrints table name for current programme. 749 @rtype: str 750 """ 751 srcTable = self.getAttr('sourceTable', **kwds) 752 if type(srcTable) is str: 753 return srcTable.replace('Source', 'TilePawPrints') 754 else: 755 return [table.replace('Source', 'TilePawPrints') 756 for table in srcTable]

757 758 #-------------------------------------------------------------------------- 759

760 - def getVariabilityTable(self, **kwds):

761 """ @returns: Variability table name for current programme. 762 @rtype: str 763 """ 764 return self.getAttr('variabilityTable', **kwds)

765 766 #-------------------------------------------------------------------------- 767

768 - def getVarFrameSetInfoTable(self, **kwds):

769 """ @returns: VarFrameSetInfo table name for current programme. 770 @rtype: str 771 """ 772 varTable = self.getAttr('variabilityTable', **kwds) 773 if type(varTable) is str: 774 return varTable.replace('iability', 'FrameSetInfo') 775 else: 776 return [table.replace('iability', 'FrameSetInfo') 777 for table in varTable]

778 779 #-------------------------------------------------------------------------- 780

781 - def getSchemaScript(self, **kwds):

782 """ 783 Get the name of the .sql script file that describes the schema for 784 tables belonging to the current programme. For non-survey programmes 785 the file name includes the NonSurvey directory name. 786 787 @param kwds: Optionally specify a keyword argument to select the 788 desired programme in the table based on an attribute value 789 that defines a unique row. For example: 790 C{getSchemaScript(programmeID=101)}. If no keyword 791 supplied, then the current programme as set with 792 L{setCurRow()} is used. If no current programme is set, 793 then None is returned. If the keyword does not reference 794 a unique row then the first row that contains this 795 attribute value is used as the current programme. 796 @type kwds: dict 797 798 @return: Schema .sql script file name (and NonSurvey directory name if 799 non-survey). 800 @rtype: str or None 801 802 """ 803 if not self._curRow and not kwds: 804 return 805 806 return self.getAttr('catalogueSchema', **kwds)

807 808 #-------------------------------------------------------------------------- 809

810 - def getProductTypes(self, **kwds):

811 """ @return: Product types to be created for the current programme. 812 @rtype: list(str) 813 """ 814 # Only types where there are products defined. 815 usedProductTypes = [pType for pType in self._db.sysc.productTypes 816 if self._db.queryEntriesExist("Required%s" % pType, 817 where="programmeID=%s" % self.getAttr("programmeID"))] 818 819 maxProductLayer = self.getAttr("sourceProdType", **kwds) 820 if maxProductLayer not in usedProductTypes: 821 return usedProductTypes 822 823 index = usedProductTypes.index(maxProductLayer) 824 825 return usedProductTypes[:index + 1]

826 827 #-------------------------------------------------------------------------- 828

829 - def getProgIDfromName(self, acronym):

830 """ 831 Converts a programme acronym into the corresponding programme ID. 832 833 @param acronym: Acronym for the programme. 834 @type acronym: str 835 836 @return: Unique programme ID. 837 @rtype: int 838 839 """ 840 return self._progNameToID.get(acronym.lower())

841 842 #-------------------------------------------------------------------------- 843

844 - def getProgIDsfromTable(self, tableName):

845 """ 846 Finds the corresponding programme ID for a given database table. Rather 847 than just looking for the acronym in the table name this function, does 848 an exact table match from the list given in the Programme table. 849 850 @param tableName: Name of a database table that potentially belongs to 851 a programme. 852 @type tableName: str 853 854 @return: Set of programmeIDs that given table belongs to. 855 @rtype: set(int) 856 857 """ 858 tableName = tableName.replace('Raw', '').replace('Astrometry', '')\ 859 .replace('Photometry', '').lower() 860 861 if 'detection20' in tableName: 862 tableName = tableName[:-6] 863 864 return set(self._progIDsForTable[tableName.lower()])

865 866 #-------------------------------------------------------------------------- 867

868 - def getProgIDList(self, onlyNonSurvey=False, onlySurvey=False, 869 excludeNew=True, excludeSV=False):

870 """ 871 Returns a list of programme IDs that are defined in the Programme table 872 of the database and are correctly set-up with detection tables. 873 874 @param onlyNonSurvey: If True, just list non-survey programmes. 875 @type onlyNonSurvey: bool 876 @param onlySurvey: If True, just list survey programmes. 877 @type onlySurvey: bool 878 @param excludeNew: If True, exclude programmes without detection 879 tables. 880 @type excludeNew: bool 881 @param excludeSV: If True, exclude science verification programmes. 882 @type excludeSV: bool 883 884 @return: A list of all of the programme IDs in the database. 885 @rtype: list(int) 886 887 """ 888 aboveID = (dbc.nonSurveyProgrammeOffset() if onlyNonSurvey else 0) 889 try: 890 belowID = (dbc.nonSurveyProgrammeOffset() if onlySurvey else 891 1 + max(self.getAll("programmeID"))) 892 except ValueError: 893 belowID = 0 # No programmes defined in database 894 895 return sorted(progID for progID in set(self.getAll("programmeID")) 896 if aboveID < progID < belowID and (not excludeSV or 897 progID not in [11, 12, 13, 14, 15]) and (not excludeNew or 898 self.getAttr("detectionTable", programmeID=progID) 899 != dbc.charDefault()))

900 901 #-------------------------------------------------------------------------- 902

903 - def getProjection(self, **kwds):

904 """ @return: Projection type, e.g. ZPN or TAN, for WCS co-ordinates in 905 the products curated in current programme. 906 @rtype: str 907 """ 908 prodType = self.getAttr("epochFrameType", **kwds) 909 if type(prodType) is str: 910 return 'ZPN' if prodType != 'tile' else 'TAN' 911 912 return [('ZPN' if p != 'tile' else 'TAN') for p in prodType]

913 914 #-------------------------------------------------------------------------- 915

916 - def isAutomated(self, **kwds):

917 """ @return: True, if this programme is automatically set up and 918 curated via AutoCurate. 919 @rtype: bool 920 """ 921 return self.getAttr('frameSetTolerance', **kwds) < 0

922 923 #-------------------------------------------------------------------------- 924

925 - def isDeep(self, **kwds):

926 """ @return: True, if this programme requires deep stack frames. 927 @rtype: bool 928 """ 929 return self.getAttr('variabilityTable', **kwds) != dbc.charDefault()

930 931 #-------------------------------------------------------------------------- 932

933 - def isLowGalLat(self, **kwds):

934 """ @return: True, if this programme is mainly based in the Galactic 935 Plane or Magellanic Clouds: high stellar density and 936 cannot use Schlegal Dust Maps 937 @rtype: bool 938 """ 939 return self.getAttr('isLowGalacticLat', **kwds) != dbc.no()

940 941 #-------------------------------------------------------------------------- 942

943 - def isMosaicked(self, **kwds):

944 """ @return: True, if this programme requires mosaic frames. 945 @rtype: bool 946 """ 947 return self._db.existsTable("RequiredMosaic") \ 948 and self._db.queryEntriesExist("RequiredMosaic", 949 "programmeID=%s" % self.getAttr('programmeID', **kwds))

950 951 #-------------------------------------------------------------------------- 952

953 - def isNonSurvey(self, **kwds):

954 """ @return: True, if this programme is a non-survey programme. 955 @rtype: bool 956 """ 957 return \ 958 self.getAttr('programmeID', **kwds) >= dbc.nonSurveyProgrammeOffset()

959 960 #-------------------------------------------------------------------------- 961

962 - def isSatelliteProg(self, **kwds):

963 """ @return: True, if this programme is a satellite programme. 964 @rtype: bool 965 """ 966 return self.getAttr('programmeID', **kwds) >= 11000

967 968 #-------------------------------------------------------------------------- 969

970 - def isShallowTiled(self, **kwds):

971 """ @return: True, if this programme's merged source product should 972 just consists of shallow tiles. 973 @rtype: bool 974 """ 975 return (self.getAttr('sourceProdType', **kwds) == 'tile' 976 and self.getAttr('programmeID', **kwds) 977 in [self._db.sysc.scienceProgs.get('VHS')])

978 979 #-------------------------------------------------------------------------- 980

981 - def isSynoptic(self, **kwds):

982 """ @return: True, if this programme requires synoptic merging. 983 @rtype: bool 984 """ 985 return self.getAttr('synopticSourceTable', **kwds) != dbc.charDefault()

986 987 #-------------------------------------------------------------------------- 988

989 - def isUkidss(self, **kwds):

990 """ @return: True, if this programme is a UKIDSS programme. 991 @rtype: bool 992 """ 993 return self._db.sysc.isWSA() \ 994 and self.getAttr('programmeID', **kwds) < dbc.calibrationProgrammeID()

995 996 #-------------------------------------------------------------------------- 997

998 - def setProgID(self, progStr):

999 """ 1000 Sets the current row to the given programme and returns the unique 1001 programme ID. 1002 1003 @param progStr: Any string describing the programme; the unique ID, 1004 the acronym or the dfsIDString. 1005 @type progStr: str 1006 1007 @return: Unique programme ID. 1008 @rtype: int 1009 1010 """ 1011 if progStr.isdigit(): 1012 progID = int(progStr) 1013 else: 1014 progID = self.getProgIDfromName(progStr.replace('/', '')) 1015 if not progID: 1016 raise SystemExit("<ERROR> No such programme: %s in database" 1017 " %s.%s" % (progStr, self._db.server, self._db.database)) 1018 1019 self.setCurRow(programmeID=progID) 1020 return progID

1021 1022 #-------------------------------------------------------------------------- 1023

1024 - def updateDetails(self):

1025 """ Updates the ProgrammeTable object with the latest database values. 1026 """ 1027 super(ProgrammeTable, self).updateDetails(uidName="programmeID") 1028 self._initialiseProgramme()

1029 1030 #-------------------------------------------------------------------------- 1031

1032 - def useSExtractor(self, **kwds):

1033 """ @return: True, if the catalogues for this programme are produced 1034 with SExtractor rather than CASU's cirdr package. 1035 @rtype: bool 1036 """ 1037 return self.getAttr('extractTool', **kwds) == 'SEX'

1038 1039 #------------------------------------------------------------------------------ 1040 # Change log: 1041 # 1042 # 5-May-2004, IAB: Original version that deprecated the stuff in the 1043 # DataTypes directory. 1044 # 18-May-2004, IAB: Added getProperty function to base DataEntity class. 1045 # 27-May-2004, NCH: Fixed cuID bug in CurationTask class so as to return 1046 # task description, not simply cuID again; pushed the 1047 # SQL in DataTable class down a layer into SqlWrappers; 1048 # removed redundant table names from metadata in DataTable. 1049 # 31-May-2004, NCH: Added more properties to Programme subclass. 1050 # 24-Jun-2004, IAB: Added more documentation in the triple quotes ''' 1051 # 2-Aug-2004, NCH: Fixed very minor typos in comments 1052 # 19-Aug-2004, NCH: Added another property to Filter. 1053 # 23-Aug-2004, ETWS: Added catalogue schema to Programme subclass colmap 1054 # 7-Sep-2004, NCH: Changed triple quotes to doubles; added more attributes 1055 # to programme subclass column map. 1056 # 8-Dec-2004, NCH: Added new data entity subclass "survey". 1057 # 18-Jan-2005, NCH: More colmaps in subclass Programme for list-driven 1058 # photometry tables. 1059 # 9-Mar-2005, ETWS: Fixed typo in DataEntity docstring 1060 # 24-Nov-2005, NCH: Added another column map for Programme instance. 1061 # 12-Jul-2006, RSC: Created class Table() which implements every function of 1062 # this module in a more powerful and simple way. Also, and 1063 # additional ProgrammeTable subclass to provide additional 1064 # specific methods for Programme data. 1065 # 3-Aug-2006, RSC: Attributes are now case-insensitive. Added 1066 # CurtationTaskTable class. 1067 # 28-Aug-2006, RSC: * New View class. 1068 # * Added getSchemaScript() method to ProgrammeTable. 1069 # * Table now stores column names as lower case. 1070 # 31-May-2007, RSC: Finally removed obsolete methods. 1071 # 25-Mar-2008, RSC: Updated ProgrammeTable to supply all the new survey table 1072 # names. 1073 # 10-Apr-2008, RSC: Removed seekAttr() method in favour of adding the seek 1074 # keywords directly into getAttr(). 1075 # 4-Sep-2008, RSC: Simplified isDeep() and isMosaicked() methods to make them 1076 # more useful, and included an isSynoptic() method. 1077

Source Code for Module wsatools.DataFactory