Package wsatools :: Module SystemConstants
[hide private]

Source Code for Module wsatools.SystemConstants

   1  #------------------------------------------------------------------------------ 
   2  #$Id: SystemConstants.py 10238 2014-03-07 12:17:46Z EckhardSutorius $ 
   3  """ 
   4     System constants for the VISTA Data Flow System. A set of global "symbols" 
   5     for use by all Python curation scripts and modules, compartmentalised here 
   6     to minimise maintenance overheads. 
   7   
   8     @author: N.C. Hambly 
   9     @org:    WFAU, IfA, University of Edinburgh 
  10   
  11     @newfield contributors: Contributors, Contributors (Alphabetical Order) 
  12     @contributors: J. Bryant, R.S. Collins, N.J.G. Cross, E. Sutorius 
  13  """ 
  14  #------------------------------------------------------------------------------ 
  15  from __future__      import division, print_function 
  16  from future_builtins import map 
  17   
  18  from   collections import namedtuple 
  19  from   operator    import attrgetter, methodcaller 
  20  import os 
  21   
  22  from wsatools.ObsCalendar import ObsCal, VistaCal, WfcamCal, OmegaCal, GesCal 
  23  #------------------------------------------------------------------------------ 
  24  # Test for environment variables on import 
  25   
  26  try: 
  27      assert os.path.exists(os.getenv('WSA_SANDBOX')), \ 
  28             'WSA_SANDBOX = %s' % os.getenv('WSA_SANDBOX') 
  29      assert os.path.exists(os.path.join(os.getenv('WSA_SANDBOX'), 
  30                            os.getenv('SRC_BRANCH'))), \ 
  31             'SRC_BRANCH = %s' % os.getenv('SRC_BRANCH') 
  32      assert os.getenv('USER') is not None, 'USER = None' 
  33  except AssertionError as error: 
  34      print("Incorrect environment variable:", error) 
  35      raise SystemExit 
  36   
  37  #------------------------------------------------------------------------------ 
  38   
  39  #: The format of tuples giving CASU file system ranges. 
  40  AlphaNum = namedtuple('AlphaNum', 'name alpha num') 
41 42 #------------------------------------------------------------------------------ 43 44 -class DepCodes(object):
45 """ A collection of deprecation code constants. 46 """ 47 constitDep = 104 #: Deep deprecated because constituent deprecated 48 deepStackOnly = 50 #: Exclude from deep stacks only. 49 extProdComp = 112 #: Component of external product. 50 incompGrp = 6 #: Incomplete group. 51 incompProv = 19 #: Incomplete provenance. 52 intermitDet = 51 #: Intermittency problem in channel 14 of detector. 53 invalidCat = 7 #: Invalid catalogue values. 54 invalidDet = 2 #: Dead detector frame. 55 invalidMet = 3 #: Invalid metadata values. 56 invalidSee = 8 #: Invalid seeing value (non-positive). 57 ignoredObs = 26 #: Observation is flagged to be ignored. 58 missingCat = 1 #: Multiframe has no catalogue where expected. 59 missingDet = 103 #: Detector missing from deep stack. 60 nonDep = 0 #: Non-deprecated. 61 poorEsoGrade = 56 #: Deprecated because EsoGrade is 'C', 'D' or 'R' 62 propDepCom = 18 #: Propagation of deprecated component to multiframe. 63 propDepDet = 102 #: Propagation of deprecated detector to detections. 64 propDepFra = 100 #: Propagation of deprecated detectors to multiframe. 65 propDepObs = 101 #: Propagation of deprecated multiframe to detectors. 66 poorPhotom = 110 #: Stack with poor photometry that is part of a deep. 67 rejectObs = 27 #: Observation is flagged to be rejected. 68 reprocAltn = 111 #: Data superseded by alternative reprocessing. 69 reprocCASU = 128 #: Data superseded due to reprocessing by CASU. 70 reprocWFAU = 255 #: Data superseded due to reprocessing by WFAU. 71 testDetect = 53 #: Test detections - propagate through, but don't 72 # use in stacking, source or variability, but do in 73 # cu19 and in BestMatch 74 75 selectDeprecated = "deprecated NOT IN (%s, %s, %s)" \ 76 % (nonDep, deepStackOnly, intermitDet) 77 78 selectNonDeprecated = selectDeprecated.replace("NOT ", "") 79 80 selectDepAndNotTest = "deprecated NOT IN (%s, %s, %s, %s)" \ 81 % (nonDep, deepStackOnly, intermitDet, testDetect) 82 83 selectNonDepAndTest = selectDepAndNotTest.replace("NOT ", "")
84
85 #------------------------------------------------------------------------------ 86 87 -class SystemConstants(object):
88 """ Constants common to the entire VDFS system. 89 """ 90 #: Observation dates for the survey. 91 obsCal = ObsCal() 92 93 #{ Numerical constants 94 minNumCPUs = 6 95 """ Minimum number of CPUs for tiling software. 96 """ 97 one_kilobyte = 1024 98 """ A kilobyte in bytes. 99 """ 100 one_megabyte = 1024 * 1024 101 """ A megabyte in bytes. 102 """ 103 one_gigabyte = 1024 * 1024 * 1024 104 """ A gigabyte in bytes. 105 """ 106 sqlDataTypeSize = dict(datetime=8, float=8, real=4, bigint=8, int=4, 107 smallint=2, tinyint=1) 108 """ Dictionary of SQL data type sizes in bytes referenced by data type 109 name. 110 """ 111 #{ Catalogue server constants 112 adminDatabase = "master" 113 """ The name of the database context within which to perform miscellaneous 114 administrative commands (e.g. attach/detach etc). For Microsoft SQL 115 Server, this database context is "master". 116 """ 117 catServerFileSep = chr(0x5c) 118 """ Path separator character for the catalogue server's O/S. 119 """ 120 catServerSharePath = 2 * catServerFileSep + os.getenv('HOST').upper() \ 121 + catServerFileSep + 'share' + catServerFileSep 122 """ Share path to curation server as seen by the catalogue server. 123 """ 124 clusterName = "ramses" 125 """ Root name of the cluster servers. 126 """ 127 loadServerVolumes = ('G', 'H', 'I') 128 """ Default load server volume configuration (for cluster servers). 129 """ 130 #{ Pixel server file system 131 curationServers = ["thoth", "menkaure", "shepseskaf", "userkaf", "sahure", 132 "neferefre", "ini", "unas"] 133 """Available curation servers 134 """ 135 criticalDisks = ["/disk01/wsa", "/disk02/wsa", "/disk05/wsa", 136 "/disk06/wsa", "/disk10/wsa", "/disk33/vsa"] 137 """ Disks running a dump system backup or moved to other disks. 138 """ 139 deprecatedDataStorage = ["/disk06/wsa", "/disk08/wsa", "/disk09/wsa"] 140 """ Disks on which development takes place, so no other data will 141 be stored there. 142 """ 143 developDisks = ["/disk47/sys", "/disk34/vsa", "/disk34/wsa", "/disk34/osa"] 144 """ Disks on which development takes place, so no other data will be stored 145 there. 146 """ 147 infinibandServers = ['ini', 'neferefre', 'kakai', 'sahure', 'userkaf', 148 'shepseskaf', 'menkaure'] 149 """ Curation servers with infiniband connections. 150 """ 151 sysPath = "/disk01/wsa/sys" 152 """ System path for lock and shell scripts. 153 """ 154 userInterfaceTempStorage = ["/disk07/wsa"] 155 """ Disks on which development takes place, so no other data will 156 be stored there. 157 """ 158 #{ Pixel server common directories 159 casu2massDir = os.path.join(developDisks[0], 'Viz2Mass', 'fits') 160 """ Directory containing CASU 2mass catalogues for wcsfit 161 """ 162 compressImDir = os.path.join('products', 'jpgs') 163 """ Directory for compressed images (jpgs). 164 """ 165 deprecatedComprImDir = os.path.join('products', 'deprecated_jpgs') 166 """ Path to directory for deprecated compressed images (jpgs). 167 """ 168 deprecatedDir = os.path.join('ingest', 'deprecated') 169 """ Directory for deprecated files. 170 """ 171 diffDir = os.path.join('products', 'diffs') 172 """ Directory for difference image products. 173 """ 174 downloadDir = os.path.join('ingest', 'dload') 175 """ Final top directory where the data will be stored. 176 """ 177 extProdDir = os.path.join(developDisks[0], 'ExternalProducts') 178 """ Directory for external products. 179 """ 180 esoExtDir = os.path.join(extProdDir, 'teamCats') 181 """ Directory for external team catalogues 182 """ 183 fitsDir = os.path.join('ingest', 'fits') 184 """ Final top directory where the data will be stored. 185 """ 186 headerDir = os.path.join('ingest', 'headers') 187 """ Final top directory where the header metadata will be stored. 188 """ 189 listDrivDir = os.path.join('products', 'list') 190 """ Directory for list driven catalogue products. 191 """ 192 mosaicDir = os.path.join('products', 'mosaics') 193 """ Directory for mosaic image products. 194 """ 195 productLogDir = os.path.join('products', 'logs') 196 """ Directory for production log files. 197 """ 198 productsDir = 'products' 199 """ Directory for WFAU produced files, ie. stacked, mosaiced, and 200 difference images. 201 """ 202 stackDir = os.path.join('products', 'stacks') 203 """ Directory for stacked image products. 204 """ 205 swarpDir = os.path.join(extProdDir, 'swarpMos') 206 """ Directory for external SWARP mosaics 207 """ 208 tempProductsDir = os.path.join('products', 'temp') 209 """ Directory for temporary products. 210 """ 211 tileDir = os.path.join('products', 'tiles') 212 """ Directory for tiled image products. 213 """ 214 #{ FITS file naming conventions 215 #: Prefix for WFAU produced FITS files. 216 wfauPrefix = 'e' 217 218 #: Prefix for dark frame FITS files. 219 darkPrefix = 'dark_' 220 221 #: Prefix for sky frame FITS files. 222 skyPrefix = 'sky_' 223 224 #: Prefix for dark frame FITS files. 225 biasPrefix = 'bias_' 226 227 #: Suffix for catalogue FITS files. 228 catSuffix = '_cat' 229 230 #: Suffix for fringe FITS files. 231 fringeSuffix = '_fringe' 232 233 #: Suffix for confidence image map FITS files. 234 confSuffix = '_conf' 235 236 #: Suffix for CPM(?) FITS files. 237 cpmSuffix = '_cpm' 238 239 #: Suffix for deep stacked FITS files. 240 deepSuffix = '_dp' 241 242 #: Suffix for science frame FITS files. 243 diffSuffix = '_diff' 244 245 #: Suffix for filtered FITS files. 246 filtSuffix = '_two' 247 248 #: Prefix for filtered FITS files. 249 filtPrefix = 'tmp_' 250 251 #: Suffix for flat frame FITS files. 252 flatSuffix = '_flat' 253 254 #: Suffix for science frame FITS files. 255 leavSuffix = '_sf' 256 257 #: Suffix for list-driven catalogue FITS files. 258 listSuffix = '_list' 259 260 #: Suffix for temporary MJD map FITS file. 261 mjdSuffix = '_mjd' 262 263 #: Suffix for mosaiced FITS files. 264 mosaicSuffix = '_mos' 265 266 #: Suffix for pre-grouted tile catalogue FITS files. 267 pregrtSuffix = '_pgrt' 268 269 #: Suffix for simple? FITS files. 270 simpleSuffix = '_simple' 271 272 #: Suffix for stacked frame FITS files. 273 stackSuffix = '_st' 274 275 #: Suffix for OSA fixed catalogue FITS files. 276 fixcatSuffix = '_fix_cat' 277 278 #: Suffix for tiled FITS files. 279 tileSuffix = '_tl' 280 281 #: Extension type for catalogue FITS files. 282 catType = '.fits' 283 284 #: Extension type for compressed JPEG thumbnail image files. 285 jpgType = '.jpg' 286 287 #: Extension type for multi-extension FITS files. 288 mefType = '.fit' 289 290 #: Extension type for uncompressed types. 291 uncType = '.unc' 292 293 #: A list of frame types ordered by precedence in a combined frame type. 294 frameTypes = ['mosaic', 'tile', 'deep', 'filt', 'leav', 'diff', 295 'stack', 'conf', 'normal'] 296 297 #{ Software file structure 298 cirdrShellScript = os.path.join(os.getenv("SCOSHOME"), "usr", "local", 299 "wsa", "cirdr", "cir_startup.bash") 300 """ Full path to the shell script to set up the correct environment for 301 CASU's cirdr package of source extractor related software. 302 """ 303 emptyFitsCatalogueFileName = "empty_catalogue.fits" 304 """ Name of the default empty catalogue file. 305 """ 306 procScript = "COMMON_StoredProcsAndFuncs.sql" 307 """ Script for generic procedures and functions for all databases. 308 """ 309 sqlScriptPath = os.path.join(os.getenv("WSA_SANDBOX"), "sql") 310 """ Path to sql scripts. 311 """ 312 srcPath = os.path.join(os.getenv("WSA_SANDBOX"), os.getenv("SRC_BRANCH")) 313 """ Path to present sandbox's source code. Useful if working 314 on a different branch of the source code. 315 """ 316 webDocsPath = os.path.join(os.getenv("WEB_DIR"), "webpages") 317 """ Path to web pages. 318 """ 319 #{ SExtractor 320 defaultSexInparFile = "defaultDetCat.inpar" 321 """ Default sex inpar file. Modelled on UDS-DR8. 322 """ 323 #{ CASU data transfer constants 324 #: Command to run ftp 325 ftp_method = "ftp" 326 327 #: Directory at CASU for list files. 328 rem_camlist_dir = "/home/wfcam" 329 330 #: Remote lock file name. 331 rem_lock_name = "readlock" 332 333 #: Remote ready to copy file name. 334 rem_ready2copy_name = "OK_TO_COPY" 335 336 #: Remote(=local) success file name. 337 rem_success_name = "SUCCESSFULLY_READ" 338 339 #: Number of parallel threads to use for scp. 340 scpThreads = 1 341 342 #: The user login name for the CASU scp server. 343 scpUser = "wfcam" 344 345 #{ Synoptic syntax constants 346 #: Substring representing uncorrelated synoptic table. 347 uncSubStr = "UNC" 348 349 #: Substring representing correlated synoptic table. 350 corSubStr = "COR" 351 352 #: Substring representing both synoptic tables. 353 bthSubStr = "BOTH" 354 355 #: Substring representing variability table. 356 varSubStr = "VAR" 357 358 #{ Common instrument properties 359 #: Channel width. 360 channelWidth = 128 361 362 #: Number of pixels in detector (X). 363 instrumentXnPix = 2048 364 365 #: Number of pixels in detector (Y). 366 instrumentYnPix = 2048 367 368 #: has microstepping? 369 hasMicroStep = False 370 371 #: has variable orientation? 372 hasVarOrient = False 373 374 #: has offset positions? 375 hasOffsetPos = False 376 377 #: Has an ESO release? 378 hasEsoRelease = False 379 380 #{ Initialiser
381 - def __new__(cls, switchName=None):
382 """ Creates a new object of type SystemConstants or its subclass if the 383 supplied argument suggest a specific subclass is more appropriate. 384 385 @param switchName: Initialised SystemConstants object or else a 386 string that contains necessary data to switch 387 on, e.g. database name, the INSTRUME FITS header 388 keyword, full path to a FITS file etc. 389 @type switchName: SystemConstants or str 390 391 @rtype: SystemConstants 392 """ 393 if not switchName: 394 return object.__new__(cls) 395 396 if str(switchName).lower().replace('test', '').startswith('v') \ 397 or os.sep + 'vsa' + os.sep in str(switchName).lower(): 398 subclass = VsaConstants 399 elif str(switchName).lower().replace('test', '').startswith('ssa') \ 400 or os.sep + 'ssa' + os.sep in str(switchName).lower(): 401 subclass = SsaConstants 402 elif str(switchName).lower().replace('test', '').startswith('o') \ 403 or os.sep + 'osa' + os.sep in str(switchName).lower(): 404 subclass = OsaConstants 405 elif str(switchName).lower().replace('test', '').startswith('ges') \ 406 or os.sep + 'ges' + os.sep in str(switchName).lower(): 407 subclass = GesConstants 408 else: 409 subclass = WsaConstants 410 411 return super(cls, subclass).__new__(subclass)
412 413 #{ Convenience methods
414 - def __str__(self):
415 """ Converting a object of this class type to a string returns the 416 classes name. 417 """ 418 return self.__class__.__name__
419
420 - def isGES(self):
421 """ @return: True, if these are GES constants. 422 @rtype: bool 423 """ 424 return isinstance(self, GesConstants)
425
426 - def isSSA(self):
427 """ @return: True, if these are SSA constants. 428 @rtype: bool 429 """ 430 return isinstance(self, SsaConstants)
431
432 - def isWSA(self):
433 """ @return: True, if these are WSA constants. 434 @rtype: bool 435 """ 436 return isinstance(self, WsaConstants)
437
438 - def isVSA(self):
439 """ @return: True, if these are VSA constants. 440 @rtype: bool 441 """ 442 return isinstance(self, VsaConstants)
443
444 - def isOSA(self):
445 """ @return: True, if these are VSA constants. 446 @rtype: bool 447 """ 448 return isinstance(self, OsaConstants)
449 450 #{ Getter methods 451 @staticmethod
452 - def getAll(constantName):
453 """ Returns the values of the given constant in all subclasses. 454 """ 455 return (attrgetter(constantName)(subClass) 456 for subClass in SystemConstants.__subclasses__() 457 if constantName in subClass.__dict__)
458 459 @staticmethod
460 - def getServerName(fsPath):
461 """ 462 The name of the server that hosts the supplied disk path. 463 464 @param fsPath: Path to anywhere within a disk file system. 465 @type fsPath: str 466 @rtype: str 467 468 """ 469 disk = fsPath.lstrip(os.sep).split(os.sep)[0] 470 allServerRaidArrays = sum(map(methodcaller("items"), 471 SystemConstants.getAll("serverRaidArray")), []) 472 473 for server, diskPaths in allServerRaidArrays: 474 disks = (diskPath.lstrip(os.sep).split(os.sep)[0] 475 for diskPath in diskPaths) 476 477 if disk in disks: 478 return server 479 480 # Cover the case of catalogue server share paths 481 return os.path.basename(os.path.abspath(fsPath))
482 483 @staticmethod
484 - def catServerVolumes(server=''):
485 """ 486 List of logical volumes on the given catalogue server. 487 488 @param server: Optionally supply name of the server, otherwise assumes 489 it's a standard ahmose-like configuration. 490 @type server: str 491 @rtype: list(str) 492 493 """ 494 if server == "hatshepsut": 495 volumes = SystemConstants.loadServerVolumes 496 elif server.startswith(SystemConstants.clusterName): 497 clusterId = server.replace(SystemConstants.clusterName, '') 498 if clusterId in ('1', '2', '10', '12', '13'): 499 volumes = SystemConstants.loadServerVolumes 500 elif clusterId == '14': 501 volumes = ['G'] 502 elif clusterId in ('3', '4', '9'): 503 volumes = ['J', 'K', 'L'] 504 elif clusterId in ('5', '6'): 505 volumes = ['M', 'N', 'O'] 506 elif clusterId in ('7', '8'): 507 volumes = ['P', 'Q', 'R'] 508 # elif clusterId == '9': 509 # volumes = ['S'] 510 elif server == "nytro": 511 volumes = ['G'] 512 else: 513 volumes = ['G', 'H', 'I', 'J'] 514 515 return [volume + ':' + SystemConstants.catServerFileSep 516 for volume in volumes]
517 518 @staticmethod
519 - def dbSharePath(fileName='', serverName=os.getenv('HOST')):
520 """ @return: Full path to the given file located on the samba share. 521 @rtype: str 522 """ 523 return os.sep + os.path.join(serverName, 'samba', 'share', 524 fileName).rstrip(os.sep)
525
526 - def dbMntPath(self, loadServer=None):
527 """ @return: Path to the NFS share mounted on the catalogue load server. 528 @rtype: str 529 """ 530 return os.path.join('/mnt', loadServer or self.loadServer)
531
532 - def calibrationSchema(self):
533 """ @return: Name of the calibration tables schema SQL script file. 534 @rtype: str 535 """ 536 return self.loadDatabase + "_CalibSchema.sql"
537
538 - def curationSchema(self):
539 """ @return: Name of the curation tables schema SQL script file. 540 @rtype: str 541 """ 542 return self.loadDatabase + "_CurationLogsSchema.sql"
543
544 - def ingestedLogsPath(self, sharePath=None):
545 """ @return: Path to the directory of ingest log files on the NFS share 546 of the catalogue load server (Windows). 547 @rtype: str 548 """ 549 return os.path.join(sharePath or self.dbSharePath(), "ingestedLogs")
550
551 - def ingestedFilesPath(self, sharePath=None):
552 """ @return: Path to the directory of files to be ingested on the NFS 553 share of the catalogue load server (Windows). 554 @rtype: str 555 """ 556 return os.path.join(sharePath or self.dbSharePath(), "ingestedFiles")
557
558 - def ingestListsPath(self, sharePath=None):
559 """ @return: Path to the directory of ingest file lists on the NFS 560 share of the catalogue load server (Windows). 561 @rtype: str 562 """ 563 return os.path.join(sharePath or self.dbMntPath(), "ingestLists")
564
565 - def metadataSchema(self):
566 """ @return: Name of the metadata tables schema SQL script file. 567 @rtype: str 568 """ 569 return self.loadDatabase + "_MultiframeSchema.sql"
570
571 - def wcsErrorFilesPath(self):
572 """ @return: Path to the directory of files with WCS errors on the NFS 573 share of the catalogue load server (Windows). 574 @rtype: str 575 """ 576 return os.path.join(self.dbSharePath(), "wcsErrorFiles")
577 578 #{ Software directory structure
579 - def calibrationDir(self):
580 """ @return: Directory for calibration log files. 581 @rtype: str 582 """ 583 return os.path.join(self.developDisks[0], 'calibration')
584
585 - def cambInfoPath(self):
586 """ @return: Path to camb info?? 587 @rtype: str 588 """ 589 return os.path.join(self.genOpsDisk, 'cambinfo')
590 591 @staticmethod
592 - def casuCodePerlPath():
593 """ @return: Path to CASU Perl scripts directory. 594 @rtype: str 595 """ 596 return os.path.join(SystemConstants.srcPath, 'CasuSource/PerlScripts')
597
598 - def corruptFilesPath(self):
599 """ @return: Path for corrupted files. 600 @rtype: str 601 """ 602 return os.path.join(self.genOpsDisk, 'ingest/corrupt')
603 604 @staticmethod
605 - def curationCodePath():
606 """ @return: Path to curation source code directory. 607 @rtype: str 608 """ 609 return os.path.join(SystemConstants.srcPath, 'curation')
610 611 @staticmethod
613 """ @return: Full path to the default empty catalogue file. 614 @rtype: str 615 """ 616 return os.path.join(SystemConstants.curationCodePath(), 'wsatools', 617 'FitsIO', SystemConstants.emptyFitsCatalogueFileName)
618
619 - def filterDataDirPath(self):
620 """ @return: Full path to the directory containing filter data 621 @rtype: str 622 """ 623 return os.path.join(self.developDisks[0], "Filters")
624
625 - def filterRefDataDirPath(self):
626 """ @return: Full path to the directory containing filter reference 627 data 628 @rtype: str 629 """ 630 return os.path.join(self.filterDataDirPath(), "Reference")
631 632 @staticmethod
633 - def helpersPath():
634 """ @return: Path to the helpers code directory. 635 @rtype: str 636 """ 637 return os.path.join(SystemConstants.curationCodePath(), "helpers")
638
639 - def illuminationDir(self):
640 """ @return: Directory for illumination table files. 641 @rtype: str 642 """ 643 return os.path.join(self.genOpsDisk, "IlluminationTables")
644
645 - def logFilePath(self, fileName=''):
646 """@return: Path to curation logs. 647 @rtype: str 648 """ 649 return os.path.join(self.genOpsDisk, "logs", fileName)
650
651 - def missingFilesFilePath(self):
652 """ @return: Path to log of missing or broken files. 653 @rtype: str 654 """ 655 return os.path.join(self.logFilePath(), "problems")
656
657 - def monitorPath(self):
658 """ @return: Path to the monitor code directory. 659 @rtype: str 660 """ 661 return os.path.join(self.curationCodePath(), 'invocations', 662 'monitoring')
663
664 - def nonSurveyRegPath(self, fileName=''):
665 """ @return: Path of the non-survey resgistration files 666 @rtype: str 667 """ 668 return os.path.join(self.genOpsDisk, 'nonSurvey', fileName)
669 670 @staticmethod
671 - def sExtractorInputsPath():
672 """ @return: Path to SExtractor inputs directory. 673 @rtype: str 674 """ 675 return os.path.join(SystemConstants.srcPath, 'SExInputs')
676 677 @staticmethod
678 - def sourcePosTablesPath():
679 """ @return: Path where Cu16 works & stores source position tables. 680 @rtype: str 681 """ 682 return SystemConstants.dbSharePath('SourcePosTables')
683 684 @staticmethod
685 - def sqlFragmentPath():
686 """ @return: Path to sql script fragments used by SqlWrappers.py. 687 @rtype: str 688 """ 689 return os.path.join(SystemConstants.sqlScriptPath, "fragments")
690 691 @staticmethod
692 - def sqlNonSurveyPath(fileName=''):
693 """ @return: Path to non-survey sql scripts. 694 @rtype: str 695 """ 696 return os.path.join(SystemConstants.sqlScriptPath, "NonSurvey", 697 fileName)
698 699 @staticmethod
700 - def sqlQualityControlPath(fileName=''):
701 """ @return: Path to quality control sql scripts. 702 @rtype: str 703 """ 704 return os.path.join(SystemConstants.sqlScriptPath, "QualityControl", 705 fileName)
706 707 @staticmethod
708 - def sqlTemplatePath(fileName=''):
709 """ @return: Path to sql script templates. 710 @rtype: str 711 """ 712 return os.path.join(SystemConstants.sqlScriptPath, "Templates", 713 fileName)
714 715 @staticmethod
716 - def sqlMonthlyDetPath(fileName='', full=True):
717 """ @return: Path to monthly detection schemas. 718 @rtype: str 719 """ 720 return (os.path.join( 721 SystemConstants.sqlScriptPath, "MonthlySQL", fileName) 722 if full else os.path.join("MonthlySQL", fileName))
723
724 - def tempWorkPath(self):
725 """ @return: Path to temporary working directory for curation scripts. 726 @rtype: str 727 """ 728 return os.sep + os.path.join(os.getenv('HOST'), "tmp", 729 os.getenv('USER') + '_' + str(os.getpid()))
730 731 @staticmethod
732 - def testOutputPath(fileName=''):
733 """ @return: Path to permanent test directory used in examing test 734 results. 735 @rtype: str 736 """ 737 if os.getenv("USER") == "njc" and os.getenv("HOST") == "djoser": 738 path = \ 739 os.path.join(os.sep, "djoser", "home", "njc", "test", fileName) 740 else: 741 path = \ 742 os.path.join(SystemConstants.developDisks[0], "test", fileName) 743 744 return path.rstrip(os.sep)
745
746 - def xferLogPath(self):
747 """ @return: Path of the transfer log files. 748 @rtype: str 749 """ 750 return os.path.join(self.genOpsDisk, 'ingest/logs')
751 752 @staticmethod
753 - def privNetCmd(server, cmd):
754 """ @return: SSH command to be run on remote WFAU curation server. 755 @rtype: str 756 """ 757 server += ("-i" if server in SystemConstants.infinibandServers and 758 os.getenv("HOST") in SystemConstants.infinibandServers else "-p") 759 760 return "ssh %s -o NumberOfPasswordPrompts=0 '%s'" % (server, cmd)
761 762 #{ Pixel server directory structure
763 - def availableRaidFileSystem(self):
764 """ @return: Disks which are available for storage. 765 @rtype: list(str) 766 """ 767 return sorted(set(self.massStorageRaidFileSystem()) 768 .difference(self.excludedRaidFileSystem()))
769
770 - def excludedRaidFileSystem(self):
771 """ @return: Disks which are excluded from the mass storage raid file 772 system. 773 @rtype: list(str) 774 """ 775 return (self.serverRaidArray['khafre'] 776 + SystemConstants.userInterfaceTempStorage 777 + SystemConstants.deprecatedDataStorage)
778
779 - def leaveSpaceOnMSRFS(self, byPercent=False):
780 """ 781 Space to be left free on the disks in the mass storage raid file 782 system. 783 784 @param byPercent: If True, free disc space is defined by percentage 785 rather than absolute values. 786 @type byPercent: bool 787 788 @return: Free space available. 789 @rtype: int 790 791 """ 792 # Space to leave on non-critical disks in kilobytes 793 nominalSpace = (0.02 if byPercent else 794 50 * self.one_gigabyte / self.one_kilobyte) 795 796 freeSpace = dict.fromkeys( 797 [disk for disk in self.availableRaidFileSystem() 798 if disk not in self.criticalDisks], nominalSpace) 799 800 # Space to leave on critical disks in kilobytes 801 criticalSpace = (0.1 if byPercent else 802 200 * self.one_gigabyte / self.one_kilobyte) 803 804 freeSpace.update(dict.fromkeys(self.criticalDisks, criticalSpace)) 805 freeSpace.update(dict.fromkeys(self.deprecatedDataStorage, 806 criticalSpace)) 807 808 # Space to leave on development disk in kilobytes 809 developSpace = (0.5 if byPercent else 810 750 * self.one_gigabyte / self.one_kilobyte) 811 812 freeSpace.update(dict.fromkeys(self.developDisks, developSpace)) 813 814 return freeSpace
815
816 - def massStorageRaidFileSystem(self):
817 """ @return: A list of paths to every mass storage raid disk. 818 @rtype: list(str) 819 """ 820 return sorted(set(element for subList in self.serverRaidArray.values() 821 for element in subList))
822 823 #{ Pixel server common directories
824 - def downloadPathList(self):
825 """ @return: Complete list of paths on all RAID discs where the data 826 will be downloaded to. 827 @rtype: list(str) 828 """ 829 return [os.path.join(raidDisk, 'ingest', 'dload') 830 for raidDisk in self.availableRaidFileSystem() 831 if raidDisk not in self.developDisks]
832 833 #{ Permanent curation log file paths
834 - def filesBrokenPath(self):
835 """ @return: Path to the log listing files that are broken in some way, 836 e.g. fail FITS verification checks or have missing header 837 keywords such as the PROV keyword. 838 @rtype: str 839 """ 840 return os.path.join(self.missingFilesFilePath(), "files_broken.log")
841
842 - def filesDuplicatedPath(self):
843 """ @return: Path to the log listing files that have duplicate 844 non-deprecated entries in the database. 845 @rtype: str 846 """ 847 return os.path.join(self.missingFilesFilePath(), 848 "files_duplicated.log")
849
850 - def filesMissingPath(self):
851 """ @return: Path to the log listing files with database entries, but 852 not existing at the location provided by the database 853 entry. 854 @rtype: str 855 """ 856 return os.path.join(self.missingFilesFilePath(), "files_missing.log")
857
858 - def filesNotIngestedPath(self):
859 """ @return: Path to the log listing files not ingested (i.e. not in 860 database, and possibly not on the pixel servers' raid 861 array). 862 @rtype: str 863 """ 864 return os.path.join(self.missingFilesFilePath(), 865 "files_not_ingested.log")
866 867 #{ FITS file naming conventions 868 @staticmethod
869 - def fileNameSuffices():
870 """ @return: A list of file name suffices ordered by precedence. 871 @rtype: list(str) 872 """ 873 return [SystemConstants.deepSuffix, SystemConstants.leavSuffix, 874 SystemConstants.diffSuffix, SystemConstants.stackSuffix, 875 SystemConstants.tileSuffix, SystemConstants.mosaicSuffix, 876 SystemConstants.filtSuffix, SystemConstants.confSuffix, '']
877 878 @staticmethod
879 - def frameTypeToSuffix():
880 """ @return: A dictionary of file name suffices referenced by frame 881 type. 882 @rtype: dict(str:str) 883 """ 884 return dict(normal='', mosaic=SystemConstants.mosaicSuffix, 885 tile=SystemConstants.tileSuffix, 886 deep=SystemConstants.deepSuffix, 887 filt=SystemConstants.filtSuffix, 888 leav=SystemConstants.leavSuffix, 889 diff=SystemConstants.diffSuffix, 890 stack=SystemConstants.stackSuffix, 891 conf=SystemConstants.confSuffix)
892
893 - def extListOfType(self, productType='stack', frameType=None):
894 """ @return: List of expected extensions in given product or database 895 frame type. 896 @rtype: list(int) 897 """ 898 if frameType: # Convert to productType 899 productType = self.productTypeOfFrame(frameType) 900 901 if productType == 'stack': 902 return range(2, self.maxHDUs + 1) 903 904 # Tile or mosaic 905 return range(2, 3)
906
907 - def productTypeOfFrame(self, frameType):
908 """ @return: Product type of given database frame type. 909 @rtype: str 910 """ 911 if not frameType.endswith("stack"): 912 raise ValueError("This is not a product: " + frameType) 913 914 # Work from highest order product down 915 for pType in reversed(self.productTypes): 916 if pType in frameType: 917 return pType
918
919 - def wfauFitsKeys(self):
920 """ @return: List of FITS header keys inserted into files by WFAU. 921 @rtype: list(str) 922 """ 923 return [self.loadDatabase + "_TIME", self.loadDatabase + "_MFID"]
924 925 #{ CASU data transfer constants
926 - def scpMethod(self, light=True):
927 """ @return: Command to run scp. 928 @rtype: str 929 """ 930 return ("hpn-scp -P %s -o TCPRcvBufPoll=yes -pq" % self.scpPort 931 if light else "scp -q")
932
933 - def sshMethod(self, light=True):
934 """ @return: Command to run ssh. 935 @rtype: str 936 """ 937 return "hpn-ssh -p %s" % self.scpPort if light else "ssh"
938
939 - def scpMainDir(self):
940 """ @return: Top directory of the data tree on the scp server. 941 @rtype: str 942 """ 943 return self.casuFileSystem[-1]
944 945 #{ Web site constants
946 - def surveyBaseUrl(self):
947 """ @return: URL to the science archive web interface. 948 @rtype: str 949 """ 950 return 'http://surveys.roe.ac.uk/%s/' % self.loadDatabase.lower()
951
952 - def surveyJspBaseUrl(self):
953 """ @return: URL to the science archive web interface including port. 954 @rtype: str 955 """ 956 return 'http://surveys.roe.ac.uk:8080/%s/' % self.loadDatabase.lower()
957
958 #------------------------------------------------------------------------------ 959 960 -class WsaConstants(SystemConstants):
961 """ Constants with specific values for the WSA. 962 """ 963 #{ Catalogue server constants 964 #: Name of load database. 965 loadDatabase = "WSA" 966 967 #: Name of load server. 968 loadServer = SystemConstants.clusterName + '1' 969 970 #: Dictionary of ingest database names on each server for split ingest dbs. 971 ingestDbForServer = {loadServer: loadDatabase} 972 973 # @@BUG: Curation isn't metadata (it's neighbour table data), but has to 974 # stay for backwards compatibility. 975 #: Additional database file groups beyond primary and logs for metadata. 976 metadataFileGroups = ["Curation", "Indices"] 977 978 #: Database tables containing metadata to be populated by CU3. 979 metadataTables = ["Multiframe", "MultiframeDetector", "CurrentAstrometry", 980 "ProgrammeFrame"] 981 982 #: Default directory for non-survey db. 983 nonSurveyDir = "WSA-NonSurvey" 984 985 #: (Deprecated) This is appended to fileNames in the database. 986 pixelServerHostName = 'djoser:' 987 988 #: List of public catalogue servers. 989 publicServers = [SystemConstants.clusterName + '5', 990 SystemConstants.clusterName + '6'] 991 992 #{ Pixel server file system 993 #: Prefix for CASU produced FITS files. 994 casuPrefix = 'w' 995 996 #: Disk for general operations output (logs, info, ...). 997 genOpsDisk = "/disk01/wsa" 998 999 #: A dictionary giving the file systems of each server. 1000 serverRaidArray = dict( 1001 djoser=[], 1002 sneferu=[], 1003 khufu=['/disk06/wsa', '/disk07/wsa', '/disk08/wsa', 1004 '/disk09/wsa'], 1005 djedefre=['/disk10/wsa', '/disk11/wsa', '/disk12/wsa', 1006 '/disk13/wsa'], 1007 khafre=['/disk14/wsa', '/disk15/wsa'], 1008 sheshonq=['/disk01/wsa', '/disk16/wsa', '/disk17/wsa', 1009 '/disk18/wsa'], 1010 osorkon=['/disk02/wsa', '/disk03/wsa', '/disk04/wsa', 1011 '/disk05/wsa'], 1012 takelot=['/disk19/wsa', '/disk23/wsa', '/disk24/wsa'], 1013 takelot2=['/disk20/wsa', '/disk21/wsa', '/disk22/wsa'], 1014 pami=['/disk29/wsa', '/disk30/wsa', '/disk31/wsa', '/disk32/wsa'], 1015 pami2=['/disk27/wsa', '/disk28/wsa', '/disk34/wsa'], 1016 piye1=[], 1017 piye2=['/disk36/wsa'], 1018 pedubaste1=['/disk37/wsa'], 1019 pedubaste2=['/disk38/wsa'], 1020 shepsesre=['/disk39/wsa', '/disk40/wsa'], 1021 wahkare=['/disk41/wsa'], 1022 shebaka=['/disk44/wsa']) 1023 1024 #: Format of versioning numbers. 1025 versFormat = "%d" 1026 1027 #: Increments of versioning numbers. 1028 versStep = 1 1029 1030 #{ Software file system 1031 #: Template schema for automated index scripts. 1032 autoTemplateIndexSchema = \ 1033 SystemConstants.sqlTemplatePath("WSANS_templateIndices.sql") 1034 1035 #: Template schema for automated neighbour scripts. 1036 autoTemplateNeighboursSchema = \ 1037 SystemConstants.sqlTemplatePath("WSA_autoTemplateNeighboursSchema.sql") 1038 1039 #: Template schema for automated scripts. 1040 autoTemplateSchema = \ 1041 SystemConstants.sqlTemplatePath("WSA_autoTemplateSchema.sql") 1042 1043 #: File name for the SQL script defining WSA table indices. 1044 indexScript = "WSA_Indices.sql" 1045 1046 #: File name for the SQL script defining non-survey specific table indices. 1047 nsIndexScript = "NonSurvey/WSA_NonSurveyIndices.sql" 1048 1049 #: Location to the script of quality control SQL statements for parsing. 1050 nsQcScript = \ 1051 SystemConstants.sqlQualityControlPath("WSA_NonSurveyQCstatements.sql") 1052 1053 #: Script for survey specific procedures and functions. 1054 surveyProcScript = "WSA_UKIDSS_StoredProcsAndFuncs.sql" 1055 1056 #: File name for the SQL script defining survey views. 1057 surveyViewScript = "WSA_UkidssViews.sql" 1058 1059 #{ Survey data constants 1060 #: Airmass start value keyword in FITS HDU. 1061 amStart = "AMSTART" 1062 1063 #: Airmass end value keyword in FITS HDU. 1064 amEnd = "AMEND" 1065 1066 #: CASU catalogue type. 1067 casuCatType = 2 1068 1069 #: CASU detection threshold dict(nustep,sigma) 1070 casuDetThresh = {1:1.5, 4:1.25, 9:1.25} 1071 1072 #: EXPTIME header keyword 1073 expTimeKeyWord = 'EXP_TIME' 1074 1075 #: Size of the field of view x and y (tuple) 1076 fieldOfView = (0.6713, 0.6713) 1077 1078 #: Name of HDU number fits key 1079 hduNumKey = "CAMNUM" 1080 1081 #: Name of the instrument 1082 instrumentName = "WFCAM" 1083 1084 #: Standard pixel scale of instrument 1085 instrumentPixScale = 0.4 1086 1087 #: List of programmeIDs of programmes which are manually set up. 1088 manualProgs = [1, 101, 102, 103] 1089 1090 #: List of programmes that need monthly detection tables 1091 monthlyDetSurveys = [] 1092 1093 #: Maximum number of extensions 1094 maxHDUs = 5 1095 1096 #: Maximum ra or dec separation for stacks in a group for deep stacking 1097 maxRaDecExtentStackDeg = 0.068 1098 1099 #: Minimum ra or dec separation for stacks in a group for deep stacking 1100 minRaDecExtentStackDeg = 0.017 1101 1102 #: Observation dates for the survey. 1103 obsCal = WfcamCal() 1104 1105 #: Pawprint area in degrees 1106 pawPrintArea = 0.207 1107 1108 #: Photometric zero-point error limit that necessitates recalibration. 1109 phtErrLimit = 0.02 1110 1111 #: Pointing keys to check 1112 pointingKeys = ["RABASE", "DECBASE", "FILTER"] 1113 1114 #: Product types 1115 productTypes = ["stack", "mosaic"] 1116 1117 #: FITS header keys that identify the project. 1118 projectKeys = ["PROJECT"] 1119 1120 #: Core radius ~ 1/instrument pixScale 1121 rCoreBasic = 2.5 1122 1123 #: Programmes 1124 scienceProgs = {"LAS": 101, "GPS": 102, "GCS": 103, "DXS": 104, "UDS": 105} 1125 1126 #: Pixel super bin size in JPEG rebinning process. 1127 scrunch = 2 1128 1129 #: Known survey name translations of WFCAM frames. 1130 surveyTranslations = {} 1131 1132 #: Name of the telescope 1133 telescopeName = "UKIRT" 1134 1135 #{ CASU data transfer constants 1136 #: List of disks where data is stored at CASU. Sorted in order of their 1137 # appearance at CASU. 1138 casuFileSystem = ['/data/apm22_a/wfcam/processed', 1139 '/data/apm22_b/wfcam/processed', 1140 '/data/apm23_d/wfcam/processed', 1141 '/data/apm24_b/wfcam/processed', 1142 '/data/apm24_c/wfcam/processed', 1143 '/data/apm22_c/wfcam/processed', 1144 '/data/apm22_d/wfcam/processed', 1145 '/data/apm23_e/wfcam/processed', 1146 '/data/apm22_e/wfcam/processed', 1147 '/data/apm23_b/wfcam/processed'] 1148 1149 1150 #: Letters and numbers that can appear in the CASU file system. 1151 casuFileSystemVariables = AlphaNum("wfcam", "abcdef", 1152 range(22, 25) + [44, 47]) 1153 1154 #: The directory where illumination tables are stored at CASU. 1155 casuIllumDir = "/data/apm7_a/mike/photomtests" 1156 1157 #: The CASU scp server port number for processed data. 1158 scpPort = 22 1159 1160 #: has microstepping? 1161 hasMicroStep = True 1162 1163 @staticmethod
1164 - def scpServer(light=True):
1165 """ @return: The CASU scp server for processed data. 1166 @rtype: str 1167 """ 1168 return '10.0.1.7' if light else 'apm7.ast.cam.ac.uk'
1169 1170 @staticmethod
1171 - def surveyFileGroups(_surveys=''):
1172 """ Returns list of file groups for a database of the given surveys. 1173 1174 @param _surveys: Survey acronyms as a space-separated string. 1175 @type _surveys: str 1176 @rtype: list(str) 1177 """ 1178 return ["Detection", "Source"]
1179
1180 #------------------------------------------------------------------------------ 1181 1182 -class GesConstants(SystemConstants):
1183 """ Constants with specific values for the GES. 1184 """ 1185 #: Name of load database. 1186 loadDatabase = "GES" 1187 1188 #: Name of load server. 1189 loadServer = SystemConstants.clusterName + '1' 1190 1191 #: Dictionary of ingest database names on each server for split ingest dbs. 1192 ingestDbForServer = {loadServer: loadDatabase} 1193 1194 #: Additional database file groups beyond primary and logs for metadata. 1195 metadataFileGroups = ["Spectrum", "Curation", "Indices"] 1196 1197 #: List of public catalogue servers. 1198 publicServers = [SystemConstants.clusterName + '5', 1199 SystemConstants.clusterName + '6'] 1200 1201 #: Programmes 1202 scienceProgs = {} 1203 1204 #: Disk for general operations output (logs, info, ...). 1205 genOpsDisk = "/disk40/ges" 1206 1207 #: A dictionary giving the file systems of each server. 1208 serverRaidArray = dict( 1209 djoser=[], 1210 sneferu=[], 1211 khufu=[], 1212 djedefre=[], 1213 khafre=[], 1214 sheshonq=[], 1215 osorkon=[], 1216 takelot=[], 1217 takelot2=[], 1218 pami=[], 1219 pami2=[], 1220 piye1=[], 1221 piye2=[], 1222 pedubaste1=[], 1223 pedubaste2=[], 1224 shepsesre=['/disk40/ges'], 1225 wahkare=[], 1226 shebaka=[]) 1227 1228 #: File name for the SQL script defining table indices. 1229 indexScript = "GES_Indices.sql" 1230 1231 #: Full path to WG analysis files for ingest. 1232 ingestDataPath = os.path.join(genOpsDisk, "ingest", "data") 1233 1234 #: Path to sql scripts. 1235 sqlScriptPath = os.path.join(os.getenv("GES_DIR") or '', "sql") 1236 1237 #: Script for survey procedures and functions for all databases. 1238 surveyProcScript = "GES_StoredProcsAndFuncs.sql" 1239 1240 #: File name for the SQL script defining survey views. 1241 surveyViewScript = "GES_Views.sql" 1242 1243 #: Observation dates for the survey. 1244 obsCal = GesCal() 1245 1246 #{ CASU data transfer constants 1247 #: List of disks where data is stored at CASU. Sorted in order of their 1248 # appearance at CASU. 1249 casuFileSystem = [''] 1250 1251 #: Letters and numbers that can appear in the CASU file system. 1252 casuFileSystemVariables = AlphaNum("ges", "a", [47]) 1253 1254 #: The CASU scp server port number for processed data. 1255 scpPort = 22 1256
1257 - def metadataSchema(self):
1258 """ @return: Name of the metadata tables schema SQL script file. 1259 @rtype: str 1260 """ 1261 return self.loadDatabase + "_SpectrumSchema.sql"
1262 1263 @staticmethod
1264 - def scpServer(light=True):
1265 """ @return: The CASU scp server for processed data. 1266 @rtype: str 1267 """ 1268 return '10.0.1.29' if light else 'apm29.ast.cam.ac.uk'
1269 1270 @staticmethod
1271 - def surveyFileGroups(_surveys=''):
1272 """ Returns list of file groups for a database of the given surveys. 1273 1274 @param _surveys: Survey acronyms as a space-separated string. 1275 @type _surveys: str 1276 @rtype: list(str) 1277 """ 1278 return []
1279
1280 #------------------------------------------------------------------------------ 1281 1282 -class SsaConstants(SystemConstants):
1283 """ Constants with specific values for the SSA. 1284 """ 1285 #{ Catalogue server constants 1286 #: Name of load database. 1287 loadDatabase = "SSA" 1288 1289 #: Name of load server. 1290 loadServer = SystemConstants.clusterName + '7' 1291 1292 #: List of public catalogue servers. 1293 publicServers = [SystemConstants.clusterName + '8', 1294 SystemConstants.clusterName + '3'] 1295 1296 #: Programmes 1297 scienceProgs = {} 1298 1299 #{ CASU data transfer constants 1300 #: List of disks where data is stored at CASU. Sorted in order of their 1301 # appearance at CASU. 1302 casuFileSystem = ['']
1303
1304 1305 #------------------------------------------------------------------------------ 1306 1307 -class VsaConstants(SystemConstants):
1308 """ Constants with specific values for the VSA. 1309 """ 1310 #{ Catalogue server constants 1311 #: Name of load database. 1312 loadDatabase = "VSA" 1313 1314 #: Name of load server. 1315 loadServer = SystemConstants.clusterName + '9' 1316 1317 #: Dictionary of ingest database names on each server for split ingest dbs. 1318 ingestDbForServer = {loadServer: loadDatabase, 1319 SystemConstants.clusterName + '14': loadDatabase + 'VVV'} 1320 1321 #: Additional database file groups beyond primary and logs for metadata. 1322 metadataFileGroups = [] 1323 1324 #: Database tables containing metadata to be populated by CU3. 1325 metadataTables = ["Multiframe", "MultiframeDetector", "CurrentAstrometry", 1326 "ProgrammeFrame", "MultiframeEsoKeys", 1327 "MultiframeDetectorEsoKeys"] 1328 1329 #: Default directory for non-survey db. 1330 nonSurveyDir = "VSA-NonSurvey" 1331 1332 #: Dictionary of OB filter setup 1333 obFiltDict = { 1334 'VVV':[('Z', 'Y'), ('J', 'H', 'Ks')], 1335 } 1336 1337 #: (Deprecated) This is appended to fileNames in the database. 1338 pixelServerHostName = '' 1339 1340 #: List of public catalogue servers. 1341 publicServers = [SystemConstants.clusterName + '3', 1342 SystemConstants.clusterName + '4'] 1343 1344 #{ Pixel server file system 1345 #: Prefix for CASU produced FITS files. 1346 casuPrefix = 'v' 1347 1348 #: Disk for general operations output (logs, info, ...). 1349 genOpsDisk = "/disk47/vsa" 1350 1351 #: A dictionary giving the file systems of each server. 1352 serverRaidArray = dict( 1353 djoser=[], 1354 sneferu=[], 1355 khufu=[], 1356 djedefre=[], 1357 khafre=[], 1358 sheshonq=[], 1359 osorkon=[], 1360 takelot=[], 1361 takelot2=[], 1362 pami=['/disk32/vsa'], 1363 pami2=['/disk33/vsa'], 1364 piye1=['/disk35/vsa'], 1365 piye2=['/disk36/vsa'], 1366 pedubaste1=['/disk37/vsa'], 1367 pedubaste2=['/disk38/vsa', '/disk43/vsa'], 1368 shepsesre=['/disk39/vsa', '/disk40/vsa'], 1369 wahkare=['/disk42/vsa'], 1370 shebaka=['/disk45/vsa', '/disk46/vsa']) 1371 1372 #: Format of versioning numbers. 1373 versFormat = "%.1f" 1374 1375 #: Increments of versioning numbers. 1376 versStep = 0.1 1377 1378 #{ Software file system 1379 #: Template schema for automated index scripts. 1380 autoTemplateIndexSchema = \ 1381 SystemConstants.sqlTemplatePath("VSA_autoTemplateIndices.sql") 1382 1383 #: Template schema for automated neighbour scripts. 1384 autoTemplateNeighboursSchema = \ 1385 SystemConstants.sqlTemplatePath("VSA_autoTemplateNeighboursSchema.sql") 1386 1387 #: Template schema for automated scripts. 1388 autoTemplateSchema = \ 1389 SystemConstants.sqlTemplatePath("VSA_autoTemplateSchema.sql") 1390 1391 #: Template schema for automated view scripts. 1392 autoTemplateViewSchema = \ 1393 SystemConstants.sqlTemplatePath("VSA_autoTemplateViews.sql") 1394 1395 #: File name for the SQL script defining table indices. 1396 indexScript = "VSA_Indices.sql" 1397 1398 #: File name for the SQL script defining non-survey table indices. 1399 nsIndexScript = "NonSurvey/VSANS_Indices.sql" 1400 1401 #: Location to the script of quality control SQL statements for parsing. 1402 nsQcScript = \ 1403 SystemConstants.sqlQualityControlPath("VSA_NonSurveyQCstatements.sql") 1404 1405 #: Script for survey procedures and functions for all databases. 1406 surveyProcScript = "VSA_PS_StoredProcsAndFuncs.sql" 1407 1408 #: File name for the SQL script defining survey views. 1409 surveyViewScript = "VSA_PS_Views.sql" 1410 1411 #{ Survey data constants 1412 #: Airmass start value keyword in FITS HDU. 1413 amStart = 'HIERARCH ESO TEL AIRM START' 1414 1415 #: Airmass end value keyword in FITS HDU. 1416 amEnd = 'HIERARCH ESO TEL AIRM END' 1417 1418 #: CASU catalogue type. 1419 casuCatType = 6 1420 1421 #: CASU detection threshold dict(nustep,sigma) 1422 casuDetThresh = {1:1.25} 1423 1424 #: EXPTIME header keyword 1425 expTimeKeyWord = 'EXPTIME' 1426 1427 #: Size of the field of view x and y (tuple) 1428 fieldOfView = (1.2921, 1.0173) 1429 1430 #: Size of the field of view x and y (tuple) TILE >= 2x exposure 1431 fieldOfViewTile = (1.4753, 1.0173) 1432 1433 #: Name of HDU number fits key 1434 hduNumKey = "HIERARCH ESO DET CHIP NO" 1435 1436 #: Name of the instrument 1437 instrumentName = "VIRCAM" 1438 1439 #: Standard pixel scale of instrument 1440 instrumentPixScale = 0.339 1441 1442 #: List of programmeIDs of programmes which are manually set up. 1443 manualProgs = [] 1444 1445 #: List of programmes that need monthly detection tables 1446 monthlyDetSurveys = [120] 1447 1448 #: Maximum number of extensions 1449 maxHDUs = 17 1450 1451 #: Maximum ra or dec separation for stacks in a group for deep stacking 1452 maxRaDecExtentStackDeg = 0.068 1453 1454 #: Minimum ra or dec separation for stacks in a group for deep stacking 1455 minRaDecExtentStackDeg = 0.017 1456 1457 #: Observation dates for the survey. 1458 obsCal = VistaCal() 1459 1460 #: Pawprint area in degrees 1461 pawPrintArea = 0.598 1462 1463 #: Photometric zero-point error limit that necessitates recalibration. 1464 phtErrLimit = 0.02 1465 1466 #: Pointing keys to check 1467 pointingKeys = ["RA", "DEC", "HIERARCH ESO INS FILT1 NAME"] 1468 1469 #: Product types 1470 productTypes = ["stack", "tile", "mosaic"] 1471 1472 #: FITS header keys that identify the project. 1473 projectKeys = ["HIERARCH ESO OBS PROG ID", "HIERARCH ESO OBS NAME", 1474 "HIERARCH ESO DPR CATG"] 1475 1476 #: Core radius ~ 1/instrument pixScale 1477 rCoreBasic = 3.0 1478 1479 #: Programmes 1480 scienceProgs = {"VHS": 110, "VVV": 120, "VMC": 130, "VIKING": 140, 1481 "VIDEO": 150, "ULTRAVISTA": 160} 1482 1483 #: Pixel super bin size in JPEG rebinning process. 1484 scrunch = 4 1485 1486 #: Known survey name translations of VISTA frames. 1487 surveyTranslations = {"MAINTENANCE": "MAINTENANCE", 1488 "60.A-9285(A)": "SV-NGC253", 1489 "60.A-9285(B)": "SV-ORION", 1490 "60.A-9292(A)": "CAL", 1491 "60.A-9293(A)": "CAL", 1492 "60.A-9253(N)": "COMM", 1493 "179.B-2002": "VVV", 1494 "179.B-2003": "VMC", 1495 "179.A-2004": "VIKING", 1496 "179.A-2005": "UltraVISTA", 1497 "179.A-2006": "VIDEO", 1498 "179.A-2010": "VHS" 1499 } 1500 1501 #: Name of the telescope 1502 telescopeName = "VISTA" 1503 1504 #: Tile area / square degrees 1505 tileArea = 1.5008 1506 1507 #: Default position angle tolerance / degrees 1508 defaultPosAngleTolerance = 15. 1509 1510 #: Width of undithered tile in x-dir: smallest nJitter=1 tile 1511 xWidthTileDith = 12707 1512 1513 #: Width of undithered tile in y-dir: could go smaller 1514 yWidthTileDith = 15593 1515 1516 #: Width of uneared tile in x-dir: detRows * (5.275 + 0.475) 1517 xWidthTileEar = 11776 1518 1519 #{ SExtractor 1520 defaultSexInparFile = "defaultDetCatVsa.inpar" 1521 """ Default sex inpar file. Modelled on Victoria's VIDEO inpar file. 1522 """ 1523 #{ CASU data transfer constants 1524 #: List of disks where data is stored at CASU. Sorted in order of their 1525 # appearance at CASU. 1526 casuFileSystem = ['/data/apm29_b/vista/processed', 1527 '/data/apm29_a/vista/processed', 1528 '/data/apm44_b/vista/processed'] 1529 1530 #: Letters and numbers that can appear in the CASU file system. 1531 casuFileSystemVariables = AlphaNum("vista", "abe", [29, 44, 49, 50]) 1532 1533 #: The directory where illumination tables are stored at CASU. 1534 casuIllumDir = "/data/apm35_a/vista/mike/photomtests/" 1535 1536 #: The CASU scp server port number for processed data. 1537 scpPort = 22 1538 1539 #: has microstepping? 1540 hasMicroStep = True 1541 1542 #: has variable orientation 1543 hasVarOrient = True 1544 1545 #: has offset positions 1546 hasOffsetPos = True 1547 1548 #: Has an ESO release? 1549 hasEsoRelease = True 1550 1551 @staticmethod
1552 - def scpServer(light=True):
1553 """ @return: The CASU scp server for processed data. 1554 @rtype: str 1555 """ 1556 return '10.0.1.29' if light else 'apm29.ast.cam.ac.uk'
1557 1558 @staticmethod
1559 - def surveyFileGroups(_surveys=''):
1560 """ Returns list of file groups for a database of the given surveys. 1561 1562 @param _surveys: Survey acronyms as a space-separated string. 1563 @type _surveys: str 1564 @rtype: list(str) 1565 """ 1566 allSurveys = "cal vhs vvv vmc viking video ultravista".split() 1567 surveys = (allSurveys if not _surveys else 1568 set(_surveys.split()).intersection(allSurveys)) 1569 1570 return [prog + filegroup 1571 for filegroup in ("Detection", "Source", "Curation", "Indices") 1572 for prog in surveys]
1573
1574 #------------------------------------------------------------------------------ 1575 1576 -class OsaConstants(SystemConstants):
1577 """ Constants with specific values for the OSA. 1578 """ 1579 #{ Catalogue server constants 1580 #: Name of load database. 1581 loadDatabase = "OSA" 1582 1583 #: Name of load server. 1584 loadServer = SystemConstants.clusterName + '1' 1585 1586 #: Dictionary of ingest database names on each server for split ingest dbs. 1587 ingestDbForServer = {loadServer: loadDatabase} 1588 1589 #: Additional database file groups beyond primary and logs for metadata. 1590 metadataFileGroups = ["Curation", "Indices"] 1591 1592 #: Database tables containing metadata to be populated by CU3. 1593 metadataTables = ["Multiframe", "MultiframeDetector", "CurrentAstrometry", 1594 "ProgrammeFrame", "MultiframeEsoKeys", 1595 "MultiframeDetectorEsoKeys"] 1596 1597 #: Default directory for non-survey db. 1598 nonSurveyDir = "OSA-NonSurvey" 1599 1600 #: (Deprecated) This is appended to fileNames in the database. 1601 pixelServerHostName = '' 1602 1603 #: List of public catalogue servers. 1604 publicServers = [SystemConstants.clusterName + '5', 1605 SystemConstants.clusterName + '6'] 1606 1607 #{ Pixel server file system 1608 #: Prefix for CASU produced FITS files. 1609 casuPrefix = 'o' 1610 1611 #: Disk for general operations output (logs, info, ...). 1612 genOpsDisk = "/disk40/osa" 1613 1614 #: A dictionary giving the file systems of each server. 1615 serverRaidArray = dict( 1616 djoser=[], 1617 sneferu=[], 1618 khufu=[], 1619 djedefre=[], 1620 khafre=[], 1621 sheshonq=[], 1622 osorkon=[], 1623 takelot=[], 1624 takelot2=[], 1625 pami=[], 1626 pami2=[], 1627 piye1=[], 1628 piye2=[], 1629 pedubaste1=[], 1630 pedubaste2=[], 1631 shepsesre=['/disk39/osa', '/disk40/osa'], 1632 wahkare=['/disk41/osa'], 1633 shebaka=['/disk47/osa']) 1634 1635 #: Format of versioning numbers. 1636 versFormat = "%.1f" 1637 1638 #: Increments of versioning numbers. 1639 versStep = 0.1 1640 1641 #{ Software file system 1642 #: Template schema for automated index scripts. 1643 autoTemplateIndexSchema = \ 1644 SystemConstants.sqlTemplatePath("OSA_autoTemplateIndices.sql") 1645 1646 #: Template schema for automated neighbour scripts. 1647 autoTemplateNeighboursSchema = \ 1648 SystemConstants.sqlTemplatePath("OSA_autoTemplateNeighboursSchema.sql") 1649 1650 #: Template schema for automated scripts. 1651 autoTemplateSchema = \ 1652 SystemConstants.sqlTemplatePath("OSA_autoTemplateSchema.sql") 1653 1654 #: Template schema for automated view scripts. 1655 autoTemplateViewSchema = \ 1656 SystemConstants.sqlTemplatePath("OSA_autoTemplateViews.sql") 1657 1658 #: File name for the SQL script defining table indices. 1659 indexScript = "OSA_Indices.sql" 1660 1661 #: File name for the SQL script defining non-survey table indices. 1662 nsIndexScript = "NonSurvey/OSANS_Indices.sql" 1663 1664 #: Script for survey procedures and functions for all databases. 1665 surveyProcScript = "OSA_PS_StoredProcsAndFuncs.sql" 1666 1667 #: File name for the SQL script defining survey views. 1668 surveyViewScript = "OSA_PS_Views.sql" 1669 1670 #{ Survey data constants 1671 #: Airmass start value keyword in FITS HDU. 1672 amStart = 'HIERARCH ESO TEL AIRM START' 1673 1674 #: Airmass end value keyword in FITS HDU. 1675 amEnd = 'HIERARCH ESO TEL AIRM END' 1676 1677 #: CASU catalogue type. 1678 # @FIXME: This needs to be changed 1679 casuCatType = 6 1680 1681 #: CASU detection threshold dict(nustep,sigma) 1682 # @TODO: threshold is 1.25, but no micro-stepping values. 1683 casuDetThresh = {1:1.25} 1684 1685 #: EXPTIME header keyword 1686 expTimeKeyWord = 'EXPTIME' 1687 1688 #: Size of the field of view x and y (tuple) 1689 fieldOfView = (1.0, 1.0) 1690 1691 #: Name of HDU number fits key 1692 hduNumKey = "CAMNUM" 1693 1694 hduNumDict = {65: 1, 66: 2, 67: 3, 68: 4, 73: 5, 74: 6, 75: 7, 76: 8, 1695 81: 9, 82:10, 83:11, 84:12, 89:13, 90:14, 91:15, 92:16, 1696 69:17, 70:18, 71:19, 72:20, 77:21, 78:22, 79:23, 80:24, 1697 85:25, 86:26, 87:27, 88:28, 93:29, 94:30, 95:31, 96:32} 1698 1699 #: Name of the instrument 1700 instrumentName = "OMEGACAM" 1701 1702 #: Standard pixel scale of instrument 1703 instrumentPixScale = 0.214 1704 1705 #: List of programmeIDs of programmes which are manually set up. 1706 manualProgs = [] 1707 1708 #: List of programmes that need monthly detection tables 1709 monthlyDetSurveys = [] 1710 1711 #: Maximum number of extensions 1712 maxHDUs = 33 1713 1714 #: Maximum ra or dec separation for stacks in a group for deep stacking 1715 maxRaDecExtentStackDeg = 0.05 1716 1717 #: Minimum ra or dec separation for stacks in a group for deep stacking 1718 minRaDecExtentStackDeg = 0.017 1719 1720 #: Observation dates for the survey. 1721 obsCal = OmegaCal() 1722 1723 #: Pawprint area in degrees 1724 pawPrintArea = 0.949 1725 1726 #: Photometric zero-point error limit that necessitates recalibration. 1727 phtErrLimit = 0.02 1728 1729 #: Pointing keys to check 1730 pointingKeys = ["RA", "DEC", "HIERARCH ESO INS FILT1 NAME"] 1731 1732 #: Product types 1733 # @TODO: May need mosaic at some point? 1734 productTypes = ["stack"] 1735 1736 #: FITS header keys that identify the project. 1737 projectKeys = ["HIERARCH ESO OBS PROG ID", "HIERARCH ESO OBS NAME", 1738 "HIERARCH ESO DPR CATG"] 1739 1740 #: Core radius ~ 1/instrument pixScale 1741 rCoreBasic = 4.65 1742 1743 #: Programmes 1744 scienceProgs = {"ATLAS": 110} 1745 1746 #: Pixel super bin size in JPEG rebinning process. 1747 scrunch = 4 1748 1749 #: Known survey name translations of VST frames. 1750 surveyTranslations = {"MAINTENANCE": "MAINTENANCE", 1751 "60.A-9038(A)": "TECHNICAL", 1752 "60.A-9800(Q)": "CAL", 1753 "177.A-3011": "ATLAS", 1754 "177.A-3016": "KIDS", 1755 "177.A-3017": "KIDS", 1756 "177.A-3018": "KIDS", 1757 "177.D-3023": "VPHAS", 1758 } 1759 1760 #: Name of the telescope 1761 telescopeName = "VST" 1762 1763 #{ CASU data transfer constants 1764 #: List of disks where data is stored at CASU. Sorted in order of their 1765 # appearance at CASU. 1766 casuFileSystem = ['/data/apm45_a/vst/processed/'] 1767 1768 #: Letters and numbers that can appear in the CASU file system. 1769 casuFileSystemVariables = AlphaNum("vst", "ab", [45, 46, 48]) 1770 1771 #: The directory where illumination tables are stored at CASU. 1772 casuIllumDir = "" 1773 1774 #: The CASU scp server port number for processed data. 1775 scpPort = 22 1776 1777 #: has variable orientation 1778 hasVarOrient = True 1779 1780 #: Has an ESO release? 1781 hasEsoRelease = True 1782 1783 @staticmethod
1784 - def scpServer(light=True):
1785 """ @return: The CASU scp server for processed data. 1786 @rtype: str 1787 """ 1788 return '10.0.1.29' if light else 'apm29.ast.cam.ac.uk'
1789 1790 @staticmethod
1791 - def surveyFileGroups(_surveys=''):
1792 """ Returns list of file groups for a database of the given surveys. 1793 1794 @param _surveys: Survey acronyms as a space-separated string. 1795 @type _surveys: str 1796 @rtype: list(str) 1797 """ 1798 return ["Detection", "Source"]
1799