1
2
3 """
4 System constants for the VISTA Data Flow System. A set of global "symbols"
5 for use by all Python curation scripts and modules, compartmentalised here
6 to minimise maintenance overheads.
7
8 @author: N.C. Hambly
9 @org: WFAU, IfA, University of Edinburgh
10
11 @newfield contributors: Contributors, Contributors (Alphabetical Order)
12 @contributors: J. Bryant, R.S. Collins, N.J.G. Cross, E. Sutorius
13 """
14
15 from __future__ import division, print_function
16 from future_builtins import map
17
18 from collections import namedtuple
19 from operator import attrgetter, methodcaller
20 import os
21
22 from wsatools.ObsCalendar import ObsCal, VistaCal, WfcamCal, OmegaCal, GesCal
23
24
25
26 try:
27 assert os.path.exists(os.getenv('WSA_SANDBOX')), \
28 'WSA_SANDBOX = %s' % os.getenv('WSA_SANDBOX')
29 assert os.path.exists(os.path.join(os.getenv('WSA_SANDBOX'),
30 os.getenv('SRC_BRANCH'))), \
31 'SRC_BRANCH = %s' % os.getenv('SRC_BRANCH')
32 assert os.getenv('USER') is not None, 'USER = None'
33 except AssertionError as error:
34 print("Incorrect environment variable:", error)
35 raise SystemExit
36
37
38
39
40 AlphaNum = namedtuple('AlphaNum', 'name alpha num')
45 """ A collection of deprecation code constants.
46 """
47 constitDep = 104
48 deepStackOnly = 50
49 extProdComp = 112
50 incompGrp = 6
51 incompProv = 19
52 intermitDet = 51
53 invalidCat = 7
54 invalidDet = 2
55 invalidMet = 3
56 invalidSee = 8
57 ignoredObs = 26
58 missingCat = 1
59 missingDet = 103
60 nonDep = 0
61 poorEsoGrade = 56
62 propDepCom = 18
63 propDepDet = 102
64 propDepFra = 100
65 propDepObs = 101
66 poorPhotom = 110
67 rejectObs = 27
68 reprocAltn = 111
69 reprocCASU = 128
70 reprocWFAU = 255
71 testDetect = 53
72
73
74
75 selectDeprecated = "deprecated NOT IN (%s, %s, %s)" \
76 % (nonDep, deepStackOnly, intermitDet)
77
78 selectNonDeprecated = selectDeprecated.replace("NOT ", "")
79
80 selectDepAndNotTest = "deprecated NOT IN (%s, %s, %s, %s)" \
81 % (nonDep, deepStackOnly, intermitDet, testDetect)
82
83 selectNonDepAndTest = selectDepAndNotTest.replace("NOT ", "")
84
88 """ Constants common to the entire VDFS system.
89 """
90
91 obsCal = ObsCal()
92
93
94 minNumCPUs = 6
95 """ Minimum number of CPUs for tiling software.
96 """
97 one_kilobyte = 1024
98 """ A kilobyte in bytes.
99 """
100 one_megabyte = 1024 * 1024
101 """ A megabyte in bytes.
102 """
103 one_gigabyte = 1024 * 1024 * 1024
104 """ A gigabyte in bytes.
105 """
106 sqlDataTypeSize = dict(datetime=8, float=8, real=4, bigint=8, int=4,
107 smallint=2, tinyint=1)
108 """ Dictionary of SQL data type sizes in bytes referenced by data type
109 name.
110 """
111
112 adminDatabase = "master"
113 """ The name of the database context within which to perform miscellaneous
114 administrative commands (e.g. attach/detach etc). For Microsoft SQL
115 Server, this database context is "master".
116 """
117 catServerFileSep = chr(0x5c)
118 """ Path separator character for the catalogue server's O/S.
119 """
120 catServerSharePath = 2 * catServerFileSep + os.getenv('HOST').upper() \
121 + catServerFileSep + 'share' + catServerFileSep
122 """ Share path to curation server as seen by the catalogue server.
123 """
124 clusterName = "ramses"
125 """ Root name of the cluster servers.
126 """
127 loadServerVolumes = ('G', 'H', 'I')
128 """ Default load server volume configuration (for cluster servers).
129 """
130
131 curationServers = ["thoth", "menkaure", "shepseskaf", "userkaf", "sahure",
132 "neferefre", "ini", "unas"]
133 """Available curation servers
134 """
135 criticalDisks = ["/disk01/wsa", "/disk02/wsa", "/disk05/wsa",
136 "/disk06/wsa", "/disk10/wsa", "/disk33/vsa"]
137 """ Disks running a dump system backup or moved to other disks.
138 """
139 deprecatedDataStorage = ["/disk06/wsa", "/disk08/wsa", "/disk09/wsa"]
140 """ Disks on which development takes place, so no other data will
141 be stored there.
142 """
143 developDisks = ["/disk47/sys", "/disk34/vsa", "/disk34/wsa", "/disk34/osa"]
144 """ Disks on which development takes place, so no other data will be stored
145 there.
146 """
147 infinibandServers = ['ini', 'neferefre', 'kakai', 'sahure', 'userkaf',
148 'shepseskaf', 'menkaure']
149 """ Curation servers with infiniband connections.
150 """
151 sysPath = "/disk01/wsa/sys"
152 """ System path for lock and shell scripts.
153 """
154 userInterfaceTempStorage = ["/disk07/wsa"]
155 """ Disks on which development takes place, so no other data will
156 be stored there.
157 """
158
159 casu2massDir = os.path.join(developDisks[0], 'Viz2Mass', 'fits')
160 """ Directory containing CASU 2mass catalogues for wcsfit
161 """
162 compressImDir = os.path.join('products', 'jpgs')
163 """ Directory for compressed images (jpgs).
164 """
165 deprecatedComprImDir = os.path.join('products', 'deprecated_jpgs')
166 """ Path to directory for deprecated compressed images (jpgs).
167 """
168 deprecatedDir = os.path.join('ingest', 'deprecated')
169 """ Directory for deprecated files.
170 """
171 diffDir = os.path.join('products', 'diffs')
172 """ Directory for difference image products.
173 """
174 downloadDir = os.path.join('ingest', 'dload')
175 """ Final top directory where the data will be stored.
176 """
177 extProdDir = os.path.join(developDisks[0], 'ExternalProducts')
178 """ Directory for external products.
179 """
180 esoExtDir = os.path.join(extProdDir, 'teamCats')
181 """ Directory for external team catalogues
182 """
183 fitsDir = os.path.join('ingest', 'fits')
184 """ Final top directory where the data will be stored.
185 """
186 headerDir = os.path.join('ingest', 'headers')
187 """ Final top directory where the header metadata will be stored.
188 """
189 listDrivDir = os.path.join('products', 'list')
190 """ Directory for list driven catalogue products.
191 """
192 mosaicDir = os.path.join('products', 'mosaics')
193 """ Directory for mosaic image products.
194 """
195 productLogDir = os.path.join('products', 'logs')
196 """ Directory for production log files.
197 """
198 productsDir = 'products'
199 """ Directory for WFAU produced files, ie. stacked, mosaiced, and
200 difference images.
201 """
202 stackDir = os.path.join('products', 'stacks')
203 """ Directory for stacked image products.
204 """
205 swarpDir = os.path.join(extProdDir, 'swarpMos')
206 """ Directory for external SWARP mosaics
207 """
208 tempProductsDir = os.path.join('products', 'temp')
209 """ Directory for temporary products.
210 """
211 tileDir = os.path.join('products', 'tiles')
212 """ Directory for tiled image products.
213 """
214
215
216 wfauPrefix = 'e'
217
218
219 darkPrefix = 'dark_'
220
221
222 skyPrefix = 'sky_'
223
224
225 biasPrefix = 'bias_'
226
227
228 catSuffix = '_cat'
229
230
231 fringeSuffix = '_fringe'
232
233
234 confSuffix = '_conf'
235
236
237 cpmSuffix = '_cpm'
238
239
240 deepSuffix = '_dp'
241
242
243 diffSuffix = '_diff'
244
245
246 filtSuffix = '_two'
247
248
249 filtPrefix = 'tmp_'
250
251
252 flatSuffix = '_flat'
253
254
255 leavSuffix = '_sf'
256
257
258 listSuffix = '_list'
259
260
261 mjdSuffix = '_mjd'
262
263
264 mosaicSuffix = '_mos'
265
266
267 pregrtSuffix = '_pgrt'
268
269
270 simpleSuffix = '_simple'
271
272
273 stackSuffix = '_st'
274
275
276 fixcatSuffix = '_fix_cat'
277
278
279 tileSuffix = '_tl'
280
281
282 catType = '.fits'
283
284
285 jpgType = '.jpg'
286
287
288 mefType = '.fit'
289
290
291 uncType = '.unc'
292
293
294 frameTypes = ['mosaic', 'tile', 'deep', 'filt', 'leav', 'diff',
295 'stack', 'conf', 'normal']
296
297
298 cirdrShellScript = os.path.join(os.getenv("SCOSHOME"), "usr", "local",
299 "wsa", "cirdr", "cir_startup.bash")
300 """ Full path to the shell script to set up the correct environment for
301 CASU's cirdr package of source extractor related software.
302 """
303 emptyFitsCatalogueFileName = "empty_catalogue.fits"
304 """ Name of the default empty catalogue file.
305 """
306 procScript = "COMMON_StoredProcsAndFuncs.sql"
307 """ Script for generic procedures and functions for all databases.
308 """
309 sqlScriptPath = os.path.join(os.getenv("WSA_SANDBOX"), "sql")
310 """ Path to sql scripts.
311 """
312 srcPath = os.path.join(os.getenv("WSA_SANDBOX"), os.getenv("SRC_BRANCH"))
313 """ Path to present sandbox's source code. Useful if working
314 on a different branch of the source code.
315 """
316 webDocsPath = os.path.join(os.getenv("WEB_DIR"), "webpages")
317 """ Path to web pages.
318 """
319
320 defaultSexInparFile = "defaultDetCat.inpar"
321 """ Default sex inpar file. Modelled on UDS-DR8.
322 """
323
324
325 ftp_method = "ftp"
326
327
328 rem_camlist_dir = "/home/wfcam"
329
330
331 rem_lock_name = "readlock"
332
333
334 rem_ready2copy_name = "OK_TO_COPY"
335
336
337 rem_success_name = "SUCCESSFULLY_READ"
338
339
340 scpThreads = 1
341
342
343 scpUser = "wfcam"
344
345
346
347 uncSubStr = "UNC"
348
349
350 corSubStr = "COR"
351
352
353 bthSubStr = "BOTH"
354
355
356 varSubStr = "VAR"
357
358
359
360 channelWidth = 128
361
362
363 instrumentXnPix = 2048
364
365
366 instrumentYnPix = 2048
367
368
369 hasMicroStep = False
370
371
372 hasVarOrient = False
373
374
375 hasOffsetPos = False
376
377
378 hasEsoRelease = False
379
380
381 - def __new__(cls, switchName=None):
382 """ Creates a new object of type SystemConstants or its subclass if the
383 supplied argument suggest a specific subclass is more appropriate.
384
385 @param switchName: Initialised SystemConstants object or else a
386 string that contains necessary data to switch
387 on, e.g. database name, the INSTRUME FITS header
388 keyword, full path to a FITS file etc.
389 @type switchName: SystemConstants or str
390
391 @rtype: SystemConstants
392 """
393 if not switchName:
394 return object.__new__(cls)
395
396 if str(switchName).lower().replace('test', '').startswith('v') \
397 or os.sep + 'vsa' + os.sep in str(switchName).lower():
398 subclass = VsaConstants
399 elif str(switchName).lower().replace('test', '').startswith('ssa') \
400 or os.sep + 'ssa' + os.sep in str(switchName).lower():
401 subclass = SsaConstants
402 elif str(switchName).lower().replace('test', '').startswith('o') \
403 or os.sep + 'osa' + os.sep in str(switchName).lower():
404 subclass = OsaConstants
405 elif str(switchName).lower().replace('test', '').startswith('ges') \
406 or os.sep + 'ges' + os.sep in str(switchName).lower():
407 subclass = GesConstants
408 else:
409 subclass = WsaConstants
410
411 return super(cls, subclass).__new__(subclass)
412
413
415 """ Converting a object of this class type to a string returns the
416 classes name.
417 """
418 return self.__class__.__name__
419
421 """ @return: True, if these are GES constants.
422 @rtype: bool
423 """
424 return isinstance(self, GesConstants)
425
427 """ @return: True, if these are SSA constants.
428 @rtype: bool
429 """
430 return isinstance(self, SsaConstants)
431
433 """ @return: True, if these are WSA constants.
434 @rtype: bool
435 """
436 return isinstance(self, WsaConstants)
437
439 """ @return: True, if these are VSA constants.
440 @rtype: bool
441 """
442 return isinstance(self, VsaConstants)
443
445 """ @return: True, if these are VSA constants.
446 @rtype: bool
447 """
448 return isinstance(self, OsaConstants)
449
450
451 @staticmethod
453 """ Returns the values of the given constant in all subclasses.
454 """
455 return (attrgetter(constantName)(subClass)
456 for subClass in SystemConstants.__subclasses__()
457 if constantName in subClass.__dict__)
458
459 @staticmethod
461 """
462 The name of the server that hosts the supplied disk path.
463
464 @param fsPath: Path to anywhere within a disk file system.
465 @type fsPath: str
466 @rtype: str
467
468 """
469 disk = fsPath.lstrip(os.sep).split(os.sep)[0]
470 allServerRaidArrays = sum(map(methodcaller("items"),
471 SystemConstants.getAll("serverRaidArray")), [])
472
473 for server, diskPaths in allServerRaidArrays:
474 disks = (diskPath.lstrip(os.sep).split(os.sep)[0]
475 for diskPath in diskPaths)
476
477 if disk in disks:
478 return server
479
480
481 return os.path.basename(os.path.abspath(fsPath))
482
483 @staticmethod
485 """
486 List of logical volumes on the given catalogue server.
487
488 @param server: Optionally supply name of the server, otherwise assumes
489 it's a standard ahmose-like configuration.
490 @type server: str
491 @rtype: list(str)
492
493 """
494 if server == "hatshepsut":
495 volumes = SystemConstants.loadServerVolumes
496 elif server.startswith(SystemConstants.clusterName):
497 clusterId = server.replace(SystemConstants.clusterName, '')
498 if clusterId in ('1', '2', '10', '12', '13'):
499 volumes = SystemConstants.loadServerVolumes
500 elif clusterId == '14':
501 volumes = ['G']
502 elif clusterId in ('3', '4', '9'):
503 volumes = ['J', 'K', 'L']
504 elif clusterId in ('5', '6'):
505 volumes = ['M', 'N', 'O']
506 elif clusterId in ('7', '8'):
507 volumes = ['P', 'Q', 'R']
508
509
510 elif server == "nytro":
511 volumes = ['G']
512 else:
513 volumes = ['G', 'H', 'I', 'J']
514
515 return [volume + ':' + SystemConstants.catServerFileSep
516 for volume in volumes]
517
518 @staticmethod
519 - def dbSharePath(fileName='', serverName=os.getenv('HOST')):
520 """ @return: Full path to the given file located on the samba share.
521 @rtype: str
522 """
523 return os.sep + os.path.join(serverName, 'samba', 'share',
524 fileName).rstrip(os.sep)
525
527 """ @return: Path to the NFS share mounted on the catalogue load server.
528 @rtype: str
529 """
530 return os.path.join('/mnt', loadServer or self.loadServer)
531
533 """ @return: Name of the calibration tables schema SQL script file.
534 @rtype: str
535 """
536 return self.loadDatabase + "_CalibSchema.sql"
537
539 """ @return: Name of the curation tables schema SQL script file.
540 @rtype: str
541 """
542 return self.loadDatabase + "_CurationLogsSchema.sql"
543
545 """ @return: Path to the directory of ingest log files on the NFS share
546 of the catalogue load server (Windows).
547 @rtype: str
548 """
549 return os.path.join(sharePath or self.dbSharePath(), "ingestedLogs")
550
552 """ @return: Path to the directory of files to be ingested on the NFS
553 share of the catalogue load server (Windows).
554 @rtype: str
555 """
556 return os.path.join(sharePath or self.dbSharePath(), "ingestedFiles")
557
559 """ @return: Path to the directory of ingest file lists on the NFS
560 share of the catalogue load server (Windows).
561 @rtype: str
562 """
563 return os.path.join(sharePath or self.dbMntPath(), "ingestLists")
564
570
572 """ @return: Path to the directory of files with WCS errors on the NFS
573 share of the catalogue load server (Windows).
574 @rtype: str
575 """
576 return os.path.join(self.dbSharePath(), "wcsErrorFiles")
577
578
580 """ @return: Directory for calibration log files.
581 @rtype: str
582 """
583 return os.path.join(self.developDisks[0], 'calibration')
584
586 """ @return: Path to camb info??
587 @rtype: str
588 """
589 return os.path.join(self.genOpsDisk, 'cambinfo')
590
591 @staticmethod
593 """ @return: Path to CASU Perl scripts directory.
594 @rtype: str
595 """
596 return os.path.join(SystemConstants.srcPath, 'CasuSource/PerlScripts')
597
599 """ @return: Path for corrupted files.
600 @rtype: str
601 """
602 return os.path.join(self.genOpsDisk, 'ingest/corrupt')
603
604 @staticmethod
606 """ @return: Path to curation source code directory.
607 @rtype: str
608 """
609 return os.path.join(SystemConstants.srcPath, 'curation')
610
611 @staticmethod
618
620 """ @return: Full path to the directory containing filter data
621 @rtype: str
622 """
623 return os.path.join(self.developDisks[0], "Filters")
624
626 """ @return: Full path to the directory containing filter reference
627 data
628 @rtype: str
629 """
630 return os.path.join(self.filterDataDirPath(), "Reference")
631
632 @staticmethod
638
640 """ @return: Directory for illumination table files.
641 @rtype: str
642 """
643 return os.path.join(self.genOpsDisk, "IlluminationTables")
644
646 """@return: Path to curation logs.
647 @rtype: str
648 """
649 return os.path.join(self.genOpsDisk, "logs", fileName)
650
652 """ @return: Path to log of missing or broken files.
653 @rtype: str
654 """
655 return os.path.join(self.logFilePath(), "problems")
656
658 """ @return: Path to the monitor code directory.
659 @rtype: str
660 """
661 return os.path.join(self.curationCodePath(), 'invocations',
662 'monitoring')
663
665 """ @return: Path of the non-survey resgistration files
666 @rtype: str
667 """
668 return os.path.join(self.genOpsDisk, 'nonSurvey', fileName)
669
670 @staticmethod
676
677 @staticmethod
679 """ @return: Path where Cu16 works & stores source position tables.
680 @rtype: str
681 """
682 return SystemConstants.dbSharePath('SourcePosTables')
683
684 @staticmethod
686 """ @return: Path to sql script fragments used by SqlWrappers.py.
687 @rtype: str
688 """
689 return os.path.join(SystemConstants.sqlScriptPath, "fragments")
690
691 @staticmethod
698
699 @staticmethod
706
707 @staticmethod
714
715 @staticmethod
723
725 """ @return: Path to temporary working directory for curation scripts.
726 @rtype: str
727 """
728 return os.sep + os.path.join(os.getenv('HOST'), "tmp",
729 os.getenv('USER') + '_' + str(os.getpid()))
730
731 @staticmethod
733 """ @return: Path to permanent test directory used in examing test
734 results.
735 @rtype: str
736 """
737 if os.getenv("USER") == "njc" and os.getenv("HOST") == "djoser":
738 path = \
739 os.path.join(os.sep, "djoser", "home", "njc", "test", fileName)
740 else:
741 path = \
742 os.path.join(SystemConstants.developDisks[0], "test", fileName)
743
744 return path.rstrip(os.sep)
745
747 """ @return: Path of the transfer log files.
748 @rtype: str
749 """
750 return os.path.join(self.genOpsDisk, 'ingest/logs')
751
752 @staticmethod
761
762
769
778
780 """
781 Space to be left free on the disks in the mass storage raid file
782 system.
783
784 @param byPercent: If True, free disc space is defined by percentage
785 rather than absolute values.
786 @type byPercent: bool
787
788 @return: Free space available.
789 @rtype: int
790
791 """
792
793 nominalSpace = (0.02 if byPercent else
794 50 * self.one_gigabyte / self.one_kilobyte)
795
796 freeSpace = dict.fromkeys(
797 [disk for disk in self.availableRaidFileSystem()
798 if disk not in self.criticalDisks], nominalSpace)
799
800
801 criticalSpace = (0.1 if byPercent else
802 200 * self.one_gigabyte / self.one_kilobyte)
803
804 freeSpace.update(dict.fromkeys(self.criticalDisks, criticalSpace))
805 freeSpace.update(dict.fromkeys(self.deprecatedDataStorage,
806 criticalSpace))
807
808
809 developSpace = (0.5 if byPercent else
810 750 * self.one_gigabyte / self.one_kilobyte)
811
812 freeSpace.update(dict.fromkeys(self.developDisks, developSpace))
813
814 return freeSpace
815
817 """ @return: A list of paths to every mass storage raid disk.
818 @rtype: list(str)
819 """
820 return sorted(set(element for subList in self.serverRaidArray.values()
821 for element in subList))
822
823
825 """ @return: Complete list of paths on all RAID discs where the data
826 will be downloaded to.
827 @rtype: list(str)
828 """
829 return [os.path.join(raidDisk, 'ingest', 'dload')
830 for raidDisk in self.availableRaidFileSystem()
831 if raidDisk not in self.developDisks]
832
833
835 """ @return: Path to the log listing files that are broken in some way,
836 e.g. fail FITS verification checks or have missing header
837 keywords such as the PROV keyword.
838 @rtype: str
839 """
840 return os.path.join(self.missingFilesFilePath(), "files_broken.log")
841
843 """ @return: Path to the log listing files that have duplicate
844 non-deprecated entries in the database.
845 @rtype: str
846 """
847 return os.path.join(self.missingFilesFilePath(),
848 "files_duplicated.log")
849
851 """ @return: Path to the log listing files with database entries, but
852 not existing at the location provided by the database
853 entry.
854 @rtype: str
855 """
856 return os.path.join(self.missingFilesFilePath(), "files_missing.log")
857
859 """ @return: Path to the log listing files not ingested (i.e. not in
860 database, and possibly not on the pixel servers' raid
861 array).
862 @rtype: str
863 """
864 return os.path.join(self.missingFilesFilePath(),
865 "files_not_ingested.log")
866
867
868 @staticmethod
877
878 @staticmethod
892
906
908 """ @return: Product type of given database frame type.
909 @rtype: str
910 """
911 if not frameType.endswith("stack"):
912 raise ValueError("This is not a product: " + frameType)
913
914
915 for pType in reversed(self.productTypes):
916 if pType in frameType:
917 return pType
918
920 """ @return: List of FITS header keys inserted into files by WFAU.
921 @rtype: list(str)
922 """
923 return [self.loadDatabase + "_TIME", self.loadDatabase + "_MFID"]
924
925
927 """ @return: Command to run scp.
928 @rtype: str
929 """
930 return ("hpn-scp -P %s -o TCPRcvBufPoll=yes -pq" % self.scpPort
931 if light else "scp -q")
932
934 """ @return: Command to run ssh.
935 @rtype: str
936 """
937 return "hpn-ssh -p %s" % self.scpPort if light else "ssh"
938
939 - def scpMainDir(self):
940 """ @return: Top directory of the data tree on the scp server.
941 @rtype: str
942 """
943 return self.casuFileSystem[-1]
944
945
947 """ @return: URL to the science archive web interface.
948 @rtype: str
949 """
950 return 'http://surveys.roe.ac.uk/%s/' % self.loadDatabase.lower()
951
953 """ @return: URL to the science archive web interface including port.
954 @rtype: str
955 """
956 return 'http://surveys.roe.ac.uk:8080/%s/' % self.loadDatabase.lower()
957
961 """ Constants with specific values for the WSA.
962 """
963
964
965 loadDatabase = "WSA"
966
967
968 loadServer = SystemConstants.clusterName + '1'
969
970
971 ingestDbForServer = {loadServer: loadDatabase}
972
973
974
975
976 metadataFileGroups = ["Curation", "Indices"]
977
978
979 metadataTables = ["Multiframe", "MultiframeDetector", "CurrentAstrometry",
980 "ProgrammeFrame"]
981
982
983 nonSurveyDir = "WSA-NonSurvey"
984
985
986 pixelServerHostName = 'djoser:'
987
988
989 publicServers = [SystemConstants.clusterName + '5',
990 SystemConstants.clusterName + '6']
991
992
993
994 casuPrefix = 'w'
995
996
997 genOpsDisk = "/disk01/wsa"
998
999
1000 serverRaidArray = dict(
1001 djoser=[],
1002 sneferu=[],
1003 khufu=['/disk06/wsa', '/disk07/wsa', '/disk08/wsa',
1004 '/disk09/wsa'],
1005 djedefre=['/disk10/wsa', '/disk11/wsa', '/disk12/wsa',
1006 '/disk13/wsa'],
1007 khafre=['/disk14/wsa', '/disk15/wsa'],
1008 sheshonq=['/disk01/wsa', '/disk16/wsa', '/disk17/wsa',
1009 '/disk18/wsa'],
1010 osorkon=['/disk02/wsa', '/disk03/wsa', '/disk04/wsa',
1011 '/disk05/wsa'],
1012 takelot=['/disk19/wsa', '/disk23/wsa', '/disk24/wsa'],
1013 takelot2=['/disk20/wsa', '/disk21/wsa', '/disk22/wsa'],
1014 pami=['/disk29/wsa', '/disk30/wsa', '/disk31/wsa', '/disk32/wsa'],
1015 pami2=['/disk27/wsa', '/disk28/wsa', '/disk34/wsa'],
1016 piye1=[],
1017 piye2=['/disk36/wsa'],
1018 pedubaste1=['/disk37/wsa'],
1019 pedubaste2=['/disk38/wsa'],
1020 shepsesre=['/disk39/wsa', '/disk40/wsa'],
1021 wahkare=['/disk41/wsa'],
1022 shebaka=['/disk44/wsa'])
1023
1024
1025 versFormat = "%d"
1026
1027
1028 versStep = 1
1029
1030
1031
1032 autoTemplateIndexSchema = \
1033 SystemConstants.sqlTemplatePath("WSANS_templateIndices.sql")
1034
1035
1036 autoTemplateNeighboursSchema = \
1037 SystemConstants.sqlTemplatePath("WSA_autoTemplateNeighboursSchema.sql")
1038
1039
1040 autoTemplateSchema = \
1041 SystemConstants.sqlTemplatePath("WSA_autoTemplateSchema.sql")
1042
1043
1044 indexScript = "WSA_Indices.sql"
1045
1046
1047 nsIndexScript = "NonSurvey/WSA_NonSurveyIndices.sql"
1048
1049
1050 nsQcScript = \
1051 SystemConstants.sqlQualityControlPath("WSA_NonSurveyQCstatements.sql")
1052
1053
1054 surveyProcScript = "WSA_UKIDSS_StoredProcsAndFuncs.sql"
1055
1056
1057 surveyViewScript = "WSA_UkidssViews.sql"
1058
1059
1060
1061 amStart = "AMSTART"
1062
1063
1064 amEnd = "AMEND"
1065
1066
1067 casuCatType = 2
1068
1069
1070 casuDetThresh = {1:1.5, 4:1.25, 9:1.25}
1071
1072
1073 expTimeKeyWord = 'EXP_TIME'
1074
1075
1076 fieldOfView = (0.6713, 0.6713)
1077
1078
1079 hduNumKey = "CAMNUM"
1080
1081
1082 instrumentName = "WFCAM"
1083
1084
1085 instrumentPixScale = 0.4
1086
1087
1088 manualProgs = [1, 101, 102, 103]
1089
1090
1091 monthlyDetSurveys = []
1092
1093
1094 maxHDUs = 5
1095
1096
1097 maxRaDecExtentStackDeg = 0.068
1098
1099
1100 minRaDecExtentStackDeg = 0.017
1101
1102
1103 obsCal = WfcamCal()
1104
1105
1106 pawPrintArea = 0.207
1107
1108
1109 phtErrLimit = 0.02
1110
1111
1112 pointingKeys = ["RABASE", "DECBASE", "FILTER"]
1113
1114
1115 productTypes = ["stack", "mosaic"]
1116
1117
1118 projectKeys = ["PROJECT"]
1119
1120
1121 rCoreBasic = 2.5
1122
1123
1124 scienceProgs = {"LAS": 101, "GPS": 102, "GCS": 103, "DXS": 104, "UDS": 105}
1125
1126
1127 scrunch = 2
1128
1129
1130 surveyTranslations = {}
1131
1132
1133 telescopeName = "UKIRT"
1134
1135
1136
1137
1138 casuFileSystem = ['/data/apm22_a/wfcam/processed',
1139 '/data/apm22_b/wfcam/processed',
1140 '/data/apm23_d/wfcam/processed',
1141 '/data/apm24_b/wfcam/processed',
1142 '/data/apm24_c/wfcam/processed',
1143 '/data/apm22_c/wfcam/processed',
1144 '/data/apm22_d/wfcam/processed',
1145 '/data/apm23_e/wfcam/processed',
1146 '/data/apm22_e/wfcam/processed',
1147 '/data/apm23_b/wfcam/processed']
1148
1149
1150
1151 casuFileSystemVariables = AlphaNum("wfcam", "abcdef",
1152 range(22, 25) + [44, 47])
1153
1154
1155 casuIllumDir = "/data/apm7_a/mike/photomtests"
1156
1157
1158 scpPort = 22
1159
1160
1161 hasMicroStep = True
1162
1163 @staticmethod
1165 """ @return: The CASU scp server for processed data.
1166 @rtype: str
1167 """
1168 return '10.0.1.7' if light else 'apm7.ast.cam.ac.uk'
1169
1170 @staticmethod
1172 """ Returns list of file groups for a database of the given surveys.
1173
1174 @param _surveys: Survey acronyms as a space-separated string.
1175 @type _surveys: str
1176 @rtype: list(str)
1177 """
1178 return ["Detection", "Source"]
1179
1183 """ Constants with specific values for the GES.
1184 """
1185
1186 loadDatabase = "GES"
1187
1188
1189 loadServer = SystemConstants.clusterName + '1'
1190
1191
1192 ingestDbForServer = {loadServer: loadDatabase}
1193
1194
1195 metadataFileGroups = ["Spectrum", "Curation", "Indices"]
1196
1197
1198 publicServers = [SystemConstants.clusterName + '5',
1199 SystemConstants.clusterName + '6']
1200
1201
1202 scienceProgs = {}
1203
1204
1205 genOpsDisk = "/disk40/ges"
1206
1207
1208 serverRaidArray = dict(
1209 djoser=[],
1210 sneferu=[],
1211 khufu=[],
1212 djedefre=[],
1213 khafre=[],
1214 sheshonq=[],
1215 osorkon=[],
1216 takelot=[],
1217 takelot2=[],
1218 pami=[],
1219 pami2=[],
1220 piye1=[],
1221 piye2=[],
1222 pedubaste1=[],
1223 pedubaste2=[],
1224 shepsesre=['/disk40/ges'],
1225 wahkare=[],
1226 shebaka=[])
1227
1228
1229 indexScript = "GES_Indices.sql"
1230
1231
1232 ingestDataPath = os.path.join(genOpsDisk, "ingest", "data")
1233
1234
1235 sqlScriptPath = os.path.join(os.getenv("GES_DIR") or '', "sql")
1236
1237
1238 surveyProcScript = "GES_StoredProcsAndFuncs.sql"
1239
1240
1241 surveyViewScript = "GES_Views.sql"
1242
1243
1244 obsCal = GesCal()
1245
1246
1247
1248
1249 casuFileSystem = ['']
1250
1251
1252 casuFileSystemVariables = AlphaNum("ges", "a", [47])
1253
1254
1255 scpPort = 22
1256
1262
1263 @staticmethod
1265 """ @return: The CASU scp server for processed data.
1266 @rtype: str
1267 """
1268 return '10.0.1.29' if light else 'apm29.ast.cam.ac.uk'
1269
1270 @staticmethod
1272 """ Returns list of file groups for a database of the given surveys.
1273
1274 @param _surveys: Survey acronyms as a space-separated string.
1275 @type _surveys: str
1276 @rtype: list(str)
1277 """
1278 return []
1279
1303
1308 """ Constants with specific values for the VSA.
1309 """
1310
1311
1312 loadDatabase = "VSA"
1313
1314
1315 loadServer = SystemConstants.clusterName + '9'
1316
1317
1318 ingestDbForServer = {loadServer: loadDatabase,
1319 SystemConstants.clusterName + '14': loadDatabase + 'VVV'}
1320
1321
1322 metadataFileGroups = []
1323
1324
1325 metadataTables = ["Multiframe", "MultiframeDetector", "CurrentAstrometry",
1326 "ProgrammeFrame", "MultiframeEsoKeys",
1327 "MultiframeDetectorEsoKeys"]
1328
1329
1330 nonSurveyDir = "VSA-NonSurvey"
1331
1332
1333 obFiltDict = {
1334 'VVV':[('Z', 'Y'), ('J', 'H', 'Ks')],
1335 }
1336
1337
1338 pixelServerHostName = ''
1339
1340
1341 publicServers = [SystemConstants.clusterName + '3',
1342 SystemConstants.clusterName + '4']
1343
1344
1345
1346 casuPrefix = 'v'
1347
1348
1349 genOpsDisk = "/disk47/vsa"
1350
1351
1352 serverRaidArray = dict(
1353 djoser=[],
1354 sneferu=[],
1355 khufu=[],
1356 djedefre=[],
1357 khafre=[],
1358 sheshonq=[],
1359 osorkon=[],
1360 takelot=[],
1361 takelot2=[],
1362 pami=['/disk32/vsa'],
1363 pami2=['/disk33/vsa'],
1364 piye1=['/disk35/vsa'],
1365 piye2=['/disk36/vsa'],
1366 pedubaste1=['/disk37/vsa'],
1367 pedubaste2=['/disk38/vsa', '/disk43/vsa'],
1368 shepsesre=['/disk39/vsa', '/disk40/vsa'],
1369 wahkare=['/disk42/vsa'],
1370 shebaka=['/disk45/vsa', '/disk46/vsa'])
1371
1372
1373 versFormat = "%.1f"
1374
1375
1376 versStep = 0.1
1377
1378
1379
1380 autoTemplateIndexSchema = \
1381 SystemConstants.sqlTemplatePath("VSA_autoTemplateIndices.sql")
1382
1383
1384 autoTemplateNeighboursSchema = \
1385 SystemConstants.sqlTemplatePath("VSA_autoTemplateNeighboursSchema.sql")
1386
1387
1388 autoTemplateSchema = \
1389 SystemConstants.sqlTemplatePath("VSA_autoTemplateSchema.sql")
1390
1391
1392 autoTemplateViewSchema = \
1393 SystemConstants.sqlTemplatePath("VSA_autoTemplateViews.sql")
1394
1395
1396 indexScript = "VSA_Indices.sql"
1397
1398
1399 nsIndexScript = "NonSurvey/VSANS_Indices.sql"
1400
1401
1402 nsQcScript = \
1403 SystemConstants.sqlQualityControlPath("VSA_NonSurveyQCstatements.sql")
1404
1405
1406 surveyProcScript = "VSA_PS_StoredProcsAndFuncs.sql"
1407
1408
1409 surveyViewScript = "VSA_PS_Views.sql"
1410
1411
1412
1413 amStart = 'HIERARCH ESO TEL AIRM START'
1414
1415
1416 amEnd = 'HIERARCH ESO TEL AIRM END'
1417
1418
1419 casuCatType = 6
1420
1421
1422 casuDetThresh = {1:1.25}
1423
1424
1425 expTimeKeyWord = 'EXPTIME'
1426
1427
1428 fieldOfView = (1.2921, 1.0173)
1429
1430
1431 fieldOfViewTile = (1.4753, 1.0173)
1432
1433
1434 hduNumKey = "HIERARCH ESO DET CHIP NO"
1435
1436
1437 instrumentName = "VIRCAM"
1438
1439
1440 instrumentPixScale = 0.339
1441
1442
1443 manualProgs = []
1444
1445
1446 monthlyDetSurveys = [120]
1447
1448
1449 maxHDUs = 17
1450
1451
1452 maxRaDecExtentStackDeg = 0.068
1453
1454
1455 minRaDecExtentStackDeg = 0.017
1456
1457
1458 obsCal = VistaCal()
1459
1460
1461 pawPrintArea = 0.598
1462
1463
1464 phtErrLimit = 0.02
1465
1466
1467 pointingKeys = ["RA", "DEC", "HIERARCH ESO INS FILT1 NAME"]
1468
1469
1470 productTypes = ["stack", "tile", "mosaic"]
1471
1472
1473 projectKeys = ["HIERARCH ESO OBS PROG ID", "HIERARCH ESO OBS NAME",
1474 "HIERARCH ESO DPR CATG"]
1475
1476
1477 rCoreBasic = 3.0
1478
1479
1480 scienceProgs = {"VHS": 110, "VVV": 120, "VMC": 130, "VIKING": 140,
1481 "VIDEO": 150, "ULTRAVISTA": 160}
1482
1483
1484 scrunch = 4
1485
1486
1487 surveyTranslations = {"MAINTENANCE": "MAINTENANCE",
1488 "60.A-9285(A)": "SV-NGC253",
1489 "60.A-9285(B)": "SV-ORION",
1490 "60.A-9292(A)": "CAL",
1491 "60.A-9293(A)": "CAL",
1492 "60.A-9253(N)": "COMM",
1493 "179.B-2002": "VVV",
1494 "179.B-2003": "VMC",
1495 "179.A-2004": "VIKING",
1496 "179.A-2005": "UltraVISTA",
1497 "179.A-2006": "VIDEO",
1498 "179.A-2010": "VHS"
1499 }
1500
1501
1502 telescopeName = "VISTA"
1503
1504
1505 tileArea = 1.5008
1506
1507
1508 defaultPosAngleTolerance = 15.
1509
1510
1511 xWidthTileDith = 12707
1512
1513
1514 yWidthTileDith = 15593
1515
1516
1517 xWidthTileEar = 11776
1518
1519
1520 defaultSexInparFile = "defaultDetCatVsa.inpar"
1521 """ Default sex inpar file. Modelled on Victoria's VIDEO inpar file.
1522 """
1523
1524
1525
1526 casuFileSystem = ['/data/apm29_b/vista/processed',
1527 '/data/apm29_a/vista/processed',
1528 '/data/apm44_b/vista/processed']
1529
1530
1531 casuFileSystemVariables = AlphaNum("vista", "abe", [29, 44, 49, 50])
1532
1533
1534 casuIllumDir = "/data/apm35_a/vista/mike/photomtests/"
1535
1536
1537 scpPort = 22
1538
1539
1540 hasMicroStep = True
1541
1542
1543 hasVarOrient = True
1544
1545
1546 hasOffsetPos = True
1547
1548
1549 hasEsoRelease = True
1550
1551 @staticmethod
1553 """ @return: The CASU scp server for processed data.
1554 @rtype: str
1555 """
1556 return '10.0.1.29' if light else 'apm29.ast.cam.ac.uk'
1557
1558 @staticmethod
1560 """ Returns list of file groups for a database of the given surveys.
1561
1562 @param _surveys: Survey acronyms as a space-separated string.
1563 @type _surveys: str
1564 @rtype: list(str)
1565 """
1566 allSurveys = "cal vhs vvv vmc viking video ultravista".split()
1567 surveys = (allSurveys if not _surveys else
1568 set(_surveys.split()).intersection(allSurveys))
1569
1570 return [prog + filegroup
1571 for filegroup in ("Detection", "Source", "Curation", "Indices")
1572 for prog in surveys]
1573
1577 """ Constants with specific values for the OSA.
1578 """
1579
1580
1581 loadDatabase = "OSA"
1582
1583
1584 loadServer = SystemConstants.clusterName + '1'
1585
1586
1587 ingestDbForServer = {loadServer: loadDatabase}
1588
1589
1590 metadataFileGroups = ["Curation", "Indices"]
1591
1592
1593 metadataTables = ["Multiframe", "MultiframeDetector", "CurrentAstrometry",
1594 "ProgrammeFrame", "MultiframeEsoKeys",
1595 "MultiframeDetectorEsoKeys"]
1596
1597
1598 nonSurveyDir = "OSA-NonSurvey"
1599
1600
1601 pixelServerHostName = ''
1602
1603
1604 publicServers = [SystemConstants.clusterName + '5',
1605 SystemConstants.clusterName + '6']
1606
1607
1608
1609 casuPrefix = 'o'
1610
1611
1612 genOpsDisk = "/disk40/osa"
1613
1614
1615 serverRaidArray = dict(
1616 djoser=[],
1617 sneferu=[],
1618 khufu=[],
1619 djedefre=[],
1620 khafre=[],
1621 sheshonq=[],
1622 osorkon=[],
1623 takelot=[],
1624 takelot2=[],
1625 pami=[],
1626 pami2=[],
1627 piye1=[],
1628 piye2=[],
1629 pedubaste1=[],
1630 pedubaste2=[],
1631 shepsesre=['/disk39/osa', '/disk40/osa'],
1632 wahkare=['/disk41/osa'],
1633 shebaka=['/disk47/osa'])
1634
1635
1636 versFormat = "%.1f"
1637
1638
1639 versStep = 0.1
1640
1641
1642
1643 autoTemplateIndexSchema = \
1644 SystemConstants.sqlTemplatePath("OSA_autoTemplateIndices.sql")
1645
1646
1647 autoTemplateNeighboursSchema = \
1648 SystemConstants.sqlTemplatePath("OSA_autoTemplateNeighboursSchema.sql")
1649
1650
1651 autoTemplateSchema = \
1652 SystemConstants.sqlTemplatePath("OSA_autoTemplateSchema.sql")
1653
1654
1655 autoTemplateViewSchema = \
1656 SystemConstants.sqlTemplatePath("OSA_autoTemplateViews.sql")
1657
1658
1659 indexScript = "OSA_Indices.sql"
1660
1661
1662 nsIndexScript = "NonSurvey/OSANS_Indices.sql"
1663
1664
1665 surveyProcScript = "OSA_PS_StoredProcsAndFuncs.sql"
1666
1667
1668 surveyViewScript = "OSA_PS_Views.sql"
1669
1670
1671
1672 amStart = 'HIERARCH ESO TEL AIRM START'
1673
1674
1675 amEnd = 'HIERARCH ESO TEL AIRM END'
1676
1677
1678
1679 casuCatType = 6
1680
1681
1682
1683 casuDetThresh = {1:1.25}
1684
1685
1686 expTimeKeyWord = 'EXPTIME'
1687
1688
1689 fieldOfView = (1.0, 1.0)
1690
1691
1692 hduNumKey = "CAMNUM"
1693
1694 hduNumDict = {65: 1, 66: 2, 67: 3, 68: 4, 73: 5, 74: 6, 75: 7, 76: 8,
1695 81: 9, 82:10, 83:11, 84:12, 89:13, 90:14, 91:15, 92:16,
1696 69:17, 70:18, 71:19, 72:20, 77:21, 78:22, 79:23, 80:24,
1697 85:25, 86:26, 87:27, 88:28, 93:29, 94:30, 95:31, 96:32}
1698
1699
1700 instrumentName = "OMEGACAM"
1701
1702
1703 instrumentPixScale = 0.214
1704
1705
1706 manualProgs = []
1707
1708
1709 monthlyDetSurveys = []
1710
1711
1712 maxHDUs = 33
1713
1714
1715 maxRaDecExtentStackDeg = 0.05
1716
1717
1718 minRaDecExtentStackDeg = 0.017
1719
1720
1721 obsCal = OmegaCal()
1722
1723
1724 pawPrintArea = 0.949
1725
1726
1727 phtErrLimit = 0.02
1728
1729
1730 pointingKeys = ["RA", "DEC", "HIERARCH ESO INS FILT1 NAME"]
1731
1732
1733
1734 productTypes = ["stack"]
1735
1736
1737 projectKeys = ["HIERARCH ESO OBS PROG ID", "HIERARCH ESO OBS NAME",
1738 "HIERARCH ESO DPR CATG"]
1739
1740
1741 rCoreBasic = 4.65
1742
1743
1744 scienceProgs = {"ATLAS": 110}
1745
1746
1747 scrunch = 4
1748
1749
1750 surveyTranslations = {"MAINTENANCE": "MAINTENANCE",
1751 "60.A-9038(A)": "TECHNICAL",
1752 "60.A-9800(Q)": "CAL",
1753 "177.A-3011": "ATLAS",
1754 "177.A-3016": "KIDS",
1755 "177.A-3017": "KIDS",
1756 "177.A-3018": "KIDS",
1757 "177.D-3023": "VPHAS",
1758 }
1759
1760
1761 telescopeName = "VST"
1762
1763
1764
1765
1766 casuFileSystem = ['/data/apm45_a/vst/processed/']
1767
1768
1769 casuFileSystemVariables = AlphaNum("vst", "ab", [45, 46, 48])
1770
1771
1772 casuIllumDir = ""
1773
1774
1775 scpPort = 22
1776
1777
1778 hasVarOrient = True
1779
1780
1781 hasEsoRelease = True
1782
1783 @staticmethod
1785 """ @return: The CASU scp server for processed data.
1786 @rtype: str
1787 """
1788 return '10.0.1.29' if light else 'apm29.ast.cam.ac.uk'
1789
1790 @staticmethod
1792 """ Returns list of file groups for a database of the given surveys.
1793
1794 @param _surveys: Survey acronyms as a space-separated string.
1795 @type _surveys: str
1796 @rtype: list(str)
1797 """
1798 return ["Detection", "Source"]
1799