1
2
3
4 """
5 Automatically runs the necessary curation tasks to prepare a programme's
6 data for release. Once the data are ingested into the load database. It
7 works for any programme, but non-surveys should be both automatically
8 prepared and released together using the NonSurveyRelease.py script, and
9 curation tasks for shallow UKIDSS surveys are best run in stages due to the
10 data quantities involved. So for practical purposes this script should be
11 used for curating the deep UKIDSS surveys, DXS, UDS, which have smaller
12 data volumes. This script cuts out many of the otherwise manual tasks
13 that must be performed for deep surveys so is useful for curating the CAL
14 data set too.
15
16 @author: N.J.G. Cross
17 @org: WFAU, IfA, University of Edinburgh
18
19 @newfield contributors: Contributors, Contributors (Alphabetical Order)
20 @contributors: R.S. Collins
21 """
22
23 from __future__ import division, print_function
24
25 import multiprocessing
26 import time
27
28 from wsatools.Automator import autoCurate
29 from wsatools.CLI import CLI
30 import wsatools.CSV as csv
31 from wsatools.DbConnect.CuSession import CuSession
32 from wsatools.DbConnect.DbSession import DbSession
33 from wsatools.Logger import Logger
34 from wsatools.ProgrammeBuilder import ProgrammeBuilder, commitSchema
35 from wsatools.SystemConstants import SystemConstants
36 import wsatools.Utilities as utils
37
38
40 """ @return: True, if the product list supplied contains only recognised
41 products.
42 @rtype: bool
43 """
44 allTypes = utils.unpackList(SystemConstants.getAll("productTypes"))
45
46 return not set(csv.values(products)).difference(allTypes)
47
48
49
50
51
52 if __name__ == '__main__':
53
54
55 CLI.progArgs += [
56 CLI.Argument("begin_date", "05A", isValOK=CLI.isDateOK),
57 CLI.Argument("end_date", "05A", isValOK=CLI.isDateOK),
58 CLI.Argument("releaseNum", '3', isValOK=lambda x: x.isdigit())]
59
60 CLI.progOpts += [
61 CLI.Option('R', "reset",
62 "perform a re-release for the requested re-release number"),
63
64 CLI.Option('f', "force",
65 "Force curation, even if mosaic provenance is incomplete"),
66
67 CLI.Option('i', "newSource",
68 "Reproduce source table"),
69
70 CLI.Option('j', "newNeigh",
71 "Reproduce neighbour tables"),
72
73 CLI.Option('l', "fields",
74 "comma separated list of highest layer of fieldIDs e.g. mosaic "
75 "fieldIDs if stacks and mosaics, tile fieldIDs if stacks and tiles or "
76 "stacks if just stacks",
77 "LIST"),
78
79 CLI.Option('n', "minProdID",
80 "minimum product ID in highest product layer",
81 "NUMBER", isValOK=lambda x: x.isdigit()),
82
83 CLI.Option('p', "preCU6_only",
84 "Runs automatic curation up to CU6, but not CU6 itself"),
85
86 CLI.Option('r', "recalib",
87 "force recalibration of intermediate stacks"),
88
89 CLI.Option('s', "reprocess",
90 "force recreation of all products of following types (stack,tile)",
91 "LIST", isValOK=isProductListOK),
92
93 CLI.Option('v', "verbose",
94 "more verbose logging"),
95
96 CLI.Option('x', "maxProdID",
97 "maximum product ID in highest product layer",
98 "NUMBER", isValOK=lambda x: x.isdigit()),
99
100 CLI.Option('y', "threads",
101 "maximum number of processors to use (minimum allowed is %s)"
102 % SystemConstants.minNumCPUs,
103 "NUMBER", str(multiprocessing.cpu_count()),
104 isValOK=lambda val: val.isdigit())]
105
106 cli = CLI("AutoCurate", "$Revision: 9016 $", __doc__)
107 Logger.isVerbose = cli.getOpt("verbose")
108 Logger.addMessage(cli.getProgDetails())
109 CLI.check64bitServer()
110 if cli.getOpt("reset") and not CLI.isConfirmed("You have requested to "
111 "perform a reprocessing. This will reset all existing data for the "
112 "current WFAU-product version number (%s)." % cli.getArg("releaseNum")):
113 exit()
114
115 dbName = cli.getArg("database").split('.')[-1]
116 logFileName = \
117 '%s_AutoCurate_%s.log' % (dbName, time.strftime('%Y%m%d_%H%M'))
118
119 try:
120 try:
121 sysc = SystemConstants(dbName)
122 dateRange = sysc.obsCal.dateRange(cli.getArg("begin_date"),
123 cli.getArg("end_date"))
124
125 except Exception as error:
126 raise CuSession.CuError(error)
127
128
129
130 if autoCurate(cli, cli.getArg("programmeID"),
131 int(cli.getArg("releaseNum")), dateRange,
132 redoSources=cli.getOpt("newSource"),
133 redoProducts=(csv.values(cli.getOpt("reprocess"))
134 if cli.getOpt("reprocess") else None),
135 redoRecal=cli.getOpt("recalib"),
136 redoNeigh=cli.getOpt("newNeigh"),
137 minProdID=cli.getOpt("minProdID"),
138 maxProdID=cli.getOpt("maxProdID"),
139 doCu6=not cli.getOpt("preCU6_only"),
140 skipMosaicCheck=cli.getOpt("force"),
141 isRerelease=cli.getOpt("reset"),
142 numCPUs=int(cli.getOpt("threads")),
143 fields=(csv.values(cli.getOpt("fields"))
144 if cli.getOpt("fields") else None)):
145
146
147 Logger.addMessage("Updating schema documentation")
148 Logger.addMessage(cli.getProgDetails("ProgrammeBuilder"))
149
150 ProgrammeBuilder.createSchemaOnly = True
151 ProgrammeBuilder(cli.getArg("programmeID"),
152 database=cli.getArg("database"),
153 isTrialRun=True,
154 userName=cli.getOpt("user")).run()
155
156 db = DbSession(autoCommit=True, cli=cli)
157 commitSchema(db, "updated", isNonSurvey=False)
158
159 except BaseException as error:
160 with utils.noInterrupt():
161 stdErrMsg = CuSession.logException(error)
162 if error.args:
163 stdErrMsg += ": see log " + logFileName
164
165 raise SystemExit(stdErrMsg)
166
167 finally:
168
169 if not cli.getOpt("test"):
170 Logger(logFileName)
171
172
173
174
175
176