Source Code for Module helpers.ProgrammeCoverageChecker

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  # $Id: ProgrammeCoverageChecker.py 8995 2012-02-09 17:57:10Z RossCollins $ 
  4  """ 
  5     Utility class to check that all multiframes in a survey are within a 
  6     product group and all used to make stacks (unless there is a limit on a 
  7     stack or different microstepping is used) 
  8   
  9   
 10     @author: N.J.G.. Cross 
 11     @org:    WFAU, IfA, University of Edinburgh 
 12  """ 
 13  #------------------------------------------------------------------------------ 
 14  from __future__      import division, print_function 
 15   
 16  from   wsatools.CLI                 import CLI 
 17  from   wsatools.DbConnect.CuSession import CuSession 
 18  from   wsatools.Logger              import Logger 
 19  import wsatools.Utilities               as utils 
 20  import wsatools.DbConnect.CommonQueries as queries 
 21  from   wsatools.SystemConstants     import DepCodes 
 22  #------------------------------------------------------------------------------ 
 23   
 24 -class ProgrammeCoverageChecker(CuSession): 
 25      """ Makes sure all multiframes have been assigned 
 26      """ 
 27      #-------------------------------------------------------------------------- 
 28      # Class constants (access as ProductAssigner.varName) 
 29      cuNum = 28 
 30   
 31      #-------------------------------------------------------------------------- 
 32   
 33      dateRange = CuSession.sysc.obsCal.dateRange()     #: Nights to stack. 
 34      maxProdID = None        #: Maximum product ID of stack to be produced. 
 35      minProdID = None        #: Minimum product ID of stack to be produced. 
 36      testProds = False       #: Test groups 
 37      testGroups = False      #: Test products 
 38      #-------------------------------------------------------------------------- 
 39   
 40 -    def _onRun(self): 
 41          """ 
 42          """ 
 43          self.allMultiframes = {} 
 44          for productType in self.programme.getProductTypes(): 
 45              frameTypeSel = queries.getFrameSelection(productType, noDeeps=True) 
 46              self.allMultiframes[productType] = set(self.archive.query( 
 47                  selectStr="m.multiframeID", 
 48                  fromStr="Multiframe as m, ProgrammeFrame as p", 
 49                  whereStr="p.programmeID=%s AND m.multiframeID=p.multiframeID " 
 50                  "AND %s AND %s AND utDate BETWEEN '%s' AND '%s'" % (self.programmeID, 
 51                  DepCodes.selectNonDeprecated, frameTypeSel, self.dateRange[0], 
 52                  self.dateRange[1]))) 
 53   
 54          if self.testGroups: 
 55              self.checkProductRequirements() 
 56              for productType in self.programme.getProductTypes(): 
 57                  self.areFieldsTooClose(productType) 
 58   
 59          if self.testProds: 
 60              self.checkProducts() 
 61   
 62      #-------------------------------------------------------------------------- 
 63   
 64 -    def checkProductRequirements(self): 
 65          """ Checks that all parts of the programme are covered and no 
 66          multiframes are missing. 
 67          """ 
 68          Logger.addMessage("Checking programme coverage for %s" 
 69                            % self.programme.getAcronym().upper()) 
 70   
 71          # @TODO: stacks that are not components of tiles 
 72          for productType in self.programme.getProductTypes(): 
 73              # select based on fields - should this be products? 
 74              multiframesSel1 = set(self.archive.query("multiframeID", 
 75                "Multiframe", "multiframeID in (%s)" 
 76                % queries.getMultiframes(self.programme, self.dateRange, 
 77                                         prodType=productType, useDeeps=False))) 
 78   
 79              # select based on multiframes in programme 
 80              self.testSelections(multiframesSel1, 
 81                                  self.allMultiframes[productType], 
 82                                  productType, "Grouping") 
 83   
 84      #-------------------------------------------------------------------------- 
 85   
 86 -    def checkProducts(self): 
 87          """ Check actual products 
 88          """ 
 89          productType = 'stack' 
 90          # @TODO: change dep codes - not use 50. 
 91          # @TODO: default seeing. Wrong size extension - cannot be stacked. 
 92          # Use Provenance to make sure that all are there 
 93          releaseNum = self.archive.query( 
 94              selectStr="max(releaseNum)", 
 95              fromStr="ProgrammeFrame", 
 96              whereStr="programmeID=%s" % self.programmeID, firstOnly=True) 
 97          # First get single epoch products 
 98          frameSetSel = queries.getFrameSelection(productType, noDeeps=True) 
 99          multiframesSel1 = self.archive.query( 
100              selectStr="m.multiframeID", 
101              fromStr="Multiframe as m,ProgrammeFrame as p", 
102              whereStr="p.programmeID=%s AND p.multiframeID=m.multiframeID AND " 
103              "%s AND %s and productID>0 and releaseNum=%s" 
104              % (self.programmeID, frameSetSel, DepCodes.selectNonDeprecated, 
105                 releaseNum)) 
106   
107          # Now get deeps - trace back the provenance. 
108          frameSetSel = queries.getFrameSelection(productType, deepOnly=True) 
109          multiframesSel1 += self.archive.query( 
110              selectStr="v.multiframeID", 
111              fromStr="Provenance as v,ProgrammeFrame as p,Multiframe as m", 
112              whereStr="p.programmeID=%s AND p.multiframeID=m.multiframeID AND " 
113              "%s AND %s and productID>0 and releaseNum=%s and " 
114              "m.multiframeID=v.combiframeID" 
115              % (self.programmeID, frameSetSel, DepCodes.selectNonDeprecated, 
116                 releaseNum)) 
117          frameTypeSel = queries.getFrameSelection(productType, noDeeps=True) 
118          multiframesSel2 = set(self.archive.query( 
119              selectStr="m.multiframeID", 
120              fromStr="Multiframe as m, ProgrammeFrame as p", 
121              whereStr="p.programmeID=%s AND m.multiframeID=p.multiframeID " 
122              "AND deprecated in (0,%s) AND %s AND utDate BETWEEN '%s' AND '%s'" 
123              % (self.programmeID, DepCodes.intermitDet, frameTypeSel, 
124                 self.dateRange[0], self.dateRange[1]))) 
125   
126          self.testSelections(set(multiframesSel1), multiframesSel2, 
127                              productType, "Product production") 
128   
129      #-------------------------------------------------------------------------- 
130   
131 -    def areFieldsTooClose(self, productType): 
132          """ Check whether any overlapping fields. 
133          """ 
134          overlappingFields = self.archive.query( 
135              selectStr="s1.fieldID as field1,s2.fieldID as field2," 
136                 "(60.*s1.stackRadius) as groupRadius1,(60*s2.stackRadius) as " 
137                 "groupRadius2,dbo.fGreatCircleDist(s1.ra,s1.dec,s2.ra,s2.dec) " 
138                 "as distanceMins", 
139              fromStr="Required%s as s1,Required%s as s2" % (productType, productType), 
140              whereStr="s1.programmeID=%s and s1.programmeID=s2.programmeID and " 
141                 "s1.fieldID!=s2.fieldID and dbo.fGreatCircleDist(s1.ra,s1.dec," 
142                 "s2.ra,s2.dec)<60.*(s1.stackRadius+s2.stackRadius)" 
143                 % (self.programmeID)) 
144          if overlappingFields: 
145              Logger.addMessage("The following fields are overlapping. " 
146                                "Problem with SetupProgramme") 
147              for field1, field2, rad1, rad2, dist in overlappingFields: 
148                  Logger.addMessage("Field %s overlaps with field %s. " 
149                                    "They should be separated by at least %s " 
150                                    "arcmins, but are separated by %s arcmins." 
151                                    % (field1, field2, rad1 + rad2, dist)) 
152   
153      #-------------------------------------------------------------------------- 
154   
155 -    def testSelections(self, multiframesSel1, multiframesSel2, productType, testType): 
156          """ 
157          """ 
158          diff1 = multiframesSel1.difference(multiframesSel2) 
159          diff2 = multiframesSel2.difference(multiframesSel1) 
160          if len(diff1) > 0 or len(diff2) > 0: 
161              Logger.addMessage("%s is not complete for %s %ss: " % 
162                                (testType.title(), self.programme.getAcronym().upper(), 
163                                 productType)) 
164              if len(diff1) > 0: 
165                  Logger.addMessage("Following multiframes are in the %s " 
166                                    "selection, but not in all science (stack) " 
167                                    "frames in %s in the time range %s - %s and " 
168                                    "with %s deprecations: %s" % 
169                                    (testType, self.programme.getAcronym().upper(), 
170                                     self.dateRange[0], self.dateRange[1], 
171                                     DepCodes.selectNonDeprecated, diff1)) 
172              if len(diff2) > 0: 
173                  Logger.addMessage("Following multiframes are in the all " 
174                                    "science (stack) frames in %s in the time " 
175                                    "range %s - %s and with %s deprecations, but " 
176                                    "not in the %s selection: %s" % 
177                                    (self.programme.getAcronym().upper(), 
178                                     self.dateRange[0], self.dateRange[1], 
179                                     DepCodes.selectNonDeprecated, testType, diff2)) 
180          else: 
181              Logger.addMessage("%s is complete for %s %ss: " % 
182                                (testType.title(), self.programme.getAcronym(), 
183                                 productType)) 
184   
185  #------------------------------------------------------------------------------ 
186  # Entry point for ProductAssigner 
187   
188  # Allow module to be imported as well as executed from the command line 
189  if __name__ == "__main__": 
190   
191      # Define specific command-line interface settings required by ProductAssigner 
192      CLI.progArgs += [ 
193        CLI.Argument("programmeID", "DXS"), 
194        CLI.Argument("begin_date", "05A", isValOK=CLI.isDateOK), 
195        CLI.Argument("end_date", "05A", isValOK=CLI.isDateOK)] 
196   
197      CLI.progOpts += [ 
198        CLI.Option('l', "products", 
199          "comma separated list of highest layer of products e.g. mosaic " 
200          "products if stacks and mosaics, tile products if stacks and tiles or " 
201          "stacks if just stacks", 
202          "LIST"), 
203        CLI.Option('g', "testGroups", 
204          "Test the grouping of products after ProgrammeSetup has been run"), 
205        CLI.Option('n', "minProdID", 
206          "start at this productID in highest layer of products", 
207          "NUMBER", isValOK=lambda val: val.isdigit()), 
208        CLI.Option('p', "testProducts", 
209          "Test the contents of stacks after CU13 has been run"), 
210        CLI.Option('v', "verbose", 
211          "more verbose logging"), 
212   
213        CLI.Option('x', "maxProdID", 
214          "finish at this productID in highest layer of products", 
215          "NUMBER", isValOK=lambda val: val.isdigit())] 
216   
217      cli = CLI(ProgrammeCoverageChecker, "$Revision: 8995 $") 
218      Logger.isVerbose = cli.getOpt("verbose") 
219      Logger.addMessage(cli.getProgDetails()) 
220      cu = ProgrammeCoverageChecker(cli.getArg("programmeID"), cli=cli) 
221      try: 
222          cu.dateRange = cu.sysc.obsCal.dateRange(cli.getArg("begin_date"), 
223                                                  cli.getArg("end_date")) 
224      except Exception as error: 
225          eType = "Invalid Option" 
226          Logger.addExceptionMessage(error, eType) 
227          raise SystemExit(eType + ": see log " + cu._log.pathName) 
228   
229      cu.dirDate = \ 
230        utils.makeDateTime(cli.getOpt("dir_date")).date.replace('-', '') 
231   
232      if cli.getOpt("maxProdID"): 
233          cu.maxProdID = int(cli.getOpt("maxProdID")) 
234   
235      if cli.getOpt("minProdID"): 
236          cu.minProdID = int(cli.getOpt("minProdID")) 
237   
238   
239      if cli.getOpt("testGroups"): 
240          cu.testGroups = cli.getOpt("testGroups") 
241      if cli.getOpt("testProducts"): 
242          cu.testProds = cli.getOpt("testProducts") 
243   
244   
245      cu.run() 
246   
247  #------------------------------------------------------------------------------ 
248