Source Code for Module helpers.UpdateConfIDs

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: UpdateConfIDs.py 8988 2012-02-08 15:20:42Z EckhardSutorius $ 
  4  """ 
  5     Update confID entries for data with different versions. 
  6   
  7     @author: E. Sutorius 
  8     @org:    WFAU, IfA, University of Edinburgh 
  9  """ 
 10  #------------------------------------------------------------------------------ 
 11  from   collections import defaultdict 
 12  import inspect 
 13  import os 
 14   
 15  from   wsatools.CLI                    import CLI 
 16  from   wsatools.DbConnect.DbSession    import DbSession 
 17  from   wsatools.File                   import File 
 18  import wsatools.FitsUtils                  as fits 
 19  from   wsatools.DbConnect.IngCuSession import IngCuSession 
 20  from   wsatools.Logger                 import ForLoopMonitor, Logger 
 21  import wsatools.DbConnect.CommonQueries    as queries 
 22  import wsatools.SystemConstants            as sysc 
 23   
 24  #------------------------------------------------------------------------------ 
 25 -class UpdateConfIDs(IngCuSession): 
 26      """Update missing confIDs. 
 27      """ 
 28      #-------------------------------------------------------------------------- 
 29   
 30 -    def __init__(self,                  
 31                   curator=CLI.getOptDef("curator"), 
 32                   database=DbSession.database, 
 33                   beginDate=CLI.getOptDef("begin"), 
 34                   endDate=CLI.getOptDef("end"), 
 35                   versionStr=CLI.getOptDef("version"), 
 36                   programme=CLI.getArgDef("programme"), 
 37                   isTrialRun=DbSession.isTrialRun, 
 38                   comment=CLI.getArgDef("comment")): 
 39   
 40          """ 
 41          @param beginDate:  First date to process, eg. 20050101. 
 42          @type  beginDate:  str 
 43          @param comment:    Descriptive comment as to why curation task is 
 44                             being performed. 
 45          @type  comment:    str 
 46          @param curator:    Name of curator. 
 47          @type  curator:    str 
 48          @param database:   Name of the database to connect to. 
 49          @type  database:   str 
 50          @param endDate:    Last date to process, eg. 20050131. 
 51          @type  endDate:    str 
 52          @param isTrialRun: If True, do not perform database modifications. 
 53          @type  isTrialRun: bool 
 54          @param programme:  Process data for given programme. 
 55          @type  programme: str 
 56          @param versionStr: Version number of the data. 
 57          @type  versionStr: str 
 58          """         
 59          # Initialize parent class 
 60          super(UpdateConfIDs, self).__init__(cuNum=-9999, 
 61                                              curator=curator, 
 62                                              comment=comment, 
 63                                              reqWorkDir=False, 
 64                                              keepWorkDir=False, 
 65                                              database=database, 
 66                                              autoCommit=False, 
 67                                              isTrialRun=isTrialRun) 
 68   
 69          beginDate, endDate = \ 
 70            self.sysc.obsCal.getDatesFromInput(beginDate, endDate) 
 71           
 72          typeTranslation = {"curator":str, 
 73                             "database":str, 
 74                             "beginDate":str, 
 75                             "endDate":str, 
 76                             "versionStr":str, 
 77                             "programme":str, 
 78                             "isTrialRun":bool, 
 79                             "comment":str} 
 80   
 81          super(UpdateConfIDs, self).attributesFromArguments( 
 82              inspect.getargspec(UpdateConfIDs.__init__)[0], locals(), 
 83              types=typeTranslation) 
 84   
 85          if not self.versionStr: 
 86              self.versionStr = str(max( 
 87                  self.sysc.obsCal.maxVersOfDate(self.beginDate), 
 88                  self.sysc.obsCal.maxVersOfDate(self.endDate))) 
 89   
 90      #-------------------------------------------------------------------------- 
 91   
 92 -    def _onRun(self): 
 93          """ Run. 
 94          """ 
 95          Logger.addMessage("Updating confIDs for %s-%s" % ( 
 96              self.sysc.loadDatabase, self.programme)) 
 97   
 98          updateConfList = [] 
 99          self._connectToDb() 
100   
101          # get fits files with default confID 
102          filesToUpdate = self.getFileNames() 
103   
104          if filesToUpdate: 
105              # get conf frames and their multiframeID 
106              confIdDict = self.getConfIDs() 
107   
108              # create a lookup dict between file mfID and conf mfID 
109              for dateVersStr in filesToUpdate: 
110                  for fileName in filesToUpdate[dateVersStr]: 
111                      date, versStr = dateVersStr.partition("_v")[::2] 
112                      availConfIDs = sorted(confIdDict[date]["%s_conf"%fileName]) 
113                      if not availConfIDs: 
114                          availConfIDs = self.getCirCpm( 
115                              filesToUpdate[dateVersStr][fileName][1], 
116                              confIdDict[date]) 
117                      try: 
118                          updateConfList.append( 
119                              (filesToUpdate[dateVersStr][fileName][0], 
120                               availConfIDs[-1][1])) 
121                      except IndexError: 
122                          if "sky" in fileName: 
123                              pass 
124                          else: 
125                              raise SystemExit( 
126                                  "<ERROR> no confIDs found for %s" % fileName) 
127              Logger.addMessage( 
128                  "%d entries will be updated in the %s" % ( 
129                      len(updateConfList), self.sysc.loadDatabase)) 
130   
131              # update confID entries in DB 
132              numUpdates = self.updateDB(updateConfList) 
133              Logger.addMessage("Updated %d entries in Multiframe." % numUpdates) 
134          else: 
135              Logger.addMessage( 
136                  "No missing confIDs in version %s from %s to %s" % ( 
137                      self.versionStr, self.beginDate, self.endDate)) 
138   
139          self._disconnectFromDb() 
140   
141          Logger.addMessage("finished.") 
142       
143      #-------------------------------------------------------------------------- 
144   
145 -    def updateDB(self, updateList): 
146          """ 
147          Update confID entries in Multiframe. 
148   
149          @param updateList: List containing multiframeID/confID pairs. 
150          @type  updateList:  
151          @return: Number of updates. 
152          @rtype:  int 
153          """ 
154          counter = 0 
155          for fileMfID, confID in sorted(updateList): 
156              num = self.archive.update("Multiframe", 
157                                        "confID=%d" % confID, 
158                                        where="multiframeID=%d" % fileMfID) 
159              counter += num 
160          return counter 
161           
162      #-------------------------------------------------------------------------- 
163   
164      @staticmethod 
165 -    def getCirCpm(fileName, confIdDict): 
166          """ 
167          Read conf frame from FITS file key CIR_CPM. 
168   
169          @return: Conf file name 
170          @rtype:  str 
171   
172          """ 
173          try: 
174              fimgptr = fits.open(fileName) 
175              circpm = fimgptr[1].header["CIR_CPM"].partition('[1]')[0] 
176              fimgptr.close() 
177              Logger.addMessage("From FITS %s: %s" % (fileName, circpm)) 
178          except AttributeError as error: 
179              Logger.addMessage("<ERROR>: %s in %s" % (error[0], fileName)) 
180   
181          return confIdDict[os.path.splitext(circpm)[0]] 
182           
183       #-------------------------------------------------------------------------- 
184   
185 -    def getConfIDs(self): 
186          """ 
187          Read data for conf frames from the given database. 
188   
189          @return: Date dict of dicts of files and their version/multiframeID. 
190          @rtype:  dict(dict(str:list(str))) 
191   
192          """ 
193          mfidList = self.archive.query( 
194              selectStr="dateVersStr, fileName, multiframeID", 
195              fromStr="FlatfileLookup", 
196              whereStr=" AND ".join([ 
197              "fileName LIKE '%conf%'", 
198              "SUBSTRING(dateVersStr,1,8) BETWEEN '%s' AND '%s'" % ( 
199                  self.beginDate, self.endDate)]) 
200              ) 
201          mfIDFileDict = defaultdict(lambda : defaultdict(list)) 
202          for dateVersStr, fileName, multiframeID in mfidList: 
203              fileObj = File(fileName) 
204              date, versStr = dateVersStr.partition("_v")[::2] 
205              mfIDFileDict[date][fileObj.root].append((versStr, multiframeID)) 
206   
207          return mfIDFileDict 
208   
209      #-------------------------------------------------------------------------- 
210   
211 -    def getFileNames(self): 
212          """ 
213          Read data for files that are missing confIDs. 
214   
215          @return: Date dict of files and their multiframeID. 
216          @rtype:  dict(dict(str:str)) 
217   
218          """ 
219          mfidList = self.archive.query( 
220              selectStr="dateVersStr, f.fileName, f.multiframeID", 
221              fromStr="FlatfileLookup f, ProgrammeFrame p, Multiframe m", 
222              whereStr=" AND ".join([ 
223                  "f.multiframeid=m.multiframeid", 
224                  "f.multiframeid=p.multiframeid", 
225                  "programmeID=%d" % self.sysc.scienceProgs.get(self.programme), 
226                  "confID<0", 
227                  "f.fileName NOT LIKE '%conf%'", 
228                  "SUBSTRING(dateVersStr,11,3) like '%s'" % self.versionStr, 
229                  "SUBSTRING(dateVersStr,1,8) BETWEEN '%s' AND '%s'" % ( 
230                      self.beginDate, self.endDate) 
231                  ]) + " OPTION (FORCE ORDER)") 
232          mfIDFileDict = defaultdict(dict) 
233          for dateVersStr, fileName, multiframeID in mfidList: 
234              fileObj = File(fileName.replace(self.sysc.pixelServerHostName, '')) 
235              mfIDFileDict[dateVersStr].update( 
236                  {os.path.join(fileObj.root): (multiframeID, fileObj.name)}) 
237          return mfIDFileDict 
238           
239   
240  #------------------------------------------------------------------------------ 
241  # Entry point for script. 
242  if __name__ == '__main__': 
243      # Define additional command-line options 
244      CLI.progArgs["comment"] = "Updating missing confIDs" 
245      CLI.progArgs += [CLI.Argument("programme", "GPS")] 
246      CLI.progOpts += [ 
247        CLI.Option('b', "begin", 
248                   "first date to process, eg. 20050101", 
249                   "DATE", str(IngCuSession.beginDateDef)), 
250        CLI.Option('e', "end", 
251                  "last date to process, eg. 20050131", 
252                  "DATE", str(IngCuSession.endDateDef)), 
253        CLI.Option('v', "version", 
254                   "version number of the data", 
255                   "STR", '')] 
256   
257      cli = CLI("UpdateConfIDs", "$Revision: 8988 $", __doc__) 
258      Logger.addMessage(cli.getProgDetails()) 
259   
260      updater = UpdateConfIDs( 
261                            cli.getOpt("curator"), 
262                            cli.getArg("database"), 
263                            cli.getOpt("begin"), 
264                            cli.getOpt("end"), 
265                            cli.getOpt("version"), 
266                            cli.getArg("programme"), 
267                            cli.getOpt("test"), 
268                            cli.getArg("comment")) 
269      updater.run() 
270