Source Code for Module helpers.FixIngestLogs

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: FixIngestLogs.py 9171 2012-05-10 16:57:22Z RossCollins $ 
  4  """ 
  5     Script to fix ingest logs. 
  6   
  7     @author: E.T.W. Sutorius 
  8     @org:    WFAU, IfA, University of Edinburgh 
  9  """ 
 10  #------------------------------------------------------------------------------ 
 11  import os 
 12  import string 
 13  import time 
 14   
 15  from   wsatools.CLI                 import CLI 
 16  from   wsatools.File                import File 
 17  from   wsatools.DbConnect.IngIngester  import IngestLogFile 
 18  from   wsatools.Logger              import Logger 
 19  from   wsatools.SystemConstants     import SystemConstants 
 20   
 21  #------------------------------------------------------------------------------ 
 22 -class FixIngestLogs(object): 
 23      """Script to fix ingest logs to allow ingests of data left on the file 
 24      share. This needs an old ingest log for the history bit, but creates 
 25      the ingest part from files on the file share. 
 26   
 27      """ 
 28   
 29      tableLookup = {4: "Detection"} # only cu4 implemented yet! 
 30   
 31      #-------------------------------------------------------------------------- 
 32   
 33 -    def __init__(self, cli): 
 34          """ 
 35          Initialise class from command-line options. 
 36   
 37          @param cli: Command-line arguments. 
 38          @type  cli: CLI 
 39   
 40          """ 
 41          self.progList = cli.getOpt("programmes").split(',') 
 42          ingLogFile = File(cli.getArg("inglog")) 
 43          if ingLogFile.ext == ".log": 
 44              self.ingLogs = [ingLogFile.name] 
 45          else: 
 46              ingLogFile.ropen() 
 47              self.ingLogs = ingLogFile.readlines(commentChar='#') 
 48          self.archive = cli.getArg("database") 
 49          outDir = cli.getOpt("outdir") 
 50          self.sysc = SystemConstants(self.archive) 
 51          self.outDir = (outDir if not "dbSharePath" in outDir 
 52                         else self.sysc.dbSharePath()) 
 53   
 54      #-------------------------------------------------------------------------- 
 55   
 56 -    def run(self): 
 57          """ 
 58          """ 
 59          for ingLogFile in self.ingLogs: 
 60              ingestLogFile = IngestLogFile(ingLogFile) 
 61              Logger.addMessage("Reading ingest log %s" % ingestLogFile.name) 
 62              tmpIngestOrder, tmpTableDict, histories = ingestLogFile.pickleRead() 
 63   
 64              ingestOrder = [] 
 65              ingestSurveys = {} 
 66              for fileName in os.listdir(self.sysc.dbSharePath()): 
 67                  if fileName.endswith('.dat') and "%s-%s" % ( 
 68                      ingestLogFile.commonPrefix, 
 69                      ingestLogFile.runNo) in fileName: 
 70                      cuNum = ingestLogFile.cuNum 
 71                      tableName = os.path.splitext(fileName)[0].rpartition('_')[2] 
 72                      ingestOrder.append(tableName) 
 73                      survey = tableName[:tableName.find( 
 74                          FixIngestLogs.tableLookup[cuNum])] 
 75                      if len(survey)>3 and self.sysc.isWSA(): 
 76                          schemaFile = "NonSurvey/%sNS%s_%sSchema.sql" % ( 
 77                              self.archive, survey, survey) 
 78                      else: 
 79                          schemaFile = "%s_%sSchema.sql" % (self.archive, survey) 
 80                      ingestSurveys[tableName] = dict(schema=schemaFile, 
 81                                                      ingfile=fileName) 
 82              if ingestOrder: 
 83                  outgestLogFile = IngestLogFile(os.path.join( 
 84                      self.outDir, os.path.split(ingLogFile)[1])) 
 85                  time.sleep(2) 
 86                  if os.path.exists(outgestLogFile.name): 
 87                      Logger.addMessage("%s exists!" % outgestLogFile.name) 
 88                      os.rename(outgestLogFile.name, outgestLogFile.name + ".old") 
 89   
 90                  outgestLogFile.pickleWrite(ingestOrder, ingestSurveys, 
 91                                             histories) 
 92                  Logger.addMessage("Ingest data written to %s" % \ 
 93                                    outgestLogFile.name) 
 94           
 95           
 96  #------------------------------------------------------------------------------ 
 97  # Entry point for script. 
 98   
 99  # Allow module to be imported as well as executed from the command line 
100  if __name__ == '__main__': 
101      # Define command-line interface settings for UpdateFitsFileNames 
102      CLI.progArgs += [CLI.Argument("inglog", "cu04.log")] 
103      CLI.progOpts += [ 
104          CLI.Option('p', "programmes", 
105                     "update only with this programme", "LIST", "all"), 
106          CLI.Option('o', "outdir", "directory where new log is written", 
107                     "DIR", 'dbSharePath')] 
108      cli = CLI(FixIngestLogs, "$Revision: 9171 $") 
109      Logger.addMessage(cli.getProgDetails()) 
110      FixIngestLogs(cli).run() 
111