Source Code for Module helpers.CleanUpPixelServer

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: CleanUpPixelServer.py 7205 2010-07-12 10:47:39Z EckhardSutorius $ 
  4  """ 
  5     Creates a shell script to clear file space and delete interleave component 
  6     files. Also gives SQL to update the database. 
  7   
  8     @author: Eckhard Sutorius 
  9     @org:    WFAU, IfA, University of Edinburgh 
 10   
 11     @newfield contributors: Contributors, Contributors (Alphabetical Order) 
 12     @contributors: R.S. Collins 
 13  """ 
 14  #------------------------------------------------------------------------------ 
 15  from   __future__      import division, print_function 
 16  import os 
 17   
 18  from   wsatools.CLI                 import CLI 
 19  from   wsatools.DbConnect.DbSession import DbSession 
 20  #------------------------------------------------------------------------------ 
 21   
 22 -def getNormalLeav(db, versionNum, dateRange): 
 23      """ Get all 'normal' files contributing to an interleaved file. 
 24      """ 
 25      return db.query("filename", "Multiframe", 
 26        whereStr="project LIKE 'u/ukidss%' AND filename LIKE '%_v" 
 27                 + versionNum + "/w%' AND " + dateRange + 
 28                 " AND filename NOT LIKE " 
 29                 "'PixelFileNoLongerAvailable:%' AND frameType LIKE 'normal' " 
 30                 "AND multiframeID IN (SELECT P.multiframeID FROM " 
 31                 "Multiframe AS M, Provenance AS P WHERE " 
 32                 "M.multiframeID=P.combiframeID AND frameType='leav')") 
 33   
 34  #------------------------------------------------------------------------------ 
 35   
 36 -def getNonStack(db, versionNum, dateRange): 
 37      """ Get all science frames that are not stacks or confs of a stack. 
 38      """ 
 39      return db.query("filename", "Multiframe", 
 40        whereStr="project LIKE 'u/ukidss%' AND filename LIKE '%_v" 
 41                 + versionNum + "/w%' AND " + dateRange + 
 42                 " AND filename NOT LIKE " 
 43                 "'PixelFileNoLongerAvailable:%' AND frameType NOT LIKE " 
 44                 "'%stack%'") 
 45   
 46  #------------------------------------------------------------------------------ 
 47   
 48 -def writeDelScript(delFile, delType, fileNames): 
 49      """ Write the shell script that deletes all files found. 
 50      """ 
 51      ofh = open(delFile, 'w') 
 52      ofh.write("#! /usr/bin/tcsh\n") 
 53      for fileName in fileNames: 
 54          if delType.lower() == "normalleav"and fileName.count('_') != 2: 
 55              print("WARNING: wrong file? " + fileName) 
 56   
 57          elif delType.lower() == "nonstack" \ 
 58            and ("_st" in fileName or not "/w2" in fileName): 
 59   
 60              print("WARNING: wrong file? " + fileName) 
 61   
 62          ofh.write("rm -f " + fileName.rpartition(':')[2] + '\n') 
 63   
 64      ofh.close() 
 65      os.chmod(delFile, 0755) 
 66   
 67  #------------------------------------------------------------------------------ 
 68  # Entry point for script. 
 69   
 70  # Allow module to be imported as well as executed from the command line 
 71  if __name__ == "__main__": 
 72      # Define command-line interface options 
 73      CLI.progArgs += [ 
 74        CLI.Argument("beginDate", "05A", isValOK=CLI.isDateOK), 
 75        CLI.Argument("endDate", "05A", isValOK=CLI.isDateOK), 
 76        CLI.Argument("version", "1", isValOK=lambda x: x.replace('.', '').isdigit() and x>'0')] 
 77   
 78      frameTypes = ['normalleav', 'nonstack'] 
 79      CLI.progOpts.remove("test") 
 80      CLI.progOpts.append(CLI.Option('f', "frame_types", 
 81          " or ".join(frameTypes), "NAME", frameTypes[0], 
 82          isValOK=lambda x: x in frameTypes)) 
 83   
 84      cli = CLI("CleanUpPixelServer", "$Revision: 7205 $", __doc__) 
 85   
 86      db = DbSession(cli=cli) 
 87      delType = cli.getOpt("frame_types") 
 88      versionnr = cli.getArg("version") 
 89      obsCal = db.sysc.obsCal 
 90      try: 
 91          dateRange = "utDate BETWEEN '%s' AND '%s'" \ 
 92            % obsCal.dateRange(cli.getArg("beginDate"), cli.getArg("endDate")) 
 93   
 94      except Exception as error: 
 95          raise SystemExit("Illegal Option: " + str(error)) 
 96   
 97      # create delete script 
 98      if delType.lower() == "normalleav": 
 99          fileNames = getNormalLeav(db, versionnr, dateRange) 
100          print("%s files found." % len(fileNames)) 
101          if fileNames: 
102              writeDelScript("deleteLeaves.script", delType, fileNames) 
103   
104      elif delType.lower() == "nonstack": 
105          fileNames = getNonStack(db, versionnr, dateRange) 
106          print("%s files found." % len(fileNames)) 
107          if fileNames: 
108              writeDelScript("deleteNonStacks.script", delType, fileNames) 
109   
110      print("To update the server, run the following SQL on the appropriate " 
111            "DBs:") 
112   
113      if delType.lower() == "normalleav": 
114          print("UPDATE Multiframe" 
115                " SET filename='PixelFileNoLongerAvailable:'+filename\n" 
116                " WHERE project LIKE 'u/ukidss%' AND filename LIKE '%_v" 
117                + versionnr + "/%'\nAND " + dateRange + "\nAND filename NOT LIKE" 
118                " 'PixelFileNoLongerAvailable:%'\n" 
119                "AND frameType LIKE 'normal' AND multiframeID IN (\n" 
120                "SELECT P.multiframeID FROM Multiframe as M, Provenance as P\n" 
121                " WHERE M.multiframeID=P.combiframeID AND frameType='leav')") 
122   
123      elif delType.lower() == "nonstack": 
124          print("UPDATE Multiframe" 
125                " SET filename='PixelFileNoLongerAvailable:'+filename\n" 
126                " WHERE project LIKE 'u/ukidss%' AND filename LIKE '%_v" 
127                + versionnr + "/w%'\nAND " + dateRange + "\n" 
128                "AND filename NOT LIKE 'PixelFileNoLongerAvailable:%'\n" 
129                "AND  frameType NOT LIKE '%stack%'") 
130   
131  #------------------------------------------------------------------------------ 
132  #-- Change log: 
133  #-- 
134  #-- 31-Oct-2006, ETWS: Original version. 
135  #-- 01-Nov-2006,   JB: Fixed SQL for mulitple uses on the same date range. 
136  #-- 08-Nov-2006,   JB: Fixed an SQL bug in the update srcipt. 
137  #-- 14-Nov-2006, ETWS: Included non-stack removal, therefore renamed from 
138  #--                    deleteInterleave.py 
139