Source Code for Module helpers.RedoFiles

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  # $Id: RedoFiles.py 7146 2010-06-29 15:10:06Z EckhardSutorius $ 
  4  """ 
  5     Upload files from CASU, deprecate existing ones, update the databases 
  6     and create a list for CU2 to 4. 
  7   
  8     @author: E. Sutorius 
  9     @org:    WFAU, IfA, University of Edinburgh 
 10  """ 
 11  #------------------------------------------------------------------------------ 
 12  import getopt 
 13  import os 
 14  import sys 
 15   
 16  from   wsatools.File                import File 
 17  import wsatools.FitsUtils               as fits 
 18  from   wsatools.DbConnect.DbSession import DbSession 
 19  import wsatools.SystemConstants         as sysc 
 20  import wsatools.Utilities               as utils 
 21  #------------------------------------------------------------------------------ 
 22   
 23 -def getFreeDisk(): 
 24      """ 
 25      Get the disk with the most free disk space. 
 26   
 27      @return: Disk name. 
 28      @rtype:  str 
 29   
 30      """ 
 31      spacePerDisk = utils.getDiskSpace( 
 32        set(sysc.massStorageRaidFileSystem())-set(sysc.excludedRaidFileSystem())) 
 33   
 34      freeDiskSpace = utils.Ratings((d, -spacePerDisk[d][1]) 
 35       for d in spacePerDisk if spacePerDisk[d][1] > sysc.leaveSpaceOnMSRFS()[d]) 
 36   
 37      return utils.Ratings((key, freeDiskSpace.rating(key)) 
 38                           for key in freeDiskSpace).getKeyByRating(0) 
 39   
 40  #------------------------------------------------------------------------------ 
 41   
 42 -def getJpgDict(fileDict): 
 43      """ 
 44      Get a dictionary of jpegs per date directory name. 
 45   
 46      @param fileDict: A dictionary containig FITS file names per dateVersDir. 
 47      @type  fileDict: dict 
 48   
 49      @return: Dict containing jpg names for each dateDir. 
 50      @rtype:  dict(str, list(str)) 
 51   
 52      """ 
 53      jpgDict = {} 
 54      for dateVersStr in fileDict: 
 55          for fileName in fileDict[dateVersStr]: 
 56              if fileName.endswith(sysc.mefType()): 
 57                  cmd = ''.join(["ls -1 /disk??/wsa/products/jpgs/", 
 58                                 dateVersStr, "/", fileName.rpartition('.')[0], 
 59                                 "_[1234].jpg"]) 
 60                  jpgDict.setdefault(dateVersStr, []) 
 61                  jpgDict.extend(x.rstrip() for x in os.popen(cmd)) 
 62      return jpgDict 
 63   
 64  #------------------------------------------------------------------------------ 
 65   
 66 -def readFileList(fileName, casuSysDict): 
 67      """ 
 68      Read the data from a log file containing FITS file names in CASU format. 
 69      @param fileName: Name of the file. 
 70      @type  fileName: str 
 71      @param casuSysDict: Dictionary containing CASU dir info. 
 72      @type  casuSysDict: dict 
 73      @return: Dict containing the file names for each date. 
 74      @rtype: dict(str: list(str)) 
 75      """ 
 76      logFile = File(fileName) 
 77      logFile.ropen() 
 78      fileNames = logFile.readlines(strip=True) 
 79      logFile.close() 
 80      fileDict = {} 
 81      for line in fileNames: 
 82          if line: 
 83              dateStr, fitsName = line.split('/') 
 84              dateVersStr = ''.join([ 
 85                  dateStr, "_v", casuSysDict[dateStr][0][1].partition('.')[0]]) 
 86              fileDict.setdefault(dateVersStr, []).append(fitsName) 
 87      return fileDict 
 88   
 89  #------------------------------------------------------------------------------ 
 90   
 91 -def readVersion(direc, archive, light): 
 92      """ 
 93      Create a dictionary for each date containing the path and version 
 94      number at CASU. 
 95      @param direc: CASU directory name. 
 96      @type  direc: str 
 97      @param light: Flag if UKLight is used. 
 98      @type  light: bool 
 99      @return: Dict containing path and version number tuples for each directory. 
100      @rtype: dict(str: (str, str)) 
101      """ 
102      cmd = ''.join(["ssh ", sysc.scp_user(), "@", 
103                     sysc.scp_server(archive, light), 
104                     " \"find ", direc, " -name 'VERSION_*' -print\""]) 
105   
106      versDirs = sorted(x.replace('\n', '') for x in os.popen(cmd)) 
107      versDict = {} 
108      for entry in versDirs: 
109          dirPath, version = os.path.split(entry) 
110          versNum = version.partition('_')[2] 
111          versDict.setdefault(os.path.basename(dirPath), []).append([dirPath, 
112                                                                     versNum]) 
113      return versDict 
114   
115  #------------------------------------------------------------------------------ 
116   
117 -def usage(): 
118      print "usage:  ./RedoFiles.py [-j/--janet] [-o/--outdisk] CASUlistfile" 
119      print "    -j/--janet:  switch to Janet" 
120      print "    -o/--outdisk:  Disk where the deprecated data is moved to, default: disk with most space." 
121      print "    CASUlistfile   file name list from CASU with data that needs to be replaced.\n" 
122   
123  #------------------------------------------------------------------------------ 
124   
125 -def main(argv): 
126      casuSysDict = {} 
127      light = True 
128      archive = "WSA" 
129      outDisk = getFreeDisk() 
130      scriptDir = "redo" 
131      os.system("rm -rf " + scriptDir) 
132   
133      try: 
134          opts, args = getopt.getopt(argv[1:], "ahjo:", 
135                                     ["archive", "help", "janet", "outdisk="]) 
136   
137      except getopt.GetoptError: 
138          # print help information and exit: 
139          print argv 
140          usage() 
141          raise SystemExit 
142   
143      for o, a in opts: 
144          if o in ("-a", "--archive"): 
145              archive = a 
146          if o in ("-j", "--janet"): 
147              light = False 
148          if o in ("-h","--help"): 
149              usage() 
150              raise SystemExit 
151          if o in ("-o", "--outdisk"): 
152              outDisk = a 
153   
154      if len(args) == 1: 
155          inputFile = args[0] 
156      else: 
157          usage() 
158          raise SystemExit 
159   
160      utils.ensureDirExist(scriptDir) 
161      # get version numbers and directory paths from CASU 
162      for direc in sysc.casuFileSystem(): 
163          casuSysDict.update(readVersion(direc, archive, light)) 
164      # get directories here 
165      disks = sysc.availableRaidFileSystem() 
166      wfauSysDict = utils.invertDict(fits.getAllPaths(disks)) 
167      # read file names from log (CASU format: <date>/<filename>) 
168      fileDict = readFileList(inputFile, casuSysDict) 
169      # get jpgDirs 
170      jpgDict = getJpgDict(fileDict) 
171      # get multiframeIDs 
172      sqlFileList = [] 
173      for dateVersStr in fileDict: 
174          wfauDir = os.path.join(wfauSysDict[dateVersStr], dateVersStr) 
175          for fileName in fileDict[dateVersStr]: 
176              sqlFileList.append(os.path.join(''.join( 
177                  [sysc.pixelServerHostName(), wfauDir]), fileName)) 
178   
179      mfIDs = DbSession().query("multiframeID, fileName", "Multiframe", 
180                                "fileName IN ('%s')" % "','".join(sqlFileList)) 
181   
182      mfIDofFile = dict((fileName.partition(sysc.fitsDir()+os.sep)[2], mfID) 
183                        for mfID, fileName in mfIDs) 
184   
185      cuRuns = {} 
186      # create scripts 
187      moveScript = File(os.path.join(scriptDir, "01_moveScript.sh")) 
188      moveScript.wopen() 
189      moveScript.writetheline("#! /bin/tcsh") 
190      updateLog = File(os.path.join(scriptDir, "02_updateMoved.log")) 
191      updateLog.wopen() 
192      updateJpgLog = File(os.path.join(scriptDir, "03_updateJpgs.log")) 
193      updateJpgLog.wopen() 
194      updateScript = File(os.path.join(scriptDir, "04_updateScript.sh")) 
195      updateScript.wopen() 
196      updateScript.writetheline(' '.join([ 
197          os.path.join(sysc.helpersPath(), "UpdateFitsFileNames.py"), 
198          "-M dbpub::UKIDSSDR2PLUS", "-D", "-f", updateLog.name])) 
199      updateScript.writetheline(' '.join([ 
200          os.path.join(sysc.helpersPath(), "UpdateFitsFileNames.py"), 
201          "-M dbload::WSA", "-D", "-f", updateLog.name])) 
202      updateScript.writetheline(' '.join([ 
203          os.path.join(sysc.helpersPath(), "UpdateJpegs.py"), 
204          "-M dbpub::UKIDSSDR2PLUS", "-j", updateJpgLog.name])) 
205      updateScript.writetheline(' '.join([ 
206          os.path.join(sysc.helpersPath(), "UpdateJpegs.py"), 
207          "-M dbload::WSA", "-j", updateJpgLog.name])) 
208      updateScript.close() 
209      cuScript = File(os.path.join(scriptDir, "07_runCUs.sh")) 
210      cuScript.wopen() 
211      transferScript = File(os.path.join(scriptDir, "05_scpcam00.sh")) 
212      transferScript.wopen() 
213      transferScript.writetheline("#! /bin/tcsh") 
214      transferMarker = "cmd" 
215      transferScript.writetheline(''.join([sysc.ftp_method(), " ", 
216                                           sysc.scp_server(archive, light), 
217                                           " << ", transferMarker])) 
218      transferScript.writetheline("bin") 
219   
220      for dateVersStr in sorted(fileDict.keys()): 
221          dateStr, versNum = dateVersStr.partition("_v")[::2] 
222          # Directory names 
223          wfauDir = os.path.join(wfauSysDict[dateVersStr], dateVersStr) 
224          deprDir = os.path.join(outDisk, sysc.deprecatedDir(), dateVersStr) 
225          utils.ensureDirExist(deprDir) 
226          deprJpgDir = os.path.join(outDisk, sysc.deprecatedComprImDir(), 
227                                    dateVersStr) 
228          utils.ensureDirExist(deprJpgDir) 
229          # log file containing list of transferred files (by version number) 
230          cuLog = File(os.path.join(scriptDir, ''.join(["06_newFiles_v", 
231                                                        versNum, ".log"]))) 
232          cuLog.aopen() 
233          for fileName in fileDict[dateVersStr]: 
234              casuPath = os.path.join(casuSysDict[dateStr][0][0], fileName) 
235              wfauPath = os.path.join(wfauDir, fileName) 
236              deprWfauPath = os.path.join(deprDir, fileName) 
237              if os.path.exists(wfauPath): 
238                  moveScript.writetheline(' '.join(["mv", wfauPath, 
239                                                    deprWfauPath])) 
240                  updateLog.writetheline(','.join([ 
241                      ''.join([sysc.pixelServerHostName(), wfauPath]), 
242                      ''.join([sysc.pixelServerHostName(), deprWfauPath])])) 
243              transferScript.writetheline(' '.join(["get", casuPath, wfauPath])) 
244              cuLog.writetheline(wfauPath) 
245          cuLog.close() 
246          # move the deprecated jpgs 
247          for fileName in jpgDict[dateVersStr]: 
248              deprJpegPath = os.path.join(deprJpgDir, os.path.basename(fileName)) 
249              moveScript.writetheline( 
250                  ' '.join(["mv", fileName, deprJpegPath])) 
251              fitsBase, extNum = \ 
252                      fileName.partition(sysc.compressImDir() \ 
253                                         + os.sep)[2].rpartition('_')[::2] 
254              updateJpgLog.writetheline(','.join([ 
255                  str(mfIDofFile[''.join([fitsBase, sysc.mefType()])]), 
256                  extNum.partition('.')[0], deprJpegPath])) 
257   
258          # create new cu commands 
259          cuRuns.setdefault("cu1", set()).add(' '.join(["./DataBuilder.py", 
260                                                        "-l", cuLog.name, 
261                                                        "-v", versNum, 
262                                                        "-m -r -x 1"])) 
263          cuRuns.setdefault("cu2", set()).add(' '.join(["./DataBuilder.py", 
264                                                        "-b", dateStr, 
265                                                        "-e", dateStr, 
266                                                        "-x 2"])) 
267          cuRuns.setdefault("cu3", set()).add(' '.join(["./DataBuilder.py", 
268                                                           "-l", cuLog.name, 
269                                                           "-v", versNum, 
270                                                           "-n -x 3"])) 
271          cuRuns.setdefault("cu4", set()).add(' '.join(["./DataBuilder.py", 
272                                                           "-l", cuLog.name, 
273                                                           "-v", versNum, 
274                                                           "-x 4"])) 
275      for cu in sorted(cuRuns.keys()): 
276          cuScript.writelines(cuRuns[cu]) 
277      moveScript.close() 
278      updateLog.close() 
279      updateJpgLog.close() 
280      cuScript.close() 
281      transferScript.writetheline("quit") 
282      transferScript.writetheline(transferMarker) 
283      transferScript.close() 
284      os.system("chmod 755 " + moveScript.name) 
285      os.system("chmod 755 " + updateScript.name) 
286      os.system("chmod 755 " + transferScript.name) 
287      os.system("chmod 755 " + cuScript.name) 
288      print ''.join(["\nRun the scripts in ", scriptDir, 
289                     " in the following order (from ", 
290                     sysc.helpersPath(), "):\n", 
291                     ">./", moveScript.name, '\n', 
292                     ">./", updateScript.name, '\n', 
293                     ">./", transferScript.name, '\n', 
294                     "and copy ", cuScript.name, " to ", 
295                     os.path.join(sysc.curationCodePath(), "invocations/cu0"), 
296                     " and run it there:\n", ">./", cuScript.name]) 
297   
298  #------------------------------------------------------------------------------ 
299   
300  if __name__ == "__main__": 
301      main(sys.argv) 
302   
303  #------------------------------------------------------------------------------ 
304  # Change log: 
305  # 
306  # 11-Sep-2007, ETWS: Original version. 
307  # 14-Sep-2007, ETWS: Included deprecation flag in updateScript calls. 
308  # 20-Sep-2007, ETWS: Moved the multiframeID writing from CU3 to CU1 command. 
309