Source Code for Module invocations.cu2.makejpegs

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: makejpegs.py 6157 2009-10-21 08:30:21Z RossCollins $ 
  4  """ 
  5     Wrapper to create jpegs automatically. Uses helpers/makelist.py, 
  6     cu2/checkcu2.py, cu2/cu2jpeg.py 
  7   
  8     @author: Eckhard Sutorius 
  9     @org:    WFAU, IfA, University of Edinburgh 
 10  """ 
 11  #------------------------------------------------------------------------------ 
 12  import dircache as dc 
 13  import getopt 
 14  import os 
 15  import sys 
 16   
 17  import wsatools.CSV             as csv 
 18  from   wsatools.File            import File 
 19  import wsatools.SystemConstants as sysc 
 20  #------------------------------------------------------------------------------ 
 21   
 22 -def usage(): 
 23      print "usage: makejpegs.py [-b/--begin startdate] [-e/--end enddate] [-k/--disk disklist] [-d/--dir directory] [-s/--subdir directory] [-M/--mosaic] [-r/--redo] version\n" 
 24      print "startdate = start date of directories to check" 
 25      print "enddate   = end date of directories to check" 
 26      print "disklist  = list of disks, eg. '/disk01/wsa,/disk02/wsa'; default is massStorageRaidFileSystem()" 
 27      print "directory = fits directory, eg. 'products/stacks default is fin_dir()" 
 28      print "version   = version number of the data dir" 
 29      print "subdir    = jpg sub-directory where products/jpgs is created; default is 'None'" 
 30      print "mosaic    = forces making mosaics, may break if size is bigger than 2 GB." 
 31      print "redo      = re-do the jpegs." 
 32   
 33  #------------------------------------------------------------------------------ 
 34   
 35 -def cleanUp(cu2Path, datumVersStr): 
 36      cmd = ' '.join(["rm -f", os.path.join(cu2Path, "*%s*.log" % datumVersStr), 
 37                      os.path.join(cu2Path, "*%s.tbl" % datumVersStr), 
 38                      os.path.join(cu2Path, "autoruncu2_%s" % datumVersStr)]) 
 39      os.system(cmd) 
 40   
 41  #------------------------------------------------------------------------------ 
 42   
 43 -def main(argv): 
 44      """Wrapper to create jpegs automatically. 
 45   
 46      startdate is the start date of directories to check. 
 47      enddate   is the end date of directories to check. 
 48      version   is the version number of the data dir. 
 49   
 50      """ 
 51      # read the arguments 
 52      try: 
 53          opts, args = getopt.getopt(argv[1:], "b:e:d:k:o:Mr", 
 54                                     ["begin=", "end=", "dir=", 
 55                                      "disk=", "outpath=", "mosaic", "redo"]) 
 56      except getopt.GetoptError: 
 57          # print help information and exit: 
 58          print argv 
 59          usage() 
 60          raise SystemExit 
 61   
 62      startdate = '00000000' 
 63      enddate = '99999999' 
 64      directory = sysc.fitsDir() 
 65      disklist = sysc.availableRaidFileSystem() 
 66      jpgPathOptFlag = '' 
 67      jpgPathOpt = '' 
 68      forceMosaic = False 
 69      reDo = False 
 70   
 71      for o, a in opts: 
 72          if o in ("-b", "--begin"): 
 73              startdate = a 
 74          elif o in ("-e", "--end"): 
 75              enddate = a 
 76          elif o in ("-d","--dir"): 
 77              directory = a 
 78          elif o in ("-k","--disk"): 
 79              disklist = a.split(',') 
 80          elif o in ("-o","--outpath"): 
 81              jpgPathOptFlag = "-o" 
 82              jpgPathOpt = a 
 83          elif  o in ("-M","--mosaic"): 
 84              forceMosaic = True 
 85          elif  o in ("-r","--redo"): 
 86              reDo = True 
 87   
 88      if len(args) == 0: 
 89          usage() 
 90          raise SystemExit 
 91   
 92      version = args[0] 
 93      datumVersStr = '_v'.join([startdate, version]) 
 94   
 95      cu2Path = os.path.join(sysc.curationCodePath(), "invocations/cu2") 
 96   
 97      cleanUp(cu2Path, datumVersStr) 
 98      # create a fits date dictionary 
 99      fitsdatedict = {} 
100      for direc in disklist: 
101          fdir = os.path.join(direc,directory) 
102          if os.path.exists(fdir): 
103              listdir = [elem for elem in dc.listdir(fdir) 
104                         if elem[:2] == "20" and elem.find('_') > 0] 
105              fitsdatedict[fdir]=listdir 
106              if len(fitsdatedict[fdir]) == 0: 
107                  del fitsdatedict[fdir] 
108   
109      # find a begin and end date 
110      maxdate = '00000000' 
111      mindate = '99999999' 
112   
113      for direc in fitsdatedict.iterkeys(): 
114          datelist = fitsdatedict[direc] 
115          datelist.sort() 
116          if datelist[0][:datelist[0].find('_')] < mindate: 
117              mindate = datelist[0][:datelist[0].find('_')] 
118          if datelist[-1][:datelist[0].find('_')] > maxdate: 
119              maxdate = datelist[-1][:datelist[0].find('_')] 
120   
121      if startdate > mindate: 
122          mindate = startdate 
123      if enddate < maxdate: 
124          maxdate = enddate 
125   
126      # get a list only of disks used 
127      fitsdatedisklist = [x[:len(disklist[0])] for x in fitsdatedict.keys()] 
128      tmplist = [] 
129      for elem in disklist: 
130          if fitsdatedisklist.count(elem) > 0: 
131              tmplist.append(elem) 
132      disklist = tmplist 
133   
134      # check for fits files and their jpegs 
135      # writes non-processed dirs to 'notprocdirs.log' and reprocessable 
136      # files to 'reprocfiles.log' 
137      cmd = ' '.join([os.path.join(cu2Path, "checkcu2.py"), 
138                      "-k", csv.join(disklist), 
139                      "-d", directory]) 
140      if forceMosaic: 
141          cmd = ' '.join([cmd, "-M"]) 
142      if reDo: 
143          cmd = ' '.join([cmd, "-r"]) 
144      cmd = ' '.join([cmd, jpgPathOptFlag, jpgPathOpt, 
145                      mindate, maxdate, version]) 
146      os.system(cmd) 
147   
148      # make lists of pixel data files for each non-processed dir 
149      notProcFile = File(os.path.join( 
150          cu2Path, "notprocdirs_%s.log" % datumVersStr)) 
151      if notProcFile.exists() and notProcFile.getsize() > 0: 
152          notProcFile.ropen() 
153          npdirlines = notProcFile.readlines() 
154          notProcFile.close() 
155          for line in npdirlines: 
156              date = line[line.rfind('/')+1:line.rfind('_')] 
157              diskstr = line[:len(disklist[0])] 
158              cmd = ' '.join([os.path.join(sysc.helpersPath(), "makelist.py"), 
159                              "-c c -n 1000", 
160                              "-o", cu2Path, 
161                              "-k", diskstr, 
162                              "-d", directory, 
163                              date, date, version]) 
164              os.system(cmd) 
165   
166      # create a list of log files for the actual making of jpegs 
167      cu2dirList = dc.listdir(cu2Path) 
168      pixlist = [] 
169      reProcFileName = os.path.join(cu2Path, "reprocfiles_%s.log" % datumVersStr) 
170      if os.path.exists(reProcFileName) \ 
171             and os.path.getsize(reProcFileName) > 0: 
172          pixlist.append(reProcFileName) 
173      for entry in cu2dirList: 
174          if "pixlist" in entry and datumVersStr in entry: 
175              pixlist.append(entry) 
176   
177      # create the autorun file 
178      autoRunFile = File(os.path.join(cu2Path, "autoruncu2_%s" % datumVersStr)) 
179      autoRunFile.wopen() 
180      cu2JpegFileName = os.path.join(cu2Path, "cu2jpeg.py") 
181      for entry in pixlist: 
182          cmd = ' '.join([cu2JpegFileName, jpgPathOptFlag, jpgPathOpt, 
183                          os.getenv('USER'), entry, "'Running CU2'"]) 
184          autoRunFile.writetheline(cmd) 
185      autoRunFile.close() 
186      autoRunFile.chmod(0744) 
187      os.system(autoRunFile.name) 
188      cleanUp(cu2Path, datumVersStr) 
189   
190  #------------------------------------------------------------------------------ 
191  # 
192  if __name__ == '__main__': 
193      main(sys.argv) 
194   
195  #------------------------------------------------------------------------------ 
196  # Change log: 
197  # 
198  #  5-Aug-2005, ETWS: Original version. 
199  #  3-Oct-2005, ETWS: Included versioning of data dirs. 
200  # 17-Nov-2005, ETWS: Included disklist parameter to makelist.py; 
201  #                    major refactoring; included disklist parameter 
202  #                    for checkcu2 
203  #  6-Jan-2006, ETWS: included input arguments (startdate, enddate, version) 
204  # 30-Jan-2006, ETWS: Included optional 'disk' and 'directory' parameters 
205  #                    to list other dirs than 'ingest/fits' 
206  # 15-Mar-2006, ETWS: Switched on the automatic processing again. 
207  #  5-Jul-2006,  RSC: Upgraded to use new CSV.py module. 
208  # 01-May-2007, ETWS: Updated for use with CU0 
209  # 10-Jul-2007, ETWS: Updated for use with test cases. 
210  # 29-Jan-2008, ETWS: Included possibility to force mosaic jpeg creation 
211  #                    and to re-process existing jpegs. 
212  #  3-Mar-2008, ETWS: Updated to use only the available RaidFileSystem. 
213  # 30-Jul-2008, ETWS: Enabled possibility to be run on multiple processors. 
214  #  1-Aug-2008,  RPB: Make sure it tidies up .tbl as well as .log files 
215