Source Code for Module helpers.makelist

  1  #! /usr/bin/env python 
  2  #------------------------------------------------------------------------------ 
  3  #$Id: makelist.py 6323 2009-11-09 22:39:17Z RossCollins $ 
  4  """ 
  5     Make file lists of one or all fits directories. 
  6   
  7     @author: E. Sutorius 
  8     @org:    WFAU, IfA, University of Edinburgh 
  9  """ 
 10  #------------------------------------------------------------------------------ 
 11  #  usage: makelist.py [-c/--crit listcrit] [-g/--group numdirs] 
 12  #                     [-n/--num numfiles] 
 13  #                     [-p/--pre prefix] startdate enddate version [disklist] 
 14  #  -c Options are: 
 15  #  'a[ll]'   all files in one list. 
 16  #  's[plit]' two lists, one containing science frames ('w*.fit'), the other 
 17  #            containing all other frames ('[^w]*.fit', 'w*_cat.fits'). 
 18  #  'c[at]'   two lists, one containing catalogue files ('w*_cat.fits'), the 
 19  #            other containing pixel files ('*.fit'). 
 20  #  None      'all' and 'split' lists. 
 21  # 
 22  #  If -n is used, the maximum number of files per list is restricted 
 23  #  to 'numfiles' (only for listcrit = 'all' or 'cat'). 
 24  # 
 25  #  -p gives every output file the given prefix. 
 26  # 
 27  #  -g combines multiple (up to numdirs) directories into one log file. 
 28  # 
 29  #  'disklist' is a comma seperated list of disks to be processed, only 
 30  #             date dirs on those disks will be listed. 
 31  #------------------------------------------------------------------------------ 
 32  import dircache 
 33  import getopt 
 34  import os 
 35  import re 
 36  import sys 
 37   
 38  import wsatools.SystemConstants as sysc 
 39  from   wsatools.File        import File 
 40  from   wsatools.FitsUtils   import FitsList 
 41  #------------------------------------------------------------------------------ 
 42   
 43 -def usage(): 
 44      print "usage: makelist.py [-c/--crit listcrit] [-g/--group numdirs] [-n/--num numfiles] [-o/--outdir] [-p/--pre prefix] [-k/--disk disklist] [-d/--dir directory] startdate enddate version\n" 
 45      print "listcrit = a[ll]/s[plit]/c[at]" 
 46      print "numdirs = number of directories per output logfile" 
 47      print "numfiles = number of files per sublist" 
 48      print "prefix = prefix of the outputfile(s)" 
 49      print "disklist = list of disks, eg. '/disk01/wsa,/disk02/wsa'; default is massStorageRaidFileSystem()" 
 50      print "directory = subdirectory, eg. 'products/stacks default is fin_dir()" 
 51      print "startdate = start date of directories to check" 
 52      print "enddate   = end date of directories to check" 
 53      print "version   = version number of the data dir" 
 54   
 55  #------------------------------------------------------------------------------ 
 56   
 57 -def main(): 
 58      """Make list(s) of one or all fits directories. 
 59   
 60      startdate is the start date. 
 61      enddate   is the end date. 
 62      filecrit  is the listing criterium: 'all': all files in one list, 'split': 
 63      lists split into science frames (starting with casuPrefix, ending with 
 64      '.fit') and others (calibration and catalogue files). 
 65      """ 
 66   
 67      try: 
 68          opts, args = getopt.getopt(sys.argv[1:], "c:d:g:hk:n:o:p:", 
 69                                     ["crit=", "dir=", "group=", "help", 
 70                                      "disk=", "num=", "outdir=", "pre="]) 
 71      except getopt.GetoptError: 
 72          # print help information and exit: 
 73          print opts,args 
 74          usage() 
 75          sys.exit(2) 
 76   
 77      listcrit = '' 
 78      maxnum = 0 
 79      prefix = "" 
 80      group = False 
 81      disklist = sysc.massStorageRaidFileSystem() 
 82      directory = sysc.fitsDir() 
 83      outDir = "./" 
 84   
 85      for o, a in opts: 
 86          if o in ("-c","--crit"): 
 87              listcrit = a 
 88   
 89          elif o in ("-h","--help"): 
 90              usage() 
 91              sys.exit(0) 
 92   
 93          elif o in ("-n","--num"): 
 94              maxnum = int(a) 
 95   
 96          elif o in ("-p","--pre"): 
 97              prefix = a 
 98   
 99          elif o in ("-g","--group"): 
100              group = True 
101              pergroup = int(a) 
102   
103          elif o in ("-d","--dir"): 
104              directory = a 
105   
106          elif o in ("-o","--outdir"): 
107              outDir = a 
108   
109          elif o in ("-k","--disk"): 
110              disklist = a.split(',') 
111   
112      if len(args) == 0: 
113          usage() 
114          sys.exit(2) 
115   
116      datumcritlow = int(args[0]) 
117      datumcrithigh = int(args[1]) 
118      versionnr = int(args[2]) 
119   
120      if listcrit.find('a') == 0: 
121          listcrit = "all" 
122      elif listcrit.find('s') == 0: 
123          listcrit = "split" 
124      elif listcrit.find('c') == 0: 
125          listcrit = "cat" 
126      else: 
127          listcrit = "undef" 
128   
129      if group and listcrit != "all": 
130          group = False 
131   
132      filecrit1 = re.compile(r'\b['+sysc.casuPrefix()+sysc.wfauPrefix()+']') 
133      filecrit2 = re.compile(sysc.mefType()+r'\b') 
134      filecrit3 = re.compile(sysc.catSuffix()+sysc.catType()+r'\b') 
135   
136      # create a dictionary listing all dates for every fits dir in disklist. 
137      fitsDirs = FitsList(prefix="cu2_") 
138      fitsDirs.createFitsDateDict(disklist, directory) 
139      fitsdatedict = fitsDirs.fitsDateDict 
140   
141      for direc in disklist: 
142          fdir = os.path.join(direc,directory) 
143          if os.path.exists(fdir): 
144              listdir = [elem for elem in dircache.listdir(fdir) 
145                         if elem[:2] == "20" and elem.find('_') > 0] 
146              fitsdatedict[fdir]=listdir 
147              if len(fitsdatedict[fdir]) == 0: 
148                  del fitsdatedict[fdir] 
149   
150      # list all files in one dir into one filelist 
151      if listcrit == "all": 
152          for direc in fitsdatedict.iterkeys(): 
153              for datumVersStr in fitsdatedict[direc]: 
154                  date = int(datumVersStr[:datumVersStr.find('_')]) 
155                  vers = fitsDirs.getVersion(datumVersStr) 
156                  if (datumcritlow <= date <= datumcrithigh) \ 
157                         and versionnr == vers: 
158                      counter = 1 
159                      if maxnum > 0: 
160                          filenumstr = '_1' 
161                      else: 
162                          filenumstr = '' 
163                      listFileName = os.path.join( 
164                          outDir, ''.join([prefix, "filelist", datumVersStr, 
165                                           filenumstr, ".log"])) 
166                      listFile = File(listFileName) 
167                      listFile.wopen() 
168                      path = os.path.join(direc, datumVersStr) 
169                      for item in dircache.listdir(path): 
170                          listFile.writetheline(os.path.join(path, item)) 
171                          if maxnum > 0: 
172                              if counter < maxnum: 
173                                  counter += 1 
174                              else: 
175                                  counter = 1 
176                                  listFile.close() 
177                                  filenumstr = '_'+str(int(filenumstr[1:])+1) 
178                                  listFileName = os.path.join( 
179                                      outDir, ''.join([prefix, "filelist", 
180                                                       datumVersStr, filenumstr, 
181                                                       ".log"])) 
182                                  listFile = File(listFileName) 
183                                  listFile.wopen() 
184                      listFile.close() 
185   
186          if group: 
187              newlfl = [] 
188              for file in dircache.listdir(outDir): 
189                  if (os.path.splitext(file)[1] == '.log') \ 
190                     and (file.find('_v')>0): 
191                      newlfl.append(file) 
192   
193              listfilelist = newlfl 
194              pref = os.path.commonprefix(listfilelist) 
195              listfilelist.sort() 
196              while len(listfilelist)>0: 
197                  if len(listfilelist) < pergroup: 
198                      pergroup = len(listfilelist) 
199                  listb = listfilelist[0] 
200                  liste = listfilelist[pergroup-1] 
201                  lfoutname = pref+listb[len(pref):listb.rfind('_')] \ 
202                              + "to" + liste[len(pref):liste.rfind('_')] \ 
203                              +".log" 
204                  if os.path.exists(lfoutname): 
205                      os.remove(lfoutname) 
206                  lfout = open(lfoutname,'w') 
207                  for _i in range(pergroup): 
208                      lfinname = listfilelist.pop(0) 
209                      print "in:",lfinname 
210                      lfin = open(lfinname,'r') 
211                      filelist = lfin.readlines() 
212                      lfin.close() 
213                      lfout.writelines(filelist) 
214                  print "written to:",lfoutname 
215                  lfout.close() 
216   
217      # list all files in one dir into a 'science frame' and an 'other' list 
218      elif listcrit == "split": 
219          for direc in fitsdatedict.iterkeys(): 
220              for datumVersStr in fitsdatedict[direc]: 
221                  date = int(datumVersStr[:datumVersStr.find('_')]) 
222                  vers = fitsDirs.getVersion(datumVersStr) 
223                  if (datumcritlow <= date <= datumcrithigh) \ 
224                         and versionnr == vers: 
225                      listFileName = os.path.join( 
226                          outDir, ''.join([prefix, "w_list", datumVersStr, ".log"])) 
227                      listFile = File(listFileName) 
228                      listFile.wopen() 
229                      otherFileName = os.path.join( 
230                          outDir, ''.join([prefix, "otherlist", datumVersStr, ".log"])) 
231                      otherFile = File(otherFileName) 
232                      otherFile.wopen() 
233                      path = os.path.join(direc,datumVersStr) 
234                      for item in dircache.listdir(path): 
235                          if (filecrit1.search(item) and filecrit2.search(item)): 
236                              listFile.writetheline(os.path.join(path, item)) 
237                          else: 
238                              otherFile.writetheline(os.path.join(path, item)) 
239                      listFile.close() 
240                      otherFile.close() 
241   
242      # list all files in one dir into a 'cat' and 'pix' list 
243      elif listcrit == "cat": 
244          for direc in fitsdatedict.iterkeys(): 
245              for datumVersStr in fitsdatedict[direc]: 
246                  date = int(datumVersStr[:datumVersStr.find('_')]) 
247                  vers = fitsDirs.getVersion(datumVersStr) 
248                  if (datumcritlow <= date <= datumcrithigh) \ 
249                         and versionnr == vers: 
250                      countercat = 1 
251                      counterpix = 1 
252                      if maxnum > 0: 
253                          filenumcatstr = '_1' 
254                          filenumpixstr = '_1' 
255                      else: 
256                          filenumcatstr = '' 
257                          filenumpixstr = '' 
258   
259                      catFileName = os.path.join( 
260                          outDir, ''.join([prefix, "catlist", datumVersStr, 
261                                           filenumcatstr, ".log"])) 
262                      catFile = File(catFileName) 
263                      catFile.wopen() 
264                      pixFileName = os.path.join( 
265                          outDir, ''.join([prefix, "pixlist", datumVersStr, 
266                                           filenumpixstr, ".log"])) 
267                      pixFile = File(pixFileName) 
268                      pixFile.wopen() 
269                      otherFileName = os.path.join( 
270                          outDir, ''.join([prefix, "otherlist", datumVersStr, ".log"])) 
271                      otherFile = File(otherFileName) 
272                      otherFile.wopen() 
273   
274                      path = os.path.join(direc, datumVersStr) 
275                      for item in dircache.listdir(path): 
276                          if (filecrit1.search(item) and filecrit3.search(item)): 
277                              catFile.writetheline(os.path.join(path, item)) 
278                              if maxnum > 0: 
279                                  if countercat<maxnum: 
280                                      countercat += 1 
281                                  else: 
282                                      countercat = 1 
283                                      catFile.close() 
284                                      filenumcatstr = '_' + str(int(filenumcatstr[1:])+1) 
285   
286                                      catFileName = os.path.join( 
287                                          outDir, ''.join([prefix, "catlist", 
288                                                           datumVersStr, filenumcatstr, 
289                                                           ".log"])) 
290                                      catFile = File(catFileName) 
291                                      catFile.wopen() 
292                          elif filecrit2.search(item): 
293                              pixFile.writetheline(os.path.join(path, item)) 
294                              if maxnum > 0: 
295                                  if counterpix<maxnum: 
296                                      counterpix += 1 
297                                  else: 
298                                      counterpix = 1 
299                                      pixFile.close() 
300                                      filenumpixstr = '_'+str(int(filenumpixstr[1:])+1) 
301                                      pixFileName = os.path.join( 
302                                          outDir, ''.join([prefix, "pixlist", 
303                                                           datumVersStr, filenumpixstr, 
304                                                           ".log"])) 
305                                      pixFile = File(pixFileName) 
306                                      pixFile.wopen() 
307                          else: 
308                              otherFile.writetheline(os.path.join(path, item)) 
309   
310                      catFile.close() 
311                      pixFile.close() 
312   
313      # since you don't know what you want you get everything 
314      elif listcrit == "undef": 
315          for direc in fitsdatedict.iterkeys(): 
316              for datumVersStr in fitsdatedict[direc]: 
317                  date = int(datumVersStr[:datumVersStr.find('_')]) 
318                  vers = fitsDirs.getVersion(datumVersStr) 
319                  if (datumcritlow <= date <= datumcrithigh) and versionnr == vers: 
320   
321   
322                      wListFileName = os.path.join(outDir, ''.join([prefix, 
323                                                                    "w_list", 
324                                                                    datumVersStr, 
325                                                                    ".log"])) 
326                      wListFile = File(wListFileName) 
327                      wListFile.wopen() 
328                      otherFileName = os.path.join(outDir, ''.join([prefix, 
329                                                                    "otherlist", 
330                                                                    datumVersStr, 
331                                                                    ".log"])) 
332                      otherFile = File(otherFileName) 
333                      otherFile.wopen() 
334                      allFileName = os.path.join(outDir, ''.join([prefix, 
335                                                                  "filelist", 
336                                                                  datumVersStr, 
337                                                                  ".log"])) 
338                      allFile = File(allFileName) 
339                      allFile.wopen() 
340                      catFileName = os.path.join(outDir, ''.join([prefix, 
341                                                                  "catlist", 
342                                                                  datumVersStr, 
343                                                                  ".log"])) 
344                      catFile = File(catFileName) 
345                      catFile.wopen() 
346                      pixFileName = os.path.join(outDir, ''.join([prefix, 
347                                                                  "pixlist", 
348                                                                  datumVersStr, 
349                                                                  ".log"])) 
350                      pixFile = File(pixFileName) 
351                      pixFile.wopen() 
352   
353                      path = os.path.join(direc, datumVersStr) 
354                      for item in dircache.listdir(path): 
355                          allFile.writetheline( 
356                              os.path.join(path, item)) 
357                          if (filecrit1.search(item) and filecrit2.search(item)): 
358                              wListFile.writetheline( 
359                                  os.path.join(path, item)) 
360                          else: 
361                              otherFile.writetheline( 
362                                  os.path.join(path, item)) 
363                          if (filecrit1.search(item) and filecrit3.search(item)): 
364                              catFile.writetheline( 
365                                  os.path.join(path, item)) 
366                          elif filecrit2.search(item): 
367                              pixFile.writetheline( 
368                                  os.path.join(path, item)) 
369                          else: 
370                              otherFile.writetheline( 
371                                  os.path.join(path, item)) 
372                      wListFile.close() 
373                      otherFile.close() 
374                      allFile.close() 
375                      catFile.close() 
376                      pixFile.close() 
377   
378  #------------------------------------------------------------------------------ 
379  # 
380  if __name__ == '__main__': 
381      main() 
382   
383  #------------------------------------------------------------------------------ 
384  # Change log: 
385  # 
386  # 13-Jul-2005, ETWS: Original version. 
387  #  5-Aug-2005, ETWS: Added in more functionality. 
388  #  3-Oct-2005, ETWS: Included version number and output file prefix. 
389  # 10-Oct-2005, ETWS: Added method to combine multiple days into one log; 
390  #                    added 'help' functionality. 
391  #  3-Nov-2005, ETWS: Added optional 'disklist' parameter to limit list 
392  #                    on a set of disks. 
393  # 17-Nov-2005, ETWS: Removed debug output. 
394  # 30-Jan-2006, ETWS: Included optional 'directory' parameter to list 
395  #                    other dirs than 'ingest/fits' 
396  # 31-Jan-2006, ETWS: Included wfauPrefix in RegExp 
397  # 28-Apr-2006, ETWS: Included check for test directories 
398  # 01-May-2007, ETWS: Updated for include into cu0 
399  # 10-May-2007, ETWS: Fixed directory parsing for non-fits files. 
400  # 13-Jun-2007, ETWS: Included FitsList class 
401