1
2
3
4 """
5 Make file lists of one or all fits directories.
6
7 @author: E. Sutorius
8 @org: WFAU, IfA, University of Edinburgh
9 """
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32 import dircache
33 import getopt
34 import os
35 import re
36 import sys
37
38 import wsatools.SystemConstants as sysc
39 from wsatools.File import File
40 from wsatools.FitsUtils import FitsList
41
42
44 print "usage: makelist.py [-c/--crit listcrit] [-g/--group numdirs] [-n/--num numfiles] [-o/--outdir] [-p/--pre prefix] [-k/--disk disklist] [-d/--dir directory] startdate enddate version\n"
45 print "listcrit = a[ll]/s[plit]/c[at]"
46 print "numdirs = number of directories per output logfile"
47 print "numfiles = number of files per sublist"
48 print "prefix = prefix of the outputfile(s)"
49 print "disklist = list of disks, eg. '/disk01/wsa,/disk02/wsa'; default is massStorageRaidFileSystem()"
50 print "directory = subdirectory, eg. 'products/stacks default is fin_dir()"
51 print "startdate = start date of directories to check"
52 print "enddate = end date of directories to check"
53 print "version = version number of the data dir"
54
55
56
58 """Make list(s) of one or all fits directories.
59
60 startdate is the start date.
61 enddate is the end date.
62 filecrit is the listing criterium: 'all': all files in one list, 'split':
63 lists split into science frames (starting with casuPrefix, ending with
64 '.fit') and others (calibration and catalogue files).
65 """
66
67 try:
68 opts, args = getopt.getopt(sys.argv[1:], "c:d:g:hk:n:o:p:",
69 ["crit=", "dir=", "group=", "help",
70 "disk=", "num=", "outdir=", "pre="])
71 except getopt.GetoptError:
72
73 print opts,args
74 usage()
75 sys.exit(2)
76
77 listcrit = ''
78 maxnum = 0
79 prefix = ""
80 group = False
81 disklist = sysc.massStorageRaidFileSystem()
82 directory = sysc.fitsDir()
83 outDir = "./"
84
85 for o, a in opts:
86 if o in ("-c","--crit"):
87 listcrit = a
88
89 elif o in ("-h","--help"):
90 usage()
91 sys.exit(0)
92
93 elif o in ("-n","--num"):
94 maxnum = int(a)
95
96 elif o in ("-p","--pre"):
97 prefix = a
98
99 elif o in ("-g","--group"):
100 group = True
101 pergroup = int(a)
102
103 elif o in ("-d","--dir"):
104 directory = a
105
106 elif o in ("-o","--outdir"):
107 outDir = a
108
109 elif o in ("-k","--disk"):
110 disklist = a.split(',')
111
112 if len(args) == 0:
113 usage()
114 sys.exit(2)
115
116 datumcritlow = int(args[0])
117 datumcrithigh = int(args[1])
118 versionnr = int(args[2])
119
120 if listcrit.find('a') == 0:
121 listcrit = "all"
122 elif listcrit.find('s') == 0:
123 listcrit = "split"
124 elif listcrit.find('c') == 0:
125 listcrit = "cat"
126 else:
127 listcrit = "undef"
128
129 if group and listcrit != "all":
130 group = False
131
132 filecrit1 = re.compile(r'\b['+sysc.casuPrefix()+sysc.wfauPrefix()+']')
133 filecrit2 = re.compile(sysc.mefType()+r'\b')
134 filecrit3 = re.compile(sysc.catSuffix()+sysc.catType()+r'\b')
135
136
137 fitsDirs = FitsList(prefix="cu2_")
138 fitsDirs.createFitsDateDict(disklist, directory)
139 fitsdatedict = fitsDirs.fitsDateDict
140
141 for direc in disklist:
142 fdir = os.path.join(direc,directory)
143 if os.path.exists(fdir):
144 listdir = [elem for elem in dircache.listdir(fdir)
145 if elem[:2] == "20" and elem.find('_') > 0]
146 fitsdatedict[fdir]=listdir
147 if len(fitsdatedict[fdir]) == 0:
148 del fitsdatedict[fdir]
149
150
151 if listcrit == "all":
152 for direc in fitsdatedict.iterkeys():
153 for datumVersStr in fitsdatedict[direc]:
154 date = int(datumVersStr[:datumVersStr.find('_')])
155 vers = fitsDirs.getVersion(datumVersStr)
156 if (datumcritlow <= date <= datumcrithigh) \
157 and versionnr == vers:
158 counter = 1
159 if maxnum > 0:
160 filenumstr = '_1'
161 else:
162 filenumstr = ''
163 listFileName = os.path.join(
164 outDir, ''.join([prefix, "filelist", datumVersStr,
165 filenumstr, ".log"]))
166 listFile = File(listFileName)
167 listFile.wopen()
168 path = os.path.join(direc, datumVersStr)
169 for item in dircache.listdir(path):
170 listFile.writetheline(os.path.join(path, item))
171 if maxnum > 0:
172 if counter < maxnum:
173 counter += 1
174 else:
175 counter = 1
176 listFile.close()
177 filenumstr = '_'+str(int(filenumstr[1:])+1)
178 listFileName = os.path.join(
179 outDir, ''.join([prefix, "filelist",
180 datumVersStr, filenumstr,
181 ".log"]))
182 listFile = File(listFileName)
183 listFile.wopen()
184 listFile.close()
185
186 if group:
187 newlfl = []
188 for file in dircache.listdir(outDir):
189 if (os.path.splitext(file)[1] == '.log') \
190 and (file.find('_v')>0):
191 newlfl.append(file)
192
193 listfilelist = newlfl
194 pref = os.path.commonprefix(listfilelist)
195 listfilelist.sort()
196 while len(listfilelist)>0:
197 if len(listfilelist) < pergroup:
198 pergroup = len(listfilelist)
199 listb = listfilelist[0]
200 liste = listfilelist[pergroup-1]
201 lfoutname = pref+listb[len(pref):listb.rfind('_')] \
202 + "to" + liste[len(pref):liste.rfind('_')] \
203 +".log"
204 if os.path.exists(lfoutname):
205 os.remove(lfoutname)
206 lfout = open(lfoutname,'w')
207 for _i in range(pergroup):
208 lfinname = listfilelist.pop(0)
209 print "in:",lfinname
210 lfin = open(lfinname,'r')
211 filelist = lfin.readlines()
212 lfin.close()
213 lfout.writelines(filelist)
214 print "written to:",lfoutname
215 lfout.close()
216
217
218 elif listcrit == "split":
219 for direc in fitsdatedict.iterkeys():
220 for datumVersStr in fitsdatedict[direc]:
221 date = int(datumVersStr[:datumVersStr.find('_')])
222 vers = fitsDirs.getVersion(datumVersStr)
223 if (datumcritlow <= date <= datumcrithigh) \
224 and versionnr == vers:
225 listFileName = os.path.join(
226 outDir, ''.join([prefix, "w_list", datumVersStr, ".log"]))
227 listFile = File(listFileName)
228 listFile.wopen()
229 otherFileName = os.path.join(
230 outDir, ''.join([prefix, "otherlist", datumVersStr, ".log"]))
231 otherFile = File(otherFileName)
232 otherFile.wopen()
233 path = os.path.join(direc,datumVersStr)
234 for item in dircache.listdir(path):
235 if (filecrit1.search(item) and filecrit2.search(item)):
236 listFile.writetheline(os.path.join(path, item))
237 else:
238 otherFile.writetheline(os.path.join(path, item))
239 listFile.close()
240 otherFile.close()
241
242
243 elif listcrit == "cat":
244 for direc in fitsdatedict.iterkeys():
245 for datumVersStr in fitsdatedict[direc]:
246 date = int(datumVersStr[:datumVersStr.find('_')])
247 vers = fitsDirs.getVersion(datumVersStr)
248 if (datumcritlow <= date <= datumcrithigh) \
249 and versionnr == vers:
250 countercat = 1
251 counterpix = 1
252 if maxnum > 0:
253 filenumcatstr = '_1'
254 filenumpixstr = '_1'
255 else:
256 filenumcatstr = ''
257 filenumpixstr = ''
258
259 catFileName = os.path.join(
260 outDir, ''.join([prefix, "catlist", datumVersStr,
261 filenumcatstr, ".log"]))
262 catFile = File(catFileName)
263 catFile.wopen()
264 pixFileName = os.path.join(
265 outDir, ''.join([prefix, "pixlist", datumVersStr,
266 filenumpixstr, ".log"]))
267 pixFile = File(pixFileName)
268 pixFile.wopen()
269 otherFileName = os.path.join(
270 outDir, ''.join([prefix, "otherlist", datumVersStr, ".log"]))
271 otherFile = File(otherFileName)
272 otherFile.wopen()
273
274 path = os.path.join(direc, datumVersStr)
275 for item in dircache.listdir(path):
276 if (filecrit1.search(item) and filecrit3.search(item)):
277 catFile.writetheline(os.path.join(path, item))
278 if maxnum > 0:
279 if countercat<maxnum:
280 countercat += 1
281 else:
282 countercat = 1
283 catFile.close()
284 filenumcatstr = '_' + str(int(filenumcatstr[1:])+1)
285
286 catFileName = os.path.join(
287 outDir, ''.join([prefix, "catlist",
288 datumVersStr, filenumcatstr,
289 ".log"]))
290 catFile = File(catFileName)
291 catFile.wopen()
292 elif filecrit2.search(item):
293 pixFile.writetheline(os.path.join(path, item))
294 if maxnum > 0:
295 if counterpix<maxnum:
296 counterpix += 1
297 else:
298 counterpix = 1
299 pixFile.close()
300 filenumpixstr = '_'+str(int(filenumpixstr[1:])+1)
301 pixFileName = os.path.join(
302 outDir, ''.join([prefix, "pixlist",
303 datumVersStr, filenumpixstr,
304 ".log"]))
305 pixFile = File(pixFileName)
306 pixFile.wopen()
307 else:
308 otherFile.writetheline(os.path.join(path, item))
309
310 catFile.close()
311 pixFile.close()
312
313
314 elif listcrit == "undef":
315 for direc in fitsdatedict.iterkeys():
316 for datumVersStr in fitsdatedict[direc]:
317 date = int(datumVersStr[:datumVersStr.find('_')])
318 vers = fitsDirs.getVersion(datumVersStr)
319 if (datumcritlow <= date <= datumcrithigh) and versionnr == vers:
320
321
322 wListFileName = os.path.join(outDir, ''.join([prefix,
323 "w_list",
324 datumVersStr,
325 ".log"]))
326 wListFile = File(wListFileName)
327 wListFile.wopen()
328 otherFileName = os.path.join(outDir, ''.join([prefix,
329 "otherlist",
330 datumVersStr,
331 ".log"]))
332 otherFile = File(otherFileName)
333 otherFile.wopen()
334 allFileName = os.path.join(outDir, ''.join([prefix,
335 "filelist",
336 datumVersStr,
337 ".log"]))
338 allFile = File(allFileName)
339 allFile.wopen()
340 catFileName = os.path.join(outDir, ''.join([prefix,
341 "catlist",
342 datumVersStr,
343 ".log"]))
344 catFile = File(catFileName)
345 catFile.wopen()
346 pixFileName = os.path.join(outDir, ''.join([prefix,
347 "pixlist",
348 datumVersStr,
349 ".log"]))
350 pixFile = File(pixFileName)
351 pixFile.wopen()
352
353 path = os.path.join(direc, datumVersStr)
354 for item in dircache.listdir(path):
355 allFile.writetheline(
356 os.path.join(path, item))
357 if (filecrit1.search(item) and filecrit2.search(item)):
358 wListFile.writetheline(
359 os.path.join(path, item))
360 else:
361 otherFile.writetheline(
362 os.path.join(path, item))
363 if (filecrit1.search(item) and filecrit3.search(item)):
364 catFile.writetheline(
365 os.path.join(path, item))
366 elif filecrit2.search(item):
367 pixFile.writetheline(
368 os.path.join(path, item))
369 else:
370 otherFile.writetheline(
371 os.path.join(path, item))
372 wListFile.close()
373 otherFile.close()
374 allFile.close()
375 catFile.close()
376 pixFile.close()
377
378
379
380 if __name__ == '__main__':
381 main()
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401