1
2
3
4 """
5 Determines the used disk space per fits file type.
6
7 @author: E.T.W. Sutorius
8 @org: WFAU, IfA, University of Edinburgh
9 """
10
11 from collections import defaultdict
12 import dircache
13 import os
14
15 from wsatools.CLI import CLI
16 from wsatools.Logger import Logger
17 from wsatools.SystemConstants import SystemConstants
18
19
21 """ Calculate filesizes.
22 """
29 self.dataArchive = dataArchive
30 self.sysc = SystemConstants(self.dataArchive)
31 self.diskList = diskList or self.sysc.availableRaidFileSystem()
32 self.subDir = subDir or self.sysc.fitsDir
33 self.semester = semester
34 self.useGByte = useGByte
35 self.typeDict = {'dark': 0, 'sky': 0, 'flat': 0, 'conf': 0,
36 '_sf_st_cat': 0, '_st_cat': 0, '_cat': 0,
37 '_sf_st': 0, '_sf_st_conf': 0,
38 '_sf': 0, '_sf_conf': 0,
39 '_st': 0, '_st_conf': 0,
40 'normal': 0, 'misc': 0, 'total': 0, 'fit': 0}
41
42
43
45 Logger.addMessage("Creating daily filelists for %s..." % self.semester)
46 self.fitsDateDict = defaultdict(list)
47 for direc in diskList:
48 fDir = os.path.join(direc, subDir)
49 if os.path.exists(fDir):
50 self.fitsDateDict[fDir].extend(
51 [elem for elem in dircache.listdir(fDir)
52 if elem[:2] == "20" and elem.find('_v') > 0])
53 self.fileSizeDict = defaultdict(dict)
54 startDate, endDate = \
55 self.sysc.obsCal.getDates(self.semester, "%Y%m%d")
56
57 for direc in self.fitsDateDict:
58 for datumVersStr in self.fitsDateDict[direc]:
59 dateStr, versNum = datumVersStr.partition('_v')[::2]
60 if versNum not in self.fileSizeDict:
61 self.fileSizeDict[versNum] = {}
62 if startDate <= dateStr <= endDate:
63 path = os.path.join(direc, datumVersStr)
64 for fileName in dircache.listdir(path):
65 if ('.fit' in fileName
66 and subDir == self.sysc.fitsDir) \
67 or ('.jpg' in fileName
68 and subDir == self.sysc.compressImDir):
69 self.fileSizeDict[versNum][fileName] = \
70 os.path.getsize(
71 os.path.join(direc, datumVersStr, fileName))
72
73
74
76 """Add up sizes for different types.
77 """
78 Logger.addMessage("Calculating sizes...")
79 typeSizeDict = defaultdict(dict)
80 catSizeDict = defaultdict(dict)
81 for versNum in sorted(self.fileSizeDict):
82 if versNum not in typeSizeDict:
83 typeSizeDict[versNum] = self.typeDict.copy()
84 catSizeDict[versNum] = self.typeDict.copy()
85 for fileName in self.fileSizeDict[versNum]:
86 typeSizeDict[versNum]['total'] += \
87 self.fileSizeDict[versNum][fileName]
88 if not 'cat' in fileName:
89 typeSizeDict[versNum]['fit'] += \
90 self.fileSizeDict[versNum][fileName]
91 if 'dark' in fileName:
92 typeSizeDict[versNum]['dark'] += \
93 self.fileSizeDict[versNum][fileName]
94 elif 'sky' in fileName:
95 typeSizeDict[versNum]['sky'] += \
96 self.fileSizeDict[versNum][fileName]
97 elif 'flat' in fileName:
98 typeSizeDict[versNum]['flat'] += \
99 self.fileSizeDict[versNum][fileName]
100 elif '_sf_st_cat' in fileName:
101 typeSizeDict[versNum]['_sf_st_cat'] += \
102 self.fileSizeDict[versNum][fileName]
103 try:
104 catSizeDict[versNum]['_sf_st_cat'] += \
105 self.fileSizeDict[versNum][fileName]
106 except KeyError:
107 pass
108 try:
109 catSizeDict[versNum]['_sf_st_cat'] += \
110 self.fileSizeDict[versNum][fileName.replace("_cat.fits", ".fit")]
111 except KeyError:
112 pass
113 try:
114 catSizeDict[versNum]['_sf_st_cat'] += \
115 self.fileSizeDict[versNum][fileName.replace("_cat.fits", "_conf.fit")]
116 except KeyError:
117 pass
118 elif '_st_cat' in fileName:
119 typeSizeDict[versNum]['_st_cat'] += \
120 self.fileSizeDict[versNum][fileName]
121 try:
122 catSizeDict[versNum]['_st_cat'] += \
123 self.fileSizeDict[versNum][fileName]
124 except KeyError:
125 pass
126 try:
127 catSizeDict[versNum]['_st_cat'] += \
128 self.fileSizeDict[versNum][fileName.replace("_cat.fits", ".fit")]
129 except KeyError:
130 pass
131 try:
132 catSizeDict[versNum]['_st_cat'] += \
133 self.fileSizeDict[versNum][fileName.replace("_cat.fits", "_conf.fit")]
134 except KeyError:
135 pass
136 elif '_cat' in fileName:
137 typeSizeDict[versNum]['_cat'] += \
138 self.fileSizeDict[versNum][fileName]
139 try:
140 catSizeDict[versNum]['_cat'] += \
141 self.fileSizeDict[versNum][fileName]
142 except KeyError:
143 pass
144 try:
145 catSizeDict[versNum]['_cat'] += \
146 self.fileSizeDict[versNum][fileName.replace("_cat.fits", ".fit")]
147 except KeyError:
148 pass
149 try:
150 catSizeDict[versNum]['_cat'] += \
151 self.fileSizeDict[versNum][fileName.replace("_cat.fits", "_conf.fit")]
152 except KeyError:
153 pass
154 elif '_sf_st_conf' in fileName:
155 typeSizeDict[versNum]['_sf_st_conf'] += \
156 self.fileSizeDict[versNum][fileName]
157 elif '_sf_st' in fileName:
158 typeSizeDict[versNum]['_sf_st'] += \
159 self.fileSizeDict[versNum][fileName]
160 elif '_sf_conf' in fileName:
161 typeSizeDict[versNum]['_sf_conf'] += \
162 self.fileSizeDict[versNum][fileName]
163 elif '_sf' in fileName:
164 typeSizeDict[versNum]['_sf'] += \
165 self.fileSizeDict[versNum][fileName]
166 elif '_st_conf' in fileName:
167 typeSizeDict[versNum]['_st_conf'] += \
168 self.fileSizeDict[versNum][fileName]
169 elif '_st' in fileName:
170 typeSizeDict[versNum]['_st'] += \
171 self.fileSizeDict[versNum][fileName]
172 elif 'conf' in fileName:
173 typeSizeDict[versNum]['conf'] += \
174 self.fileSizeDict[versNum][fileName]
175 elif '%s20' % self.sysc.casuPrefix in fileName \
176 and fileName.count('_') == 1 \
177 and '.fit' in fileName:
178 typeSizeDict[versNum]['normal'] += \
179 self.fileSizeDict[versNum][fileName]
180 elif '%s20' % self.sysc.casuPrefix in fileName \
181 and fileName.count('_') == 2 \
182 and '.jpg' in fileName:
183 typeSizeDict[versNum]['normal'] += \
184 self.fileSizeDict[versNum][fileName]
185 else:
186 print fileName
187 typeSizeDict[versNum]['misc'] += \
188 self.fileSizeDict[versNum][fileName]
189
190 return typeSizeDict, catSizeDict
191
192
193
194
196 _logFile = Logger("FileSizes%s_%s.log" % (
197 self.semester, os.path.basename(self.subDir)), path=os.curdir)
198 self.createDailyLists(self.diskList, self.subDir)
199 typeSizeDict, catSizeDict = self.calcSize()
200 for versNum in sorted(typeSizeDict):
201 Logger.addMessage("%d files in %sv%s" % (
202 len(self.fileSizeDict[versNum]), self.semester, versNum))
203 for sizeType in sorted(typeSizeDict[versNum]):
204 if typeSizeDict[versNum]['total'] > 0:
205 if self.useGByte:
206 format = "[%s] %-12s: %12.6f\n"
207 sizeVal = typeSizeDict[versNum][sizeType]/ \
208 (1.*self.sysc.one_gigabyte)
209 else:
210 format = "[%s] %-12s: %12d\n"
211 sizeVal = typeSizeDict[versNum][sizeType]
212
213 print(format % (versNum, sizeType, sizeVal)),
214 Logger._logText.append(format % (
215 versNum, sizeType, sizeVal))
216
217 for sizeType in sorted(catSizeDict[versNum]):
218 if typeSizeDict[versNum]['total'] > 0 and "cat" in sizeType:
219 if self.useGByte:
220 format = "[%s] %-12s: %12.6f\n"
221 sizeVal = catSizeDict[versNum][sizeType]/ \
222 (1.*self.sysc.one_gigabyte)
223 else:
224 format = "[%s] %-12s: %12d\n"
225 sizeVal = catSizeDict[versNum][sizeType]
226 print(format % (versNum, sizeType+"F", sizeVal)),
227 Logger._logText.append(format % (
228 versNum, sizeType+"F", sizeVal))
229
230
231
232
233
234 if __name__ == "__main__":
235 CLI.progArgs.append(CLI.Argument("archive", "WSA"))
236 CLI.progArgs.append(CLI.Argument("semester", "01X"))
237 CLI.progOpts += [
238 CLI.Option('k', "disklist",
239 "list of RAID disk paths including the archive name, "
240 "eg. '/disk01/wsa,/disk02/wsa' (depending on given archive).",
241 "LIST", None),
242 CLI.Option('s', "subdir",
243 "sub-directory containing FITS date directories.",
244 "DIR", None),
245 CLI.Option('G', "giga",
246 "Output in GByte.")]
247
248 cli = CLI(GetFileSizes.__name__, "$Revision: 10139 $", GetFileSizes.__doc__)
249 Logger.addMessage(cli.getProgDetails())
250
251 task = GetFileSizes(cli.getOpt("disklist"),
252 cli.getOpt("subdir"),
253 cli.getOpt("giga"),
254 cli.getArg("semester"),
255 cli.getArg("archive"))
256 task.run()
257
258
259