1
2
3
4 """
5 Move the redone JPGs from their redo/ dirs into the original directories.
6
7 @author: Eckhard Sutorius
8 @org: WFAU, IfA, University of Edinburgh
9 """
10
11 from collections import defaultdict
12 import inspect
13 import os
14
15 from wsatools.CLI import CLI
16 from wsatools.DbConnect.DbSession import DbSession
17 from wsatools.File import File
18 import wsatools.FitsUtils as fits
19 from wsatools.Logger import ForLoopMonitor
20 from wsatools.DbConnect.IngCuSession import IngCuSession
21 from wsatools.DbConnect.IngIngester import IngestLogger as Logger
22 from wsatools.SystemConstants import SystemConstants
23 import wsatools.Utilities as utils
26 """
27 Replace old JPGs with reprocessed ones.
28 """
39 """
40 @param curator: Name of curator.
41 @type curator: str
42 @param comment: Descriptive comment as to why curation task is
43 being performed.
44 @type comment: str
45 @param beginDate: First date to process, eg. 20050101.
46 @type beginDate: int
47 @param endDate: Last date to process, eg. 20050131.
48 @type endDate: int
49 @param fileType: The file type to be replaced.
50 @type fileType: str
51 @param isTrialRun: If True, do not perform database modifications.
52 @type isTrialRun: bool
53 @param redoDir Path to the directory, where re-done JPGs reside.
54 @type redoDir str
55 @param versionStr: Version number of the data.
56 @type versionStr: str
57
58 """
59
60 typeTranslation = {"curator":str,
61 "database":str,
62 "beginDate":type(IngCuSession.beginDateDef),
63 "endDate":type(IngCuSession.endDateDef),
64 "versionStr":str,
65 "redoDir":str,
66 "fileType":list,
67 "isTrialRun":bool,
68 "comment":str}
69
70 super(ReplaceJpgs, self).attributesFromArguments(
71 inspect.getargspec(ReplaceJpgs.__init__)[0], locals(),
72 types=typeTranslation)
73
74
75 super(ReplaceJpgs, self).__init__(cuNum=0,
76 curator=self.curator,
77 comment=self.comment,
78 reqWorkDir=False,
79 database=self.database,
80 autoCommit=False,
81 isTrialRun=self.isTrialRun)
82
83
84 self.jpegDirs = fits.FitsList(self.sysc, 'jpg')
85 self.jpegDirs.createFitsDateDict(
86 ingestDirectory=self.sysc.compressImDir,
87 beginDateStr=str(self.beginDate), endDateStr=str(self.endDate),
88 versionStr=self.versionStr, forceLists=True)
89 if self.isTrialRun:
90 print("JD:", self.jpegDirs.invFitsDateDict)
91
92
93 self.redoDirs = fits.FitsList(self.sysc, 'rjpg')
94 self.redoDirs.createFitsDateDict(
95 disklist=[self.redoDir],
96 ingestDirectory=self.redoDir,
97 beginDateStr=str(self.beginDate), endDateStr=str(self.endDate),
98 versionStr=self.versionStr, forceLists=True)
99 if self.isTrialRun:
100 print("RD:", self.redoDirs.invFitsDateDict)
101
102
103
137
138
139
140 @staticmethod
142 """Check if all files are created.
143
144 @param origJpgsDict: Dictionary of the original JPGs.
145 @type origJpgsDict: dict
146 @param redoJpgsDict: Dictionary of the re-done JPGs.
147 @type redoJpgsDict: dict
148
149 @return: Dict of all JPGs where no re-done JPG is found.
150 @rtype: dict
151
152 """
153 missingDict = defaultdict(lambda : defaultdict(list))
154 for dateVersStr in sorted(origJpgsDict):
155 for fileID in origJpgsDict[dateVersStr]:
156 if fileID in redoJpgsDict[dateVersStr]:
157 for fileName in origJpgsDict[dateVersStr][fileID]:
158 theFile = File(fileName)
159 if os.path.join(theFile.subdir, theFile.base) \
160 not in redoJpgsDict[dateVersStr][fileID]:
161 missingDict[dateVersStr][fileID].append(
162 theFile.name)
163 del theFile
164 else:
165 missingDict[dateVersStr][fileID] = \
166 origJpgsDict[dateVersStr][fileID]
167 return missingDict
168
169
170
171 @staticmethod
173 """ Get all the files in the directory.
174
175 @param dirName: Directory path.
176 @type dirName: str
177 @param fileType: The file type to be replaced.
178 @type fileType: str
179 @param fullPath: If true write the full path else only the fileID.
180 @type fullPath: bool
181
182 @return: Dict of all JPGs where no re-done JPG is found.
183 @rtype: dict
184
185 """
186 tmpDict = defaultdict(list)
187 Logger.addMessage("reading %s ..." % dirName)
188 for path, dirs, files in os.walk(dirName):
189
190 for dotDir in (".svn", ".emacs_backups"):
191 if dotDir in dirs:
192 dirs.remove(dotDir)
193 for fileName in files:
194 if not fileName.startswith(".stats"):
195 theFile = File(os.path.join(path, fileName))
196 theType = theFile.ftype
197 thePath = (theFile.name if fullPath
198 else os.path.join(theFile.subdir, theFile.base))
199 if "all" in fileType \
200 or any(x in theType for x in fileType):
201 tmpDict[theFile.fileID.rpartition('_')[0]].append(
202 thePath)
203 del theFile
204 return tmpDict
205
206
207
208 - def getJpgs(self, theDirs, fileType, fullPath=True):
209 """Create a dictionary containing info about the paths to each JPG.
210
211 @param theDirs: Contains info about all the JPG dirs.
212 @type theDirs: FitsList obj
213 @param fileType: The file type to be replaced.
214 @type fileType: str
215 @param fullPath: If true write the full path else only the fileID.
216 @type fullPath: bool
217
218 @return: Dict of all JPGs by dateVersStr and fileID.
219 @rtype: dict
220
221 """
222 jpgSum = 0
223 jpegDict = defaultdict(lambda : defaultdict(list))
224 for dateVersStr in theDirs.invFitsDateDict:
225 for jpgDir in theDirs.invFitsDateDict[dateVersStr]:
226 jpegDict[dateVersStr].update(self.getFilesInDir(
227 os.path.join(jpgDir, dateVersStr), fileType, fullPath))
228 return jpegDict
229
230
231
232 - def movJpgs(self, origJpgsDict, redoJpgsDict):
266
267
268
269
270
271 if __name__ == '__main__':
272 CLI.progOpts += [
273 CLI.Option('T', "type", "file type to be replaced",
274 "TYPE", "all"),
275 CLI.Option('b', "begin", "first date to process, eg. 20050101",
276 "DATE", str(IngCuSession.beginDateDef)),
277 CLI.Option('e', "end", "last date to process, eg. 20050131",
278 "DATE", str(IngCuSession.endDateDef)),
279 CLI.Option('v', "version", "version number of the data",
280 "STR", '1'),
281 CLI.Option('r', "redodisk",
282 "Path where the re-done /<archive>/redo/%s subdirectory"
283 " is." % SystemConstants.compressImDir, "PATH",
284 os.path.dirname(SystemConstants.developDisks[0]))]
285
286 cli = CLI(ReplaceJpgs, "$Revision: 10105 $")
287 Logger.addMessage(cli.getProgDetails())
288 redoDir = os.path.join(cli.getOpt("redodisk"),
289 cli.getArg("database").rpartition('.')[2].lower(),
290 "redo", SystemConstants.compressImDir)
291 Logger.addMessage("The re-done JPGs are looked up in: %s" % redoDir)
292
293 repl = ReplaceJpgs(cli.getOpt("curator"),
294 cli.getArg("database"),
295 cli.getOpt("begin"),
296 cli.getOpt("end"),
297 cli.getOpt("version"),
298 redoDir,
299 cli.getOpt("type"),
300 cli.getOpt("test"),
301 cli.getArg("comment"))
302 repl.run()
303