1
2
3
4 """
5 Create jpeg thumbnail pages for QC.
6
7 @author: E. Sutorius
8 @org: WFAU, IfA, University of Edinburgh
9 """
10
11 from numpy import arange
12 from collections import defaultdict
13 import Image
14 import inspect
15 import multiprocessing
16 import os
17 import subprocess
18 import sys
19
20 from wsatools.CLI import CLI
21 import wsatools.DbConnect.DbConstants as dbc
22 from wsatools.DbConnect.DbSession import DbSession, Join
23 from wsatools.File import File, HTMLFile, PickleFile
24 from wsatools.DbConnect.IngCuSession import IngCuSession
25 from wsatools.Logger import Logger
26 import wsatools.Utilities as utils
27
28
30 return os.stat(jpgPath).st_mtime > os.stat(thumbPath).st_mtime
31
32
33
35
36
37
38 numprocesses = 1
39 maxChunkSize = 100
40
41
57
58
59
61 """
62 Calculate JPGs.
63 @param args: Arguments for createThumbnailImage.
64 @type args: args
65
66 """
67 result = self.createThumbnailImage(*args)
68 return '%s(%s) = %s' % (self.createThumbnailImage.__name__, args,
69 result)
70
71
72
74 """
75 Creates a thumbnail from a JPEG compressed image.
76
77 @param origJpg: Full path to the JPG file.
78 @type origJpg: str
79 @param thumbPath: Path to thumbnail directory.
80 @type thumbPath: str
81 """
82 outJpgName = os.path.join(thumbPath, "%s_th.jpg" %
83 os.path.splitext(os.path.basename(origJpg))[0])
84 try:
85 if not os.path.exists(outJpgName) \
86 or checkTimestamps(origJpg, outJpgName):
87 im = Image.open(origJpg)
88 im.thumbnail((CreateJpgThumbs._thSize, CreateJpgThumbs._thSize),
89 Image.ANTIALIAS)
90 im.save(outJpgName, "JPEG")
91 status = True
92 except IOError:
93 Logger.addMessage(
94 "Cannot create thumbnail for %s" % origJpg)
95 status = False
96 return status
97
98
99
102
103
104
106 """
107 Create JPG thumbnails pages for (external) QC.
108
109 """
110
111
112 overwrite = False
113 onlyHTML = False
114 _surveyNameDef = "ukidss"
115
116
117 _jpgThumbPath = "/disk08/%s/products/jpgthumbs"
118 _thSize = 192
119 _thPerPage=100
120
121
122 fitsRatings = utils.Ratings({"stack": 10,
123 "conf": 30,
124 "science": 50,
125 "sciconf":55,
126 "basic": 90})
127 fitsTypes = {"stack": ["_st", "stack"],
128 "conf": ["_conf", "stack conf"],
129 "science": ["w20", "normal"],
130 "sciconf": ["_conf", "normal conf"],
131 "basic": ["misc", "calib"]}
132
133
134
135 - def __init__(self,
136 curator=CLI.getOptDef("curator"),
137 database=DbSession.database,
138 beginDate=CLI.getOptDef("begin"),
139 endDate=CLI.getOptDef("end"),
140 versionStr=CLI.getOptDef("version"),
141 surveyNames=CLI.getOptDef("survey"),
142 cleanUp=CLI.getOptDef("cleanup"),
143 fromFile=CLI.getOptDef("fromfile"),
144 numThreads=CLI.getOptDef("threads"),
145 isTrialRun=DbSession.isTrialRun,
146 comment=CLI.getArgDef("comment")):
147 """
148 @param beginDate: First date to process, eg. 20050101.
149 @type beginDate: int
150 @param cleanUp: Remove HTML files before processing.
151 @type cleanUp: bool
152 @param comment: Descriptive comment as to why curation task is
153 being performed.
154 @type comment: str
155 @param endDate: Last date to process, eg. 20050131.
156 @type endDate: int
157 @param fromFile: Pickled list of existing JPGs.
158 @type fromFile: str
159 @param isTrialRun: If True, do not perform database modifications.
160 @type isTrialRun: bool
161 @param numThreads: Number of multiprocessing threads.
162 @type numThreads: int
163 @param surveyNames: List of surveys to process
164 @type surveyNames: list(str)
165 @param versionStr: Version number of the data or 'all' or 'last'.
166 @type versionStr: str
167
168 """
169 typeTranslation = {"curator":str,
170 "database":str,
171 "beginDate":str,
172 "endDate":str,
173 "versionStr":str,
174 "surveyNames":list,
175 "cleanUp":bool,
176 "onlyHTML":bool,
177 "fromFile":list,
178 "numThreads":int,
179 "isTrialRun":bool,
180 "comment":str}
181
182 super(CreateJpgThumbs, self).attributesFromArguments(
183 inspect.getargspec(CreateJpgThumbs.__init__)[0], locals(),
184 types=typeTranslation)
185
186
187 super(CreateJpgThumbs, self).__init__(cuNum=0,
188 curator=self.curator,
189 comment=self.comment,
190 reqWorkDir=False,
191 database=self.database,
192 autoCommit=False,
193 isTrialRun=self.isTrialRun)
194 self.jpgThumbDict = defaultdict(dict)
195 self._jpgThumbPath %= self.sysc.loadDatabase.lower()
196
197
198 ThumbCalc.numprocesses = (multiprocessing.cpu_count()
199 if self.numThreads == 0 else self.numThreads)
200
201
202
204 """ Create thumbnails.
205 """
206 self.obsCal = self.sysc.obsCal
207 self.beginDate, self.endDate = \
208 self.obsCal.getDatesFromInput(self.beginDate, self.endDate)
209
210 semList = [sem for sem in self.obsCal.getSemList()
211 if self.obsCal.checkDate(self.beginDate) <= sem <= \
212 self.obsCal.checkDate(self.endDate)]
213
214 for semester in semList:
215 semBeginDate, semEndDate = \
216 self.obsCal.getDates(semester, "%Y%m%d")
217 beginDate = max(self.beginDate, semBeginDate)
218 endDate = min(self.endDate, semEndDate)
219 versList = []
220 if self.versionStr.replace('.', '').isdigit():
221 versList.append(self.versionStr)
222 elif self.versionStr == "all":
223 versList.extend(self.obsCal.versNums[semester])
224 else:
225 versList.append(self.obsCal.maxVers(semester))
226
227 for versStr in versList:
228 Logger.addMessage("Creating thumbnails for %s: %s - %s v%s" %
229 (semester, beginDate, endDate, versStr))
230 jpgDateDict = self._getJpgDict(beginDate, endDate, versStr)
231 if not self.isTrialRun and jpgDateDict:
232 self._createThumbs(jpgDateDict)
233 webPageDict = self._createHtml(semester)
234
235 if not self.isTrialRun and self.jpgThumbDict:
236 self._createTopHtml(webPageDict, semester, versList)
237 Logger.addMessage("finished %s." % semester)
238 timestamp = utils.makeTimeStamp().replace(' ','_')
239 Logger.dump(file("CreateJpgThumbs_%s.log" % timestamp, 'w'))
240
241
242
243 - def _getJpgDict(self, startDate, endDate, versionStr):
244 """ Create a dictionary of JPGs per day and survey.
245 """
246 startDateStr = "%s_v%s" % (startDate, versionStr)
247 endDateStr = "%s_v%s" % (endDate, versionStr)
248 selectStr = "programmeID, dateVersStr, compFile"
249 fromStr = Join(["ProgrammeFrame", "FlatFileLookup",
250 "MultiframeDetector"], ["multiframeID"])
251 whereStr = " AND ".join([
252 "compFile NOT LIKE '%s'" % dbc.charDefault(),
253 "dateVersStr BETWEEN '%s' AND '%s'" % (startDateStr, endDateStr),
254 "dateVersStr LIKE '%%v%s'" % versionStr])
255
256 if not self.isTrialRun:
257 self._connectToDb()
258 self._createProgrammeTranslation()
259 jpgList = []
260 if self.fromFile and any(['_v%s' % versionStr in x
261 for x in self.fromFile]):
262 Logger.addMessage("Reading data from file(s) %s" % \
263 self.fromFile)
264 for fileName in self.fromFile:
265 inFile = PickleFile(fileName)
266 jpgList.extend(list(inFile.pickleRead())[0])
267 else:
268 Logger.addMessage("Querying database from %s to %s" % (
269 startDateStr, endDateStr))
270 jpgList = self.archive.query(selectStr, fromStr, whereStr)
271 outFile = PickleFile("jpgList_%s_%s.data" % (startDateStr,
272 endDateStr))
273 outFile.pickleWrite(jpgList)
274 self._disconnectFromDb()
275
276
277 deco = [(compFileName[compFileName.rfind(
278 '/',compFileName.rfind('/')-1):],
279 programmeID, date, compFileName)
280 for programmeID, date, compFileName in jpgList]
281 deco.sort()
282 jpgList = [(programmeID, date, compFileName)
283 for _, programmeID, date, compFileName in deco]
284
285 jpgDict = defaultdict(dict)
286 Logger.setEchoOff()
287 for programmeID, date, compFileName in jpgList:
288 if os.path.exists(compFileName.rpartition(':')[2]):
289 jpgDict[programmeID].setdefault(date, []).append(
290 compFileName.rpartition(':')[2])
291 else:
292 Logger.addMessage("JPEG file doesn't exist: %s" % \
293 compFileName.rpartition(':')[2])
294 Logger.setEchoOn()
295
296 progOrderRatedDict = \
297 self.getProcessOrder(jpgDict.keys(), include=self.surveyNames)
298
299 return dict((k, jpgDict.get(k, {}))
300 for k in progOrderRatedDict.keys())
301 else:
302 print "SELECT ", selectStr
303 print "FROM ", fromStr.fromStr
304 print "WHERE %s AND %s" % (fromStr.whereStr, whereStr)
305 return dict()
306
307
308
310 """Create thumbnails from existing JPGs.
311 """
312 for programmeID in jpgDateDict:
313 progName = self._progNameOfID[programmeID]
314 progName = progName.replace('U/UKIDSS/','').replace('/', '')
315
316 for date in sorted(jpgDateDict[programmeID]):
317 jpgRatingsDict = utils.Ratings({})
318
319 Logger.addMessage(
320 "Creating thumbnails for %s: %s" % (progName, date))
321 thumbPath = os.path.join(self._jpgThumbPath, date, "thumbs")
322 utils.ensureDirExist(thumbPath)
323
324 if not CreateJpgThumbs.onlyHTML \
325 or CreateJpgThumbs.overwrite:
326
327 inputs = [(origJpg, thumbPath)
328 for origJpg in jpgDateDict[programmeID][date]]
329
330 if len(inputs) < ThumbCalc.maxChunkSize/2:
331 ThumbCalc.numprocesses = 1
332
333
334 c = ThumbCalc()
335 c.pooling(inputs)
336
337 self.jpgThumbDict[programmeID].setdefault(date, {})["data"] = {}
338 for origJpg in jpgDateDict[programmeID][date]:
339 origShort = origJpg[origJpg.rfind(
340 '/', origJpg.rfind('/')-1):]
341 jpgRatingsDict[(origShort, origJpg)] = \
342 self._getRating(origJpg)
343 self.jpgThumbDict[programmeID][date]["data"][origJpg] = \
344 os.path.join(thumbPath, "%s_th.jpg" %
345 os.path.splitext(os.path.basename(origJpg))[0])
346
347 self.jpgThumbDict[programmeID][date]["ratings"] = jpgRatingsDict
348
349
350
352 webPageDict = defaultdict(dict)
353
354 Logger.addMessage("Creating web pages...")
355 for programmeID in sorted(self.jpgThumbDict):
356 progName = self._progNameOfID[programmeID]
357 progName = progName.replace('U/UKIDSS/','').replace('/', '')
358
359 for date in sorted(self.jpgThumbDict[programmeID]):
360 Logger.addMessage(
361 "Creating HTML for %s: %s" % (progName, date))
362 htmlPath = os.path.join(self._jpgThumbPath, date, "html")
363 utils.ensureDirExist(htmlPath)
364 topHtmlDoc = os.path.join(self._jpgThumbPath, "html",
365 "%sthumbs_%s.html" % (progName, semester))
366 pageCounter = 0
367 counter = 0
368 jpgRatings = self.jpgThumbDict[programmeID][
369 date]["ratings"]
370 dataLength = jpgRatings.getCountsPerRating()
371 htmlFileBaseName = "JPGthumbs_%s_%s" % (progName, date)
372 if self.cleanUp:
373 self._cleanUp("%s/%s*.html" % (
374 htmlPath, htmlFileBaseName))
375 webPageDict[programmeID].setdefault(date, []).extend(
376 [os.path.join(htmlPath, "%s_1.html" % htmlFileBaseName),
377 dataLength])
378 htmlFile = pageLinks = oldRating = None
379
380 for origShort, orig in jpgRatings:
381
382 if counter != 0 \
383 and jpgRatings[(origShort, orig)] != oldRating:
384 self._endPage(htmlFile, pageLinks)
385 counter = 0
386
387 if counter%CreateJpgThumbs._thPerPage == 0 \
388 or (counter != 0 \
389 and jpgRatings[(origShort, orig)] != oldRating):
390 pageCounter += 1
391 htmlFileName = os.path.join(
392 htmlPath, "%s_%s.html" % \
393 (htmlFileBaseName, pageCounter))
394 htmlFile, pageLinks = self._startPage(
395 htmlFileName, pageCounter, dataLength,
396 progName, date, topHtmlDoc)
397
398
399 thumbnail = File(self.jpgThumbDict[programmeID][
400 date]["data"][orig])
401 origJpg = File(orig)
402 oldRating = jpgRatings[(origShort, orig)]
403 if counter%4 == 0:
404 tableEntry = "<tr>"
405
406 while int(thumbnail.root.rsplit('_', 2)[1]) != counter%4+1:
407 tableEntry += ''.join(
408 ["<td align=\"center\" class='v'>",
409 "No JPEG available",
410 "</td>"])
411 if counter%4 == 3:
412 tableEntry += "</tr>"
413 counter += 1
414
415 tableEntry += ''.join(
416 ["<td align=\"center\" class='v'>",
417 "<a href=\"%s\">" % origJpg.name,
418 "<IMG height=\"%s\"" % self._thSize,
419 " src=\"%s\"" % thumbnail.name,
420 " border=0 alt=\"%s\"" % thumbnail.base,
421 " title=\"%s\">" % thumbnail.root,
422 "<br>%s</a></td>" % origJpg.root])
423
424 if counter%4 == 3:
425 tableEntry += "</tr>"
426 htmlFile.writetheline(tableEntry)
427 counter += 1
428 if counter%CreateJpgThumbs._thPerPage == 0:
429 self._endPage(htmlFile, pageLinks)
430 counter = 0
431 return webPageDict
432
433
434
448
449
450
451 - def _startPage(self, htmlFileName, pageCounter, dataLength,
452 progName, date, htmlLink):
453 htmlFile = HTMLFile(htmlFileName)
454 htmlFile.wopen()
455 pageLinks = self._createPageLinks(htmlFile.base, dataLength)
456
457 htmlFile.writeHeader("%s JPEG thumbnails %s: %s (p.%s)" % (
458 self.sysc.loadDatabase, progName, date, pageCounter),
459 ["%scommon/wsaserver.css" % self.sysc.surveyBaseUrl(),
460 "a:link{color:#99FF33;}\na:visited{color:#99ffff}\n"])
461 htmlFile.writelines([
462 "<div align=\"center\"><h1><a href=\"%s\">%s</a> %s v%s (p.%s)</h1>" % (
463 htmlLink, progName, self.obsCal.formatDate(date.partition('_v')[0],
464 outFormat="%d/%m/%Y"),
465 date.partition('_v')[2], pageCounter),
466 pageLinks, "</div><br><p>"])
467
468 tableHead = ''.join([
469 "<div align=\"center\">",
470 "<TABLE border=0 cellspacing=1 cellpadding=4 width='800px'>",
471 ("<col width='%spx'>" % self._thSize)*4])
472 htmlFile.writetheline(tableHead)
473 return htmlFile, pageLinks
474
475
476
477 - def _endPage(self, htmlFile, pageLinks):
478 htmlFile.writetheline("</TABLE></div>")
479 htmlFile.writelines(["<P><br><center>", pageLinks, "</center><br>"])
480
481 htmlFile.writeFoot()
482 htmlFile.close()
483
484
485
487 Logger.addMessage("Creating top level web pages...")
488 htmlPath = os.path.join(self._jpgThumbPath, "html")
489 utils.ensureDirExist(htmlPath)
490 topHtmlDoc = os.path.join(htmlPath, "JPGthumbs.html")
491 for programmeID in webPageDict:
492 progName = self._progNameOfID[programmeID]
493 progName = progName.replace('U/UKIDSS/','').replace('/', '')
494 Logger.addMessage("Creating HTML for %s: %s" % (
495 progName, semester))
496 htmlFileName = "%sthumbs_%s.html" % (progName, semester)
497 htmlFile = HTMLFile(os.path.join(htmlPath,
498 htmlFileName))
499 htmlFile.wopen()
500
501 htmlFile.writeHeader(
502 "%s JPEG thumbnails %s" % (self.sysc.loadDatabase, progName),
503 ["%scommon/wsaserver.css" % self.sysc.surveyBaseUrl(),
504 "a:link{color:#99FF33;}\na:visited{color:#99ffff}\n" + \
505 ".noth{font-size:10pt; background-color:#000050; " + \
506 "font-weight: 100; color: #446688; border-width: 3; " + \
507 "border-color=#001144; border-style:groove;" + \
508 " padding:2px;}\n" + \
509 ".vers{font-size:11pt; background-color:#000000; " + \
510 "font-weight: 100; color: #ffffff; border-width: 0; " + \
511 "border-style:none; padding:0px;}\n"])
512 htmlFile.writetheline("<center><h1><a href=\"%s\">%s</a></h1></center>" % (topHtmlDoc, progName))
513 monthList = self.obsCal.getSemMonthList(semester)
514 dayList = self.obsCal.getSemDayList(semester)
515 for monthNo in monthList:
516 htmlFile.writetheline("<center><h3>%s %s</h3></center>" % (
517 self.obsCal.getMonth(monthNo).title(), monthNo[:4]))
518 tableHead = ''.join([
519 "<div align=\"center\">",
520 "<TABLE border='0' width='840px'>",
521 "<colgroup width='40px' span='17'></colgroup>"])
522
523 htmlFile.writetheline(tableHead)
524 for versionStr in versionList:
525 counter = 0
526 dayLine = ""
527 for day in dayList:
528 if day[:6] == monthNo:
529 if counter%16 == 0:
530 dayLine = "<tr><td align='center' " + \
531 "class='vers'>" + \
532 "V%s</td>" % versionStr
533 dateVersStr = "%s_v%s" % (day, versionStr)
534 if dateVersStr in webPageDict[programmeID]:
535 entry = "<a href=\"%s\">%s</a>" % (
536 webPageDict[programmeID][dateVersStr][0],
537 day[6:])
538 else:
539 entry = day[6:]
540 dayLine += \
541 "<td align=\"center\" class='noth'>%s</td>" % entry
542 counter += 1
543 if counter%16 == 0:
544 dayLine += "</tr>\n"
545 htmlFile.writetheline(dayLine)
546 dayLine = ""
547 counter = 0
548 dayLine += "</tr>\n"
549 htmlFile.writetheline(dayLine)
550 dayLine = ""
551 if dayLine:
552 htmlFile.writetheline(dayLine+"</tr>")
553 htmlFile.writetheline("</TABLE></div>")
554
555 htmlFile.writeFoot()
556 htmlFile.close()
557
558
559
560 - def _createPageLinks(self, currentPage, dataLengthDict):
561 pageLinks = ' '.join(["<table border='0'","align='center'",
562 "cellspacing='4'", "width='800px'>"])
563 pageNum = 0
564 for fType, rating in CreateJpgThumbs.fitsRatings.items():
565 if rating in dataLengthDict:
566 num = dataLengthDict.get(rating, 0)
567 maxPages = max(1, num//CreateJpgThumbs._thPerPage \
568 + (num%CreateJpgThumbs._thPerPage>0))
569 pageLinks += "<tr><td>%s:</td><td>" % \
570 CreateJpgThumbs.fitsTypes[fType][1]
571 for i in range(pageNum, pageNum+maxPages):
572 page = "%s_%s.html" % (
573 currentPage.rsplit('_', 1)[0], (i+1))
574 if page == currentPage:
575 pageLinks += " (%s)" % (i+1)
576 else:
577 pageLinks += ' '.join([
578 "<a href=\"%s\">" % page, "[%s]</a>" % (i+1)])
579
580 pageNum += maxPages
581 pageLinks += "</td></tr>\n"
582 pageLinks += "</table>\n"
583 return pageLinks
584
585
586
588 try:
589 retcode = subprocess.call("rm -f %s" % nameRegExp, shell=True)
590 if retcode < 0:
591 print >>sys.stderr, "Child was terminated by signal", -retcode
592 except OSError, e:
593 print >>sys.stderr, "Execution failed:", e
594
595
596
597
598
599 if __name__ == '__main__':
600 CLI.progOpts += [CLI.Option('s', "survey",
601 "Survey name(s), can be 'all', 'ukidss'",
602 "NAME", CreateJpgThumbs._surveyNameDef),
603 CLI.Option('b', "begin",
604 "first date/semester to process",
605 "DATE", "05A", isValOK=CLI.isDateOK),
606 CLI.Option('C', "cleanup",
607 "remove HTML files before processing"),
608 CLI.Option('e', "end",
609 "last date/semester to process",
610 "DATE", "08A", isValOK=CLI.isDateOK),
611 CLI.Option('v', "version",
612 "version number of the data or 'all', 'last'",
613 "STR", 'all'),
614 CLI.Option('y', "threads", "number of processors to use",
615 "INT", '0'),
616 CLI.Option('F', "fromfile", "read JPG list from file",
617 "STR", ''),
618 CLI.Option('H', "htmlonly", "create HTML pages only"),
619 CLI.Option('f', "force",
620 "overwrite existing thumbnails")]
621
622 cli = CLI(CreateJpgThumbs, "$Revision: 8979 $")
623 Logger.addMessage(cli.getProgDetails())
624
625 CreateJpgThumbs.overwrite = cli.getOpt("force")
626 CreateJpgThumbs.onlyHTML = cli.getOpt("htmlonly")
627
628 makeThumbs = CreateJpgThumbs(cli.getOpt("curator"),
629 cli.getArg("database"),
630 cli.getOpt("begin"),
631 cli.getOpt("end"),
632 cli.getOpt("version"),
633 cli.getOpt("survey"),
634 cli.getOpt("cleanup"),
635 cli.getOpt("fromfile"),
636 cli.getOpt("threads"),
637 cli.getOpt("test"),
638 cli.getArg("comment"))
639 makeThumbs.run()
640
641
642