1
2
3
4 """
5 Lists CASU disks to get file time information to be able to sync our files
6 with CASU's.
7
8 @author: E.T.W. Sutorius
9 @org: WFAU, IfA, University of Edinburgh
10 """
11
12 import getopt
13 import os
14 import sys
15
16 from wsatools.File import PickleFile
17 import wsatools.SystemConstants as sysc
18 import wsatools.Utilities as utils
19
20 dataArchive = "WSA"
21 light = True
22 loglines = []
23 verboseTag = True
24
25
27 print("""Usage:
28 lsCasuDirs.py [-d/--disks casudisks] [-b/--begin startdate] [-e/--end enddate]
29 [-j/--janet] [-o/--outdir outdir] [-r/--read filename]
30 [-v/--verbose]
31 startdate: first date dir to be checked (default: 0)
32 enddate: last date to be checked (default: 99999999)
33 casudisks: comma separated list of disks at CASU (default: %r)
34 outdir: log file directory (default: %s)
35 -j/--janet uses the JANET connection instead of UKLight
36 -r/--read read pickled data file instead of creating it
37 -v/--verbose prints log messages also onto screen""" % \
38 (sysc.casuFileSystem(), sysc.cambInfoPath()))
39
40
41
43 """
44 Create a dictionary containing CASU info ordered by disk from a list
45 of CASU disks.
46 """
47 casuDict = {}
48 for entry in aList:
49 d5 = entry[1:].split('/')[-1]
50 dateVersStr = "%s_v%s" % (d5, versDict[int(d5)])
51 casuDict[dateVersStr] = {}
52 if begin <= int(d5) <= end:
53 cmd = "ssh " + scp_user() + "@" + scp_server() \
54 + " 'ls --indicator-style=none -o" \
55 +" --time-style=full-iso " + entry + "'"
56
57 for line in os.popen(cmd):
58 if len(line.split()) == 8:
59 access, nol, uid, size, yyyymmdd, time, timezone, name = \
60 line.split()
61
62 if os.path.splitext(name)[1] in ([".fit",".fits"]):
63 casuDict[dateVersStr].setdefault(
64 os.path.basename(name), []).extend([
65 int(d5), versDict[int(d5)], access, nol, uid,
66 size, yyyymmdd, time, timezone, name])
67 casuDict["FORMAT"] = ['date', 'version', 'access', 'nol', 'uid',
68 'size', 'yyyymmdd', 'time', 'timezone', 'name']
69 return casuDict
70
71
72
74 """
75 Get the successfully-read directories.
76 """
77
78 cmd = "ssh " + scp_user() + "@" + scp_server() \
79 + " 'find " + direc + " -name SUCCESSFULLY_READ -print'"
80
81
82
83
84 srfh = os.popen(cmd)
85 sr_dirs_n = srfh.readlines()
86 status = srfh.close()
87 if status is None:
88 sr_dirs = sorted(os.path.dirname(x.rstrip()) for x in sr_dirs_n)
89 dirlist = [dirs for dirs in sr_dirs
90 if stdate <= int(os.path.basename(dirs)) <= edate]
91 if not dirlist:
92 make_log(loglines, ' '.join([
93 "Dates from", str(stdate), "to", str(edate),
94 "not available on", direc]), True)
95 else:
96 srDirsDict = createCasuDict(sr_dirs, stdate, edate, versDict)
97 make_log(loglines, ' '.join([
98 "Dates from", str(stdate), "to", str(edate),
99 "found on", direc]), verboseTag)
100 else:
101 make_log(loglines, "ERROR can't read " + direc, True)
102
103 return srDirsDict
104
105
106
108 """
109 Get the version number of the data.
110 """
111
112 cmd = "ssh " + scp_user() + "@" + scp_server() \
113 + " \"find " + direc + " -name 'VERSION_*' -print\""
114 dvfh = os.popen(cmd)
115 vers_dirs_n = dvfh.readlines()
116 status = dvfh.close()
117 vers_dirs_dict = {}
118 if status is None:
119 for x in sorted(vers_dirs_n):
120 x = x.replace('\n', '')
121 date = int(os.path.basename(os.path.dirname(x)))
122 version = os.path.basename(x).lstrip("VERSION_")
123 vers_dirs_dict[date] = version
124 else:
125 make_log(loglines,"ERROR can't read " + direc, True)
126 return vers_dirs_dict, status
127
128
129
130 -def make_log(logList, logText, verbose=False):
131 """Print log to screen and append to list."""
132 if verbose:
133 print logText
134 logList.append(logText)
135 return logList
136
137
138
142
146
147
148
149
150
151 if __name__ == "__main__":
152 try:
153 opts, args = getopt.getopt(sys.argv[1:], "b:e:d:jhr:o:v",
154 ["begin=", "end=", "disks=", "janet",
155 "help", "read=", "outdir", "verbose"])
156 if args:
157 raise getopt.GetoptError
158 except getopt.GetoptError:
159
160 print("ERROR: " + sys.argv)
161 usage()
162 sys.exit(2)
163
164 disklist = []
165 startdate = 0
166 enddate = 99999999
167 outdir = os.curdir
168 scp_casudirs = sysc.casuFileSystem()
169 read = False
170
171 for o, a in opts:
172 if o in ("-b", "--begin"):
173 startdate = int(a)
174 if o in ("-e", "--end"):
175 enddate = int(a)
176 if o in ("-d","--disks"):
177 scp_casudirs = a.split(',')
178 if o in ("-h","--help"):
179 usage()
180 sys.exit(0)
181 if o in ("-o","--outdir"):
182 outdir = a
183 if o in ("-j", "--janet"):
184 light = False
185 if o in ("-r", "--read"):
186 inFileName = a
187 read = True
188 if o in ("-v", "--verbose"):
189 verboseTag = True
190
191 utils.ensureDirExist(outdir)
192
193 casuDirs = {}
194 if not read:
195
196 casuversdict = {}
197 make_log(loglines, "Reading version info...", True)
198 nonAvailDisks = []
199 for direc in scp_casudirs:
200 versions, status = readCasuVersion(direc)
201 casuversdict.update(versions)
202 if status:
203 nonAvailDisks.append(direc)
204 scp_casudirs = list(set(scp_casudirs).difference(nonAvailDisks))
205
206 for direc in scp_casudirs:
207 make_log(loglines, "Checking " + direc + " ...", True)
208
209 casuDirs.setdefault(direc, {}).update(readSrDirs(direc,
210 startdate,
211 enddate,
212 casuversdict))
213 casuDirsLogFile = PickleFile("casuDirsStats_%s_%s.log" %
214 (startdate, enddate))
215 casuDirsLogFile.pickleWrite(casuDirs)
216 else:
217 print "reading", inFileName, "..."
218 casuDirsLogFile = PickleFile(inFileName)
219 casuDirs = list(casuDirsLogFile.pickleRead())[0]
220 print casuDirs.keys()
221 casuFileDict = {}
222 for disk in casuDirs:
223 format = casuDirs[disk].pop("FORMAT")
224 print disk, format
225 for dateVersStr in casuDirs[disk]:
226 casuFileDict[dateVersStr] = casuDirs[disk][dateVersStr]
227 print dateVersStr, ':', len(casuFileDict[dateVersStr])
228 print casuFileDict[dateVersStr]
229
230
231
232
233
234