1
2
3 """
4 Run external processes with logging of message output.
5
6 Usage
7 =====
8
9 To run an external process, e.g. C{mkmerge}, in an "sh" shell, with just
10 error messages logged::
11 import wsatools.ExternalProcess as extp
12 try:
13 extp.run("mkmerge -opt=1 arg")
14 except ExternalProcess.Error as error:
15 Logger.addExceptionMessage(error)
16 raise SystemExit
17
18 However, you don't always need to do this and it is faster to not run in an
19 shell. To avoid running the shell, supply the command as a list of strings
20 (with just a single item if there are no arguments)::
21
22 extp.run(["mkmerge", "-opt=1", "arg"])
23
24 To disable an ExternalProcess.Error exception from being raised if any
25 output is sent to stdErr (i.e. there is an error), just issue this keyword
26 argument::
27
28 raiseOnError=False
29
30 This is important for certain CASU programs, for example, that send normal
31 output to stdErr. In this case, stderr is redirected to stdout. Note that
32 ExternalProcess will always raise OSError exceptions when they occur
33 irrespective of this setting.
34
35 Turn on various L{run()} options to enhance logging features. The full log
36 as a list of lines is always returned, but doesn't need to be assigned to
37 a variable. If you wish to inspect the program's terminal output, either
38 do::
39
40 output = extp.run("mkmerge -opt=1 arg")
41
42 or::
43
44 for line in extp.run("mkmerge -opt=1 arg"):
45 Logger.addMessage(line)
46
47 The output is always stripped. To get iterable raw output in a direct
48 replacement for os.popen() use the L{out()} method. However, this should
49 only be used when output to stderr does not need to be monitored for
50 exception purposes::
51
52 for line in extp.out("ls"):
53 Logger.addMessage(line)
54
55 To add/change any environment variables for just this run of the command,
56 issue this keyword argument::
57
58 env={"ENV_VARIABLE": "value", ...}
59
60 If you want to parse return codes then both parseStdOut and raiseOnError
61 must be set to False, or else access subprocess.Popen directly. In general,
62 it's better to parse stdErr than return codes, since it contains more
63 information and if a command fails then it generally sends something to
64 stdErr anyway. However, this does depend on your particular command.
65
66 @author: R.S. Collins
67 @org: WFAU, IfA, University of Edinburgh
68
69 @newfield contributors: Contributors, Contributors (Alphabetical Order)
70 @contributors: I.A. Bond, E. Sutorius
71
72 @note: Always try to replace most standard shell commands, e.g. ls, rm, mv,
73 cp, chmod etc. with shutil/glob/os equivalents instead of using this
74 module.
75
76 @todo: If we really want to be pedantic we should enclose the Popen call in
77 a with-block, e.g. "with Popen() as proc:".
78 """
79
80 from __future__ import division, print_function
81
82 from subprocess import Popen, PIPE, STDOUT
83 import time
84
85 from wsatools.Logger import Logger
86
87
89 """ Exception thrown if some problem occurs when running the external
90 program, e.g. it crashes.
91 """
92 stdErr = None
93
95 """
96 @param msg: Error message.
97 @type msg: str
98 @param stdErr: List of lines sent to stdErr.
99 @type stdErr: list(str)
100
101 """
102 self.stdErr = stdErr
103 super(Error, self).__init__(msg)
104
105
106
107 -def out(command, stdIn='', cwd=None, env=None, close_fds=True,
108 isVerbose=True):
109 """
110 Straight replacement for iterative parsing of os.popen(), it's just
111 L{run()} with a simplified interface, so see this function for description.
112
113 @note: This method cannot raise exceptions if there is output to stdErr, so
114 you should always use L{run()} if this is required.
115
116 @return: Iterable file object for stdout.
117 @rtype: file
118
119 """
120 return run(command, stdIn, cwd=cwd, env=env, close_fds=close_fds,
121 isVerbose=isVerbose, _isIterable=True)
122
123
124
125 -def run(command, stdIn='', raiseOnError=True, parseStdOut=True, cwd=None,
126 env=None, close_fds=True, isVerbose=True, _isIterable=False,
127 ignoreMsgs=None):
128 """
129 Run the given external program. Unless overridden, an exception is thrown
130 on error and the output is logged.
131
132 @param command: Command string to execute. Use a single string with
133 all arguments to run in shell, use a list of the
134 command with arguments as separate elements to not
135 run in a shell (faster if shell not needed).
136 @type command: str or list(str)
137 @param stdIn: Optionally supply some input for stdin.
138 @type stdIn: str
139 @param raiseOnError: If True, if the external process sends anything to
140 stdErr then an exception is raised and the complete
141 programme is logged. Otherwise, stdErr is just always
142 redirected to stdOut.
143 @type raiseOnError: bool
144 @param parseStdOut: If True, stdout is captured, not print to screen, and
145 returned by this function, otherwise stdout is left
146 alone and will be sent to terminal as normal.
147 @type parseStdOut: bool
148 @param cwd: Run the external process with this directory as its
149 working directory.
150 @type cwd: str
151 @param env: Environment variables for the external process.
152 @type env: dict(str:str)
153 @param close_fds: If True, close all open file-like objects before
154 executing external process.
155 @type close_fds: bool
156 @param isVerbose: If False, don't log the full command that was
157 executed, even when Logger is in verbose mode.
158 @type isVerbose: bool
159 @param _isIterable: Return an iterable stdout. NB: Use the L{out()}
160 function instead of this option.
161 @type _isIterable: bool
162 @param ignoreMsgs: List of strings that if they appear in stderr should
163 override the raiseOnError if it is set to True.
164 @type ignoreMsgs: list(str)
165
166 @return: Messages sent to stdout if parsed, otherwise an iterable file
167 object for stdout if _isIterable, else a return a code.
168 @rtype: list(str) or file or int
169
170 """
171 cmdStr = (command if isinstance(command, str) else ' '.join(command))
172 if isVerbose:
173 Logger.addMessage(cmdStr, alwaysLog=False)
174
175 parseStdOut = parseStdOut or _isIterable
176 parseStdErr = raiseOnError and not _isIterable
177 isMemError = False
178 while True:
179 try:
180 proc = Popen(command, shell=isinstance(command, str),
181 stdin=(PIPE if stdIn else None),
182 stdout=(PIPE if parseStdOut else None),
183 stderr=(PIPE if parseStdErr else STDOUT),
184 close_fds=close_fds, cwd=cwd, env=env)
185 except OSError as error:
186 if "[Errno 12] Cannot allocate memory" not in str(error):
187 raise
188 if not isMemError:
189 Logger.addMessage("Memory allocation problem; delaying...")
190 isMemError = True
191 close_fds = True
192 time.sleep(60)
193 else:
194 if isMemError:
195 Logger.addMessage("Problem fixed; continuing...")
196 break
197
198 if stdIn:
199 proc.stdin.write(stdIn + '\n')
200 proc.stdin.flush()
201
202 if _isIterable:
203 return proc.stdout
204
205 stdOut = []
206 stdErr = []
207 try:
208 if parseStdOut:
209
210
211 stdOut = [line.strip() for line in proc.stdout.readlines()]
212 if raiseOnError:
213 stdErr = [line.strip() for line in proc.stderr]
214
215 if not parseStdOut and not raiseOnError:
216 return proc.wait()
217
218
219
220
221
222
223
224
225
226
227
228 except IOError as error:
229
230
231
232 if "Interrupted system call" in str(error):
233 raise KeyboardInterrupt
234 raise
235
236
237 if stdErr and ignoreMsgs:
238 for stdErrStr in stdErr[:]:
239 if any(msg in stdErrStr for msg in ignoreMsgs):
240 stdErr.remove(stdErrStr)
241
242 if stdErr:
243 Logger.setEchoOn()
244 if raiseOnError and (not isVerbose or not Logger.isVerbose):
245 Logger.addMessage(cmdStr)
246
247 for line in stdOut:
248 Logger.addMessage(line)
249
250 for line in stdErr:
251 Logger.addMessage('# ' + line)
252
253 if raiseOnError:
254 cmd = cmdStr.split(';')[-1].split()[0]
255 if cmd == "python":
256 cmd = ' '.join(cmdStr.split()[:2])
257
258 raise Error(cmd + " failed", stdErr)
259
260 return stdOut
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293