common/pyradlib/pyrad_proc.py

# -*- coding: utf-8 -*-
''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
2016 - Georg Mischler

Use as:
        from pyradlib.pyrad_proc import PIPE, Error, ProcMixin

For a single-file installation, include the contents of this file
at the same place (minus the __future__ import below).
'''
from __future__ import division, print_function, unicode_literals

import sys
import subprocess
PIPE = subprocess.PIPE


class Error(Exception): pass


class ProcMixin():
        '''Process and pipeline management for Python Radiance scripts
        '''
        def raise_on_error(self, actstr, e):
                try: self._strtypes
                except AttributeError: self.__configure_subprocess()
                if hasattr(e, 'strerror'): eb = e.strerror
                elif isinstance(e, self._strtypes): eb = e
                else: eb = e
                if isinstance(eb, type(b'')):
                        estr = eb.decode(encoding='utf-8', errors='ignore')
                else: estr = eb
                raise Error('Unable to %s - %s' % (actstr, estr)) #

        def __configure_subprocess(self):
                '''Prevent subprocess module failure in frozen scripts on Windows.
                   Prevent console windows from popping up when not console based.
                   Make sure we use the version-specific string types.
                '''
                # On Windows, sys.stdxxx may not be available when:
                # - built as *.exe with "pyinstaller --noconsole"
                # - invoked via CreateProcess() and stream not redirected
                try:
                        sys.__stdin__.fileno()
                        self._stdin = sys.stdin
                except: self._stdin = PIPE
                try:
                        sys.__stdout__.fileno()
                        self._stdout = sys.stdout
                except: self._stdout = PIPE
                try:
                        sys.__stderr__.fileno()
                        self._stderr = sys.stderr
                # keep subprocesses from opening their own console.
                except: self._stderr = PIPE
                if hasattr(subprocess, 'STARTUPINFO'):
                        si = subprocess.STARTUPINFO()
                        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                        self._pipeargs = {'startupinfo':si}
                else: self._pipeargs = {}
                # type names vary between Py2.7 and 3.x
                self._strtypes = (type(b''), type(u''))

        def qjoin(self, sl):
                '''Join a list with quotes around each element containing whitespace.
                We only use this to display command lines on sys.stderr, the actual
                Popen() calls are made with the original list.
                '''
                def _q(s):
                        if ' ' in s or '\t' in s or ';' in s:
                                return "'" + s + "'"
                        return s
                return  ' '.join([_q(s) for s in sl])

        def __parse_args(self, _in, out):
                try: self._strtypes
                except AttributeError: self.__configure_subprocess()
                instr = ''
                if _in == PIPE:
                        stdin = _in
                elif isinstance(_in, self._strtypes):
                        if not getattr(self, 'donothing', None):
                                stdin = open(_in, 'rb')
                        else: stdin = None
                        instr = ' < "%s"' % _in
                elif hasattr(_in, 'read'):
                        stdin = _in
                        instr = ' < "%s"' % _in.name
                else: stdin = self._stdin
                outstr = ''
                if out == PIPE:
                        stdout = out
                elif isinstance(out, self._strtypes):
                        if not getattr(self, 'donothing', None):
                                stdout = open(out, 'wb')
                        else: stdout = None
                        outstr = ' > "%s"' % out
                elif hasattr(out, 'write'):
                        stdout = out
                        outstr = ' > "%s"' % out.name
                else: stdout = self._stdout
                return stdin, stdout, instr, outstr

        def call_one(self, cmdl, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a single subprocess, possibly with an incoming and outgoing
                pipe at each end.
                - cmdl
                  A list of strings, leading with the name of the executable (without
                  the *.exe suffix), followed by individual arguments.
                - actstr
                  A text string of the form "do something".
                  Used in verbose mode as "### do someting\\n### [command line]"
                  Used in error messages as "Scriptname: Unable to do something".
                - _in / out
                  What to do with the input and output pipes of the process:
                  * a filename as string
                    Open file and use for reading/writing.
                  * a file like object
                    Use for reading/writing
                  * PIPE
                    Pipe will be available in returned object for reading/writing.
                  * None (default)
                    System stdin/stdout will be used if available
                If _in or out is a PIPE, the caller should call p.wait() on the
                returned Popen instance after writing to and closing it.
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try: p = subprocess.Popen(cmdl,
                                        stdin=stdin, stdout=stdout, stderr=self._stderr,
                                        universal_newlines=universal_newlines, **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr, e)
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p.wait()
                                if res != 0:
                                        self.raise_on_error(actstr,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl)+instr+outstr+'\n'))
                        return p

        def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
                        universal_newlines=False):
                '''Create two processes, chained via a pipe, possibly with an incoming
                and outgoing pipe at each end.
                Returns a tuple of two Popen instances.
                Arguments are equivalent to call_one(), with _in and out applying
                to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on both
                returned popen instances after writing to and closing the first on .
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr_1)
                        sys.stderr.write('### %s \n' % actstr_2)
                        sys.stderr.write(self.qjoin(cmdl_1) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try: p1 = subprocess.Popen(cmdl_1,
                                        stdin=stdin, stdout=PIPE, stderr=self._stderr,
                                        **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr_1, e)
                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                p2 = subprocess.Popen(cmdl_2,
                                                stdin=p1.stdout, stdout=stdout, stderr=self._stderr, 
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                p1.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr_2, e)
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p1.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_1,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_1)))
                                res = p2.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_2,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_2)))
                        return p1, p2

        def call_many(self, cmdlines, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a series of N processes, chained via pipes, possibly with an
                incoming and outgoing pipe at each end.
                Returns a tuple of N subprocess.Popen instances.
                Depending on the values of _in and out, the first and last may be
                available to write to or read from respectively.
                Most arguments are equivalent to call_one(), with
                - cmdlines
                  a list of N command argument lists
                - _in / out
                  applying to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on all
                returned Popen instances after writing to and closing the first one.
                '''
                if len(cmdlines) == 1:
                        # other than direct call_one(), this returns a one-item tuple!
                        return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
                                        universal_newlines=universal_newlines),)
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                procs = []
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try:
                                prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
                                                stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                procs.append(prevproc)
                        except Exception as e:
                                self.raise_on_error(actstr, e)

                for cmdl in cmdlines[1:-1]:
                        if getattr(self, 'verbose', None):
                                sys.stderr.write(self.qjoin(cmdl) + ' | ')
                        if not getattr(self, 'donothing', None):
                                try:
                                        nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
                                                        stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                        procs.append(nextproc)
                                        prevproc.stdout.close()
                                        prevproc = nextproc
                                except Exception as e:
                                        self.raise_on_error(actstr, e)

                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
                                                stdout=stdout, stderr=self._stderr,
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                prevproc.stdout.close()
                                procs.append(lastproc)
                                prevproc.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr, e)

                        if stdin != PIPE and stdout!= PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                for proc, cmdl in zip(procs, cmdlines):
                                        res = proc.wait()
                                        if res != 0:
                                                self.raise_on_error(actstr,
                                                                'Nonzero exit (%d) from command [%s].'
                                                                % (res, self.qjoin(cmdl)))
                        return procs


### end of proc_mixin.py
Revision:	1.5
Committed:	Wed Apr 27 16:29:48 2016 UTC (8 years ago) by schorsch
Content type:	text/x-python
Branch:	MAIN
CVS Tags:	rad5R4, rad5R2, rad5R3, HEAD
Changes since 1.4:	+3 -3 lines
Log Message:	yet some more robustness
#	Content
1	# -- coding: utf-8 --
2	''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
3	2016 - Georg Mischler
4
5	Use as:
6	from pyradlib.pyrad_proc import PIPE, Error, ProcMixin
7
8	For a single-file installation, include the contents of this file
9	at the same place (minus the __future__ import below).
10	'''
11	from __future__ import division, print_function, unicode_literals
12
13	import sys
14	import subprocess
15	PIPE = subprocess.PIPE
16
17
18	class Error(Exception): pass
19
20
21	class ProcMixin():
22	'''Process and pipeline management for Python Radiance scripts
23	'''
24	def raise_on_error(self, actstr, e):
25	try: self._strtypes
26	except AttributeError: self.__configure_subprocess()
27	if hasattr(e, 'strerror'): eb = e.strerror
28	elif isinstance(e, self._strtypes): eb = e
29	else: eb = e
30	if isinstance(eb, type(b'')):
31	estr = eb.decode(encoding='utf-8', errors='ignore')
32	else: estr = eb
33	raise Error('Unable to %s - %s' % (actstr, estr)) #
34
35	def __configure_subprocess(self):
36	'''Prevent subprocess module failure in frozen scripts on Windows.
37	Prevent console windows from popping up when not console based.
38	Make sure we use the version-specific string types.
39	'''
40	# On Windows, sys.stdxxx may not be available when:
41	# - built as *.exe with "pyinstaller --noconsole"
42	# - invoked via CreateProcess() and stream not redirected
43	try:
44	sys.__stdin__.fileno()
45	self._stdin = sys.stdin
46	except: self._stdin = PIPE
47	try:
48	sys.__stdout__.fileno()
49	self._stdout = sys.stdout
50	except: self._stdout = PIPE
51	try:
52	sys.__stderr__.fileno()
53	self._stderr = sys.stderr
54	# keep subprocesses from opening their own console.
55	except: self._stderr = PIPE
56	if hasattr(subprocess, 'STARTUPINFO'):
57	si = subprocess.STARTUPINFO()
58	si.dwFlags \|= subprocess.STARTF_USESHOWWINDOW
59	self._pipeargs = {'startupinfo':si}
60	else: self._pipeargs = {}
61	# type names vary between Py2.7 and 3.x
62	self._strtypes = (type(b''), type(u''))
63
64	def qjoin(self, sl):
65	'''Join a list with quotes around each element containing whitespace.
66	We only use this to display command lines on sys.stderr, the actual
67	Popen() calls are made with the original list.
68	'''
69	def _q(s):
70	if ' ' in s or '\t' in s or ';' in s:
71	return "'" + s + "'"
72	return s
73	return ' '.join([_q(s) for s in sl])
74
75	def __parse_args(self, _in, out):
76	try: self._strtypes
77	except AttributeError: self.__configure_subprocess()
78	instr = ''
79	if _in == PIPE:
80	stdin = _in
81	elif isinstance(_in, self._strtypes):
82	if not getattr(self, 'donothing', None):
83	stdin = open(_in, 'rb')
84	else: stdin = None
85	instr = ' < "%s"' % _in
86	elif hasattr(_in, 'read'):
87	stdin = _in
88	instr = ' < "%s"' % _in.name
89	else: stdin = self._stdin
90	outstr = ''
91	if out == PIPE:
92	stdout = out
93	elif isinstance(out, self._strtypes):
94	if not getattr(self, 'donothing', None):
95	stdout = open(out, 'wb')
96	else: stdout = None
97	outstr = ' > "%s"' % out
98	elif hasattr(out, 'write'):
99	stdout = out
100	outstr = ' > "%s"' % out.name
101	else: stdout = self._stdout
102	return stdin, stdout, instr, outstr
103
104	def call_one(self, cmdl, actstr, _in=None, out=None,
105	universal_newlines=False):
106	'''Create a single subprocess, possibly with an incoming and outgoing
107	pipe at each end.
108	- cmdl
109	A list of strings, leading with the name of the executable (without
110	the *.exe suffix), followed by individual arguments.
111	- actstr
112	A text string of the form "do something".
113	Used in verbose mode as "### do someting\\n### [command line]"
114	Used in error messages as "Scriptname: Unable to do something".
115	- _in / out
116	What to do with the input and output pipes of the process:
117	* a filename as string
118	Open file and use for reading/writing.
119	* a file like object
120	Use for reading/writing
121	* PIPE
122	Pipe will be available in returned object for reading/writing.
123	* None (default)
124	System stdin/stdout will be used if available
125	If _in or out is a PIPE, the caller should call p.wait() on the
126	returned Popen instance after writing to and closing it.
127	'''
128	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
129	if getattr(self, 'verbose', None):
130	sys.stderr.write('### %s \n' % actstr)
131	sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
132	if not getattr(self, 'donothing', None):
133	try: p = subprocess.Popen(cmdl,
134	stdin=stdin, stdout=stdout, stderr=self._stderr,
135	universal_newlines=universal_newlines, **self._pipeargs)
136	except Exception as e:
137	self.raise_on_error(actstr, e)
138	if stdin != PIPE and stdout != PIPE:
139	# caller needs to wait after reading or writing (else deadlock)
140	res = p.wait()
141	if res != 0:
142	self.raise_on_error(actstr,
143	'Nonzero exit (%d) from command [%s].'
144	% (res, self.qjoin(cmdl)+instr+outstr+'\n'))
145	return p
146
147	def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
148	universal_newlines=False):
149	'''Create two processes, chained via a pipe, possibly with an incoming
150	and outgoing pipe at each end.
151	Returns a tuple of two Popen instances.
152	Arguments are equivalent to call_one(), with _in and out applying
153	to the ends of the chain.
154	If _in or out is PIPE, the caller should call p.wait() on both
155	returned popen instances after writing to and closing the first on .
156	'''
157	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
158	if getattr(self, 'verbose', None):
159	sys.stderr.write('### %s \n' % actstr_1)
160	sys.stderr.write('### %s \n' % actstr_2)
161	sys.stderr.write(self.qjoin(cmdl_1) + instr + ' \| ')
162	if not getattr(self, 'donothing', None):
163	try: p1 = subprocess.Popen(cmdl_1,
164	stdin=stdin, stdout=PIPE, stderr=self._stderr,
165	**self._pipeargs)
166	except Exception as e:
167	self.raise_on_error(actstr_1, e)
168	if getattr(self, 'verbose', None):
169	sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
170	if not getattr(self, 'donothing', None):
171	try:
172	p2 = subprocess.Popen(cmdl_2,
173	stdin=p1.stdout, stdout=stdout, stderr=self._stderr,
174	universal_newlines=universal_newlines, **self._pipeargs)
175	p1.stdout.close()
176	except Exception as e:
177	self.raise_on_error(actstr_2, e)
178	if stdin != PIPE and stdout != PIPE:
179	# caller needs to wait after reading or writing (else deadlock)
180	res = p1.wait()
181	if res != 0:
182	self.raise_on_error(actstr_1,
183	'Nonzero exit (%d) from command [%s].'
184	% (res, self.qjoin(cmdl_1)))
185	res = p2.wait()
186	if res != 0:
187	self.raise_on_error(actstr_2,
188	'Nonzero exit (%d) from command [%s].'
189	% (res, self.qjoin(cmdl_2)))
190	return p1, p2
191
192	def call_many(self, cmdlines, actstr, _in=None, out=None,
193	universal_newlines=False):
194	'''Create a series of N processes, chained via pipes, possibly with an
195	incoming and outgoing pipe at each end.
196	Returns a tuple of N subprocess.Popen instances.
197	Depending on the values of _in and out, the first and last may be
198	available to write to or read from respectively.
199	Most arguments are equivalent to call_one(), with
200	- cmdlines
201	a list of N command argument lists
202	- _in / out
203	applying to the ends of the chain.
204	If _in or out is PIPE, the caller should call p.wait() on all
205	returned Popen instances after writing to and closing the first one.
206	'''
207	if len(cmdlines) == 1:
208	# other than direct call_one(), this returns a one-item tuple!
209	return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
210	universal_newlines=universal_newlines),)
211	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
212	procs = []
213	if getattr(self, 'verbose', None):
214	sys.stderr.write('### %s \n' % actstr)
215	sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' \| ')
216	if not getattr(self, 'donothing', None):
217	try:
218	prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
219	stdout=PIPE, stderr=self._stderr, **self._pipeargs)
220	procs.append(prevproc)
221	except Exception as e:
222	self.raise_on_error(actstr, e)
223
224	for cmdl in cmdlines[1:-1]:
225	if getattr(self, 'verbose', None):
226	sys.stderr.write(self.qjoin(cmdl) + ' \| ')
227	if not getattr(self, 'donothing', None):
228	try:
229	nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
230	stdout=PIPE, stderr=self._stderr, **self._pipeargs)
231	procs.append(nextproc)
232	prevproc.stdout.close()
233	prevproc = nextproc
234	except Exception as e:
235	self.raise_on_error(actstr, e)
236
237	if getattr(self, 'verbose', None):
238	sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
239	if not getattr(self, 'donothing', None):
240	try:
241	lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
242	stdout=stdout, stderr=self._stderr,
243	universal_newlines=universal_newlines, **self._pipeargs)
244	prevproc.stdout.close()
245	procs.append(lastproc)
246	prevproc.stdout.close()
247	except Exception as e:
248	self.raise_on_error(actstr, e)
249
250	if stdin != PIPE and stdout!= PIPE:
251	# caller needs to wait after reading or writing (else deadlock)
252	for proc, cmdl in zip(procs, cmdlines):
253	res = proc.wait()
254	if res != 0:
255	self.raise_on_error(actstr,
256	'Nonzero exit (%d) from command [%s].'
257	% (res, self.qjoin(cmdl)))
258	return procs
259
260
261	### end of proc_mixin.py