common/pyradlib/pyrad_proc.py

# -*- coding: utf-8 -*-
''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
2016 - Georg Mischler

Use as:
        from pyradlib.pyrad_proc import PIPE, Error, ProcMixin

For a single-file installation, include the contents of this file
at the same place (minus the __future__ import below).
'''
from __future__ import division, print_function, unicode_literals

import sys
import subprocess
PIPE = subprocess.PIPE


class Error(Exception): pass


class ProcMixin():
        '''Process and pipeline management for Python Radiance scripts
        '''
        def raise_on_error(self, actstr, e):
                if hasattr(e, 'strerror'): eb = e.strerror
                elif isinstance(e, self._strtypes): eb = e
                else: eb = e
                if isinstance(eb, type(b'')):
                        estr = eb.decode(encoding='utf-8', errors='ignore')
                else: estr = eb
                raise Error('Unable to %s - %s' % (actstr, estr)) #

        def __configure_subprocess(self):
                '''Prevent subprocess module failure in frozen scripts on Windows.
                   Prevent console windows from popping up when not console based.
                   Make sure we use the version-specific string types.
                '''
                # On Windows, sys.stdxxx may not be available when:
                # - built as *.exe with "pyinstaller --noconsole"
                # - invoked via CreateProcess() and stream not redirected
                try:
                        sys.__stdin__.fileno()
                        self._stdin = sys.stdin
                except: self._stdin = PIPE
                try:
                        sys.__stdout__.fileno()
                        self._stdout = sys.stdout
                except: self._stdout = PIPE
                try:
                        sys.__stderr__.fileno()
                        self._stderr = sys.stderr
                # keep subprocesses from opening their own console.
                except: self._stderr = PIPE
                if hasattr(subprocess, 'STARTUPINFO'):
                        si = subprocess.STARTUPINFO()
                        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                        self._pipeargs = {'startupinfo':si}
                else: self._pipeargs = {}
                # type names vary between Py2.7 and 3.x
                self._strtypes = (type(b''), type(u''))
                # private attribute to indicate established configuration
                self.__proc_mixin_setup = True

        def qjoin(self, sl):
                '''Join a list with quotes around each element containing whitespace.
                We only use this to display command lines on sys.stderr, the actual
                Popen() calls are made with the original list.
                '''
                def _q(s):
                        if ' ' in s or '\t' in s or ';' in s:
                                return "'" + s + "'"
                        return s
                return  ' '.join([_q(s) for s in sl])

        def __parse_args(self, _in, out):
                try: self.__proc_mixin_setup
                except AttributeError: self.__configure_subprocess()
                instr = ''
                if _in == PIPE:
                        stdin = _in
                elif isinstance(_in, self._strtypes):
                        if self.donothing: stdin = None
                        else: stdin = open(_in, 'rb')
                        instr = ' < "%s"' % _in
                elif hasattr(_in, 'read'):
                        stdin = _in
                        instr = ' < "%s"' % _in.name
                else: stdin = self._stdin
                outstr = ''
                if out == PIPE:
                        stdout = out
                elif isinstance(out, self._strtypes):
                        if self.donothing: stdout = None
                        else: stdout = open(out, 'wb')
                        outstr = ' > "%s"' % out
                elif hasattr(out, 'write'):
                        stdout = out
                        outstr = ' > "%s"' % out.name
                else: stdout = self._stdout
                return stdin, stdout, instr, outstr

        def call_one(self, cmdl, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a single subprocess, possibly with an incoming and outgoing
                pipe at each end.
                - cmdl
                  A list of strings, leading with the name of the executable (without
                  the *.exe suffix), followed by individual arguments.
                - actstr
                  A text string of the form "do something".
                  Used in verbose mode as "### do someting\\n### [command line]"
                  Used in error messages as "Scriptname: Unable to do something".
                - _in / out
                  What to do with the input and output pipes of the process:
                  * a filename as string
                    Open file and use for reading/writing.
                  * a file like object
                    Use for reading/writing
                  * PIPE
                    Pipe will be available in returned object for reading/writing.
                  * None (default)
                    System stdin/stdout will be used if available
                If _in or out is a PIPE, the caller should call p.wait() on the
                returned Popen instance after writing to and closing it.
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try: p = subprocess.Popen(cmdl,
                                        stdin=stdin, stdout=stdout, stderr=self._stderr,
                                        universal_newlines=universal_newlines, **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr, e)
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p.wait()
                                if res != 0:
                                        self.raise_on_error(actstr,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl)+instr+outstr+'\n'))
                        return p

        def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
                        universal_newlines=False):
                '''Create two processes, chained via a pipe, possibly with an incoming
                and outgoing pipe at each end.
                Returns a tuple of two Popen instances.
                Arguments are equivalent to call_one(), with _in and out applying
                to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on both
                returned popen instances after writing to and closing the first on .
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr_1)
                        sys.stderr.write('### %s \n' % actstr_2)
                        sys.stderr.write(self.qjoin(cmdl_1) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try: p1 = subprocess.Popen(cmdl_1,
                                        stdin=stdin, stdout=PIPE, stderr=self._stderr,
                                        **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr_1, e)
                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                p2 = subprocess.Popen(cmdl_2,
                                                stdin=p1.stdout, stdout=stdout, stderr=self._stderr, 
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                p1.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr_2, e)
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p1.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_1,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_1)))
                                res = p2.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_2,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_2)))
                        return p1, p2

        def call_many(self, cmdlines, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a series of N processes, chained via pipes, possibly with an
                incoming and outgoing pipe at each end.
                Returns a tuple of N subprocess.Popen instances.
                Depending on the values of _in and out, the first and last may be
                available to write to or read from respectively.
                Most arguments are equivalent to call_one(), with
                - cmdlines
                  a list of N command argument lists
                - _in / out
                  applying to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on all
                returned Popen instances after writing to and closing the first one.
                '''
                if len(cmdlines) == 1:
                        # other than direct call_one(), this returns a one-item tuple!
                        return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
                                        universal_newlines=universal_newlines),)
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                procs = []
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try:
                                prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
                                                stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                procs.append(prevproc)
                        except Exception as e:
                                self.raise_on_error(actstr, e)

                for cmdl in cmdlines[1:-1]:
                        if getattr(self, 'verbose', None):
                                sys.stderr.write(self.qjoin(cmdl) + ' | ')
                        if not getattr(self, 'donothing', None):
                                try:
                                        nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
                                                        stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                        procs.append(nextproc)
                                        prevproc.stdout.close()
                                        prevproc = nextproc
                                except Exception as e:
                                        self.raise_on_error(actstr, e)

                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
                                                stdout=stdout, stderr=self._stderr,
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                prevproc.stdout.close()
                                procs.append(lastproc)
                                prevproc.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr, e)

                        if stdin != PIPE and stdout!= PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                for proc, cmdl in zip(procs, cmdlines):
                                        res = proc.wait()
                                        if res != 0:
                                                self.raise_on_error(actstr,
                                                                'Nonzero exit (%d) from command [%s].'
                                                                % (res, self.qjoin(cmdl)))
                        return procs


### end of proc_mixin.py
Revision:	1.3
Committed:	Sat Apr 2 15:52:48 2016 UTC (9 years, 3 months ago) by schorsch
Content type:	text/x-python
Branch:	MAIN
Changes since 1.2:	+17 -9 lines
Log Message:	more robustness
#	Content
1	# -- coding: utf-8 --
2	''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
3	2016 - Georg Mischler
4
5	Use as:
6	from pyradlib.pyrad_proc import PIPE, Error, ProcMixin
7
8	For a single-file installation, include the contents of this file
9	at the same place (minus the __future__ import below).
10	'''
11	from __future__ import division, print_function, unicode_literals
12
13	import sys
14	import subprocess
15	PIPE = subprocess.PIPE
16
17
18	class Error(Exception): pass
19
20
21	class ProcMixin():
22	'''Process and pipeline management for Python Radiance scripts
23	'''
24	def raise_on_error(self, actstr, e):
25	if hasattr(e, 'strerror'): eb = e.strerror
26	elif isinstance(e, self._strtypes): eb = e
27	else: eb = e
28	if isinstance(eb, type(b'')):
29	estr = eb.decode(encoding='utf-8', errors='ignore')
30	else: estr = eb
31	raise Error('Unable to %s - %s' % (actstr, estr)) #
32
33	def __configure_subprocess(self):
34	'''Prevent subprocess module failure in frozen scripts on Windows.
35	Prevent console windows from popping up when not console based.
36	Make sure we use the version-specific string types.
37	'''
38	# On Windows, sys.stdxxx may not be available when:
39	# - built as *.exe with "pyinstaller --noconsole"
40	# - invoked via CreateProcess() and stream not redirected
41	try:
42	sys.__stdin__.fileno()
43	self._stdin = sys.stdin
44	except: self._stdin = PIPE
45	try:
46	sys.__stdout__.fileno()
47	self._stdout = sys.stdout
48	except: self._stdout = PIPE
49	try:
50	sys.__stderr__.fileno()
51	self._stderr = sys.stderr
52	# keep subprocesses from opening their own console.
53	except: self._stderr = PIPE
54	if hasattr(subprocess, 'STARTUPINFO'):
55	si = subprocess.STARTUPINFO()
56	si.dwFlags \|= subprocess.STARTF_USESHOWWINDOW
57	self._pipeargs = {'startupinfo':si}
58	else: self._pipeargs = {}
59	# type names vary between Py2.7 and 3.x
60	self._strtypes = (type(b''), type(u''))
61	# private attribute to indicate established configuration
62	self.__proc_mixin_setup = True
63
64	def qjoin(self, sl):
65	'''Join a list with quotes around each element containing whitespace.
66	We only use this to display command lines on sys.stderr, the actual
67	Popen() calls are made with the original list.
68	'''
69	def _q(s):
70	if ' ' in s or '\t' in s or ';' in s:
71	return "'" + s + "'"
72	return s
73	return ' '.join([_q(s) for s in sl])
74
75	def __parse_args(self, _in, out):
76	try: self.__proc_mixin_setup
77	except AttributeError: self.__configure_subprocess()
78	instr = ''
79	if _in == PIPE:
80	stdin = _in
81	elif isinstance(_in, self._strtypes):
82	if self.donothing: stdin = None
83	else: stdin = open(_in, 'rb')
84	instr = ' < "%s"' % _in
85	elif hasattr(_in, 'read'):
86	stdin = _in
87	instr = ' < "%s"' % _in.name
88	else: stdin = self._stdin
89	outstr = ''
90	if out == PIPE:
91	stdout = out
92	elif isinstance(out, self._strtypes):
93	if self.donothing: stdout = None
94	else: stdout = open(out, 'wb')
95	outstr = ' > "%s"' % out
96	elif hasattr(out, 'write'):
97	stdout = out
98	outstr = ' > "%s"' % out.name
99	else: stdout = self._stdout
100	return stdin, stdout, instr, outstr
101
102	def call_one(self, cmdl, actstr, _in=None, out=None,
103	universal_newlines=False):
104	'''Create a single subprocess, possibly with an incoming and outgoing
105	pipe at each end.
106	- cmdl
107	A list of strings, leading with the name of the executable (without
108	the *.exe suffix), followed by individual arguments.
109	- actstr
110	A text string of the form "do something".
111	Used in verbose mode as "### do someting\\n### [command line]"
112	Used in error messages as "Scriptname: Unable to do something".
113	- _in / out
114	What to do with the input and output pipes of the process:
115	* a filename as string
116	Open file and use for reading/writing.
117	* a file like object
118	Use for reading/writing
119	* PIPE
120	Pipe will be available in returned object for reading/writing.
121	* None (default)
122	System stdin/stdout will be used if available
123	If _in or out is a PIPE, the caller should call p.wait() on the
124	returned Popen instance after writing to and closing it.
125	'''
126	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
127	if getattr(self, 'verbose', None):
128	sys.stderr.write('### %s \n' % actstr)
129	sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
130	if not getattr(self, 'donothing', None):
131	try: p = subprocess.Popen(cmdl,
132	stdin=stdin, stdout=stdout, stderr=self._stderr,
133	universal_newlines=universal_newlines, **self._pipeargs)
134	except Exception as e:
135	self.raise_on_error(actstr, e)
136	if stdin != PIPE and stdout != PIPE:
137	# caller needs to wait after reading or writing (else deadlock)
138	res = p.wait()
139	if res != 0:
140	self.raise_on_error(actstr,
141	'Nonzero exit (%d) from command [%s].'
142	% (res, self.qjoin(cmdl)+instr+outstr+'\n'))
143	return p
144
145	def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
146	universal_newlines=False):
147	'''Create two processes, chained via a pipe, possibly with an incoming
148	and outgoing pipe at each end.
149	Returns a tuple of two Popen instances.
150	Arguments are equivalent to call_one(), with _in and out applying
151	to the ends of the chain.
152	If _in or out is PIPE, the caller should call p.wait() on both
153	returned popen instances after writing to and closing the first on .
154	'''
155	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
156	if getattr(self, 'verbose', None):
157	sys.stderr.write('### %s \n' % actstr_1)
158	sys.stderr.write('### %s \n' % actstr_2)
159	sys.stderr.write(self.qjoin(cmdl_1) + instr + ' \| ')
160	if not getattr(self, 'donothing', None):
161	try: p1 = subprocess.Popen(cmdl_1,
162	stdin=stdin, stdout=PIPE, stderr=self._stderr,
163	**self._pipeargs)
164	except Exception as e:
165	self.raise_on_error(actstr_1, e)
166	if getattr(self, 'verbose', None):
167	sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
168	if not getattr(self, 'donothing', None):
169	try:
170	p2 = subprocess.Popen(cmdl_2,
171	stdin=p1.stdout, stdout=stdout, stderr=self._stderr,
172	universal_newlines=universal_newlines, **self._pipeargs)
173	p1.stdout.close()
174	except Exception as e:
175	self.raise_on_error(actstr_2, e)
176	if stdin != PIPE and stdout != PIPE:
177	# caller needs to wait after reading or writing (else deadlock)
178	res = p1.wait()
179	if res != 0:
180	self.raise_on_error(actstr_1,
181	'Nonzero exit (%d) from command [%s].'
182	% (res, self.qjoin(cmdl_1)))
183	res = p2.wait()
184	if res != 0:
185	self.raise_on_error(actstr_2,
186	'Nonzero exit (%d) from command [%s].'
187	% (res, self.qjoin(cmdl_2)))
188	return p1, p2
189
190	def call_many(self, cmdlines, actstr, _in=None, out=None,
191	universal_newlines=False):
192	'''Create a series of N processes, chained via pipes, possibly with an
193	incoming and outgoing pipe at each end.
194	Returns a tuple of N subprocess.Popen instances.
195	Depending on the values of _in and out, the first and last may be
196	available to write to or read from respectively.
197	Most arguments are equivalent to call_one(), with
198	- cmdlines
199	a list of N command argument lists
200	- _in / out
201	applying to the ends of the chain.
202	If _in or out is PIPE, the caller should call p.wait() on all
203	returned Popen instances after writing to and closing the first one.
204	'''
205	if len(cmdlines) == 1:
206	# other than direct call_one(), this returns a one-item tuple!
207	return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
208	universal_newlines=universal_newlines),)
209	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
210	procs = []
211	if getattr(self, 'verbose', None):
212	sys.stderr.write('### %s \n' % actstr)
213	sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' \| ')
214	if not getattr(self, 'donothing', None):
215	try:
216	prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
217	stdout=PIPE, stderr=self._stderr, **self._pipeargs)
218	procs.append(prevproc)
219	except Exception as e:
220	self.raise_on_error(actstr, e)
221
222	for cmdl in cmdlines[1:-1]:
223	if getattr(self, 'verbose', None):
224	sys.stderr.write(self.qjoin(cmdl) + ' \| ')
225	if not getattr(self, 'donothing', None):
226	try:
227	nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
228	stdout=PIPE, stderr=self._stderr, **self._pipeargs)
229	procs.append(nextproc)
230	prevproc.stdout.close()
231	prevproc = nextproc
232	except Exception as e:
233	self.raise_on_error(actstr, e)
234
235	if getattr(self, 'verbose', None):
236	sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
237	if not getattr(self, 'donothing', None):
238	try:
239	lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
240	stdout=stdout, stderr=self._stderr,
241	universal_newlines=universal_newlines, **self._pipeargs)
242	prevproc.stdout.close()
243	procs.append(lastproc)
244	prevproc.stdout.close()
245	except Exception as e:
246	self.raise_on_error(actstr, e)
247
248	if stdin != PIPE and stdout!= PIPE:
249	# caller needs to wait after reading or writing (else deadlock)
250	for proc, cmdl in zip(procs, cmdlines):
251	res = proc.wait()
252	if res != 0:
253	self.raise_on_error(actstr,
254	'Nonzero exit (%d) from command [%s].'
255	% (res, self.qjoin(cmdl)))
256	return procs
257
258
259	### end of proc_mixin.py