common/pyradlib/pyrad_proc.py

# -*- coding: utf-8 -*-
''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
2016 - Georg Mischler

Use as:
        from pyradlib.pyrad_proc import PIPE, Error, ProcMixin

For a single-file installation, include the contents of this file
at the same place (minus the __future__ import below).
'''
from __future__ import division, print_function, unicode_literals

import sys
import subprocess
PIPE = subprocess.PIPE


class Error(Exception): pass


class ProcMixin():
        '''Process and pipeline management for Python Radiance scripts
        '''
        def raise_on_error(self, actstr, e):
                raise Error('Unable to %s - %s' % (actstr, str(e)))

        def __configure_subprocess(self):
                '''Prevent subprocess module failure in frozen scripts on Windows.
                   Prevent console windows from popping up when not console based.
                   Make sure we use the version-specific string types.
                '''
                # On Windows, sys.stdxxx may not be available when:
                # - built as *.exe with "pyinstaller --noconsole"
                # - invoked via CreateProcess() and stream not redirected
                try:
                        sys.__stdin__.fileno()
                        self._stdin = sys.stdin
                except: self._stdin = PIPE
                try:
                        sys.__stdout__.fileno()
                        self._stdout = sys.stdout
                except: self._stdout = PIPE
                try:
                        sys.__stderr__.fileno()
                        self._stderr = sys.stderr
                # keep subprocesses from opening their own console.
                except: self._stderr = PIPE
                if hasattr(subprocess, 'STARTUPINFO'):
                        si = subprocess.STARTUPINFO()
                        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                        self._pipeargs = {'startupinfo':si}
                else: self._pipeargs = {}
                # type names vary between Py2.7 and 3.x
                self._strtypes = (type(b''), type(u''))
                # private attribute to indicate established configuration
                self.__proc_mixin_setup = True

        def qjoin(self, sl):
                '''Join a list with quotes around each element containing whitespace.
                We only use this to display command lines on sys.stderr, the actual
                Popen() calls are made with the original list.
                '''
                def _q(s):
                        if ' ' in s or '\t' in s or ';' in s:
                                return "'" + s + "'"
                        return s
                return  ' '.join([_q(s) for s in sl])

        def __parse_args(self, _in, out):
                try: self.__proc_mixin_setup
                except AttributeError: self.__configure_subprocess()
                instr = ''
                if _in == PIPE:
                        stdin = _in
                elif isinstance(_in, self._strtypes):
                        stdin = open(_in, 'rb')
                        instr = ' < "%s"' % _in
                elif hasattr(_in, 'read'):
                        stdin = _in
                        instr = ' < "%s"' % _in.name
                else: stdin = self._stdin
                outstr = ''
                if out == PIPE:
                        stdout = out
                elif isinstance(out, self._strtypes):
                        stdout = open(out, 'wb')
                        outstr = ' > "%s"' % out
                elif hasattr(out, 'write'):
                        stdout = out
                        outstr = ' > "%s"' % out.name
                else: stdout = self._stdout
                return stdin, stdout, instr, outstr

        def call_one(self, cmdl, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a single subprocess, possibly with an incoming and outgoing
                pipe at each end.
                - cmdl
                  A list of strings, leading with the name of the executable (without
                  the *.exe suffix), followed by individual arguments.
                - actstr
                  A text string of the form "do something".
                  Used in verbose mode as "### do someting\\n### [command line]"
                  Used in error messages as "Scriptname: Unable to do something".
                - _in / out
                  What to do with the input and output pipes of the process:
                  * a filename as string
                    Open file and use for reading/writing.
                  * a file like object
                    Use for reading/writing
                  * PIPE
                    Pipe will be available in returned object for reading/writing.
                  * None (default)
                    System stdin/stdout will be used if available
                If _in or out is a PIPE, the caller should call p.wait() on the
                returned Popen instance after writing to and closing it.
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try: p = subprocess.Popen(cmdl,
                                        stdin=stdin, stdout=stdout, stderr=self._stderr,
                                        universal_newlines=universal_newlines, **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p.wait()
                                if res != 0:
                                        self.raise_on_error(actstr,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl)+instr+outstr+'\n'))
                        return p

        def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
                        universal_newlines=False):
                '''Create two processes, chained via a pipe, possibly with an incoming
                and outgoing pipe at each end.
                Returns a tuple of two Popen instances.
                Arguments are equivalent to call_one(), with _in and out applying
                to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on both
                returned popen instances after writing to and closing the first on .
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr_1)
                        sys.stderr.write('### %s \n' % actstr_2)
                        sys.stderr.write(self.qjoin(cmdl_1) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try: p1 = subprocess.Popen(cmdl_1,
                                        stdin=stdin, stdout=PIPE, stderr=self._stderr,
                                        **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr_1, str(e))
                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                p2 = subprocess.Popen(cmdl_2,
                                                stdin=p1.stdout, stdout=stdout, stderr=self._stderr, 
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                p1.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr_2, str(e))
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p1.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_1,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_1)))
                                res = p2.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_2,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_2)))
                        return p1, p2

        def call_many(self, cmdlines, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a series of N processes, chained via pipes, possibly with an
                incoming and outgoing pipe at each end.
                Returns a tuple of N subprocess.Popen instances.
                Depending on the values of _in and out, the first and last may be
                available to write to or read from respectively.
                Most arguments are equivalent to call_one(), with
                - cmdlines
                  a list of N command argument lists
                - _in / out
                  applying to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on all
                returned Popen instances after writing to and closing the first one.
                '''
                if len(cmdlines) == 1:
                        # other than direct call_one(), this returns a one-item tuple!
                        return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
                                        universal_newlines=universal_newlines),)
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                procs = []
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try:
                                prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
                                                stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                procs.append(prevproc)
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))

                for cmdl in cmdlines[1:-1]:
                        if getattr(self, 'verbose', None):
                                sys.stderr.write(self.qjoin(cmdl) + ' | ')
                        if not getattr(self, 'donothing', None):
                                try:
                                        nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
                                                        stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                        procs.append(nextproc)
                                        prevproc.stdout.close()
                                        prevproc = nextproc
                                except Exception as e:
                                        self.raise_on_error(actstr, str(e))

                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
                                                stdout=stdout, stderr=self._stderr,
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                prevproc.stdout.close()
                                procs.append(lastproc)
                                prevproc.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))

                        if stdin != PIPE and stdout!= PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                for proc, cmdl in zip(procs, cmdlines):
                                        res = proc.wait()
                                        if res != 0:
                                                self.raise_on_error(actstr,
                                                                'Nonzero exit (%d) from command [%s].'
                                                                % (res, self.qjoin(cmdl)))
                        return procs


### end of proc_mixin.py
Revision:	1.2
Committed:	Wed Mar 30 18:02:06 2016 UTC (8 years, 2 months ago) by schorsch
Content type:	text/x-python
Branch:	MAIN
Changes since 1.1:	+42 -59 lines
Log Message:	unify argument parsing
#	Content
1	# -- coding: utf-8 --
2	''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
3	2016 - Georg Mischler
4
5	Use as:
6	from pyradlib.pyrad_proc import PIPE, Error, ProcMixin
7
8	For a single-file installation, include the contents of this file
9	at the same place (minus the __future__ import below).
10	'''
11	from __future__ import division, print_function, unicode_literals
12
13	import sys
14	import subprocess
15	PIPE = subprocess.PIPE
16
17
18	class Error(Exception): pass
19
20
21	class ProcMixin():
22	'''Process and pipeline management for Python Radiance scripts
23	'''
24	def raise_on_error(self, actstr, e):
25	raise Error('Unable to %s - %s' % (actstr, str(e)))
26
27	def __configure_subprocess(self):
28	'''Prevent subprocess module failure in frozen scripts on Windows.
29	Prevent console windows from popping up when not console based.
30	Make sure we use the version-specific string types.
31	'''
32	# On Windows, sys.stdxxx may not be available when:
33	# - built as *.exe with "pyinstaller --noconsole"
34	# - invoked via CreateProcess() and stream not redirected
35	try:
36	sys.__stdin__.fileno()
37	self._stdin = sys.stdin
38	except: self._stdin = PIPE
39	try:
40	sys.__stdout__.fileno()
41	self._stdout = sys.stdout
42	except: self._stdout = PIPE
43	try:
44	sys.__stderr__.fileno()
45	self._stderr = sys.stderr
46	# keep subprocesses from opening their own console.
47	except: self._stderr = PIPE
48	if hasattr(subprocess, 'STARTUPINFO'):
49	si = subprocess.STARTUPINFO()
50	si.dwFlags \|= subprocess.STARTF_USESHOWWINDOW
51	self._pipeargs = {'startupinfo':si}
52	else: self._pipeargs = {}
53	# type names vary between Py2.7 and 3.x
54	self._strtypes = (type(b''), type(u''))
55	# private attribute to indicate established configuration
56	self.__proc_mixin_setup = True
57
58	def qjoin(self, sl):
59	'''Join a list with quotes around each element containing whitespace.
60	We only use this to display command lines on sys.stderr, the actual
61	Popen() calls are made with the original list.
62	'''
63	def _q(s):
64	if ' ' in s or '\t' in s or ';' in s:
65	return "'" + s + "'"
66	return s
67	return ' '.join([_q(s) for s in sl])
68
69	def __parse_args(self, _in, out):
70	try: self.__proc_mixin_setup
71	except AttributeError: self.__configure_subprocess()
72	instr = ''
73	if _in == PIPE:
74	stdin = _in
75	elif isinstance(_in, self._strtypes):
76	stdin = open(_in, 'rb')
77	instr = ' < "%s"' % _in
78	elif hasattr(_in, 'read'):
79	stdin = _in
80	instr = ' < "%s"' % _in.name
81	else: stdin = self._stdin
82	outstr = ''
83	if out == PIPE:
84	stdout = out
85	elif isinstance(out, self._strtypes):
86	stdout = open(out, 'wb')
87	outstr = ' > "%s"' % out
88	elif hasattr(out, 'write'):
89	stdout = out
90	outstr = ' > "%s"' % out.name
91	else: stdout = self._stdout
92	return stdin, stdout, instr, outstr
93
94	def call_one(self, cmdl, actstr, _in=None, out=None,
95	universal_newlines=False):
96	'''Create a single subprocess, possibly with an incoming and outgoing
97	pipe at each end.
98	- cmdl
99	A list of strings, leading with the name of the executable (without
100	the *.exe suffix), followed by individual arguments.
101	- actstr
102	A text string of the form "do something".
103	Used in verbose mode as "### do someting\\n### [command line]"
104	Used in error messages as "Scriptname: Unable to do something".
105	- _in / out
106	What to do with the input and output pipes of the process:
107	* a filename as string
108	Open file and use for reading/writing.
109	* a file like object
110	Use for reading/writing
111	* PIPE
112	Pipe will be available in returned object for reading/writing.
113	* None (default)
114	System stdin/stdout will be used if available
115	If _in or out is a PIPE, the caller should call p.wait() on the
116	returned Popen instance after writing to and closing it.
117	'''
118	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
119	if getattr(self, 'verbose', None):
120	sys.stderr.write('### %s \n' % actstr)
121	sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
122	if not getattr(self, 'donothing', None):
123	try: p = subprocess.Popen(cmdl,
124	stdin=stdin, stdout=stdout, stderr=self._stderr,
125	universal_newlines=universal_newlines, **self._pipeargs)
126	except Exception as e:
127	self.raise_on_error(actstr, str(e))
128	if stdin != PIPE and stdout != PIPE:
129	# caller needs to wait after reading or writing (else deadlock)
130	res = p.wait()
131	if res != 0:
132	self.raise_on_error(actstr,
133	'Nonzero exit (%d) from command [%s].'
134	% (res, self.qjoin(cmdl)+instr+outstr+'\n'))
135	return p
136
137	def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
138	universal_newlines=False):
139	'''Create two processes, chained via a pipe, possibly with an incoming
140	and outgoing pipe at each end.
141	Returns a tuple of two Popen instances.
142	Arguments are equivalent to call_one(), with _in and out applying
143	to the ends of the chain.
144	If _in or out is PIPE, the caller should call p.wait() on both
145	returned popen instances after writing to and closing the first on .
146	'''
147	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
148	if getattr(self, 'verbose', None):
149	sys.stderr.write('### %s \n' % actstr_1)
150	sys.stderr.write('### %s \n' % actstr_2)
151	sys.stderr.write(self.qjoin(cmdl_1) + instr + ' \| ')
152	if not getattr(self, 'donothing', None):
153	try: p1 = subprocess.Popen(cmdl_1,
154	stdin=stdin, stdout=PIPE, stderr=self._stderr,
155	**self._pipeargs)
156	except Exception as e:
157	self.raise_on_error(actstr_1, str(e))
158	if getattr(self, 'verbose', None):
159	sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
160	if not getattr(self, 'donothing', None):
161	try:
162	p2 = subprocess.Popen(cmdl_2,
163	stdin=p1.stdout, stdout=stdout, stderr=self._stderr,
164	universal_newlines=universal_newlines, **self._pipeargs)
165	p1.stdout.close()
166	except Exception as e:
167	self.raise_on_error(actstr_2, str(e))
168	if stdin != PIPE and stdout != PIPE:
169	# caller needs to wait after reading or writing (else deadlock)
170	res = p1.wait()
171	if res != 0:
172	self.raise_on_error(actstr_1,
173	'Nonzero exit (%d) from command [%s].'
174	% (res, self.qjoin(cmdl_1)))
175	res = p2.wait()
176	if res != 0:
177	self.raise_on_error(actstr_2,
178	'Nonzero exit (%d) from command [%s].'
179	% (res, self.qjoin(cmdl_2)))
180	return p1, p2
181
182	def call_many(self, cmdlines, actstr, _in=None, out=None,
183	universal_newlines=False):
184	'''Create a series of N processes, chained via pipes, possibly with an
185	incoming and outgoing pipe at each end.
186	Returns a tuple of N subprocess.Popen instances.
187	Depending on the values of _in and out, the first and last may be
188	available to write to or read from respectively.
189	Most arguments are equivalent to call_one(), with
190	- cmdlines
191	a list of N command argument lists
192	- _in / out
193	applying to the ends of the chain.
194	If _in or out is PIPE, the caller should call p.wait() on all
195	returned Popen instances after writing to and closing the first one.
196	'''
197	if len(cmdlines) == 1:
198	# other than direct call_one(), this returns a one-item tuple!
199	return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
200	universal_newlines=universal_newlines),)
201	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
202	procs = []
203	if getattr(self, 'verbose', None):
204	sys.stderr.write('### %s \n' % actstr)
205	sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' \| ')
206	if not getattr(self, 'donothing', None):
207	try:
208	prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
209	stdout=PIPE, stderr=self._stderr, **self._pipeargs)
210	procs.append(prevproc)
211	except Exception as e:
212	self.raise_on_error(actstr, str(e))
213
214	for cmdl in cmdlines[1:-1]:
215	if getattr(self, 'verbose', None):
216	sys.stderr.write(self.qjoin(cmdl) + ' \| ')
217	if not getattr(self, 'donothing', None):
218	try:
219	nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
220	stdout=PIPE, stderr=self._stderr, **self._pipeargs)
221	procs.append(nextproc)
222	prevproc.stdout.close()
223	prevproc = nextproc
224	except Exception as e:
225	self.raise_on_error(actstr, str(e))
226
227	if getattr(self, 'verbose', None):
228	sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
229	if not getattr(self, 'donothing', None):
230	try:
231	lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
232	stdout=stdout, stderr=self._stderr,
233	universal_newlines=universal_newlines, **self._pipeargs)
234	prevproc.stdout.close()
235	procs.append(lastproc)
236	prevproc.stdout.close()
237	except Exception as e:
238	self.raise_on_error(actstr, str(e))
239
240	if stdin != PIPE and stdout!= PIPE:
241	# caller needs to wait after reading or writing (else deadlock)
242	for proc, cmdl in zip(procs, cmdlines):
243	res = proc.wait()
244	if res != 0:
245	self.raise_on_error(actstr,
246	'Nonzero exit (%d) from command [%s].'
247	% (res, self.qjoin(cmdl)))
248	return procs
249
250
251	### end of proc_mixin.py