common/pyradlib/pyrad_proc.py

# -*- coding: utf-8 -*-
''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
2016 - Georg Mischler

Use as:
        from pyradlib.pyrad_proc import PIPE, Error, ProcMixin

For a single-file installation, include the contents of this file
at the same place (minus the __future__ import below).
'''
from __future__ import division, print_function, unicode_literals

import sys
import subprocess
PIPE = subprocess.PIPE


class Error(Exception): pass


class ProcMixin():
        '''Process and pipeline management for Python Radiance scripts
        '''
        def raise_on_error(self, actstr, e):
                raise Error('Unable to %s - %s' % (actstr, str(e)))

        def __configure_subprocess(self):
                '''Prevent subprocess module failure in frozen scripts on Windows.
                   Prevent console windows from popping up when not console based.
                   Make sure we use the version-specific string types.
                '''
                # On Windows, sys.stdxxx may not be available when:
                # - built as *.exe with "pyinstaller --noconsole"
                # - invoked via CreateProcess() and stream not redirected
                try:
                        sys.__stdin__.fileno()
                        self._stdin = sys.stdin
                except: self._stdin = PIPE
                try:
                        sys.__stdout__.fileno()
                        self._stdout = sys.stdout
                except: self._stdout = PIPE
                try:
                        sys.__stderr__.fileno()
                        self._stderr = sys.stderr
                # keep subprocesses from opening their own console.
                except: self._stderr = PIPE
                if hasattr(subprocess, 'STARTUPINFO'):
                        si = subprocess.STARTUPINFO()
                        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                        self._pipeargs = {'startupinfo':si}
                else: self._pipeargs = {}
                # type names vary between Py2.7 and 3.x
                self._strtypes = (type(b''), type(u''))
                # private attribute to indicate established configuration
                self.__proc_mixin_setup = True

        def qjoin(self, sl):
                '''Join a list with quotes around each element containing whitespace.
                We only use this to display command lines on sys.stderr, the actual
                Popen() calls are made with the original list.
                '''
                def _q(s):
                        if ' ' in s or '\t' in s or ';' in s:
                                return "'" + s + "'"
                        return s
                return  ' '.join([_q(s) for s in sl])

        def __parse_args(self, _in, out):
                try: self.__proc_mixin_setup
                except AttributeError: self.__configure_subprocess()
                instr = ''
                if _in == PIPE:
                        stdin = _in
                elif isinstance(_in, self._strtypes):
                        stdin = open(_in, 'rb')
                        instr = ' < "%s"' % _in
                elif hasattr(_in, 'read'):
                        stdin = _in
                        instr = ' < "%s"' % _in.name
                else: stdin = self._stdin
                outstr = ''
                if out == PIPE:
                        stdout = out
                elif isinstance(out, self._strtypes):
                        stdout = open(out, 'wb')
                        outstr = ' > "%s"' % out
                elif hasattr(out, 'write'):
                        stdout = out
                        outstr = ' > "%s"' % out.name
                else: stdout = self._stdout
                return stdin, stdout, instr, outstr

        def call_one(self, cmdl, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a single subprocess, possibly with an incoming and outgoing
                pipe at each end.
                - cmdl
                  A list of strings, leading with the name of the executable (without
                  the *.exe suffix), followed by individual arguments.
                - actstr
                  A text string of the form "do something".
                  Used in verbose mode as "### do someting\\n### [command line]"
                  Used in error messages as "Scriptname: Unable to do something".
                - _in / out
                  What to do with the input and output pipes of the process:
                  * a filename as string
                    Open file and use for reading/writing.
                  * a file like object
                    Use for reading/writing
                  * PIPE
                    Pipe will be available in returned object for reading/writing.
                  * None (default)
                    System stdin/stdout will be used if available
                If _in or out is a PIPE, the caller should call p.wait() on the
                returned Popen instance after writing to and closing it.
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try: p = subprocess.Popen(cmdl,
                                        stdin=stdin, stdout=stdout, stderr=self._stderr,
                                        universal_newlines=universal_newlines, **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p.wait()
                                if res != 0:
                                        self.raise_on_error(actstr,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl)+instr+outstr+'\n'))
                        return p

        def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
                        universal_newlines=False):
                '''Create two processes, chained via a pipe, possibly with an incoming
                and outgoing pipe at each end.
                Returns a tuple of two Popen instances.
                Arguments are equivalent to call_one(), with _in and out applying
                to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on both
                returned popen instances after writing to and closing the first on .
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr_1)
                        sys.stderr.write('### %s \n' % actstr_2)
                        sys.stderr.write(self.qjoin(cmdl_1) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try: p1 = subprocess.Popen(cmdl_1,
                                        stdin=stdin, stdout=PIPE, stderr=self._stderr,
                                        **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr_1, str(e))
                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                p2 = subprocess.Popen(cmdl_2,
                                                stdin=p1.stdout, stdout=stdout, stderr=self._stderr, 
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                p1.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr_2, str(e))
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p1.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_1,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_1)))
                                res = p2.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_2,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_2)))
                        return p1, p2

        def call_many(self, cmdlines, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a series of N processes, chained via pipes, possibly with an
                incoming and outgoing pipe at each end.
                Returns a tuple of N subprocess.Popen instances.
                Depending on the values of _in and out, the first and last may be
                available to write to or read from respectively.
                Most arguments are equivalent to call_one(), with
                - cmdlines
                  a list of N command argument lists
                - _in / out
                  applying to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on all
                returned Popen instances after writing to and closing the first one.
                '''
                if len(cmdlines) == 1:
                        # other than direct call_one(), this returns a one-item tuple!
                        return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
                                        universal_newlines=universal_newlines),)
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                procs = []
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try:
                                prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
                                                stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                procs.append(prevproc)
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))

                for cmdl in cmdlines[1:-1]:
                        if getattr(self, 'verbose', None):
                                sys.stderr.write(self.qjoin(cmdl) + ' | ')
                        if not getattr(self, 'donothing', None):
                                try:
                                        nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
                                                        stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                        procs.append(nextproc)
                                        prevproc.stdout.close()
                                        prevproc = nextproc
                                except Exception as e:
                                        self.raise_on_error(actstr, str(e))

                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
                                                stdout=stdout, stderr=self._stderr,
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                prevproc.stdout.close()
                                procs.append(lastproc)
                                prevproc.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))

                        if stdin != PIPE and stdout!= PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                for proc, cmdl in zip(procs, cmdlines):
                                        res = proc.wait()
                                        if res != 0:
                                                self.raise_on_error(actstr,
                                                                'Nonzero exit (%d) from command [%s].'
                                                                % (res, self.qjoin(cmdl)))
                        return procs


### end of proc_mixin.py
Revision:	1.2
Committed:	Wed Mar 30 18:02:06 2016 UTC (9 years, 1 month ago) by schorsch
Content type:	text/x-python
Branch:	MAIN
Changes since 1.1:	+42 -59 lines
Log Message:	unify argument parsing
#	User	Rev	Content
1	schorsch	1.1	# -- coding: utf-8 --
2			''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
3			2016 - Georg Mischler
4
5			Use as:
6			from pyradlib.pyrad_proc import PIPE, Error, ProcMixin
7
8			For a single-file installation, include the contents of this file
9			at the same place (minus the __future__ import below).
10			'''
11			from __future__ import division, print_function, unicode_literals
12
13			import sys
14			import subprocess
15			PIPE = subprocess.PIPE
16
17
18			class Error(Exception): pass
19
20
21			class ProcMixin():
22			'''Process and pipeline management for Python Radiance scripts
23			'''
24			def raise_on_error(self, actstr, e):
25			raise Error('Unable to %s - %s' % (actstr, str(e)))
26
27			def __configure_subprocess(self):
28			'''Prevent subprocess module failure in frozen scripts on Windows.
29			Prevent console windows from popping up when not console based.
30	schorsch	1.2	Make sure we use the version-specific string types.
31	schorsch	1.1	'''
32			# On Windows, sys.stdxxx may not be available when:
33			# - built as *.exe with "pyinstaller --noconsole"
34			# - invoked via CreateProcess() and stream not redirected
35			try:
36			sys.__stdin__.fileno()
37			self._stdin = sys.stdin
38			except: self._stdin = PIPE
39			try:
40			sys.__stdout__.fileno()
41			self._stdout = sys.stdout
42			except: self._stdout = PIPE
43			try:
44			sys.__stderr__.fileno()
45			self._stderr = sys.stderr
46			# keep subprocesses from opening their own console.
47			except: self._stderr = PIPE
48			if hasattr(subprocess, 'STARTUPINFO'):
49			si = subprocess.STARTUPINFO()
50			si.dwFlags \|= subprocess.STARTF_USESHOWWINDOW
51			self._pipeargs = {'startupinfo':si}
52			else: self._pipeargs = {}
53			# type names vary between Py2.7 and 3.x
54			self._strtypes = (type(b''), type(u''))
55			# private attribute to indicate established configuration
56			self.__proc_mixin_setup = True
57
58			def qjoin(self, sl):
59			'''Join a list with quotes around each element containing whitespace.
60			We only use this to display command lines on sys.stderr, the actual
61			Popen() calls are made with the original list.
62			'''
63			def _q(s):
64			if ' ' in s or '\t' in s or ';' in s:
65			return "'" + s + "'"
66			return s
67			return ' '.join([_q(s) for s in sl])
68
69	schorsch	1.2	def __parse_args(self, _in, out):
70			try: self.__proc_mixin_setup
71			except AttributeError: self.__configure_subprocess()
72			instr = ''
73			if _in == PIPE:
74			stdin = _in
75			elif isinstance(_in, self._strtypes):
76			stdin = open(_in, 'rb')
77			instr = ' < "%s"' % _in
78			elif hasattr(_in, 'read'):
79			stdin = _in
80			instr = ' < "%s"' % _in.name
81			else: stdin = self._stdin
82			outstr = ''
83			if out == PIPE:
84			stdout = out
85			elif isinstance(out, self._strtypes):
86			stdout = open(out, 'wb')
87			outstr = ' > "%s"' % out
88			elif hasattr(out, 'write'):
89			stdout = out
90			outstr = ' > "%s"' % out.name
91			else: stdout = self._stdout
92			return stdin, stdout, instr, outstr
93
94	schorsch	1.1	def call_one(self, cmdl, actstr, _in=None, out=None,
95			universal_newlines=False):
96			'''Create a single subprocess, possibly with an incoming and outgoing
97			pipe at each end.
98			- cmdl
99			A list of strings, leading with the name of the executable (without
100			the *.exe suffix), followed by individual arguments.
101			- actstr
102			A text string of the form "do something".
103			Used in verbose mode as "### do someting\\n### [command line]"
104			Used in error messages as "Scriptname: Unable to do something".
105			- _in / out
106			What to do with the input and output pipes of the process:
107			* a filename as string
108			Open file and use for reading/writing.
109			* a file like object
110			Use for reading/writing
111			* PIPE
112			Pipe will be available in returned object for reading/writing.
113			* None (default)
114			System stdin/stdout will be used if available
115			If _in or out is a PIPE, the caller should call p.wait() on the
116			returned Popen instance after writing to and closing it.
117			'''
118	schorsch	1.2	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
119	schorsch	1.1	if getattr(self, 'verbose', None):
120			sys.stderr.write('### %s \n' % actstr)
121	schorsch	1.2	sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
122	schorsch	1.1	if not getattr(self, 'donothing', None):
123	schorsch	1.2	try: p = subprocess.Popen(cmdl,
124			stdin=stdin, stdout=stdout, stderr=self._stderr,
125	schorsch	1.1	universal_newlines=universal_newlines, **self._pipeargs)
126			except Exception as e:
127			self.raise_on_error(actstr, str(e))
128			if stdin != PIPE and stdout != PIPE:
129			# caller needs to wait after reading or writing (else deadlock)
130			res = p.wait()
131			if res != 0:
132			self.raise_on_error(actstr,
133			'Nonzero exit (%d) from command [%s].'
134	schorsch	1.2	% (res, self.qjoin(cmdl)+instr+outstr+'\n'))
135	schorsch	1.1	return p
136
137			def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
138			universal_newlines=False):
139			'''Create two processes, chained via a pipe, possibly with an incoming
140			and outgoing pipe at each end.
141			Returns a tuple of two Popen instances.
142			Arguments are equivalent to call_one(), with _in and out applying
143			to the ends of the chain.
144			If _in or out is PIPE, the caller should call p.wait() on both
145			returned popen instances after writing to and closing the first on .
146			'''
147	schorsch	1.2	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
148	schorsch	1.1	if getattr(self, 'verbose', None):
149			sys.stderr.write('### %s \n' % actstr_1)
150			sys.stderr.write('### %s \n' % actstr_2)
151	schorsch	1.2	sys.stderr.write(self.qjoin(cmdl_1) + instr + ' \| ')
152	schorsch	1.1	if not getattr(self, 'donothing', None):
153	schorsch	1.2	try: p1 = subprocess.Popen(cmdl_1,
154			stdin=stdin, stdout=PIPE, stderr=self._stderr,
155			**self._pipeargs)
156	schorsch	1.1	except Exception as e:
157			self.raise_on_error(actstr_1, str(e))
158			if getattr(self, 'verbose', None):
159	schorsch	1.2	sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
160	schorsch	1.1	if not getattr(self, 'donothing', None):
161			try:
162	schorsch	1.2	p2 = subprocess.Popen(cmdl_2,
163			stdin=p1.stdout, stdout=stdout, stderr=self._stderr,
164	schorsch	1.1	universal_newlines=universal_newlines, **self._pipeargs)
165			p1.stdout.close()
166			except Exception as e:
167			self.raise_on_error(actstr_2, str(e))
168			if stdin != PIPE and stdout != PIPE:
169			# caller needs to wait after reading or writing (else deadlock)
170			res = p1.wait()
171			if res != 0:
172			self.raise_on_error(actstr_1,
173			'Nonzero exit (%d) from command [%s].'
174			% (res, self.qjoin(cmdl_1)))
175			res = p2.wait()
176			if res != 0:
177			self.raise_on_error(actstr_2,
178			'Nonzero exit (%d) from command [%s].'
179			% (res, self.qjoin(cmdl_2)))
180			return p1, p2
181
182			def call_many(self, cmdlines, actstr, _in=None, out=None,
183			universal_newlines=False):
184			'''Create a series of N processes, chained via pipes, possibly with an
185			incoming and outgoing pipe at each end.
186			Returns a tuple of N subprocess.Popen instances.
187			Depending on the values of _in and out, the first and last may be
188			available to write to or read from respectively.
189			Most arguments are equivalent to call_one(), with
190			- cmdlines
191			a list of N command argument lists
192			- _in / out
193			applying to the ends of the chain.
194			If _in or out is PIPE, the caller should call p.wait() on all
195			returned Popen instances after writing to and closing the first one.
196			'''
197			if len(cmdlines) == 1:
198			# other than direct call_one(), this returns a one-item tuple!
199			return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
200			universal_newlines=universal_newlines),)
201	schorsch	1.2	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
202	schorsch	1.1	procs = []
203			if getattr(self, 'verbose', None):
204			sys.stderr.write('### %s \n' % actstr)
205	schorsch	1.2	sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' \| ')
206	schorsch	1.1	if not getattr(self, 'donothing', None):
207			try:
208			prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
209			stdout=PIPE, stderr=self._stderr, **self._pipeargs)
210			procs.append(prevproc)
211			except Exception as e:
212			self.raise_on_error(actstr, str(e))
213
214			for cmdl in cmdlines[1:-1]:
215			if getattr(self, 'verbose', None):
216			sys.stderr.write(self.qjoin(cmdl) + ' \| ')
217			if not getattr(self, 'donothing', None):
218			try:
219			nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
220			stdout=PIPE, stderr=self._stderr, **self._pipeargs)
221			procs.append(nextproc)
222			prevproc.stdout.close()
223			prevproc = nextproc
224			except Exception as e:
225			self.raise_on_error(actstr, str(e))
226
227			if getattr(self, 'verbose', None):
228	schorsch	1.2	sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
229	schorsch	1.1	if not getattr(self, 'donothing', None):
230			try:
231			lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
232			stdout=stdout, stderr=self._stderr,
233			universal_newlines=universal_newlines, **self._pipeargs)
234			prevproc.stdout.close()
235			procs.append(lastproc)
236			prevproc.stdout.close()
237			except Exception as e:
238			self.raise_on_error(actstr, str(e))
239
240			if stdin != PIPE and stdout!= PIPE:
241			# caller needs to wait after reading or writing (else deadlock)
242			for proc, cmdl in zip(procs, cmdlines):
243			res = proc.wait()
244			if res != 0:
245			self.raise_on_error(actstr,
246			'Nonzero exit (%d) from command [%s].'
247			% (res, self.qjoin(cmdl)))
248			return procs
249
250
251			### end of proc_mixin.py