common/pyradlib/pyrad_proc.py

# -*- coding: utf-8 -*-
''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
2016 - Georg Mischler

Use as:
        from pyradlib.pyrad_proc import PIPE, Error, ProcMixin

For a single-file installation, include the contents of this file
at the same place (minus the __future__ import below).
'''
from __future__ import division, print_function, unicode_literals

import sys
import subprocess
PIPE = subprocess.PIPE


class Error(Exception): pass


class ProcMixin():
        '''Process and pipeline management for Python Radiance scripts
        '''
        def raise_on_error(self, actstr, e):
                if hasattr(e, 'strerror'): eb = e.strerror
                elif isinstance(e, self._strtypes): eb = e
                else: eb = e
                if isinstance(eb, type(b'')):
                        estr = eb.decode(encoding='utf-8', errors='ignore')
                else: estr = eb
                raise Error('Unable to %s - %s' % (actstr, estr)) #

        def __configure_subprocess(self):
                '''Prevent subprocess module failure in frozen scripts on Windows.
                   Prevent console windows from popping up when not console based.
                   Make sure we use the version-specific string types.
                '''
                # On Windows, sys.stdxxx may not be available when:
                # - built as *.exe with "pyinstaller --noconsole"
                # - invoked via CreateProcess() and stream not redirected
                try:
                        sys.__stdin__.fileno()
                        self._stdin = sys.stdin
                except: self._stdin = PIPE
                try:
                        sys.__stdout__.fileno()
                        self._stdout = sys.stdout
                except: self._stdout = PIPE
                try:
                        sys.__stderr__.fileno()
                        self._stderr = sys.stderr
                # keep subprocesses from opening their own console.
                except: self._stderr = PIPE
                if hasattr(subprocess, 'STARTUPINFO'):
                        si = subprocess.STARTUPINFO()
                        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                        self._pipeargs = {'startupinfo':si}
                else: self._pipeargs = {}
                # type names vary between Py2.7 and 3.x
                self._strtypes = (type(b''), type(u''))
                # private attribute to indicate established configuration
                self.__proc_mixin_setup = True

        def qjoin(self, sl):
                '''Join a list with quotes around each element containing whitespace.
                We only use this to display command lines on sys.stderr, the actual
                Popen() calls are made with the original list.
                '''
                def _q(s):
                        if ' ' in s or '\t' in s or ';' in s:
                                return "'" + s + "'"
                        return s
                return  ' '.join([_q(s) for s in sl])

        def __parse_args(self, _in, out):
                try: self.__proc_mixin_setup
                except AttributeError: self.__configure_subprocess()
                instr = ''
                if _in == PIPE:
                        stdin = _in
                elif isinstance(_in, self._strtypes):
                        if not getattr(self, 'donothing', None):
                                stdin = open(_in, 'rb')
                        else: stdin = None
                        instr = ' < "%s"' % _in
                elif hasattr(_in, 'read'):
                        stdin = _in
                        instr = ' < "%s"' % _in.name
                else: stdin = self._stdin
                outstr = ''
                if out == PIPE:
                        stdout = out
                elif isinstance(out, self._strtypes):
                        if not getattr(self, 'donothing', None):
                                stdout = open(out, 'wb')
                        else: stdout = None
                        outstr = ' > "%s"' % out
                elif hasattr(out, 'write'):
                        stdout = out
                        outstr = ' > "%s"' % out.name
                else: stdout = self._stdout
                return stdin, stdout, instr, outstr

        def call_one(self, cmdl, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a single subprocess, possibly with an incoming and outgoing
                pipe at each end.
                - cmdl
                  A list of strings, leading with the name of the executable (without
                  the *.exe suffix), followed by individual arguments.
                - actstr
                  A text string of the form "do something".
                  Used in verbose mode as "### do someting\\n### [command line]"
                  Used in error messages as "Scriptname: Unable to do something".
                - _in / out
                  What to do with the input and output pipes of the process:
                  * a filename as string
                    Open file and use for reading/writing.
                  * a file like object
                    Use for reading/writing
                  * PIPE
                    Pipe will be available in returned object for reading/writing.
                  * None (default)
                    System stdin/stdout will be used if available
                If _in or out is a PIPE, the caller should call p.wait() on the
                returned Popen instance after writing to and closing it.
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try: p = subprocess.Popen(cmdl,
                                        stdin=stdin, stdout=stdout, stderr=self._stderr,
                                        universal_newlines=universal_newlines, **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr, e)
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p.wait()
                                if res != 0:
                                        self.raise_on_error(actstr,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl)+instr+outstr+'\n'))
                        return p

        def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
                        universal_newlines=False):
                '''Create two processes, chained via a pipe, possibly with an incoming
                and outgoing pipe at each end.
                Returns a tuple of two Popen instances.
                Arguments are equivalent to call_one(), with _in and out applying
                to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on both
                returned popen instances after writing to and closing the first on .
                '''
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr_1)
                        sys.stderr.write('### %s \n' % actstr_2)
                        sys.stderr.write(self.qjoin(cmdl_1) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try: p1 = subprocess.Popen(cmdl_1,
                                        stdin=stdin, stdout=PIPE, stderr=self._stderr,
                                        **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr_1, e)
                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                p2 = subprocess.Popen(cmdl_2,
                                                stdin=p1.stdout, stdout=stdout, stderr=self._stderr, 
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                p1.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr_2, e)
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p1.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_1,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_1)))
                                res = p2.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_2,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_2)))
                        return p1, p2

        def call_many(self, cmdlines, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a series of N processes, chained via pipes, possibly with an
                incoming and outgoing pipe at each end.
                Returns a tuple of N subprocess.Popen instances.
                Depending on the values of _in and out, the first and last may be
                available to write to or read from respectively.
                Most arguments are equivalent to call_one(), with
                - cmdlines
                  a list of N command argument lists
                - _in / out
                  applying to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on all
                returned Popen instances after writing to and closing the first one.
                '''
                if len(cmdlines) == 1:
                        # other than direct call_one(), this returns a one-item tuple!
                        return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
                                        universal_newlines=universal_newlines),)
                stdin, stdout, instr, outstr = self.__parse_args(_in, out)
                procs = []
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' | ')
                if not getattr(self, 'donothing', None):
                        try:
                                prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
                                                stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                procs.append(prevproc)
                        except Exception as e:
                                self.raise_on_error(actstr, e)

                for cmdl in cmdlines[1:-1]:
                        if getattr(self, 'verbose', None):
                                sys.stderr.write(self.qjoin(cmdl) + ' | ')
                        if not getattr(self, 'donothing', None):
                                try:
                                        nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
                                                        stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                        procs.append(nextproc)
                                        prevproc.stdout.close()
                                        prevproc = nextproc
                                except Exception as e:
                                        self.raise_on_error(actstr, e)

                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
                if not getattr(self, 'donothing', None):
                        try:
                                lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
                                                stdout=stdout, stderr=self._stderr,
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                prevproc.stdout.close()
                                procs.append(lastproc)
                                prevproc.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr, e)

                        if stdin != PIPE and stdout!= PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                for proc, cmdl in zip(procs, cmdlines):
                                        res = proc.wait()
                                        if res != 0:
                                                self.raise_on_error(actstr,
                                                                'Nonzero exit (%d) from command [%s].'
                                                                % (res, self.qjoin(cmdl)))
                        return procs


### end of proc_mixin.py
Revision:	1.4
Committed:	Tue Apr 19 21:17:41 2016 UTC (9 years ago) by schorsch
Content type:	text/x-python
Branch:	MAIN
Changes since 1.3:	+6 -4 lines
Log Message:	more robustness with donothing
#	User	Rev	Content
1	schorsch	1.1	# -- coding: utf-8 --
2			''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
3			2016 - Georg Mischler
4
5			Use as:
6			from pyradlib.pyrad_proc import PIPE, Error, ProcMixin
7
8			For a single-file installation, include the contents of this file
9			at the same place (minus the __future__ import below).
10			'''
11			from __future__ import division, print_function, unicode_literals
12
13			import sys
14			import subprocess
15			PIPE = subprocess.PIPE
16
17
18			class Error(Exception): pass
19
20
21			class ProcMixin():
22			'''Process and pipeline management for Python Radiance scripts
23			'''
24			def raise_on_error(self, actstr, e):
25	schorsch	1.3	if hasattr(e, 'strerror'): eb = e.strerror
26			elif isinstance(e, self._strtypes): eb = e
27			else: eb = e
28			if isinstance(eb, type(b'')):
29			estr = eb.decode(encoding='utf-8', errors='ignore')
30			else: estr = eb
31			raise Error('Unable to %s - %s' % (actstr, estr)) #
32	schorsch	1.1
33			def __configure_subprocess(self):
34			'''Prevent subprocess module failure in frozen scripts on Windows.
35			Prevent console windows from popping up when not console based.
36	schorsch	1.2	Make sure we use the version-specific string types.
37	schorsch	1.1	'''
38			# On Windows, sys.stdxxx may not be available when:
39			# - built as *.exe with "pyinstaller --noconsole"
40			# - invoked via CreateProcess() and stream not redirected
41			try:
42			sys.__stdin__.fileno()
43			self._stdin = sys.stdin
44			except: self._stdin = PIPE
45			try:
46			sys.__stdout__.fileno()
47			self._stdout = sys.stdout
48			except: self._stdout = PIPE
49			try:
50			sys.__stderr__.fileno()
51			self._stderr = sys.stderr
52			# keep subprocesses from opening their own console.
53			except: self._stderr = PIPE
54			if hasattr(subprocess, 'STARTUPINFO'):
55			si = subprocess.STARTUPINFO()
56			si.dwFlags \|= subprocess.STARTF_USESHOWWINDOW
57			self._pipeargs = {'startupinfo':si}
58			else: self._pipeargs = {}
59			# type names vary between Py2.7 and 3.x
60			self._strtypes = (type(b''), type(u''))
61			# private attribute to indicate established configuration
62			self.__proc_mixin_setup = True
63
64			def qjoin(self, sl):
65			'''Join a list with quotes around each element containing whitespace.
66			We only use this to display command lines on sys.stderr, the actual
67			Popen() calls are made with the original list.
68			'''
69			def _q(s):
70			if ' ' in s or '\t' in s or ';' in s:
71			return "'" + s + "'"
72			return s
73			return ' '.join([_q(s) for s in sl])
74
75	schorsch	1.2	def __parse_args(self, _in, out):
76			try: self.__proc_mixin_setup
77			except AttributeError: self.__configure_subprocess()
78			instr = ''
79			if _in == PIPE:
80			stdin = _in
81			elif isinstance(_in, self._strtypes):
82	schorsch	1.4	if not getattr(self, 'donothing', None):
83			stdin = open(_in, 'rb')
84			else: stdin = None
85	schorsch	1.2	instr = ' < "%s"' % _in
86			elif hasattr(_in, 'read'):
87			stdin = _in
88			instr = ' < "%s"' % _in.name
89			else: stdin = self._stdin
90			outstr = ''
91			if out == PIPE:
92			stdout = out
93			elif isinstance(out, self._strtypes):
94	schorsch	1.4	if not getattr(self, 'donothing', None):
95			stdout = open(out, 'wb')
96			else: stdout = None
97	schorsch	1.2	outstr = ' > "%s"' % out
98			elif hasattr(out, 'write'):
99			stdout = out
100			outstr = ' > "%s"' % out.name
101			else: stdout = self._stdout
102			return stdin, stdout, instr, outstr
103
104	schorsch	1.1	def call_one(self, cmdl, actstr, _in=None, out=None,
105			universal_newlines=False):
106			'''Create a single subprocess, possibly with an incoming and outgoing
107			pipe at each end.
108			- cmdl
109			A list of strings, leading with the name of the executable (without
110			the *.exe suffix), followed by individual arguments.
111			- actstr
112			A text string of the form "do something".
113			Used in verbose mode as "### do someting\\n### [command line]"
114			Used in error messages as "Scriptname: Unable to do something".
115			- _in / out
116			What to do with the input and output pipes of the process:
117			* a filename as string
118			Open file and use for reading/writing.
119			* a file like object
120			Use for reading/writing
121			* PIPE
122			Pipe will be available in returned object for reading/writing.
123			* None (default)
124			System stdin/stdout will be used if available
125			If _in or out is a PIPE, the caller should call p.wait() on the
126			returned Popen instance after writing to and closing it.
127			'''
128	schorsch	1.2	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
129	schorsch	1.1	if getattr(self, 'verbose', None):
130			sys.stderr.write('### %s \n' % actstr)
131	schorsch	1.2	sys.stderr.write(self.qjoin(cmdl) + instr + outstr + '\n')
132	schorsch	1.1	if not getattr(self, 'donothing', None):
133	schorsch	1.2	try: p = subprocess.Popen(cmdl,
134			stdin=stdin, stdout=stdout, stderr=self._stderr,
135	schorsch	1.1	universal_newlines=universal_newlines, **self._pipeargs)
136			except Exception as e:
137	schorsch	1.3	self.raise_on_error(actstr, e)
138	schorsch	1.1	if stdin != PIPE and stdout != PIPE:
139			# caller needs to wait after reading or writing (else deadlock)
140			res = p.wait()
141			if res != 0:
142			self.raise_on_error(actstr,
143			'Nonzero exit (%d) from command [%s].'
144	schorsch	1.2	% (res, self.qjoin(cmdl)+instr+outstr+'\n'))
145	schorsch	1.1	return p
146
147			def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
148			universal_newlines=False):
149			'''Create two processes, chained via a pipe, possibly with an incoming
150			and outgoing pipe at each end.
151			Returns a tuple of two Popen instances.
152			Arguments are equivalent to call_one(), with _in and out applying
153			to the ends of the chain.
154			If _in or out is PIPE, the caller should call p.wait() on both
155			returned popen instances after writing to and closing the first on .
156			'''
157	schorsch	1.2	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
158	schorsch	1.1	if getattr(self, 'verbose', None):
159			sys.stderr.write('### %s \n' % actstr_1)
160			sys.stderr.write('### %s \n' % actstr_2)
161	schorsch	1.2	sys.stderr.write(self.qjoin(cmdl_1) + instr + ' \| ')
162	schorsch	1.1	if not getattr(self, 'donothing', None):
163	schorsch	1.2	try: p1 = subprocess.Popen(cmdl_1,
164			stdin=stdin, stdout=PIPE, stderr=self._stderr,
165			**self._pipeargs)
166	schorsch	1.1	except Exception as e:
167	schorsch	1.3	self.raise_on_error(actstr_1, e)
168	schorsch	1.1	if getattr(self, 'verbose', None):
169	schorsch	1.2	sys.stderr.write(self.qjoin(cmdl_2) + outstr + '\n')
170	schorsch	1.1	if not getattr(self, 'donothing', None):
171			try:
172	schorsch	1.2	p2 = subprocess.Popen(cmdl_2,
173			stdin=p1.stdout, stdout=stdout, stderr=self._stderr,
174	schorsch	1.1	universal_newlines=universal_newlines, **self._pipeargs)
175			p1.stdout.close()
176			except Exception as e:
177	schorsch	1.3	self.raise_on_error(actstr_2, e)
178	schorsch	1.1	if stdin != PIPE and stdout != PIPE:
179			# caller needs to wait after reading or writing (else deadlock)
180			res = p1.wait()
181			if res != 0:
182			self.raise_on_error(actstr_1,
183			'Nonzero exit (%d) from command [%s].'
184			% (res, self.qjoin(cmdl_1)))
185			res = p2.wait()
186			if res != 0:
187			self.raise_on_error(actstr_2,
188			'Nonzero exit (%d) from command [%s].'
189			% (res, self.qjoin(cmdl_2)))
190			return p1, p2
191
192			def call_many(self, cmdlines, actstr, _in=None, out=None,
193			universal_newlines=False):
194			'''Create a series of N processes, chained via pipes, possibly with an
195			incoming and outgoing pipe at each end.
196			Returns a tuple of N subprocess.Popen instances.
197			Depending on the values of _in and out, the first and last may be
198			available to write to or read from respectively.
199			Most arguments are equivalent to call_one(), with
200			- cmdlines
201			a list of N command argument lists
202			- _in / out
203			applying to the ends of the chain.
204			If _in or out is PIPE, the caller should call p.wait() on all
205			returned Popen instances after writing to and closing the first one.
206			'''
207			if len(cmdlines) == 1:
208			# other than direct call_one(), this returns a one-item tuple!
209			return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
210			universal_newlines=universal_newlines),)
211	schorsch	1.2	stdin, stdout, instr, outstr = self.__parse_args(_in, out)
212	schorsch	1.1	procs = []
213			if getattr(self, 'verbose', None):
214			sys.stderr.write('### %s \n' % actstr)
215	schorsch	1.2	sys.stderr.write(self.qjoin(cmdlines[0]) + instr + ' \| ')
216	schorsch	1.1	if not getattr(self, 'donothing', None):
217			try:
218			prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
219			stdout=PIPE, stderr=self._stderr, **self._pipeargs)
220			procs.append(prevproc)
221			except Exception as e:
222	schorsch	1.3	self.raise_on_error(actstr, e)
223	schorsch	1.1
224			for cmdl in cmdlines[1:-1]:
225			if getattr(self, 'verbose', None):
226			sys.stderr.write(self.qjoin(cmdl) + ' \| ')
227			if not getattr(self, 'donothing', None):
228			try:
229			nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
230			stdout=PIPE, stderr=self._stderr, **self._pipeargs)
231			procs.append(nextproc)
232			prevproc.stdout.close()
233			prevproc = nextproc
234			except Exception as e:
235	schorsch	1.3	self.raise_on_error(actstr, e)
236	schorsch	1.1
237			if getattr(self, 'verbose', None):
238	schorsch	1.2	sys.stderr.write(self.qjoin(cmdlines[-1]) + outstr + '\n')
239	schorsch	1.1	if not getattr(self, 'donothing', None):
240			try:
241			lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
242			stdout=stdout, stderr=self._stderr,
243			universal_newlines=universal_newlines, **self._pipeargs)
244			prevproc.stdout.close()
245			procs.append(lastproc)
246			prevproc.stdout.close()
247			except Exception as e:
248	schorsch	1.3	self.raise_on_error(actstr, e)
249	schorsch	1.1
250			if stdin != PIPE and stdout!= PIPE:
251			# caller needs to wait after reading or writing (else deadlock)
252			for proc, cmdl in zip(procs, cmdlines):
253			res = proc.wait()
254			if res != 0:
255			self.raise_on_error(actstr,
256			'Nonzero exit (%d) from command [%s].'
257			% (res, self.qjoin(cmdl)))
258			return procs
259
260
261			### end of proc_mixin.py