common/pyradlib/pyrad_proc.py

# -*- coding: utf-8 -*-
''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
2016 - Georg Mischler

Use as:
        from pyradlib.pyrad_proc import PIPE, Error, ProcMixin

For a single-file installation, include the contents of this file
at the same place (minus the __future__ import below).
'''
from __future__ import division, print_function, unicode_literals

import sys
import subprocess
PIPE = subprocess.PIPE


class Error(Exception): pass


class ProcMixin():
        '''Process and pipeline management for Python Radiance scripts
        '''
        def raise_on_error(self, actstr, e):
                raise Error('Unable to %s - %s' % (actstr, str(e)))

        def __configure_subprocess(self):
                '''Prevent subprocess module failure in frozen scripts on Windows.
                   Prevent console windows from popping up when not console based.
                   Make sure we use the version specific string types.
                '''
                # On Windows, sys.stdxxx may not be available when:
                # - built as *.exe with "pyinstaller --noconsole"
                # - invoked via CreateProcess() and stream not redirected
                try:
                        sys.__stdin__.fileno()
                        self._stdin = sys.stdin
                except: self._stdin = PIPE
                try:
                        sys.__stdout__.fileno()
                        self._stdout = sys.stdout
                except: self._stdout = PIPE
                try:
                        sys.__stderr__.fileno()
                        self._stderr = sys.stderr
                # keep subprocesses from opening their own console.
                except: self._stderr = PIPE
                if hasattr(subprocess, 'STARTUPINFO'):
                        si = subprocess.STARTUPINFO()
                        si.dwFlags |= subprocess.STARTF_USESHOWWINDOW
                        self._pipeargs = {'startupinfo':si}
                else: self._pipeargs = {}
                # type names vary between Py2.7 and 3.x
                self._strtypes = (type(b''), type(u''))
                # private attribute to indicate established configuration
                self.__proc_mixin_setup = True

        def qjoin(self, sl):
                '''Join a list with quotes around each element containing whitespace.
                We only use this to display command lines on sys.stderr, the actual
                Popen() calls are made with the original list.
                '''
                def _q(s):
                        if ' ' in s or '\t' in s or ';' in s:
                                return "'" + s + "'"
                        return s
                return  ' '.join([_q(s) for s in sl])

        def call_one(self, cmdl, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a single subprocess, possibly with an incoming and outgoing
                pipe at each end.
                - cmdl
                  A list of strings, leading with the name of the executable (without
                  the *.exe suffix), followed by individual arguments.
                - actstr
                  A text string of the form "do something".
                  Used in verbose mode as "### do someting\\n### [command line]"
                  Used in error messages as "Scriptname: Unable to do something".
                - _in / out
                  What to do with the input and output pipes of the process:
                  * a filename as string
                    Open file and use for reading/writing.
                  * a file like object
                    Use for reading/writing
                  * PIPE
                    Pipe will be available in returned object for reading/writing.
                  * None (default)
                    System stdin/stdout will be used if available
                If _in or out is a PIPE, the caller should call p.wait() on the
                returned Popen instance after writing to and closing it.
                '''
                try: self.__proc_mixin_setup
                except AttributeError: self.__configure_subprocess()
                if _in == PIPE: stdin = _in
                elif isinstance(_in, self._strtypes): stdin = open(_in, 'rb')
                elif hasattr(_in, 'read'): stdin = _in
                else: stdin = self._stdin
                if out == PIPE: stdout = out
                elif isinstance(out, self._strtypes): stdout = open(out, 'wb')
                elif hasattr(out, 'write'): stdout = out
                else: stdout = self._stdout
                displ = cmdl[:]
                if isinstance(_in, self._strtypes): displ[:0] = [_in, '>']
                if isinstance(out, self._strtypes): displ.extend(['>', out])
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(displ) + '\n')
                if not getattr(self, 'donothing', None):
                        try: p = subprocess.Popen(cmdl, stdin=stdin, stdout=stdout,
                                        stderr=self._stderr,
                                        universal_newlines=universal_newlines, **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p.wait()
                                if res != 0:
                                        self.raise_on_error(actstr,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(displ)))
                        return p

        def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
                        universal_newlines=False):
                '''Create two processes, chained via a pipe, possibly with an incoming
                and outgoing pipe at each end.
                Returns a tuple of two Popen instances.
                Arguments are equivalent to call_one(), with _in and out applying
                to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on both
                returned popen instances after writing to and closing the first on .
                '''
                try: self.__proc_mixin_setup
                except AttributeError: self.__configure_subprocess()
                if _in == PIPE: stdin = _in
                elif isinstance(_in, self._strtypes): stdin = open(_in, 'rb')
                elif hasattr(_in, 'read'): stdin = _in
                else: stdin = self._stdin
                outendstr = '\n'
                if out == PIPE:
                        stdout = out
                elif isinstance(out, self._strtypes):
                        stdout = open(out, 'wb')
                        outendstr = ' > "%s"\n' % out
                elif hasattr(out, 'write'):
                        stdout = out
                        outendstr = ' > "%s"\n' % out.name
                else: stdout = self._stdout
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr_1)
                        sys.stderr.write('### %s \n' % actstr_2)
                        sys.stderr.write(self.qjoin(cmdl_1) + ' | ')
                if not getattr(self, 'donothing', None):
                        try: p1 = subprocess.Popen(cmdl_1, stdin=stdin,
                                        stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                        except Exception as e:
                                self.raise_on_error(actstr_1, str(e))
                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdl_2) + outendstr)
                if not getattr(self, 'donothing', None):
                        try:
                                p2 = subprocess.Popen(cmdl_2, stdin=p1.stdout, stdout=stdout,
                                                stderr=self._stderr, 
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                p1.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr_2, str(e))
                        if stdin != PIPE and stdout != PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                res = p1.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_1,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_1)))
                                res = p2.wait()
                                if res != 0:
                                        self.raise_on_error(actstr_2,
                                                        'Nonzero exit (%d) from command [%s].'
                                                        % (res, self.qjoin(cmdl_2)))
                        return p1, p2


        def call_many(self, cmdlines, actstr, _in=None, out=None,
                        universal_newlines=False):
                '''Create a series of N processes, chained via pipes, possibly with an
                incoming and outgoing pipe at each end.
                Returns a tuple of N subprocess.Popen instances.
                Depending on the values of _in and out, the first and last may be
                available to write to or read from respectively.
                Most arguments are equivalent to call_one(), with
                - cmdlines
                  a list of N command argument lists
                - _in / out
                  applying to the ends of the chain.
                If _in or out is PIPE, the caller should call p.wait() on all
                returned Popen instances after writing to and closing the first one.
                '''
                if len(cmdlines) == 1:
                        # other than direct call_one(), this returns a one-item tuple!
                        return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
                                        universal_newlines=universal_newlines),)
                try: self.__proc_mixin_setup
                except AttributeError: self.__configure_subprocess()
                if _in == PIPE: stdin = _in
                elif isinstance(_in, self._strtypes): stdin = open(_in, 'rb')
                elif hasattr(_in, 'read'): stdin = _in
                else: stdin = self._stdin
                outendstr = '\n'
                if out == PIPE:
                        stdout = out
                elif isinstance(out, self._strtypes):
                        stdout = open(out, 'wb')
                        outendstr = ' > "%s"\n' % out
                elif hasattr(out, 'write'):
                        stdout = out
                        outendstr = ' > "%s"\n' % out.name
                else: stdout = self._stdout
                procs = []
                if getattr(self, 'verbose', None):
                        sys.stderr.write('### %s \n' % actstr)
                        sys.stderr.write(self.qjoin(cmdlines[0]) + ' | ')
                if not getattr(self, 'donothing', None):
                        try:
                                prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
                                                stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                procs.append(prevproc)
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))

                for cmdl in cmdlines[1:-1]:
                        if getattr(self, 'verbose', None):
                                sys.stderr.write(self.qjoin(cmdl) + ' | ')
                        if not getattr(self, 'donothing', None):
                                try:
                                        nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
                                                        stdout=PIPE, stderr=self._stderr, **self._pipeargs)
                                        procs.append(nextproc)
                                        prevproc.stdout.close()
                                        prevproc = nextproc
                                except Exception as e:
                                        self.raise_on_error(actstr, str(e))

                if getattr(self, 'verbose', None):
                        sys.stderr.write(self.qjoin(cmdlines[-1]) + outendstr)
                if not getattr(self, 'donothing', None):
                        try:
                                lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
                                                stdout=stdout, stderr=self._stderr,
                                                universal_newlines=universal_newlines, **self._pipeargs)
                                prevproc.stdout.close()
                                procs.append(lastproc)
                                prevproc.stdout.close()
                        except Exception as e:
                                self.raise_on_error(actstr, str(e))

                        if stdin != PIPE and stdout!= PIPE:
                                # caller needs to wait after reading or writing (else deadlock)
                                for proc, cmdl in zip(procs, cmdlines):
                                        res = proc.wait()
                                        if res != 0:
                                                self.raise_on_error(actstr,
                                                                'Nonzero exit (%d) from command [%s].'
                                                                % (res, self.qjoin(cmdl)))
                        return procs


### end of proc_mixin.py
Revision:	1.1
Committed:	Mon Mar 28 17:52:11 2016 UTC (9 years, 7 months ago) by schorsch
Content type:	text/x-python
Branch:	MAIN
Log Message:	Support library for test suite and other Pytho scripts
#	User	Rev	Content
1	schorsch	1.1	# -- coding: utf-8 --
2			''' pyrad_proc.py - Process and pipeline management for Python Radiance scripts
3			2016 - Georg Mischler
4
5			Use as:
6			from pyradlib.pyrad_proc import PIPE, Error, ProcMixin
7
8			For a single-file installation, include the contents of this file
9			at the same place (minus the __future__ import below).
10			'''
11			from __future__ import division, print_function, unicode_literals
12
13			import sys
14			import subprocess
15			PIPE = subprocess.PIPE
16
17
18			class Error(Exception): pass
19
20
21			class ProcMixin():
22			'''Process and pipeline management for Python Radiance scripts
23			'''
24			def raise_on_error(self, actstr, e):
25			raise Error('Unable to %s - %s' % (actstr, str(e)))
26
27			def __configure_subprocess(self):
28			'''Prevent subprocess module failure in frozen scripts on Windows.
29			Prevent console windows from popping up when not console based.
30			Make sure we use the version specific string types.
31			'''
32			# On Windows, sys.stdxxx may not be available when:
33			# - built as *.exe with "pyinstaller --noconsole"
34			# - invoked via CreateProcess() and stream not redirected
35			try:
36			sys.__stdin__.fileno()
37			self._stdin = sys.stdin
38			except: self._stdin = PIPE
39			try:
40			sys.__stdout__.fileno()
41			self._stdout = sys.stdout
42			except: self._stdout = PIPE
43			try:
44			sys.__stderr__.fileno()
45			self._stderr = sys.stderr
46			# keep subprocesses from opening their own console.
47			except: self._stderr = PIPE
48			if hasattr(subprocess, 'STARTUPINFO'):
49			si = subprocess.STARTUPINFO()
50			si.dwFlags \|= subprocess.STARTF_USESHOWWINDOW
51			self._pipeargs = {'startupinfo':si}
52			else: self._pipeargs = {}
53			# type names vary between Py2.7 and 3.x
54			self._strtypes = (type(b''), type(u''))
55			# private attribute to indicate established configuration
56			self.__proc_mixin_setup = True
57
58			def qjoin(self, sl):
59			'''Join a list with quotes around each element containing whitespace.
60			We only use this to display command lines on sys.stderr, the actual
61			Popen() calls are made with the original list.
62			'''
63			def _q(s):
64			if ' ' in s or '\t' in s or ';' in s:
65			return "'" + s + "'"
66			return s
67			return ' '.join([_q(s) for s in sl])
68
69			def call_one(self, cmdl, actstr, _in=None, out=None,
70			universal_newlines=False):
71			'''Create a single subprocess, possibly with an incoming and outgoing
72			pipe at each end.
73			- cmdl
74			A list of strings, leading with the name of the executable (without
75			the *.exe suffix), followed by individual arguments.
76			- actstr
77			A text string of the form "do something".
78			Used in verbose mode as "### do someting\\n### [command line]"
79			Used in error messages as "Scriptname: Unable to do something".
80			- _in / out
81			What to do with the input and output pipes of the process:
82			* a filename as string
83			Open file and use for reading/writing.
84			* a file like object
85			Use for reading/writing
86			* PIPE
87			Pipe will be available in returned object for reading/writing.
88			* None (default)
89			System stdin/stdout will be used if available
90			If _in or out is a PIPE, the caller should call p.wait() on the
91			returned Popen instance after writing to and closing it.
92			'''
93			try: self.__proc_mixin_setup
94			except AttributeError: self.__configure_subprocess()
95			if _in == PIPE: stdin = _in
96			elif isinstance(_in, self._strtypes): stdin = open(_in, 'rb')
97			elif hasattr(_in, 'read'): stdin = _in
98			else: stdin = self._stdin
99			if out == PIPE: stdout = out
100			elif isinstance(out, self._strtypes): stdout = open(out, 'wb')
101			elif hasattr(out, 'write'): stdout = out
102			else: stdout = self._stdout
103			displ = cmdl[:]
104			if isinstance(_in, self._strtypes): displ[:0] = [_in, '>']
105			if isinstance(out, self._strtypes): displ.extend(['>', out])
106			if getattr(self, 'verbose', None):
107			sys.stderr.write('### %s \n' % actstr)
108			sys.stderr.write(self.qjoin(displ) + '\n')
109			if not getattr(self, 'donothing', None):
110			try: p = subprocess.Popen(cmdl, stdin=stdin, stdout=stdout,
111			stderr=self._stderr,
112			universal_newlines=universal_newlines, **self._pipeargs)
113			except Exception as e:
114			self.raise_on_error(actstr, str(e))
115			if stdin != PIPE and stdout != PIPE:
116			# caller needs to wait after reading or writing (else deadlock)
117			res = p.wait()
118			if res != 0:
119			self.raise_on_error(actstr,
120			'Nonzero exit (%d) from command [%s].'
121			% (res, self.qjoin(displ)))
122			return p
123
124			def call_two(self, cmdl_1, cmdl_2, actstr_1, actstr_2, _in=None, out=None,
125			universal_newlines=False):
126			'''Create two processes, chained via a pipe, possibly with an incoming
127			and outgoing pipe at each end.
128			Returns a tuple of two Popen instances.
129			Arguments are equivalent to call_one(), with _in and out applying
130			to the ends of the chain.
131			If _in or out is PIPE, the caller should call p.wait() on both
132			returned popen instances after writing to and closing the first on .
133			'''
134			try: self.__proc_mixin_setup
135			except AttributeError: self.__configure_subprocess()
136			if _in == PIPE: stdin = _in
137			elif isinstance(_in, self._strtypes): stdin = open(_in, 'rb')
138			elif hasattr(_in, 'read'): stdin = _in
139			else: stdin = self._stdin
140			outendstr = '\n'
141			if out == PIPE:
142			stdout = out
143			elif isinstance(out, self._strtypes):
144			stdout = open(out, 'wb')
145			outendstr = ' > "%s"\n' % out
146			elif hasattr(out, 'write'):
147			stdout = out
148			outendstr = ' > "%s"\n' % out.name
149			else: stdout = self._stdout
150			if getattr(self, 'verbose', None):
151			sys.stderr.write('### %s \n' % actstr_1)
152			sys.stderr.write('### %s \n' % actstr_2)
153			sys.stderr.write(self.qjoin(cmdl_1) + ' \| ')
154			if not getattr(self, 'donothing', None):
155			try: p1 = subprocess.Popen(cmdl_1, stdin=stdin,
156			stdout=PIPE, stderr=self._stderr, **self._pipeargs)
157			except Exception as e:
158			self.raise_on_error(actstr_1, str(e))
159			if getattr(self, 'verbose', None):
160			sys.stderr.write(self.qjoin(cmdl_2) + outendstr)
161			if not getattr(self, 'donothing', None):
162			try:
163			p2 = subprocess.Popen(cmdl_2, stdin=p1.stdout, stdout=stdout,
164			stderr=self._stderr,
165			universal_newlines=universal_newlines, **self._pipeargs)
166			p1.stdout.close()
167			except Exception as e:
168			self.raise_on_error(actstr_2, str(e))
169			if stdin != PIPE and stdout != PIPE:
170			# caller needs to wait after reading or writing (else deadlock)
171			res = p1.wait()
172			if res != 0:
173			self.raise_on_error(actstr_1,
174			'Nonzero exit (%d) from command [%s].'
175			% (res, self.qjoin(cmdl_1)))
176			res = p2.wait()
177			if res != 0:
178			self.raise_on_error(actstr_2,
179			'Nonzero exit (%d) from command [%s].'
180			% (res, self.qjoin(cmdl_2)))
181			return p1, p2
182
183
184			def call_many(self, cmdlines, actstr, _in=None, out=None,
185			universal_newlines=False):
186			'''Create a series of N processes, chained via pipes, possibly with an
187			incoming and outgoing pipe at each end.
188			Returns a tuple of N subprocess.Popen instances.
189			Depending on the values of _in and out, the first and last may be
190			available to write to or read from respectively.
191			Most arguments are equivalent to call_one(), with
192			- cmdlines
193			a list of N command argument lists
194			- _in / out
195			applying to the ends of the chain.
196			If _in or out is PIPE, the caller should call p.wait() on all
197			returned Popen instances after writing to and closing the first one.
198			'''
199			if len(cmdlines) == 1:
200			# other than direct call_one(), this returns a one-item tuple!
201			return (self.call_one(cmdlines[0], actstr, _in=_in, out=out,
202			universal_newlines=universal_newlines),)
203			try: self.__proc_mixin_setup
204			except AttributeError: self.__configure_subprocess()
205			if _in == PIPE: stdin = _in
206			elif isinstance(_in, self._strtypes): stdin = open(_in, 'rb')
207			elif hasattr(_in, 'read'): stdin = _in
208			else: stdin = self._stdin
209			outendstr = '\n'
210			if out == PIPE:
211			stdout = out
212			elif isinstance(out, self._strtypes):
213			stdout = open(out, 'wb')
214			outendstr = ' > "%s"\n' % out
215			elif hasattr(out, 'write'):
216			stdout = out
217			outendstr = ' > "%s"\n' % out.name
218			else: stdout = self._stdout
219			procs = []
220			if getattr(self, 'verbose', None):
221			sys.stderr.write('### %s \n' % actstr)
222			sys.stderr.write(self.qjoin(cmdlines[0]) + ' \| ')
223			if not getattr(self, 'donothing', None):
224			try:
225			prevproc = subprocess.Popen(cmdlines[0], stdin=stdin,
226			stdout=PIPE, stderr=self._stderr, **self._pipeargs)
227			procs.append(prevproc)
228			except Exception as e:
229			self.raise_on_error(actstr, str(e))
230
231			for cmdl in cmdlines[1:-1]:
232			if getattr(self, 'verbose', None):
233			sys.stderr.write(self.qjoin(cmdl) + ' \| ')
234			if not getattr(self, 'donothing', None):
235			try:
236			nextproc = subprocess.Popen(cmdl, stdin=prevproc.stdout,
237			stdout=PIPE, stderr=self._stderr, **self._pipeargs)
238			procs.append(nextproc)
239			prevproc.stdout.close()
240			prevproc = nextproc
241			except Exception as e:
242			self.raise_on_error(actstr, str(e))
243
244			if getattr(self, 'verbose', None):
245			sys.stderr.write(self.qjoin(cmdlines[-1]) + outendstr)
246			if not getattr(self, 'donothing', None):
247			try:
248			lastproc = subprocess.Popen(cmdlines[-1], stdin=prevproc.stdout,
249			stdout=stdout, stderr=self._stderr,
250			universal_newlines=universal_newlines, **self._pipeargs)
251			prevproc.stdout.close()
252			procs.append(lastproc)
253			prevproc.stdout.close()
254			except Exception as e:
255			self.raise_on_error(actstr, str(e))
256
257			if stdin != PIPE and stdout!= PIPE:
258			# caller needs to wait after reading or writing (else deadlock)
259			for proc, cmdl in zip(procs, cmdlines):
260			res = proc.wait()
261			if res != 0:
262			self.raise_on_error(actstr,
263			'Nonzero exit (%d) from command [%s].'
264			% (res, self.qjoin(cmdl)))
265			return procs
266
267
268			### end of proc_mixin.py