improve testing
authorrobin
Tue, 28 Sep 2021 17:06:17 +0100
changeset 108 ec1e97acaa60
parent 107 5a2d8906a279
child 109 c929a89bd8d6
improve testing
test/runAll.py
test/testRXPbasic.py
test/test_xmltestsuite.py
--- a/test/runAll.py	Tue Sep 28 17:04:49 2021 +0100
+++ b/test/runAll.py	Tue Sep 28 17:06:17 2021 +0100
@@ -1,22 +1,73 @@
+class Unbuffered(object):
+    def __init__(self, stream):
+        self.stream = stream
+    def write(self, data):
+        self.stream.write(data)
+        self.stream.flush()
+    def writelines(self, datas):
+        self.stream.writelines(datas)
+        self.stream.flush()
+    def __getattr__(self, attr):
+        return getattr(self.stream, attr)
+
 def main():
-    import sys, os
+    import sys, os, psutil, platform
     sys.__old_stderr__ = sys.stderr
-    sys.__stderr__ = sys.stderr = sys.stdout
+    sys.__stderr__ = sys.stderr = sys.stdout = Unbuffered(sys.stdout)
     wd = os.path.dirname(os.path.abspath(sys.argv[0]))
     sys.path.insert(0,wd)
     os.chdir(wd)
-    
+    platform_attrs = [
+        "architecture",
+        "java_ver",
+        "libc_ver",
+        "machine",
+        "mac_ver",
+        "node",
+        "platform",
+        "processor",
+        "python_branch",
+        "python_build",
+        "python_compiler",
+        "python_implementation",
+        "python_revision",
+        "python_version",
+        "python_version_tuple",
+        "release",
+        "system",
+        "system_alias",
+        "uname",
+        "version",
+        "win32_edition",
+        "win32_is_iot",
+        "win32_ver",
+        ]
+    print(f'+++++ {sys.platform}')
+    print(f'+++++ {sys.byteorder}')
+    for a in platform_attrs:
+        if hasattr(platform,a):
+            try:
+                print(f'+++++ platform.{a}()={getattr(platform,a)()}')
+            except:
+                print(f'!!!!! platform.{a}()=cannot be determined')
+
     #special case import to allow reportlab to modify the envirnment in development
     try:
-        import reportlab
+      import reportlab
     except ImportError:
-        pass
+      pass
 
     import leaktest, testRXPbasic, test_xmltestsuite
 
     leaktest.main(100)
     testRXPbasic.main()
-    test_xmltestsuite.main()
+    if 1 or platform.system()!='Darwin':
+        verbose=int(os.environ.get('VERBOSE','0'))
+        singles=int(os.environ.get('SINGLES','0'))
+        test_xmltestsuite.main(verbose, singles)
+    else:
+        test_xmltestsuite.main(3,1)
+    print(f'+++++ open Files={psutil.Process().open_files()!r}')
 
 if __name__=='__main__':
     main()
--- a/test/testRXPbasic.py	Tue Sep 28 17:04:49 2021 +0100
+++ b/test/testRXPbasic.py	Tue Sep 28 17:06:17 2021 +0100
@@ -1,11 +1,5 @@
-from __future__ import unicode_literals
-try:
-	from future_builtins import ascii
-except ImportError:
-	pass
-import traceback, sys, os
-_pyRXP = None
-_logf = open('pyRXP_test.log','w')
+import traceback, sys, os, pyRXPU
+_logf = None
 _bad = 0
 _total = 0
 def _dot(c,write=sys.stdout.write):
@@ -36,7 +30,7 @@
 
 def goodTest(x,t,tb=0,inOnly=0,**kw):
 	try:
-		P=_pyRXP.Parser(**kw)
+		P=pyRXPU.Parser(**kw)
 		r = P(x)
 		rb = 0
 	except:
@@ -56,7 +50,7 @@
 	if type(r) is type(''):
 		r = r.replace('\r','\\r')
 		r = r.replace('\n','\\n')
-	plog('%s.Parser(%s)(%s)'%(_pyRXP.__name__,s[2:],repr(x)))
+	plog('%s.Parser(%s)(%s)'%(pyRXPU.__name__,s[2:],repr(x)))
 	if (inOnly and t in r) or (r==t) and rb==tb:
 		plogn('OK')
 		_dot('.')
@@ -104,12 +98,10 @@
 def eoDTD(s):
 	return DTDs.get(os.path.basename(s),None)
 
-def _runTests(pyRXP):
-	global _pyRXP
-	_pyRXP = pyRXP
-	plogn('############# Testing %s=%8.8X'%(pyRXP.__name__,id(_pyRXP)))
+def _runTests():
+	plogn('############# Testing %s=%8.8X'%(pyRXPU.__name__,id(pyRXPU)))
 	try:
-		for k,v in pyRXP.parser_flags.items(): eval('pyRXP.Parser(%s=%d)' % (k,v))
+		for k,v in pyRXPU.parser_flags.items(): eval('pyRXPU.Parser(%s=%d)' % (k,v))
 		plogn('Parser keywords OK')
 		_dot('.')
 	except:
@@ -117,7 +109,7 @@
 		plogn('Parser keywords BAD')
 		_dot('E')
 	try:
-		for k,v in pyRXP.parser_flags.items(): eval('pyRXP.Parser()("<a/>",%s=%d)' % (k,v))
+		for k,v in pyRXPU.parser_flags.items(): eval('pyRXPU.Parser()("<a/>",%s=%d)' % (k,v))
 		plogn('Parser().parse keywords OK')
 		_dot('.')
 	except:
@@ -126,7 +118,7 @@
 		_dot('E')
 
 	try:
-		P=_pyRXP.Parser()
+		P=pyRXPU.Parser()
 		plog('Parser()=%r' % P)
 		plog('Parser().__class__=%r' % P.__class__)
 		plog('type(Parser())=%r\n\n' % type(P))
@@ -156,8 +148,8 @@
 	goodTest('<!--comment--><a/>',('a', None, None, None),ReturnComments=1)
 	failTest('<?xml version="1.0" encoding="LATIN-1"?></a>',"error Unknown declared encoding LATIN-1\nInternal error, ParserPush failed!\n")
 	goodTest('<?work version="1.0" encoding="utf-8"?><a/>',[('<?',{'name':'work'}, ['version="1.0" encoding="utf-8"'],None), ('a', None, None, None)],IgnorePlacementErrors=1,ReturnList=1,ReturnProcessingInstructions=1,ReturnComments=1)
-	goodTest('<a>\nHello\n<b>cruel\n</b>\nWorld\n</a>',('a', None, ['\nHello\n', ('b', None, ['cruel\n'], (('aaa', 2, 3), ('aaa', 3, 4))), '\nWorld\n'], (('aaa', 0, 3), ('aaa', 5, 4))),fourth=pyRXP.recordLocation,srcName='aaa')
-	goodTest('<a aname="ANAME" aother="AOTHER">\nHello\n<b bname="BNAME" bother="BOTHER">cruel\n</b>\nWorld\n</a>',('a', {"aname": "ANAME", "aother": "AOTHER"}, ['\nHello\n', ('b', {"bname": "BNAME", "bother": "BOTHER"}, ['cruel\n'], (('aaa', 2, 33), ('aaa', 3, 4))), '\nWorld\n'], (('aaa', 0, 33), ('aaa', 5, 4))),fourth=pyRXP.recordLocation,srcName='aaa')
+	goodTest('<a>\nHello\n<b>cruel\n</b>\nWorld\n</a>',('a', None, ['\nHello\n', ('b', None, ['cruel\n'], (('aaa', 2, 3), ('aaa', 3, 4))), '\nWorld\n'], (('aaa', 0, 3), ('aaa', 5, 4))),fourth=pyRXPU.recordLocation,srcName='aaa')
+	goodTest('<a aname="ANAME" aother="AOTHER">\nHello\n<b bname="BNAME" bother="BOTHER">cruel\n</b>\nWorld\n</a>',('a', {"aname": "ANAME", "aother": "AOTHER"}, ['\nHello\n', ('b', {"bname": "BNAME", "bother": "BOTHER"}, ['cruel\n'], (('aaa', 2, 33), ('aaa', 3, 4))), '\nWorld\n'], (('aaa', 0, 33), ('aaa', 5, 4))),fourth=pyRXPU.recordLocation,srcName='aaa')
 	goodTest('<a><![CDATA[<a>]]></a>',('a', None, ['<a>'], None))
 	goodTest('<a><![CDATA[<a>]]></a>',('a', None, [('<![CDATA[', None, ['<a>'], None)], None),ReturnCDATASectionsAsTuples=1)
 	goodTest('''<foo:A xmlns:foo="http://www.foo.org/"><foo:B><foo:C xmlns:foo="http://www.bar.org/"><foo:D>abcd</foo:D></foo:C></foo:B><foo:B/><A>bare A<C>bare C</C><B>bare B</B></A><A xmlns="http://default.reportlab.com/" xmlns:bongo="http://bongo.reportlab.com/">default ns A<bongo:A>bongo A</bongo:A><B>default NS B</B></A></foo:A>''',('{http://www.foo.org/}A', {'xmlns:foo': 'http://www.foo.org/'}, [('{http://www.foo.org/}B', None, [('{http://www.bar.org/}C', {'xmlns:foo': 'http://www.bar.org/'}, [('{http://www.bar.org/}D', None, ['abcd'], None)], None)], None), ('{http://www.foo.org/}B', None, None, None), ('A', None, ['bare A', ('C', None, ['bare C'], None), ('B', None, ['bare B'], None)], None), ('{http://default.reportlab.com/}A', {'xmlns': 'http://default.reportlab.com/', 'xmlns:bongo': 'http://bongo.reportlab.com/'}, ['default ns A', ('{http://bongo.reportlab.com/}A', None, ['bongo A'], None), ('{http://default.reportlab.com/}B', None, ['default NS B'], None)], None)], None),XMLNamespaces=1,ReturnNamespaceAttributes=1)
@@ -180,17 +172,17 @@
 	failTest('<!DOCTYPE foo SYSTEM "badt-have-unicode-content.dtd"><foo><a>aaa</a><b>bbbb</b></foo>',"error Error: Couldn't open dtd entity badt-have-unicode-content.dtd\\n in unnamed entity at line 1 char 54 of [unknown]",inOnly=1,NoNoDTDWarning=0,eoCB=eoDTD)
 
 def main():
-	#import pyRXP
-	import pyRXPU
 	if '__doc__' in sys.argv:
 		print(pyRXPU.__doc__)
 	else:
-		if pyRXPU: _runTests(pyRXPU)
-		msg = ("\n%d tests" % _total) + (', %d FAILED!' % _bad if _bad else '\n\nOK')
-		print(msg)
-		plogn(msg)
-		if _bad:
-			sys.exit('failure in testRXPBasic')
+		global _logf
+		with open('pyRXP_test.log','w') as _logf:
+			_runTests()
+			msg = ("\n%d tests" % _total) + (', %d FAILED!' % _bad if _bad else '\n\nOK')
+			print(msg)
+			plogn(msg)
+			if _bad:
+				sys.exit('failure in testRXPBasic')
 
 if __name__=='__main__': #noruntests
 	main()
--- a/test/test_xmltestsuite.py	Tue Sep 28 17:04:49 2021 +0100
+++ b/test/test_xmltestsuite.py	Tue Sep 28 17:06:17 2021 +0100
@@ -13,7 +13,7 @@
 __author__ = 'Stuart Bishop <stuart@stuartbishop.net>'
 
 import unittest, zipfile, sys, os, os.path, codecs
-debug = int(os.environ.get('RL_DEBUG','0'))
+debug = int(os.environ.get('DEBUG_PYRXP','0'))
 import pyRXPU
 
 # Debug is to help me trace down memory bugs
@@ -22,13 +22,18 @@
 class test_pyRXPU(unittest.TestCase):
 
 	def parse(self,filename,**kw):
-		if debug: print('About to parse %s' % filename, file=sys.stderr)
+		if debug&2: print('##### About to parse %s' % filename, file=sys.stderr)
 		kw = kw.copy()
 		kw['ReturnComments'] = 0
 		kw['ExpandEmpty'] = 1
 		kw['XMLLessThan'] = 1
 		kw['ReturnProcessingInstructions'] = 1
 		kw['ReturnList'] = 1
+		if debug&4:
+			def eocb(s):
+				print(f'+++++ eocb({s!r})',file=sys.stderr,flush=True)
+				return s
+			kw['eoCB'] = eocb
 		parser = pyRXPU.Parser(**kw)
 		# Change directory in case we are loading entities from cwd
 		retdir = os.getcwd()
@@ -37,12 +42,15 @@
 		try:
 			with open(n,'rb') as f:
 				xml = f.read()
-			return parser.parse(xml)
+			r = parser.parse(xml)
+			if debug:
+				print(f'##### r={ascii(r)}',file=sys.stderr,flush=True)
+			return r
 		finally:
 			os.chdir(retdir)
-			if debug: print('Done parsing   %s' % filename, file=sys.stderr)
-			if debug: print('='*60, file=sys.stderr)
-			if debug==1: time.sleep(1)
+			if debug&2: print('Done parsing   %s' % filename, file=sys.stderr,flush=True)
+			if debug&2: print('='*60, file=sys.stderr,flush=True)
+			if debug&16: time.sleep(1)
 
 	def getcanonical(self,filename):
 		''' Parse in the named file, and return it as canonical XML '''
@@ -90,28 +98,29 @@
 		with codecs.open(outname,mode='rb',encoding='utf8') as f:
 			outxml = f.read()
 		self.assertEqual(inxml,outxml,'%s != %s' % (inname,outname))
+		if debug&8: print(f'##### _test_valid({inname!r}) OK')  
 
 	def _test_invalid_parse(self,inname):
 		try:
 			self.parse(inname,Validate=0)
 		except pyRXPU.error:
-			pass
+			if debug&8: print(f'##### _test_invalid_parse({inname!r}) OK')  
 
 	def _test_invalid_validate(self,inname):
 		try:
 			self.parse(inname,Validate=1)
-			self.fail('Failed to detect validity error in %r' % inname)
+			self.fail('!!!!! Failed to detect validity error in %r' % inname)
 		except pyRXPU.error:
-			pass
+			if debug&8: print(f'##### _test_invalid_validate({inname!r}) OK')  
 
 	def _test_notwf(self,inname):
 		try:
 			self.parse(inname,Validate=0)
 			self.fail(
-				'Failed to detect that %r was not well formed' % inname
+				'!!!!! Failed to detect that %r was not well formed' % inname
 				)
 		except pyRXPU.error:
-			pass
+			if debug&8: print(f'##### _test_notwf({inname!r}) OK')  
 
 def buildup_test(cls=test_pyRXPU,I=[]):
 	''' Add test methods to the TestCase '''
@@ -120,12 +129,16 @@
 	cls.notwf = []
 	testdir = os.path.dirname(__file__)
 	try:
-		zipf = zipfile.ZipFile(os.path.join(testdir,'xmltest.zip'))
+		zipfName = os.path.join(testdir,'xmltest.zip')
+		zipf = zipfile.ZipFile(zipfName)
 	except:
-		print("Can't locate file xmltest.zip\nPerhaps it should be downloaded from\nhttp://www.reportlab.com/ftp/xmltest.zip\nor\nftp://ftp.jclark.com/pub/xml/xmltest.zip\n", file=sys.stderr)
+		print("!!!!! Can't locate file xmltest.zip\nPerhaps it should be downloaded from\nhttp://www.reportlab.com/ftp/xmltest.zip\nor\nftp://ftp.jclark.com/pub/xml/xmltest.zip\n", file=sys.stderr)
 		raise
+	else:
+		if debug&8:
+			print(f'##### obtained {zipf!r} from {zipfName!r}') 
 
-	for zipname in zipf.namelist():
+	for zipname in sorted(zipf.namelist()):
 		#skip broken tests
 		#if sys.platform=='win32':
 		#	if zipname in ('xmltest/valid/ext-sa/014.xml',
@@ -139,9 +152,10 @@
 		if not os.path.isdir(dir):
 			os.makedirs(dir)
 		if not os.path.isfile(osname):
-			f = open(osname,'wb')
-			f.write(zipf.read(zipname))
-			f.close()
+			if debug&8: print(f'##### creating {osname!r}...',end='...')
+			with open(osname,'wb') as f:
+				f.write(zipf.read(zipname))
+			if debug&8: print('written',flush=True)
 		if I and zipname not in I: continue
 
 		# Add input files to our lists
@@ -154,14 +168,20 @@
 				outname = os.path.join(dir,'out',os.path.basename(osname))
 				cls.valid.append( (osname,outname) )
 
+	if debug&8:
+		print(f'##### |{cls!r}.valid| = {len(cls.valid)}',flush=True)
+		print(f'##### |{cls!r}.invalid| = {len(cls.invalid)}',flush=True)
+		print(f'##### |{cls!r}.notwf| = {len(cls.notwf)}',flush=True)
+
 	# Add 'valid' tests
 	for inname,outname in cls.valid:
 		num = int(os.path.splitext(os.path.basename(inname))[0])
 		dir = os.path.split(os.path.split(inname)[0])[1]
-		mname = 'test_Valid_%s_%03d' % (dir,num)
+		mname = ('test_Valid_%s_%03d' % (dir,num)).replace('-','_')
 		def doTest(self,inname=inname,outname=outname):
 			self._test_valid(inname,outname)
 		setattr(cls,mname,doTest)
+	if debug&8: print('##### valid tests created')
 
 	# Add 'invalid' tests
 	for inname in cls.invalid:
@@ -174,22 +194,38 @@
 		def doTest(self,inname=inname):
 			self._test_invalid_validate(inname)
 		setattr(cls,mname,doTest)
+	if debug&8: print('##### invalid tests created')
 
 	# Add 'not wellformed' tests
 	for inname in cls.notwf:
 		num = int(os.path.splitext(os.path.basename(inname))[0])
 		dir = os.path.split(os.path.split(inname)[0])[1]
-		mname = 'test_NotWellFormed_%s_%03d' % (dir,num)
+		mname = ('test_NotWellFormed_%s_%03d' % (dir,num)).replace('-','_')
 		def doTest(self,inname=inname):
 			self._test_notwf(inname)
 		setattr(cls,mname,doTest)
+	if debug&8: print('##### notwf tests created')
 
-def main():
+def makeTests(debug=0):
+	globals()['debug'] = debug
 	I = filter(lambda a: a[:2]=='-I',sys.argv)
 	for i in I: sys.argv.remove(i)
 	I = list(map(lambda x: x[2:],I))
 	buildup_test(I=I)
-	unittest.main(module=__name__)
+
+def main(verbose=0, singles=0):
+	makeTests(debug=debug)
+	if singles:
+		T = sorted([t for t in test_pyRXPU.__dict__ if t.startswith("test_")])
+		for t in T:
+			print(f'+++++ test_xmltestsuite.test_pyRXPU.{t}',end='.....',flush=True)
+			r = os.system(' '.join((sys.executable, 'test_xmltestsuite.py', f'test_pyRXPU.{t}')))
+			if r:
+				print(f'FAIL !!!!! {t} exited with error {r} !!!!!')
+			else:
+				print(' OK')
+	else:
+		unittest.main(module=__name__,exit=False)
 
 if __name__ == '__main__':
 	main()