You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

508 lines
23 KiB

#! /usr/bin/env python3
import os
import subprocess
import sys
import platform
import argparse
import re
from glob import glob
# Tests require at least CPython 3.3. If your default python3 executable
# is of lower version, you can point MICROPY_CPYTHON3 environment var
# to the correct executable.
if == 'nt':
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../windows/micropython.exe')
CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')
MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../unix/micropython')
# mpy-cross is only needed if --via-mpy command-line arg is passed
MPYCROSS = os.getenv('MICROPY_MPYCROSS', '../mpy-cross/mpy-cross')
# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale
os.environ['PYTHONIOENCODING'] = 'utf-8'
def rm_f(fname):
if os.path.exists(fname):
# unescape wanted regex chars and escape unwanted ones
def convert_regex_escapes(line):
cs = []
escape = False
for c in str(line, 'utf8'):
if escape:
escape = False
elif c == '\\':
escape = True
elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'):
cs.append('\\' + c)
# accept carriage-return(s) before final newline
if cs[-1] == '\n':
cs[-1] = '\r*\n'
return bytes(''.join(cs), 'utf8')
def run_micropython(pyb, args, test_file, is_special=False):
special_tests = ('micropython/', 'basics/', 'basics/', 'thread/')
if pyb is None:
# run on PC
if test_file.startswith(('cmdline/', 'feature_check/')) or test_file in special_tests:
# special handling for tests of the unix cmdline program
is_special = True
if is_special:
# check for any cmdline options needed for this test
with open(test_file, 'rb') as f:
line = f.readline()
if line.startswith(b'# cmdline:'):
# subprocess.check_output on Windows only accepts strings, not bytes
args += [str(c, 'utf-8') for c in line[10:].strip().split()]
# run the test, possibly with redirected input
if 'repl_' in test_file:
# Need to use a PTY to test command line editing
import pty
except ImportError:
# in case pty module is not available, like on Windows
return b'SKIP\n'
import select
def get(required=False):
rv = b''
while True:
ready =[master], [], [], 0.02)
if ready[0] == [master]:
rv +=, 1024)
if not required or rv:
return rv
def send_get(what):
os.write(master, what)
return get()
with open(test_file, 'rb') as f:
# instead of: output_mupy = subprocess.check_output(args, stdin=f)
master, slave = pty.openpty()
p = subprocess.Popen(args, stdin=slave, stdout=slave,
stderr=subprocess.STDOUT, bufsize=0)
banner = get(True)
output_mupy = banner + b''.join(send_get(line) for line in f)
send_get(b'\x04') # exit the REPL, so coverage info is saved
output_mupy = subprocess.check_output(args + [test_file])
except subprocess.CalledProcessError:
return b'CRASH'
# a standard test run on PC
# create system command
cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit]
if args.heapsize is not None:
cmdlist.extend(['-X', 'heapsize=' + args.heapsize])
# if running via .mpy, first compile the .py file
if args.via_mpy:
subprocess.check_output([MPYCROSS, '-mcache-lookup-bc', '-o', 'mpytest.mpy', test_file])
cmdlist.extend(['-m', 'mpytest'])
# run the actual test
e = {"MICROPYPATH": os.getcwd() + ":", "LANG": "en_US.UTF-8"}
p = subprocess.Popen(cmdlist, env=e, stdout=subprocess.PIPE)
output_mupy = b''
while p.poll() is None:
output_mupy +=
output_mupy +=
if p.returncode != 0:
output_mupy = b'CRASH'
# clean up if we had an intermediate .mpy file
if args.via_mpy:
# run on pyboard
import pyboard
output_mupy = pyb.execfile(test_file)
except pyboard.PyboardError:
output_mupy = b'CRASH'
# canonical form for all ports/platforms is to use \n for end-of-line
output_mupy = output_mupy.replace(b'\r\n', b'\n')
# don't try to convert the output if we should skip this test
if output_mupy in (b'SKIP\n', b'CRASH'):
return output_mupy
if is_special or test_file in special_tests:
# convert parts of the output that are not stable across runs
with open(test_file + '.exp', 'rb') as f:
lines_exp = []
for line in f.readlines():
if line == b'########\n':
line = (line,)
line = (line, re.compile(convert_regex_escapes(line)))
lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')]
if output_mupy.endswith(b'\n'):
lines_mupy = lines_mupy[:-1] # remove erroneous last empty line
i_mupy = 0
for i in range(len(lines_exp)):
if lines_exp[i][0] == b'########\n':
# 8x #'s means match 0 or more whole lines
line_exp = lines_exp[i + 1]
skip = 0
while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]):
skip += 1
if i_mupy + skip >= len(lines_mupy):
lines_mupy[i_mupy] = b'######## FAIL\n'
del lines_mupy[i_mupy:i_mupy + skip]
lines_mupy.insert(i_mupy, b'########\n')
i_mupy += 1
# a regex
if lines_exp[i][1].match(lines_mupy[i_mupy]):
lines_mupy[i_mupy] = lines_exp[i][0]
#print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG
i_mupy += 1
if i_mupy >= len(lines_mupy):
output_mupy = b''.join(lines_mupy)
return output_mupy
def run_feature_check(pyb, args, base_path, test_file):
return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True)
def run_tests(pyb, tests, args, base_path="."):
test_count = 0
testcase_count = 0
passed_count = 0
failed_tests = []
skipped_tests = []
skip_tests = set()
skip_native = False
skip_int_big = False
skip_set_type = False
skip_async = False
skip_const = False
# Check if micropython.native is supported, and skip such tests if it's not
native = run_feature_check(pyb, args, base_path, '')
if native == b'CRASH':
skip_native = True
# Check if arbitrary-precision integers are supported, and skip such tests if it's not
native = run_feature_check(pyb, args, base_path, '')
if native != b'1000000000000000000000000000000000000000000000\n':
skip_int_big = True
# Check if set type (and set literals) is supported, and skip such tests if it's not
native = run_feature_check(pyb, args, base_path, '')
if native == b'CRASH':
skip_set_type = True
# Check if async/await keywords are supported, and skip such tests if it's not
native = run_feature_check(pyb, args, base_path, '')
if native == b'CRASH':
skip_async = True
# Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not
native = run_feature_check(pyb, args, base_path, '')
if native == b'CRASH':
skip_const = True
# Check if emacs repl is supported, and skip such tests if it's not
t = run_feature_check(pyb, args, base_path, '')
if not 'True' in str(t, 'ascii'):
upy_byteorder = run_feature_check(pyb, args, base_path, '')
upy_float_precision = int(run_feature_check(pyb, args, base_path, ''))
has_complex = run_feature_check(pyb, args, base_path, '') == b'complex\n'
has_coverage = run_feature_check(pyb, args, base_path, '') == b'coverage\n'
cpy_byteorder = subprocess.check_output([CPYTHON3, base_path + '/feature_check/'])
skip_endian = (upy_byteorder != cpy_byteorder)
# Some tests shouldn't be run under Travis CI
if os.getenv('TRAVIS') == 'true':
skip_tests.add('thread/') # has reliability issues
skip_tests.add('thread/') # has reliability issues
skip_tests.add('thread/') # has reliability issues
skip_tests.add('thread/') # has reliability issues
if upy_float_precision == 0:
if upy_float_precision < 32:
skip_tests.add('float/') # requires fp32, there's instead
skip_tests.add('float/') # requires fp32, there's instead
skip_tests.add('float/') # requires fp32
skip_tests.add('float/') # requires fp32
if upy_float_precision < 64:
skip_tests.add('float/') # tested by float/ instead
if not has_complex:
if not has_coverage:
# Some tests shouldn't be run on a PC
if pyb is None:
# unix build does not have the GIL so can't run thread mutation tests
for t in tests:
if t.startswith('thread/mutate_'):
# Some tests shouldn't be run on pyboard
if pyb is not None:
skip_tests.add('basics/') # warning is not printed
skip_tests.add('micropython/') # output is very different to PC output
skip_tests.add('extmod/') # raw memory access not supported
if == 'wipy':
skip_tests.add('misc/') # requires error reporting full
skip_tests.add('misc/') # requires stack checking enabled
skip_tests.add('misc/') # requires stack checking enabled
skip_tests.add('misc/') # requires stack checking enabled
skip_tests.update({'extmod/' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes
skip_tests.add('extmod/') # requires zlib
skip_tests.add('extmod/') # uheapq not supported by WiPy
skip_tests.add('extmod/') # requires urandom
skip_tests.add('extmod/') # requires urandom
elif == 'esp8266':
skip_tests.add('misc/') # too large
elif == 'minimal':
skip_tests.add('misc/') # too large
skip_tests.add('micropython/') # don't assume line numbers are stored
# Some tests are known to fail on 64-bit machines
if pyb is None and platform.architecture()[0] == '64bit':
# Some tests use unsupported features on Windows
if == 'nt':
skip_tests.add('import/') # works but CPython prints forward slashes
# Some tests are known to fail with native emitter
# Remove them from the below when they work
if args.emit == 'native':
skip_tests.update({'basics/' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_return generator_send'.split()}) # require yield
skip_tests.update({'basics/' % t for t in 'bytes_gen class_store_class globals_del string_join'.split()}) # require yield
skip_tests.update({'basics/' % t for t in 'def await await2 for for2 with with2'.split()}) # require yield
skip_tests.update({'basics/' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs
skip_tests.update({'basics/' % t for t in 'with_break with_continue with_return'.split()}) # require complete with support
skip_tests.add('basics/') # requires generators
skip_tests.add('basics/') # seems to randomly fail
skip_tests.add('basics/') # requires yield
skip_tests.add('basics/') # requires yield
skip_tests.add('basics/') # requires checking for unbound local
skip_tests.add('basics/') # requires checking for unbound local
skip_tests.add('basics/') # raise from is not supported
skip_tests.add('basics/') # requires yield_value
skip_tests.add('basics/') # requires proper try finally code
skip_tests.add('basics/') # requires proper try finally code
skip_tests.add('basics/') # requires proper try finally code
skip_tests.add('basics/') # requires checking for unbound local
skip_tests.add('import/') # requires yield_value
skip_tests.add('misc/') # requires raise_varargs
skip_tests.add('misc/') # requires yield
skip_tests.add('misc/') # because native doesn't have proper traceback info
skip_tests.add('misc/') # sys.exc_info() is not supported for native
skip_tests.add('micropython/') # because native doesn't have proper traceback info
skip_tests.add('micropython/') # requires generators
skip_tests.add('micropython/') # native code doesn't check pending events
for test_file in tests:
test_file = test_file.replace('\\', '/')
test_basename = os.path.basename(test_file)
test_name = os.path.splitext(test_basename)[0]
is_native = test_name.startswith("native_") or test_name.startswith("viper_")
is_endian = test_name.endswith("_endian")
is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig")
is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset")
is_async = test_name.startswith("async_")
is_const = test_name.startswith("const")
skip_it = test_file in skip_tests
skip_it |= skip_native and is_native
skip_it |= skip_endian and is_endian
skip_it |= skip_int_big and is_int_big
skip_it |= skip_set_type and is_set_type
skip_it |= skip_async and is_async
skip_it |= skip_const and is_const
if skip_it:
print("skip ", test_file)
# get expected output
test_file_expected = test_file + '.exp'
if os.path.isfile(test_file_expected):
# expected output given by a file, so read that in
with open(test_file_expected, 'rb') as f:
output_expected =
# run CPython to work out expected output
e = {"PYTHONPATH": os.getcwd(),
"PATH": os.environ["PATH"],
"LANG": "en_US.UTF-8"}
p = subprocess.Popen([CPYTHON3, '-B', test_file], env=e, stdout=subprocess.PIPE)
output_expected = b''
while p.poll() is None:
output_expected +=
output_expected +=
if p.returncode != 0:
output_expected = b'CPYTHON3 CRASH'
elif args.write_exp:
with open(test_file_expected, 'wb') as f:
# canonical form for all host platforms is to use \n for end-of-line
output_expected = output_expected.replace(b'\r\n', b'\n')
if args.write_exp:
# run MicroPython
output_mupy = run_micropython(pyb, args, test_file)
if output_mupy == b'SKIP\n':
print("skip ", test_file)
testcase_count += len(output_expected.splitlines())
filename_expected = test_basename + ".exp"
filename_mupy = test_basename + ".out"
if output_expected == output_mupy:
print("pass ", test_file)
passed_count += 1
with open(filename_expected, "wb") as f:
with open(filename_mupy, "wb") as f:
print("### Expected")
print("### Actual")
print("FAIL ", test_file)
test_count += 1
print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))
print("{} tests passed".format(passed_count))
if len(skipped_tests) > 0:
print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests)))
if len(failed_tests) > 0:
print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests)))
return False
# all tests succeeded
return True
def main():
cmd_parser = argparse.ArgumentParser(description='Run tests for MicroPython.')
cmd_parser.add_argument('--target', default='unix', help='the target platform')
cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard')
cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device')
cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')
cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')
cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')
cmd_parser.add_argument('--write-exp', action='store_true', help='save .exp files to run tests w/o CPython')
cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)')
cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)')
cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first')
cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests')
cmd_parser.add_argument('files', nargs='*', help='input test files')
args = cmd_parser.parse_args()
EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'minimal')
import pyboard
pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)
elif == 'unix':
pyb = None
raise ValueError('target must be either %s or unix' % ", ".join(EXTERNAL_TARGETS))
if len(args.files) == 0:
if args.test_dirs is None:
if == 'pyboard':
# run pyboard tests
test_dirs = ('basics', 'micropython', 'float', 'misc', 'stress', 'extmod', 'pyb', 'pybnative', 'inlineasm')
elif in ('esp8266', 'minimal'):
test_dirs = ('basics', 'micropython', 'float', 'misc', 'extmod')
elif == 'wipy':
# run WiPy tests
test_dirs = ('basics', 'micropython', 'misc', 'extmod', 'wipy')
# run PC tests
test_dirs = ('basics', 'micropython', 'float', 'import', 'io', 'misc', 'stress', 'unicode', 'extmod', 'unix', 'cmdline')
# run tests from these directories
test_dirs = args.test_dirs
tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)
# tests explicitly given
tests = args.files
if not args.keep_path:
# clear search path to make sure tests use only builtin modules
os.environ['MICROPYPATH'] = ''
# Even if we run completely different tests in a different directory,
# we need to access feature_check's from the same directory as the
# run-tests script itself.
base_path = os.path.dirname(sys.argv[0]) or "."
res = run_tests(pyb, tests, args, base_path)
if pyb:
if not res:
if __name__ == "__main__":