runtest.py: order of magnitude faster.

C#, VB and Lua tests are broken though.

4 files changed, 205 insertions, 30 deletions

diff --git a/Makefile b/Makefile
index 674ad71..6405422 100644
--- a/Makefile
+++ b/Makefile
@@ -29,18 +29,21 @@ stepA = stepA_mal
 EXCLUDE_TESTS += test^bash^step5 # no stack exhaustion or completion
 EXCLUDE_TESTS += test^c^step5    # segfault
 EXCLUDE_TESTS += test^cs^step5   # fatal stack overflow fault
+EXCLUDE_TESTS += test^haskell^step5 # test completes
 EXCLUDE_TESTS += test^make^step5 # no TCO capability/step
 EXCLUDE_TESTS += test^mal^step5  # no TCO capability/step
 EXCLUDE_TESTS += test^go^step5   # test completes, even at 100,000
 EXCLUDE_TESTS += test^php^step5  # test completes, even at 100,000
+EXCLUDE_TESTS += test^racket^step5 # test completes
 EXCLUDE_TESTS += test^ruby^step5 # test completes, even at 100,000
 EXCLUDE_TESTS += test^rust^step5 # no catching stack overflows
 EXCLUDE_TESTS += test^ocaml^step5 # test completes, even at 1,000,000
 
 # interop tests now implemented yet
-EXCLUDE_TESTS += test^cs^stepA test^java^stepA test^mal^stepA \
-		 test^mal^step0 test^php^stepA test^ps^stepA \
-		 test^python^stepA test^ruby^stepA
+EXCLUDE_TESTS += test^cs^stepA test^go^stepA test^haskell^stepA \
+		 test^java^stepA test^mal^stepA test^mal^step0 \
+		 test^php^stepA test^ps^stepA test^python^stepA \
+		 test^ruby^stepA
 
 EXCLUDE_PERFS = perf^mal  # TODO: fix this
 
@@ -112,7 +115,7 @@ vb_RUNSTEP =      mono ../$(2) --raw $(3)
 
 # Extra options to pass to runtest.py
 cs_TEST_OPTS =  --redirect
-mal_TEST_OPTS = --redirect --start-timeout 60 --test-timeout 120
+mal_TEST_OPTS = --start-timeout 60 --test-timeout 120
 vb_TEST_OPTS =  --redirect
 
 
diff --git a/docs/TODO b/docs/TODO
index e5a4fc1..0452fe0 100644
--- a/docs/TODO
+++ b/docs/TODO
@@ -9,8 +9,8 @@ All:
       of iterations per second
     - redefine (defmacro!) as (def! (macro*))
     - runtest expect fixes:
-        - stop using expect, so we can drop --raw option
-        - fix long lines in runtext/expect
+        * stop using expect, so we can drop --raw option
+        - fix long line splitting in runtest
     - regular expression matching in runtest
     - add re (use in rep) everywhere and use that (to avoid printing)
     - Implement/fix interop
diff --git a/runtest-old.py b/runtest-old.py
new file mode 100755
index 0000000..aacd770
--- /dev/null
+++ b/runtest-old.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+
+import os, sys, re
+import argparse
+
+# http://pexpect.sourceforge.net/pexpect.html
+from pexpect import spawn, EOF, TIMEOUT
+
+# TODO: do we need to support '\n' too
+sep = "\r\n"
+rundir = None
+
+parser = argparse.ArgumentParser(
+        description="Run a test file against a Mal implementation")
+parser.add_argument('--rundir',
+        help="change to the directory before running tests")
+parser.add_argument('--start-timeout', default=10, type=int,
+        help="default timeout for initial prompt")
+parser.add_argument('--test-timeout', default=20, type=int,
+        help="default timeout for each individual test action")
+parser.add_argument('--pre-eval', default=None, type=str,
+        help="Mal code to evaluate prior to running the test")
+parser.add_argument('--redirect', action='store_true',
+        help="Run implementation in bash and redirect output to /dev/null")
+
+parser.add_argument('test_file', type=argparse.FileType('r'),
+        help="a test file formatted as with mal test data")
+parser.add_argument('mal_cmd', nargs="*",
+        help="Mal implementation command line. Use '--' to "
+             "specify a Mal command line with dashed options.")
+
+args = parser.parse_args(sys.argv[1:])
+test_data = args.test_file.read().split('\n')
+
+if args.rundir: os.chdir(args.rundir)
+
+if args.redirect:
+    # Redirect to try and force raw mode (no ASCII codes)
+    p = spawn('/bin/bash -c "' + " ".join(args.mal_cmd) + ' |tee /dev/null"')
+else:
+    p = spawn(args.mal_cmd[0], args.mal_cmd[1:])
+
+
+test_idx = 0
+def read_test(data):
+    global test_idx
+    form, output, ret = None, "", None
+    while data:
+        test_idx += 1
+        line = data.pop(0)
+        if re.match(r"^\s*$", line):   # blank line
+            continue
+        elif line[0:3] == ";;;":       # ignore comment
+            continue
+        elif line[0:2] == ";;":        # output comment
+            print line[3:]
+            continue
+        elif line[0:2] == ";":         # unexpected comment
+            print "Test data error at line %d:\n%s" % (test_idx, line)
+            return None, None, None, test_idx
+        form = line   # the line is a form to send
+
+        # Now find the output and return value
+        while data:
+            line = data[0]
+            if line[0:3] == ";=>":
+                ret = line[3:].replace('\\r', '\r').replace('\\n', '\n')
+                test_idx += 1
+                data.pop(0)
+                break
+            elif line[0:2] == "; ":
+                output = output + line[2:] + sep
+                test_idx += 1
+                data.pop(0)
+            else:
+                ret = "*"
+                break
+        if ret: break
+
+    return form, output, ret, test_idx
+
+def assert_prompt(timeout):
+    # Wait for the initial prompt
+    idx = p.expect(['user> ', 'mal-user> ', EOF, TIMEOUT],
+                timeout=timeout)
+    if idx not in [0,1]:
+        print "Did not get 'user> ' or 'mal-user> ' prompt"
+        print "    Got      : %s" % repr(p.before)
+        sys.exit(1)
+
+
+# Wait for the initial prompt
+assert_prompt(args.start_timeout)
+
+# Send the pre-eval code if any
+if args.pre_eval:
+    sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval)
+    p.sendline(args.pre_eval)
+    assert_prompt(args.test_timeout)
+
+fail_cnt = 0
+
+while test_data:
+    form, out, ret, line_num = read_test(test_data)
+    if form == None:
+        break
+    sys.stdout.write("TEST: %s -> [%s,%s]" % (form, repr(out), repr(ret)))
+    sys.stdout.flush()
+    expected = "%s%s%s%s" % (form, sep, out, ret)
+
+    p.sendline(form)
+    try:
+        idx = p.expect(['\r\nuser> ', '\nuser> ',
+                        '\r\nmal-user> ', '\nmal-user> '],
+                       timeout=args.test_timeout)
+        #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
+        if ret == "*" or p.before == expected:
+            print " -> SUCCESS"
+        else:
+            print " -> FAIL (line %d):" % line_num
+            print "    Expected : %s" % repr(expected)
+            print "    Got      : %s" % repr(p.before)
+            fail_cnt += 1
+    except EOF:
+        print "Got EOF"
+        sys.exit(1)
+    except TIMEOUT:
+        print "Got TIMEOUT, received: %s" % repr(p.before)
+        sys.exit(1)
+
+if fail_cnt > 0:
+    print "FAILURES: %d" % fail_cnt
+    sys.exit(2)
+sys.exit(0)
diff --git a/runtest.py b/runtest.py
index aacd770..858c634 100755
--- a/runtest.py
+++ b/runtest.py
@@ -1,13 +1,15 @@
 #!/usr/bin/env python
 
 import os, sys, re
-import argparse
+import argparse, time
 
-# http://pexpect.sourceforge.net/pexpect.html
-from pexpect import spawn, EOF, TIMEOUT
+import pty
+from subprocess import Popen, STDOUT, PIPE
+from select import select
 
 # TODO: do we need to support '\n' too
 sep = "\r\n"
+#sep = "\n"
 rundir = None
 
 parser = argparse.ArgumentParser(
@@ -29,16 +31,53 @@ parser.add_argument('mal_cmd', nargs="*",
         help="Mal implementation command line. Use '--' to "
              "specify a Mal command line with dashed options.")
 
+class Runner():
+    def __init__(self, args, redirect=False):
+        print "args: %s" % repr(args)
+        if redirect:
+            print "using redirect"
+            self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
+            self.stdin = self.p.stdin
+            self.stdout = self.p.stdout
+        else:
+            # provide tty to get 'interactive' readline to work
+            master, slave = pty.openpty()
+            self.p = Popen(args, bufsize=0, stdin=slave, stdout=slave, stderr=STDOUT)
+            self.stdin = os.fdopen(master, 'r+b', 0)
+            self.stdout = self.stdin
+
+        #print "started"
+        self.buf = ""
+        self.last_prompt = ""
+
+    def read_to_prompt(self, prompts, timeout):
+        end_time = time.time() + timeout
+        while time.time() < end_time:
+            [outs,_,_] = select([self.stdin], [], [], 1)
+            if self.stdin in outs:
+                new_data = self.stdin.read(1)
+                #print "new_data: '%s'" % new_data
+                self.buf += new_data
+                for prompt in prompts:
+                    regexp = re.compile(prompt)
+                    match = regexp.search(self.buf)
+                    if match:
+                        end = match.end()
+                        buf = self.buf[0:end-len(prompt)]
+                        self.buf = self.buf[end:]
+                        self.last_prompt = prompt
+                        return buf
+        return None
+
+    def write(self, str):
+        self.stdout.write(str)
+
 args = parser.parse_args(sys.argv[1:])
 test_data = args.test_file.read().split('\n')
 
 if args.rundir: os.chdir(args.rundir)
 
-if args.redirect:
-    # Redirect to try and force raw mode (no ASCII codes)
-    p = spawn('/bin/bash -c "' + " ".join(args.mal_cmd) + ' |tee /dev/null"')
-else:
-    p = spawn(args.mal_cmd[0], args.mal_cmd[1:])
+r = Runner(args.mal_cmd, redirect=args.redirect)
 
 
 test_idx = 0
@@ -81,11 +120,13 @@ def read_test(data):
 
 def assert_prompt(timeout):
     # Wait for the initial prompt
-    idx = p.expect(['user> ', 'mal-user> ', EOF, TIMEOUT],
-                timeout=timeout)
-    if idx not in [0,1]:
+    header = r.read_to_prompt(['user> ', 'mal-user> '], timeout=timeout)
+    if not header == None:
+        if header:
+            print "Started with:\n%s" % header
+    else:
         print "Did not get 'user> ' or 'mal-user> ' prompt"
-        print "    Got      : %s" % repr(p.before)
+        print "    Got      : %s" % repr(r.buf)
         sys.exit(1)
 
 
@@ -95,7 +136,7 @@ assert_prompt(args.start_timeout)
 # Send the pre-eval code if any
 if args.pre_eval:
     sys.stdout.write("RUNNING pre-eval: %s" % args.pre_eval)
-    p.sendline(args.pre_eval)
+    p.write(args.pre_eval)
     assert_prompt(args.test_timeout)
 
 fail_cnt = 0
@@ -108,24 +149,21 @@ while test_data:
     sys.stdout.flush()
     expected = "%s%s%s%s" % (form, sep, out, ret)
 
-    p.sendline(form)
+    r.write(form + "\n")
     try:
-        idx = p.expect(['\r\nuser> ', '\nuser> ',
-                        '\r\nmal-user> ', '\nmal-user> '],
-                       timeout=args.test_timeout)
+        res = r.read_to_prompt(['\r\nuser> ', '\nuser> ',
+                                '\r\nmal-user> ', '\nmal-user> '],
+                                timeout=args.test_timeout)
         #print "%s,%s,%s" % (idx, repr(p.before), repr(p.after))
-        if ret == "*" or p.before == expected:
+        if ret == "*" or res == expected:
             print " -> SUCCESS"
         else:
             print " -> FAIL (line %d):" % line_num
             print "    Expected : %s" % repr(expected)
-            print "    Got      : %s" % repr(p.before)
+            print "    Got      : %s" % repr(res)
             fail_cnt += 1
-    except EOF:
-        print "Got EOF"
-        sys.exit(1)
-    except TIMEOUT:
-        print "Got TIMEOUT, received: %s" % repr(p.before)
+    except:
+        print "Got Exception"
         sys.exit(1)
 
 if fail_cnt > 0: