All: perf test, Makefile refactor, add *host-language*

Other:
- bash,make,postscript: quasiquote of vectors
- Fix Java slurp
- add time function to core.mal
    - switches on *host-language* for make time-secs vs time-ms
- Ignore */experiments directories

1 files changed, 53 insertions, 29 deletions

diff --git a/Makefile b/Makefile
index e986df4..2a99452 100644
--- a/Makefile
+++ b/Makefile
@@ -25,6 +25,7 @@ step9 = step9_interop
 stepA = stepA_more
 
 EXCLUDE_TESTS = test^make^step5 test^mal^step0 test^mal^step5 test^mal^step9 test^java^step9 test^cs^step9
+EXCLUDE_PERFS = perf^mal  # TODO: fix this
 
 #
 # Utility functions
@@ -32,32 +33,36 @@ EXCLUDE_TESTS = test^make^step5 test^mal^step0 test^mal^step5 test^mal^step9 tes
 
 STEP_TEST_FILES = $(strip $(wildcard $(1)/tests/$($(2)).mal) $(wildcard tests/$($(2)).mal))
 
-bash_STEP_TO_PROG = bash/$($(1)).sh
-c_STEP_TO_PROG = c/$($(1))
+bash_STEP_TO_PROG =    bash/$($(1)).sh
+c_STEP_TO_PROG =       c/$($(1))
 clojure_STEP_TO_PROG = clojure/src/$($(1)).clj
-cs_STEP_TO_PROG = cs/$($(1)).exe
-java_STEP_TO_PROG = java/src/main/java/mal/$($(1)).java
-js_STEP_TO_PROG = js/$($(1)).js
-make_STEP_TO_PROG = make/$($(1)).mk
-mal_STEP_TO_PROG = mal/$($(1)).mal
-php_STEP_TO_PROG = php/$($(1)).php
-ps_STEP_TO_PROG = ps/$($(1)).ps
-python_STEP_TO_PROG = python/$($(1)).py
-ruby_STEP_TO_PROG = ruby/$($(1)).rb
-
-
-bash_RUNTEST = ../runtest.py $(4) ../$(1) -- bash ../$(2) $(5)
-c_RUNTEST = ../runtest.py $(4) ../$(1) -- ../$(2) $(5)
-clojure_RUNTEST = ../runtest.py $(4) ../$(1) -- lein with-profile +$(3) trampoline run $(5)
-cs_RUNTEST = ../runtest.py --redirect $(4) ../$(1) -- mono --debug ../$(2) --raw $(5)
-java_RUNTEST = ../runtest.py $(4) ../$(1) -- mvn -quiet exec:java -Dexec.mainClass="mal.$($(3))" -Dexec.args="--raw$(if $(5), $(5),)"
-js_RUNTEST = ../runtest.py $(4) ../$(1) -- node ../$(2) $(5)
-make_RUNTEST = ../runtest.py $(4) ../$(1) -- make -f ../$(2) $(5)
-mal_RUNTEST = $(call $(MAL_IMPL)_RUNTEST,$(1),$(call $(MAL_IMPL)_STEP_TO_PROG,stepA),stepA,--start-timeout 30 --test-timeout 120,../$(2))
-php_RUNTEST = ../runtest.py $(4) ../$(1) -- php ../$(2) $(5)
-ps_RUNTEST = ../runtest.py $(4) ../$(1) -- "gs -q -dNODISPLAY -- ../$(2) $(5)"
-python_RUNTEST = ../runtest.py $(4) ../$(1) -- $(PYTHON) ../$(2) $(5)
-ruby_RUNTEST = ../runtest.py $(4) ../$(1) -- ruby ../$(2) $(5)
+cs_STEP_TO_PROG =      cs/$($(1)).exe
+java_STEP_TO_PROG =    java/src/main/java/mal/$($(1)).java
+js_STEP_TO_PROG =      js/$($(1)).js
+make_STEP_TO_PROG =    make/$($(1)).mk
+mal_STEP_TO_PROG =     mal/$($(1)).mal
+php_STEP_TO_PROG =     php/$($(1)).php
+ps_STEP_TO_PROG =      ps/$($(1)).ps
+python_STEP_TO_PROG =  python/$($(1)).py
+ruby_STEP_TO_PROG =    ruby/$($(1)).rb
+
+
+bash_RUNSTEP =    bash ../$(2) $(3)
+c_RUNSTEP =       ../$(2) $(3)
+clojure_RUNSTEP = lein with-profile +$(1) trampoline run $(3)
+cs_RUNSTEP =      mono ../$(2) --raw $(3)
+java_RUNSTEP =    mvn -quiet exec:java -Dexec.mainClass="mal.$($(1))" -Dexec.args="--raw$(if $(3), $(3),)"
+js_RUNSTEP =      node ../$(2) $(3)
+make_RUNSTEP =    make -f ../$(2) $(3)
+mal_RUNSTEP =     $(call $(MAL_IMPL)_RUNSTEP,$(1),$(call $(MAL_IMPL)_STEP_TO_PROG,stepA),../$(2),")  #"
+php_RUNSTEP =     php ../$(2) $(3)
+ps_RUNSTEP =      $(4)gs -q -dNODISPLAY -- ../$(2) $(3)$(4)
+python_RUNSTEP =  $(PYTHON) ../$(2) $(3)
+ruby_RUNSTEP =    ruby ../$(2) $(3)
+
+# Extra options to pass to runtest.py
+cs_TEST_OPTS =  --redirect
+mal_TEST_OPTS = --start-timeout 30 --test-timeout 120
 
 
 # Derived lists
@@ -72,6 +77,8 @@ ALL_TESTS = $(filter-out $(EXCLUDE_TESTS),\
 IMPL_STATS = $(foreach impl,$(IMPLS),stats^$(impl))
 IMPL_STATS_LISP = $(foreach impl,$(IMPLS),stats-lisp^$(impl))
 
+IMPL_PERF = $(filter-out $(EXCLUDE_PERFS),$(foreach impl,$(IMPLS),perf^$(impl)))
+
 #
 # Build rules
 #
@@ -95,8 +102,8 @@ $(ALL_TESTS): $$(call $$(word 2,$$(subst ^, ,$$(@)))_STEP_TO_PROG,$$(word 3,$$(s
 	    $(foreach test,$(call STEP_TEST_FILES,$(impl),$(step)),\
 	      echo '----------------------------------------------'; \
 	      echo 'Testing $@, step file: $+, test file: $(test)'; \
-	      echo 'Running: $(call $(impl)_RUNTEST,$(test),$(+),$(step))'; \
-	      $(call $(impl)_RUNTEST,$(test),$(+),$(step)))))
+	      echo 'Running: ../runtest.py $(call $(impl)_TEST_OPTS) ../$(test) -- $(call $(impl)_RUNSTEP,$(step),$(+))'; \
+	      ../runtest.py $(call $(impl)_TEST_OPTS) ../$(test) -- $(call $(impl)_RUNSTEP,$(step),$(+)))))
 
 test: $(ALL_TESTS)
 tests: $(ALL_TESTS)
@@ -104,6 +111,9 @@ tests: $(ALL_TESTS)
 
 # Stats rules
 
+stats: $(IMPL_STATS)
+stats-lisp: $(IMPL_STATS_LISP)
+
 .SECONDEXPANSION:
 $(IMPL_STATS):
 	@echo "----------------------------------------------"; \
@@ -118,5 +128,19 @@ $(IMPL_STATS_LISP):
 	  echo "Stats (lisp only) for $(impl):"; \
 	  $(MAKE) --no-print-directory -C $(impl) stats-lisp)
 
-stats: $(IMPL_STATS)
-stats-lisp: $(IMPL_STATS_LISP)
+
+# Performance test rules
+
+perf: $(IMPL_PERF)
+
+.SECONDEXPANSION:
+$(IMPL_PERF):
+	@echo "----------------------------------------------"; \
+	$(foreach impl,$(word 2,$(subst ^, ,$(@))),\
+	  cd $(if $(filter mal,$(impl)),$(MAL_IMPL),$(impl)); \
+	  echo "Performance test for $(impl):"; \
+	  echo 'Running: $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf1.mal)'; \
+          $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf1.mal); \
+	  echo 'Running: $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf2.mal)'; \
+          $(call $(impl)_RUNSTEP,stepA,$(call $(impl)_STEP_TO_PROG,stepA),../tests/perf2.mal))
+