Enhance test system

- Tests are now named
- Testlogs are preserved between runs
- Tests can be run through Valgrind Memcheck tool

test/.gitignore

@@ -1,3 +1,4 @@
 load_dump
 loadf_dumpf
 loads_dumps
+testlogs

test/Makefile.am

@@ -2,6 +2,10 @@ check_PROGRAMS = load_dump loadf_dumpf loads_dumps
 
 AM_CPPFLAGS = -I$(top_srcdir)/src
 AM_CFLAGS = -Wall -Werror
+LDFLAGS = -static  # for speed and Valgrind
 LDADD = ../src/libjansson.la
 
 TESTS = test-invalid test-valid
+
+clean-local:
+	rm -rf testlogs

test/run-test

@@ -1,27 +1,50 @@
-cleanup() {
+VALGRIND_CMDLINE="valgrind --leak-check=full --show-reachable=yes --track-origins=yes -q"
-    rm -rf $TMPDIR
-}
-trap cleanup 0
 
 run_testprog() {
     local prog=$1
-    local input=$2
+    local prefix=$2
+    if [ -n "$VALGRIND" ]; then
+        local runner="$VALGRIND_CMDLINE "
+    fi
+
     case "$prog" in
         load_dump)
-            ./$prog $input $TMPDIR/output 2>$TMPDIR/error
+            $runner./$prog \
+                $prefix.in \
+                $prefix.$prog.stdout \
+                2>$prefix.$prog.stderr
             ;;
         *)
-            ./$prog <$input >$TMPDIR/output 2>$TMPDIR/error
+            $runner./$prog \
+                <$prefix.in \
+                >$prefix.$prog.stdout \
+                2>$prefix.$prog.stderr
             ;;
     esac
+
+    if [ -n "$VALGRIND" ]; then
+        # Check for Valgrind error output. The valgrind option
+        # --error-exitcode is not enough because Valgrind doesn't
+        # think unfreed allocs are errors.
+        if grep -E -q '^==[0-9]+== ' $prefix.$prog.stderr; then
+            echo "### $prefix ($prog) failed:" >&2
+            echo "valgrind detected an error" >&2
+            echo "for details, see test/$prefix.$prog.stderr" >&2
+            exit 1
+        fi
+    fi
 }
 
+rm -rf testlogs
+
 for testfile in $TESTFILES; do
-    mkdir -p $TMPDIR
+    tmpdir="testlogs/`basename $testfile`"
-    ${srcdir}/split-testfile.py $testfile $TMPDIR | while read input output; do
+    mkdir -p $tmpdir
-        run_test load_dump $input $output
+    ${srcdir}/split-testfile.py $testfile $tmpdir | while read name; do
-        run_test loadf_dumpf $input $output
+        run_test load_dump $tmpdir/$name
-        run_test loads_dumps $input $output
+        run_test loadf_dumpf $tmpdir/$name
+        run_test loads_dumps $tmpdir/$name
+        echo -n '.'
     done || exit 1
-    rm -rf $TMPDIR
+    echo
 done

test/split-testfile.py

@@ -3,14 +3,12 @@
 import os
 import sys
 
-def open_files(outdir, i):
+def open_files(outdir, i, name):
-    return (open(os.path.join(outdir, 'test%02d.in' % i), 'w'),
+    basename = '%02d_%s' % (i, name)
-            open(os.path.join(outdir, 'test%02d.out' % i), 'w'))
+    print basename
-
+    input_path = os.path.join(outdir, basename + '.in')
-def close_files(input, output):
+    output_path = os.path.join(outdir, basename + '.out')
-    print os.path.basename(input.name), os.path.basename(output.name)
+    return open(input_path, 'w'), open(output_path, 'w')
-    input.close()
-    output.close()
 
 def main():
     if len(sys.argv) != 3:
@@ -24,23 +22,28 @@ def main():
         print >>sys.stderr, 'output directory %r does not exist!' % outdir
         return 1
 
-    i = 0
+    n = 0
-    input, output = open_files(outdir, i)
+    current = None
-    current = input
+    input, output = None, None
 
     for line in open(infile):
-        if line == '====\n':
+        if line.startswith('==== '):
-            current = output
+            n += 1
-        elif line == '========\n':
+            if input is not None and output is not None:
-            close_files(input, output)
+                input.close()
-            i += 1
+                output.close()
-            input, output = open_files(outdir, i)
+            input, output = open_files(outdir, n, line[5:line.find(' ====\n')])
             current = input
+        elif line == '====\n':
+            current = output
         else:
             current.write(line)
 
-    close_files(input, output)
+    if input is not None and output is not None:
-    print >>sys.stderr, "%s: %d test cases" % (infile, i + 1)
+        input.close()
+        output.close()
+
+    print >>sys.stderr, "%s: %d test cases" % (infile, n)
 
 if __name__ == '__main__':
     sys.exit(main() or 0)

test/test-invalid

@@ -1,20 +1,20 @@
 #!/bin/sh
 
 TESTFILES="${srcdir}/testdata/invalid ${srcdir}/testdata/invalid-unicode"
-TMPDIR="tmp"
 
 run_test() {
     local prog=$1
-    local input=$2
+    local prefix=$2
-    local output=$3
+
-    run_testprog $prog $TMPDIR/$input
+    run_testprog $prog $prefix
-    if ! cmp $TMPDIR/$output $TMPDIR/error >/dev/null; then
+    if ! cmp $prefix.out $prefix.$prog.stderr >/dev/null; then
-        echo "### $input ($prog) failed:" >&2
+        echo >&2
-        cat $TMPDIR/$input >&2
+        echo "### $prefix ($prog) failed:" >&2
+        cat $prefix.in >&2
         echo "### expected output:" >&2
-        cat $TMPDIR/$output >&2
+        cat $prefix.out >&2
         echo "### actual output:" >&2
-        cat $TMPDIR/error >&2
+        cat $prefix.$prog.stderr >&2
         exit 1
     fi
 }

test/test-valid

@@ -1,30 +1,29 @@
 #!/bin/sh
 
 TESTFILES="${srcdir}/testdata/valid"
-TMPDIR="tmp"
 
 run_test() {
     local prog=$1
-    local input=$2
+    local prefix=$2
-    local output=$3
+
-    run_testprog $prog $TMPDIR/$input
+    run_testprog $prog $prefix
-    if ! ${srcdir}/json-compare.py $TMPDIR/$input $TMPDIR/output \
+
-        >$TMPDIR/cmp-output
+    if ! ${srcdir}/json-compare.py $prefix.in $prefix.$prog.stdout \
+        >$prefix.$prog.cmp-stdout
     then
-        echo "### $input ($prog) failed:" >&2
+        echo >&2
-        cat $TMPDIR/$input >&2
+        echo "### $prefix ($prog) failed:" >&2
-        if [ -f $TMPDIR/output ]; then
+        cat $prefix.in >&2
+        if [ -f $prefix.$prog.stdout ]; then
             echo "### output:" >&2
-            cat $TMPDIR/output >&2
+            cat $prefix.$prog.stdout >&2
         fi
-        if [ -s $TMPDIR/cmp-output ]; then
+        if [ -s $prefix.$prog.stdout ]; then
             echo "### compare output:" >&2
-            cat $TMPDIR/cmp-output >&2
+            cat $prefix.$prog.cmp-stdout >&2
         fi
         exit 1
     fi
-    rm -f $TMPDIR/output
-    rm -f $TMPDIR/cmp-output
 }
 
 . ${srcdir}/run-test

test/testdata/invalid

@@ -1,107 +1,108 @@
+==== empty ====
 ====
 1
 '[' or '{' expected near end of file
-========
+==== null ====
 null
 ====
 1
 '[' or '{' expected near 'null'
-========
+==== lone-open-brace ====
 {
 ====
 2
 string or '}' expected near end of file
-========
+==== lone-open-bracket ====
 [
 ====
 2
 ']' expected near end of file
-========
+==== bracket-comma ====
 [,
 ====
 1
 unexpected token near ','
-========
+==== bracket-one-comma ====
 [1,
 ====
 2
 ']' expected near end of file
-========
+==== unterminated-string ====
 ["a
 ====
 1
 unexpected newline near '"a'
-========
+==== unterminated-array ====
 ["a"
 ====
 2
 ']' expected near end of file
-========
+==== apostrophe ====
 ['
 ====
 1
 invalid token near '''
-========
+==== brace-comma ====
 {,
 ====
 1
 string or '}' expected near ','
-========
+==== unterminated-empty-key ====
 {"
 ====
 1
 unexpected newline near '"'
-========
+==== unterminated-key ====
 {"a
 ====
 1
 unexpected newline near '"a'
-========
+==== object-no-colon ====
 {"a"
 ====
 2
 ':' expected near end of file
-========
+==== object-apostrophes ====
 {'a'
 ====
 1
 string or '}' expected near '''
-========
+==== object-no-value ====
 {"a":
 ====
 2
 unexpected token near end of file
-========
+==== object-unterminated-value ====
 {"a":"a
 ====
 1
 unexpected newline near '"a'
-========
+==== object-garbage-at-end ====
 {"a":"a" 123}
 ====
 1
 '}' expected near '123'
-========
+==== unterminated-object-and-array ====
 {[
 ====
 1
 string or '}' expected near '['
-========
+==== unterminated-array-and-object ====
 [{
 ====
 2
 string or '}' expected near end of file
-========
+==== object-in-unterminated-array ====
 [{}
 ====
 2
 ']' expected near end of file
-========
+==== extra-comma-in-array ====
 [1,]
 ====
 1
 unexpected token near ']'
-========
+==== extra-command-in-multiline-array ====
 [1,
 2,
 3,
@@ -111,67 +112,67 @@ unexpected token near ']'
 ====
 6
 unexpected token near ']'
-========
+==== real-truncated-at-point ====
 [1.]
 ====
 1
 invalid token near '1.'
-========
+==== real-truncated-at-e ====
 [1e]
 ====
 1
 invalid token near '1e'
-========
+==== real-garbage-after-e ====
 [1ea]
 ====
 1
 invalid token near '1e'
-========
+==== integer-starting-with-zero ====
 [012]
 ====
 1
 invalid token near '0'
-========
+==== negative-integer-starting-with-zero ====
 [-012]
 ====
 1
 invalid token near '-0'
-========
+==== invalid-identifier ====
 [troo
 ====
 1
 invalid token near 'troo'
-========
+==== invalid-escap ====
 ["\a <-- invalid escape"]
 ====
 1
 invalid escape near '"\'
-========
+==== tab-character-in-string ====
 ["	 <-- tab character"]
 ====
 1
 control character 0x9 near '"'
-========
+==== null-byte-in-string ====
 ["\u0000 (null byte not allowed)"]
 ====
 1
 \u0000 is not allowed
-========
+==== truncated-unicode-surrogate ====
 ["\uDADA (first surrogate without the second)"]
 ====
 1
 invalid Unicode '\uDADA'
-========
+==== invalid-second-surrogate ====
 ["\uD888\u3210 (first surrogate and invalid second surrogate)"]
 ====
 1
 invalid Unicode '\uD888\u3210'
-========
+==== lone-second-surrogate ====
 ["\uDFAA (second surrogate on it's own)"]
 ====
 1
 invalid Unicode '\uDFAA'
-========
+==== unicode-identifier ====
 å
 ====
 1

test/testdata/invalid-unicode

@@ -1,88 +1,89 @@
+==== lone-invalid-utf-8 ====
 å
 ====
 -1
 unable to decode byte 0xe5 at position 0
-========
+==== invalid-utf-8-in-string ====
 ["å <-- invalid UTF-8"]
 ====
 -1
 unable to decode byte 0xe5 at position 2
-========
+==== invalid-utf-8-in-array ====
 [å]
 ====
 -1
 unable to decode byte 0xe5 at position 1
-========
+==== invalid-utf-8-in-identifier ====
 [aå]
 ====
 -1
 unable to decode byte 0xe5 at position 2
-========
+==== invalid-utf-8-in-escape ====
 ["\uå"]
 ====
 -1
 unable to decode byte 0xe5 at position 4
-========
+==== invalid-utf-8-after-backslash ====
 ["\å"]
 ====
 -1
 unable to decode byte 0xe5 at position 3
-========
+==== invalid-utf-8-in-int ====
 [0å]
 ====
 -1
 unable to decode byte 0xe5 at position 2
-========
+==== invalid-utf-8-in-bigger-int ====
 [123å]
 ====
 -1
 unable to decode byte 0xe5 at position 4
-========
+==== invalid-utf-8-in-real-after-e ====
 [1eå]
 ====
 -1
 unable to decode byte 0xe5 at position 3
-========
+==== invalid-utf-8-in-exponent ====
 [1e1å]
 ====
 -1
 unable to decode byte 0xe5 at position 4
-========
+==== lone-utf-8-continuation-byte ====
 ["<22>"]
 ====
 -1
 unable to decode byte 0x81 at position 2
-========
+==== overlong-ascii-encoding ====
 ["Á"]
 ====
 -1
 unable to decode byte 0xc1 at position 2
-========
+==== restricted-utf-8 ====
 ["ý"]
 ====
 -1
 unable to decode byte 0xfd at position 2
-========
+==== not-in-unicode-range ====
 ["ô¿¿¿"]
 ====
 -1
 unable to decode byte 0xf4 at position 2
-========
+==== overlong-3-byte-encoding ====
 ["à€¢ <-- overlong encoding"]
 ====
 -1
 unable to decode byte 0xe0 at position 2
-========
+==== overlong-4-byte-encoding ====
 ["ð€€¢ <-- overlong encoding"]
 ====
 -1
 unable to decode byte 0xf0 at position 2
-========
+==== truncated-utf-8 ====
 ["àÿ <-- truncated UTF-8"]
 ====
 -1
 unable to decode byte 0xe0 at position 2
-========
+==== encoded-surrogate-half ====
 ["í¢« <-- encoded surrogate half"]
 ====
 -1

test/testdata/valid

@@ -1,67 +1,68 @@
+==== empty-string ====
 [""]
-========
+==== short-string ====
 ["a"]
-========
+==== simple-ascii-string ====
 ["abcdefghijklmnopqrstuvwxyz1234567890 "]
-========
+==== utf-8-string ====
 ["€þıœəßð some utf-8 ĸʒ×ŋµåäö𝄞"]
-========
+==== string-escapes ====
 ["\"\\\/\b\f\n\r\t"]
-========
+==== one-byte-utf-8 ====
 ["\u002c one-byte UTF-8"]
-========
+==== two-byte-utf-8 ====
 ["\u0123 two-byte UTF-8"]
-========
+==== three-byte-utf-8 ====
 ["\u0821 three-byte UTF-8"]
-========
+==== utf-surrogate-four-byte-encoding ====
 ["\uD834\uDD1E surrogate, four-byte UTF-8"]
-========
+==== escaped-utf-control-char ====
 ["\u0012 escaped control character"]
-========
+==== simple-int-0 ====
 [0]
-========
+==== simple-int-1 ====
 [1]
-========
+==== simple-int-123 ====
 [123]
-========
+==== negative-zero ====
 [-0]
-========
+==== negative-one ====
 [-1]
-========
+==== negative-int ====
 [-123]
-========
+==== simple-real ====
 [123.456789]
-========
+==== real-exponent ====
 [123e45]
-========
+==== real-capital-e ====
 [1E22]
-========
+==== real-positive-exponent ====
 [1e+2]
-========
+==== real-negative-exponent ====
 [1e-2]
-========
+==== real-capital-e-positive-exponent ====
 [1E+2]
-========
+==== real-capital-e-negative-exponent ====
 [1E-2]
-========
+==== real-fraction-exponent ====
 [123.456e78]
-========
+==== true ====
 [true]
-========
+==== false ====
 [false]
-========
+==== null ====
 [null]
-========
+==== empty-array ====
 []
-========
+==== empty-object-in-array ====
 [{}]
-========
+==== complex-array ====
 [1,2,3,4,
 "a", "b", "c",
 {"foo": "bar", "core": "dump"},
 true, false, true, true, null, false
 ]
-========
+==== empty-object ====
 {}
-========
+==== simple-object ====
 {"a":[]}