Enhance test system

- Tests are now named
- Testlogs are preserved between runs
- Tests can be run through Valgrind Memcheck tool
This commit is contained in:
Petri Lehtinen 2009-07-26 22:44:11 +03:00
parent d3b3055a34
commit f60c563bab
9 changed files with 170 additions and 137 deletions

1
test/.gitignore vendored
View File

@ -1,3 +1,4 @@
load_dump
loadf_dumpf
loads_dumps
testlogs

View File

@ -2,6 +2,10 @@ check_PROGRAMS = load_dump loadf_dumpf loads_dumps
AM_CPPFLAGS = -I$(top_srcdir)/src
AM_CFLAGS = -Wall -Werror
LDFLAGS = -static # for speed and Valgrind
LDADD = ../src/libjansson.la
TESTS = test-invalid test-valid
clean-local:
rm -rf testlogs

View File

@ -1,27 +1,50 @@
cleanup() {
rm -rf $TMPDIR
}
trap cleanup 0
VALGRIND_CMDLINE="valgrind --leak-check=full --show-reachable=yes --track-origins=yes -q"
run_testprog() {
local prog=$1
local input=$2
local prefix=$2
if [ -n "$VALGRIND" ]; then
local runner="$VALGRIND_CMDLINE "
fi
case "$prog" in
load_dump)
./$prog $input $TMPDIR/output 2>$TMPDIR/error
$runner./$prog \
$prefix.in \
$prefix.$prog.stdout \
2>$prefix.$prog.stderr
;;
*)
./$prog <$input >$TMPDIR/output 2>$TMPDIR/error
$runner./$prog \
<$prefix.in \
>$prefix.$prog.stdout \
2>$prefix.$prog.stderr
;;
esac
if [ -n "$VALGRIND" ]; then
# Check for Valgrind error output. The valgrind option
# --error-exitcode is not enough because Valgrind doesn't
# think unfreed allocs are errors.
if grep -E -q '^==[0-9]+== ' $prefix.$prog.stderr; then
echo "### $prefix ($prog) failed:" >&2
echo "valgrind detected an error" >&2
echo "for details, see test/$prefix.$prog.stderr" >&2
exit 1
fi
fi
}
rm -rf testlogs
for testfile in $TESTFILES; do
mkdir -p $TMPDIR
${srcdir}/split-testfile.py $testfile $TMPDIR | while read input output; do
run_test load_dump $input $output
run_test loadf_dumpf $input $output
run_test loads_dumps $input $output
tmpdir="testlogs/`basename $testfile`"
mkdir -p $tmpdir
${srcdir}/split-testfile.py $testfile $tmpdir | while read name; do
run_test load_dump $tmpdir/$name
run_test loadf_dumpf $tmpdir/$name
run_test loads_dumps $tmpdir/$name
echo -n '.'
done || exit 1
rm -rf $TMPDIR
echo
done

View File

@ -3,14 +3,12 @@
import os
import sys
def open_files(outdir, i):
return (open(os.path.join(outdir, 'test%02d.in' % i), 'w'),
open(os.path.join(outdir, 'test%02d.out' % i), 'w'))
def close_files(input, output):
print os.path.basename(input.name), os.path.basename(output.name)
input.close()
output.close()
def open_files(outdir, i, name):
basename = '%02d_%s' % (i, name)
print basename
input_path = os.path.join(outdir, basename + '.in')
output_path = os.path.join(outdir, basename + '.out')
return open(input_path, 'w'), open(output_path, 'w')
def main():
if len(sys.argv) != 3:
@ -24,23 +22,28 @@ def main():
print >>sys.stderr, 'output directory %r does not exist!' % outdir
return 1
i = 0
input, output = open_files(outdir, i)
current = input
n = 0
current = None
input, output = None, None
for line in open(infile):
if line == '====\n':
current = output
elif line == '========\n':
close_files(input, output)
i += 1
input, output = open_files(outdir, i)
if line.startswith('==== '):
n += 1
if input is not None and output is not None:
input.close()
output.close()
input, output = open_files(outdir, n, line[5:line.find(' ====\n')])
current = input
elif line == '====\n':
current = output
else:
current.write(line)
close_files(input, output)
print >>sys.stderr, "%s: %d test cases" % (infile, i + 1)
if input is not None and output is not None:
input.close()
output.close()
print >>sys.stderr, "%s: %d test cases" % (infile, n)
if __name__ == '__main__':
sys.exit(main() or 0)

View File

@ -1,20 +1,20 @@
#!/bin/sh
TESTFILES="${srcdir}/testdata/invalid ${srcdir}/testdata/invalid-unicode"
TMPDIR="tmp"
run_test() {
local prog=$1
local input=$2
local output=$3
run_testprog $prog $TMPDIR/$input
if ! cmp $TMPDIR/$output $TMPDIR/error >/dev/null; then
echo "### $input ($prog) failed:" >&2
cat $TMPDIR/$input >&2
local prefix=$2
run_testprog $prog $prefix
if ! cmp $prefix.out $prefix.$prog.stderr >/dev/null; then
echo >&2
echo "### $prefix ($prog) failed:" >&2
cat $prefix.in >&2
echo "### expected output:" >&2
cat $TMPDIR/$output >&2
cat $prefix.out >&2
echo "### actual output:" >&2
cat $TMPDIR/error >&2
cat $prefix.$prog.stderr >&2
exit 1
fi
}

View File

@ -1,30 +1,29 @@
#!/bin/sh
TESTFILES="${srcdir}/testdata/valid"
TMPDIR="tmp"
run_test() {
local prog=$1
local input=$2
local output=$3
run_testprog $prog $TMPDIR/$input
if ! ${srcdir}/json-compare.py $TMPDIR/$input $TMPDIR/output \
>$TMPDIR/cmp-output
local prefix=$2
run_testprog $prog $prefix
if ! ${srcdir}/json-compare.py $prefix.in $prefix.$prog.stdout \
>$prefix.$prog.cmp-stdout
then
echo "### $input ($prog) failed:" >&2
cat $TMPDIR/$input >&2
if [ -f $TMPDIR/output ]; then
echo >&2
echo "### $prefix ($prog) failed:" >&2
cat $prefix.in >&2
if [ -f $prefix.$prog.stdout ]; then
echo "### output:" >&2
cat $TMPDIR/output >&2
cat $prefix.$prog.stdout >&2
fi
if [ -s $TMPDIR/cmp-output ]; then
if [ -s $prefix.$prog.stdout ]; then
echo "### compare output:" >&2
cat $TMPDIR/cmp-output >&2
cat $prefix.$prog.cmp-stdout >&2
fi
exit 1
fi
rm -f $TMPDIR/output
rm -f $TMPDIR/cmp-output
}
. ${srcdir}/run-test

69
test/testdata/invalid vendored
View File

@ -1,107 +1,108 @@
==== empty ====
====
1
'[' or '{' expected near end of file
========
==== null ====
null
====
1
'[' or '{' expected near 'null'
========
==== lone-open-brace ====
{
====
2
string or '}' expected near end of file
========
==== lone-open-bracket ====
[
====
2
']' expected near end of file
========
==== bracket-comma ====
[,
====
1
unexpected token near ','
========
==== bracket-one-comma ====
[1,
====
2
']' expected near end of file
========
==== unterminated-string ====
["a
====
1
unexpected newline near '"a'
========
==== unterminated-array ====
["a"
====
2
']' expected near end of file
========
==== apostrophe ====
['
====
1
invalid token near '''
========
==== brace-comma ====
{,
====
1
string or '}' expected near ','
========
==== unterminated-empty-key ====
{"
====
1
unexpected newline near '"'
========
==== unterminated-key ====
{"a
====
1
unexpected newline near '"a'
========
==== object-no-colon ====
{"a"
====
2
':' expected near end of file
========
==== object-apostrophes ====
{'a'
====
1
string or '}' expected near '''
========
==== object-no-value ====
{"a":
====
2
unexpected token near end of file
========
==== object-unterminated-value ====
{"a":"a
====
1
unexpected newline near '"a'
========
==== object-garbage-at-end ====
{"a":"a" 123}
====
1
'}' expected near '123'
========
==== unterminated-object-and-array ====
{[
====
1
string or '}' expected near '['
========
==== unterminated-array-and-object ====
[{
====
2
string or '}' expected near end of file
========
==== object-in-unterminated-array ====
[{}
====
2
']' expected near end of file
========
==== extra-comma-in-array ====
[1,]
====
1
unexpected token near ']'
========
==== extra-command-in-multiline-array ====
[1,
2,
3,
@ -111,67 +112,67 @@ unexpected token near ']'
====
6
unexpected token near ']'
========
==== real-truncated-at-point ====
[1.]
====
1
invalid token near '1.'
========
==== real-truncated-at-e ====
[1e]
====
1
invalid token near '1e'
========
==== real-garbage-after-e ====
[1ea]
====
1
invalid token near '1e'
========
==== integer-starting-with-zero ====
[012]
====
1
invalid token near '0'
========
==== negative-integer-starting-with-zero ====
[-012]
====
1
invalid token near '-0'
========
==== invalid-identifier ====
[troo
====
1
invalid token near 'troo'
========
==== invalid-escap ====
["\a <-- invalid escape"]
====
1
invalid escape near '"\'
========
==== tab-character-in-string ====
[" <-- tab character"]
====
1
control character 0x9 near '"'
========
==== null-byte-in-string ====
["\u0000 (null byte not allowed)"]
====
1
\u0000 is not allowed
========
==== truncated-unicode-surrogate ====
["\uDADA (first surrogate without the second)"]
====
1
invalid Unicode '\uDADA'
========
==== invalid-second-surrogate ====
["\uD888\u3210 (first surrogate and invalid second surrogate)"]
====
1
invalid Unicode '\uD888\u3210'
========
==== lone-second-surrogate ====
["\uDFAA (second surrogate on it's own)"]
====
1
invalid Unicode '\uDFAA'
========
==== unicode-identifier ====
å
====
1

View File

@ -1,88 +1,89 @@
==== lone-invalid-utf-8 ====
å
====
-1
unable to decode byte 0xe5 at position 0
========
==== invalid-utf-8-in-string ====
["å <-- invalid UTF-8"]
====
-1
unable to decode byte 0xe5 at position 2
========
==== invalid-utf-8-in-array ====
[å]
====
-1
unable to decode byte 0xe5 at position 1
========
==== invalid-utf-8-in-identifier ====
[aå]
====
-1
unable to decode byte 0xe5 at position 2
========
==== invalid-utf-8-in-escape ====
["\uå"]
====
-1
unable to decode byte 0xe5 at position 4
========
==== invalid-utf-8-after-backslash ====
["\å"]
====
-1
unable to decode byte 0xe5 at position 3
========
==== invalid-utf-8-in-int ====
[0å]
====
-1
unable to decode byte 0xe5 at position 2
========
==== invalid-utf-8-in-bigger-int ====
[123å]
====
-1
unable to decode byte 0xe5 at position 4
========
==== invalid-utf-8-in-real-after-e ====
[1eå]
====
-1
unable to decode byte 0xe5 at position 3
========
==== invalid-utf-8-in-exponent ====
[1e1å]
====
-1
unable to decode byte 0xe5 at position 4
========
==== lone-utf-8-continuation-byte ====
["<22>"]
====
-1
unable to decode byte 0x81 at position 2
========
==== overlong-ascii-encoding ====
["Á"]
====
-1
unable to decode byte 0xc1 at position 2
========
==== restricted-utf-8 ====
["ý"]
====
-1
unable to decode byte 0xfd at position 2
========
==== not-in-unicode-range ====
["ô¿¿¿"]
====
-1
unable to decode byte 0xf4 at position 2
========
==== overlong-3-byte-encoding ====
["à€¢ <-- overlong encoding"]
====
-1
unable to decode byte 0xe0 at position 2
========
==== overlong-4-byte-encoding ====
["ð€€¢ <-- overlong encoding"]
====
-1
unable to decode byte 0xf0 at position 2
========
==== truncated-utf-8 ====
["àÿ <-- truncated UTF-8"]
====
-1
unable to decode byte 0xe0 at position 2
========
==== encoded-surrogate-half ====
["í¢« <-- encoded surrogate half"]
====
-1

63
test/testdata/valid vendored
View File

@ -1,67 +1,68 @@
==== empty-string ====
[""]
========
==== short-string ====
["a"]
========
==== simple-ascii-string ====
["abcdefghijklmnopqrstuvwxyz1234567890 "]
========
==== utf-8-string ====
["€þıœəßð some utf-8 ĸʒ×ŋµåäö𝄞"]
========
==== string-escapes ====
["\"\\\/\b\f\n\r\t"]
========
==== one-byte-utf-8 ====
["\u002c one-byte UTF-8"]
========
==== two-byte-utf-8 ====
["\u0123 two-byte UTF-8"]
========
==== three-byte-utf-8 ====
["\u0821 three-byte UTF-8"]
========
==== utf-surrogate-four-byte-encoding ====
["\uD834\uDD1E surrogate, four-byte UTF-8"]
========
==== escaped-utf-control-char ====
["\u0012 escaped control character"]
========
==== simple-int-0 ====
[0]
========
==== simple-int-1 ====
[1]
========
==== simple-int-123 ====
[123]
========
==== negative-zero ====
[-0]
========
==== negative-one ====
[-1]
========
==== negative-int ====
[-123]
========
==== simple-real ====
[123.456789]
========
==== real-exponent ====
[123e45]
========
==== real-capital-e ====
[1E22]
========
==== real-positive-exponent ====
[1e+2]
========
==== real-negative-exponent ====
[1e-2]
========
==== real-capital-e-positive-exponent ====
[1E+2]
========
==== real-capital-e-negative-exponent ====
[1E-2]
========
==== real-fraction-exponent ====
[123.456e78]
========
==== true ====
[true]
========
==== false ====
[false]
========
==== null ====
[null]
========
==== empty-array ====
[]
========
==== empty-object-in-array ====
[{}]
========
==== complex-array ====
[1,2,3,4,
"a", "b", "c",
{"foo": "bar", "core": "dump"},
true, false, true, true, null, false
]
========
==== empty-object ====
{}
========
==== simple-object ====
{"a":[]}