Refactor the test system

This commit is contained in:
Petri Lehtinen 2009-12-14 23:01:36 +02:00
parent 50031440a3
commit 3add1cf361
385 changed files with 880 additions and 1117 deletions

View File

@ -23,7 +23,8 @@ AC_CONFIG_FILES([
doc/Makefile
src/Makefile
test/Makefile
test/testdata/Makefile
test/testprogs/Makefile
test/bin/Makefile
test/suites/Makefile
test/suites/api/Makefile
])
AC_OUTPUT

18
test/.gitignore vendored
View File

@ -1,10 +1,8 @@
loadf_dumpf
loads_dumps
load_file_dump_file
testlogs
testprogs/test_array
testprogs/test_dump
testprogs/test_load
testprogs/test_number
testprogs/test_object
testprogs/test_simple
logs
bin/json_process
suites/api/test_array
suites/api/test_dump
suites/api/test_load
suites/api/test_number
suites/api/test_object
suites/api/test_simple

View File

@ -1,22 +1,10 @@
DIST_SUBDIRS = testprogs testdata
SUBDIRS = testprogs
SUBDIRS = bin suites
EXTRA_DIST = scripts
check_PROGRAMS = loadf_dumpf loads_dumps load_file_dump_file
AM_CPPFLAGS = -I$(top_srcdir)/src
AM_CFLAGS = -Wall -Werror
LDFLAGS = -static # for speed and Valgrind
LDADD = ../src/libjansson.la
TESTS = test-api test-invalid test-valid
EXTRA_DIST = \
test-api \
test-invalid \
test-valid \
run-test \
json-compare.py \
split-testfile.py
TESTS = run-suites
TESTS_ENVIRONMENT = \
top_srcdir=$(top_srcdir) \
top_builddir=$(top_builddir)
clean-local:
rm -rf testlogs
rm -rf logs

6
test/bin/Makefile.am Normal file
View File

@ -0,0 +1,6 @@
check_PROGRAMS = json_process
AM_CPPFLAGS = -I$(top_srcdir)/src
AM_CFLAGS = -Wall -Werror
LDFLAGS = -static # for speed and Valgrind
LDADD = $(top_builddir)/src/libjansson.la

66
test/bin/json_process.c Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
*
* Jansson is free software; you can redistribute it and/or modify
* it under the terms of the MIT license. See LICENSE for details.
*/
#include <stdio.h>
#include <stdlib.h>
#include <jansson.h>
static int getenv_int(const char *name)
{
char *value, *end;
long result;
value = getenv(name);
if(!value)
return 0;
result = strtol(value, &end, 10);
if(*end != '\0')
return 0;
return (int)result;
}
int main(int argc, char *argv[])
{
int indent = 0;
unsigned int flags = 0;
json_t *json;
json_error_t error;
if(argc != 1) {
fprintf(stderr, "usage: %s\n", argv[0]);
return 2;
}
indent = getenv_int("JSON_INDENT");
if(indent < 0 || indent > 255) {
fprintf(stderr, "invalid value for JSON_INDENT: %d\n", indent);
return 2;
}
if(indent > 0)
flags |= JSON_INDENT(indent);
if(getenv_int("JSON_COMPACT") > 0)
flags |= JSON_COMPACT;
if(getenv_int("JSON_ENSURE_ASCII"))
flags |= JSON_ENSURE_ASCII;
json = json_loadf(stdin, &error);
if(!json) {
fprintf(stderr, "%d\n%s\n", error.line, error.text);
return 1;
}
json_dumpf(json, stdout, flags);
json_decref(json);
return 0;
}

View File

@ -1,45 +0,0 @@
#!/usr/bin/python
#
# Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
#
# Jansson is free software; you can redistribute it and/or modify
# it under the terms of the MIT license. See LICENSE for details.
import sys
try:
import json
except ImportError:
import simplejson as json
def load(filename):
try:
jsonfile = open(filename)
except IOError, err:
print >>sys.stderr, "unable to load %s: %s" % \
(filename, err.strerror)
sys.exit(1)
try:
jsondata = json.load(jsonfile)
except ValueError, err:
print "%s is malformed: %s" % (filename, err)
sys.exit(1)
finally:
jsonfile.close()
return jsondata
def main():
if len(sys.argv) != 3:
print >>sys.stderr, "usage: %s json1 json2" % sys.argv[0]
return 2
json1 = load(sys.argv[1])
json2 = load(sys.argv[2])
if json1 == json2:
return 0
else:
return 1
if __name__ == '__main__':
sys.exit(main() or 0)

View File

@ -1,31 +0,0 @@
/*
* Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
*
* Jansson is free software; you can redistribute it and/or modify
* it under the terms of the MIT license. See LICENSE for details.
*/
#include <stdio.h>
#include <jansson.h>
int main(int argc, char *argv[])
{
json_t *json;
json_error_t error;
if(argc != 3) {
fprintf(stderr, "usage: %s infile outfile\n", argv[0]);
return 2;
}
json = json_load_file(argv[1], &error);
if(!json) {
fprintf(stderr, "%d\n%s\n", error.line, error.text);
return 1;
}
json_dump_file(json, argv[2], 0);
json_decref(json);
return 0;
}

View File

@ -1,33 +0,0 @@
/*
* Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
*
* Jansson is free software; you can redistribute it and/or modify
* it under the terms of the MIT license. See LICENSE for details.
*/
#include <stdio.h>
#include <jansson.h>
int main(int argc, char *argv[])
{
json_t *json;
json_error_t error;
if(argc != 1) {
fprintf(stderr, "usage: %s\n", argv[0]);
return 2;
}
json = json_loadf(stdin, &error);
if(!json) {
fprintf(stderr, "%d\n%s\n", error.line, error.text);
return 1;
}
/* loadf_dumpf indents, others don't, so dumping with and without
indenting is tested */
json_dumpf(json, stdout, JSON_INDENT(4));
json_decref(json);
return 0;
}

View File

@ -1,47 +0,0 @@
/*
* Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
*
* Jansson is free software; you can redistribute it and/or modify
* it under the terms of the MIT license. See LICENSE for details.
*/
#include <stdio.h>
#include <stdlib.h>
#include <jansson.h>
#define BUFFER_SIZE (256 * 1024)
int main(int argc, char *argv[])
{
json_t *json;
json_error_t error;
int count;
char buffer[BUFFER_SIZE];
char *result;
if(argc != 1) {
fprintf(stderr, "usage: %s\n", argv[0]);
return 2;
}
count = fread(buffer, 1, BUFFER_SIZE, stdin);
if(count < 0 || count >= BUFFER_SIZE) {
fprintf(stderr, "unable to read input\n");
return 1;
}
buffer[count] = '\0';
json = json_loads(buffer, &error);
if(!json) {
fprintf(stderr, "%d\n%s\n", error.line, error.text);
return 1;
}
result = json_dumps(json, 0);
json_decref(json);
puts(result);
free(result);
return 0;
}

46
test/run-suites Executable file
View File

@ -0,0 +1,46 @@
#!/bin/sh
while [ -n "$1" ]; do
suite=$1
if [ -x $top_srcdir/test/suites/$suite/run ]; then
SUITES="$SUITES $suite"
else
echo "No such suite: $suite"
exit 1
fi
shift
done
if [ -z "$SUITES" ]; then
suitedirs=$top_srcdir/test/suites/*
for suitedir in $suitedirs; do
if [ -x $suitedir/run ]; then
SUITES="$SUITES `basename $suitedir`"
fi
done
fi
export suites_srcdir=$top_srcdir/test/suites
export suites_builddir=suites
export scriptdir=$top_srcdir/test/scripts
export logdir=logs
export bindir=bin
passed=0
failed=0
for suite in $SUITES; do
echo "Suite: $suite"
if $suites_srcdir/$suite/run $suite; then
passed=$(($passed+1))
else
failed=$(($failed+1))
fi
done
if [ $failed -gt 0 ]; then
echo "$failed of $((passed+failed)) test suites failed"
exit 1
else
echo "$passed test suites passed"
rm -rf $logdir
fi

View File

@ -1,57 +0,0 @@
# Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
#
# Jansson is free software; you can redistribute it and/or modify
# it under the terms of the MIT license. See LICENSE for details.
VALGRIND_CMDLINE="valgrind --leak-check=full --show-reachable=yes --track-origins=yes -q"
run_testprog() {
local prog=$1
local prefix=$2
if [ -n "$VALGRIND" ]; then
local runner="$VALGRIND_CMDLINE "
fi
case "$prog" in
load_file_dump_file)
$runner./$prog \
$prefix.in \
$prefix.$prog.stdout \
2>$prefix.$prog.stderr
;;
*)
$runner./$prog \
<$prefix.in \
>$prefix.$prog.stdout \
2>$prefix.$prog.stderr
;;
esac
if [ -n "$VALGRIND" ]; then
# Check for Valgrind error output. The valgrind option
# --error-exitcode is not enough because Valgrind doesn't
# think unfreed allocs are errors.
if grep -E -q '^==[0-9]+== ' $prefix.$prog.stderr; then
echo "### $prefix ($prog) failed:" >&2
echo "valgrind detected an error" >&2
echo "for details, see test/$prefix.$prog.stderr" >&2
exit 1
fi
fi
}
for testfile in $TESTFILES; do
tmpdir="testlogs/`basename $testfile`"
rm -rf $tmpdir
mkdir -p $tmpdir
if echo "$testfile" | grep -q -E -e '-strip$'; then
opts="--strip"
fi
${srcdir}/split-testfile.py $opts $testfile $tmpdir | while read name; do
run_test loadf_dumpf $tmpdir/$name
run_test loads_dumps $tmpdir/$name
run_test load_file_dump_file $tmpdir/$name
echo -n '.'
done || exit 1
echo
done

71
test/scripts/run-tests.sh Normal file
View File

@ -0,0 +1,71 @@
# Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
#
# Jansson is free software; you can redistribute it and/or modify
# it under the terms of the MIT license. See LICENSE for details.
json_process=$bindir/json_process
suite_name=$1
suite_srcdir=$suites_srcdir/$suite_name
suite_builddir=$suites_builddir/$suite_name
suite_log=$logdir/$suite_name
[ -z "$VERBOSE" ] && VERBOSE=0
. $scriptdir/valgrind.sh
rm -rf $suite_log
mkdir -p $suite_log
for test_path in $suite_srcdir/*; do
test_name=$(basename $test_path)
test_builddir=$suite_builddir/$test_name
test_log=$suite_log/$test_name
[ "$test_name" = "run" ] && continue
is_test || continue
rm -rf $test_log
mkdir -p $test_log
if [ $VERBOSE -eq 1 ]; then
echo -n "$name... "
fi
if run_test; then
# Success
if [ $VERBOSE -eq 1 ]; then
echo "ok"
else
echo -n "."
fi
rm -rf $test_log
else
# Failure
if [ $VERBOSE -eq 1 ]; then
echo "FAILED"
else
echo -n "F"
fi
fi
done
if [ $VERBOSE -eq 0 ]; then
echo
fi
if [ -n "$(ls -A $suite_log)" ]; then
for test_log in $suite_log/*; do
test_name=$(basename $test_log)
test_path=$suite_srcdir/$test_name
echo "================================================================="
echo "$suite_name/$test_name"
echo "================================================================="
show_error
echo
done
echo "================================================================="
exit 1
else
rm -rf $suite_log
fi

32
test/scripts/valgrind.sh Normal file
View File

@ -0,0 +1,32 @@
# Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
#
# Jansson is free software; you can redistribute it and/or modify
# it under the terms of the MIT license. See LICENSE for details.
[ -z "$VALGRIND" ] && VALGRIND=0
VALGRIND_CMDLINE="valgrind --leak-check=full --show-reachable=yes --track-origins=yes -q"
if [ $VALGRIND -eq 1 ]; then
json_process="$VALGRIND_CMDLINE $json_process"
fi
valgrind_check() {
if [ $VALGRIND -eq 1 ]; then
# Check for Valgrind error output. The valgrind option
# --error-exitcode is not enough because Valgrind doesn't
# think unfreed allocs are errors.
if grep -E -q '^==[0-9]+== ' $1; then
touch $test_log/valgrind_error
return 1
fi
fi
}
valgrind_show_error() {
if [ $VALGRIND -eq 1 -a -f $test_log/valgrind_error ]; then
echo "valgrind detected an error"
return 0
fi
return 1
}

View File

@ -1,68 +0,0 @@
#!/usr/bin/python
#
# Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
#
# Jansson is free software; you can redistribute it and/or modify
# it under the terms of the MIT license. See LICENSE for details.
import os
import sys
from optparse import OptionParser
def strip_file(filename):
with open(filename) as fobj:
data = fobj.read()
with open(filename, 'w') as fobj:
fobj.write(data.strip())
def open_files(outdir, i, name):
basename = '%02d_%s' % (i, name)
print basename
input_path = os.path.join(outdir, basename + '.in')
output_path = os.path.join(outdir, basename + '.out')
return open(input_path, 'w'), open(output_path, 'w')
def main():
parser = OptionParser('usage: %prog [options] inputfile outputdir')
parser.add_option('--strip', help='strip whitespace from input',
action='store_true', default=False)
options, args = parser.parse_args()
if len(args) != 2:
parser.print_help()
return 2
infile = os.path.normpath(args[0])
outdir = os.path.normpath(args[1])
if not os.path.exists(outdir):
print >>sys.stderr, 'output directory %r does not exist!' % outdir
return 1
n = 0
current = None
input, output = None, None
for line in open(infile):
if line.startswith('==== '):
n += 1
if input is not None and output is not None:
input.close()
output.close()
if options.strip:
strip_file(input.name)
input, output = open_files(outdir, n, line[5:line.find(' ====\n')])
current = input
elif line == '====\n':
current = output
else:
current.write(line)
if input is not None and output is not None:
input.close()
output.close()
print >>sys.stderr, "%s: %d test cases" % (infile, n)
if __name__ == '__main__':
sys.exit(main() or 0)

View File

@ -1 +1,2 @@
SUBDIRS = api
EXTRA_DIST = invalid invalid-strip invalid-unicode valid valid-strip

View File

@ -1,4 +1,10 @@
check_PROGRAMS = test_array test_dump test_load test_simple test_number test_object
check_PROGRAMS = \
test_array \
test_dump \
test_load \
test_simple \
test_number \
test_object
test_array_SOURCES = test_array.c util.h
test_dump_SOURCES = test_dump.c util.h
@ -10,4 +16,4 @@ test_object_SOURCES = test_object.c util.h
AM_CPPFLAGS = -I$(top_srcdir)/src
AM_CFLAGS = -Wall -Werror
LDFLAGS = -static # for speed and Valgrind
LDADD = ../../src/libjansson.la
LDADD = $(top_builddir)/src/libjansson.la

21
test/suites/api/run Executable file
View File

@ -0,0 +1,21 @@
#!/bin/sh
#
# Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
#
# Jansson is free software; you can redistribute it and/or modify
# it under the terms of the MIT license. See LICENSE for details.
is_test() {
test "${test_name%.c}" != "$test_name"
}
run_test() {
$suite_builddir/${test_name%.c} >$test_log/stdout 2>$test_log/stderr
}
show_error() {
valgrind_show_error && return
cat $test_log/stderr
}
. $top_srcdir/test/scripts/run-tests.sh

View File

@ -0,0 +1,2 @@
1
invalid token near '''

View File

@ -0,0 +1 @@
['

View File

@ -0,0 +1,2 @@
1
'[' or '{' expected near 'a'

View File

@ -0,0 +1 @@

View File

@ -0,0 +1,2 @@
1
string or '}' expected near ','

View File

@ -0,0 +1 @@
{,

View File

@ -0,0 +1,2 @@
1
unexpected token near ','

View File

@ -0,0 +1 @@
[,

View File

@ -0,0 +1,2 @@
1
']' expected near end of file

View File

@ -0,0 +1 @@
[1,

View File

@ -0,0 +1,2 @@
1
'[' or '{' expected near end of file

View File

View File

@ -0,0 +1,2 @@
1
unexpected token near ']'

View File

@ -0,0 +1 @@
[1,]

View File

@ -0,0 +1,2 @@
6
unexpected token near ']'

View File

@ -0,0 +1,6 @@
[1,
2,
3,
4,
5,
]

View File

@ -0,0 +1,2 @@
2
end of file expected near 'foo'

View File

@ -0,0 +1,2 @@
[1,2,3]
foo

View File

@ -0,0 +1,2 @@
1
end of file expected near 'foo'

View File

@ -0,0 +1 @@
[1,2,3]foo

View File

@ -0,0 +1,2 @@
1
invalid token near '0'

View File

@ -0,0 +1 @@
[012]

View File

@ -0,0 +1,2 @@
1
invalid escape near '"\'

View File

@ -0,0 +1 @@
["\a <-- invalid escape"]

View File

@ -0,0 +1,2 @@
1
invalid token near 'troo'

View File

@ -0,0 +1 @@
[troo

View File

@ -0,0 +1,2 @@
1
']' expected near 'foo'

View File

@ -0,0 +1 @@
[-123foo]

View File

@ -0,0 +1,2 @@
1
']' expected near 'foo'

View File

@ -0,0 +1 @@
[-123.123foo]

View File

@ -0,0 +1,2 @@
1
invalid Unicode '\uD888\u3210'

View File

@ -0,0 +1 @@
["\uD888\u3210 (first surrogate and invalid second surrogate)"]

View File

@ -0,0 +1,2 @@
1
string or '}' expected near end of file

View File

@ -0,0 +1 @@
{

View File

@ -0,0 +1,2 @@
1
']' expected near end of file

View File

@ -0,0 +1 @@
[

View File

@ -0,0 +1,2 @@
1
invalid Unicode '\uDFAA'

View File

@ -0,0 +1 @@
["\uDFAA (second surrogate on it's own)"]

View File

@ -0,0 +1,2 @@
1
invalid token near '-'

View File

@ -0,0 +1 @@
[-foo]

View File

@ -0,0 +1,2 @@
1
invalid token near '-0'

View File

@ -0,0 +1 @@
[-012]

View File

@ -0,0 +1,2 @@
1
\u0000 is not allowed

View File

@ -0,0 +1 @@
["\u0000 (null byte not allowed)"]

View File

@ -0,0 +1,2 @@
1
'[' or '{' expected near 'null'

View File

@ -0,0 +1 @@
null

View File

@ -0,0 +1,2 @@
1
string or '}' expected near '''

View File

@ -0,0 +1 @@
{'a'

View File

@ -0,0 +1,2 @@
1
'}' expected near '123'

View File

@ -0,0 +1 @@
{"a":"a" 123}

View File

@ -0,0 +1,2 @@
1
']' expected near end of file

View File

@ -0,0 +1 @@
[{}

View File

@ -0,0 +1,2 @@
1
':' expected near end of file

View File

@ -0,0 +1 @@
{"a"

View File

@ -0,0 +1,2 @@
1
unexpected token near end of file

View File

@ -0,0 +1 @@
{"a":

View File

@ -0,0 +1,2 @@
1
premature end of input near '"a'

View File

@ -0,0 +1 @@
{"a":"a

View File

@ -0,0 +1,2 @@
1
invalid token near '1e'

View File

@ -0,0 +1 @@
[1ea]

View File

@ -0,0 +1,2 @@
1
real number overflow near '-123123e100000'

View File

@ -0,0 +1 @@
[-123123e100000]

View File

@ -0,0 +1,2 @@
1
real number overflow near '123123e100000'

View File

@ -0,0 +1 @@
[123123e100000]

View File

@ -0,0 +1,2 @@
1
invalid token near '1e'

View File

@ -0,0 +1 @@
[1e]

View File

@ -0,0 +1,2 @@
1
invalid token near '1.'

View File

@ -0,0 +1 @@
[1.]

View File

@ -0,0 +1,2 @@
1
real number underflow near '123e-10000000'

View File

@ -0,0 +1 @@
[123e-10000000]

27
test/suites/invalid-strip/run Executable file
View File

@ -0,0 +1,27 @@
#!/bin/sh
#
# Copyright (c) 2009 Petri Lehtinen <petri@digip.org>
#
# Jansson is free software; you can redistribute it and/or modify
# it under the terms of the MIT license. See LICENSE for details.
is_test() {
test -d $test_path
}
run_test() {
$json_process <$test_path/input >$test_log/stdout 2>$test_log/stderr
valgrind_check $test_log/stderr || return 1
cmp -s $test_path/error $test_log/stderr
}
show_error() {
valgrind_show_error && return
echo "EXPECTED ERROR:"
nl -bn $test_path/error
echo "ACTUAL ERROR:"
nl -bn $test_log/stderr
}
. $top_srcdir/test/scripts/run-tests.sh

View File

@ -0,0 +1,2 @@
1
control character 0x9 near '"'

View File

@ -0,0 +1 @@
[" <-- tab character"]

View File

@ -0,0 +1,2 @@
1
too big negative integer near '-123123123123123'

View File

@ -0,0 +1 @@
[-123123123123123]

View File

@ -0,0 +1,2 @@
1
too big integer near '123123123123123'

Some files were not shown because too many files have changed in this diff Show More