Add stripped unit tests for decoder

That is, test cases where there's no newline or other whitespace at
the beginning or end of input. This was implemented by adding a
--strip option to split-testfile to strip the input file after writing
it.

The actual test JSON texts are the same as testdata/invalid and
testdata/valid. The expected output of the invalid cases had to be
adjusted a bit: because there's no newline at the end, some of the
line numbers needed to be changed.

test/run-test

@@ -44,7 +44,10 @@ for testfile in $TESTFILES; do
     tmpdir="testlogs/`basename $testfile`"
     rm -rf $tmpdir
     mkdir -p $tmpdir
-    ${srcdir}/split-testfile.py $testfile $tmpdir | while read name; do
+    if echo "$testfile" | grep -q -E -e '-strip$'; then
+        opts="--strip"
+    fi
+    ${srcdir}/split-testfile.py $opts $testfile $tmpdir | while read name; do
         run_test loadf_dumpf $tmpdir/$name
         run_test loads_dumps $tmpdir/$name
         run_test load_file_dump_file $tmpdir/$name

test/split-testfile.py

@@ -7,6 +7,13 @@
 
 import os
 import sys
+from optparse import OptionParser
+
+def strip_file(filename):
+    with open(filename) as fobj:
+        data = fobj.read()
+    with open(filename, 'w') as fobj:
+        fobj.write(data.strip())
 
 def open_files(outdir, i, name):
     basename = '%02d_%s' % (i, name)
@@ -16,12 +23,17 @@ def open_files(outdir, i, name):
     return open(input_path, 'w'), open(output_path, 'w')
 
 def main():
-    if len(sys.argv) != 3:
+    parser = OptionParser('usage: %prog [options] inputfile outputdir')
-        print 'usage: %s input-file output-directory' % sys.argv[0]
+    parser.add_option('--strip', help='strip whitespace from input',
+                      action='store_true', default=False)
+    options, args = parser.parse_args()
+
+    if len(args) != 2:
+        parser.print_help()
         return 2
 
-    infile = os.path.normpath(sys.argv[1])
+    infile = os.path.normpath(args[0])
-    outdir = os.path.normpath(sys.argv[2])
+    outdir = os.path.normpath(args[1])
 
     if not os.path.exists(outdir):
         print >>sys.stderr, 'output directory %r does not exist!' % outdir
@@ -37,6 +49,8 @@ def main():
             if input is not None and output is not None:
                 input.close()
                 output.close()
+                if options.strip:
+                    strip_file(input.name)
             input, output = open_files(outdir, n, line[5:line.find(' ====\n')])
             current = input
         elif line == '====\n':

test/test-invalid

@@ -5,7 +5,7 @@
 # Jansson is free software; you can redistribute it and/or modify
 # it under the terms of the MIT license. See LICENSE for details.
 
-TESTFILES="${srcdir}/testdata/invalid ${srcdir}/testdata/invalid-unicode"
+TESTFILES="${srcdir}/testdata/invalid ${srcdir}/testdata/invalid-strip ${srcdir}/testdata/invalid-unicode"
 
 run_test() {
     local prog=$1

test/test-valid

@@ -5,7 +5,7 @@
 # Jansson is free software; you can redistribute it and/or modify
 # it under the terms of the MIT license. See LICENSE for details.
 
-TESTFILES="${srcdir}/testdata/valid"
+TESTFILES="${srcdir}/testdata/valid ${srcdir}/testdata/valid-strip"
 
 run_test() {
     local prog=$1

test/testdata/invalid-strip

@@ -0,0 +1,195 @@
+==== empty ====
+====
+1
+'[' or '{' expected near end of file
+==== null ====
+null
+====
+1
+'[' or '{' expected near 'null'
+==== lone-open-brace ====
+{
+====
+1
+string or '}' expected near end of file
+==== lone-open-bracket ====
+[
+====
+1
+']' expected near end of file
+==== bracket-comma ====
+[,
+====
+1
+unexpected token near ','
+==== bracket-one-comma ====
+[1,
+====
+1
+']' expected near end of file
+==== unterminated-string ====
+["a
+====
+1
+premature end of input near '"a'
+==== unterminated-array ====
+["a"
+====
+1
+']' expected near end of file
+==== apostrophe ====
+['
+====
+1
+invalid token near '''
+==== brace-comma ====
+{,
+====
+1
+string or '}' expected near ','
+==== unterminated-empty-key ====
+{"
+====
+1
+premature end of input near '"'
+==== unterminated-key ====
+{"a
+====
+1
+premature end of input near '"a'
+==== object-no-colon ====
+{"a"
+====
+1
+':' expected near end of file
+==== object-apostrophes ====
+{'a'
+====
+1
+string or '}' expected near '''
+==== object-no-value ====
+{"a":
+====
+1
+unexpected token near end of file
+==== object-unterminated-value ====
+{"a":"a
+====
+1
+premature end of input near '"a'
+==== object-garbage-at-end ====
+{"a":"a" 123}
+====
+1
+'}' expected near '123'
+==== unterminated-object-and-array ====
+{[
+====
+1
+string or '}' expected near '['
+==== unterminated-array-and-object ====
+[{
+====
+1
+string or '}' expected near end of file
+==== object-in-unterminated-array ====
+[{}
+====
+1
+']' expected near end of file
+==== extra-comma-in-array ====
+[1,]
+====
+1
+unexpected token near ']'
+==== extra-command-in-multiline-array ====
+[1,
+2,
+3,
+4,
+5,
+]
+====
+6
+unexpected token near ']'
+==== real-truncated-at-point ====
+[1.]
+====
+1
+invalid token near '1.'
+==== real-truncated-at-e ====
+[1e]
+====
+1
+invalid token near '1e'
+==== real-garbage-after-e ====
+[1ea]
+====
+1
+invalid token near '1e'
+==== integer-starting-with-zero ====
+[012]
+====
+1
+invalid token near '0'
+==== negative-integer-starting-with-zero ====
+[-012]
+====
+1
+invalid token near '-0'
+==== invalid-identifier ====
+[troo
+====
+1
+invalid token near 'troo'
+==== invalid-escap ====
+["\a <-- invalid escape"]
+====
+1
+invalid escape near '"\'
+==== tab-character-in-string ====
+["	 <-- tab character"]
+====
+1
+control character 0x9 near '"'
+==== null-byte-in-string ====
+["\u0000 (null byte not allowed)"]
+====
+1
+\u0000 is not allowed
+==== truncated-unicode-surrogate ====
+["\uDADA (first surrogate without the second)"]
+====
+1
+invalid Unicode '\uDADA'
+==== invalid-second-surrogate ====
+["\uD888\u3210 (first surrogate and invalid second surrogate)"]
+====
+1
+invalid Unicode '\uD888\u3210'
+==== lone-second-surrogate ====
+["\uDFAA (second surrogate on it's own)"]
+====
+1
+invalid Unicode '\uDFAA'
+==== unicode-identifier ====
+å
+====
+1
+'[' or '{' expected near 'å'
+==== ascii-unicode-identifier ====
+aå
+====
+1
+'[' or '{' expected near 'a'
+==== garbage-at-the-end ====
+[1,2,3]foo
+====
+1
+end of file expected near 'foo'
+==== garbage-after-newline ====
+[1,2,3]
+foo
+====
+2
+end of file expected near 'foo'

test/testdata/valid-strip

@@ -0,0 +1,68 @@
+==== empty-string ====
+[""]
+==== short-string ====
+["a"]
+==== simple-ascii-string ====
+["abcdefghijklmnopqrstuvwxyz1234567890 "]
+==== utf-8-string ====
+["€þıœəßð some utf-8 ĸʒ×ŋµåäö𝄞"]
+==== string-escapes ====
+["\"\\\/\b\f\n\r\t"]
+==== one-byte-utf-8 ====
+["\u002c one-byte UTF-8"]
+==== two-byte-utf-8 ====
+["\u0123 two-byte UTF-8"]
+==== three-byte-utf-8 ====
+["\u0821 three-byte UTF-8"]
+==== utf-surrogate-four-byte-encoding ====
+["\uD834\uDD1E surrogate, four-byte UTF-8"]
+==== escaped-utf-control-char ====
+["\u0012 escaped control character"]
+==== simple-int-0 ====
+[0]
+==== simple-int-1 ====
+[1]
+==== simple-int-123 ====
+[123]
+==== negative-zero ====
+[-0]
+==== negative-one ====
+[-1]
+==== negative-int ====
+[-123]
+==== simple-real ====
+[123.456789]
+==== real-exponent ====
+[123e45]
+==== real-capital-e ====
+[1E22]
+==== real-positive-exponent ====
+[1e+2]
+==== real-negative-exponent ====
+[1e-2]
+==== real-capital-e-positive-exponent ====
+[1E+2]
+==== real-capital-e-negative-exponent ====
+[1E-2]
+==== real-fraction-exponent ====
+[123.456e78]
+==== true ====
+[true]
+==== false ====
+[false]
+==== null ====
+[null]
+==== empty-array ====
+[]
+==== empty-object-in-array ====
+[{}]
+==== complex-array ====
+[1,2,3,4,
+"a", "b", "c",
+{"foo": "bar", "core": "dump"},
+true, false, true, true, null, false
+]
+==== empty-object ====
+{}
+==== simple-object ====
+{"a":[]}