diff --git a/.travis.yml b/.travis.yml index 9d4afb9..eea7e88 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,53 @@ language: python +sudo: false + +# Effectively the list below is for Linux; MacOS is kept separately +os: + - linux + addons: apt: packages: - bash - dash - zsh + - ash + - busybox + - ksh + +env: +# global: +# - DEBUG=99 +# - TEST_PATTERN='test/*.sh' + matrix: + - SHELL_PROGS=bash + - SHELL_PROGS=ash + - SHELL_PROGS=zsh + - SHELL_PROGS=dash + - SHELL_PROGS=busybox +# - SHELL_PROGS=ksh +# - SHELL_PROGS=ksh88 +# - SHELL_PROGS=ksh93 + +# Note that this does not inherit env: or pacages from above +matrix: + include: + - os: macos + env: SHELL_PROGS=bash + - os: macos + env: SHELL_PROGS=dash + +before_install: +- if [ "$TRAVIS_OS_NAME" = "osx" ] ; then brew update; brew install busybox bash ash dash zsh ksh ; fi # Whatever the current shebang, replace with hardcoded shell -script: > - sed -i '1s@.*@#!/usr/bin/env bash@' JSON.sh && ./all-tests.sh && - sed -i '1s@.*@#!/usr/bin/env zsh@' JSON.sh && ./all-tests.sh && - sed -i '1s@.*@#!/usr/bin/env dash@' JSON.sh && ./all-tests.sh +#script: > +# sed -i '1s@.*@#!/usr/bin/env bash@' JSON.sh && ./all-tests.sh && +# sed -i '1s@.*@#!/usr/bin/env zsh@' JSON.sh && ./all-tests.sh && +# sed -i '1s@.*@#!/usr/bin/env dash@' JSON.sh && ./all-tests.sh + +# This version of the script can use specified shell to source and test JSON.sh +# Note that some platforms can lack some interpreters, so a test run +# is effectively skipped and the test looks green if that's the case +script: ./all-tests.sh diff --git a/JSON.sh b/JSON.sh index 0778633..532a751 100755 --- a/JSON.sh +++ b/JSON.sh @@ -1,4 +1,163 @@ #!/bin/sh +# +# Copyright (C) 2014-2015 Dominic Tarr +# Copyright (C) 2015-2017 Eaton +# +#! \file JSON.sh +# \brief A json parser written in shell-script +# \author Dominic Tarr +# \author "aidanhs" added +# \author Jim Klimov +# \details Based on Dominic Tarr JSON.sh +# https://github.com/dominictarr/JSON.sh/blob/master/JSON.sh +# including "aidanhs" added support beyond bash - +# for ash/dash/zsh shells and busybox shell, +# Forked and further modified by Eaton / Jim Klimov +# https://github.com/jimklimov/JSON.sh +# +# NOTE: This script may be used standalone or sourced into your interpreter. +# For the latter use-case it is recommended to pre-set JSONSH_SOURCED=yes +# +# The MIT License (MIT) +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# Apache License, Version 2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE: The script code requires a shell with support for "local" +# keyword; this includes not only BASH but some more nowadays. +# TODO: Add a way to detect supported shell interpreter features +# (e.g. double-brackets, local keyword, regex expressions...) +# to either refuse running in a shell or pick an implementation +# for certain code paths. + +get_shellname() { + # Linux + if [ -x "/proc/$$/exe" ] ; then + readlink "/proc/$$/exe" 2>/dev/null && return 0 + ls -la "/proc/$$/exe" | sed -e 's,^[^\/]*/,/,' -e 's,^.* -> ,,' 2>/dev/null && return 0 + fi + + # Solaris/illumos + if [ -x "/proc/$$/path/a.out" ] ; then + readlink "/proc/$$/path/a.out" 2>/dev/null && return 0 + ls -la "/proc/$$/path/a.out" | sed -e 's,^[^\/]*/,/,' -e 's,^.* -> ,,' 2>/dev/null && return 0 + fi + + # By far this is most portable approach... although forking makes it slow + ps -e -o pid,comm | while read _PID _COMM ; do + [ "$$" = "$_PID" ] && echo "$_COMM" && return 0 + done + + return 1 +} + +get_shellbasename() { + basename "$(get_shellname)" || echo "sh" +} + +# Flag that can be passed by caller - if we can not detect the interpreter (or +# know it as unsupported), may we try to re-execute with a more capable one? +[ -n "${SHELL_CANREEXEC-}" ] || SHELL_CANREEXEC=yes +SHELL_BASENAME="$(get_shellbasename)" + +# Got support for regular expressions in extended-test [[ "$1" =~ ${regex} ]] ? +SHELL_REGEX=no +# Got support for pattern substitution in curly braces ${varname/pat/subst} ? +SHELL_TWOSLASH=no + +SHELL_SUPPORTED=yes +case "$SHELL_BASENAME" in + bash) + SHELL_REGEX=yes + SHELL_TWOSLASH=yes + ;; + dash|ash) # The spartan bare minimum + ;; + busybox*) + SHELL_TWOSLASH=yes + SHELL_BASENAME=busybox + ;; + #ksh93) ;; + #ksh88) ;; + #ksh) ;; + zsh) + SHELL_REGEX=yes + SHELL_TWOSLASH=yes + ;; + *) + if [ -n "${BASH-}" ] || [ -n "${BASH_VERSION-}" ] || [ -n "${ZSH_VERSION-}" ] ; then + SHELL_REGEX=yes + SHELL_TWOSLASH=yes + SHELL_BASENAME=bash + elif [ -n "${ZSH_VERSION-}" ] ; then + SHELL_REGEX=yes + SHELL_TWOSLASH=yes + SHELL_BASENAME=zsh + else + SHELL_SUPPORTED=no + fi + ;; +esac + +# TODO: detect having been sourced into non-bash shells? +if [ -z "${JSONSH_SOURCED-}" ]; then + case "$SHELL_BASENAME" in + bash) + if [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] ; then + JSONSH_SOURCED=no + fi + if [ -n "${BASH-}" ] && [ "$0" = "-bash" ] ; then + JSONSH_SOURCED=yes + fi + ;; + *) JSONSH_SOURCED=no ;; + esac +fi + +if [ "$SHELL_SUPPORTED" != "yes" ]; then + echo "Unknown shell for JSON.sh: $SHELL_BASENAME" >&2 + if [ "$SHELL_CANREEXEC" != no ] && [ "$JSONSH_SOURCED" != "yes" ] ; then + # NOTE: This can break scripts which source this file and are not in bash + for _TRY_SHELL in bash dash zsh ash busybox false ; do + [ "$_TRY_SHELL" = busybox ] && _TRY_SHELL="busybox sh" + ( $_TRY_SHELL -c "date" >/dev/null 2>/dev/null ) && break + done + + echo "ERROR: JSON.sh requires to be run with BASH/ZSH/ASH/DASH interpreter! Subshelling due to SHELL_CANREEXEC=$SHELL_CANREEXEC : $_TRY_SHELL ..." >&2 + ( $_TRY_SHELL "$0" "$@" ) + exit $? + else + echo "WARNING: Not changing shell because SHELL_CANREEXEC=$SHELL_CANREEXEC or JSONSH_SOURCED=$JSONSH_SOURCED - but JSON parsing can fail later on" >&2 + fi +fi throw() { echo "$*" >&2 @@ -9,27 +168,160 @@ BRIEF=0 LEAFONLY=0 PRUNE=0 NO_HEAD=0 +ALLOWEMPTYINPUT=1 NORMALIZE_SOLIDUS=0 +SORTDATA_OBJ="" +SORTDATA_ARR="" +NORMALIZE=0 +NORMALIZE_NUMBERS=0 +NORMALIZE_NUMBERS_FORMAT='%.6f' +NORMALIZE_NUMBERS_STRIP=0 +EXTRACT_JPATH="" +TOXIC_NEWLINE=0 +COOKASTRING=0 +COOKASTRING_INPUT="" + +findbin() { + # Locates a named binary or one from path, prints to stdout + BIN="" + for P in "$@" ; do case "$P" in + /*) [ -x "$P" ] && BIN="$P" && break;; + *) BIN="$(which "$P" 2>/dev/null | tail -1)" && [ -n "$BIN" ] && [ -x "$BIN" ] && break || BIN="";; + esac; done + [ -n "$BIN" ] && [ -x "$BIN" ] && echo "$BIN" && return 0 + return 1 +} + +# May be passed by caller; also may pass AWK_OPTS *for it* then +[ -z "$AWK" ] && AWK_OPTS="" && \ + AWK="$(findbin /usr/xpg4/bin/awk gawk nawk oawk awk)" +# Error-checked in one optional place it may be needed + +# Different OSes have different greps... we like a GNU one +[ -z "$GGREP" ] && \ + GGREP="$(findbin /{usr,opt}/{gnu,sfw}/bin/grep ggrep /usr/xpg4/bin/grep grep)" +[ -n "$GGREP" ] && [ -x "$GGREP" ] || throw "No GNU GREP was found!" + +[ -z "$GEGREP" ] && \ + GEGREP="$(findbin /{usr,opt}/{gnu,sfw}/bin/egrep gegrep /usr/xpg4/bin/egrep egrep)" +[ -n "$GEGREP" ] && [ -x "$GEGREP" ] || throw "No GNU EGREP was found!" + +[ -z "$GSORT" ] && \ + GSORT="$(findbin /{usr,opt}/{gnu,sfw}/bin/sort gsort sort /usr/xpg4/bin/sort)" +[ -n "$GSORT" ] && [ -x "$GSORT" ] || throw "No GNU SORT was found!" + +[ -z "$GSED" ] && \ + GSED="$(findbin /{usr,opt}/{gnu,sfw}/bin/sed /usr/xpg4/bin/sed gsed sed)" +[ -n "$GSED" ] && [ -x "$GSED" ] || throw "No GNU SED was found!" usage() { echo - echo "Usage: JSON.sh [-b] [-l] [-p] [-s] [-h]" + echo "Usage: JSON.sh [-b] [-l] [-p] [ -P] [-s] [--no-newline] [-d] \ " + echo " [-x 'regex'] [-S|-S='args'] [-N|-N='args'] \ " + echo " [-Nnx|-Nnx='fmtstr'|-Nn|-Nn='fmtstr'] < markup.json" + echo " JSON.sh [-h]" + echo "-h - This help text." echo echo "-p - Prune empty. Exclude fields with empty values." + echo "-P - Pedantic mode, forbids acception of empty input documents." echo "-l - Leaf only. Only show leaf nodes, which stops data duplication." echo "-b - Brief. Combines 'Leaf only' and 'Prune empty' options." echo "-n - No-head. Do not show nodes that have no path (lines that start with [])." echo "-s - Remove escaping of the solidus symbol (straight slash)." - echo "-h - This help text." + echo "-x 'regex' - rather than showing all document from the root element," + echo " extract the items rooted at path(s) matching the regex (see the" + echo " comma-separated list of nested hierarchy names in general output," + echo " brackets not included) e.g. regex='^\"level1obj\",\"level2arr\",0'" + echo "--no-newline - rather than concatenating detected line breaks in markup," + echo " return with error when this is seen in input" + echo "-d - Enable debugging traces to stderr (repeat or use -d=NUM to bump)" + echo + echo "Sorting is also available, although limited to single-line strings in" + echo "the markup (multilines are automatically escaped into backslash+n):" + echo "-S - Sort the contents of items in JSON markup and leaf-list markup:" + echo " 'sort' objects by key names and then values, and arrays by values" + echo "-S='args' - use 'sort \$args' for content sorting, e.g. use -S='-n -r'" + echo " for reverse numeric sort" + echo "-So|-So='args' - enable sorting (with given arguments) only for objects" + echo "-Sa|-Sa='args' - enable sorting (with given arguments) only for arrays" + echo + echo "An input JSON markup can be normalized into single-line no-whitespace:" + echo "-N - Normalize the input JSON markup into a single-line JSON output;" + echo " in this mode syntax and spacing are normalized, data order remains" + echo "-N='args' - Normalize the input JSON markup into a single-line JSON" + echo " output with contents sorted like for -S='args', e.g. use -N='-n'" + echo " This is equivalent to -N -S='args', just more compact to write" + echo "-No='args' - enable sorting (with given arguments) only for objects" + echo "-Na='args' - enable sorting (with given arguments) only for arrays" + echo + echo "Numeric values can be normalized (e.g. convert engineering into layman)" + echo "-Nn='fmtstr' - printf the detected numeric values with the fmtstr conversion" + echo "-Nn - assume 'fmtstr'='%.6f' (with 6 precision digits after period)" + echo "-Nnx - -Nn + strip trailing zeroes and trailing period (for whole numbers)" + echo + echo "To help JSON-related scripting, with '-Q' an input plaintext can be cooked" + echo "into a string valid for JSON (backslashes, quotes and newlines escaped," + echo "with no trailing newline); after cooking, the script exits:" + echo ' COOKEDSTRING="$(somecommand 2>&1 | JSON.sh -Q)"' + echo "A '-QQ' mode also exists to cook a (single) command-line argument:" + echo ' COOKEDSTRING="$(JSON.sh -QQ "$SAVED_INPUT")"' + echo "This can also be used to pack JSON in JSON. Note that '-QQ' ignores stdin." echo } +validate_debuglevel() { + ### Beside command-line, debugging can be enabled by envvars from the caller + { [ x"${DEBUG-}" = xy ] || [ x"${DEBUG-}" = xyes ] ; } && DEBUG=1 + [ -n "${DEBUG-}" ] && [ "${DEBUG-}" -ge 0 ] 2>/dev/null || DEBUG=0 +} + +unquote() { + # Remove single or double quotes surrounding the token + $GSED "s,^'\(.*\)'\$,\1," 2>/dev/null | \ + $GSED 's,^\"\(.*\)\"$,\1,' 2>/dev/null +} + +### Empty and non-numeric and non-positive values should be filtered out here +is_positive() { + [ -n "$1" ] && [ "$1" -gt 0 ] 2>/dev/null +} +default_posval() { + eval is_positive "\$$1" || eval "$1"="$2" +} + +print_debug() { + # Required params: + # $1 Debug level of the message + # $2.. The message to print to stderr (if $DEBUG>=$1) + DL="$1" + shift + [ "$DEBUG" -ge "$DL" ] 2>/dev/null && \ + printf '[%s]DEBUG(%s): %s\n' "$$" "$DL" "$*" >&2 + : +} + +tee_stderr() { + TEE_TAG="TEE_STDERR: " + [ -n "$1" ] && TEE_TAG="$1:" + [ -n "$2" ] && [ "$2" -ge 0 ] 2>/dev/null && \ + TEE_DEBUG="$2" || \ + TEE_DEBUG=$DEBUGLEVEL_PRINTTOKEN_PIPELINE + + ### If debug is not enabled, skip tee'ing quickly with little impact + ### The first "grep" should ensure that input for "while" has a trailing newline + [ "$DEBUG" -lt "$TEE_DEBUG" ] 2>/dev/null && cat || \ + $GGREP '' | while IFS= read -r LINE; do + printf '%s\n' "$LINE" + print_debug "$TEE_DEBUG" "$TEE_TAG" "$LINE" + done + : +} + parse_options() { set -- "$@" - local ARGN=$# - while [ "$ARGN" -ne 0 ] + while [ "$#" -gt 0 ] do - case $1 in + case "$1" in -h) usage exit 0 ;; @@ -41,24 +333,96 @@ parse_options() { ;; -p) PRUNE=1 ;; + -P) ALLOWEMPTYINPUT=0 + ;; -n) NO_HEAD=1 ;; -s) NORMALIZE_SOLIDUS=1 ;; - ?*) echo "ERROR: Unknown option." + -N) NORMALIZE=1 + ;; + -N=*) NORMALIZE=1 + SORTDATA_OBJ="$GSORT $(echo "$1" | $GSED 's,^-N=,,' 2>/dev/null | unquote )" + SORTDATA_ARR="$GSORT $(echo "$1" | $GSED 's,^-N=,,' 2>/dev/null | unquote )" + ;; + -No=*) NORMALIZE=1 + SORTDATA_OBJ="$GSORT $(echo "$1" | $GSED 's,^-No=,,' 2>/dev/null | unquote )" + ;; + -Na=*) NORMALIZE=1 + SORTDATA_ARR="$GSORT $(echo "$1" | $GSED 's,^-Na=,,' 2>/dev/null | unquote )" + ;; + -Nnx) NORMALIZE_NUMBERS_STRIP=1 + NORMALIZE_NUMBERS=1 + ;; + -Nnx=*) NORMALIZE_NUMBERS_STRIP=1 + NORMALIZE_NUMBERS=1 + NORMALIZE_NUMBERS_FORMAT="$(echo "$1" | $GSED 's,^-Nnx=,,' 2>/dev/null | unquote )" + ;; + -Nn) NORMALIZE_NUMBERS=1 + ;; + -Nn=*) NORMALIZE_NUMBERS=1 + NORMALIZE_NUMBERS_FORMAT="$(echo "$1" | $GSED 's,^-Nn=,,' 2>/dev/null | unquote )" + ;; + -S) SORTDATA_OBJ="$GSORT" + SORTDATA_ARR="$GSORT" + ;; + -So) SORTDATA_OBJ="$GSORT" + ;; + -Sa) SORTDATA_ARR="$GSORT" + ;; + -S=*) + SORTDATA_OBJ="$GSORT $(echo "$1" | $GSED 's,^-S=,,' 2>/dev/null | unquote )" + SORTDATA_ARR="$GSORT $(echo "$1" | $GSED 's,^-S=,,' 2>/dev/null | unquote )" + ;; + -So=*) + SORTDATA_OBJ="$GSORT $(echo "$1" | $GSED 's,^-So=,,' 2>/dev/null | unquote )" + ;; + -Sa=*) + SORTDATA_ARR="$GSORT $(echo "$1" | $GSED 's,^-Sa=,,' 2>/dev/null | unquote )" + ;; + -x) EXTRACT_JPATH="$2" + shift + ;; + -x=*) EXTRACT_JPATH="$(echo "$1" | $GSED 's,^-x=,,' 2>/dev/null)" + ;; + --no-newline) + TOXIC_NEWLINE=1 + ;; + -d) DEBUG="$(expr $DEBUG + 1)" + JSONSH_DEBUGGING_SETUP=notdone + JSONSH_DEBUGGING_REPORT=notdone + ;; + -d=*) DEBUG="$(echo "$1" | $GSED 's,^-d=,,' 2>/dev/null)" + JSONSH_DEBUGGING_SETUP=notdone + JSONSH_DEBUGGING_REPORT=notdone + ;; + -Q) COOKASTRING=1 + ;; + -QQ) COOKASTRING=2 + COOKASTRING_INPUT="$2" + shift + ;; + ?*) echo "ERROR: Unknown option: '$1'." usage exit 0 ;; esac shift 1 - ARGN=$((ARGN-1)) done + + validate_debuglevel + + # For normalized data, we do the whole job and just return the top object + [ "$NORMALIZE" = 1 ] && BRIEF=0 && LEAFONLY=0 && PRUNE=0 } -awk_egrep () { - local pattern_string=$1 +awk_egrep() { + pattern_string="$1" + + [ -z "${AWK-}" ] && throw "No AWK found!" + [ ! -x "$AWK" ] && throw "Not executable AWK='$AWK'!" - gawk '{ + "${AWK}" $AWK_OPTS '{ while ($0) { start=match($0, pattern); token=substr($0, start, RLENGTH); @@ -68,141 +432,474 @@ awk_egrep () { }' pattern="$pattern_string" } -tokenize () { - local GREP +strip_newlines() { + # replace line returns inside strings in input with \n string + local ILINE + local LINESTRIP + local NUMQ + local ODD + local INSTRING=0 + local LINENUM=0 + + # The first "grep" should ensure that input for "while" has a trailing newline + $GGREP '' | \ + tee_stderr BEFORE_STRIP $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \ + while IFS="" read -r ILINE; do + if [ "$SHELL_TWOSLASH" = yes ]; then + # Remove escaped quotes: + LINESTRIP="${ILINE//\\\"}" + # Remove all chars but remaining quotes: + LINESTRIP="${LINESTRIP//[^\"]}" + else + LINESTRIP="$(echo "$ILINE" | $GSED -e 's,\\\",,g' -e 's,[^\"],,g')" + fi + # Count unescaped quotes: + NUMQ="${#LINESTRIP}" + ODD="$(expr $NUMQ % 2)" + LINENUM="$(expr $LINENUM + 1)" + + if [ "$ODD" -eq 1 ] && [ "$INSTRING" -eq 0 ]; then + [ "$TOXIC_NEWLINE" = 1 ] && \ + echo "ERROR: Invalid JSON markup detected: newline in a string value: at line #$LINENUM" >&2 && \ + exit 121 + printf '%s\\n' "$ILINE" + INSTRING=1 + continue + fi + + if [ "$ODD" -eq 1 ] && [ "$INSTRING" -eq 1 ]; then + printf '%s\n' "$ILINE" + INSTRING=0 + continue + fi + + if [ "$ODD" -eq 0 ] && [ "$INSTRING" -eq 1 ]; then + printf '%s\\n' "$ILINE" + continue + fi + + if [ "$ODD" -eq 0 ] && [ "$INSTRING" -eq 0 ]; then + printf '%s\n' "$ILINE" + continue + fi + done + : +} + +cook_a_string() { + ### Escape backslashes, double-quotes, tabs and newlines, in this order + $GGREP '' | $GSED -e 's,\\,\\\\,g' -e 's,\",\\",g' -e 's,\t,\\t,g' 2>/dev/null | \ + { FIRST=''; while IFS="" read -r ILINE; do + printf '%s%s' "$FIRST" "$ILINE" + [ -n "$FIRST" ] || FIRST='\n' + done; } + : +} + +cook_a_string_arg() { + # Use routine above to cook a string passed as "$1" unless it is trivial + [ -z "$1" ] && return 0 + # Strangely, for some OSes it does not suffice that all chars must be from + # the first pattern - should explicitly test that some are not forbidden + IS_TRIVIAL=no + if [ "$SHELL_REGEX" = yes ]; then + # Bash-compatible regex support required for code below. + if ! [[ "$1" =~ [\\\"] ]] >/dev/null && \ + [[ "$1" =~ ^[A-Za-z0-9\ \-\.\+\/\@\:\;\!\%\,\&\(\)\{\}]*$ ]] >/dev/null \ + ; then IS_TRIVIAL=yes; fi + else + # Support for other shells if bash-regex syntax + # is not supported there (e.g. revert to sed/grep/awk)... + if [ -n "$(echo "$1" | $GEGREP -v '[\\\"]' | $GEGREP '^[A-Za-z0-9\ \-\.\+\/\@\:\;\!\%\,\&\(\)\{\}]*$' )" ] \ + ; then IS_TRIVIAL=yes; fi + fi + + if [ "$IS_TRIVIAL" = yes ] ; then + print_debug $DEBUGLEVEL_PRINTTOKEN_PIPELINE "cook_a_string_arg(): input trivial, not cooking: '$1'" + echo "$1" + return 0 + fi + + print_debug $DEBUGLEVEL_PRINTTOKEN_PIPELINE "cook_a_string_arg(): input not trivial, cooking: '$1'" + echo "$1" | cook_a_string +} + +tokenize() { + local GREP_O local ESCAPE local CHAR - if echo "test string" | egrep -ao --color=never "test" >/dev/null 2>&1 + if echo "test string" | $GEGREP -ao --color=never "test" >/dev/null 2>&1 then - GREP='egrep -ao --color=never' - else - GREP='egrep -ao' + GREP_O="$GEGREP -ao --color=never" + elif echo "test string" | $GEGREP -ao "test" >/dev/null 2>&1 + then + GREP_O="$GEGREP -ao" + elif echo "test string" | $GEGREP -o "test" >/dev/null 2>&1 + then + GREP_O="$GEGREP -o" fi - if echo "test string" | egrep -o "test" >/dev/null 2>&1 + if [ -n "$GREP_O" ] && echo "test string" | $GREP_O "test" >/dev/null 2>&1 then ESCAPE='(\\[^u[:cntrl:]]|\\u[0-9a-fA-F]{4})' CHAR='[^[:cntrl:]"\\]' else - GREP=awk_egrep + GREP_O=awk_egrep ESCAPE='(\\\\[^u[:cntrl:]]|\\u[0-9a-fA-F]{4})' CHAR='[^[:cntrl:]"\\\\]' fi - local STRING="\"$CHAR*($ESCAPE$CHAR*)*\"" - local NUMBER='-?(0|[1-9][0-9]*)([.][0-9]*)?([eE][+-]?[0-9]*)?' + # Allow tabs inside strings + local CHART="($CHAR|[[:blank:]])" + local STRINGVAL="$CHART*($ESCAPE$CHART*)*" + local STRING="(\"$STRINGVAL\")" + local NUMBER='[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?' local KEYWORD='null|false|true' local SPACE='[[:space:]]+' # Force zsh to expand $A into multiple words - local is_wordsplit_disabled=$(unsetopt 2>/dev/null | grep -c '^shwordsplit$') - if [ $is_wordsplit_disabled != 0 ]; then setopt shwordsplit; fi - $GREP "$STRING|$NUMBER|$KEYWORD|$SPACE|." | egrep -v "^$SPACE$" - if [ $is_wordsplit_disabled != 0 ]; then unsetopt shwordsplit; fi + is_wordsplit_disabled="$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')" + if [ "$is_wordsplit_disabled" != 0 ]; then setopt shwordsplit; fi + tee_stderr BEFORE_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \ + $GREP_O "$STRING|$NUMBER|$KEYWORD|$SPACE|." | $GEGREP -v "^$SPACE$" | \ + tee_stderr AFTER_TOKENIZER $DEBUGLEVEL_PRINTTOKEN_PIPELINE + RES=$? + if [ "$is_wordsplit_disabled" != 0 ]; then unsetopt shwordsplit; fi + unset is_wordsplit_disabled || true + return $RES } -parse_array () { +parse_array() { local index=0 local ary='' + local aryml='' read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(1):" "token='$token'" case "$token" in ']') ;; *) while : do parse_value "$1" "$index" - index=$((index+1)) - ary="$ary""$value" + index="$(expr $index + 1)" + ary="$ary""$value" + if [ -n "$SORTDATA_ARR" ]; then + [ -z "$aryml" ] && aryml="$value" || aryml="$aryml +$value" + fi read -r token case "$token" in ']') break ;; ',') ary="$ary," ;; - *) throw "EXPECTED , or ] GOT ${token:-EOF}" ;; + *) throw "EXPECTED ',' or ']' GOT '${token:-EOF}'" ;; esac read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse_array(3):" "token='$token'" done ;; esac - [ "$BRIEF" -eq 0 ] && value=$(printf '[%s]' "$ary") || value= + if [ -n "$SORTDATA_ARR" ]; then + ary="$(printf '%s\n' "$aryml" | $SORTDATA_ARR | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null)" + fi + [ "$BRIEF" = 0 ] && value="$(printf '[%s]' "$ary")" || value="" : } -parse_object () { - local key +parse_object() { + local key='' local obj='' + local objml='' read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(1):" "token='$token'" case "$token" in '}') ;; *) while : do case "$token" in - '"'*'"') key=$token ;; - *) throw "EXPECTED string GOT ${token:-EOF}" ;; + '"'*'"') key="$token" ;; + *) throw "EXPECTED string GOT '${token:-EOF}'" ;; esac read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(2):" "token='$token'" case "$token" in ':') ;; - *) throw "EXPECTED : GOT ${token:-EOF}" ;; + *) throw "EXPECTED : GOT '${token:-EOF}'" ;; esac read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(3):" "token='$token'" parse_value "$1" "$key" - obj="$obj$key:$value" + obj="$obj$key:$value" + if [ -n "$SORTDATA_OBJ" ]; then + [ -z "$objml" ] && objml="$key:$value" || objml="$objml +$key:$value" + fi read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(4):" "token='$token'" case "$token" in '}') break ;; ',') obj="$obj," ;; - *) throw "EXPECTED , or } GOT ${token:-EOF}" ;; + *) throw "EXPECTED ',' or '}' GOT '${token:-EOF}'" ;; esac read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse_object(5):" "token='$token'" done ;; esac - [ "$BRIEF" -eq 0 ] && value=$(printf '{%s}' "$obj") || value= + if [ -n "$SORTDATA_OBJ" ]; then + obj="$(printf '%s\n' "$objml" | $SORTDATA_OBJ | tr '\n' ',' | $GSED 's|,*$||' 2>/dev/null | $GSED 's|^,*||' 2>/dev/null)" + fi + [ "$BRIEF" = 0 ] && value="$(printf '{%s}' "$obj")" || value="" : } -parse_value () { - local jpath="${1:+$1,}$2" isleaf=0 isempty=0 print=0 +REGEX_NUMBER='^[+-]?([.][0-9]+|(0+|[1-9][0-9]*)([.][0-9]*)?)([eE][+-]?[0-9]*)?$' +parse_value() { + local jpath="${1:+$1,}$2" + local isleaf=0 + local isempty=0 + local print=0 case "$token" in - '{') parse_object "$jpath" ;; - '[') parse_array "$jpath" ;; + '{') parse_object "$jpath" + [ "$value" = '{}' ] && isempty=1 + ;; + '[') parse_array "$jpath" + [ "$value" = '[]' ] && isempty=1 + ;; # At this point, the only valid single-character tokens are digits. - ''|[!0-9]) throw "EXPECTED value GOT ${token:-EOF}" ;; - *) value=$token + ''|[!0-9]) if [ "$ALLOWEMPTYINPUT" = 1 ] && [ -z "$jpath" ] && [ -z "$token" ]; then + print_debug $DEBUGLEVEL_PRINTPATHVAL \ + 'Got a NULL document as input (no jpath, no token)' >&2 + value='{}' + else + throw "EXPECTED value GOT '${token:-EOF}'" + fi ;; + +*|-*|[0-9]*|.*) # Potential number - separate hit in case for efficiency + print_debug $DEBUGLEVEL_PRINTPATHVAL \ + "token '$token' is a suspected number" >&2 + # TODO: Bash regex and more if's + DO_NORMALIZE=no + if [ "$NORMALIZE_NUMBERS" = 1 ] ; then + if [ "$SHELL_REGEX" = yes ]; then + [[ "$token" =~ ${REGEX_NUMBER} ]] && DO_NORMALIZE=yes + else + [ -n "$(echo "$token" | $GEGREP "${REGEX_NUMBER}" )" ] && DO_NORMALIZE=yes + fi + fi + + if [ "$DO_NORMALIZE" = yes ]; then + value="$(printf "$NORMALIZE_NUMBERS_FORMAT" "$token")" || \ + value="$token" + print_debug $DEBUGLEVEL_PRINTPATHVAL "normalized numeric token" \ + "'$token' into '$value'" >&2 + if [ "$NORMALIZE_NUMBERS_STRIP" = 1 ]; then + valuetmp="$(echo "$value" | $GSED -e 's,0*$,,g' -e 's,\.$,,' 2>/dev/null)" && \ + value="$valuetmp" + unset valuetmp + print_debug $DEBUGLEVEL_PRINTPATHVAL "stripped numeric token" \ + "'$token' into '$value'" >&2 + fi + else + # Not a number or no normalization - process like default + value="$token" + if [ "$NORMALIZE_SOLIDUS" = 1 ]; then + if [ "$SHELL_TWOSLASH" = yes ] ; then + value="${value//\\\//\/}" + else + value="$(echo "$value" | $GSED 's#\\/#/#g')" + fi + fi + fi + isleaf=1 + { [ "$value" = '""' ] || [ "$value" = '' ] ; } && isempty=1 + ;; + *) value="$token" # if asked, replace solidus ("\/") in json strings with normalized value: "/" - [ "$NORMALIZE_SOLIDUS" -eq 1 ] && value=$(echo "$value" | sed 's#\\/#/#g') + if [ "$NORMALIZE_SOLIDUS" = 1 ]; then + if [ "$SHELL_TWOSLASH" = yes ] ; then + value="${value//\\\//\/}" + else + value="$(echo "$value" | $GSED 's#\\/#/#g')" + fi + fi isleaf=1 [ "$value" = '""' ] && isempty=1 ;; esac + + if [ "$NORMALIZE" = 1 ]; then + # Ensure a "true" output from the "if" for "return" + if [ "$jpath" != '' ]; then : ; else + print_debug $DEBUGLEVEL_PRINTPATHVAL \ + "Non-root keys were skipped due to normalization mode" + printf "%s\n" "$value" + fi + return + fi + + ### Skip printing larger objects in brief mode [ "$value" = '' ] && return [ "$NO_HEAD" -eq 1 ] && [ -z "$jpath" ] && return [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 0 ] && print=1 - [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && [ $PRUNE -eq 0 ] && print=1 - [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 1 ] && [ "$isempty" -eq 0 ] && print=1 + [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && [ $PRUNE -eq 0 ] && print=2 + [ "$LEAFONLY" -eq 0 ] && [ "$PRUNE" -eq 1 ] && [ "$isempty" -eq 0 ] && print=3 [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 1 ] && \ - [ $PRUNE -eq 1 ] && [ $isempty -eq 0 ] && print=1 - [ "$print" -eq 1 ] && printf "[%s]\t%s\n" "$jpath" "$value" + [ $PRUNE -eq 1 ] && [ $isempty -eq 0 ] && print=4 + + ### A special case of an empty array or object - for leaf printing + ### without pruning, we are interested in these: + [ "$LEAFONLY" -eq 1 ] && [ "$isleaf" -eq 0 ] && [ "$isempty" -eq 1 ] && \ + [ $PRUNE -eq 0 ] && print=5 + + if [ "$print" -ne 0 ] && [ -n "$EXTRACT_JPATH" ] ; then + if [ "$SHELL_REGEX" = yes ]; then + ### BASH regex matching: + [[ "${jpath}" =~ ${EXTRACT_JPATH} ]] || print=-1 + else + [ -n "$(echo "${jpath}" | $GEGREP "${EXTRACT_JPATH}")" ] || print=-1 + fi + fi + + print_debug $DEBUGLEVEL_PRINTPATHVAL \ + "JPATH='$jpath' VALUE='$value' B='$BRIEF'" \ + "isleaf='$isleaf'/L='$LEAFONLY' isempty='$isempty'/P='$PRUNE':" \ + "print='$print'" >&2 + + [ "$print" -gt 0 ] && printf "[%s]\t%s\n" "$jpath" "$value" : } -parse () { +parse() { read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse(1):" "token='$token'" parse_value read -r token + print_debug $DEBUGLEVEL_PRINTTOKEN "parse(2):" "token='$token'" case "$token" in '') ;; - *) throw "EXPECTED EOF GOT $token" ;; + *) throw "EXPECTED EOF GOT '$token'" ;; esac } -if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]); -then +smart_parse() { + strip_newlines | \ + tokenize | if [ -n "$SORTDATA_OBJ$SORTDATA_ARR" ] ; then + ### Any type of sort was enabled + ( NORMALIZE=1 LEAFONLY=0 BRIEF=0 parse ) \ + | tokenize | parse + else + parse + fi +} + +JSONSH_DEBUGGING_SETUP=notdone +JSONSH_DEBUGGING_REPORT=notdone +JSONSH_DEBUGGING_DEFAULTS=notdone +jsonsh_debugging_defaults() { + [ x"$JSONSH_DEBUGGING_DEFAULTS" = xdone ] && return 0 + ### Caller can disable specific debuggers by setting their level too high + validate_debuglevel + default_posval DEBUGLEVEL_PRINTPATHVAL 1 + default_posval DEBUGLEVEL_PRINTTOKEN 2 + default_posval DEBUGLEVEL_PRINTTOKEN_PIPELINE 3 + default_posval DEBUGLEVEL_TRACE_X 4 + default_posval DEBUGLEVEL_TRACE_V 5 + default_posval DEBUGLEVEL_MERGE_ERROUT 4 + JSONSH_DEBUGGING_DEFAULTS="done" +} + +jsonsh_debugging_setup() { + [ x"$JSONSH_DEBUGGING_SETUP" = xdone ] && return 0 + # Note that the CLI options enable some debug level + + [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \ + exec 2>&1 + [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ] && \ + set +v + [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ] && \ + set -x + + JSONSH_DEBUGGING_SETUP="done" +} + +jsonsh_debugging_report() { + [ x"$JSONSH_DEBUGGING_REPORT" = xdone ] && return 0 + # Note that the CLI options enable some debug level + + [ "$DEBUG" -ge "$DEBUGLEVEL_MERGE_ERROUT" ] && \ + echo "[$$]DEBUG: Merge stderr and stdout for easier tracing with less" \ + "(DEBUGLEVEL_MERGE_ERROUT=$DEBUGLEVEL_MERGE_ERROUT)" >&2 + [ "$DEBUG" -gt 0 ] && \ + echo "[$$]DEBUG: Enabled (debugging level $DEBUG)" >&2 + [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTPATHVAL" ] && \ + echo "[$$]DEBUG: Enabled tracing of path:value printing decisions" \ + "(DEBUGLEVEL_PRINTPATHVAL=$DEBUGLEVEL_PRINTPATHVAL)" >&2 + [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ] && \ + echo "[$$]DEBUG: Enabled printing of each processed token" \ + "(DEBUGLEVEL_PRINTTOKEN=$DEBUGLEVEL_PRINTTOKEN)" >&2 + [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ] && \ + echo "[$$]DEBUG: Enabled tracing of read-in token conversions" \ + "(DEBUGLEVEL_PRINTTOKEN_PIPELINE=$DEBUGLEVEL_PRINTTOKEN_PIPELINE)" >&2 + [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_V" ] && \ + echo "[$$]DEBUG: Enable execution tracing (-v)" \ + "(DEBUGLEVEL_TRACE_V=$DEBUGLEVEL_TRACE_V)" >&2 + [ "$DEBUG" -ge "$DEBUGLEVEL_TRACE_X" ] && \ + echo "[$$]DEBUG: Enable execution tracing (-x)" \ + "(DEBUGLEVEL_TRACE_X=$DEBUGLEVEL_TRACE_X)" >&2 + + JSONSH_DEBUGGING_REPORT="done" +} + +jsonsh_cli() { + # All the logic needed to parse the CLI options and the JSON stdin + # for a common case "cat file.json | tokenize | parse" can suffice + # NOTE: If the caller sets up some specific different debugging envvars + # then consider changing JSONSH_DEBUGGING_SETUP and JSONSH_DEBUGGING_REPORT + # to e.g. "notdone" as well parse_options "$@" - tokenize | parse + jsonsh_debugging_setup + jsonsh_debugging_report + if [ "$COOKASTRING" -eq 2 ]; then + if [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN" ] || \ + [ "$DEBUG" -ge "$DEBUGLEVEL_PRINTTOKEN_PIPELINE" ] ; then + echo "[$$]DEBUG: Cooking an argument into JSON string and exiting:" "$1" >&2 + fi + cook_a_string_arg "$COOKASTRING_INPUT" + else + tee_stderr RAW_INPUT $DEBUGLEVEL_PRINTTOKEN_PIPELINE | \ + case "$COOKASTRING" in + 1) cook_a_string ;; + *) smart_parse ;; + esac + fi +} + +jsonsh_cli_subshell() ( + # Same as above, but isolated in a subshell (no variables come back) + jsonsh_cli "$@" + exit $? +) + + +########################################################### +### Active logic +jsonsh_debugging_defaults + +# If NOT sourced into a bash script, parse stdin and quit +#[ "${JSONSH_SOURCED-}" != yes ] || \ +#if [ "$0" = "$BASH_SOURCE[0]" ] || [ "$0" = "$BASH_SOURCE" ] || [ -z "${BASH-}" ] || [ -z "$BASH_SOURCE" ]; \ +if [ "${JSONSH_SOURCED-}" != yes ] +then + jsonsh_cli "$@" + exit $? fi +#if ([ "$0" = "$BASH_SOURCE" ] || ! [ -n "$BASH_SOURCE" ]); +#then +# parse_options "$@" +# tokenize | parse +#fi + # vi: expandtab sw=2 ts=2 diff --git a/README.md b/README.md index ebe9648..1822cb5 100755 --- a/README.md +++ b/README.md @@ -4,8 +4,21 @@ yo, so it's a json parser written in shell, compatible with ash, bash, dash and [![travis](https://secure.travis-ci.org/dominictarr/JSON.sh.png?branch=master)](https://travis-ci.org/dominictarr/JSON.sh) -pipe json to it, and it traverses the json objects and prints out the -path to the current object (as a JSON array) and then the object, without whitespace. +Pipe JSON to it, and it traverses the json objects and prints out the +path to the current object (as a JSON array) and then the object, +without whitespace between syntactic constructs (whitespace inside +content strings is preserved). + +Further features include the ability to "extract" the JSON paths which +match a specified regex, ability to sort the content (array and object +items), and ability to "cook" raw text into escaped strings acceptable +for passing in JSON markup. Finally, there is a mode to "normalize" an +input JSON markup into a single-line no-extra-whitespace string, with +optional sorting applied, so that the script can be used as a filter to +normalize two JSON documents so they can be compared for differences. + +A simple example follows here, and more complex ones are presented +after the break for command-line options: ``` bash $ json_parse < package.json @@ -22,6 +35,24 @@ $ json_parse < package.json # ... etc ``` +* Pruning of empty arrays/structs (from line-by-line output) is supported +as well as for strings-only before: +```bash +:; ELINE='{"emptyarr":[],"emptyobj":{},"emptystr":""}' + +# Here you see them... +:; echo -E "$ELINE" | ./JSON.sh +["emptyarr"] [] +["emptyobj"] {} +["emptystr"] "" +[] {"emptyarr":[],"emptyobj":{},"emptystr":""} + +# Here you don't ;) +:; echo -E "$ELINE" | ./JSON.sh -p +[] {"emptyarr":[],"emptyobj":{},"emptystr":""} +``` + + a more complex example: ``` bash @@ -31,24 +62,505 @@ curl registry.npmjs.org/express | ./JSON.sh | egrep '\["versions","[^"]*"\]' ## Options --b +### Usual queries, full or filtered +``` bash +Usage: JSON.sh [-b] [-l] [-p] [-x 'regex'] [-S|-S='args'] [--no-newline] [-d] +``` + +* `-b` > Brief output. Combines 'Leaf only' and 'Prune empty' options. --l +* `-l` > Leaf only. Only show leaf nodes, which stops data duplication. --p -> Prune empty. Exclude fields with empty values. +* `-p` +> Prune empty. Exclude fields with empty values (strings, arrays, objects). --n +* `-n` > No-head. Don't show nodes that have no path. Normally these output a leading '[]', which you can't use in a bash array. --s +* `-s` > Remove escaping of the solidus symbol (stright slash). --h +* `-x 'regex'` or `-x='regex'` +> "Extract" - rather than showing all document from the root element, +extract the items rooted at path(s) matching the regex (see the +comma-separated list of nested hierarchy names in general output, +brackets not included) e.g. `-x='^"level1obj","level2arr",0'` + +Sorting is also available, although limited to single-line strings in +the markup (multilines are automatically escaped into backslash+n): + +* `-S` +> Sort the contents of items in JSON markup and leaf-list markup: +`sort` objects by key names and then values, and arrays by values + +* `-S='args'` +> Use `sort $args` for content sorting, e.g. use `-S='-n -r'` for +reverse numeric sort + +* `-So='args'` and/or `-Sa='args'`, or `-So` or `-Sa` +> Only enable `sort` and set the arguments for either objects (`-So`) +or arrays/tuples (`-Sa`). This way sorting of tuples can be avoided +to keep data in valid order (as defined by the programmatic users of +the markup) and/or different rules can be used for arrays vs. objects. +Essentially, the singular `-S{='args'}` option just enables both the +`-Sa` and `-So` options with the same values. + +Other options: + +* `--no-newline` +> rather than concatenating detected line breaks in markup, return +with error when this is seen in input + +* `-d [-d...]` or `-d=NUM` +> Enable debugging traces to `stderr` (repeat or use `-d=NUM` to bump, +see the script source for details on what can be debugged and how to +select what you want) + + +### Normalization (with optional sorting) +``` bash +Usage: JSON.sh [-N|-N='args'] [-d] < markup.json +``` + +An input JSON markup can be normalized into single-line no-whitespace: +* `-N` +> Normalize the input JSON markup into a single-line JSON output; +in this mode syntax and spacing are normalized, data order remains + +* `-N='args'` +> Normalize the input JSON markup into a single-line JSON output with +contents sorted like for `-S='args'`, e.g. use `-N='-n'`. +This is equivalent to `-N -S='args'`, just more compact to write. + +* `-No='args'` and/or `-Na='args'` +> Normalize with sorting like above, but only enable and set the `sort` +arguments for either objects (`-No`) or arrays/tuples (`-Na`). This way +sorting of tuples can be avoided to keep data in valid order (as defined +by the programmatic users of the markup). + +### Cook raw data +``` bash +Usage: COOKEDSTRING="`somecommand 2>&1 | ./JSON.sh -Q`" +``` + +Cooking: +* `-Q` +> To help JSON-related scripting, a block of input plaintext can be +"cooked" into a string valid for JSON (backslashes, quotes and newlines +escaped, with no trailing newline); after cooking, the script exits. +This mode can also be used to pack JSON into JSON. + +### Ask for help +``` bash +Usage: JSON.sh [-h] +``` + +Helping: +* `-h` > Show help text. +## Complex usage examples + +A picture shows more than a thousand words, heh? +So here is a few thousand words for you, to display the new features ;) + +* First, define a complicated value contrived just for show-off. This example +can be found in the source checkout as `tests/valid/documented_example.json`. +It may be arguable that newlines in the markup are invalid... but for some +practical example, if the actual data is generated by some shell-script +(storing a copy of a multiline file or command output, etc.) - then the new +features in `JSON.sh` precisely allow to turn that into (more) valid markup ;) +```bash +:; LINE='{"var1":"val1","split +key":"value","var0":"escaped \" quote","splitValue":"there + are a newline and three spaces (one after \"there\" and two before \"are\")", +"array":["z","a","b",3,20,0,"","","\"" +,"escaping\"several\"\" +quote\"s and +newlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines", +"var38":"","emptyarr":[],"emptyobj":{}, +"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"}, + {"var":"val2","str":"z"},{"var":"val2","str":"x"}, + {"var":"val1","str":"S"},{"var":"val1","str":"\""}, +{"var":"val1","str":5},{"var":"val1","str":"5"}]}' + +### Examples related to sorting will also need this to be reproducible: +:; LANG=C; LC_ALL=C; export LANG; export LC_ALL +``` + +* Note also the use of `echo -E` to avoid shell's processing of escaped +characters (such as `\\\n` in `var8`): +```bash +:; echo -E "$LINE" +{"var1":"val1","split +key":"value","var0":"escaped \" quote","splitValue":"there + are a newline and three spaces (one after \"there\" and two before \"are\")", +"array":["z","a","b",3,20,0,"","","\"" +,"escaping\"several\"\" +quote\"s and +newlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines", +"var38":"","emptyarr":[],"emptyobj":{}, +"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"}, + {"var":"val2","str":"z"},{"var":"val2","str":"x"}, + {"var":"val1","str":"S"},{"var":"val1","str":"\""}, +{"var":"val1","str":5},{"var":"val1","str":"5"}]} +``` + +* There is a mode to detect invalid input (due to newlines in strings): +```bash +:; echo "$LINE" | ./JSON.sh --no-newline +Invalid JSON markup detected: newline in a string value: at line #1 +EXPECTED value GOT EOF +``` + +* Otherwise automatic conversion of these takes place: +```bash +:; echo -E "$LINE" | ./JSON.sh +["var1"] "val1" +["split\nkey"] "value" +["var0"] "escaped \" quote" +["splitValue"] "there\n are a newline and three spaces (one after \"there\" and two before \"are\")" +["array",0] "z" +["array",1] "a" +["array",2] "b" +["array",3] 3 +["array",4] 20 +["array",5] 0 +["array",6] "" +["array",7] "" +["array",8] "\"" +["array",9] "escaping\"several\"\"\nquote\"s and\nnewlines" +["array"] ["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"] +["aNumber"] 1 +["var8"] "string\nwith\nproper\\\nnewlines" +["var38"] "" +["emptyarr"] [] +["emptyobj"] {} +["arrOfObjs",0,"var"] "val1" +["arrOfObjs",0,"str"] "s" +["arrOfObjs",0] {"var":"val1","str":"s"} +["arrOfObjs",1,"var"] "val30" +["arrOfObjs",1,"str"] "s" +["arrOfObjs",1] {"var":"val30","str":"s"} +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",2,"str"] "z" +["arrOfObjs",2] {"var":"val2","str":"z"} +["arrOfObjs",3,"var"] "val2" +["arrOfObjs",3,"str"] "x" +["arrOfObjs",3] {"var":"val2","str":"x"} +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",4,"str"] "S" +["arrOfObjs",4] {"var":"val1","str":"S"} +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",5,"str"] "\"" +["arrOfObjs",5] {"var":"val1","str":"\""} +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",6,"str"] 5 +["arrOfObjs",6] {"var":"val1","str":5} +["arrOfObjs",7,"var"] "val1" +["arrOfObjs",7,"str"] "5" +["arrOfObjs",7] {"var":"val1","str":"5"} +["arrOfObjs"] [{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}] +[] {"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]} +``` + +* Returning a valid JSON markup string without the JSON path (i.e. use of +`JSON.sh` as a filter to convert scripted output into more valid JSON: +```bash +:; echo -E "$LINE" | ./JSON.sh -N +{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]} +``` + +* Sorted output with defaults taken by `sort` program in your OS, i.e. +alphabetic order (where `3` is greater than `20`), etc. and according to +currently exported locale/collation (influencing order of numbers over +punctuation over letters, sorting of letters with diacritics, etc.): +```bash +:; $ echo -E "$LINE" | ./JSON.sh -S +["aNumber"] 1 +["arrOfObjs",0,"str"] "5" +["arrOfObjs",0,"var"] "val1" +["arrOfObjs",0] {"str":"5","var":"val1"} +["arrOfObjs",1,"str"] "S" +["arrOfObjs",1,"var"] "val1" +["arrOfObjs",1] {"str":"S","var":"val1"} +["arrOfObjs",2,"str"] "\"" +["arrOfObjs",2,"var"] "val1" +["arrOfObjs",2] {"str":"\"","var":"val1"} +["arrOfObjs",3,"str"] "s" +["arrOfObjs",3,"var"] "val1" +["arrOfObjs",3] {"str":"s","var":"val1"} +["arrOfObjs",4,"str"] "s" +["arrOfObjs",4,"var"] "val30" +["arrOfObjs",4] {"str":"s","var":"val30"} +["arrOfObjs",5,"str"] "x" +["arrOfObjs",5,"var"] "val2" +["arrOfObjs",5] {"str":"x","var":"val2"} +["arrOfObjs",6,"str"] "z" +["arrOfObjs",6,"var"] "val2" +["arrOfObjs",6] {"str":"z","var":"val2"} +["arrOfObjs",7,"str"] 5 +["arrOfObjs",7,"var"] "val1" +["arrOfObjs",7] {"str":5,"var":"val1"} +["arrOfObjs"] [{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}] +["array",0] "" +["array",1] "" +["array",2] "\"" +["array",3] "a" +["array",4] "b" +["array",5] "escaping\"several\"\"\nquote\"s and\nnewlines" +["array",6] "z" +["array",7] 0 +["array",8] 20 +["array",9] 3 +["array"] ["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,20,3] +["emptyarr"] [] +["emptyobj"] {} +["splitValue"] "there\n are a newline and three spaces (one after \"there\" and two before \"are\")" +["split\nkey"] "value" +["var0"] "escaped \" quote" +["var1"] "val1" +["var38"] "" +["var8"] "string\nwith\nproper\\\nnewlines" +[] {"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,20,3],"emptyarr":[],"emptyobj":{},"splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"} +``` + +* Sorting with parameters, several can be passed as a single quoted string - +for example we request numeric (`20` is greater than `3` - though only for +standalone number tokens) and reversed (`a` is after `z`) sorting: +```bash +:; echo -E "$LINE" | ./JSON.sh -S='-r -n' +["var8"] "string\nwith\nproper\\\nnewlines" +["var38"] "" +["var1"] "val1" +["var0"] "escaped \" quote" +["split\nkey"] "value" +["splitValue"] "there\n are a newline and three spaces (one after \"there\" and two before \"are\")" +["emptyobj"] {} +["emptyarr"] [] +["array",0] 20 +["array",1] 3 +["array",2] 0 +["array",3] "z" +["array",4] "escaping\"several\"\"\nquote\"s and\nnewlines" +["array",5] "b" +["array",6] "a" +["array",7] "\"" +["array",8] "" +["array",9] "" +["array"] [20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""] +["arrOfObjs",0,"var"] "val30" +["arrOfObjs",0,"str"] "s" +["arrOfObjs",0] {"var":"val30","str":"s"} +["arrOfObjs",1,"var"] "val2" +["arrOfObjs",1,"str"] "z" +["arrOfObjs",1] {"var":"val2","str":"z"} +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",2,"str"] "x" +["arrOfObjs",2] {"var":"val2","str":"x"} +["arrOfObjs",3,"var"] "val1" +["arrOfObjs",3,"str"] 5 +["arrOfObjs",3] {"var":"val1","str":5} +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",4,"str"] "s" +["arrOfObjs",4] {"var":"val1","str":"s"} +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",5,"str"] "\"" +["arrOfObjs",5] {"var":"val1","str":"\""} +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",6,"str"] "S" +["arrOfObjs",6] {"var":"val1","str":"S"} +["arrOfObjs",7,"var"] "val1" +["arrOfObjs",7,"str"] "5" +["arrOfObjs",7] {"var":"val1","str":"5"} +["arrOfObjs"] [{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}] +["aNumber"] 1 +[] {"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1} +``` + +* Normalized output can also be sorted, upon request - although *NOTE* that if +your document schema has arrays whose order of items has syntactic meaning for +your application (aka "tuples"), such ordering will likely make the document +invalid for your application's use-case - and in such case you might want to +use `-No{='args'}` to only sort objects; this warning *should* be irrelevant +for objects (the `{"key":value}` pairs) though: +```bash +:; echo -E "$LINE" | ./JSON.sh -N='-n' +{"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"} + +:; echo -E "$LINE" | ./JSON.sh -N=-r +{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","emptyobj":{},"emptyarr":[],"array":[3,20,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1} + +:; echo -E "$LINE" | ./JSON.sh -N="-r -n" +{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1} + +### Normalize sorting only objects (arrays/tuples remain in original order): +:; echo -E "$LINE" | ./JSON.sh -No='-r -n' +{"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","emptystr":"","emptyobj":{},"emptyarr":[],"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}],"array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1} + +:; echo -E "$LINE" | ./JSON.sh -No='-n' +{"aNumber":1,"array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"arrOfObjs":[{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"z","var":"val2"},{"str":"x","var":"val2"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":5,"var":"val1"},{"str":"5","var":"val1"}],"emptyarr":[],"emptyobj":{},"emptystr":"","split\nkey":"value","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"} +``` + +* And note that the normalized output returns (maybe sorted) JSON markup of +the top-level item without whitespaces between syntactic elements, and other +`JSON.sh` modifiers are essentially ignored (`-x` option is detailed below): +```bash +:; echo -E "$LINE" | ./JSON.sh -x 'empty' -N +{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]} + +### Normalization mode can still be used for validation of input markup though: +:; echo -E "$LINE" | ./JSON.sh --no-newline -N +Invalid JSON markup detected: newline in a string value: at line #1 +EXPECTED value GOT EOF +``` + +* As developers of the script itself, we can debug why something is or is not +printed and thanks to which logical block (interesting excerpts copypasted); +several keys have been defined to pring different debug values if the debug +level is big enough, and can be easily redefined by `export` from the caller +(see `JSON.sh` source): +```bash +:; echo -E "$LINE" | ./JSON.sh -p -l -d +# Leaf value printed +=== KEY='"var1"' VALUE='"val1"' B='1' isleaf='1'/L='1' isempty='0'/P='1': print='4' +["var1"] "val1" +# Empty value pruned from line-by-line output (not from JSON markup, not from index numbering): +=== KEY='"array",6' VALUE='""' B='1' isleaf='1'/L='1' isempty='1'/P='1': print='0' +=== KEY='"array",7' VALUE='""' B='1' isleaf='1'/L='1' isempty='1'/P='1': print='0' +=== KEY='"var38"' VALUE='""' B='1' isleaf='1'/L='1' isempty='1'/P='1': print='0' +# Empty arrays and objects are NOW also pruned on request (this is different from brief mode which just does not output objects/arrays at all): +=== KEY='"emptyarr"' VALUE='[]' B='0' isleaf='0'/L='1' isempty='1'/P='1': print='0' +=== KEY='"emptyobj"' VALUE='{}' B='0' isleaf='0'/L='1' isempty='1'/P='1': print='0' +# Non-leaf items skipped from line-by-line printing: +=== KEY='"arrOfObjs",7' VALUE='{"var":"val1","str":"5"}' B='0' isleaf='0'/L='1' isempty='0'/P='1': print='0' +=== KEY='"arrOfObjs"' VALUE='[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]' B='0' isleaf='0'/L='1' isempty='0'/P='1': print='0' +=== KEY='' VALUE='...' B='0' isleaf='0'/L='1' isempty='0'/P='1': print='0' +``` + +* Last but not least, we now have an "extractor" to simplify scripted requests +to particular entries by their jpaths, which helps scripted interaction with +the JSON markup: +```bash +:; echo -E "$LINE" | ./JSON.sh -x 'empty' +["emptyarr"] [] +["emptyobj"] {} + +:; echo -E "$LINE" | ./JSON.sh -x 'var' +["var1"] "val1" +["var0"] "escaped \" quote" +["var8"] "string\nwith\nproper\\\nnewlines" +["var38"] "" +["arrOfObjs",0,"var"] "val1" +["arrOfObjs",1,"var"] "val30" +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",3,"var"] "val2" +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",7,"var"] "val1" + +# Regex can be used: +:; echo -E "$LINE" | ./JSON.sh -x '^\"var' +["var1"] "val1" +["var0"] "escaped \" quote" +["var8"] "string\nwith\nproper\\\nnewlines" +["var38"] "" + +:; echo -E "$LINE" | ./JSON.sh -x 'var\"$' +["arrOfObjs",0,"var"] "val1" +["arrOfObjs",1,"var"] "val30" +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",3,"var"] "val2" +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",7,"var"] "val1" + +# You can also pick array elements... +:; echo -E "$LINE" | ./JSON.sh -x 'arrOfObjs\",[0-9]*$' +["arrOfObjs",0] {"var":"val1","str":"s"} +["arrOfObjs",1] {"var":"val30","str":"s"} +["arrOfObjs",2] {"var":"val2","str":"z"} +["arrOfObjs",3] {"var":"val2","str":"x"} +["arrOfObjs",4] {"var":"val1","str":"S"} +["arrOfObjs",5] {"var":"val1","str":"\""} +["arrOfObjs",6] {"var":"val1","str":5} +["arrOfObjs",7] {"var":"val1","str":"5"} + +#... unless of course you use leaf-only mode: +:; echo -E "$LINE" | ./JSON.sh -x 'arrOfObjs\",[0-9]*$' -l + +#...or you can pick just the contents of the arrays: +:; echo -E "$LINE" | ./JSON.sh -x 'arrOfObjs\",[0-9]+,.+$' +["arrOfObjs",0,"var"] "val1" +["arrOfObjs",0,"str"] "s" +["arrOfObjs",1,"var"] "val30" +["arrOfObjs",1,"str"] "s" +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",2,"str"] "z" +["arrOfObjs",3,"var"] "val2" +["arrOfObjs",3,"str"] "x" +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",4,"str"] "S" +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",5,"str"] "\"" +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",6,"str"] 5 +["arrOfObjs",7,"var"] "val1" +["arrOfObjs",7,"str"] "5" + +#...or perhaps just the items in these arrays starting with an "s": +:; echo -E "$LINE" | ./JSON.sh -x 'arrOfObjs\",[0-9]+,\"s.+$' +["arrOfObjs",0,"str"] "s" +["arrOfObjs",1,"str"] "s" +["arrOfObjs",2,"str"] "z" +["arrOfObjs",3,"str"] "x" +["arrOfObjs",4,"str"] "S" +["arrOfObjs",5,"str"] "\"" +["arrOfObjs",6,"str"] 5 +["arrOfObjs",7,"str"] "5" + +# Note that only jpaths (not contents themselves) are matched: +:; echo -E "$LINE" | ./JSON.sh -x '\n' -l +["split\nkey"] "value" +``` + +* Another new feature to help scripting is "cooking" of input strings into +escaped JSON that should be valid markup (with no trailing newline as well); +this currently allows to escape newlines, backslashes and TAB characters which +otherwise made the `JSON.sh` parser sad: +```bash +:; RAWLINE='[ This is text +It has +Several "lines" +maybe \n escaped \" +}' + +:; ESCAPED="`echo -E "$RAWLINE" | ./JSON.sh -Q`"; echo -E "'$ESCAPED'" +'[ This is text\nIt has\nSeveral \"lines\"\nmaybe \\n escaped \\\"\n}' +``` + +For a more practical example, let's turn some text-file dumps into +JSON markup with escaped newlines: + +```bash +:; ( echo '['; for F in /etc/motd /etc/release ; do \ + printf '{"filename":"'"$F"'","contents":"%s"},\n' \ + "`cat "$F"`"; done; echo '{}]' ) | ./JSON.sh -N +[{"filename":"/etc/motd","contents":"The Illumos Project SunOS 5.11 illumos-ad69a33 January 2015"},{"filename":"/etc/release","contents":" OpenIndiana Development oi_151.1.8 X86 (powered by illumos)\n Copyright 2011 Oracle and/or its affiliates. All rights reserved.\n Use is subject to license terms.\n Assembled 19 February 2013"},{}] +``` + +Escaping for TAB characters in string contents during "cooking", as well as +toleration during processing, can be seen in `/etc/motd` of this example both +above and below: +``` +:; cat /etc/motd | ./JSON.sh -Q ; echo "" +The Illumos Project\tSunOS 5.11\tillumos-ad69a33\tJanuary 2015 +``` + ## Cool Links * [step-/JSON.awk](https://github.com/step-/JSON.awk) JSON.sh ported to awk diff --git a/all-tests.sh b/all-tests.sh index 48b40f6..141645f 100755 --- a/all-tests.sh +++ b/all-tests.sh @@ -1,34 +1,101 @@ #!/bin/sh -cd "${0%/*}" +# This script can now test with various shell interpreters +# which you can pass in a space-separated list of SHELL_PROGS +# To use old behavior : export SHELL_PROGS="-" +# This script runs one or more actual sub-tests named in TEST_PATTERN +# (the value may be a shell wildcard). +cd "$(dirname "$0")" #set -e -fail=0 -tests=0 -passed=0 -#all_tests=${__dirname:} -#echo PLAN ${#all_tests} -for test in test/*.sh ; -do - tests=$((tests+1)) - echo TEST: $test - ./$test - ret=$? - if [ $ret -eq 0 ] ; then - echo OK: ---- $test - passed=$((passed+1)) - else - echo FAIL: $test $fail - fail=$((fail+ret)) - fi + +overall_exitcode=0 +jsonsh_tests() ( + [ -z "${SHELL_PROG-}" ] && SHELL_PROG="" + fail=0 + tests=0 + passed=0 + fail_names="" + #all_tests=${__dirname:} + #echo PLAN ${#all_tests} + for test in $TEST_PATTERN + do + tests="$(expr $tests + 1)" + echo "TEST: $test" + # TODO: find a way to use the current shell-interpreter program to + # run sub-tests (simple sourcing fails ATM because scripts start + # with "cd `dirname $0`")... + #( . "./$test" ) + $SHELL_PROG "./$test" + ret=$? + if [ $ret -eq 0 ] ; then + echo "OK: ---- $test" + passed="$(expr $passed + 1)" + else + echo "FAIL: $test ($ret)" + fail="$(expr $fail + 1)" + fail_names="$fail_names $test" + fi + done + + if [ "$fail" = 0 ]; then + printf '===== SUCCESS ' + exitcode=0 + else + printf '===== FAILURE ' + exitcode=1 + fi + + # Note the leading space if populated + [ -z "$fail_names" ] || fail_names=" : failed for$fail_names" + + # Note the leading space if populated + [ -z "$SHELL_PROG" ] || fail_names="$fail_names interpreted by '$SHELL_PROG'" + + printf ": passed $passed / $tests tests$fail_names\n" + exit $exitcode +) + +OKAY_SHELLS="" +FAIL_SHELLS="" +SKIP_SHELLS="" +[ -n "$SHELL_PROGS" ] || SHELL_PROGS="bash dash ash zsh busybox" +#SHELL_PROGS="$SHELL_PROGS ksh ksh88 ksh93" +[ -n "$TEST_PATTERN" ] || TEST_PATTERN='test/*.sh' +export TEST_PATTERN + +# Force zsh to expand $A into multiple words +is_wordsplit_disabled="$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')" +if [ "$is_wordsplit_disabled" != 0 ]; then setopt shwordsplit; fi + +for SHELL_PROG in $SHELL_PROGS ; do + [ "$SHELL_PROG" = "busybox" ] && SHELL_PROG="busybox sh" + { [ "$SHELL_PROG" = "-" ] || [ "$SHELL_PROG" = " " ] ; } && \ + SHELL_PROG='' + + if [ -n "$SHELL_PROG" ] ; then + if $SHELL_PROG -c "date" >/dev/null 2>&1 ; then : ; else + echo "=== SKIP missing shell : $SHELL_PROG" + echo "" + SKIP_SHELLS="$SKIP_SHELLS $SHELL_PROG" + continue + fi + export SHELL_PROG + echo "=== TESTING WITH shell interpreter : $SHELL_PROG" + else + unset SHELL_PROG + echo "=== TESTING WITH default shell interpreter e.g. likely with /bin/sh, whatever this is in your OS" + fi + + jsonsh_tests && OKAY_SHELLS="$OKAY_SHELLS $SHELL_PROG" || \ + { overall_exitcode=$? ; FAIL_SHELLS="$FAIL_SHELLS $SHELL_PROG" ; } + echo "" done -if [ $fail -eq 0 ]; then - /bin/echo -n 'SUCCESS ' - exitcode=0 -else - /bin/echo -n 'FAILURE ' - exitcode=1 -fi -echo $passed / $tests -exit $exitcode +if [ "${is_wordsplit_disabled-}" != 0 ]; then unsetopt shwordsplit; is_wordsplit_disabled=0; fi + +echo "OVERALL RESULT:" +echo "OKAY_SHELLS = $OKAY_SHELLS" +echo "FAIL_SHELLS = $FAIL_SHELLS" +echo "SKIP_SHELLS = $SKIP_SHELLS" +exit $overall_exitcode diff --git a/test/cook-test.sh b/test/cook-test.sh new file mode 100755 index 0000000..55532e7 --- /dev/null +++ b/test/cook-test.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +cd "$(dirname "$0")" + +# Can't detect sourcing in sh, so immediately terminate the attempt to parse +JSONSH_SOURCED=yes +. ../JSON.sh '$OUT'" + else + echo "not ok $i - '$INPUT' => '$OUT' (expected '$EXPECT')" + fails="$(expr $fails+1)" + fi +} + +fails=0 +i=0 + +echo "1..4" +cooktest 'a@b' 'a@b' +cooktest 'a"b' 'a\"b' +cooktest 'a\"b' 'a\\\"b' +cooktest 'a b' 'a b' + +echo "$fails test(s) failed" +exit $fails diff --git a/test/invalid-test.sh b/test/invalid-test.sh index f776326..84b434e 100755 --- a/test/invalid-test.sh +++ b/test/invalid-test.sh @@ -1,29 +1,45 @@ #!/bin/sh -cd ${0%/*} +cd "$(dirname "$0")" + +# Can't detect sourcing in sh, so immediately terminate the attempt to parse +JSONSH_SOURCED=yes +. ../JSON.sh /tmp/JSON.sh_outlog 2> /tmp/JSON.sh_errlog + i="$(expr $i + 1)" + if jsonsh_cli < "$input" > "${tmp}"JSON.sh_outlog 2> "${tmp}"JSON.sh_errlog then - echo "not ok $i - cat $input | ../JSON.sh should fail" + echo "not ok $i - cat $input | ../JSON.sh should have failed" #this should be indented with '#' at the start. echo "OUTPUT WAS >>>" - cat /tmp/JSON.sh_outlog + cat "${tmp}"JSON.sh_outlog + echo "ERRORS WAS >>>" + cat "${tmp}"JSON.sh_errlog echo "<<<" - fails=$((fails+1)) + fails="$(expr $fails + 1)" else - echo "ok $i - $input was rejected" - echo "#" `cat /tmp/JSON.sh_errlog` + echo "ok $i - $input was rejected as expected" + echo "# `cat "${tmp}"JSON.sh_errlog`" fi done + echo "$fails test(s) failed" exit $fails diff --git a/test/invalid/bad_numbers.json b/test/invalid/bad_numbers.json new file mode 100644 index 0000000..894b15e --- /dev/null +++ b/test/invalid/bad_numbers.json @@ -0,0 +1,3 @@ +[++2,--1,-+3,+-4,-f,+x, +1e++4] + diff --git a/test/no-head-test.sh b/test/no-head-test.sh index a297b92..5231a97 100755 --- a/test/no-head-test.sh +++ b/test/no-head-test.sh @@ -1,27 +1,49 @@ #!/bin/sh -cd ${0%/*} -tmp=${TEMP:-/tmp} -tmp=${tmp%%/}/ # Avoid duplicate // +cd "$(dirname "$0")" + +# Can't detect sourcing in sh, so immediately terminate the attempt to parse +JSONSH_SOURCED=yes +. ../JSON.sh $expected - i=$((i+1)) - if ! ../JSON.sh -n < "$input" | diff -u - "$expected" + expected="${tmp}$(basename "$input" .json).no-head" + # NOTE: The echo trick is required to ensure EOLs for both empty and populated results + printf '%s\n' "$(egrep -v '^\[]' < "$(dirname "$input")/$(basename "$input" .json).parsed")" > "$expected" + i="$(expr $i + 1)" + # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters + JSONSH_OUT="$(jsonsh_cli -n < "$input")" && \ + printf '%s\n' "$JSONSH_OUT" | diff -u - "$expected" + JSONSH_RES=$? + if [ "$JSONSH_RES" != 0 ] then echo "not ok $i - $input" - fails=$((fails+1)) + fails="$(expr $fails + 1)" + printf ">>> JSONSH_OUT='%s'\n" "$JSONSH_OUT" + echo ">>> EXPECTED : `ls -la $expected`" + cat "$expected" else - echo "ok $i - $input" + echo "ok $i - $input" fi done + echo "$fails test(s) failed" exit $fails diff --git a/test/parse-test.sh b/test/parse-test.sh index fb2cfdf..6b88909 100755 --- a/test/parse-test.sh +++ b/test/parse-test.sh @@ -1,8 +1,9 @@ #!/bin/sh -cd ${0%/*} +cd "$(dirname "$0")" # Can't detect sourcing in sh, so immediately terminate the attempt to parse +JSONSH_SOURCED=yes . ../JSON.sh /tmp/json_ttest_expected - if echo "$input" | tokenize | diff -u - /tmp/json_ttest_expected + i="$(expr $i + 1)" + input="$1"; shift + expected="$(printf '%s\n' "$@")" + echo "$expected" > "${tmp}"json_ttest_expected + + # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters + JSONSH_OUT="$(echo "$input" | tokenize)" && \ + printf '%s\n' "$JSONSH_OUT" | diff -u - "${tmp}"json_ttest_expected + JSONSH_RES=$? + if [ "$JSONSH_RES" = 0 ] then - echo "ok $i - $input" - else + echo "ok $i - $input" + else echo "not ok $i - $input" - fails=$((fails+1)) + fails="$(expr $fails + 1)" fi } @@ -46,10 +61,14 @@ ttest '[ null , -110e10, "null" ]' \ ttest '{"e": false}' '{' '"e"' ':' 'false' '}' ttest '{"e": "string"}' '{' '"e"' ':' '"string"' '}' -if ! cat ../package.json | tokenize >/dev/null +i="$(expr $i + 1)" +input="Tokenizing the 'package.json' file" +if tokenize < ../package.json >/dev/null then - fails=$((fails+1)) - echo "Tokenizing package.json failed!" + echo "ok $i - $input" +else + echo "not ok $i - $input" + fails="$(expr $fails + 1)" fi echo "$fails test(s) failed" diff --git a/test/valid-test.sh b/test/valid-test.sh index 88e8f66..a32507f 100755 --- a/test/valid-test.sh +++ b/test/valid-test.sh @@ -1,21 +1,111 @@ #!/bin/sh -cd ${0%/*} +# NOTE: Developer of a new feature can pre-create expected result files: +# JSON_TEST_GENERATE=auto conditionally (don't replace nonempty files) +# JSON_TEST_GENERATE=yes recreate (replace existing results if any) + +# To disambiguate tests on sorting, use one locale +LANG=C +LC_ALL=C +export LANG LC_ALL + +cd "$(dirname "$0")" + +# Can't detect sourcing in sh, so immediately terminate the attempt to parse +JSONSH_SOURCED=yes +. ../JSON.sh &2 && exit 1 + +tests="$(echo "$FILES" | wc -l)" +### We currently have up to 8 extensions to consider per test +tests="$(expr $tests \* 8)" +echo "1..$tests" + +set -x + +# Force zsh to expand $A into multiple words +is_wordsplit_disabled="$(unsetopt 2>/dev/null | grep -c '^shwordsplit$')" +if [ "$is_wordsplit_disabled" != 0 ]; then setopt shwordsplit; fi + +for input in $FILES do - expected="${input%.json}.parsed" - i=$((i+1)) - if ! ../JSON.sh < "$input" | diff -u - "$expected" - then - echo "not ok $i - $input" - fails=$((fails+1)) - else - echo "ok $i - $input" - fi + if [ "${is_wordsplit_disabled-}" != 0 ]; then unsetopt shwordsplit; is_wordsplit_disabled=0; fi + for EXT in parsed sorted normalized normalized_sorted \ + numnormalized numnormalized_stripped \ + normalized_numnormalized normalized_numnormalized_stripped \ + ; do + if [ ! -f "$input" ]; then + echo "error - missing input file '$input', assuming all its tests failed" + fails="$(expr $fails + 8)" + break + fi + +# expected="${input%.json}.$EXT" + expected="$(dirname "$input")/$(basename "$input" .json).$EXT" + if [ -f "$expected" -o -n "$JSON_TEST_GENERATE" ]; then + i="$(expr $i + 1)" + case "$EXT" in + sorted) OPTIONS="-S='-n -r'" ;; + normalized) OPTIONS="-N" ;; + normalized_sorted) OPTIONS="-N=-n" ;; + numnormalized) OPTIONS="-Nn=%.12f" ;; + numnormalized_stripped) OPTIONS="-Nnx" ;; + normalized_numnormalized) OPTIONS="-N=-n -Nn=%.12f" ;; + normalized_numnormalized_stripped) OPTIONS="-N=-n -Nnx" ;; + parsed|*) OPTIONS="" ;; + esac + if [ "$JSON_TEST_GENERATE" = yes ] || \ + [ "$JSON_TEST_GENERATE" = auto -a ! -s "$expected" ] + then + if ! eval jsonsh_cli $OPTIONS < "$input" > "$expected" + then + echo "generation not ok $i - $input $EXT" + fails="$(expr $fails + 1)" + mv -f "$expected" "$expected.failed" + else + echo "generation ok $i - $input $EXT" + passes="$(expr $passes + 1)" + generated="$(expr $generated + 1)" + fi + continue + fi + + # Such explicit chaining is equivalent to "pipefail" in non-Bash interpreters + JSONSH_OUT="$(eval jsonsh_cli $OPTIONS < "$input")" && \ + printf '%s\n' "$JSONSH_OUT" | diff -u - "${expected}" + JSONSH_RES=$? + if [ "$JSONSH_RES" != 0 ] + then + echo "not ok $i - $input $EXT" + fails="$(expr $fails + 1)" + printf ">>> JSONSH_OUT='%s'\n" "$JSONSH_OUT" + echo ">>> EXPECTED : `ls -la $expected`" + cat "$expected" + else + echo "ok $i - $input $EXT" + passes="$(expr $passes + 1)" + fi + else + # echo "skip (missing result file) - $input $EXT" + skips="$(expr $skips + 1)" + fi + done done + +[ -n "$JSON_TEST_GENERATE" ] && echo "$generated expected results generated" +[ -n "$skips" ] && echo "$skips test(s) skipped (missing expected results file)" +echo "$passes test(s) succeeded" echo "$fails test(s) failed" exit $fails diff --git a/test/valid/array.normalized b/test/valid/array.normalized new file mode 100644 index 0000000..ee44631 --- /dev/null +++ b/test/valid/array.normalized @@ -0,0 +1 @@ +[1,2,3,"hello"] diff --git a/test/valid/array.normalized_numnormalized b/test/valid/array.normalized_numnormalized new file mode 100644 index 0000000..c70d6ea --- /dev/null +++ b/test/valid/array.normalized_numnormalized @@ -0,0 +1 @@ +["hello",1.000000000000,2.000000000000,3.000000000000] diff --git a/test/valid/array.normalized_numnormalized_stripped b/test/valid/array.normalized_numnormalized_stripped new file mode 100644 index 0000000..b5a9be5 --- /dev/null +++ b/test/valid/array.normalized_numnormalized_stripped @@ -0,0 +1 @@ +["hello",1,2,3] diff --git a/test/valid/array.normalized_sorted b/test/valid/array.normalized_sorted new file mode 100644 index 0000000..b5a9be5 --- /dev/null +++ b/test/valid/array.normalized_sorted @@ -0,0 +1 @@ +["hello",1,2,3] diff --git a/test/valid/array.numnormalized b/test/valid/array.numnormalized new file mode 100644 index 0000000..02487eb --- /dev/null +++ b/test/valid/array.numnormalized @@ -0,0 +1,5 @@ +[0] 1.000000000000 +[1] 2.000000000000 +[2] 3.000000000000 +[3] "hello" +[] [1.000000000000,2.000000000000,3.000000000000,"hello"] diff --git a/test/valid/array.numnormalized_stripped b/test/valid/array.numnormalized_stripped new file mode 100644 index 0000000..d564cd9 --- /dev/null +++ b/test/valid/array.numnormalized_stripped @@ -0,0 +1,5 @@ +[0] 1 +[1] 2 +[2] 3 +[3] "hello" +[] [1,2,3,"hello"] diff --git a/test/valid/array.sorted b/test/valid/array.sorted new file mode 100644 index 0000000..0bcd687 --- /dev/null +++ b/test/valid/array.sorted @@ -0,0 +1,5 @@ +[0] 3 +[1] 2 +[2] 1 +[3] "hello" +[] [3,2,1,"hello"] diff --git a/test/valid/array_with_empty_elements.json b/test/valid/array_with_empty_elements.json new file mode 100644 index 0000000..3bde6fa --- /dev/null +++ b/test/valid/array_with_empty_elements.json @@ -0,0 +1,6 @@ +["", {"k1":"v1", + "k2":"","k3":1e15, +"k0":0,"arr":[3,1,2,4]}, + "", + "av1", +"v1",0] diff --git a/test/valid/array_with_empty_elements.normalized b/test/valid/array_with_empty_elements.normalized new file mode 100644 index 0000000..76bdb17 --- /dev/null +++ b/test/valid/array_with_empty_elements.normalized @@ -0,0 +1 @@ +["",{"k1":"v1","k2":"","k3":1e15,"k0":0,"arr":[3,1,2,4]},"","av1","v1",0] diff --git a/test/valid/array_with_empty_elements.normalized_numnormalized b/test/valid/array_with_empty_elements.normalized_numnormalized new file mode 100644 index 0000000..b86bcb5 --- /dev/null +++ b/test/valid/array_with_empty_elements.normalized_numnormalized @@ -0,0 +1 @@ +["","","av1","v1",0.000000000000,{"arr":[1.000000000000,2.000000000000,3.000000000000,4.000000000000],"k0":0.000000000000,"k1":"v1","k2":"","k3":1000000000000000.000000000000}] diff --git a/test/valid/array_with_empty_elements.normalized_numnormalized_stripped b/test/valid/array_with_empty_elements.normalized_numnormalized_stripped new file mode 100644 index 0000000..9894eb5 --- /dev/null +++ b/test/valid/array_with_empty_elements.normalized_numnormalized_stripped @@ -0,0 +1 @@ +["","","av1","v1",0,{"arr":[1,2,3,4],"k0":0,"k1":"v1","k2":"","k3":1000000000000000}] diff --git a/test/valid/array_with_empty_elements.normalized_sorted b/test/valid/array_with_empty_elements.normalized_sorted new file mode 100644 index 0000000..4d70dca --- /dev/null +++ b/test/valid/array_with_empty_elements.normalized_sorted @@ -0,0 +1 @@ +["","","av1","v1",0,{"arr":[1,2,3,4],"k0":0,"k1":"v1","k2":"","k3":1e15}] diff --git a/test/valid/array_with_empty_elements.numnormalized b/test/valid/array_with_empty_elements.numnormalized new file mode 100644 index 0000000..5a37807 --- /dev/null +++ b/test/valid/array_with_empty_elements.numnormalized @@ -0,0 +1,16 @@ +[0] "" +[1,"k1"] "v1" +[1,"k2"] "" +[1,"k3"] 1000000000000000.000000000000 +[1,"k0"] 0.000000000000 +[1,"arr",0] 3.000000000000 +[1,"arr",1] 1.000000000000 +[1,"arr",2] 2.000000000000 +[1,"arr",3] 4.000000000000 +[1,"arr"] [3.000000000000,1.000000000000,2.000000000000,4.000000000000] +[1] {"k1":"v1","k2":"","k3":1000000000000000.000000000000,"k0":0.000000000000,"arr":[3.000000000000,1.000000000000,2.000000000000,4.000000000000]} +[2] "" +[3] "av1" +[4] "v1" +[5] 0.000000000000 +[] ["",{"k1":"v1","k2":"","k3":1000000000000000.000000000000,"k0":0.000000000000,"arr":[3.000000000000,1.000000000000,2.000000000000,4.000000000000]},"","av1","v1",0.000000000000] diff --git a/test/valid/array_with_empty_elements.numnormalized_stripped b/test/valid/array_with_empty_elements.numnormalized_stripped new file mode 100644 index 0000000..1405a83 --- /dev/null +++ b/test/valid/array_with_empty_elements.numnormalized_stripped @@ -0,0 +1,16 @@ +[0] "" +[1,"k1"] "v1" +[1,"k2"] "" +[1,"k3"] 1000000000000000 +[1,"k0"] 0 +[1,"arr",0] 3 +[1,"arr",1] 1 +[1,"arr",2] 2 +[1,"arr",3] 4 +[1,"arr"] [3,1,2,4] +[1] {"k1":"v1","k2":"","k3":1000000000000000,"k0":0,"arr":[3,1,2,4]} +[2] "" +[3] "av1" +[4] "v1" +[5] 0 +[] ["",{"k1":"v1","k2":"","k3":1000000000000000,"k0":0,"arr":[3,1,2,4]},"","av1","v1",0] diff --git a/test/valid/array_with_empty_elements.parsed b/test/valid/array_with_empty_elements.parsed new file mode 100644 index 0000000..3ec2517 --- /dev/null +++ b/test/valid/array_with_empty_elements.parsed @@ -0,0 +1,16 @@ +[0] "" +[1,"k1"] "v1" +[1,"k2"] "" +[1,"k3"] 1e15 +[1,"k0"] 0 +[1,"arr",0] 3 +[1,"arr",1] 1 +[1,"arr",2] 2 +[1,"arr",3] 4 +[1,"arr"] [3,1,2,4] +[1] {"k1":"v1","k2":"","k3":1e15,"k0":0,"arr":[3,1,2,4]} +[2] "" +[3] "av1" +[4] "v1" +[5] 0 +[] ["",{"k1":"v1","k2":"","k3":1e15,"k0":0,"arr":[3,1,2,4]},"","av1","v1",0] diff --git a/test/valid/array_with_empty_elements.sorted b/test/valid/array_with_empty_elements.sorted new file mode 100644 index 0000000..10c308c --- /dev/null +++ b/test/valid/array_with_empty_elements.sorted @@ -0,0 +1,16 @@ +[0,"k3"] 1e15 +[0,"k2"] "" +[0,"k1"] "v1" +[0,"k0"] 0 +[0,"arr",0] 4 +[0,"arr",1] 3 +[0,"arr",2] 2 +[0,"arr",3] 1 +[0,"arr"] [4,3,2,1] +[0] {"k3":1e15,"k2":"","k1":"v1","k0":0,"arr":[4,3,2,1]} +[1] 0 +[2] "v1" +[3] "av1" +[4] "" +[5] "" +[] [{"k3":1e15,"k2":"","k1":"v1","k0":0,"arr":[4,3,2,1]},0,"v1","av1","",""] diff --git a/test/valid/documented_example.json b/test/valid/documented_example.json new file mode 100644 index 0000000..11b8cbf --- /dev/null +++ b/test/valid/documented_example.json @@ -0,0 +1,13 @@ +{"var1":"val1","split +key":"value","var0":"escaped \" quote","splitValue":"there + are a newline and three spaces (one after \"there\" and two before \"are\")", +"array":["z","a","b",3,20,0,"","","\"" +,"escaping\"several\"\" +quote\"s and +newlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines", +"var38":"","emptyarr":[],"emptyobj":{},"emptystr":"", +"arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"}, + {"var":"val2","str":"z"},{"var":"val2","str":"x"}, + {"var":"val1","str":"S"},{"var":"val1","str":"\""}, +{"var":"val1","str":5},{"var":"val1","str":"5"}]} + diff --git a/test/valid/documented_example.normalized b/test/valid/documented_example.normalized new file mode 100644 index 0000000..7f5de8f --- /dev/null +++ b/test/valid/documented_example.normalized @@ -0,0 +1 @@ +{"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"emptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]} diff --git a/test/valid/documented_example.normalized_numnormalized b/test/valid/documented_example.normalized_numnormalized new file mode 100644 index 0000000..a927426 --- /dev/null +++ b/test/valid/documented_example.normalized_numnormalized @@ -0,0 +1 @@ +{"aNumber":1.000000000000,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5.000000000000,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0.000000000000,3.000000000000,20.000000000000],"emptyarr":[],"emptyobj":{},"emptystr":"","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"} diff --git a/test/valid/documented_example.normalized_numnormalized_stripped b/test/valid/documented_example.normalized_numnormalized_stripped new file mode 100644 index 0000000..6aed423 --- /dev/null +++ b/test/valid/documented_example.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"emptystr":"","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"} diff --git a/test/valid/documented_example.normalized_sorted b/test/valid/documented_example.normalized_sorted new file mode 100644 index 0000000..6aed423 --- /dev/null +++ b/test/valid/documented_example.normalized_sorted @@ -0,0 +1 @@ +{"aNumber":1,"arrOfObjs":[{"str":"5","var":"val1"},{"str":"S","var":"val1"},{"str":"\"","var":"val1"},{"str":"s","var":"val1"},{"str":"s","var":"val30"},{"str":"x","var":"val2"},{"str":"z","var":"val2"},{"str":5,"var":"val1"}],"array":["","","\"","a","b","escaping\"several\"\"\nquote\"s and\nnewlines","z",0,3,20],"emptyarr":[],"emptyobj":{},"emptystr":"","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","split\nkey":"value","var0":"escaped \" quote","var1":"val1","var38":"","var8":"string\nwith\nproper\\\nnewlines"} diff --git a/test/valid/documented_example.numnormalized b/test/valid/documented_example.numnormalized new file mode 100644 index 0000000..d7b020b --- /dev/null +++ b/test/valid/documented_example.numnormalized @@ -0,0 +1,47 @@ +["var1"] "val1" +["split\nkey"] "value" +["var0"] "escaped \" quote" +["splitValue"] "there\n are a newline and three spaces (one after \"there\" and two before \"are\")" +["array",0] "z" +["array",1] "a" +["array",2] "b" +["array",3] 3.000000000000 +["array",4] 20.000000000000 +["array",5] 0.000000000000 +["array",6] "" +["array",7] "" +["array",8] "\"" +["array",9] "escaping\"several\"\"\nquote\"s and\nnewlines" +["array"] ["z","a","b",3.000000000000,20.000000000000,0.000000000000,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"] +["aNumber"] 1.000000000000 +["var8"] "string\nwith\nproper\\\nnewlines" +["var38"] "" +["emptyarr"] [] +["emptyobj"] {} +["emptystr"] "" +["arrOfObjs",0,"var"] "val1" +["arrOfObjs",0,"str"] "s" +["arrOfObjs",0] {"var":"val1","str":"s"} +["arrOfObjs",1,"var"] "val30" +["arrOfObjs",1,"str"] "s" +["arrOfObjs",1] {"var":"val30","str":"s"} +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",2,"str"] "z" +["arrOfObjs",2] {"var":"val2","str":"z"} +["arrOfObjs",3,"var"] "val2" +["arrOfObjs",3,"str"] "x" +["arrOfObjs",3] {"var":"val2","str":"x"} +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",4,"str"] "S" +["arrOfObjs",4] {"var":"val1","str":"S"} +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",5,"str"] "\"" +["arrOfObjs",5] {"var":"val1","str":"\""} +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",6,"str"] 5.000000000000 +["arrOfObjs",6] {"var":"val1","str":5.000000000000} +["arrOfObjs",7,"var"] "val1" +["arrOfObjs",7,"str"] "5" +["arrOfObjs",7] {"var":"val1","str":"5"} +["arrOfObjs"] [{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5.000000000000},{"var":"val1","str":"5"}] +[] {"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3.000000000000,20.000000000000,0.000000000000,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1.000000000000,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"emptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5.000000000000},{"var":"val1","str":"5"}]} diff --git a/test/valid/documented_example.numnormalized_stripped b/test/valid/documented_example.numnormalized_stripped new file mode 100644 index 0000000..78ee6dd --- /dev/null +++ b/test/valid/documented_example.numnormalized_stripped @@ -0,0 +1,47 @@ +["var1"] "val1" +["split\nkey"] "value" +["var0"] "escaped \" quote" +["splitValue"] "there\n are a newline and three spaces (one after \"there\" and two before \"are\")" +["array",0] "z" +["array",1] "a" +["array",2] "b" +["array",3] 3 +["array",4] 20 +["array",5] 0 +["array",6] "" +["array",7] "" +["array",8] "\"" +["array",9] "escaping\"several\"\"\nquote\"s and\nnewlines" +["array"] ["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"] +["aNumber"] 1 +["var8"] "string\nwith\nproper\\\nnewlines" +["var38"] "" +["emptyarr"] [] +["emptyobj"] {} +["emptystr"] "" +["arrOfObjs",0,"var"] "val1" +["arrOfObjs",0,"str"] "s" +["arrOfObjs",0] {"var":"val1","str":"s"} +["arrOfObjs",1,"var"] "val30" +["arrOfObjs",1,"str"] "s" +["arrOfObjs",1] {"var":"val30","str":"s"} +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",2,"str"] "z" +["arrOfObjs",2] {"var":"val2","str":"z"} +["arrOfObjs",3,"var"] "val2" +["arrOfObjs",3,"str"] "x" +["arrOfObjs",3] {"var":"val2","str":"x"} +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",4,"str"] "S" +["arrOfObjs",4] {"var":"val1","str":"S"} +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",5,"str"] "\"" +["arrOfObjs",5] {"var":"val1","str":"\""} +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",6,"str"] 5 +["arrOfObjs",6] {"var":"val1","str":5} +["arrOfObjs",7,"var"] "val1" +["arrOfObjs",7,"str"] "5" +["arrOfObjs",7] {"var":"val1","str":"5"} +["arrOfObjs"] [{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}] +[] {"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"emptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]} diff --git a/test/valid/documented_example.parsed b/test/valid/documented_example.parsed new file mode 100644 index 0000000..78ee6dd --- /dev/null +++ b/test/valid/documented_example.parsed @@ -0,0 +1,47 @@ +["var1"] "val1" +["split\nkey"] "value" +["var0"] "escaped \" quote" +["splitValue"] "there\n are a newline and three spaces (one after \"there\" and two before \"are\")" +["array",0] "z" +["array",1] "a" +["array",2] "b" +["array",3] 3 +["array",4] 20 +["array",5] 0 +["array",6] "" +["array",7] "" +["array",8] "\"" +["array",9] "escaping\"several\"\"\nquote\"s and\nnewlines" +["array"] ["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"] +["aNumber"] 1 +["var8"] "string\nwith\nproper\\\nnewlines" +["var38"] "" +["emptyarr"] [] +["emptyobj"] {} +["emptystr"] "" +["arrOfObjs",0,"var"] "val1" +["arrOfObjs",0,"str"] "s" +["arrOfObjs",0] {"var":"val1","str":"s"} +["arrOfObjs",1,"var"] "val30" +["arrOfObjs",1,"str"] "s" +["arrOfObjs",1] {"var":"val30","str":"s"} +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",2,"str"] "z" +["arrOfObjs",2] {"var":"val2","str":"z"} +["arrOfObjs",3,"var"] "val2" +["arrOfObjs",3,"str"] "x" +["arrOfObjs",3] {"var":"val2","str":"x"} +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",4,"str"] "S" +["arrOfObjs",4] {"var":"val1","str":"S"} +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",5,"str"] "\"" +["arrOfObjs",5] {"var":"val1","str":"\""} +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",6,"str"] 5 +["arrOfObjs",6] {"var":"val1","str":5} +["arrOfObjs",7,"var"] "val1" +["arrOfObjs",7,"str"] "5" +["arrOfObjs",7] {"var":"val1","str":"5"} +["arrOfObjs"] [{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}] +[] {"var1":"val1","split\nkey":"value","var0":"escaped \" quote","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","array":["z","a","b",3,20,0,"","","\"","escaping\"several\"\"\nquote\"s and\nnewlines"],"aNumber":1,"var8":"string\nwith\nproper\\\nnewlines","var38":"","emptyarr":[],"emptyobj":{},"emptystr":"","arrOfObjs":[{"var":"val1","str":"s"},{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":"S"},{"var":"val1","str":"\""},{"var":"val1","str":5},{"var":"val1","str":"5"}]} diff --git a/test/valid/documented_example.sorted b/test/valid/documented_example.sorted new file mode 100644 index 0000000..fc2bb14 --- /dev/null +++ b/test/valid/documented_example.sorted @@ -0,0 +1,47 @@ +["var8"] "string\nwith\nproper\\\nnewlines" +["var38"] "" +["var1"] "val1" +["var0"] "escaped \" quote" +["split\nkey"] "value" +["splitValue"] "there\n are a newline and three spaces (one after \"there\" and two before \"are\")" +["emptystr"] "" +["emptyobj"] {} +["emptyarr"] [] +["array",0] 20 +["array",1] 3 +["array",2] 0 +["array",3] "z" +["array",4] "escaping\"several\"\"\nquote\"s and\nnewlines" +["array",5] "b" +["array",6] "a" +["array",7] "\"" +["array",8] "" +["array",9] "" +["array"] [20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""] +["arrOfObjs",0,"var"] "val30" +["arrOfObjs",0,"str"] "s" +["arrOfObjs",0] {"var":"val30","str":"s"} +["arrOfObjs",1,"var"] "val2" +["arrOfObjs",1,"str"] "z" +["arrOfObjs",1] {"var":"val2","str":"z"} +["arrOfObjs",2,"var"] "val2" +["arrOfObjs",2,"str"] "x" +["arrOfObjs",2] {"var":"val2","str":"x"} +["arrOfObjs",3,"var"] "val1" +["arrOfObjs",3,"str"] 5 +["arrOfObjs",3] {"var":"val1","str":5} +["arrOfObjs",4,"var"] "val1" +["arrOfObjs",4,"str"] "s" +["arrOfObjs",4] {"var":"val1","str":"s"} +["arrOfObjs",5,"var"] "val1" +["arrOfObjs",5,"str"] "\"" +["arrOfObjs",5] {"var":"val1","str":"\""} +["arrOfObjs",6,"var"] "val1" +["arrOfObjs",6,"str"] "S" +["arrOfObjs",6] {"var":"val1","str":"S"} +["arrOfObjs",7,"var"] "val1" +["arrOfObjs",7,"str"] "5" +["arrOfObjs",7] {"var":"val1","str":"5"} +["arrOfObjs"] [{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}] +["aNumber"] 1 +[] {"var8":"string\nwith\nproper\\\nnewlines","var38":"","var1":"val1","var0":"escaped \" quote","split\nkey":"value","splitValue":"there\n are a newline and three spaces (one after \"there\" and two before \"are\")","emptystr":"","emptyobj":{},"emptyarr":[],"array":[20,3,0,"z","escaping\"several\"\"\nquote\"s and\nnewlines","b","a","\"","",""],"arrOfObjs":[{"var":"val30","str":"s"},{"var":"val2","str":"z"},{"var":"val2","str":"x"},{"var":"val1","str":5},{"var":"val1","str":"s"},{"var":"val1","str":"\""},{"var":"val1","str":"S"},{"var":"val1","str":"5"}],"aNumber":1} diff --git a/test/valid/embedded.normalized b/test/valid/embedded.normalized new file mode 100644 index 0000000..f327913 --- /dev/null +++ b/test/valid/embedded.normalized @@ -0,0 +1 @@ +{"foo":"{\"foo\":\"bar\"}"} diff --git a/test/valid/embedded.normalized_numnormalized b/test/valid/embedded.normalized_numnormalized new file mode 100644 index 0000000..f327913 --- /dev/null +++ b/test/valid/embedded.normalized_numnormalized @@ -0,0 +1 @@ +{"foo":"{\"foo\":\"bar\"}"} diff --git a/test/valid/embedded.normalized_numnormalized_stripped b/test/valid/embedded.normalized_numnormalized_stripped new file mode 100644 index 0000000..f327913 --- /dev/null +++ b/test/valid/embedded.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"foo":"{\"foo\":\"bar\"}"} diff --git a/test/valid/embedded.normalized_sorted b/test/valid/embedded.normalized_sorted new file mode 100644 index 0000000..f327913 --- /dev/null +++ b/test/valid/embedded.normalized_sorted @@ -0,0 +1 @@ +{"foo":"{\"foo\":\"bar\"}"} diff --git a/test/valid/embedded.numnormalized b/test/valid/embedded.numnormalized new file mode 100644 index 0000000..041eaf9 --- /dev/null +++ b/test/valid/embedded.numnormalized @@ -0,0 +1,2 @@ +["foo"] "{\"foo\":\"bar\"}" +[] {"foo":"{\"foo\":\"bar\"}"} diff --git a/test/valid/embedded.numnormalized_stripped b/test/valid/embedded.numnormalized_stripped new file mode 100644 index 0000000..041eaf9 --- /dev/null +++ b/test/valid/embedded.numnormalized_stripped @@ -0,0 +1,2 @@ +["foo"] "{\"foo\":\"bar\"}" +[] {"foo":"{\"foo\":\"bar\"}"} diff --git a/test/valid/embedded.sorted b/test/valid/embedded.sorted new file mode 100644 index 0000000..041eaf9 --- /dev/null +++ b/test/valid/embedded.sorted @@ -0,0 +1,2 @@ +["foo"] "{\"foo\":\"bar\"}" +[] {"foo":"{\"foo\":\"bar\"}"} diff --git a/test/valid/empty_array.normalized b/test/valid/empty_array.normalized new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/test/valid/empty_array.normalized @@ -0,0 +1 @@ +[] diff --git a/test/valid/empty_array.normalized_numnormalized b/test/valid/empty_array.normalized_numnormalized new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/test/valid/empty_array.normalized_numnormalized @@ -0,0 +1 @@ +[] diff --git a/test/valid/empty_array.normalized_numnormalized_stripped b/test/valid/empty_array.normalized_numnormalized_stripped new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/test/valid/empty_array.normalized_numnormalized_stripped @@ -0,0 +1 @@ +[] diff --git a/test/valid/empty_array.normalized_sorted b/test/valid/empty_array.normalized_sorted new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/test/valid/empty_array.normalized_sorted @@ -0,0 +1 @@ +[] diff --git a/test/valid/empty_array.numnormalized b/test/valid/empty_array.numnormalized new file mode 100644 index 0000000..d24d150 --- /dev/null +++ b/test/valid/empty_array.numnormalized @@ -0,0 +1 @@ +[] [] diff --git a/test/valid/empty_array.numnormalized_stripped b/test/valid/empty_array.numnormalized_stripped new file mode 100644 index 0000000..d24d150 --- /dev/null +++ b/test/valid/empty_array.numnormalized_stripped @@ -0,0 +1 @@ +[] [] diff --git a/test/valid/empty_array.sorted b/test/valid/empty_array.sorted new file mode 100644 index 0000000..d24d150 --- /dev/null +++ b/test/valid/empty_array.sorted @@ -0,0 +1 @@ +[] [] diff --git a/test/invalid/empty.json b/test/valid/empty_document.json similarity index 100% rename from test/invalid/empty.json rename to test/valid/empty_document.json diff --git a/test/valid/empty_document.normalized b/test/valid/empty_document.normalized new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test/valid/empty_document.normalized @@ -0,0 +1 @@ +{} diff --git a/test/valid/empty_document.normalized_numnormalized b/test/valid/empty_document.normalized_numnormalized new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test/valid/empty_document.normalized_numnormalized @@ -0,0 +1 @@ +{} diff --git a/test/valid/empty_document.normalized_numnormalized_stripped b/test/valid/empty_document.normalized_numnormalized_stripped new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test/valid/empty_document.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{} diff --git a/test/valid/empty_document.normalized_sorted b/test/valid/empty_document.normalized_sorted new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test/valid/empty_document.normalized_sorted @@ -0,0 +1 @@ +{} diff --git a/test/valid/empty_document.numnormalized b/test/valid/empty_document.numnormalized new file mode 100644 index 0000000..4cdea2a --- /dev/null +++ b/test/valid/empty_document.numnormalized @@ -0,0 +1 @@ +[] {} diff --git a/test/valid/empty_document.numnormalized_stripped b/test/valid/empty_document.numnormalized_stripped new file mode 100644 index 0000000..4cdea2a --- /dev/null +++ b/test/valid/empty_document.numnormalized_stripped @@ -0,0 +1 @@ +[] {} diff --git a/test/valid/empty_document.parsed b/test/valid/empty_document.parsed new file mode 100644 index 0000000..4cdea2a --- /dev/null +++ b/test/valid/empty_document.parsed @@ -0,0 +1 @@ +[] {} diff --git a/test/valid/empty_document.sorted b/test/valid/empty_document.sorted new file mode 100644 index 0000000..4cdea2a --- /dev/null +++ b/test/valid/empty_document.sorted @@ -0,0 +1 @@ +[] {} diff --git a/test/valid/empty_object.normalized b/test/valid/empty_object.normalized new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test/valid/empty_object.normalized @@ -0,0 +1 @@ +{} diff --git a/test/valid/empty_object.normalized_numnormalized b/test/valid/empty_object.normalized_numnormalized new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test/valid/empty_object.normalized_numnormalized @@ -0,0 +1 @@ +{} diff --git a/test/valid/empty_object.normalized_numnormalized_stripped b/test/valid/empty_object.normalized_numnormalized_stripped new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test/valid/empty_object.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{} diff --git a/test/valid/empty_object.normalized_sorted b/test/valid/empty_object.normalized_sorted new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/test/valid/empty_object.normalized_sorted @@ -0,0 +1 @@ +{} diff --git a/test/valid/empty_object.numnormalized b/test/valid/empty_object.numnormalized new file mode 100644 index 0000000..4cdea2a --- /dev/null +++ b/test/valid/empty_object.numnormalized @@ -0,0 +1 @@ +[] {} diff --git a/test/valid/empty_object.numnormalized_stripped b/test/valid/empty_object.numnormalized_stripped new file mode 100644 index 0000000..4cdea2a --- /dev/null +++ b/test/valid/empty_object.numnormalized_stripped @@ -0,0 +1 @@ +[] {} diff --git a/test/valid/empty_object.sorted b/test/valid/empty_object.sorted new file mode 100644 index 0000000..4cdea2a --- /dev/null +++ b/test/valid/empty_object.sorted @@ -0,0 +1 @@ +[] {} diff --git a/test/valid/generate-results.sh b/test/valid/generate-results.sh new file mode 100755 index 0000000..5a6b781 --- /dev/null +++ b/test/valid/generate-results.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +# Generate the reference results in a standardized manner +# (script options, extensions, locale) +LANG=C +LC_ALL=C +export LANG +export C + +JSONSH=../../JSON.sh + +generate() { + F="$1" + [ -s "$F" ] || return + + B="$(basename "$F" .json)" + echo "=== Generating results for '$F'..." + RES=0 + + EXT=parsed + $JSONSH < "$F" > "$B.$EXT" || \ + { RES=$?; echo "ERROR with $EXT"; } + + EXT=sorted + $JSONSH -S="-n -r" < "$F" > "$B.$EXT" || \ + { RES=$?; echo "ERROR with $EXT"; } + + EXT=normalized + $JSONSH -N < "$F" > "$B.$EXT" || \ + { RES=$?; echo "ERROR with $EXT"; } + + EXT=normalized_sorted + $JSONSH -N='-n' < "$F" > "$B.$EXT" || \ + { RES=$?; echo "ERROR with $EXT"; } + + return $RES +} + +if [ $# -gt 0 ]; then + for F in "$@" ; do + generate "$F" + done +else + for F in *.json ; do + generate "$F" + done +fi diff --git a/test/valid/many_object.normalized b/test/valid/many_object.normalized new file mode 100644 index 0000000..abfdad2 --- /dev/null +++ b/test/valid/many_object.normalized @@ -0,0 +1 @@ +{"key1":"string","key2":3573} diff --git a/test/valid/many_object.normalized_numnormalized b/test/valid/many_object.normalized_numnormalized new file mode 100644 index 0000000..376ddac --- /dev/null +++ b/test/valid/many_object.normalized_numnormalized @@ -0,0 +1 @@ +{"key1":"string","key2":3573.000000000000} diff --git a/test/valid/many_object.normalized_numnormalized_stripped b/test/valid/many_object.normalized_numnormalized_stripped new file mode 100644 index 0000000..abfdad2 --- /dev/null +++ b/test/valid/many_object.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"key1":"string","key2":3573} diff --git a/test/valid/many_object.normalized_sorted b/test/valid/many_object.normalized_sorted new file mode 100644 index 0000000..abfdad2 --- /dev/null +++ b/test/valid/many_object.normalized_sorted @@ -0,0 +1 @@ +{"key1":"string","key2":3573} diff --git a/test/valid/many_object.numnormalized b/test/valid/many_object.numnormalized new file mode 100644 index 0000000..b04a65b --- /dev/null +++ b/test/valid/many_object.numnormalized @@ -0,0 +1,3 @@ +["key1"] "string" +["key2"] 3573.000000000000 +[] {"key1":"string","key2":3573.000000000000} diff --git a/test/valid/many_object.numnormalized_stripped b/test/valid/many_object.numnormalized_stripped new file mode 100644 index 0000000..fd1fccb --- /dev/null +++ b/test/valid/many_object.numnormalized_stripped @@ -0,0 +1,3 @@ +["key1"] "string" +["key2"] 3573 +[] {"key1":"string","key2":3573} diff --git a/test/valid/many_object.sorted b/test/valid/many_object.sorted new file mode 100644 index 0000000..7ea3be5 --- /dev/null +++ b/test/valid/many_object.sorted @@ -0,0 +1,3 @@ +["key2"] 3573 +["key1"] "string" +[] {"key2":3573,"key1":"string"} diff --git a/test/valid/multiline_escapedquotes.json b/test/valid/multiline_escapedquotes.json new file mode 100644 index 0000000..4315c38 --- /dev/null +++ b/test/valid/multiline_escapedquotes.json @@ -0,0 +1,5 @@ +{"s1 +s2 \" s3 ": "abs" , +"s4": "qwe", "d2": "qwer +t +yu","d2":123} diff --git a/test/valid/multiline_escapedquotes.normalized b/test/valid/multiline_escapedquotes.normalized new file mode 100644 index 0000000..6eea35d --- /dev/null +++ b/test/valid/multiline_escapedquotes.normalized @@ -0,0 +1 @@ +{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer\nt\nyu","d2":123} diff --git a/test/valid/multiline_escapedquotes.normalized_numnormalized b/test/valid/multiline_escapedquotes.normalized_numnormalized new file mode 100644 index 0000000..d049f7e --- /dev/null +++ b/test/valid/multiline_escapedquotes.normalized_numnormalized @@ -0,0 +1 @@ +{"d2":"qwer\nt\nyu","d2":123.000000000000,"s1\ns2 \" s3 ":"abs","s4":"qwe"} diff --git a/test/valid/multiline_escapedquotes.normalized_numnormalized_stripped b/test/valid/multiline_escapedquotes.normalized_numnormalized_stripped new file mode 100644 index 0000000..fe90225 --- /dev/null +++ b/test/valid/multiline_escapedquotes.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"d2":"qwer\nt\nyu","d2":123,"s1\ns2 \" s3 ":"abs","s4":"qwe"} diff --git a/test/valid/multiline_escapedquotes.normalized_sorted b/test/valid/multiline_escapedquotes.normalized_sorted new file mode 100644 index 0000000..fe90225 --- /dev/null +++ b/test/valid/multiline_escapedquotes.normalized_sorted @@ -0,0 +1 @@ +{"d2":"qwer\nt\nyu","d2":123,"s1\ns2 \" s3 ":"abs","s4":"qwe"} diff --git a/test/valid/multiline_escapedquotes.numnormalized b/test/valid/multiline_escapedquotes.numnormalized new file mode 100644 index 0000000..9b72344 --- /dev/null +++ b/test/valid/multiline_escapedquotes.numnormalized @@ -0,0 +1,5 @@ +["s1\ns2 \" s3 "] "abs" +["s4"] "qwe" +["d2"] "qwer\nt\nyu" +["d2"] 123.000000000000 +[] {"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer\nt\nyu","d2":123.000000000000} diff --git a/test/valid/multiline_escapedquotes.numnormalized_stripped b/test/valid/multiline_escapedquotes.numnormalized_stripped new file mode 100644 index 0000000..7a8aee6 --- /dev/null +++ b/test/valid/multiline_escapedquotes.numnormalized_stripped @@ -0,0 +1,5 @@ +["s1\ns2 \" s3 "] "abs" +["s4"] "qwe" +["d2"] "qwer\nt\nyu" +["d2"] 123 +[] {"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer\nt\nyu","d2":123} diff --git a/test/valid/multiline_escapedquotes.parsed b/test/valid/multiline_escapedquotes.parsed new file mode 100644 index 0000000..7a8aee6 --- /dev/null +++ b/test/valid/multiline_escapedquotes.parsed @@ -0,0 +1,5 @@ +["s1\ns2 \" s3 "] "abs" +["s4"] "qwe" +["d2"] "qwer\nt\nyu" +["d2"] 123 +[] {"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer\nt\nyu","d2":123} diff --git a/test/valid/multiline_escapedquotes.sorted b/test/valid/multiline_escapedquotes.sorted new file mode 100644 index 0000000..ecbd9e9 --- /dev/null +++ b/test/valid/multiline_escapedquotes.sorted @@ -0,0 +1,5 @@ +["s4"] "qwe" +["s1\ns2 \" s3 "] "abs" +["d2"] 123 +["d2"] "qwer\nt\nyu" +[] {"s4":"qwe","s1\ns2 \" s3 ":"abs","d2":123,"d2":"qwer\nt\nyu"} diff --git a/test/valid/multiline_escapedquotes_indented.json b/test/valid/multiline_escapedquotes_indented.json new file mode 100644 index 0000000..74dacab --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.json @@ -0,0 +1,5 @@ +{"s1 +s2 \" s3 ": "abs" , +"s4": "qwe", "d2": "qwer +t + yu","d2":123} diff --git a/test/valid/multiline_escapedquotes_indented.normalized b/test/valid/multiline_escapedquotes_indented.normalized new file mode 100644 index 0000000..2b65429 --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.normalized @@ -0,0 +1 @@ +{"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer \nt\n yu","d2":123} diff --git a/test/valid/multiline_escapedquotes_indented.normalized_numnormalized b/test/valid/multiline_escapedquotes_indented.normalized_numnormalized new file mode 100644 index 0000000..64a0716 --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.normalized_numnormalized @@ -0,0 +1 @@ +{"d2":"qwer \nt\n yu","d2":123.000000000000,"s1\ns2 \" s3 ":"abs","s4":"qwe"} diff --git a/test/valid/multiline_escapedquotes_indented.normalized_numnormalized_stripped b/test/valid/multiline_escapedquotes_indented.normalized_numnormalized_stripped new file mode 100644 index 0000000..4521493 --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"d2":"qwer \nt\n yu","d2":123,"s1\ns2 \" s3 ":"abs","s4":"qwe"} diff --git a/test/valid/multiline_escapedquotes_indented.normalized_sorted b/test/valid/multiline_escapedquotes_indented.normalized_sorted new file mode 100644 index 0000000..4521493 --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.normalized_sorted @@ -0,0 +1 @@ +{"d2":"qwer \nt\n yu","d2":123,"s1\ns2 \" s3 ":"abs","s4":"qwe"} diff --git a/test/valid/multiline_escapedquotes_indented.numnormalized b/test/valid/multiline_escapedquotes_indented.numnormalized new file mode 100644 index 0000000..bc66cee --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.numnormalized @@ -0,0 +1,5 @@ +["s1\ns2 \" s3 "] "abs" +["s4"] "qwe" +["d2"] "qwer \nt\n yu" +["d2"] 123.000000000000 +[] {"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer \nt\n yu","d2":123.000000000000} diff --git a/test/valid/multiline_escapedquotes_indented.numnormalized_stripped b/test/valid/multiline_escapedquotes_indented.numnormalized_stripped new file mode 100644 index 0000000..cb7fbcb --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.numnormalized_stripped @@ -0,0 +1,5 @@ +["s1\ns2 \" s3 "] "abs" +["s4"] "qwe" +["d2"] "qwer \nt\n yu" +["d2"] 123 +[] {"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer \nt\n yu","d2":123} diff --git a/test/valid/multiline_escapedquotes_indented.parsed b/test/valid/multiline_escapedquotes_indented.parsed new file mode 100644 index 0000000..cb7fbcb --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.parsed @@ -0,0 +1,5 @@ +["s1\ns2 \" s3 "] "abs" +["s4"] "qwe" +["d2"] "qwer \nt\n yu" +["d2"] 123 +[] {"s1\ns2 \" s3 ":"abs","s4":"qwe","d2":"qwer \nt\n yu","d2":123} diff --git a/test/valid/multiline_escapedquotes_indented.sorted b/test/valid/multiline_escapedquotes_indented.sorted new file mode 100644 index 0000000..bae5acb --- /dev/null +++ b/test/valid/multiline_escapedquotes_indented.sorted @@ -0,0 +1,5 @@ +["s4"] "qwe" +["s1\ns2 \" s3 "] "abs" +["d2"] 123 +["d2"] "qwer \nt\n yu" +[] {"s4":"qwe","s1\ns2 \" s3 ":"abs","d2":123,"d2":"qwer \nt\n yu"} diff --git a/test/valid/multiline_simple_key.json b/test/valid/multiline_simple_key.json new file mode 100644 index 0000000..38d71d1 --- /dev/null +++ b/test/valid/multiline_simple_key.json @@ -0,0 +1,2 @@ +{"s1 +s2": "abs"} \ No newline at end of file diff --git a/test/valid/multiline_simple_key.normalized b/test/valid/multiline_simple_key.normalized new file mode 100644 index 0000000..39302df --- /dev/null +++ b/test/valid/multiline_simple_key.normalized @@ -0,0 +1 @@ +{"s1\ns2":"abs"} diff --git a/test/valid/multiline_simple_key.normalized_numnormalized b/test/valid/multiline_simple_key.normalized_numnormalized new file mode 100644 index 0000000..39302df --- /dev/null +++ b/test/valid/multiline_simple_key.normalized_numnormalized @@ -0,0 +1 @@ +{"s1\ns2":"abs"} diff --git a/test/valid/multiline_simple_key.normalized_numnormalized_stripped b/test/valid/multiline_simple_key.normalized_numnormalized_stripped new file mode 100644 index 0000000..39302df --- /dev/null +++ b/test/valid/multiline_simple_key.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"s1\ns2":"abs"} diff --git a/test/valid/multiline_simple_key.normalized_sorted b/test/valid/multiline_simple_key.normalized_sorted new file mode 100644 index 0000000..39302df --- /dev/null +++ b/test/valid/multiline_simple_key.normalized_sorted @@ -0,0 +1 @@ +{"s1\ns2":"abs"} diff --git a/test/valid/multiline_simple_key.numnormalized b/test/valid/multiline_simple_key.numnormalized new file mode 100644 index 0000000..af9dee7 --- /dev/null +++ b/test/valid/multiline_simple_key.numnormalized @@ -0,0 +1,2 @@ +["s1\ns2"] "abs" +[] {"s1\ns2":"abs"} diff --git a/test/valid/multiline_simple_key.numnormalized_stripped b/test/valid/multiline_simple_key.numnormalized_stripped new file mode 100644 index 0000000..af9dee7 --- /dev/null +++ b/test/valid/multiline_simple_key.numnormalized_stripped @@ -0,0 +1,2 @@ +["s1\ns2"] "abs" +[] {"s1\ns2":"abs"} diff --git a/test/valid/multiline_simple_key.parsed b/test/valid/multiline_simple_key.parsed new file mode 100644 index 0000000..af9dee7 --- /dev/null +++ b/test/valid/multiline_simple_key.parsed @@ -0,0 +1,2 @@ +["s1\ns2"] "abs" +[] {"s1\ns2":"abs"} diff --git a/test/valid/multiline_simple_key.sorted b/test/valid/multiline_simple_key.sorted new file mode 100644 index 0000000..af9dee7 --- /dev/null +++ b/test/valid/multiline_simple_key.sorted @@ -0,0 +1,2 @@ +["s1\ns2"] "abs" +[] {"s1\ns2":"abs"} diff --git a/test/valid/multiline_simple_value.json b/test/valid/multiline_simple_value.json new file mode 100644 index 0000000..022ac26 --- /dev/null +++ b/test/valid/multiline_simple_value.json @@ -0,0 +1,2 @@ +{"s":"ab c +d e"} \ No newline at end of file diff --git a/test/valid/multiline_simple_value.normalized b/test/valid/multiline_simple_value.normalized new file mode 100644 index 0000000..3ee6927 --- /dev/null +++ b/test/valid/multiline_simple_value.normalized @@ -0,0 +1 @@ +{"s":"ab c\nd e"} diff --git a/test/valid/multiline_simple_value.normalized_numnormalized b/test/valid/multiline_simple_value.normalized_numnormalized new file mode 100644 index 0000000..3ee6927 --- /dev/null +++ b/test/valid/multiline_simple_value.normalized_numnormalized @@ -0,0 +1 @@ +{"s":"ab c\nd e"} diff --git a/test/valid/multiline_simple_value.normalized_numnormalized_stripped b/test/valid/multiline_simple_value.normalized_numnormalized_stripped new file mode 100644 index 0000000..3ee6927 --- /dev/null +++ b/test/valid/multiline_simple_value.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"s":"ab c\nd e"} diff --git a/test/valid/multiline_simple_value.normalized_sorted b/test/valid/multiline_simple_value.normalized_sorted new file mode 100644 index 0000000..3ee6927 --- /dev/null +++ b/test/valid/multiline_simple_value.normalized_sorted @@ -0,0 +1 @@ +{"s":"ab c\nd e"} diff --git a/test/valid/multiline_simple_value.numnormalized b/test/valid/multiline_simple_value.numnormalized new file mode 100644 index 0000000..00b40af --- /dev/null +++ b/test/valid/multiline_simple_value.numnormalized @@ -0,0 +1,2 @@ +["s"] "ab c\nd e" +[] {"s":"ab c\nd e"} diff --git a/test/valid/multiline_simple_value.numnormalized_stripped b/test/valid/multiline_simple_value.numnormalized_stripped new file mode 100644 index 0000000..00b40af --- /dev/null +++ b/test/valid/multiline_simple_value.numnormalized_stripped @@ -0,0 +1,2 @@ +["s"] "ab c\nd e" +[] {"s":"ab c\nd e"} diff --git a/test/valid/multiline_simple_value.parsed b/test/valid/multiline_simple_value.parsed new file mode 100644 index 0000000..00b40af --- /dev/null +++ b/test/valid/multiline_simple_value.parsed @@ -0,0 +1,2 @@ +["s"] "ab c\nd e" +[] {"s":"ab c\nd e"} diff --git a/test/valid/multiline_simple_value.sorted b/test/valid/multiline_simple_value.sorted new file mode 100644 index 0000000..00b40af --- /dev/null +++ b/test/valid/multiline_simple_value.sorted @@ -0,0 +1,2 @@ +["s"] "ab c\nd e" +[] {"s":"ab c\nd e"} diff --git a/test/valid/nested_array.normalized b/test/valid/nested_array.normalized new file mode 100644 index 0000000..97aade5 --- /dev/null +++ b/test/valid/nested_array.normalized @@ -0,0 +1 @@ +[1,[],[4,"hello",{}],{"array":[]}] diff --git a/test/valid/nested_array.normalized_numnormalized b/test/valid/nested_array.normalized_numnormalized new file mode 100644 index 0000000..874f2f0 --- /dev/null +++ b/test/valid/nested_array.normalized_numnormalized @@ -0,0 +1 @@ +[["hello",{},4.000000000000],[],{"array":[]},1.000000000000] diff --git a/test/valid/nested_array.normalized_numnormalized_stripped b/test/valid/nested_array.normalized_numnormalized_stripped new file mode 100644 index 0000000..3cb16fb --- /dev/null +++ b/test/valid/nested_array.normalized_numnormalized_stripped @@ -0,0 +1 @@ +[["hello",{},4],[],{"array":[]},1] diff --git a/test/valid/nested_array.normalized_sorted b/test/valid/nested_array.normalized_sorted new file mode 100644 index 0000000..3cb16fb --- /dev/null +++ b/test/valid/nested_array.normalized_sorted @@ -0,0 +1 @@ +[["hello",{},4],[],{"array":[]},1] diff --git a/test/valid/nested_array.numnormalized b/test/valid/nested_array.numnormalized new file mode 100644 index 0000000..6e33067 --- /dev/null +++ b/test/valid/nested_array.numnormalized @@ -0,0 +1,9 @@ +[0] 1.000000000000 +[1] [] +[2,0] 4.000000000000 +[2,1] "hello" +[2,2] {} +[2] [4.000000000000,"hello",{}] +[3,"array"] [] +[3] {"array":[]} +[] [1.000000000000,[],[4.000000000000,"hello",{}],{"array":[]}] diff --git a/test/valid/nested_array.numnormalized_stripped b/test/valid/nested_array.numnormalized_stripped new file mode 100644 index 0000000..4638c15 --- /dev/null +++ b/test/valid/nested_array.numnormalized_stripped @@ -0,0 +1,9 @@ +[0] 1 +[1] [] +[2,0] 4 +[2,1] "hello" +[2,2] {} +[2] [4,"hello",{}] +[3,"array"] [] +[3] {"array":[]} +[] [1,[],[4,"hello",{}],{"array":[]}] diff --git a/test/valid/nested_array.sorted b/test/valid/nested_array.sorted new file mode 100644 index 0000000..f323363 --- /dev/null +++ b/test/valid/nested_array.sorted @@ -0,0 +1,9 @@ +[0] 1 +[1,"array"] [] +[1] {"array":[]} +[2] [] +[3,0] 4 +[3,1] {} +[3,2] "hello" +[3] [4,{},"hello"] +[] [1,{"array":[]},[],[4,{},"hello"]] diff --git a/test/valid/nested_object.normalized b/test/valid/nested_object.normalized new file mode 100644 index 0000000..68c057f --- /dev/null +++ b/test/valid/nested_object.normalized @@ -0,0 +1 @@ +{"object":{"key":"value","empty":{}},"number":5} diff --git a/test/valid/nested_object.normalized_numnormalized b/test/valid/nested_object.normalized_numnormalized new file mode 100644 index 0000000..220de56 --- /dev/null +++ b/test/valid/nested_object.normalized_numnormalized @@ -0,0 +1 @@ +{"number":5.000000000000,"object":{"empty":{},"key":"value"}} diff --git a/test/valid/nested_object.normalized_numnormalized_stripped b/test/valid/nested_object.normalized_numnormalized_stripped new file mode 100644 index 0000000..efe1ce2 --- /dev/null +++ b/test/valid/nested_object.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"number":5,"object":{"empty":{},"key":"value"}} diff --git a/test/valid/nested_object.normalized_sorted b/test/valid/nested_object.normalized_sorted new file mode 100644 index 0000000..efe1ce2 --- /dev/null +++ b/test/valid/nested_object.normalized_sorted @@ -0,0 +1 @@ +{"number":5,"object":{"empty":{},"key":"value"}} diff --git a/test/valid/nested_object.numnormalized b/test/valid/nested_object.numnormalized new file mode 100644 index 0000000..61ab077 --- /dev/null +++ b/test/valid/nested_object.numnormalized @@ -0,0 +1,5 @@ +["object","key"] "value" +["object","empty"] {} +["object"] {"key":"value","empty":{}} +["number"] 5.000000000000 +[] {"object":{"key":"value","empty":{}},"number":5.000000000000} diff --git a/test/valid/nested_object.numnormalized_stripped b/test/valid/nested_object.numnormalized_stripped new file mode 100644 index 0000000..8609e30 --- /dev/null +++ b/test/valid/nested_object.numnormalized_stripped @@ -0,0 +1,5 @@ +["object","key"] "value" +["object","empty"] {} +["object"] {"key":"value","empty":{}} +["number"] 5 +[] {"object":{"key":"value","empty":{}},"number":5} diff --git a/test/valid/nested_object.sorted b/test/valid/nested_object.sorted new file mode 100644 index 0000000..8609e30 --- /dev/null +++ b/test/valid/nested_object.sorted @@ -0,0 +1,5 @@ +["object","key"] "value" +["object","empty"] {} +["object"] {"key":"value","empty":{}} +["number"] 5 +[] {"object":{"key":"value","empty":{}},"number":5} diff --git a/test/valid/number.normalized b/test/valid/number.normalized new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/test/valid/number.normalized @@ -0,0 +1 @@ +3 diff --git a/test/valid/number.normalized_numnormalized b/test/valid/number.normalized_numnormalized new file mode 100644 index 0000000..81a5cdb --- /dev/null +++ b/test/valid/number.normalized_numnormalized @@ -0,0 +1 @@ +3.000000000000 diff --git a/test/valid/number.normalized_numnormalized_stripped b/test/valid/number.normalized_numnormalized_stripped new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/test/valid/number.normalized_numnormalized_stripped @@ -0,0 +1 @@ +3 diff --git a/test/valid/number.normalized_sorted b/test/valid/number.normalized_sorted new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/test/valid/number.normalized_sorted @@ -0,0 +1 @@ +3 diff --git a/test/valid/number.numnormalized b/test/valid/number.numnormalized new file mode 100644 index 0000000..4c0c749 --- /dev/null +++ b/test/valid/number.numnormalized @@ -0,0 +1 @@ +[] 3.000000000000 diff --git a/test/valid/number.numnormalized_stripped b/test/valid/number.numnormalized_stripped new file mode 100644 index 0000000..2a1fecb --- /dev/null +++ b/test/valid/number.numnormalized_stripped @@ -0,0 +1 @@ +[] 3 diff --git a/test/valid/number.sorted b/test/valid/number.sorted new file mode 100644 index 0000000..2a1fecb --- /dev/null +++ b/test/valid/number.sorted @@ -0,0 +1 @@ +[] 3 diff --git a/test/valid/number_representations.json b/test/valid/number_representations.json new file mode 100644 index 0000000..7c528b1 --- /dev/null +++ b/test/valid/number_representations.json @@ -0,0 +1,12 @@ +[1,0,-1, +00,-00.40, +2e3, +2.1e2, +2.2e-4, +-1.3e+2, +1e14, +-.3, +.5, +0.4, ++3.25] + diff --git a/test/valid/number_representations.normalized b/test/valid/number_representations.normalized new file mode 100644 index 0000000..76f6b87 --- /dev/null +++ b/test/valid/number_representations.normalized @@ -0,0 +1 @@ +[1,0,-1,00,-00.40,2e3,2.1e2,2.2e-4,-1.3e+2,1e14,-.3,.5,0.4,+3.25] diff --git a/test/valid/number_representations.normalized_numnormalized b/test/valid/number_representations.normalized_numnormalized new file mode 100644 index 0000000..ff0e4c7 --- /dev/null +++ b/test/valid/number_representations.normalized_numnormalized @@ -0,0 +1 @@ +[-130.000000000000,-1.000000000000,-0.400000000000,-0.300000000000,0.000000000000,0.000000000000,0.000220000000,0.400000000000,0.500000000000,1.000000000000,3.250000000000,210.000000000000,2000.000000000000,100000000000000.000000000000] diff --git a/test/valid/number_representations.normalized_numnormalized_stripped b/test/valid/number_representations.normalized_numnormalized_stripped new file mode 100644 index 0000000..e77be75 --- /dev/null +++ b/test/valid/number_representations.normalized_numnormalized_stripped @@ -0,0 +1 @@ +[-130,-1,-0.4,-0.3,0,0,0.00022,0.4,0.5,1,3.25,210,2000,100000000000000] diff --git a/test/valid/number_representations.normalized_sorted b/test/valid/number_representations.normalized_sorted new file mode 100644 index 0000000..305b358 --- /dev/null +++ b/test/valid/number_representations.normalized_sorted @@ -0,0 +1 @@ +[-1.3e+2,-1,-00.40,-.3,+3.25,0,00,0.4,.5,1,1e14,2e3,2.1e2,2.2e-4] diff --git a/test/valid/number_representations.numnormalized b/test/valid/number_representations.numnormalized new file mode 100644 index 0000000..da48718 --- /dev/null +++ b/test/valid/number_representations.numnormalized @@ -0,0 +1,15 @@ +[0] 1.000000000000 +[1] 0.000000000000 +[2] -1.000000000000 +[3] 0.000000000000 +[4] -0.400000000000 +[5] 2000.000000000000 +[6] 210.000000000000 +[7] 0.000220000000 +[8] -130.000000000000 +[9] 100000000000000.000000000000 +[10] -0.300000000000 +[11] 0.500000000000 +[12] 0.400000000000 +[13] 3.250000000000 +[] [1.000000000000,0.000000000000,-1.000000000000,0.000000000000,-0.400000000000,2000.000000000000,210.000000000000,0.000220000000,-130.000000000000,100000000000000.000000000000,-0.300000000000,0.500000000000,0.400000000000,3.250000000000] diff --git a/test/valid/number_representations.numnormalized_stripped b/test/valid/number_representations.numnormalized_stripped new file mode 100644 index 0000000..13e9b8c --- /dev/null +++ b/test/valid/number_representations.numnormalized_stripped @@ -0,0 +1,15 @@ +[0] 1 +[1] 0 +[2] -1 +[3] 0 +[4] -0.4 +[5] 2000 +[6] 210 +[7] 0.00022 +[8] -130 +[9] 100000000000000 +[10] -0.3 +[11] 0.5 +[12] 0.4 +[13] 3.25 +[] [1,0,-1,0,-0.4,2000,210,0.00022,-130,100000000000000,-0.3,0.5,0.4,3.25] diff --git a/test/valid/number_representations.parsed b/test/valid/number_representations.parsed new file mode 100644 index 0000000..9435ad5 --- /dev/null +++ b/test/valid/number_representations.parsed @@ -0,0 +1,15 @@ +[0] 1 +[1] 0 +[2] -1 +[3] 00 +[4] -00.40 +[5] 2e3 +[6] 2.1e2 +[7] 2.2e-4 +[8] -1.3e+2 +[9] 1e14 +[10] -.3 +[11] .5 +[12] 0.4 +[13] +3.25 +[] [1,0,-1,00,-00.40,2e3,2.1e2,2.2e-4,-1.3e+2,1e14,-.3,.5,0.4,+3.25] diff --git a/test/valid/number_representations.sorted b/test/valid/number_representations.sorted new file mode 100644 index 0000000..d1982f9 --- /dev/null +++ b/test/valid/number_representations.sorted @@ -0,0 +1,15 @@ +[0] 2.2e-4 +[1] 2.1e2 +[2] 2e3 +[3] 1e14 +[4] 1 +[5] .5 +[6] 0.4 +[7] 00 +[8] 0 +[9] +3.25 +[10] -.3 +[11] -00.40 +[12] -1 +[13] -1.3e+2 +[] [2.2e-4,2.1e2,2e3,1e14,1,.5,0.4,00,0,+3.25,-.3,-00.40,-1,-1.3e+2] diff --git a/test/valid/object.normalized b/test/valid/object.normalized new file mode 100644 index 0000000..f523ccf --- /dev/null +++ b/test/valid/object.normalized @@ -0,0 +1 @@ +{"key":"Value"} diff --git a/test/valid/object.normalized_numnormalized b/test/valid/object.normalized_numnormalized new file mode 100644 index 0000000..f523ccf --- /dev/null +++ b/test/valid/object.normalized_numnormalized @@ -0,0 +1 @@ +{"key":"Value"} diff --git a/test/valid/object.normalized_numnormalized_stripped b/test/valid/object.normalized_numnormalized_stripped new file mode 100644 index 0000000..f523ccf --- /dev/null +++ b/test/valid/object.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"key":"Value"} diff --git a/test/valid/object.normalized_sorted b/test/valid/object.normalized_sorted new file mode 100644 index 0000000..f523ccf --- /dev/null +++ b/test/valid/object.normalized_sorted @@ -0,0 +1 @@ +{"key":"Value"} diff --git a/test/valid/object.numnormalized b/test/valid/object.numnormalized new file mode 100644 index 0000000..9f50711 --- /dev/null +++ b/test/valid/object.numnormalized @@ -0,0 +1,2 @@ +["key"] "Value" +[] {"key":"Value"} diff --git a/test/valid/object.numnormalized_stripped b/test/valid/object.numnormalized_stripped new file mode 100644 index 0000000..9f50711 --- /dev/null +++ b/test/valid/object.numnormalized_stripped @@ -0,0 +1,2 @@ +["key"] "Value" +[] {"key":"Value"} diff --git a/test/valid/object.sorted b/test/valid/object.sorted new file mode 100644 index 0000000..9f50711 --- /dev/null +++ b/test/valid/object.sorted @@ -0,0 +1,2 @@ +["key"] "Value" +[] {"key":"Value"} diff --git a/test/valid/singleline_escapedquotes_key.json b/test/valid/singleline_escapedquotes_key.json new file mode 100644 index 0000000..1aaa032 --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.json @@ -0,0 +1 @@ +{"s1 \" s2": "abs"} \ No newline at end of file diff --git a/test/valid/singleline_escapedquotes_key.normalized b/test/valid/singleline_escapedquotes_key.normalized new file mode 100644 index 0000000..938af32 --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.normalized @@ -0,0 +1 @@ +{"s1 \" s2":"abs"} diff --git a/test/valid/singleline_escapedquotes_key.normalized_numnormalized b/test/valid/singleline_escapedquotes_key.normalized_numnormalized new file mode 100644 index 0000000..938af32 --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.normalized_numnormalized @@ -0,0 +1 @@ +{"s1 \" s2":"abs"} diff --git a/test/valid/singleline_escapedquotes_key.normalized_numnormalized_stripped b/test/valid/singleline_escapedquotes_key.normalized_numnormalized_stripped new file mode 100644 index 0000000..938af32 --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"s1 \" s2":"abs"} diff --git a/test/valid/singleline_escapedquotes_key.normalized_sorted b/test/valid/singleline_escapedquotes_key.normalized_sorted new file mode 100644 index 0000000..938af32 --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.normalized_sorted @@ -0,0 +1 @@ +{"s1 \" s2":"abs"} diff --git a/test/valid/singleline_escapedquotes_key.numnormalized b/test/valid/singleline_escapedquotes_key.numnormalized new file mode 100644 index 0000000..66e89aa --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.numnormalized @@ -0,0 +1,2 @@ +["s1 \" s2"] "abs" +[] {"s1 \" s2":"abs"} diff --git a/test/valid/singleline_escapedquotes_key.numnormalized_stripped b/test/valid/singleline_escapedquotes_key.numnormalized_stripped new file mode 100644 index 0000000..66e89aa --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.numnormalized_stripped @@ -0,0 +1,2 @@ +["s1 \" s2"] "abs" +[] {"s1 \" s2":"abs"} diff --git a/test/valid/singleline_escapedquotes_key.parsed b/test/valid/singleline_escapedquotes_key.parsed new file mode 100644 index 0000000..66e89aa --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.parsed @@ -0,0 +1,2 @@ +["s1 \" s2"] "abs" +[] {"s1 \" s2":"abs"} diff --git a/test/valid/singleline_escapedquotes_key.sorted b/test/valid/singleline_escapedquotes_key.sorted new file mode 100644 index 0000000..66e89aa --- /dev/null +++ b/test/valid/singleline_escapedquotes_key.sorted @@ -0,0 +1,2 @@ +["s1 \" s2"] "abs" +[] {"s1 \" s2":"abs"} diff --git a/test/valid/singleline_escapedquotes_value.json b/test/valid/singleline_escapedquotes_value.json new file mode 100644 index 0000000..78dc4f2 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.json @@ -0,0 +1 @@ +{"s1 s2": "quoted \"substring\" value"} \ No newline at end of file diff --git a/test/valid/singleline_escapedquotes_value.normalized b/test/valid/singleline_escapedquotes_value.normalized new file mode 100644 index 0000000..075a978 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.normalized @@ -0,0 +1 @@ +{"s1 s2":"quoted \"substring\" value"} diff --git a/test/valid/singleline_escapedquotes_value.normalized_numnormalized b/test/valid/singleline_escapedquotes_value.normalized_numnormalized new file mode 100644 index 0000000..075a978 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.normalized_numnormalized @@ -0,0 +1 @@ +{"s1 s2":"quoted \"substring\" value"} diff --git a/test/valid/singleline_escapedquotes_value.normalized_numnormalized_stripped b/test/valid/singleline_escapedquotes_value.normalized_numnormalized_stripped new file mode 100644 index 0000000..075a978 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"s1 s2":"quoted \"substring\" value"} diff --git a/test/valid/singleline_escapedquotes_value.normalized_sorted b/test/valid/singleline_escapedquotes_value.normalized_sorted new file mode 100644 index 0000000..075a978 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.normalized_sorted @@ -0,0 +1 @@ +{"s1 s2":"quoted \"substring\" value"} diff --git a/test/valid/singleline_escapedquotes_value.numnormalized b/test/valid/singleline_escapedquotes_value.numnormalized new file mode 100644 index 0000000..1339ac3 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.numnormalized @@ -0,0 +1,2 @@ +["s1 s2"] "quoted \"substring\" value" +[] {"s1 s2":"quoted \"substring\" value"} diff --git a/test/valid/singleline_escapedquotes_value.numnormalized_stripped b/test/valid/singleline_escapedquotes_value.numnormalized_stripped new file mode 100644 index 0000000..1339ac3 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.numnormalized_stripped @@ -0,0 +1,2 @@ +["s1 s2"] "quoted \"substring\" value" +[] {"s1 s2":"quoted \"substring\" value"} diff --git a/test/valid/singleline_escapedquotes_value.parsed b/test/valid/singleline_escapedquotes_value.parsed new file mode 100644 index 0000000..1339ac3 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.parsed @@ -0,0 +1,2 @@ +["s1 s2"] "quoted \"substring\" value" +[] {"s1 s2":"quoted \"substring\" value"} diff --git a/test/valid/singleline_escapedquotes_value.sorted b/test/valid/singleline_escapedquotes_value.sorted new file mode 100644 index 0000000..1339ac3 --- /dev/null +++ b/test/valid/singleline_escapedquotes_value.sorted @@ -0,0 +1,2 @@ +["s1 s2"] "quoted \"substring\" value" +[] {"s1 s2":"quoted \"substring\" value"} diff --git a/test/valid/string.normalized b/test/valid/string.normalized new file mode 100644 index 0000000..31f592f --- /dev/null +++ b/test/valid/string.normalized @@ -0,0 +1 @@ +"hello this is a string" diff --git a/test/valid/string.normalized_numnormalized b/test/valid/string.normalized_numnormalized new file mode 100644 index 0000000..31f592f --- /dev/null +++ b/test/valid/string.normalized_numnormalized @@ -0,0 +1 @@ +"hello this is a string" diff --git a/test/valid/string.normalized_numnormalized_stripped b/test/valid/string.normalized_numnormalized_stripped new file mode 100644 index 0000000..31f592f --- /dev/null +++ b/test/valid/string.normalized_numnormalized_stripped @@ -0,0 +1 @@ +"hello this is a string" diff --git a/test/valid/string.normalized_sorted b/test/valid/string.normalized_sorted new file mode 100644 index 0000000..31f592f --- /dev/null +++ b/test/valid/string.normalized_sorted @@ -0,0 +1 @@ +"hello this is a string" diff --git a/test/valid/string.numnormalized b/test/valid/string.numnormalized new file mode 100644 index 0000000..b1fb986 --- /dev/null +++ b/test/valid/string.numnormalized @@ -0,0 +1 @@ +[] "hello this is a string" diff --git a/test/valid/string.numnormalized_stripped b/test/valid/string.numnormalized_stripped new file mode 100644 index 0000000..b1fb986 --- /dev/null +++ b/test/valid/string.numnormalized_stripped @@ -0,0 +1 @@ +[] "hello this is a string" diff --git a/test/valid/string.sorted b/test/valid/string.sorted new file mode 100644 index 0000000..b1fb986 --- /dev/null +++ b/test/valid/string.sorted @@ -0,0 +1 @@ +[] "hello this is a string" diff --git a/test/valid/string_in_array.normalized b/test/valid/string_in_array.normalized new file mode 100644 index 0000000..89179f6 --- /dev/null +++ b/test/valid/string_in_array.normalized @@ -0,0 +1 @@ +["hello this is a string"] diff --git a/test/valid/string_in_array.normalized_numnormalized b/test/valid/string_in_array.normalized_numnormalized new file mode 100644 index 0000000..89179f6 --- /dev/null +++ b/test/valid/string_in_array.normalized_numnormalized @@ -0,0 +1 @@ +["hello this is a string"] diff --git a/test/valid/string_in_array.normalized_numnormalized_stripped b/test/valid/string_in_array.normalized_numnormalized_stripped new file mode 100644 index 0000000..89179f6 --- /dev/null +++ b/test/valid/string_in_array.normalized_numnormalized_stripped @@ -0,0 +1 @@ +["hello this is a string"] diff --git a/test/valid/string_in_array.normalized_sorted b/test/valid/string_in_array.normalized_sorted new file mode 100644 index 0000000..89179f6 --- /dev/null +++ b/test/valid/string_in_array.normalized_sorted @@ -0,0 +1 @@ +["hello this is a string"] diff --git a/test/valid/string_in_array.numnormalized b/test/valid/string_in_array.numnormalized new file mode 100644 index 0000000..a49e6d9 --- /dev/null +++ b/test/valid/string_in_array.numnormalized @@ -0,0 +1,2 @@ +[0] "hello this is a string" +[] ["hello this is a string"] diff --git a/test/valid/string_in_array.numnormalized_stripped b/test/valid/string_in_array.numnormalized_stripped new file mode 100644 index 0000000..a49e6d9 --- /dev/null +++ b/test/valid/string_in_array.numnormalized_stripped @@ -0,0 +1,2 @@ +[0] "hello this is a string" +[] ["hello this is a string"] diff --git a/test/valid/string_in_array.sorted b/test/valid/string_in_array.sorted new file mode 100644 index 0000000..a49e6d9 --- /dev/null +++ b/test/valid/string_in_array.sorted @@ -0,0 +1,2 @@ +[0] "hello this is a string" +[] ["hello this is a string"] diff --git a/test/valid/string_in_object.normalized b/test/valid/string_in_object.normalized new file mode 100644 index 0000000..357dfdc --- /dev/null +++ b/test/valid/string_in_object.normalized @@ -0,0 +1 @@ +{"key":"hello this is a string"} diff --git a/test/valid/string_in_object.normalized_numnormalized b/test/valid/string_in_object.normalized_numnormalized new file mode 100644 index 0000000..357dfdc --- /dev/null +++ b/test/valid/string_in_object.normalized_numnormalized @@ -0,0 +1 @@ +{"key":"hello this is a string"} diff --git a/test/valid/string_in_object.normalized_numnormalized_stripped b/test/valid/string_in_object.normalized_numnormalized_stripped new file mode 100644 index 0000000..357dfdc --- /dev/null +++ b/test/valid/string_in_object.normalized_numnormalized_stripped @@ -0,0 +1 @@ +{"key":"hello this is a string"} diff --git a/test/valid/string_in_object.normalized_sorted b/test/valid/string_in_object.normalized_sorted new file mode 100644 index 0000000..357dfdc --- /dev/null +++ b/test/valid/string_in_object.normalized_sorted @@ -0,0 +1 @@ +{"key":"hello this is a string"} diff --git a/test/valid/string_in_object.numnormalized b/test/valid/string_in_object.numnormalized new file mode 100644 index 0000000..e266552 --- /dev/null +++ b/test/valid/string_in_object.numnormalized @@ -0,0 +1,2 @@ +["key"] "hello this is a string" +[] {"key":"hello this is a string"} diff --git a/test/valid/string_in_object.numnormalized_stripped b/test/valid/string_in_object.numnormalized_stripped new file mode 100644 index 0000000..e266552 --- /dev/null +++ b/test/valid/string_in_object.numnormalized_stripped @@ -0,0 +1,2 @@ +["key"] "hello this is a string" +[] {"key":"hello this is a string"} diff --git a/test/valid/string_in_object.sorted b/test/valid/string_in_object.sorted new file mode 100644 index 0000000..e266552 --- /dev/null +++ b/test/valid/string_in_object.sorted @@ -0,0 +1,2 @@ +["key"] "hello this is a string" +[] {"key":"hello this is a string"} diff --git a/test/valid/tab_escape.normalized b/test/valid/tab_escape.normalized new file mode 100644 index 0000000..b7e42b8 --- /dev/null +++ b/test/valid/tab_escape.normalized @@ -0,0 +1 @@ +"hello\tworld" diff --git a/test/valid/tab_escape.normalized_numnormalized b/test/valid/tab_escape.normalized_numnormalized new file mode 100644 index 0000000..b7e42b8 --- /dev/null +++ b/test/valid/tab_escape.normalized_numnormalized @@ -0,0 +1 @@ +"hello\tworld" diff --git a/test/valid/tab_escape.normalized_numnormalized_stripped b/test/valid/tab_escape.normalized_numnormalized_stripped new file mode 100644 index 0000000..b7e42b8 --- /dev/null +++ b/test/valid/tab_escape.normalized_numnormalized_stripped @@ -0,0 +1 @@ +"hello\tworld" diff --git a/test/valid/tab_escape.normalized_sorted b/test/valid/tab_escape.normalized_sorted new file mode 100644 index 0000000..b7e42b8 --- /dev/null +++ b/test/valid/tab_escape.normalized_sorted @@ -0,0 +1 @@ +"hello\tworld" diff --git a/test/valid/tab_escape.numnormalized b/test/valid/tab_escape.numnormalized new file mode 100644 index 0000000..ee69dd9 --- /dev/null +++ b/test/valid/tab_escape.numnormalized @@ -0,0 +1 @@ +[] "hello\tworld" diff --git a/test/valid/tab_escape.numnormalized_stripped b/test/valid/tab_escape.numnormalized_stripped new file mode 100644 index 0000000..ee69dd9 --- /dev/null +++ b/test/valid/tab_escape.numnormalized_stripped @@ -0,0 +1 @@ +[] "hello\tworld" diff --git a/test/valid/tab_escape.sorted b/test/valid/tab_escape.sorted new file mode 100644 index 0000000..ee69dd9 --- /dev/null +++ b/test/valid/tab_escape.sorted @@ -0,0 +1 @@ +[] "hello\tworld"