mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2024-12-25 02:44:36 +00:00
JSON Schema to GBNF integration tests (#7790)
* Adding simple bare-bones test for end-to-end integration test for json validation against auto-generated JSON-schema grammars. * Adding additional examples as documented in #7789 . Also adding the ability to automatically output improperly failing grammars to debug output files so they can more easily be examined in the gbnf-validator program. * Uncommenting formerly commented tests so that they fail for others who are attempting to reproduce the bugs. * Merging improved schema test methods added by @ochafik in #7797 * Adding #define to temporarily remove failing tests so that this PR can pass CI, but still be useful for other PRs that want to leverage the framework. * Fixing nits from ochafik. Removing escape slashes, adding additional failing cases, fixing some other strings. * Fixing grammar indentation to be consistent throughout file.
This commit is contained in:
Clint Herron
GitHub
parent
557b653dc9
commit
c5a8d4b749
2
Makefile
2
Makefile
@ -1051,7 +1051,7 @@ tests/test-grammar-parser: tests/test-grammar-parser.cpp ggml.o llama.o grammar-
|
|||||||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||||
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||||
|
|
||||||
tests/test-grammar-integration: tests/test-grammar-integration.cpp ggml.o llama.o grammar-parser.o $(OBJS)
|
tests/test-grammar-integration: tests/test-grammar-integration.cpp json-schema-to-grammar.o ggml.o llama.o grammar-parser.o $(OBJS)
|
||||||
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
$(CXX) $(CXXFLAGS) -c $< -o $(call GET_OBJ_FILE, $<)
|
||||||
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
$(CXX) $(CXXFLAGS) $(filter-out %.h $<,$^) $(call GET_OBJ_FILE, $<) -o $@ $(LDFLAGS)
|
||||||
|
|
||||||
|
|||||||
@ -7,11 +7,16 @@
|
|||||||
#include "ggml.h"
|
#include "ggml.h"
|
||||||
#include "llama.h"
|
#include "llama.h"
|
||||||
#include "grammar-parser.h"
|
#include "grammar-parser.h"
|
||||||
|
#include "json-schema-to-grammar.h"
|
||||||
#include "unicode.h"
|
#include "unicode.h"
|
||||||
#include <cassert>
|
#include <cassert>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
using json = nlohmann::ordered_json;
|
||||||
|
|
||||||
|
//#define INCLUDE_FAILING_TESTS 1
|
||||||
|
|
||||||
static llama_grammar* build_grammar(const std::string & grammar_str) {
|
static llama_grammar* build_grammar(const std::string & grammar_str) {
|
||||||
auto parsed_grammar = grammar_parser::parse(grammar_str.c_str());
|
auto parsed_grammar = grammar_parser::parse(grammar_str.c_str());
|
||||||
|
|
||||||
@ -65,8 +70,8 @@ static bool match_string(const std::string & input, llama_grammar* grammar) {
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
static void test_grammar(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
|
static void test(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
|
||||||
fprintf(stderr, "⚫ Testing %s. Grammar: %s\n", test_desc.c_str(), grammar_str.c_str());
|
fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());
|
||||||
fflush(stderr);
|
fflush(stderr);
|
||||||
|
|
||||||
auto grammar = build_grammar(grammar_str);
|
auto grammar = build_grammar(grammar_str);
|
||||||
@ -85,6 +90,23 @@ static void test_grammar(const std::string & test_desc, const std::string & gram
|
|||||||
|
|
||||||
if (!matched) {
|
if (!matched) {
|
||||||
fprintf(stderr, "❌ (failed to match)\n");
|
fprintf(stderr, "❌ (failed to match)\n");
|
||||||
|
|
||||||
|
// DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.
|
||||||
|
// DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf
|
||||||
|
FILE* grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w");
|
||||||
|
if (grammar_file) {
|
||||||
|
fprintf(grammar_file, "%s", grammar_str.c_str());
|
||||||
|
fclose(grammar_file);
|
||||||
|
}
|
||||||
|
|
||||||
|
// DEBUG: Write the test string to test-grammar-integration.string.txt
|
||||||
|
FILE* string_file = fopen("test-grammar-integration.string.txt", "w");
|
||||||
|
if (string_file) {
|
||||||
|
fprintf(string_file, "%s", test_string.c_str());
|
||||||
|
fclose(string_file);
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf test-grammar-integration.string.txt\n\n");
|
||||||
} else {
|
} else {
|
||||||
fprintf(stdout, "✅︎\n");
|
fprintf(stdout, "✅︎\n");
|
||||||
}
|
}
|
||||||
@ -118,6 +140,12 @@ static void test_grammar(const std::string & test_desc, const std::string & gram
|
|||||||
// Clean up allocated memory
|
// Clean up allocated memory
|
||||||
llama_grammar_free(grammar);
|
llama_grammar_free(grammar);
|
||||||
}
|
}
|
||||||
|
static void test_grammar(const std::string & test_desc, const std::string & grammar_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
|
||||||
|
test(test_desc + ". Grammar: " + grammar_str, grammar_str, passing_strings, failing_strings);
|
||||||
|
}
|
||||||
|
static void test_schema(const std::string & test_desc, const std::string & schema_str, const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {
|
||||||
|
test(test_desc + ". Schema: " + schema_str, json_schema_to_grammar(json::parse(schema_str)), passing_strings, failing_strings);
|
||||||
|
}
|
||||||
|
|
||||||
static void test_simple_grammar() {
|
static void test_simple_grammar() {
|
||||||
// Test case for a simple grammar
|
// Test case for a simple grammar
|
||||||
@ -400,10 +428,11 @@ static void test_quantifiers() {
|
|||||||
static void test_failure_missing_root() {
|
static void test_failure_missing_root() {
|
||||||
fprintf(stderr, "⚫ Testing missing root node:\n");
|
fprintf(stderr, "⚫ Testing missing root node:\n");
|
||||||
// Test case for a grammar that is missing a root rule
|
// Test case for a grammar that is missing a root rule
|
||||||
const std::string grammar_str = R"""(rot ::= expr
|
const std::string grammar_str = R"""(
|
||||||
expr ::= term ("+" term)*
|
rot ::= expr
|
||||||
term ::= number
|
expr ::= term ("+" term)*
|
||||||
number ::= [0-9]+)""";
|
term ::= number
|
||||||
|
number ::= [0-9]+)""";
|
||||||
|
|
||||||
grammar_parser::parse_state parsed_grammar = grammar_parser::parse(grammar_str.c_str());
|
grammar_parser::parse_state parsed_grammar = grammar_parser::parse(grammar_str.c_str());
|
||||||
|
|
||||||
@ -420,10 +449,10 @@ static void test_failure_missing_reference() {
|
|||||||
|
|
||||||
// Test case for a grammar that is missing a referenced rule
|
// Test case for a grammar that is missing a referenced rule
|
||||||
const std::string grammar_str =
|
const std::string grammar_str =
|
||||||
R"""(root ::= expr
|
R"""(root ::= expr
|
||||||
expr ::= term ("+" term)*
|
expr ::= term ("+" term)*
|
||||||
term ::= numero
|
term ::= numero
|
||||||
number ::= [0-9]+)""";
|
number ::= [0-9]+)""";
|
||||||
|
|
||||||
fprintf(stderr, " Expected error: ");
|
fprintf(stderr, " Expected error: ");
|
||||||
|
|
||||||
@ -445,29 +474,558 @@ static void test_failure_left_recursion() {
|
|||||||
|
|
||||||
// Test more complicated left recursion detection
|
// Test more complicated left recursion detection
|
||||||
const std::string medium_str = R"""(
|
const std::string medium_str = R"""(
|
||||||
root ::= asdf
|
root ::= asdf
|
||||||
asdf ::= "a" | asdf "a"
|
asdf ::= "a" | asdf "a"
|
||||||
)""";
|
)""";
|
||||||
assert(test_build_grammar_fails(medium_str));
|
assert(test_build_grammar_fails(medium_str));
|
||||||
|
|
||||||
// Test even more complicated left recursion detection
|
// Test even more complicated left recursion detection
|
||||||
const std::string hard_str = R"""(
|
const std::string hard_str = R"""(
|
||||||
root ::= asdf
|
root ::= asdf
|
||||||
asdf ::= "a" | foo "b"
|
asdf ::= "a" | foo "b"
|
||||||
foo ::= "c" | asdf "d" | "e")""";
|
foo ::= "c" | asdf "d" | "e")""";
|
||||||
assert(test_build_grammar_fails(hard_str));
|
assert(test_build_grammar_fails(hard_str));
|
||||||
|
|
||||||
// Test yet even more complicated left recursion detection
|
// Test yet even more complicated left recursion detection
|
||||||
const std::string hardest_str = R"""(
|
const std::string hardest_str = R"""(
|
||||||
root ::= asdf
|
root ::= asdf
|
||||||
asdf ::= "a" | foo "b"
|
asdf ::= "a" | foo "b"
|
||||||
foo ::= "c" | empty asdf "d" | "e"
|
foo ::= "c" | empty asdf "d" | "e"
|
||||||
empty ::= "blah" | )""";
|
empty ::= "blah" | )""";
|
||||||
assert(test_build_grammar_fails(hardest_str));
|
assert(test_build_grammar_fails(hardest_str));
|
||||||
|
|
||||||
fprintf(stderr, " ✅︎ Passed\n");
|
fprintf(stderr, " ✅︎ Passed\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static void test_json_schema() {
|
||||||
|
// Note that this is similar to the regular grammar tests,
|
||||||
|
// but we convert each json schema to a grammar before parsing.
|
||||||
|
// Otherwise, this test structure is the same.
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"empty schema (object)",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"{}",
|
||||||
|
R"""({"foo": "bar"})""",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"",
|
||||||
|
"[]",
|
||||||
|
"null",
|
||||||
|
"\"\"",
|
||||||
|
"true",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"exotic formats (list)",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"items": [
|
||||||
|
{ "format": "date" },
|
||||||
|
{ "format": "uuid" },
|
||||||
|
{ "format": "time" },
|
||||||
|
{ "format": "date-time" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
// "{}", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
|
||||||
|
// "[]", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
|
||||||
|
R"""(["2012-04-23", "12345678-1234-1234-1234-1234567890ab", "18:25:43.511Z", "2012-04-23T18:25:43.511Z"])""",
|
||||||
|
//R"""(["2012-04-23","12345678-1234-1234-1234-1234567890ab"])""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
|
||||||
|
//R"""({"foo": "bar"})""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
R"""(["foo", "bar"])""",
|
||||||
|
R"""(["12345678-1234-1234-1234-1234567890ab"])""",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"string",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"\"foo\"",
|
||||||
|
"\"bar\"",
|
||||||
|
"\"\"",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"{}",
|
||||||
|
"\"foo\": \"bar\"",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"string w/ min length 1",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"\"foo\"",
|
||||||
|
"\"bar\"",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"\"\"",
|
||||||
|
"{}",
|
||||||
|
"\"foo\": \"bar\"",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"string w/ min length 3",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 3
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"\"foo\"",
|
||||||
|
"\"bar\"",
|
||||||
|
"\"foobar\"",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"\"\"",
|
||||||
|
"\"f\"",
|
||||||
|
"\"fo\"",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"string w/ max length",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"maxLength": 3
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"\"foo\"",
|
||||||
|
"\"bar\"",
|
||||||
|
"\"\"",
|
||||||
|
"\"f\"",
|
||||||
|
"\"fo\"",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"\"foobar\"",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"string w/ min & max length",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "string",
|
||||||
|
"minLength": 1,
|
||||||
|
"maxLength": 4
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"\"foo\"",
|
||||||
|
"\"bar\"",
|
||||||
|
"\"f\"",
|
||||||
|
"\"barf\"",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"\"\"",
|
||||||
|
"\"barfo\"",
|
||||||
|
"\"foobar\"",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"boolean",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "boolean"
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"true",
|
||||||
|
"false",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"\"\"",
|
||||||
|
"\"true\"",
|
||||||
|
"True",
|
||||||
|
"FALSE",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"integer",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"0",
|
||||||
|
"12345",
|
||||||
|
"1234567890123456"
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"",
|
||||||
|
"01",
|
||||||
|
"007",
|
||||||
|
"12345678901234567"
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"string const",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"const": "foo"
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"\"foo\"",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"foo",
|
||||||
|
"\"bar\"",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"non-string const",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"const": true
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"true",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"",
|
||||||
|
"foo",
|
||||||
|
"\"true\"",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"non-string const",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"enum": ["red", "amber", "green", null, 42, ["foo"]]
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"\"red\"",
|
||||||
|
"null",
|
||||||
|
"42",
|
||||||
|
"[\"foo\"]",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"",
|
||||||
|
"420",
|
||||||
|
"true",
|
||||||
|
"foo",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"min+max items",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": ["number", "integer"]
|
||||||
|
},
|
||||||
|
"minItems": 3,
|
||||||
|
"maxItems": 5
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
"[1, 2, 3]",
|
||||||
|
"[1, 2, 3, 4]",
|
||||||
|
"[1, 2, 3, 4, 5]",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
"[1, 2]",
|
||||||
|
"[1, 2, 3, 4, 5, 6]",
|
||||||
|
"1"
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
// Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
|
||||||
|
test_schema(
|
||||||
|
"object properties",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"number": { "type": "number" },
|
||||||
|
"street_name": { "type": "string" },
|
||||||
|
"street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",
|
||||||
|
// "By default, leaving out properties is valid"
|
||||||
|
R"""({ "street_name": "Pennsylvania" })""",
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
|
||||||
|
// "By extension, even an empty object is valid"
|
||||||
|
R"""({})""",
|
||||||
|
// "By default, providing additional properties is valid"
|
||||||
|
#ifdef INCLUDE_FAILING_TESTS
|
||||||
|
// TODO: The following should pass, but currently FAILS. Additional properties should be permitted by default.
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
|
||||||
|
// TODO: Spaces should be permitted around enum values, but currently they fail to pass.
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
|
||||||
|
#endif
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
// Change datatype from number to string
|
||||||
|
R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",
|
||||||
|
// Reorder properties
|
||||||
|
R"""({ "street_name": "Pennsylvania", "number": 1600 })""",
|
||||||
|
// Reorder properties
|
||||||
|
R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
|
||||||
|
// Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)
|
||||||
|
test_schema(
|
||||||
|
"object properties, additionalProperties: true",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"number": { "type": "number" },
|
||||||
|
"street_name": { "type": "string" },
|
||||||
|
"street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
|
||||||
|
},
|
||||||
|
"additionalProperties": true
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
// "By extension, even an empty object is valid"
|
||||||
|
R"""({})""",
|
||||||
|
#ifdef INCLUDE_FAILING_TESTS
|
||||||
|
// TODO: Following line should pass and doesn't
|
||||||
|
R"""({"number":1600,"street_name":"Pennsylvania","street_type":"Avenue"})""",
|
||||||
|
// "By default, leaving out properties is valid"
|
||||||
|
// TODO: Following line should pass and doesn't
|
||||||
|
R"""({ "street_name": "Pennsylvania" })""",
|
||||||
|
// TODO: Following line should pass and doesn't
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
|
||||||
|
// "By default, providing additional properties is valid"
|
||||||
|
// TODO: The following should pass, but currently FAILS. Additional properties should be permitted by default.
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",
|
||||||
|
// TODO: Spaces should be permitted around enum values, but currently they fail to pass.
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
|
||||||
|
#endif
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
// Change datatype from number to string
|
||||||
|
R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",
|
||||||
|
// Reorder properties
|
||||||
|
R"""({ "street_name": "Pennsylvania", "number": 1600, "street_type":"Avenue"})""",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
// Additional properties: false
|
||||||
|
test_schema(
|
||||||
|
"required + optional props each in original order",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"number": { "type": "number" },
|
||||||
|
"street_name": { "type": "string" },
|
||||||
|
"street_type": { "enum": ["Street", "Avenue", "Boulevard"] }
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
R"""({ "street_name": "Pennsylvania" })""",
|
||||||
|
R"""({ "number": 1600, "street_type":"Avenue"})""",
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania" })""",
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",
|
||||||
|
#ifdef INCLUDE_FAILING_TESTS
|
||||||
|
// TODO: Spaces should be permitted around enum values, but currently they fail to pass.
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",
|
||||||
|
#endif
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
// Reorder properties
|
||||||
|
R"""({ "street_type": "Avenue", "number": 1600 })""",
|
||||||
|
// Add "direction"
|
||||||
|
R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue", "direction": "NW" })""",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
test_schema(
|
||||||
|
"required + optional props each in original order",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"properties": {
|
||||||
|
"b": {"type": "string"},
|
||||||
|
"a": {"type": "string"},
|
||||||
|
"d": {"type": "string"},
|
||||||
|
"c": {"type": "string"}
|
||||||
|
},
|
||||||
|
"required": ["a", "b"],
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
R"""({"b": "foo", "a": "bar"})""",
|
||||||
|
R"""({"b":"foo","a":"bar","d":"qux"})""",
|
||||||
|
R"""({"b":"foo", "a":"bar", "d":"qux", "c":"baz"})""",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
R"""({"a": "foo", "b": "bar"})""",
|
||||||
|
R"""({"b": "bar"})""",
|
||||||
|
R"""({"a": "foo", "c": "baz"})""",
|
||||||
|
R"""({"a":"foo", "b":"bar", "c":"baz", "d":"qux"})""",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
// NOTE: Example from https://json-schema.org/learn/getting-started-step-by-step#define-required-properties
|
||||||
|
test_schema(
|
||||||
|
"required props",
|
||||||
|
// Schema
|
||||||
|
R"""(
|
||||||
|
{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"$id": "https://example.com/product.schema.json",
|
||||||
|
"title": "Product",
|
||||||
|
"description": "A product from Acme's catalog",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"productId": {
|
||||||
|
"description": "The unique identifier for a product",
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"productName": {
|
||||||
|
"description": "Name of the product",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"price": {
|
||||||
|
"description": "The price of the product",
|
||||||
|
"type": "number",
|
||||||
|
"exclusiveMinimum": 0
|
||||||
|
},
|
||||||
|
"tags": {
|
||||||
|
"description": "Tags for the product",
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"minItems": 1,
|
||||||
|
"uniqueItems": true
|
||||||
|
},
|
||||||
|
"dimensions": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"length": {
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
"width": {
|
||||||
|
"type": "number"
|
||||||
|
},
|
||||||
|
"height": {
|
||||||
|
"type": "number"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [ "length", "width", "height" ]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [ "productId", "productName", "price" ]
|
||||||
|
}
|
||||||
|
)""",
|
||||||
|
// Passing strings
|
||||||
|
{
|
||||||
|
R"""({"productId": 1, "productName": "A green door", "price": 12.50})""",
|
||||||
|
R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green"]})""",
|
||||||
|
R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green"], "dimensions": {"length": 785, "width": 250.5, "height": -0.359}})""",
|
||||||
|
},
|
||||||
|
// Failing strings
|
||||||
|
{
|
||||||
|
R"""({})""", // Missing all required properties
|
||||||
|
R"""({"productName": "A green door", "price": 12.50, "productId": 1})""", // Out of order properties
|
||||||
|
// TODO: The following line should fail, but currently it passes. `exclusiveMinimum` is not supported, as it would likely be too difficult to implement.
|
||||||
|
// Perhaps special checks for minimum and maximum values of 0 could be added (since that's relatively easy to do with grammars), but anything else would likely be too complex.
|
||||||
|
// R"""({"productId": 1, "productName": "A green door", "price": -12.50})""",
|
||||||
|
R"""({"productId": 1, "productName": "A green door"})""", // Missing required property (price)
|
||||||
|
R"""({"productName": "A green door", "price": 12.50})""", // Missing required property (productId)
|
||||||
|
R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": []})""", // tags is empty, but minItems is 1
|
||||||
|
R"""({"productId": 1, "productName": "A green door", "price": 12.50, "dimensions": {"length": 785, "width": 250.5, "height": -0.359}, "tags": ["home", "green"]})""", // Tags and dimensions are out of order
|
||||||
|
// TODO: The following line should fail, but currently it passes. `uniqueItems` is not supported, as it would likely be too difficult to implement.
|
||||||
|
// R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green", "home"]})""",
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
int main() {
|
int main() {
|
||||||
fprintf(stdout, "Running grammar integration tests...\n");
|
fprintf(stdout, "Running grammar integration tests...\n");
|
||||||
test_simple_grammar();
|
test_simple_grammar();
|
||||||
@ -477,6 +1035,7 @@ int main() {
|
|||||||
test_failure_missing_root();
|
test_failure_missing_root();
|
||||||
test_failure_missing_reference();
|
test_failure_missing_reference();
|
||||||
test_failure_left_recursion();
|
test_failure_left_recursion();
|
||||||
|
test_json_schema();
|
||||||
fprintf(stdout, "All tests passed.\n");
|
fprintf(stdout, "All tests passed.\n");
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user