From cbaadc92942c50aab599a9e4c163afc1f44f7c26 Mon Sep 17 00:00:00 2001 From: Olivier Chafik Date: Thu, 11 Apr 2024 19:47:34 +0100 Subject: [PATCH] grammars: 1.5x faster inference w/ complex grammars (vector reserves / reuses) (#6609) * grammars: reserve rejects & next candidates * grammars: reuse new_stacks * grammars: fix missing sig change in llama.h * grammars: fix test (api changed) * grammars: update gbnf-validator.cpp * grammars: simpler syntax (no swap) --- examples/gbnf-validator/gbnf-validator.cpp | 2 +- llama.cpp | 16 ++++++++++------ llama.h | 5 +++-- tests/test-grammar-integration.cpp | 6 +++--- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/examples/gbnf-validator/gbnf-validator.cpp b/examples/gbnf-validator/gbnf-validator.cpp index e4c0c1689..091069ffa 100644 --- a/examples/gbnf-validator/gbnf-validator.cpp +++ b/examples/gbnf-validator/gbnf-validator.cpp @@ -17,7 +17,7 @@ static bool llama_sample_grammar_string(struct llama_grammar * grammar, const st size_t pos = 0; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { auto prev_stacks = grammar->stacks; - grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); + llama_grammar_accept(grammar->rules, prev_stacks, *it, grammar->stacks); if (grammar->stacks.empty()) { error_pos = pos; error_msg = "Unexpected character '" + unicode_cpt_to_utf8(*it) + "'"; diff --git a/llama.cpp b/llama.cpp index b6e2ade91..ad07059c4 100644 --- a/llama.cpp +++ b/llama.cpp @@ -11912,12 +11912,13 @@ static void llama_grammar_advance_stack( // be positioned at a character range (see `llama_grammar_advance_stack`), and // produces the N possible stacks if the given char is accepted at those // positions -std::vector> llama_grammar_accept( +void llama_grammar_accept( const std::vector> & rules, const std::vector> & stacks, - const uint32_t chr) { + const uint32_t chr, + std::vector> & new_stacks) { - std::vector> new_stacks; + new_stacks.clear(); for (const auto & stack : stacks) { if (stack.empty()) { @@ -11936,8 +11937,6 @@ std::vector> llama_grammar_accept( llama_grammar_advance_stack(rules, new_stack, new_stacks); } } - - return new_stacks; } static std::vector llama_grammar_reject_candidates( @@ -11951,6 +11950,7 @@ static std::vector llama_grammar_reject_candidates_for_ const std::vector & candidates) { std::vector rejects; + rejects.reserve(candidates.size()); if (stack.empty()) { for (const auto & tok : candidates) { @@ -11964,6 +11964,8 @@ static std::vector llama_grammar_reject_candidates_for_ const llama_grammar_element * stack_pos = stack.back(); std::vector next_candidates; + next_candidates.reserve(candidates.size()); + for (const auto & tok : candidates) { if (*tok.code_points == 0) { // reached end of full codepoints in token, reject iff it ended in a partial sequence @@ -12771,8 +12773,10 @@ void llama_grammar_accept_token(struct llama_context * ctx, struct llama_grammar // Note terminating 0 in decoded string const auto decoded = decode_utf8(piece, grammar->partial_utf8); const auto & code_points = decoded.first; + std::vector> tmp_new_stacks; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { - grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); + llama_grammar_accept(grammar->rules, grammar->stacks, *it, tmp_new_stacks); + grammar->stacks = tmp_new_stacks; } grammar->partial_utf8 = decoded.second; GGML_ASSERT(!grammar->stacks.empty()); diff --git a/llama.h b/llama.h index b770a275f..b5da686f7 100644 --- a/llama.h +++ b/llama.h @@ -1097,10 +1097,11 @@ const std::vector> & llama_internal struct llama_context * ctx ); -std::vector> llama_grammar_accept( +void llama_grammar_accept( const std::vector> & rules, const std::vector> & stacks, - const uint32_t chr); + const uint32_t chr, + std::vector> & new_stacks); std::pair, llama_partial_utf8> decode_utf8( const std::string & src, diff --git a/tests/test-grammar-integration.cpp b/tests/test-grammar-integration.cpp index 0a9c3b6f5..2d8f228e3 100644 --- a/tests/test-grammar-integration.cpp +++ b/tests/test-grammar-integration.cpp @@ -38,7 +38,7 @@ number ::= [0-9]+)"""; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { auto prev_stacks = grammar->stacks; - grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); + llama_grammar_accept(grammar->rules, prev_stacks, *it, grammar->stacks); assert(!grammar->stacks.empty()); } @@ -138,7 +138,7 @@ ws ::= [ \t\n\r]?)"""; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { ++pos; auto prev_stacks = grammar->stacks; - grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); + llama_grammar_accept(grammar->rules, prev_stacks, *it, grammar->stacks); // Expect that each code point will not cause the grammar to fail if (grammar->stacks.empty()) { @@ -173,7 +173,7 @@ ws ::= [ \t\n\r]?)"""; for (auto it = code_points.begin(), end = code_points.end() - 1; it != end; ++it) { auto prev_stacks = grammar->stacks; - grammar->stacks = llama_grammar_accept(grammar->rules, grammar->stacks, *it); + llama_grammar_accept(grammar->rules, prev_stacks, *it, grammar->stacks); if (grammar->stacks.empty()) { parse_failed = true; break;