From 02629d98f17f927d629312964821c8e919257e0f Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Fri, 20 Sep 2024 11:41:51 +0300 Subject: [PATCH] llama : make llm_tokenizer more private ggml-ci --- src/llama-vocab.cpp | 190 ++++++++++++++++++++++---------------------- src/llama-vocab.h | 34 ++++---- src/llama.cpp | 15 ++-- 3 files changed, 118 insertions(+), 121 deletions(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index a73038af6..80ca7335d 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -79,6 +79,15 @@ struct naive_trie { // impl // +struct llm_tokenizer { + llm_tokenizer() {} + virtual ~llm_tokenizer() = default; +}; + +llama_vocab::~llama_vocab() { + delete tokenizer; +} + int llama_vocab::find_bpe_rank(const std::string & token_left, const std::string & token_right) const { GGML_ASSERT(token_left.find(' ') == std::string::npos); GGML_ASSERT(token_left.find('\n') == std::string::npos); @@ -188,13 +197,12 @@ struct llm_bigram_spm { }; struct llm_tokenizer_spm : llm_tokenizer { - llm_tokenizer_spm(const llama_vocab & vocab) : llm_tokenizer(vocab) {} + llm_tokenizer_spm(const llama_vocab & /*vocab*/) : llm_tokenizer() {} }; struct llm_tokenizer_spm_session { - - llm_tokenizer_spm_session(const llm_tokenizer & tokenizer) : - spm_tokenizer(static_cast(tokenizer)) {} + llm_tokenizer_spm_session(const llama_vocab & vocab) : vocab(vocab), + spm_tokenizer(static_cast(vocab.tokenizer)) {} void tokenize(const std::string & text, std::vector & output) { @@ -257,7 +265,6 @@ struct llm_tokenizer_spm_session { private: void resegment(llm_symbol & symbol, std::vector & output) { - const auto & vocab = spm_tokenizer.vocab; auto text = std::string(symbol.text, symbol.n); auto token = vocab.token_to_id.find(text); @@ -287,7 +294,6 @@ private: if (left == -1 || right == -1) { return; } - const auto & vocab = spm_tokenizer.vocab; const std::string text = std::string(symbols[left].text, symbols[left].n + symbols[right].n); auto token = vocab.token_to_id.find(text); @@ -313,7 +319,8 @@ private: rev_merge[text] = std::make_pair(left, right); } - const llm_tokenizer_spm & spm_tokenizer; + const llama_vocab & vocab; + const llm_tokenizer_spm * spm_tokenizer; // currently unused std::vector symbols; llm_bigram_spm::queue work_queue; @@ -360,7 +367,7 @@ struct llm_bigram_bpe { }; struct llm_tokenizer_bpe : llm_tokenizer { - llm_tokenizer_bpe(const llama_vocab & vocab) : llm_tokenizer(vocab) { + llm_tokenizer_bpe(const llama_vocab & vocab) : llm_tokenizer() { GGML_ASSERT(vocab.type == LLAMA_VOCAB_TYPE_BPE); switch (vocab.type_pre) { case LLAMA_VOCAB_PRE_TYPE_LLAMA3: @@ -473,16 +480,14 @@ struct llm_tokenizer_bpe : llm_tokenizer { }; struct llm_tokenizer_bpe_session { + llm_tokenizer_bpe_session(const llama_vocab & vocab) : vocab(vocab), + bpe_tokenizer(static_cast(vocab.tokenizer)) {} - llm_tokenizer_bpe_session(const llm_tokenizer & tokenizer) : - bpe_tokenizer(static_cast(tokenizer)) {} - - void append(const llama_vocab::id token_id, std::vector & output) const { + static void append(const llama_vocab::id token_id, std::vector & output) { output.push_back(token_id); } bool append_bos(std::vector & output) const { - const auto & vocab = bpe_tokenizer.vocab; if (vocab.tokenizer_add_bos) { GGML_ASSERT(vocab.special_bos_id != -1); output.push_back(vocab.special_bos_id); @@ -492,7 +497,6 @@ struct llm_tokenizer_bpe_session { } bool append_eos(std::vector & output) const { - const auto & vocab = bpe_tokenizer.vocab; if (vocab.tokenizer_add_eos) { GGML_ASSERT(vocab.special_eos_id != -1); output.push_back(vocab.special_eos_id); @@ -502,7 +506,6 @@ struct llm_tokenizer_bpe_session { } void check_double_bos_eos(const std::vector & output) const { - const auto & vocab = bpe_tokenizer.vocab; if (vocab.tokenizer_add_bos && output.size() >= 2 && output[1] == vocab.special_bos_id) { LLAMA_LOG_WARN( "%s: Added a BOS token to the prompt as specified by the model but the prompt " @@ -519,12 +522,11 @@ struct llm_tokenizer_bpe_session { void tokenize(const std::string & text, std::vector & output) { int final_prev_index = -1; - const auto word_collection = unicode_regex_split(text, bpe_tokenizer.regex_exprs); - const auto & vocab = bpe_tokenizer.vocab; + const auto word_collection = unicode_regex_split(text, bpe_tokenizer->regex_exprs); symbols_final.clear(); - for (auto & word : word_collection) { + for (const auto & word : word_collection) { work_queue = llm_bigram_bpe::queue(); symbols.clear(); @@ -627,7 +629,6 @@ private: if (left == -1 || right == -1) { return; } - const auto & vocab = bpe_tokenizer.vocab; std::string left_token = std::string(symbols[left].text, symbols[left].n); std::string right_token = std::string(symbols[right].text, symbols[right].n); @@ -650,7 +651,8 @@ private: work_queue.push(bigram); } - const llm_tokenizer_bpe & bpe_tokenizer; + const llama_vocab & vocab; + const llm_tokenizer_bpe * bpe_tokenizer; std::vector symbols; std::vector symbols_final; @@ -662,16 +664,14 @@ private: // struct llm_tokenizer_wpm : llm_tokenizer { - llm_tokenizer_wpm(const llama_vocab & vocab) : llm_tokenizer(vocab) {} + llm_tokenizer_wpm(const llama_vocab & /*vocab*/) : llm_tokenizer() {} }; struct llm_tokenizer_wpm_session { - - llm_tokenizer_wpm_session(const llm_tokenizer & tokenizer) - : wpm_tokenizer(static_cast(tokenizer)) {} + llm_tokenizer_wpm_session(const llama_vocab & vocab) : vocab(vocab), + wpm_tokenizer(static_cast(vocab.tokenizer)) {} void tokenize(const std::string & text, std::vector & output) { - const auto & vocab = wpm_tokenizer.vocab; const auto & token_map = vocab.token_to_id; // normalize and split by whitespace std::vector words = preprocess(text); @@ -719,7 +719,7 @@ struct llm_tokenizer_wpm_session { } // TODO: reduce string copies by using cpts_offs array - std::vector preprocess(const std::string & text) const { + static std::vector preprocess(const std::string & text) { const std::vector cpts_nfd = unicode_cpts_normalize_nfd(unicode_cpts_from_utf8(text)); std::vector words(1, ""); @@ -772,7 +772,8 @@ struct llm_tokenizer_wpm_session { } private: - const llm_tokenizer_wpm & wpm_tokenizer; + const llama_vocab & vocab; + const llm_tokenizer_wpm * wpm_tokenizer; }; // @@ -780,7 +781,7 @@ private: // struct llm_tokenizer_ugm : llm_tokenizer { - llm_tokenizer_ugm(const llama_vocab & vocab) : llm_tokenizer(vocab) { + llm_tokenizer_ugm(const llama_vocab & vocab) : llm_tokenizer() { if (vocab.precompiled_charsmap.size() > 0) { size_t charsmap_offset = 0; @@ -847,9 +848,8 @@ struct llm_tokenizer_ugm : llm_tokenizer { }; struct llm_tokenizer_ugm_session { - - llm_tokenizer_ugm_session(const llm_tokenizer & tokenizer) - : ugm_tokenizer(static_cast(tokenizer)) {} + llm_tokenizer_ugm_session(const llama_vocab & vocab) : vocab(vocab), + ugm_tokenizer(static_cast(vocab.tokenizer)) {} /* This implementation is based on SentencePiece optimized Viterbi algorithm for * unigram language models. The general idea is to: @@ -867,7 +867,6 @@ struct llm_tokenizer_ugm_session { void tokenize(const std::string & text, std::vector & output) { // get current size of output (for reversal later) size_t output_size = output.size(); - const auto & vocab = ugm_tokenizer.vocab; // normalize the input first std::string normalized; @@ -890,7 +889,7 @@ struct llm_tokenizer_ugm_session { // traverse the token matcher trie to find a matching token bool single_codepoint_token_found = false; const struct best_tokenization & current_best = tokenization_results[input_offset]; - const struct naive_trie * node = ugm_tokenizer.token_matcher.traverse(normalized[prefix_offset++]); + const struct naive_trie * node = ugm_tokenizer->token_matcher.traverse(normalized[prefix_offset++]); while (prefix_offset <= input_len && node != NULL) { // check if we found valid token in prefix @@ -920,7 +919,7 @@ struct llm_tokenizer_ugm_session { // if we didn't find a valid token corresponding to the whole UTF code point // then use unknown token as the tokenization of this UTF code point if (!single_codepoint_token_found) { - const double challenger_score = current_best.score_sum + ugm_tokenizer.unknown_token_score; + const double challenger_score = current_best.score_sum + ugm_tokenizer->unknown_token_score; prefix_offset = input_offset + n_utf8_code_units; struct best_tokenization & current_champ = tokenization_results[prefix_offset]; if (challenger_score > current_champ.score_sum) { @@ -963,9 +962,8 @@ private: void normalize(const std::string& input, std::string * normalized) { normalized->clear(); normalized->reserve(input.size() * 3); - const auto & vocab = ugm_tokenizer.vocab; - const std::string space = vocab.tokenizer_escape_whitespaces ? ugm_tokenizer.escaped_space : " "; + const std::string space = vocab.tokenizer_escape_whitespaces ? ugm_tokenizer->escaped_space : " "; bool shall_prepend_space = !vocab.tokenizer_treat_whitespace_as_suffix && vocab.tokenizer_add_space_prefix; bool shall_append_space = vocab.tokenizer_treat_whitespace_as_suffix && vocab.tokenizer_add_space_prefix; @@ -1061,7 +1059,7 @@ private: // if input prefix matches some user-defined token return this token as normalization result auto user_defined_token_match = - ugm_tokenizer.user_defined_token_matcher.get_longest_prefix(&input[input_offset], input.size() - input_offset); + ugm_tokenizer->user_defined_token_matcher.get_longest_prefix(&input[input_offset], input.size() - input_offset); if (user_defined_token_match.second > 0) { return { &input[input_offset], user_defined_token_match.second, user_defined_token_match.second }; } @@ -1069,8 +1067,8 @@ private: size_t longest_prefix_length = 0; size_t longest_prefix_offset = 0; - if (ugm_tokenizer.xcda_array_size > 0) { - struct xcda_array_view xcda_view(ugm_tokenizer.xcda_array, ugm_tokenizer.xcda_array_size); + if (ugm_tokenizer->xcda_array_size > 0) { + struct xcda_array_view xcda_view(ugm_tokenizer->xcda_array, ugm_tokenizer->xcda_array_size); // Find the longest normalized sequence matching the input prefix by walking // the XOR-compressed compact double array (XCDA) starting from the root node @@ -1106,26 +1104,27 @@ private: if (longest_prefix_length > 0) { // we have a match, so return the replacement sequence - if (longest_prefix_offset >= ugm_tokenizer.prefix_replacements_size) { + if (longest_prefix_offset >= ugm_tokenizer->prefix_replacements_size) { throw std::runtime_error("Index out of array bounds in precompiled charsmap!"); } - const char * prefix_replacement = &(ugm_tokenizer.prefix_replacements)[longest_prefix_offset]; + const char * prefix_replacement = &(ugm_tokenizer->prefix_replacements)[longest_prefix_offset]; return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length }; - } else { - // check if the input prefix contains a valid sequence of UTF-8 code units - try { - // if yes, return this sequence unmodified - size_t prefix_offset = input_offset; - unicode_cpt_from_utf8(input, prefix_offset); - return { &input[input_offset], prefix_offset - input_offset, prefix_offset - input_offset }; - } catch (std::invalid_argument & /*ex*/) { - // if no, consume 1 byte and return U+FFFD - REPLACEMENT CHARACTER - return { "\xEF\xBF\xBD", 3, 1 }; - } + } + + // check if the input prefix contains a valid sequence of UTF-8 code units + try { + // if yes, return this sequence unmodified + size_t prefix_offset = input_offset; + unicode_cpt_from_utf8(input, prefix_offset); + return { &input[input_offset], prefix_offset - input_offset, prefix_offset - input_offset }; + } catch (std::invalid_argument & /*ex*/) { + // if no, consume 1 byte and return U+FFFD - REPLACEMENT CHARACTER + return { "\xEF\xBF\xBD", 3, 1 }; } } - const llm_tokenizer_ugm & ugm_tokenizer; + const llama_vocab & vocab; + const llm_tokenizer_ugm * ugm_tokenizer; }; // @@ -1187,7 +1186,7 @@ static std::vector llama_unescape_rwkv_token(const std::string & escape } struct llm_tokenizer_rwkv : llm_tokenizer { - llm_tokenizer_rwkv(const llama_vocab & vocab) : llm_tokenizer(vocab) { + llm_tokenizer_rwkv(const llama_vocab & vocab) : llm_tokenizer() { // RWKV supports arbitrary byte tokens, but the vocab struct only supports string tokens. // For now, we decode the vocab here into the lookup we'll use for tokenization. @@ -1203,13 +1202,11 @@ struct llm_tokenizer_rwkv : llm_tokenizer { }; struct llm_tokenizer_rwkv_session { - - llm_tokenizer_rwkv_session(const llm_tokenizer & tokenizer) - : rwkv_tokenizer(static_cast(tokenizer)) {} + llm_tokenizer_rwkv_session(const llama_vocab & vocab) : vocab(vocab), + rwkv_tokenizer(static_cast(*vocab.tokenizer)) {} void tokenize(const std::string & text, std::vector & output) { uint32_t position = 0; - const auto & vocab = rwkv_tokenizer.vocab; while (position < text.size()) { const struct naive_trie * node = rwkv_tokenizer.token_matcher.traverse(text[position]); if (node == NULL) { @@ -1237,9 +1234,32 @@ struct llm_tokenizer_rwkv_session { } private: + const llama_vocab & vocab; const llm_tokenizer_rwkv & rwkv_tokenizer; }; +void llama_vocab::init_tokenizer() { + switch (type) { + case LLAMA_VOCAB_TYPE_SPM: + tokenizer = new llm_tokenizer_spm(*this); + break; + case LLAMA_VOCAB_TYPE_BPE: + tokenizer = new llm_tokenizer_bpe(*this); + break; + case LLAMA_VOCAB_TYPE_WPM: + tokenizer = new llm_tokenizer_wpm(*this); + break; + case LLAMA_VOCAB_TYPE_UGM: + tokenizer = new llm_tokenizer_ugm(*this); + break; + case LLAMA_VOCAB_TYPE_RWKV: + tokenizer = new llm_tokenizer_rwkv(*this); + break; + default: + GGML_ABORT("unsupported vocab type"); + } +} + // // (de-) tokenize // @@ -1301,7 +1321,7 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list< // if a fragment is text ( not yet processed ) if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { - auto & raw_text = fragment.raw_text; + const auto & raw_text = fragment.raw_text; auto raw_text_base_offset = fragment.offset; auto raw_text_base_length = fragment.length; @@ -1400,11 +1420,15 @@ static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list< } } -std::vector llama_tokenize_internal(const llm_tokenizer * tokenizer, - std::string raw_text, bool add_special, bool parse_special) { +std::vector llama_tokenize_internal( + const llama_vocab & vocab, + std::string raw_text, + bool add_special, + bool parse_special) { + GGML_ASSERT(vocab.tokenizer && "Tokenizer not initialized. Call llama_vocab::init_tokenizer() first."); + std::vector output; std::forward_list fragment_buffer; - const llama_vocab & vocab = tokenizer->vocab; if (!raw_text.empty()) { fragment_buffer.emplace_front(raw_text, 0, raw_text.length()); @@ -1440,7 +1464,7 @@ std::vector llama_tokenize_internal(const llm_tokenizer * token LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); #endif llama_escape_whitespace(raw_text); - llm_tokenizer_spm_session session(*tokenizer); + llm_tokenizer_spm_session session(vocab); session.tokenize(raw_text, output); is_prev_special = false; } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) @@ -1463,7 +1487,7 @@ std::vector llama_tokenize_internal(const llm_tokenizer * token } break; case LLAMA_VOCAB_TYPE_BPE: { - llm_tokenizer_bpe_session session(*tokenizer); + llm_tokenizer_bpe_session session(vocab); // it calls some other methods that are not exist in llm_tokenizer, // here just cast it to bpe tokenizer object if (add_special) { @@ -1494,7 +1518,7 @@ std::vector llama_tokenize_internal(const llm_tokenizer * token output.push_back(vocab.special_cls_id); } - llm_tokenizer_wpm_session session(*tokenizer); + llm_tokenizer_wpm_session session(vocab); for (const auto & fragment : fragment_buffer) { if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { @@ -1520,7 +1544,7 @@ std::vector llama_tokenize_internal(const llm_tokenizer * token GGML_ASSERT(vocab.special_bos_id != -1); output.push_back(vocab.special_bos_id); } - llm_tokenizer_ugm_session session(*tokenizer); + llm_tokenizer_ugm_session session(vocab); for (const auto & fragment : fragment_buffer) { if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { @@ -1548,7 +1572,7 @@ std::vector llama_tokenize_internal(const llm_tokenizer * token } break; case LLAMA_VOCAB_TYPE_RWKV: { - llm_tokenizer_rwkv_session session(*tokenizer); + llm_tokenizer_rwkv_session session(vocab); for (const auto & fragment : fragment_buffer) { if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); @@ -1570,32 +1594,6 @@ std::vector llama_tokenize_internal(const llm_tokenizer * token return output; } -llm_tokenizer * llama_create_tokenizer(const llama_vocab & vocab) { - llm_tokenizer * tokenizer; - - switch (vocab.type) { - case LLAMA_VOCAB_TYPE_SPM: - tokenizer = new llm_tokenizer_spm(vocab); - break; - case LLAMA_VOCAB_TYPE_BPE: - tokenizer = new llm_tokenizer_bpe(vocab); - break; - case LLAMA_VOCAB_TYPE_WPM: - tokenizer = new llm_tokenizer_wpm(vocab); - break; - case LLAMA_VOCAB_TYPE_UGM: - tokenizer = new llm_tokenizer_ugm(vocab); - break; - case LLAMA_VOCAB_TYPE_RWKV: - tokenizer = new llm_tokenizer_rwkv(vocab); - break; - default: - GGML_ABORT("fatal error"); - } - - return tokenizer; -} - llama_token llama_byte_to_token_impl(const llama_vocab & vocab, uint8_t ch) { GGML_ASSERT(llama_vocab_get_type(vocab) != LLAMA_VOCAB_TYPE_NONE); static const char * hex = "0123456789ABCDEF"; @@ -1700,14 +1698,14 @@ llama_token llama_token_eom_impl(const struct llama_vocab & vocab) { } int32_t llama_tokenize_impl( - const llm_tokenizer * tokenizer, + const struct llama_vocab & vocab, const char * text, int32_t text_len, llama_token * tokens, int32_t n_tokens_max, bool add_special, bool parse_special) { - auto res = llama_tokenize_internal(tokenizer, std::string(text, text_len), add_special, parse_special); + auto res = llama_tokenize_internal(vocab, std::string(text, text_len), add_special, parse_special); if (n_tokens_max < (int) res.size()) { // LLAMA_LOG_ERROR("%s: too many tokens\n", __func__); return -((int) res.size()); @@ -1831,6 +1829,8 @@ int32_t llama_detokenize_impl( int32_t text_len_max, bool remove_special, bool unparse_special) { + GGML_ASSERT(vocab.tokenizer && "Tokenizer not initialized. Call llama_vocab::init_tokenizer() first."); + int32_t avail = text_len_max; int32_t total = 0; diff --git a/src/llama-vocab.h b/src/llama-vocab.h index 0025a8ab7..18956f44c 100644 --- a/src/llama-vocab.h +++ b/src/llama-vocab.h @@ -7,6 +7,8 @@ #include #include +struct llm_tokenizer; + struct llama_vocab { using id = llama_token; using token = std::string; @@ -61,14 +63,14 @@ struct llama_vocab { std::vector precompiled_charsmap; + llm_tokenizer * tokenizer = nullptr; + + llama_vocab() = default; + ~llama_vocab(); + int find_bpe_rank(const std::string & token_left, const std::string & token_right) const; -}; -struct llm_tokenizer { - llm_tokenizer(const llama_vocab & vocab) : vocab(vocab) {} - virtual ~llm_tokenizer() = default; - - const llama_vocab & vocab; + void init_tokenizer(); }; // @@ -78,13 +80,11 @@ struct llm_tokenizer { // TODO: rename to llama_tokenize_impl // TODO: This should probably be in llama.h std::vector llama_tokenize_internal( - const llm_tokenizer * tokenizer, + const llama_vocab & vocab, std::string raw_text, bool add_special, bool parse_special = false); -llm_tokenizer * llama_create_tokenizer(const llama_vocab & vocab); - // TODO: move the API below as member functions of llama_vocab llama_token llama_byte_to_token_impl(const llama_vocab & vocab, uint8_t ch); @@ -115,13 +115,13 @@ llama_token llama_token_eot_impl (const struct llama_vocab & vocab); llama_token llama_token_eom_impl (const struct llama_vocab & vocab); int32_t llama_tokenize_impl( - const llm_tokenizer * tokenizer, - const char * text, - int32_t text_len, - llama_token * tokens, - int32_t n_tokens_max, - bool add_special, - bool parse_special); + const struct llama_vocab & vocab, + const char * text, + int32_t text_len, + llama_token * tokens, + int32_t n_tokens_max, + bool add_special, + bool parse_special); // does not write null-terminator to buf int32_t llama_token_to_piece_impl( @@ -139,4 +139,4 @@ int32_t llama_detokenize_impl( char * text, int32_t text_len_max, bool remove_special, - bool unparse_special); + bool unparse_special); \ No newline at end of file diff --git a/src/llama.cpp b/src/llama.cpp index 6ecbe3d26..ba845766d 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -2846,9 +2846,8 @@ struct llama_model { std::string name = "n/a"; - llama_hparams hparams = {}; - llama_vocab vocab; - llm_tokenizer * tokenizer; + llama_hparams hparams = {}; + llama_vocab vocab; struct ggml_tensor * tok_embd; struct ggml_tensor * type_embd; @@ -2924,8 +2923,6 @@ struct llama_model { while (!lora_adapters.empty()) { llama_lora_adapter_free(*lora_adapters.begin()); } - - delete tokenizer; } }; @@ -6407,7 +6404,7 @@ static void llm_load_vocab( } GGML_ASSERT(vocab.id_to_token.size() == vocab.token_to_id.size()); - model.tokenizer = llama_create_tokenizer(vocab); + vocab.init_tokenizer(); // determine the newline token: LLaMA "<0x0A>" == 10 == '\n', Falcon 193 == '\n' if (vocab.type == LLAMA_VOCAB_TYPE_SPM) { @@ -6458,11 +6455,11 @@ static void llm_load_vocab( } else if (vocab.type == LLAMA_VOCAB_TYPE_WPM) { vocab.linefeed_id = vocab.special_pad_id; } else if (vocab.type == LLAMA_VOCAB_TYPE_RWKV) { - const std::vector ids = llama_tokenize_internal(model.tokenizer, "\n", false); + const std::vector ids = llama_tokenize_internal(model.vocab, "\n", false); GGML_ASSERT(!ids.empty() && "model vocab missing newline token"); vocab.linefeed_id = ids[0]; } else { - const std::vector ids = llama_tokenize_internal(model.tokenizer, "\xC4\x8A", false); // U+010A + const std::vector ids = llama_tokenize_internal(model.vocab, "\xC4\x8A", false); // U+010A GGML_ASSERT(!ids.empty() && "model vocab missing newline token"); vocab.linefeed_id = ids[0]; } @@ -20890,7 +20887,7 @@ int32_t llama_tokenize( int32_t n_tokens_max, bool add_special, bool parse_special) { - return llama_tokenize_impl(model->tokenizer, text, text_len, tokens, n_tokens_max, add_special, parse_special); + return llama_tokenize_impl(model->vocab, text, text_len, tokens, n_tokens_max, add_special, parse_special); } int32_t llama_token_to_piece(