root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-26 11:24:35 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

llama : escape all U+2581 in a string (#2750)

Browse Source
This commit is contained in:
Georgi Gerganov 2023-08-24 12:26:01 +03:00 committed by GitHub
parent 6e91a1b070
commit c3e53b421a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
llama.cpp
View File

@ -3004,11 +3004,8 @@ static std::string llama_escape_whitespace(const std::string& text) {
return result; return result;
} }
static std::string llama_unescape_whitespace(const std::string& word) { static void llama_unescape_whitespace(std::string & word) {
if (word.length() >= 3 && word.substr(0, 3) == "\xe2\x96\x81") { replace_all(word, "\xe2\x96\x81", " ");
return std::string(" ") + word.substr(3);
}
return word;
} }
struct llm_symbol { struct llm_symbol {
@ -5822,7 +5819,7 @@ int llama_token_to_str_with_model(const struct llama_model * model, llama_token
if (llama_is_normal_token(model->vocab, token)) { if (llama_is_normal_token(model->vocab, token)) {
std::string result = model->vocab.id_to_token[token].text; std::string result = model->vocab.id_to_token[token].text;
if (llama_vocab_get_type(model->vocab) == LLAMA_VOCAB_TYPE_SPM) { if (llama_vocab_get_type(model->vocab) == LLAMA_VOCAB_TYPE_SPM) {
result = llama_unescape_whitespace(result); llama_unescape_whitespace(result);
} }
if (length < (int) result.length()) { if (length < (int) result.length()) {
return -result.length(); return -result.length();