root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-27 03:44:35 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity

llama : fix Viking pre-tokenizer regex

Browse Source 
The order was previously wrong, which caused errors in some tests.
This commit is contained in:
Francis Couture-Harpin 2024-07-08 16:34:39 -04:00
parent f9d42c598b
commit 31a1b0eeaa
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llama.cpp
View File

@ -15440,8 +15440,8 @@ struct llm_tokenizer_bpe {
break; break;
case LLAMA_VOCAB_PRE_TYPE_VIKING: case LLAMA_VOCAB_PRE_TYPE_VIKING:
regex_exprs = { regex_exprs = {
"\\p{N}",
" ?[^(\\s|.,!?…。，、।۔،)]+", " ?[^(\\s|.,!?…。，、।۔،)]+",
"\\p{N}",
}; };
break; break;
default: default: