root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-11 11:11:46 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

special test

Browse Source
This commit is contained in:
Howard Su 2023-07-03 09:51:04 +08:00
parent 6caa06638f
commit 751e51ceda
1 changed files with 8 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
tests/test-tokenizer-0.cpp
View File

@ -8,13 +8,14 @@
static const std::map<std::string, std::vector<llama_token>> & k_tests()
{
static std::map<std::string, std::vector<llama_token>> _k_tests = {
{ "Hello World", { 1, 10994, 2787, }, },
{ " Hello World", { 1, 15043, 2787, }, },
{ " Hello World!", { 1, 15043, 2787, 29991, }, },
{ " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, },
{ "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, },
{ "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, },
{ ">>>>ANSWER<<", { 1, 6778, 6778, 2190, 23066, 1001, 9314,}, },
// { "Hello World", { 1, 10994, 2787, }, },
// { " Hello World", { 1, 15043, 2787, }, },
// { " Hello World!", { 1, 15043, 2787, 29991, }, },
// { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, },
// { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, },
// { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, },
// { ">>>>ANSWER<<", { 1, 6778, 6778, 2190, 23066, 1001, 9314,}, },
{ "\xe2\x96\x81\xe2\x80\x93", { 1, 100, }, },
};
return _k_tests;
};