root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-11 19:21:46 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

More tests

Browse Source
This commit is contained in:
Howard Su 2023-07-04 18:15:25 +08:00
parent 751e51ceda
commit ca150b7725
1 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tests/test-tokenizer-0.cpp
View File

@ -8,14 +8,14 @@
static const std::map<std::string, std::vector<llama_token>> & k_tests() static const std::map<std::string, std::vector<llama_token>> & k_tests()
{ {
static std::map<std::string, std::vector<llama_token>> _k_tests = { static std::map<std::string, std::vector<llama_token>> _k_tests = {
// { "Hello World", { 1, 10994, 2787, }, }, { "Hello World", { 1, 10994, 2787, }, },
// { " Hello World", { 1, 15043, 2787, }, }, { " Hello World", { 1, 15043, 2787, }, },
// { " Hello World!", { 1, 15043, 2787, 29991, }, }, { " Hello World!", { 1, 15043, 2787, 29991, }, },
// { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, }, { " this is 🦙.cpp", { 1, 445, 338, 29871, 243, 162, 169, 156, 29889, 8223, }, },
// { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, }, { "w048 7tuijk dsdfhu", { 1, 29893, 29900, 29946, 29947, 29871, 29955, 9161, 13535, 18031, 2176, 6905, }, },
// { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, }, { "нещо на Български", { 1, 821, 4851, 665, 1386, 29713, 1305, }, },
// { ">>>>ANSWER<<", { 1, 6778, 6778, 2190, 23066, 1001, 9314,}, }, { "\xe6\x88\x91\xe4\xbb\xac\xe5\xa4\xa7\xe5\xae\xb6\xe4\xb8\x80\xe8\xb5\xb7", { 1, 30672, 31381, 30257, 30613, 30287, 31558, }, },
{ "\xe2\x96\x81\xe2\x80\x93", { 1, 100, }, }, { ">>>>ANSWER<<", {1, 5099, 6778, 2190, 23066, 1001, 9314}, },
}; };
return _k_tests; return _k_tests;
}; };
@ -84,11 +84,19 @@ int main(int argc, char **argv) {
fprintf(stderr, "%6d, ", t); fprintf(stderr, "%6d, ", t);
} }
fprintf(stderr, "\n"); fprintf(stderr, "\n");
for (const auto & t : test_kv.second) {
fprintf(stderr, "%7s ", llama_token_to_str(ctx, t));
}
fprintf(stderr, "\n");
fprintf(stderr, "%s : got tokens: ", __func__); fprintf(stderr, "%s : got tokens: ", __func__);
for (const auto & t : res) { for (const auto & t : res) {
fprintf(stderr, "%6d, ", t); fprintf(stderr, "%6d, ", t);
} }
fprintf(stderr, "\n"); fprintf(stderr, "\n");
for (const auto & t : res) {
fprintf(stderr, "%7s ", llama_token_to_str(ctx, t));
}
fprintf(stderr, "\n");
llama_free_model(model); llama_free_model(model);
llama_free(ctx); llama_free(ctx);