From c77fabb1f9b77d6665065da494cb918af71c1dc0 Mon Sep 17 00:00:00 2001
From: klosax <131523366+klosax@users.noreply.github.com>
Date: Tue, 1 Aug 2023 14:32:53 +0200
Subject: [PATCH] gptneox-main.cpp : special tokens

---
 gptneox-main.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/gptneox-main.cpp b/gptneox-main.cpp
index 02fbc9fba..fa95cb772 100644
--- a/gptneox-main.cpp
+++ b/gptneox-main.cpp
@@ -257,6 +257,11 @@ bool gpt_neox_model_load(const std::string & fname, gpt_neox_model & model, gpt_
             vocab.id_to_token[i] = word;
         }
 
+        keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.bos_token_id"); if( keyidx != -1 ) {       printf("bos id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); }
+        keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.eos_token_id"); if( keyidx != -1 ) {       printf("eos id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); }
+        keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.unknown_token_id"); if( keyidx != -1 ) {   printf("unk id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); }
+        keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.separator_token_id"); if( keyidx != -1 ) { printf("sep id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); }
+        keyidx = gguf_find_key(ggufctx, "tokenizer.ggml.padding_token_id"); if( keyidx != -1 ) {   printf("pad id = %d\n", gguf_get_val_u32(ggufctx, keyidx) ); }
 
     }