chat-template: fix jinja tests (make safe a passthrough)

tests/chat/contexts/tool_use.json

@@ -21,7 +21,7 @@
     {
       "role": "tool",
       "name": "ipython",
-      "content": {"stdout": "Hello, World!"}
+      "content": "{\"stdout\": \"Hello, World!\"}"
     },
     {
       "role": "assistant",
@@ -48,7 +48,7 @@
     {
       "role": "tool",
       "name": "test",
-      "content": true
+      "content": "true"
     },
     {
       "role": "assistant",
@@ -75,7 +75,7 @@
     {
       "role": "tool",
       "name": "brave_search",
-      "content": {"title":"Truth: don't ask the web, ask an LLM instead!","url":"https://en.wikipedia.org/wiki/Truth"}
+      "content": "{\"title\":\"Truth: don't ask the web, ask an LLM instead!\",\"url\":\"https://en.wikipedia.org/wiki/Truth\"}"
     },
     {
       "role": "assistant",

tests/chat/goldens/NousResearch-Hermes-2-Pro-Llama-3-8B-tool_use-tool_use.txt

@@ -27,7 +27,7 @@ Print a hello world message with python.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-{'stdout': 'Hello, World!'}
+{"stdout": "Hello, World!"}
 </tool_response>
 <|im_end|><|im_start|>assistant
 Anything else?<|im_end|>
@@ -39,7 +39,7 @@ Test a tautology.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-True
+true
 </tool_response>
 <|im_end|><|im_start|>assistant
 Truth is definitely true.<|im_end|>
@@ -51,7 +51,7 @@ Check it on the web.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-{'title': "Truth: don't ask the web, ask an LLM instead!", 'url': 'https://en.wikipedia.org/wiki/Truth'}
+{"title":"Truth: don't ask the web, ask an LLM instead!","url":"https://en.wikipedia.org/wiki/Truth"}
 </tool_response>
 <|im_end|><|im_start|>assistant
 I don't need the web to answer you but I did check, as you asked. What now?<|im_end|>

tests/chat/goldens/NousResearch-Hermes-2-Pro-Mistral-7B-tool_use-tool_use.txt

@@ -27,7 +27,7 @@ Print a hello world message with python.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-{'stdout': 'Hello, World!'}
+{"stdout": "Hello, World!"}
 </tool_response>
 <|im_end|><|im_start|>assistant
 Anything else?<|im_end|>
@@ -39,7 +39,7 @@ Test a tautology.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-True
+true
 </tool_response>
 <|im_end|><|im_start|>assistant
 Truth is definitely true.<|im_end|>
@@ -51,7 +51,7 @@ Check it on the web.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-{'title': "Truth: don't ask the web, ask an LLM instead!", 'url': 'https://en.wikipedia.org/wiki/Truth'}
+{"title":"Truth: don't ask the web, ask an LLM instead!","url":"https://en.wikipedia.org/wiki/Truth"}
 </tool_response>
 <|im_end|><|im_start|>assistant
 I don't need the web to answer you but I did check, as you asked. What now?<|im_end|>

tests/chat/goldens/NousResearch-Hermes-3-Llama-3.1-70B-tool_use-tool_use.txt

@@ -27,7 +27,7 @@ Print a hello world message with python.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-{'stdout': 'Hello, World!'}
+{"stdout": "Hello, World!"}
 </tool_response>
 <|im_end|><|im_start|>assistant
 Anything else?<|im_end|>
@@ -39,7 +39,7 @@ Test a tautology.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-True
+true
 </tool_response>
 <|im_end|><|im_start|>assistant
 Truth is definitely true.<|im_end|>
@@ -51,7 +51,7 @@ Check it on the web.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>tool
 <tool_response>
-{'title': "Truth: don't ask the web, ask an LLM instead!", 'url': 'https://en.wikipedia.org/wiki/Truth'}
+{"title":"Truth: don't ask the web, ask an LLM instead!","url":"https://en.wikipedia.org/wiki/Truth"}
 </tool_response>
 <|im_end|><|im_start|>assistant
 I don't need the web to answer you but I did check, as you asked. What now?<|im_end|>

tests/chat/goldens/Qwen-Qwen2.5-7B-Instruct-tool_use.txt

@@ -25,7 +25,7 @@ Print a hello world message with python.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>user
 <tool_response>
-{'stdout': 'Hello, World!'}
+{"stdout": "Hello, World!"}
 </tool_response><|im_end|>
 <|im_start|>assistant
 Anything else?<|im_end|>
@@ -37,7 +37,7 @@ Test a tautology.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>user
 <tool_response>
-True
+true
 </tool_response><|im_end|>
 <|im_start|>assistant
 Truth is definitely true.<|im_end|>
@@ -49,7 +49,7 @@ Check it on the web.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>user
 <tool_response>
-{'title': "Truth: don't ask the web, ask an LLM instead!", 'url': 'https://en.wikipedia.org/wiki/Truth'}
+{"title":"Truth: don't ask the web, ask an LLM instead!","url":"https://en.wikipedia.org/wiki/Truth"}
 </tool_response><|im_end|>
 <|im_start|>assistant
 I don't need the web to answer you but I did check, as you asked. What now?<|im_end|>

tests/chat/goldens/Qwen-Qwen2.5-Math-7B-Instruct-tool_use.txt

@@ -25,7 +25,7 @@ Print a hello world message with python.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>user
 <tool_response>
-{'stdout': 'Hello, World!'}
+{"stdout": "Hello, World!"}
 </tool_response><|im_end|>
 <|im_start|>assistant
 Anything else?<|im_end|>
@@ -37,7 +37,7 @@ Test a tautology.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>user
 <tool_response>
-True
+true
 </tool_response><|im_end|>
 <|im_start|>assistant
 Truth is definitely true.<|im_end|>
@@ -49,7 +49,7 @@ Check it on the web.<|im_end|>
 </tool_call><|im_end|>
 <|im_start|>user
 <tool_response>
-{'title': "Truth: don't ask the web, ask an LLM instead!", 'url': 'https://en.wikipedia.org/wiki/Truth'}
+{"title":"Truth: don't ask the web, ask an LLM instead!","url":"https://en.wikipedia.org/wiki/Truth"}
 </tool_response><|im_end|>
 <|im_start|>assistant
 I don't need the web to answer you but I did check, as you asked. What now?<|im_end|>

tests/chat/goldens/deepseek-ai-deepseek-coder-33b-instruct-simple.txt

@@ -0,0 +1,7 @@
+<|startoftext|>You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer
+### Instruction:
+What's your favourite LLM framework?
+### Response:
+llama.cpp!
+<|EOT|>
+### Response:

tests/chat/goldens/deepseek-ai-deepseek-coder-33b-instruct-system.txt

@@ -0,0 +1,6 @@
+<|startoftext|>You only tell the truth.### Instruction:
+What's your favourite LLM framework?
+### Response:
+llama.cpp!
+<|EOT|>
+### Response:

tests/chat/goldens/meetkai-functionary-medium-v3.1-simple.txt

@@ -0,0 +1,11 @@
+<|startoftext|><|start_header_id|>system<|end_header_id|>
+
+
+Cutting Knowledge Date: December 2023
+
+<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+

tests/chat/goldens/meetkai-functionary-medium-v3.1-system.txt

@@ -0,0 +1,13 @@
+<|startoftext|><|start_header_id|>system<|end_header_id|>
+
+
+Cutting Knowledge Date: December 2023
+
+<|eot_id|><|start_header_id|>system<|end_header_id|>
+
+You only tell the truth.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+

tests/chat/goldens/meetkai-functionary-medium-v3.1-tool_use.txt

@@ -0,0 +1,66 @@
+<|startoftext|><|start_header_id|>system<|end_header_id|>
+
+
+Cutting Knowledge Date: December 2023
+
+
+You have access to the following functions:
+
+Use the function 'ipython' to 'Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.'
+{"name": "ipython", "description": "Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": "The code to run in the ipython interpreter."}}, "required": ["code"]}}
+
+Use the function 'brave_search' to 'Executes a web search with Brave.'
+{"name": "brave_search", "description": "Executes a web search with Brave.", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "The query to search for."}}, "required": ["query"]}}
+
+Use the function 'wolfram_alpha' to 'Executes a query with Wolfram Alpha.'
+{"name": "wolfram_alpha", "description": "Executes a query with Wolfram Alpha.", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "The query to execute."}}, "required": ["query"]}}
+
+Use the function 'test' to 'Runs a test.'
+{"name": "test", "description": "Runs a test.", "parameters": {"type": "object", "properties": {"condition": {"type": "boolean", "description": "The condition to test."}}, "required": ["condition"]}}
+
+
+Think very carefully before calling functions.
+If a you choose to call a function ONLY reply in the following format:
+<{start_tag}={function_name}>{parameters}{end_tag}
+where
+
+start_tag => `<function`
+parameters => a JSON dict with the function argument name as key and function argument value as value.
+end_tag => `</function>`
+
+Here is an example,
+<function=example_function_name>{"example_name": "example_value"}</function>
+
+Reminder:
+- If looking for real time information use relevant functions before falling back to brave_search
+- Function calls MUST follow the specified format, start with <function= and end with </function>
+- Required parameters MUST be specified
+- Only call one function at a time
+- Put the entire function call reply on one line
+
+<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Print a hello world message with python.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+<function=ipython>{"code": "print('Hello, World!')"}</function><|eom_id|><|start_header_id|>ipython<|end_header_id|>
+
+{"stdout": "Hello, World!"}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+Anything else?<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Test a tautology.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+<function=test>{"condition":true}</function><|eom_id|><|start_header_id|>ipython<|end_header_id|>
+
+true<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+Truth is definitely true.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Check it on the web.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+<function=brave_search>{"query": "what is truth anyway am I right?"}</function><|eom_id|><|start_header_id|>ipython<|end_header_id|>
+
+{"title":"Truth: don't ask the web, ask an LLM instead!","url":"https://en.wikipedia.org/wiki/Truth"}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+I don't need the web to answer you but I did check, as you asked. What now?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+

tests/chat/goldens/meetkai-functionary-medium-v3.2-simple.txt

@@ -0,0 +1,21 @@
+<|startoftext|><|start_header_id|>system<|end_header_id|>
+
+You are capable of executing available function(s) if required.
+Only execute function(s) when absolutely necessary.
+Ask for the required input to:recipient==all
+Use JSON for function arguments.
+Respond in this format:
+>>>${recipient}
+${content}
+Available functions:
+// Supported function definitions that should be called when necessary.
+namespace functions {
+
+} // namespace functions<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>all
+llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>

tests/chat/goldens/meetkai-functionary-medium-v3.2-system.txt

@@ -0,0 +1,23 @@
+<|startoftext|><|start_header_id|>system<|end_header_id|>
+
+You are capable of executing available function(s) if required.
+Only execute function(s) when absolutely necessary.
+Ask for the required input to:recipient==all
+Use JSON for function arguments.
+Respond in this format:
+>>>${recipient}
+${content}
+Available functions:
+// Supported function definitions that should be called when necessary.
+namespace functions {
+
+} // namespace functions<|eot_id|><|start_header_id|>system<|end_header_id|>
+
+You only tell the truth.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>all
+llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>

tests/chat/goldens/meetkai-functionary-medium-v3.2-tool_use.txt

@@ -0,0 +1,70 @@
+<|startoftext|><|start_header_id|>system<|end_header_id|>
+
+You are capable of executing available function(s) if required.
+Only execute function(s) when absolutely necessary.
+Ask for the required input to:recipient==all
+Use JSON for function arguments.
+Respond in this format:
+>>>${recipient}
+${content}
+Available functions:
+// Supported function definitions that should be called when necessary.
+namespace functions {
+
+// Runs code in an ipython interpreter and returns the result of the execution after 60 seconds.
+type ipython = (_: {
+// The code to run in the ipython interpreter.
+code: string,
+}) => any;
+
+// Executes a web search with Brave.
+type brave_search = (_: {
+// The query to search for.
+query: string,
+}) => any;
+
+// Executes a query with Wolfram Alpha.
+type wolfram_alpha = (_: {
+// The query to execute.
+query: string,
+}) => any;
+
+// Runs a test.
+type test = (_: {
+// The condition to test.
+condition: boolean,
+}) => any;
+
+} // namespace functions<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Print a hello world message with python.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>ipython
+{"code": "print('Hello, World!')"}<|eot_id|><|start_header_id|>tool<|end_header_id|>
+
+{"stdout": "Hello, World!"}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>all
+Anything else?<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Test a tautology.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>test
+{"condition":true}<|eot_id|><|start_header_id|>tool<|end_header_id|>
+
+true<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>all
+Truth is definitely true.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
+Check it on the web.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>brave_search
+{"query": "what is truth anyway am I right?"}<|eot_id|><|start_header_id|>tool<|end_header_id|>
+
+{"title":"Truth: don't ask the web, ask an LLM instead!","url":"https://en.wikipedia.org/wiki/Truth"}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>all
+I don't need the web to answer you but I did check, as you asked. What now?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
+>>>

tests/chat/goldens/meta-llama-Meta-Llama-3.1-8B-Instruct-tool_use.txt

@@ -96,7 +96,7 @@ Print a hello world message with python.<|eot_id|><|start_header_id|>assistant<|
 
 {"name": "ipython", "parameters": {"code": "print('Hello, World!')"}}<|eom_id|><|start_header_id|>ipython<|end_header_id|>
 
-{"stdout": "Hello, World!"}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+"{\"stdout\": \"Hello, World!\"}"<|eot_id|><|start_header_id|>assistant<|end_header_id|>
 
 Anything else?<|eot_id|><|start_header_id|>user<|end_header_id|>
 
@@ -104,7 +104,7 @@ Test a tautology.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
 
 {"name": "test", "parameters": {"condition": true}}<|eom_id|><|start_header_id|>ipython<|end_header_id|>
 
-True<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+"true"<|eot_id|><|start_header_id|>assistant<|end_header_id|>
 
 Truth is definitely true.<|eot_id|><|start_header_id|>user<|end_header_id|>
 
@@ -112,7 +112,7 @@ Check it on the web.<|eot_id|><|start_header_id|>assistant<|end_header_id|>
 
 <|python_tag|>brave_search.call(query="what is truth anyway am I right?")<|eom_id|><|start_header_id|>ipython<|end_header_id|>
 
-{"title": "Truth: don't ask the web, ask an LLM instead!", "url": "https://en.wikipedia.org/wiki/Truth"}<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+"{\"title\":\"Truth: don't ask the web, ask an LLM instead!\",\"url\":\"https://en.wikipedia.org/wiki/Truth\"}"<|eot_id|><|start_header_id|>assistant<|end_header_id|>
 
 I don't need the web to answer you but I did check, as you asked. What now?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
 

tests/chat/templates/deepseek-ai-deepseek-coder-33b-instruct.jinja

@@ -0,0 +1,26 @@
+{% if not add_generation_prompt is defined %}
+{% set add_generation_prompt = false %}
+{% endif %}
+{%- set ns = namespace(found=false) -%}
+{%- for message in messages -%}
+    {%- if message['role'] == 'system' -%}
+        {%- set ns.found = true -%}
+    {%- endif -%}
+{%- endfor -%}
+{{bos_token}}{%- if not ns.found -%}
+{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\n'}}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message['role'] == 'system' %}
+{{ message['content'] }}
+    {%- else %}
+        {%- if message['role'] == 'user' %}
+{{'### Instruction:\n' + message['content'] + '\n'}}
+        {%- else %}
+{{'### Response:\n' + message['content'] + '\n<|EOT|>\n'}}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{% if add_generation_prompt %}
+{{'### Response:'}}
+{% endif %}

tests/update_jinja_goldens.py

@@ -34,6 +34,8 @@ model_ids = [
     "bofenghuang/vigogne-2-70b-chat",
     "deepseek-ai/deepseek-coder-33b-instruct",
     "indischepartij/MiniCPM-3B-OpenHermes-2.5-v2",
+    "meetkai/functionary-medium-v3.2",
+    "meetkai/functionary-medium-v3.1",
     "microsoft/Phi-3-medium-4k-instruct",
     "microsoft/Phi-3-mini-4k-instruct",
     "microsoft/Phi-3-small-8k-instruct",
@@ -51,10 +53,6 @@ model_ids = [
     "teknium/OpenHermes-2.5-Mistral-7B",
     "TheBloke/FusionNet_34Bx2_MoE-AWQ",
 
-    # Python update goldens broken:
-    "meetkai/functionary-medium-v3.2",
-    "meetkai/functionary-medium-v3.1",
-
     # C++ minja templating broken:
     # "CohereForAI/c4ai-command-r-plus",
     # "THUDM/chatglm3-6b",
@@ -106,6 +104,7 @@ def handle_chat_template(model_id, variant, template_src):
         extensions=[
             jinja2.ext.loopcontrols
         ])
+    env.filters['safe'] = lambda x: x
     env.filters['tojson'] = tojson
     env.globals['raise_exception'] = raise_exception
     env.globals['strftime_now'] = strftime_now