root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2025-01-13 04:00:16 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

tool-call: test/fix functionary-medium-v3.1's template (can "look" like llama3.1 template)

Browse Source
This commit is contained in:
ochafik 2024-09-26 05:56:15 +01:00
parent 8e4a9bad8a
commit 0c870133d8
8 changed files with 116 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
common/tool-call.cpp
View File

@ -191,6 +191,16 @@ static llama_tool_calls parse_functionary_tool_calls(const std::string& input, c
}
static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) {
// This version of Functionary still supports the llama 3.1 tool call format for the python tool.
static std::regex python_tag_regex(R"(<\|python_tag\|>([\s\S\n]*)$)");
std::smatch match;
if (std::regex_search(input, match, python_tag_regex)) {
return {
match.prefix().str(), {
{"ipython", (json {{"code", match[1].str()}}).dump()},
}
};
}
static std::regex function_regex(R"(<function=(\w+)>)");
static std::regex close_regex(R"(</function>)");
return parse_functionary_tool_calls(input, function_regex, close_regex);
@ -205,12 +215,12 @@ static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input
llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) {
if (needs_hermes_pro_tool_call(chat_template)) {
return parse_hermes_tool_calls(input);
} else if (needs_llama_3_1_tool_call(chat_template)) {
return parse_llama_3_1_tool_calls(tools, input);
} else if (needs_functionary_v3_tool_call(chat_template)) {
return parse_functionary_v3_tool_calls(input);
} else if (needs_functionary_v3_llama_3_1_tool_call(chat_template)) {
return parse_functionary_v3_llama_3_1_tool_calls(input);
} else if (needs_llama_3_1_tool_call(chat_template)) {
return parse_llama_3_1_tool_calls(tools, input);
} else {
throw std::runtime_error("Unsupported chat template for tool calls");
}

18
examples/server/tests/features/tool_call.feature
View File

@ -12,17 +12,16 @@ Feature: llama.cpp server
And 8192 KV cache size
And 32 as batch size
And 2 slots
And 64 server max tokens to predict
And prometheus compatible metrics exposed
And jinja templates are enabled
@wip
Scenario Outline: OAI Compatibility w/ required tool
Given a chat template file ../../../tests/chat/templates/<template_name>.jinja
And the server is starting
And the server is healthy
And a model test
And <n> max tokens to predict
And <n_predict> max tokens to predict
And a user prompt write a hello world in python
And a tool choice <tool_choice>
And tools <tools>
@ -30,11 +29,14 @@ Feature: llama.cpp server
Then tool <tool_name> is called with arguments <tool_arguments>
Examples: Prompts
| template_name | n | tool_name | tool_arguments | tool_choice | tools |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meetkai-functionary-medium-v3.2 | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.2 | 64 | ipython | {"code": "Yes,"} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| template_name | n_predict | tool_name | tool_arguments | tool_choice | tools |
| meetkai-functionary-medium-v3.1 | 128 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "Yes, you can."} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meetkai-functionary-medium-v3.2 | 128 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meetkai-functionary-medium-v3.2 | 128 | ipython | {"code": "Yes,"} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] |
| meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] |
Scenario: OAI Compatibility w/ no tool
Given a chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja

11
tests/chat/goldens/meetkai-functionary-medium-v3.1-simple.txt Normal file
View File

@ -0,0 +1,11 @@
<|startoftext|><|start_header_id|>system<|end_header_id|>
Cutting Knowledge Date: December 2023
<|eot_id|><|start_header_id|>user<|end_header_id|>
What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|>

13
tests/chat/goldens/meetkai-functionary-medium-v3.1-system.txt Normal file
View File

@ -0,0 +1,13 @@
<|startoftext|><|start_header_id|>system<|end_header_id|>
Cutting Knowledge Date: December 2023
<|eot_id|><|start_header_id|>system<|end_header_id|>
You only tell the truth.<|eot_id|><|start_header_id|>user<|end_header_id|>
What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|>
llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|>

1
tests/chat/goldens/meetkai-functionary-medium-v3.1-tool_use.txt Normal file
View File

@ -0,0 +1 @@
ERROR: can only concatenate str (not "dict") to str

58
tests/chat/templates/meetkai-functionary-medium-v3.1.jinja Normal file
View File

@ -0,0 +1,58 @@
{# version=v3-llama3.1 #}{%- if not tools is defined -%}
{%- set tools = none -%}
{%- endif -%}
{%- set has_code_interpreter = tools | selectattr("type", "equalto", "code_interpreter") | list | length > 0 -%}
{%- if has_code_interpreter -%}
{%- set tools = tools | rejectattr("type", "equalto", "code_interpreter") | list -%}
{%- endif -%}
{#- System message + builtin tools #}
{{- bos_token + "<|start_header_id|>system<|end_header_id|>\n\n" }}
{%- if has_code_interpreter %}
{{- "Environment: ipython\n\n" }}
{%- else -%}
{{ "\n"}}
{%- endif %}
{{- "Cutting Knowledge Date: December 2023\n\n" }}
{%- if tools %}
{{- "\nYou have access to the following functions:\n\n" }}
{%- for t in tools %}
{%- if "type" in t -%}
{{ "Use the function '"|safe + t["function"]["name"] + "' to '"|safe + t["function"]["description"] + "'\n"|safe + t["function"] | tojson() }}
{%- else -%}
{{ "Use the function '"|safe + t["name"] + "' to '"|safe + t["description"] + "'\n"|safe + t | tojson() }}
{%- endif -%}
{{- "\n\n" }}
{%- endfor %}
{{- '\nThink very carefully before calling functions.\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => `<function`\nparameters => a JSON dict with the function argument name as key and function argument value as value.\nend_tag => `</function>`\n\nHere is an example,\n<function=example_function_name>{"example_name": "example_value"}</function>\n\nReminder:\n- If looking for real time information use relevant functions before falling back to brave_search\n- Function calls MUST follow the specified format, start with <function= and end with </function>\n- Required parameters MUST be specified\n- Only call one function at a time\n- Put the entire function call reply on one line\n\n' -}}
{%- endif %}
{{- "<|eot_id|>" -}}
{%- for message in messages -%}
{%- if message['role'] == 'user' or message['role'] == 'system' -%}
{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }}
{%- elif message['role'] == 'tool' -%}
{{ '<|start_header_id|>ipython<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }}
{%- else -%}
{{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'}}
{%- if message['content'] -%}
{{ message['content'] }}
{%- endif -%}
{%- if 'tool_calls' in message and message['tool_calls'] -%}
{%- for tool_call in message['tool_calls'] -%}
{%- if tool_call["function"]["name"] == "python" -%}
{{ '<|python_tag|>' + tool_call['function']['arguments'] }}
{%- else -%}
{{ '<function=' + tool_call['function']['name'] + '>' + tool_call['function']['arguments'] + '</function>' }}
{%- endif -%}
{%- endfor -%}
{{ '<|eom_id|>' }}
{%- else -%}
{{ '<|eot_id|>' }}
{%- endif -%}
{%- endif -%}
{%- endfor -%}
{%- if add_generation_prompt -%}
{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
{%- endif -%}

9
tests/test-tool-call.cpp
View File

@ -116,6 +116,15 @@ int main() {
}}
},
});
test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl,
"<function=test>{ } </function> ",
" ",
json {{
{"function", {
{"name", "test"},
{"arguments", "{}"}
}}
}});
std::string llama_3_1_like_tmpl = "Llama 3.1 template should have <|start_header_id|> and <|python_tag|> inside it";
test_parse_tool_call(tools, llama_3_1_like_tmpl,

2
tests/update_jinja_goldens.py
View File

@ -26,6 +26,7 @@ import jinja2.ext
import re
# import requests
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
model_ids = [
@ -33,6 +34,7 @@ model_ids = [
"NousResearch/Hermes-2-Pro-Llama-3-8B",
"NousResearch/Hermes-2-Pro-Mistral-7B",
"meetkai/functionary-medium-v3.2",
"meetkai/functionary-medium-v3.1",
"Qwen/Qwen2-7B-Instruct",
"Qwen/Qwen2-VL-7B-Instruct",
"Qwen/Qwen2.5-7B-Instruct",