diff --git a/common/tool-call.cpp b/common/tool-call.cpp index ca25b8038..ea7753b4e 100644 --- a/common/tool-call.cpp +++ b/common/tool-call.cpp @@ -191,6 +191,16 @@ static llama_tool_calls parse_functionary_tool_calls(const std::string& input, c } static llama_tool_calls parse_functionary_v3_llama_3_1_tool_calls(const std::string& input) { + // This version of Functionary still supports the llama 3.1 tool call format for the python tool. + static std::regex python_tag_regex(R"(<\|python_tag\|>([\s\S\n]*)$)"); + std::smatch match; + if (std::regex_search(input, match, python_tag_regex)) { + return { + match.prefix().str(), { + {"ipython", (json {{"code", match[1].str()}}).dump()}, + } + }; + } static std::regex function_regex(R"()"); static std::regex close_regex(R"()"); return parse_functionary_tool_calls(input, function_regex, close_regex); @@ -205,12 +215,12 @@ static llama_tool_calls parse_functionary_v3_tool_calls(const std::string& input llama_tool_calls parse_tool_calls(const json & tools, const std::string & chat_template, const std::string& input) { if (needs_hermes_pro_tool_call(chat_template)) { return parse_hermes_tool_calls(input); - } else if (needs_llama_3_1_tool_call(chat_template)) { - return parse_llama_3_1_tool_calls(tools, input); } else if (needs_functionary_v3_tool_call(chat_template)) { return parse_functionary_v3_tool_calls(input); } else if (needs_functionary_v3_llama_3_1_tool_call(chat_template)) { return parse_functionary_v3_llama_3_1_tool_calls(input); + } else if (needs_llama_3_1_tool_call(chat_template)) { + return parse_llama_3_1_tool_calls(tools, input); } else { throw std::runtime_error("Unsupported chat template for tool calls"); } diff --git a/examples/server/tests/features/tool_call.feature b/examples/server/tests/features/tool_call.feature index 81c427bdb..4991ed7b3 100644 --- a/examples/server/tests/features/tool_call.feature +++ b/examples/server/tests/features/tool_call.feature @@ -12,17 +12,16 @@ Feature: llama.cpp server And 8192 KV cache size And 32 as batch size And 2 slots - And 64 server max tokens to predict And prometheus compatible metrics exposed And jinja templates are enabled - @wip + Scenario Outline: OAI Compatibility w/ required tool Given a chat template file ../../../tests/chat/templates/.jinja And the server is starting And the server is healthy And a model test - And max tokens to predict + And max tokens to predict And a user prompt write a hello world in python And a tool choice And tools @@ -30,11 +29,14 @@ Feature: llama.cpp server Then tool is called with arguments Examples: Prompts - | template_name | n | tool_name | tool_arguments | tool_choice | tools | - | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | - | meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | - | meetkai-functionary-medium-v3.2 | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | - | meetkai-functionary-medium-v3.2 | 64 | ipython | {"code": "Yes,"} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | + | template_name | n_predict | tool_name | tool_arguments | tool_choice | tools | + | meetkai-functionary-medium-v3.1 | 128 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | + | meetkai-functionary-medium-v3.1 | 128 | ipython | {"code": "Yes, you can."} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | + | meetkai-functionary-medium-v3.2 | 128 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | + | meetkai-functionary-medium-v3.2 | 128 | ipython | {"code": "Yes,"} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | + | meta-llama-Meta-Llama-3.1-8B-Instruct | 64 | test | {} | required | [{"type":"function", "function": {"name": "test", "description": "", "parameters": {"type": "object", "properties": {}}}}] | + | meta-llama-Meta-Llama-3.1-8B-Instruct | 16 | ipython | {"code": "it and "} | required | [{"type":"function", "function": {"name": "ipython", "description": "", "parameters": {"type": "object", "properties": {"code": {"type": "string", "description": ""}}, "required": ["code"]}}}] | + Scenario: OAI Compatibility w/ no tool Given a chat template file ../../../tests/chat/templates/meta-llama-Meta-Llama-3.1-8B-Instruct.jinja diff --git a/tests/chat/goldens/meetkai-functionary-medium-v3.1-simple.txt b/tests/chat/goldens/meetkai-functionary-medium-v3.1-simple.txt new file mode 100644 index 000000000..415215244 --- /dev/null +++ b/tests/chat/goldens/meetkai-functionary-medium-v3.1-simple.txt @@ -0,0 +1,11 @@ +<|startoftext|><|start_header_id|>system<|end_header_id|> + + +Cutting Knowledge Date: December 2023 + +<|eot_id|><|start_header_id|>user<|end_header_id|> + +What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|> + diff --git a/tests/chat/goldens/meetkai-functionary-medium-v3.1-system.txt b/tests/chat/goldens/meetkai-functionary-medium-v3.1-system.txt new file mode 100644 index 000000000..3239384b6 --- /dev/null +++ b/tests/chat/goldens/meetkai-functionary-medium-v3.1-system.txt @@ -0,0 +1,13 @@ +<|startoftext|><|start_header_id|>system<|end_header_id|> + + +Cutting Knowledge Date: December 2023 + +<|eot_id|><|start_header_id|>system<|end_header_id|> + +You only tell the truth.<|eot_id|><|start_header_id|>user<|end_header_id|> + +What's your favourite LLM framework?<|eot_id|><|start_header_id|>assistant<|end_header_id|> + +llama.cpp!<|eot_id|><|start_header_id|>assistant<|end_header_id|> + diff --git a/tests/chat/goldens/meetkai-functionary-medium-v3.1-tool_use.txt b/tests/chat/goldens/meetkai-functionary-medium-v3.1-tool_use.txt new file mode 100644 index 000000000..2cc3c7a8e --- /dev/null +++ b/tests/chat/goldens/meetkai-functionary-medium-v3.1-tool_use.txt @@ -0,0 +1 @@ +ERROR: can only concatenate str (not "dict") to str \ No newline at end of file diff --git a/tests/chat/templates/meetkai-functionary-medium-v3.1.jinja b/tests/chat/templates/meetkai-functionary-medium-v3.1.jinja new file mode 100644 index 000000000..29d64a215 --- /dev/null +++ b/tests/chat/templates/meetkai-functionary-medium-v3.1.jinja @@ -0,0 +1,58 @@ +{# version=v3-llama3.1 #}{%- if not tools is defined -%} + {%- set tools = none -%} +{%- endif -%} + +{%- set has_code_interpreter = tools | selectattr("type", "equalto", "code_interpreter") | list | length > 0 -%} +{%- if has_code_interpreter -%} + {%- set tools = tools | rejectattr("type", "equalto", "code_interpreter") | list -%} +{%- endif -%} + +{#- System message + builtin tools #} +{{- bos_token + "<|start_header_id|>system<|end_header_id|>\n\n" }} +{%- if has_code_interpreter %} + {{- "Environment: ipython\n\n" }} +{%- else -%} + {{ "\n"}} +{%- endif %} +{{- "Cutting Knowledge Date: December 2023\n\n" }} +{%- if tools %} + {{- "\nYou have access to the following functions:\n\n" }} + {%- for t in tools %} + {%- if "type" in t -%} + {{ "Use the function '"|safe + t["function"]["name"] + "' to '"|safe + t["function"]["description"] + "'\n"|safe + t["function"] | tojson() }} + {%- else -%} + {{ "Use the function '"|safe + t["name"] + "' to '"|safe + t["description"] + "'\n"|safe + t | tojson() }} + {%- endif -%} + {{- "\n\n" }} + {%- endfor %} + {{- '\nThink very carefully before calling functions.\nIf a you choose to call a function ONLY reply in the following format:\n<{start_tag}={function_name}>{parameters}{end_tag}\nwhere\n\nstart_tag => ` a JSON dict with the function argument name as key and function argument value as value.\nend_tag => ``\n\nHere is an example,\n{"example_name": "example_value"}\n\nReminder:\n- If looking for real time information use relevant functions before falling back to brave_search\n- Function calls MUST follow the specified format, start with \n- Required parameters MUST be specified\n- Only call one function at a time\n- Put the entire function call reply on one line\n\n' -}} +{%- endif %} +{{- "<|eot_id|>" -}} + +{%- for message in messages -%} + {%- if message['role'] == 'user' or message['role'] == 'system' -%} + {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }} + {%- elif message['role'] == 'tool' -%} + {{ '<|start_header_id|>ipython<|end_header_id|>\n\n' + message['content'] + '<|eot_id|>' }} + {%- else -%} + {{ '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'}} + {%- if message['content'] -%} + {{ message['content'] }} + {%- endif -%} + {%- if 'tool_calls' in message and message['tool_calls'] -%} + {%- for tool_call in message['tool_calls'] -%} + {%- if tool_call["function"]["name"] == "python" -%} + {{ '<|python_tag|>' + tool_call['function']['arguments'] }} + {%- else -%} + {{ '' + tool_call['function']['arguments'] + '' }} + {%- endif -%} + {%- endfor -%} + {{ '<|eom_id|>' }} + {%- else -%} + {{ '<|eot_id|>' }} + {%- endif -%} + {%- endif -%} +{%- endfor -%} +{%- if add_generation_prompt -%} + {{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }} +{%- endif -%} \ No newline at end of file diff --git a/tests/test-tool-call.cpp b/tests/test-tool-call.cpp index b43aca067..a454780e1 100644 --- a/tests/test-tool-call.cpp +++ b/tests/test-tool-call.cpp @@ -116,6 +116,15 @@ int main() { }} }, }); + test_parse_tool_call(tools, functionary_v3_llama_3_1_like_tmpl, + "{ } ", + " ", + json {{ + {"function", { + {"name", "test"}, + {"arguments", "{}"} + }} + }}); std::string llama_3_1_like_tmpl = "Llama 3.1 template should have <|start_header_id|> and <|python_tag|> inside it"; test_parse_tool_call(tools, llama_3_1_like_tmpl, diff --git a/tests/update_jinja_goldens.py b/tests/update_jinja_goldens.py index f5ffc851d..5c9302690 100644 --- a/tests/update_jinja_goldens.py +++ b/tests/update_jinja_goldens.py @@ -26,6 +26,7 @@ import jinja2.ext import re # import requests +logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) model_ids = [ @@ -33,6 +34,7 @@ model_ids = [ "NousResearch/Hermes-2-Pro-Llama-3-8B", "NousResearch/Hermes-2-Pro-Mistral-7B", "meetkai/functionary-medium-v3.2", + "meetkai/functionary-medium-v3.1", "Qwen/Qwen2-7B-Instruct", "Qwen/Qwen2-VL-7B-Instruct", "Qwen/Qwen2.5-7B-Instruct",