root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-25 02:44:36 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Revert "main : alternative instruct mode (Vicuna support, etc.) (#863)" (#982)

Browse Source 
This reverts commit f4d277ae17.
This commit is contained in:
Pavol Rusnak 2023-04-14 21:58:43 +02:00 committed by GitHub
parent 489093548c
commit c85e03d12e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 121 additions and 445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
configs/alpaca-native-enhanced.txt
View File

@ -1,21 +0,0 @@
--ctx_size 2048
--batch_size 16
--repeat_penalty 1.15
--temp 0.4
--top_k 30
--top_p 0.18
--interactive-first
--keep -1
--ins-prefix-bos
--ins-prefix "\n\nUser: "
--ins-suffix "\n\nAssistant: "
--reverse-prompt "User: "
-p "You are an AI language model designed to assist the User by answering their questions, offering advice, and engaging in casual conversation in a friendly, helpful, and informative manner. You respond clearly, coherently, and you consider the conversation history.
User: Hey, how's it going?
Assistant: Hey there! I'm doing great, thank you. What can I help you with today? Let's have a fun chat!"

9
configs/alpaca.txt
View File

@ -1,9 +0,0 @@
--clean-interface
--interactive-first
--keep -1
--ins-prefix-bos
--ins-prefix "\n\n### Instruction:\n\n"
--ins-suffix "\n\n### Response:\n\n"
--reverse-prompt "### Instruction:\n\n"
-p "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n"

15
configs/chat-with-bob.txt
View File

@ -1,15 +0,0 @@
--interactive-first
--keep -1
--ins-prefix-bos
--ins-prefix "\nUser: "
--ins-suffix "\nBob: "
--reverse-prompt "User: "
--rm-trailing-space-workaround
-p "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
User: Hello, Bob.
Bob: Hello. How may I help you today?
User: Please tell me the largest city in Europe.
Bob: Sure. The largest city in Europe is Moscow, the capital of Russia."

3
configs/llama.txt
View File

@ -1,3 +0,0 @@
--interactive-first
--keep -1
--temp 0.1

7
configs/vicuna-simple.txt
View File

@ -1,7 +0,0 @@
--interactive-first
--keep -1
--ins-prefix-bos
--ins-prefix "\n### Human: "
--ins-suffix "\n### Assistant: "
--reverse-prompt "### Human: "
--rm-trailing-space-workaround

8
configs/vicuna-stop.txt
View File

@ -1,8 +0,0 @@
--interactive-first
--keep -1
--ins-prefix-bos
--ins-prefix "\n### Human: "
--ins-suffix "\n### Assistant: "
--reverse-prompt "### Human: "
--stop-prompt "### Assistant: "
--rm-trailing-space-workaround

9
configs/vicuna.txt
View File

@ -1,9 +0,0 @@
--interactive-first
--keep -1
--ins-prefix-bos
--ins-prefix "\n### Human: "
--ins-suffix "\n### Assistant: "
--reverse-prompt "### Human: "
--rm-trailing-space-workaround
-p "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."

284
examples/common.cpp
View File

@ -2,13 +2,10 @@
#include <cassert> #include <cassert>
#include <cstring> #include <cstring>
#include <iostream>
#include <fstream> #include <fstream>
#include <sstream>
#include <string> #include <string>
#include <iterator> #include <iterator>
#include <algorithm> #include <algorithm>
#include <regex>
#if defined (_WIN32) #if defined (_WIN32)
#include <fcntl.h> #include <fcntl.h>
@ -26,43 +23,6 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int
#define CP_UTF8 65001 #define CP_UTF8 65001
#endif #endif
void split_args(const std::string & args_string, std::vector<std::string> & output_args)
{
std::string current_arg = "";
bool in_quotes = false;
char quote_type;
for (char c : args_string) {
if (c == '"' || c == '\'') {
if (!in_quotes) {
in_quotes = true;
quote_type = c;
} else if (quote_type == c) {
in_quotes = false;
} else {
current_arg += c;
}
} else if (in_quotes) {
current_arg += c;
} else if (std::isspace(c)) {
if (current_arg != "") {
output_args.push_back(current_arg);
current_arg = "";
}
} else {
current_arg += c;
}
}
if (current_arg != "") {
output_args.push_back(current_arg);
}
}
std::string unescape(const std::string & str) {
return std::regex_replace(str, std::regex("\\\\n"), "\n");
}
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) { bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
// determine sensible default number of threads. // determine sensible default number of threads.
// std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0. // std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0.
@ -80,11 +40,28 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
std::string arg; std::string arg;
gpt_params default_params; gpt_params default_params;
// get additional arguments from config files
std::vector<std::string> args;
for (int i = 1; i < argc; i++) { for (int i = 1; i < argc; i++) {
arg = argv[i]; arg = argv[i];
if (arg == "--config") {
if (arg == "-s" || arg == "--seed") {
if (++i >= argc) {
invalid_param = true;
break;
}
params.seed = std::stoi(argv[i]);
} else if (arg == "-t" || arg == "--threads") {
if (++i >= argc) {
invalid_param = true;
break;
}
params.n_threads = std::stoi(argv[i]);
} else if (arg == "-p" || arg == "--prompt") {
if (++i >= argc) {
invalid_param = true;
break;
}
params.prompt = argv[i];
} else if (arg == "-f" || arg == "--file") {
if (++i >= argc) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
@ -95,153 +72,85 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
invalid_param = true; invalid_param = true;
break; break;
} }
std::string args_string;
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(args_string));
if (args_string.back() == '\n') {
args_string.pop_back();
}
split_args(args_string, args);
for (int j = 0; j < args.size(); j++) {
args[j] = unescape(args[j]);
}
} else {
args.emplace_back(argv[i]);
}
}
// parse args
int args_c = static_cast<int>(args.size());
for (int i = 0; i < args_c && !invalid_param; i++) {
arg = args[i];
if (arg == "-s" || arg == "--seed") {
if (++i >= args_c) {
invalid_param = true;
break;
}
params.seed = std::stoi(args[i]);
} else if (arg == "-t" || arg == "--threads") {
if (++i >= args_c) {
invalid_param = true;
break;
}
params.n_threads = std::stoi(args[i]);
} else if (arg == "-p" || arg == "--prompt") {
if (++i >= args_c) {
invalid_param = true;
break;
}
params.prompt = args[i];
} else if (arg == "-f" || arg == "--file") {
if (++i >= args_c) {
invalid_param = true;
break;
}
std::ifstream file(args[i]);
if (!file) {
fprintf(stderr, "error: failed to open file '%s'\n", args[i].c_str());
invalid_param = true;
break;
}
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt)); std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
if (params.prompt.back() == '\n') { if (params.prompt.back() == '\n') {
params.prompt.pop_back(); params.prompt.pop_back();
} }
} else if (arg == "-n" || arg == "--n_predict") { } else if (arg == "-n" || arg == "--n_predict") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.n_predict = std::stoi(args[i]); params.n_predict = std::stoi(argv[i]);
} else if (arg == "--top_k") { } else if (arg == "--top_k") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.top_k = std::stoi(args[i]); params.top_k = std::stoi(argv[i]);
} else if (arg == "-c" || arg == "--ctx_size") { } else if (arg == "-c" || arg == "--ctx_size") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.n_ctx = std::stoi(args[i]); params.n_ctx = std::stoi(argv[i]);
} else if (arg == "--memory_f32") { } else if (arg == "--memory_f32") {
params.memory_f16 = false; params.memory_f16 = false;
} else if (arg == "--top_p") { } else if (arg == "--top_p") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.top_p = std::stof(args[i]); params.top_p = std::stof(argv[i]);
} else if (arg == "--temp") { } else if (arg == "--temp") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.temp = std::stof(args[i]); params.temp = std::stof(argv[i]);
} else if (arg == "--repeat_last_n") { } else if (arg == "--repeat_last_n") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.repeat_last_n = std::stoi(args[i]); params.repeat_last_n = std::stoi(argv[i]);
} else if (arg == "--repeat_penalty") { } else if (arg == "--repeat_penalty") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.repeat_penalty = std::stof(args[i]); params.repeat_penalty = std::stof(argv[i]);
} else if (arg == "-b" || arg == "--batch_size") { } else if (arg == "-b" || arg == "--batch_size") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.n_batch = std::stoi(args[i]); params.n_batch = std::stoi(argv[i]);
params.n_batch = std::min(512, params.n_batch); params.n_batch = std::min(512, params.n_batch);
} else if (arg == "--keep") { } else if (arg == "--keep") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.n_keep = std::stoi(args[i]); params.n_keep = std::stoi(argv[i]);
} else if (arg == "-m" || arg == "--model") { } else if (arg == "-m" || arg == "--model") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.model = args[i]; params.model = argv[i];
} else if (arg == "-i" || arg == "--interactive") { } else if (arg == "-i" || arg == "--interactive") {
params.interactive = true; params.interactive = true;
} else if (arg == "--embedding") { } else if (arg == "--embedding") {
params.embedding = true; params.embedding = true;
} else if (arg == "--clean-interface") {
params.clean_interface = true;
} else if (arg == "--interactive-start") { } else if (arg == "--interactive-start") {
params.interactive = true; params.interactive = true;
} else if (arg == "--interactive-first") { } else if (arg == "--interactive-first") {
params.interactive_start = true; params.interactive_start = true;
} else if (arg == "-ins" || arg == "--instruct") { } else if (arg == "-ins" || arg == "--instruct") {
fprintf(stderr, "\n\nWarning: instruct mode is deprecated! Use: \n" params.instruct = true;
"--clean-interface "
"--interactive-first "
"--keep -1 "
"--ins-prefix-bos "
"--ins-prefix \"\\n\\n### Instruction:\\n\\n\" "
"--ins-suffix \"\\n\\n### Response:\\n\\n\" "
"-r \"### Instruction:\\n\\n\" "
"\n\n");
// params.instruct = true;
params.clean_interface = true;
params.interactive_start = true;
params.n_keep = -1;
params.instruct_prefix_bos = true;
params.instruct_prefix = "\n\n### Instruction:\n\n";
params.instruct_suffix = "\n\n### Response:\n\n";
params.antiprompt.push_back("### Instruction:\n\n");
} else if (arg == "--color") { } else if (arg == "--color") {
params.use_color = true; params.use_color = true;
} else if (arg == "--disable-multiline") {
params.multiline_mode = false;
} else if (arg == "--mlock") { } else if (arg == "--mlock") {
params.use_mlock = true; params.use_mlock = true;
} else if (arg == "--no-mmap") { } else if (arg == "--no-mmap") {
@ -251,94 +160,65 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
} else if (arg == "--verbose-prompt") { } else if (arg == "--verbose-prompt") {
params.verbose_prompt = true; params.verbose_prompt = true;
} else if (arg == "-r" || arg == "--reverse-prompt") { } else if (arg == "-r" || arg == "--reverse-prompt") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.antiprompt.push_back(args[i]); params.antiprompt.push_back(argv[i]);
} else if (arg == "--stop-prompt") {
if (++i >= args_c) {
invalid_param = true;
break;
}
params.stopprompt.push_back(args[i]);
} else if (arg == "--rm-trailing-space-workaround") {
params.rm_trailing_space_workaround = true;
} else if (arg == "--perplexity") { } else if (arg == "--perplexity") {
params.perplexity = true; params.perplexity = true;
} else if (arg == "--ignore-eos") { } else if (arg == "--ignore-eos") {
params.ignore_eos = true; params.ignore_eos = true;
} else if (arg == "--n_parts") { } else if (arg == "--n_parts") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.n_parts = std::stoi(args[i]); params.n_parts = std::stoi(argv[i]);
} else if (arg == "-h" || arg == "--help") { } else if (arg == "-h" || arg == "--help") {
gpt_print_usage(argv[0], default_params); gpt_print_usage(argc, argv, default_params);
exit(0); exit(0);
} else if (arg == "--random-prompt") { } else if (arg == "--random-prompt") {
params.random_prompt = true; params.random_prompt = true;
} else if (arg == "--in-prefix") { } else if (arg == "--in-prefix") {
if (++i >= args_c) { if (++i >= argc) {
invalid_param = true; invalid_param = true;
break; break;
} }
params.input_prefix = args[i]; params.input_prefix = argv[i];
} else if (arg == "--ins-prefix-bos") {
params.instruct_prefix_bos = true;
} else if (arg == "--ins-prefix") {
if (++i >= args_c) {
invalid_param = true;
break;
}
params.instruct_prefix = args[i];
} else if (arg == "--ins-suffix-bos") {
params.instruct_suffix_bos = true;
} else if (arg == "--ins-suffix") {
if (++i >= args_c) {
invalid_param = true;
break;
}
params.instruct_suffix = args[i];
} else { } else {
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str()); fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
gpt_print_usage(argv[0], default_params); gpt_print_usage(argc, argv, default_params);
exit(1); exit(1);
} }
} }
if (invalid_param) { if (invalid_param) {
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
gpt_print_usage(argv[0], default_params); gpt_print_usage(argc, argv, default_params);
exit(1); exit(1);
} }
return true; return true;
} }
void gpt_print_usage(char * argv_0, const gpt_params & params) { void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
fprintf(stderr, "usage: %s [options]\n", argv_0); fprintf(stderr, "usage: %s [options]\n", argv[0]);
fprintf(stderr, "\n"); fprintf(stderr, "\n");
fprintf(stderr, "options:\n"); fprintf(stderr, "options:\n");
fprintf(stderr, " -h, --help show this help message and exit\n"); fprintf(stderr, " -h, --help show this help message and exit\n");
fprintf(stderr, " -i, --interactive run in interactive mode\n"); fprintf(stderr, " -i, --interactive run in interactive mode\n");
fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n"); fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n");
fprintf(stderr, " --clean-interface hides input prefix & suffix and displays '>' instead\n"); fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n");
fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n"); fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n");
fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n"); fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n");
fprintf(stderr, " specified more than once for multiple prompts).\n"); fprintf(stderr, " specified more than once for multiple prompts).\n");
fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n"); fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n");
fprintf(stderr, " --disable-multiline disable multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n");
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n"); fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n");
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n"); fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
fprintf(stderr, " prompt to start generation with (default: empty)\n"); fprintf(stderr, " prompt to start generation with (default: empty)\n");
fprintf(stderr, " --random-prompt start with a randomized prompt.\n"); fprintf(stderr, " --random-prompt start with a randomized prompt.\n");
fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n"); fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n");
fprintf(stderr, " --ins-prefix STRING (instruct) prefix user inputs with tokenized string (default: empty)\n");
fprintf(stderr, " --ins-prefix-bos (instruct) prepend bos token to instruct prefix.\n");
fprintf(stderr, " --ins-suffix STRING (instruct) suffix user inputs with tokenized string (default: empty)\n");
fprintf(stderr, " --ins-suffix-bos (instruct) prepend bos token to instruct suffix.\n");
fprintf(stderr, " -f FNAME, --file FNAME\n"); fprintf(stderr, " -f FNAME, --file FNAME\n");
fprintf(stderr, " prompt file to start generation.\n"); fprintf(stderr, " prompt file to start generation.\n");
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict); fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict);
@ -448,61 +328,3 @@ void win32_utf8_encode(const std::wstring & wstr, std::string & str) {
str = strTo; str = strTo;
} }
#endif #endif
bool get_input_text(std::string & input_text, bool eof_toggled_multiline_mode) {
bool another_line = true;
bool is_eof_multiline_toggled = false;
do {
std::string line;
#if defined(_WIN32)
auto & stdcin = std::wcin;
std::wstring wline;
if (!std::getline(stdcin, wline)) {
// input stream is bad or EOF received
if (stdcin.bad()) {
fprintf(stderr, "%s: error: input stream bad\n", __func__);
return 1;
}
}
win32_utf8_encode(wline, line);
#else
auto & stdcin = std::cin;
if (!std::getline(stdcin, line)) {
// input stream is bad or EOF received
if (stdcin.bad()) {
fprintf(stderr, "%s: error: input stream bad\n", __func__);
return 1;
}
}
#endif
if (stdcin.eof()) {
stdcin.clear();
stdcin.seekg(0, std::ios::beg);
if (!eof_toggled_multiline_mode) {
another_line = false;
} else {
is_eof_multiline_toggled = !is_eof_multiline_toggled;
if (is_eof_multiline_toggled) {
input_text += line;
continue;
}
}
}
if (!eof_toggled_multiline_mode) {
if (line.empty() || line.back() != '\\') {
another_line = false;
} else {
line.pop_back(); // Remove the continue character
}
} else {
if (!is_eof_multiline_toggled) {
another_line = false;
}
}
input_text += line;
if (another_line) {
input_text += '\n'; // Append the line to the result
}
} while (another_line);
return true;
}

18
examples/common.h
View File

@ -15,13 +15,13 @@
struct gpt_params { struct gpt_params {
int32_t seed = -1; // RNG seed int32_t seed = -1; // RNG seed
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); // max 4 threads (default) int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
int32_t n_predict = 128; // new tokens to predict int32_t n_predict = 128; // new tokens to predict
int32_t repeat_last_n = 64; // last n tokens to penalize int32_t repeat_last_n = 64; // last n tokens to penalize
int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions) int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions)
int32_t n_ctx = 512; // context size int32_t n_ctx = 512; // context size
int32_t n_batch = 8; // batch size for prompt processing int32_t n_batch = 8; // batch size for prompt processing
int32_t n_keep = 0; // number of tokens to keep from initial prompt (-1 for all) int32_t n_keep = 0; // number of tokens to keep from initial prompt
// sampling parameters // sampling parameters
int32_t top_k = 40; int32_t top_k = 40;
@ -33,15 +33,8 @@ struct gpt_params {
std::string prompt = ""; std::string prompt = "";
std::string input_prefix = ""; // string to prefix user inputs with std::string input_prefix = ""; // string to prefix user inputs with
std::string instruct_prefix = ""; // prefix user inputs with tokenized string
bool instruct_prefix_bos = false; // prepend bos token to instruct prefix
std::string instruct_suffix = ""; // suffix user inputs with tokenized string
bool instruct_suffix_bos = false; // prepend bos token to instruct suffix
std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted
std::vector<std::string> stopprompt; // string upon seeing which more user input is prompted (without adding instruct prefixes and suffixes)
bool rm_trailing_space_workaround = false; // workaround for removing trailing space from reverse/stop prompts
bool memory_f16 = true; // use f16 instead of f32 for memory kv bool memory_f16 = true; // use f16 instead of f32 for memory kv
bool random_prompt = false; // do not randomize prompt if none provided bool random_prompt = false; // do not randomize prompt if none provided
@ -58,14 +51,11 @@ struct gpt_params {
bool use_mlock = false; // use mlock to keep model in memory bool use_mlock = false; // use mlock to keep model in memory
bool mem_test = false; // compute maximum memory usage bool mem_test = false; // compute maximum memory usage
bool verbose_prompt = false; // print prompt tokens before generation bool verbose_prompt = false; // print prompt tokens before generation
bool clean_interface = false; // hides input prefix & suffix and displays '>'
bool multiline_mode = true; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows
}; };
bool gpt_params_parse(int argc, char ** argv, gpt_params & params); bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
void gpt_print_usage(char * argv_0, const gpt_params & params); void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
std::string gpt_random_prompt(std::mt19937 & rng); std::string gpt_random_prompt(std::mt19937 & rng);
@ -105,5 +95,3 @@ void set_console_color(console_state & con_st, console_color_t color);
void win32_console_init(bool enable_color); void win32_console_init(bool enable_color);
void win32_utf8_encode(const std::wstring & wstr, std::string & str); void win32_utf8_encode(const std::wstring & wstr, std::string & str);
#endif #endif
bool get_input_text(std::string & input_text, bool escape_newline_mode);

166
examples/main/main.cpp
View File

@ -30,8 +30,7 @@ static bool is_interacting = false;
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
void sigint_handler(int signo) { void sigint_handler(int signo) {
set_console_color(con_st, CONSOLE_COLOR_DEFAULT); set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
fflush(stdout); printf("\n"); // this also force flush stdout.
fflush(stderr);
if (signo == SIGINT) { if (signo == SIGINT) {
if (!is_interacting) { if (!is_interacting) {
is_interacting=true; is_interacting=true;
@ -90,8 +89,6 @@ int main(int argc, char ** argv) {
params.prompt = gpt_random_prompt(rng); params.prompt = gpt_random_prompt(rng);
} }
bool instruct_mode = !params.instruct_prefix.empty() || !params.instruct_suffix.empty();
// params.prompt = R"(// this function checks if the number n is prime // params.prompt = R"(// this function checks if the number n is prime
//bool is_prime(int n) {)"; //bool is_prime(int n) {)";
@ -156,20 +153,22 @@ int main(int argc, char ** argv) {
} }
// number of tokens to keep when resetting context // number of tokens to keep when resetting context
if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size()) { if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size() || params.instruct) {
params.n_keep = (int)embd_inp.size(); params.n_keep = (int)embd_inp.size();
} }
// prefix & suffix for instruct mode // prefix & suffix for instruct mode
const auto inp_pfx = ::llama_tokenize(ctx, params.instruct_prefix, params.instruct_prefix_bos); const auto inp_pfx = ::llama_tokenize(ctx, "\n\n### Instruction:\n\n", true);
std::string instruct_suffix = params.instruct_suffix; const auto inp_sfx = ::llama_tokenize(ctx, "\n\n### Response:\n\n", false);
if (params.rm_trailing_space_workaround) {
if (instruct_suffix.back() == ' ') { instruct_suffix.pop_back(); } // in instruct mode, we inject a prefix and a suffix to each input by the user
if (params.instruct) {
params.interactive_start = true;
params.antiprompt.push_back("### Instruction:\n\n");
} }
const auto inp_sfx = ::llama_tokenize(ctx, instruct_suffix, params.instruct_suffix_bos);
// enable interactive mode if reverse prompt or interactive start is specified // enable interactive mode if reverse prompt or interactive start is specified
if (params.antiprompt.size() != 0 || params.stopprompt.size() != 0 || params.interactive_start) { if (params.antiprompt.size() != 0 || params.interactive_start) {
params.interactive = true; params.interactive = true;
} }
@ -211,21 +210,10 @@ int main(int argc, char ** argv) {
fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str()); fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str());
} }
} }
if (params.stopprompt.size()) {
for (auto stopprompt : params.stopprompt) {
fprintf(stderr, "Stop prompt: '%s'\n", stopprompt.c_str());
}
}
if (!params.input_prefix.empty()) { if (!params.input_prefix.empty()) {
fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str()); fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str());
} }
if (!params.instruct_prefix.empty()) {
fprintf(stderr, "Instruct prefix %s: '%s'\n", params.instruct_prefix_bos ? "(with bos token)" : "", params.instruct_prefix.c_str());
}
if (!params.instruct_suffix.empty()) {
fprintf(stderr, "Instruct suffix %s: '%s'\n", params.instruct_suffix_bos ? "(with bos token)" : "", params.instruct_suffix.c_str());
}
} }
fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n", fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n",
params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty); params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty);
@ -241,29 +229,12 @@ int main(int argc, char ** argv) {
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32) #if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
" - Press Ctrl+C to interject at any time.\n" " - Press Ctrl+C to interject at any time.\n"
#endif #endif
); " - Press Return to return control to LLaMa.\n"
if (params.multiline_mode) {
fprintf(stderr, " - Press Return to return control to LLaMa.\n"
#if defined (_WIN32)
" - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to toggle.\n\n");
#else
" - [MULTILINE MODE] Press Ctrl+D (EOF) to toggle.\n\n");
#endif
}
else {
fprintf(stderr, " - Press Return to return control to LLaMa.\n"
" - If you want to submit another line, end your input in '\\'.\n\n"); " - If you want to submit another line, end your input in '\\'.\n\n");
}
is_interacting = params.interactive_start; is_interacting = params.interactive_start;
} }
struct Antiprompt { bool is_antiprompt = false;
bool any = false;
bool trailing_space = false;
size_t len;
bool is_stop_prompt = false;
} antiprompt;
bool input_noecho = false; bool input_noecho = false;
int n_past = 0; int n_past = 0;
@ -333,7 +304,7 @@ int main(int argc, char ** argv) {
} }
// replace end of text token with newline token when in interactive mode // replace end of text token with newline token when in interactive mode
if (id == llama_token_eos() && params.interactive && !instruct_mode) { if (id == llama_token_eos() && params.interactive && !params.instruct) {
id = llama_token_newline.front(); id = llama_token_newline.front();
if (params.antiprompt.size() != 0) { if (params.antiprompt.size() != 0) {
// tokenize and inject first reverse prompt // tokenize and inject first reverse prompt
@ -379,72 +350,27 @@ int main(int argc, char ** argv) {
// check if we should prompt the user for more // check if we should prompt the user for more
if (params.interactive && (int) embd_inp.size() <= n_consumed) { if (params.interactive && (int) embd_inp.size() <= n_consumed) {
// check for reverse prompt or stop prompt // check for reverse prompt
if (params.antiprompt.size() || params.stopprompt.size()) { if (params.antiprompt.size()) {
std::string last_output; std::string last_output;
for (auto id : last_n_tokens) { for (auto id : last_n_tokens) {
last_output += llama_token_to_str(ctx, id); last_output += llama_token_to_str(ctx, id);
} }
antiprompt.any = false; is_antiprompt = false;
antiprompt.is_stop_prompt = false;
// Check if each of the reverse prompts appears at the end of the output. // Check if each of the reverse prompts appears at the end of the output.
for (std::string & prompt : params.antiprompt) { for (std::string & antiprompt : params.antiprompt) {
if (params.rm_trailing_space_workaround) { if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) {
antiprompt.trailing_space = prompt.back() == ' ';
antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
}
if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
is_interacting = true; is_interacting = true;
antiprompt.any = true; is_antiprompt = true;
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
fflush(stdout);
break;
}
}
if (!antiprompt.any) {
for (std::string & prompt : params.stopprompt) {
if (params.rm_trailing_space_workaround) {
antiprompt.trailing_space = prompt.back() == ' ';
antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
}
if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
is_interacting = true;
antiprompt.any = true;
antiprompt.is_stop_prompt = true;
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
fflush(stdout); fflush(stdout);
break; break;
} }
} }
} }
}
if (n_past > 0 && is_interacting)
{
std::string buffer;
if (!params.clean_interface && !params.instruct_prefix.empty() && !antiprompt.any) {
// avoid printing again user's new line (TODO: try to revert enter press and print newline)
int i = params.instruct_prefix.front() == '\n' ? 1 : 0;
for (; i < inp_pfx.size(); i++) {
printf("%s", llama_token_to_str(ctx, inp_pfx[i]));
}
fflush(stdout);
}
if (params.rm_trailing_space_workaround) {
// add only if not stopprompt (as stopprompt could be used to pause
// assistant and then continue without input - adding back trailing
// space may mess it up.)
if (!antiprompt.is_stop_prompt && antiprompt.any && antiprompt.trailing_space) {
// add back removed trailing space to buffer(workaround)
buffer += ' ';
if (!params.clean_interface) {
printf("%s", buffer.c_str());
}
fflush(stdout);
}
}
if (n_past > 0 && is_interacting) {
// potentially set color to indicate we are taking user input // potentially set color to indicate we are taking user input
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT); set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
@ -453,45 +379,49 @@ int main(int argc, char ** argv) {
signal(SIGINT, sigint_handler); signal(SIGINT, sigint_handler);
#endif #endif
if (params.clean_interface) { if (params.instruct) {
printf("\n> "); printf("\n> ");
} }
std::string buffer;
if (!params.input_prefix.empty()) { if (!params.input_prefix.empty()) {
buffer += params.input_prefix; buffer += params.input_prefix;
printf("%s", buffer.c_str()); printf("%s", buffer.c_str());
} }
if (!get_input_text(buffer, params.multiline_mode)) { std::string line;
// input stream is bad bool another_line = true;
return 1; do {
#if defined(_WIN32)
std::wstring wline;
if (!std::getline(std::wcin, wline)) {
// input stream is bad or EOF received
return 0;
} }
if (!antiprompt.is_stop_prompt) { win32_utf8_encode(wline, line);
buffer += "\n"; #else
if (!std::getline(std::cin, line)) {
// input stream is bad or EOF received
return 0;
} }
#endif
if (line.empty() || line.back() != '\\') {
another_line = false;
} else {
line.pop_back(); // Remove the continue character
}
buffer += line + '\n'; // Append the line to the result
} while (another_line);
// done taking input, reset color // done taking input, reset color
set_console_color(con_st, CONSOLE_COLOR_DEFAULT); set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
if (!params.clean_interface && !params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) {
// avoid printing again user's new line (TODO: try to revert enter press and print newline)
int i = params.instruct_suffix.front() == '\n' ? 1 : 0;
for (; i < inp_sfx.size(); i++) {
printf("%s", llama_token_to_str(ctx, inp_sfx[i]));
}
// if (remove trailing space workaround) {
// We won't add back removed trailing space here, because assistant continues here,
// and it may mess up it's output (remove trailing space workaround).
// }
fflush(stdout);
}
// Add tokens to embd only if the input buffer is non-empty // Add tokens to embd only if the input buffer is non-empty
// Entering a empty line lets the user pass control back // Entering a empty line lets the user pass control back
if (buffer.length() > 1) { if (buffer.length() > 1) {
// insert input prefix // instruct mode: insert instruction prefix
if (!params.instruct_prefix.empty() && !antiprompt.any) { if (params.instruct && !is_antiprompt) {
n_consumed = embd_inp.size(); n_consumed = embd_inp.size();
embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end()); embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end());
} }
@ -499,8 +429,8 @@ int main(int argc, char ** argv) {
auto line_inp = ::llama_tokenize(ctx, buffer, false); auto line_inp = ::llama_tokenize(ctx, buffer, false);
embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end()); embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
// insert response suffix // instruct mode: insert response suffix
if (!params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) { if (params.instruct) {
embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end()); embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end());
} }
@ -517,7 +447,7 @@ int main(int argc, char ** argv) {
// end of text token // end of text token
if (!embd.empty() && embd.back() == llama_token_eos()) { if (!embd.empty() && embd.back() == llama_token_eos()) {
if (instruct_mode) { if (params.instruct) {
is_interacting = true; is_interacting = true;
} else { } else {
fprintf(stderr, " [end of text]\n"); fprintf(stderr, " [end of text]\n");

1
prompts/alpaca.txt Normal file
View File

@ -0,0 +1 @@
Below is an instruction that describes a task. Write a response that appropriately completes the request.

7
prompts/chat-with-bob.txt Normal file
View File

@ -0,0 +1,7 @@
Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
User: Hello, Bob.
Bob: Hello. How may I help you today?
User: Please tell me the largest city in Europe.
Bob: Sure. The largest city in Europe is Moscow, the capital of Russia.
User: