mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2024-12-24 02:14:35 +00:00
This reverts commit f4d277ae17.
This commit is contained in:
Pavol Rusnak
GitHub
parent
489093548c
commit
c85e03d12e
@ -1,21 +0,0 @@
|
||||
--ctx_size 2048
|
||||
--batch_size 16
|
||||
--repeat_penalty 1.15
|
||||
--temp 0.4
|
||||
--top_k 30
|
||||
--top_p 0.18
|
||||
|
||||
--interactive-first
|
||||
--keep -1
|
||||
|
||||
--ins-prefix-bos
|
||||
--ins-prefix "\n\nUser: "
|
||||
--ins-suffix "\n\nAssistant: "
|
||||
--reverse-prompt "User: "
|
||||
|
||||
-p "You are an AI language model designed to assist the User by answering their questions, offering advice, and engaging in casual conversation in a friendly, helpful, and informative manner. You respond clearly, coherently, and you consider the conversation history.
|
||||
|
||||
User: Hey, how's it going?
|
||||
|
||||
Assistant: Hey there! I'm doing great, thank you. What can I help you with today? Let's have a fun chat!"
|
||||
|
||||
@ -1,9 +0,0 @@
|
||||
--clean-interface
|
||||
--interactive-first
|
||||
--keep -1
|
||||
--ins-prefix-bos
|
||||
--ins-prefix "\n\n### Instruction:\n\n"
|
||||
--ins-suffix "\n\n### Response:\n\n"
|
||||
--reverse-prompt "### Instruction:\n\n"
|
||||
|
||||
-p "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n"
|
||||
@ -1,15 +0,0 @@
|
||||
--interactive-first
|
||||
--keep -1
|
||||
--ins-prefix-bos
|
||||
--ins-prefix "\nUser: "
|
||||
--ins-suffix "\nBob: "
|
||||
--reverse-prompt "User: "
|
||||
--rm-trailing-space-workaround
|
||||
|
||||
-p "Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
|
||||
|
||||
User: Hello, Bob.
|
||||
Bob: Hello. How may I help you today?
|
||||
User: Please tell me the largest city in Europe.
|
||||
Bob: Sure. The largest city in Europe is Moscow, the capital of Russia."
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
--interactive-first
|
||||
--keep -1
|
||||
--temp 0.1
|
||||
@ -1,7 +0,0 @@
|
||||
--interactive-first
|
||||
--keep -1
|
||||
--ins-prefix-bos
|
||||
--ins-prefix "\n### Human: "
|
||||
--ins-suffix "\n### Assistant: "
|
||||
--reverse-prompt "### Human: "
|
||||
--rm-trailing-space-workaround
|
||||
@ -1,8 +0,0 @@
|
||||
--interactive-first
|
||||
--keep -1
|
||||
--ins-prefix-bos
|
||||
--ins-prefix "\n### Human: "
|
||||
--ins-suffix "\n### Assistant: "
|
||||
--reverse-prompt "### Human: "
|
||||
--stop-prompt "### Assistant: "
|
||||
--rm-trailing-space-workaround
|
||||
@ -1,9 +0,0 @@
|
||||
--interactive-first
|
||||
--keep -1
|
||||
--ins-prefix-bos
|
||||
--ins-prefix "\n### Human: "
|
||||
--ins-suffix "\n### Assistant: "
|
||||
--reverse-prompt "### Human: "
|
||||
--rm-trailing-space-workaround
|
||||
|
||||
-p "A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."
|
||||
@ -2,13 +2,10 @@
|
||||
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <iterator>
|
||||
#include <algorithm>
|
||||
#include <regex>
|
||||
|
||||
#if defined (_WIN32)
|
||||
#include <fcntl.h>
|
||||
@ -26,43 +23,6 @@ extern "C" __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int
|
||||
#define CP_UTF8 65001
|
||||
#endif
|
||||
|
||||
void split_args(const std::string & args_string, std::vector<std::string> & output_args)
|
||||
{
|
||||
std::string current_arg = "";
|
||||
bool in_quotes = false;
|
||||
char quote_type;
|
||||
|
||||
for (char c : args_string) {
|
||||
if (c == '"' || c == '\'') {
|
||||
if (!in_quotes) {
|
||||
in_quotes = true;
|
||||
quote_type = c;
|
||||
} else if (quote_type == c) {
|
||||
in_quotes = false;
|
||||
} else {
|
||||
current_arg += c;
|
||||
}
|
||||
} else if (in_quotes) {
|
||||
current_arg += c;
|
||||
} else if (std::isspace(c)) {
|
||||
if (current_arg != "") {
|
||||
output_args.push_back(current_arg);
|
||||
current_arg = "";
|
||||
}
|
||||
} else {
|
||||
current_arg += c;
|
||||
}
|
||||
}
|
||||
|
||||
if (current_arg != "") {
|
||||
output_args.push_back(current_arg);
|
||||
}
|
||||
}
|
||||
|
||||
std::string unescape(const std::string & str) {
|
||||
return std::regex_replace(str, std::regex("\\\\n"), "\n");
|
||||
}
|
||||
|
||||
bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
|
||||
// determine sensible default number of threads.
|
||||
// std::thread::hardware_concurrency may not be equal to the number of cores, or may return 0.
|
||||
@ -80,11 +40,28 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
|
||||
std::string arg;
|
||||
gpt_params default_params;
|
||||
|
||||
// get additional arguments from config files
|
||||
std::vector<std::string> args;
|
||||
for (int i = 1; i < argc; i++) {
|
||||
arg = argv[i];
|
||||
if (arg == "--config") {
|
||||
|
||||
if (arg == "-s" || arg == "--seed") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.seed = std::stoi(argv[i]);
|
||||
} else if (arg == "-t" || arg == "--threads") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.n_threads = std::stoi(argv[i]);
|
||||
} else if (arg == "-p" || arg == "--prompt") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.prompt = argv[i];
|
||||
} else if (arg == "-f" || arg == "--file") {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
@ -95,153 +72,85 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
std::string args_string;
|
||||
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(args_string));
|
||||
if (args_string.back() == '\n') {
|
||||
args_string.pop_back();
|
||||
}
|
||||
split_args(args_string, args);
|
||||
for (int j = 0; j < args.size(); j++) {
|
||||
args[j] = unescape(args[j]);
|
||||
}
|
||||
} else {
|
||||
args.emplace_back(argv[i]);
|
||||
}
|
||||
}
|
||||
|
||||
// parse args
|
||||
int args_c = static_cast<int>(args.size());
|
||||
for (int i = 0; i < args_c && !invalid_param; i++) {
|
||||
arg = args[i];
|
||||
|
||||
if (arg == "-s" || arg == "--seed") {
|
||||
if (++i >= args_c) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.seed = std::stoi(args[i]);
|
||||
} else if (arg == "-t" || arg == "--threads") {
|
||||
if (++i >= args_c) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.n_threads = std::stoi(args[i]);
|
||||
} else if (arg == "-p" || arg == "--prompt") {
|
||||
if (++i >= args_c) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.prompt = args[i];
|
||||
} else if (arg == "-f" || arg == "--file") {
|
||||
if (++i >= args_c) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
std::ifstream file(args[i]);
|
||||
if (!file) {
|
||||
fprintf(stderr, "error: failed to open file '%s'\n", args[i].c_str());
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
std::copy(std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>(), back_inserter(params.prompt));
|
||||
if (params.prompt.back() == '\n') {
|
||||
params.prompt.pop_back();
|
||||
}
|
||||
} else if (arg == "-n" || arg == "--n_predict") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.n_predict = std::stoi(args[i]);
|
||||
params.n_predict = std::stoi(argv[i]);
|
||||
} else if (arg == "--top_k") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.top_k = std::stoi(args[i]);
|
||||
params.top_k = std::stoi(argv[i]);
|
||||
} else if (arg == "-c" || arg == "--ctx_size") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.n_ctx = std::stoi(args[i]);
|
||||
params.n_ctx = std::stoi(argv[i]);
|
||||
} else if (arg == "--memory_f32") {
|
||||
params.memory_f16 = false;
|
||||
} else if (arg == "--top_p") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.top_p = std::stof(args[i]);
|
||||
params.top_p = std::stof(argv[i]);
|
||||
} else if (arg == "--temp") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.temp = std::stof(args[i]);
|
||||
params.temp = std::stof(argv[i]);
|
||||
} else if (arg == "--repeat_last_n") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.repeat_last_n = std::stoi(args[i]);
|
||||
params.repeat_last_n = std::stoi(argv[i]);
|
||||
} else if (arg == "--repeat_penalty") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.repeat_penalty = std::stof(args[i]);
|
||||
params.repeat_penalty = std::stof(argv[i]);
|
||||
} else if (arg == "-b" || arg == "--batch_size") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.n_batch = std::stoi(args[i]);
|
||||
params.n_batch = std::stoi(argv[i]);
|
||||
params.n_batch = std::min(512, params.n_batch);
|
||||
} else if (arg == "--keep") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.n_keep = std::stoi(args[i]);
|
||||
params.n_keep = std::stoi(argv[i]);
|
||||
} else if (arg == "-m" || arg == "--model") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.model = args[i];
|
||||
params.model = argv[i];
|
||||
} else if (arg == "-i" || arg == "--interactive") {
|
||||
params.interactive = true;
|
||||
} else if (arg == "--embedding") {
|
||||
params.embedding = true;
|
||||
} else if (arg == "--clean-interface") {
|
||||
params.clean_interface = true;
|
||||
} else if (arg == "--interactive-start") {
|
||||
params.interactive = true;
|
||||
} else if (arg == "--interactive-first") {
|
||||
params.interactive_start = true;
|
||||
} else if (arg == "-ins" || arg == "--instruct") {
|
||||
fprintf(stderr, "\n\nWarning: instruct mode is deprecated! Use: \n"
|
||||
"--clean-interface "
|
||||
"--interactive-first "
|
||||
"--keep -1 "
|
||||
"--ins-prefix-bos "
|
||||
"--ins-prefix \"\\n\\n### Instruction:\\n\\n\" "
|
||||
"--ins-suffix \"\\n\\n### Response:\\n\\n\" "
|
||||
"-r \"### Instruction:\\n\\n\" "
|
||||
"\n\n");
|
||||
// params.instruct = true;
|
||||
params.clean_interface = true;
|
||||
params.interactive_start = true;
|
||||
params.n_keep = -1;
|
||||
params.instruct_prefix_bos = true;
|
||||
params.instruct_prefix = "\n\n### Instruction:\n\n";
|
||||
params.instruct_suffix = "\n\n### Response:\n\n";
|
||||
params.antiprompt.push_back("### Instruction:\n\n");
|
||||
params.instruct = true;
|
||||
} else if (arg == "--color") {
|
||||
params.use_color = true;
|
||||
} else if (arg == "--disable-multiline") {
|
||||
params.multiline_mode = false;
|
||||
} else if (arg == "--mlock") {
|
||||
params.use_mlock = true;
|
||||
} else if (arg == "--no-mmap") {
|
||||
@ -251,94 +160,65 @@ bool gpt_params_parse(int argc, char ** argv, gpt_params & params) {
|
||||
} else if (arg == "--verbose-prompt") {
|
||||
params.verbose_prompt = true;
|
||||
} else if (arg == "-r" || arg == "--reverse-prompt") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.antiprompt.push_back(args[i]);
|
||||
} else if (arg == "--stop-prompt") {
|
||||
if (++i >= args_c) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.stopprompt.push_back(args[i]);
|
||||
} else if (arg == "--rm-trailing-space-workaround") {
|
||||
params.rm_trailing_space_workaround = true;
|
||||
params.antiprompt.push_back(argv[i]);
|
||||
} else if (arg == "--perplexity") {
|
||||
params.perplexity = true;
|
||||
} else if (arg == "--ignore-eos") {
|
||||
params.ignore_eos = true;
|
||||
} else if (arg == "--n_parts") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.n_parts = std::stoi(args[i]);
|
||||
params.n_parts = std::stoi(argv[i]);
|
||||
} else if (arg == "-h" || arg == "--help") {
|
||||
gpt_print_usage(argv[0], default_params);
|
||||
gpt_print_usage(argc, argv, default_params);
|
||||
exit(0);
|
||||
} else if (arg == "--random-prompt") {
|
||||
params.random_prompt = true;
|
||||
} else if (arg == "--in-prefix") {
|
||||
if (++i >= args_c) {
|
||||
if (++i >= argc) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.input_prefix = args[i];
|
||||
} else if (arg == "--ins-prefix-bos") {
|
||||
params.instruct_prefix_bos = true;
|
||||
} else if (arg == "--ins-prefix") {
|
||||
if (++i >= args_c) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.instruct_prefix = args[i];
|
||||
} else if (arg == "--ins-suffix-bos") {
|
||||
params.instruct_suffix_bos = true;
|
||||
} else if (arg == "--ins-suffix") {
|
||||
if (++i >= args_c) {
|
||||
invalid_param = true;
|
||||
break;
|
||||
}
|
||||
params.instruct_suffix = args[i];
|
||||
params.input_prefix = argv[i];
|
||||
} else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
gpt_print_usage(argv[0], default_params);
|
||||
gpt_print_usage(argc, argv, default_params);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
if (invalid_param) {
|
||||
fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());
|
||||
gpt_print_usage(argv[0], default_params);
|
||||
gpt_print_usage(argc, argv, default_params);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
void gpt_print_usage(char * argv_0, const gpt_params & params) {
|
||||
fprintf(stderr, "usage: %s [options]\n", argv_0);
|
||||
void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
|
||||
fprintf(stderr, "usage: %s [options]\n", argv[0]);
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "options:\n");
|
||||
fprintf(stderr, " -h, --help show this help message and exit\n");
|
||||
fprintf(stderr, " -i, --interactive run in interactive mode\n");
|
||||
fprintf(stderr, " --interactive-first run in interactive mode and wait for input right away\n");
|
||||
fprintf(stderr, " --clean-interface hides input prefix & suffix and displays '>' instead\n");
|
||||
fprintf(stderr, " -ins, --instruct run in instruction mode (use with Alpaca models)\n");
|
||||
fprintf(stderr, " -r PROMPT, --reverse-prompt PROMPT\n");
|
||||
fprintf(stderr, " run in interactive mode and poll user input upon seeing PROMPT (can be\n");
|
||||
fprintf(stderr, " specified more than once for multiple prompts).\n");
|
||||
fprintf(stderr, " --color colorise output to distinguish prompt and user input from generations\n");
|
||||
fprintf(stderr, " --disable-multiline disable multiline mode (use Ctrl+D on Linux/Mac and Ctrl+Z then Return on Windows to toggle multiline)\n");
|
||||
fprintf(stderr, " -s SEED, --seed SEED RNG seed (default: -1, use random seed for <= 0)\n");
|
||||
fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads);
|
||||
fprintf(stderr, " -p PROMPT, --prompt PROMPT\n");
|
||||
fprintf(stderr, " prompt to start generation with (default: empty)\n");
|
||||
fprintf(stderr, " --random-prompt start with a randomized prompt.\n");
|
||||
fprintf(stderr, " --in-prefix STRING string to prefix user inputs with (default: empty)\n");
|
||||
fprintf(stderr, " --ins-prefix STRING (instruct) prefix user inputs with tokenized string (default: empty)\n");
|
||||
fprintf(stderr, " --ins-prefix-bos (instruct) prepend bos token to instruct prefix.\n");
|
||||
fprintf(stderr, " --ins-suffix STRING (instruct) suffix user inputs with tokenized string (default: empty)\n");
|
||||
fprintf(stderr, " --ins-suffix-bos (instruct) prepend bos token to instruct suffix.\n");
|
||||
fprintf(stderr, " -f FNAME, --file FNAME\n");
|
||||
fprintf(stderr, " prompt file to start generation.\n");
|
||||
fprintf(stderr, " -n N, --n_predict N number of tokens to predict (default: %d, -1 = infinity)\n", params.n_predict);
|
||||
@ -448,61 +328,3 @@ void win32_utf8_encode(const std::wstring & wstr, std::string & str) {
|
||||
str = strTo;
|
||||
}
|
||||
#endif
|
||||
|
||||
bool get_input_text(std::string & input_text, bool eof_toggled_multiline_mode) {
|
||||
bool another_line = true;
|
||||
bool is_eof_multiline_toggled = false;
|
||||
do {
|
||||
std::string line;
|
||||
#if defined(_WIN32)
|
||||
auto & stdcin = std::wcin;
|
||||
std::wstring wline;
|
||||
if (!std::getline(stdcin, wline)) {
|
||||
// input stream is bad or EOF received
|
||||
if (stdcin.bad()) {
|
||||
fprintf(stderr, "%s: error: input stream bad\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
win32_utf8_encode(wline, line);
|
||||
#else
|
||||
auto & stdcin = std::cin;
|
||||
if (!std::getline(stdcin, line)) {
|
||||
// input stream is bad or EOF received
|
||||
if (stdcin.bad()) {
|
||||
fprintf(stderr, "%s: error: input stream bad\n", __func__);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (stdcin.eof()) {
|
||||
stdcin.clear();
|
||||
stdcin.seekg(0, std::ios::beg);
|
||||
if (!eof_toggled_multiline_mode) {
|
||||
another_line = false;
|
||||
} else {
|
||||
is_eof_multiline_toggled = !is_eof_multiline_toggled;
|
||||
if (is_eof_multiline_toggled) {
|
||||
input_text += line;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!eof_toggled_multiline_mode) {
|
||||
if (line.empty() || line.back() != '\\') {
|
||||
another_line = false;
|
||||
} else {
|
||||
line.pop_back(); // Remove the continue character
|
||||
}
|
||||
} else {
|
||||
if (!is_eof_multiline_toggled) {
|
||||
another_line = false;
|
||||
}
|
||||
}
|
||||
input_text += line;
|
||||
if (another_line) {
|
||||
input_text += '\n'; // Append the line to the result
|
||||
}
|
||||
} while (another_line);
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -14,14 +14,14 @@
|
||||
//
|
||||
|
||||
struct gpt_params {
|
||||
int32_t seed = -1; // RNG seed
|
||||
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency()); // max 4 threads (default)
|
||||
int32_t n_predict = 128; // new tokens to predict
|
||||
int32_t repeat_last_n = 64; // last n tokens to penalize
|
||||
int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions)
|
||||
int32_t n_ctx = 512; // context size
|
||||
int32_t n_batch = 8; // batch size for prompt processing
|
||||
int32_t n_keep = 0; // number of tokens to keep from initial prompt (-1 for all)
|
||||
int32_t seed = -1; // RNG seed
|
||||
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
|
||||
int32_t n_predict = 128; // new tokens to predict
|
||||
int32_t repeat_last_n = 64; // last n tokens to penalize
|
||||
int32_t n_parts = -1; // amount of model parts (-1 = determine from model dimensions)
|
||||
int32_t n_ctx = 512; // context size
|
||||
int32_t n_batch = 8; // batch size for prompt processing
|
||||
int32_t n_keep = 0; // number of tokens to keep from initial prompt
|
||||
|
||||
// sampling parameters
|
||||
int32_t top_k = 40;
|
||||
@ -33,15 +33,8 @@ struct gpt_params {
|
||||
std::string prompt = "";
|
||||
std::string input_prefix = ""; // string to prefix user inputs with
|
||||
|
||||
std::string instruct_prefix = ""; // prefix user inputs with tokenized string
|
||||
bool instruct_prefix_bos = false; // prepend bos token to instruct prefix
|
||||
std::string instruct_suffix = ""; // suffix user inputs with tokenized string
|
||||
bool instruct_suffix_bos = false; // prepend bos token to instruct suffix
|
||||
|
||||
std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted
|
||||
std::vector<std::string> stopprompt; // string upon seeing which more user input is prompted (without adding instruct prefixes and suffixes)
|
||||
|
||||
bool rm_trailing_space_workaround = false; // workaround for removing trailing space from reverse/stop prompts
|
||||
|
||||
bool memory_f16 = true; // use f16 instead of f32 for memory kv
|
||||
bool random_prompt = false; // do not randomize prompt if none provided
|
||||
@ -58,14 +51,11 @@ struct gpt_params {
|
||||
bool use_mlock = false; // use mlock to keep model in memory
|
||||
bool mem_test = false; // compute maximum memory usage
|
||||
bool verbose_prompt = false; // print prompt tokens before generation
|
||||
|
||||
bool clean_interface = false; // hides input prefix & suffix and displays '>'
|
||||
bool multiline_mode = true; // enables multi-line mode, to send input press CTRL+D on Linux/Max, Ctrl+Z then Return on Windows
|
||||
};
|
||||
|
||||
bool gpt_params_parse(int argc, char ** argv, gpt_params & params);
|
||||
|
||||
void gpt_print_usage(char * argv_0, const gpt_params & params);
|
||||
void gpt_print_usage(int argc, char ** argv, const gpt_params & params);
|
||||
|
||||
std::string gpt_random_prompt(std::mt19937 & rng);
|
||||
|
||||
@ -105,5 +95,3 @@ void set_console_color(console_state & con_st, console_color_t color);
|
||||
void win32_console_init(bool enable_color);
|
||||
void win32_utf8_encode(const std::wstring & wstr, std::string & str);
|
||||
#endif
|
||||
|
||||
bool get_input_text(std::string & input_text, bool escape_newline_mode);
|
||||
|
||||
@ -30,8 +30,7 @@ static bool is_interacting = false;
|
||||
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
|
||||
void sigint_handler(int signo) {
|
||||
set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
|
||||
fflush(stdout);
|
||||
fflush(stderr);
|
||||
printf("\n"); // this also force flush stdout.
|
||||
if (signo == SIGINT) {
|
||||
if (!is_interacting) {
|
||||
is_interacting=true;
|
||||
@ -90,8 +89,6 @@ int main(int argc, char ** argv) {
|
||||
params.prompt = gpt_random_prompt(rng);
|
||||
}
|
||||
|
||||
bool instruct_mode = !params.instruct_prefix.empty() || !params.instruct_suffix.empty();
|
||||
|
||||
// params.prompt = R"(// this function checks if the number n is prime
|
||||
//bool is_prime(int n) {)";
|
||||
|
||||
@ -156,20 +153,22 @@ int main(int argc, char ** argv) {
|
||||
}
|
||||
|
||||
// number of tokens to keep when resetting context
|
||||
if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size()) {
|
||||
if (params.n_keep < 0 || params.n_keep > (int)embd_inp.size() || params.instruct) {
|
||||
params.n_keep = (int)embd_inp.size();
|
||||
}
|
||||
|
||||
// prefix & suffix for instruct mode
|
||||
const auto inp_pfx = ::llama_tokenize(ctx, params.instruct_prefix, params.instruct_prefix_bos);
|
||||
std::string instruct_suffix = params.instruct_suffix;
|
||||
if (params.rm_trailing_space_workaround) {
|
||||
if (instruct_suffix.back() == ' ') { instruct_suffix.pop_back(); }
|
||||
const auto inp_pfx = ::llama_tokenize(ctx, "\n\n### Instruction:\n\n", true);
|
||||
const auto inp_sfx = ::llama_tokenize(ctx, "\n\n### Response:\n\n", false);
|
||||
|
||||
// in instruct mode, we inject a prefix and a suffix to each input by the user
|
||||
if (params.instruct) {
|
||||
params.interactive_start = true;
|
||||
params.antiprompt.push_back("### Instruction:\n\n");
|
||||
}
|
||||
const auto inp_sfx = ::llama_tokenize(ctx, instruct_suffix, params.instruct_suffix_bos);
|
||||
|
||||
// enable interactive mode if reverse prompt or interactive start is specified
|
||||
if (params.antiprompt.size() != 0 || params.stopprompt.size() != 0 || params.interactive_start) {
|
||||
if (params.antiprompt.size() != 0 || params.interactive_start) {
|
||||
params.interactive = true;
|
||||
}
|
||||
|
||||
@ -211,21 +210,10 @@ int main(int argc, char ** argv) {
|
||||
fprintf(stderr, "Reverse prompt: '%s'\n", antiprompt.c_str());
|
||||
}
|
||||
}
|
||||
if (params.stopprompt.size()) {
|
||||
for (auto stopprompt : params.stopprompt) {
|
||||
fprintf(stderr, "Stop prompt: '%s'\n", stopprompt.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
if (!params.input_prefix.empty()) {
|
||||
fprintf(stderr, "Input prefix: '%s'\n", params.input_prefix.c_str());
|
||||
}
|
||||
if (!params.instruct_prefix.empty()) {
|
||||
fprintf(stderr, "Instruct prefix %s: '%s'\n", params.instruct_prefix_bos ? "(with bos token)" : "", params.instruct_prefix.c_str());
|
||||
}
|
||||
if (!params.instruct_suffix.empty()) {
|
||||
fprintf(stderr, "Instruct suffix %s: '%s'\n", params.instruct_suffix_bos ? "(with bos token)" : "", params.instruct_suffix.c_str());
|
||||
}
|
||||
}
|
||||
fprintf(stderr, "sampling: temp = %f, top_k = %d, top_p = %f, repeat_last_n = %i, repeat_penalty = %f\n",
|
||||
params.temp, params.top_k, params.top_p, params.repeat_last_n, params.repeat_penalty);
|
||||
@ -241,29 +229,12 @@ int main(int argc, char ** argv) {
|
||||
#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)
|
||||
" - Press Ctrl+C to interject at any time.\n"
|
||||
#endif
|
||||
);
|
||||
if (params.multiline_mode) {
|
||||
fprintf(stderr, " - Press Return to return control to LLaMa.\n"
|
||||
#if defined (_WIN32)
|
||||
" - [MULTILINE MODE] Press Ctrl+Z then Return (EOF) to toggle.\n\n");
|
||||
#else
|
||||
" - [MULTILINE MODE] Press Ctrl+D (EOF) to toggle.\n\n");
|
||||
#endif
|
||||
}
|
||||
else {
|
||||
fprintf(stderr, " - Press Return to return control to LLaMa.\n"
|
||||
" - If you want to submit another line, end your input in '\\'.\n\n");
|
||||
}
|
||||
" - Press Return to return control to LLaMa.\n"
|
||||
" - If you want to submit another line, end your input in '\\'.\n\n");
|
||||
is_interacting = params.interactive_start;
|
||||
}
|
||||
|
||||
struct Antiprompt {
|
||||
bool any = false;
|
||||
bool trailing_space = false;
|
||||
size_t len;
|
||||
bool is_stop_prompt = false;
|
||||
} antiprompt;
|
||||
|
||||
bool is_antiprompt = false;
|
||||
bool input_noecho = false;
|
||||
|
||||
int n_past = 0;
|
||||
@ -333,7 +304,7 @@ int main(int argc, char ** argv) {
|
||||
}
|
||||
|
||||
// replace end of text token with newline token when in interactive mode
|
||||
if (id == llama_token_eos() && params.interactive && !instruct_mode) {
|
||||
if (id == llama_token_eos() && params.interactive && !params.instruct) {
|
||||
id = llama_token_newline.front();
|
||||
if (params.antiprompt.size() != 0) {
|
||||
// tokenize and inject first reverse prompt
|
||||
@ -379,72 +350,27 @@ int main(int argc, char ** argv) {
|
||||
// check if we should prompt the user for more
|
||||
if (params.interactive && (int) embd_inp.size() <= n_consumed) {
|
||||
|
||||
// check for reverse prompt or stop prompt
|
||||
if (params.antiprompt.size() || params.stopprompt.size()) {
|
||||
// check for reverse prompt
|
||||
if (params.antiprompt.size()) {
|
||||
std::string last_output;
|
||||
for (auto id : last_n_tokens) {
|
||||
last_output += llama_token_to_str(ctx, id);
|
||||
}
|
||||
|
||||
antiprompt.any = false;
|
||||
antiprompt.is_stop_prompt = false;
|
||||
is_antiprompt = false;
|
||||
// Check if each of the reverse prompts appears at the end of the output.
|
||||
for (std::string & prompt : params.antiprompt) {
|
||||
if (params.rm_trailing_space_workaround) {
|
||||
antiprompt.trailing_space = prompt.back() == ' ';
|
||||
antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
|
||||
}
|
||||
if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
|
||||
for (std::string & antiprompt : params.antiprompt) {
|
||||
if (last_output.find(antiprompt.c_str(), last_output.length() - antiprompt.length(), antiprompt.length()) != std::string::npos) {
|
||||
is_interacting = true;
|
||||
antiprompt.any = true;
|
||||
is_antiprompt = true;
|
||||
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
|
||||
fflush(stdout);
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!antiprompt.any) {
|
||||
for (std::string & prompt : params.stopprompt) {
|
||||
if (params.rm_trailing_space_workaround) {
|
||||
antiprompt.trailing_space = prompt.back() == ' ';
|
||||
antiprompt.len = prompt.length() - (antiprompt.trailing_space ? 1 : 0);
|
||||
}
|
||||
if (last_output.find(prompt.c_str(), last_output.length() - antiprompt.len, antiprompt.len) != std::string::npos) {
|
||||
is_interacting = true;
|
||||
antiprompt.any = true;
|
||||
antiprompt.is_stop_prompt = true;
|
||||
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
|
||||
fflush(stdout);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (n_past > 0 && is_interacting)
|
||||
{
|
||||
std::string buffer;
|
||||
if (!params.clean_interface && !params.instruct_prefix.empty() && !antiprompt.any) {
|
||||
// avoid printing again user's new line (TODO: try to revert enter press and print newline)
|
||||
int i = params.instruct_prefix.front() == '\n' ? 1 : 0;
|
||||
for (; i < inp_pfx.size(); i++) {
|
||||
printf("%s", llama_token_to_str(ctx, inp_pfx[i]));
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
if (params.rm_trailing_space_workaround) {
|
||||
// add only if not stopprompt (as stopprompt could be used to pause
|
||||
// assistant and then continue without input - adding back trailing
|
||||
// space may mess it up.)
|
||||
if (!antiprompt.is_stop_prompt && antiprompt.any && antiprompt.trailing_space) {
|
||||
// add back removed trailing space to buffer(workaround)
|
||||
buffer += ' ';
|
||||
if (!params.clean_interface) {
|
||||
printf("%s", buffer.c_str());
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
}
|
||||
|
||||
if (n_past > 0 && is_interacting) {
|
||||
// potentially set color to indicate we are taking user input
|
||||
set_console_color(con_st, CONSOLE_COLOR_USER_INPUT);
|
||||
|
||||
@ -453,45 +379,49 @@ int main(int argc, char ** argv) {
|
||||
signal(SIGINT, sigint_handler);
|
||||
#endif
|
||||
|
||||
if (params.clean_interface) {
|
||||
if (params.instruct) {
|
||||
printf("\n> ");
|
||||
}
|
||||
|
||||
std::string buffer;
|
||||
if (!params.input_prefix.empty()) {
|
||||
buffer += params.input_prefix;
|
||||
printf("%s", buffer.c_str());
|
||||
}
|
||||
|
||||
if (!get_input_text(buffer, params.multiline_mode)) {
|
||||
// input stream is bad
|
||||
return 1;
|
||||
}
|
||||
if (!antiprompt.is_stop_prompt) {
|
||||
buffer += "\n";
|
||||
}
|
||||
std::string line;
|
||||
bool another_line = true;
|
||||
do {
|
||||
#if defined(_WIN32)
|
||||
std::wstring wline;
|
||||
if (!std::getline(std::wcin, wline)) {
|
||||
// input stream is bad or EOF received
|
||||
return 0;
|
||||
}
|
||||
win32_utf8_encode(wline, line);
|
||||
#else
|
||||
if (!std::getline(std::cin, line)) {
|
||||
// input stream is bad or EOF received
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
if (line.empty() || line.back() != '\\') {
|
||||
another_line = false;
|
||||
} else {
|
||||
line.pop_back(); // Remove the continue character
|
||||
}
|
||||
buffer += line + '\n'; // Append the line to the result
|
||||
} while (another_line);
|
||||
|
||||
// done taking input, reset color
|
||||
set_console_color(con_st, CONSOLE_COLOR_DEFAULT);
|
||||
|
||||
if (!params.clean_interface && !params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) {
|
||||
// avoid printing again user's new line (TODO: try to revert enter press and print newline)
|
||||
int i = params.instruct_suffix.front() == '\n' ? 1 : 0;
|
||||
for (; i < inp_sfx.size(); i++) {
|
||||
printf("%s", llama_token_to_str(ctx, inp_sfx[i]));
|
||||
}
|
||||
// if (remove trailing space workaround) {
|
||||
// We won't add back removed trailing space here, because assistant continues here,
|
||||
// and it may mess up it's output (remove trailing space workaround).
|
||||
// }
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
// Add tokens to embd only if the input buffer is non-empty
|
||||
// Entering a empty line lets the user pass control back
|
||||
if (buffer.length() > 1) {
|
||||
|
||||
// insert input prefix
|
||||
if (!params.instruct_prefix.empty() && !antiprompt.any) {
|
||||
// instruct mode: insert instruction prefix
|
||||
if (params.instruct && !is_antiprompt) {
|
||||
n_consumed = embd_inp.size();
|
||||
embd_inp.insert(embd_inp.end(), inp_pfx.begin(), inp_pfx.end());
|
||||
}
|
||||
@ -499,8 +429,8 @@ int main(int argc, char ** argv) {
|
||||
auto line_inp = ::llama_tokenize(ctx, buffer, false);
|
||||
embd_inp.insert(embd_inp.end(), line_inp.begin(), line_inp.end());
|
||||
|
||||
// insert response suffix
|
||||
if (!params.instruct_suffix.empty() && !antiprompt.is_stop_prompt) {
|
||||
// instruct mode: insert response suffix
|
||||
if (params.instruct) {
|
||||
embd_inp.insert(embd_inp.end(), inp_sfx.begin(), inp_sfx.end());
|
||||
}
|
||||
|
||||
@ -517,7 +447,7 @@ int main(int argc, char ** argv) {
|
||||
|
||||
// end of text token
|
||||
if (!embd.empty() && embd.back() == llama_token_eos()) {
|
||||
if (instruct_mode) {
|
||||
if (params.instruct) {
|
||||
is_interacting = true;
|
||||
} else {
|
||||
fprintf(stderr, " [end of text]\n");
|
||||
|
||||
1
prompts/alpaca.txt
Normal file
1
prompts/alpaca.txt
Normal file
@ -0,0 +1 @@
|
||||
Below is an instruction that describes a task. Write a response that appropriately completes the request.
|
||||
7
prompts/chat-with-bob.txt
Normal file
7
prompts/chat-with-bob.txt
Normal file
@ -0,0 +1,7 @@
|
||||
Transcript of a dialog, where the User interacts with an Assistant named Bob. Bob is helpful, kind, honest, good at writing, and never fails to answer the User's requests immediately and with precision.
|
||||
|
||||
User: Hello, Bob.
|
||||
Bob: Hello. How may I help you today?
|
||||
User: Please tell me the largest city in Europe.
|
||||
Bob: Sure. The largest city in Europe is Moscow, the capital of Russia.
|
||||
User:
|
||||
Loading…
Reference in New Issue
Block a user