root/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggerganov/llama.cpp.git synced 2024-12-25 02:44:36 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

server : add completion mode (no chat) (#3582)

Browse Source
This commit is contained in:
Aarni Koskela 2023-10-12 15:51:53 +09:00 committed by GitHub
parent 6b3ae4da92
commit b016596d90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 2317 additions and 2061 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4187
examples/server/index.html.hpp
View File

File diff suppressed because it is too large Load Diff

163
examples/server/public/index.html
View File

@ -136,6 +136,11 @@
display: block; display: block;
} }
fieldset label.slim {
margin: 0 0.5em;
display: inline;
}
header, footer { header, footer {
text-align: center; text-align: center;
} }
@ -145,6 +150,14 @@
color: #888; color: #888;
} }
.mode-chat textarea[name=prompt] {
height: 4.5em;
}
.mode-completion textarea[name=prompt] {
height: 10em;
}
@keyframes loading-bg-wipe { @keyframes loading-bg-wipe {
0% { 0% {
@ -187,7 +200,7 @@
template: "{{prompt}}\n\n{{history}}\n{{char}}:", template: "{{prompt}}\n\n{{history}}\n{{char}}:",
historyTemplate: "{{name}}: {{message}}", historyTemplate: "{{name}}: {{message}}",
transcript: [], transcript: [],
type: "chat", type: "chat", // "chat" | "completion"
char: "Llama", char: "Llama",
user: "User", user: "User",
}) })
@ -365,13 +378,44 @@
return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key])); return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
} }
async function runLlama(prompt, llamaParams, char) {
const currentMessages = [];
const history = session.value.transcript;
if (controller.value) {
throw new Error("already running");
}
controller.value = new AbortController();
for await (const chunk of llama(prompt, llamaParams, {controller: controller.value})) {
const data = chunk.data;
if (data.stop) {
while (
currentMessages.length > 0 &&
currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
) {
currentMessages.pop();
}
transcriptUpdate([...history, [char, currentMessages]])
console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data);
} else {
currentMessages.push(data);
transcriptUpdate([...history, [char, currentMessages]])
}
if (data.timings) {
llamaStats.value = data.timings;
}
}
controller.value = null;
}
// send message to server // send message to server
const chat = async (msg) => { const chat = async (msg) => {
if (controller.value) { if (controller.value) {
console.log('already running...'); console.log('already running...');
return; return;
} }
controller.value = new AbortController();
transcriptUpdate([...session.value.transcript, ["{{user}}", msg]]) transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
@ -391,42 +435,25 @@
).join("\n"), ).join("\n"),
}); });
const currentMessages = []; await runLlama(prompt, {
const history = session.value.transcript
const llamaParams = {
...params.value, ...params.value,
stop: ["</s>", template("{{char}}:"), template("{{user}}:")], stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
}, "{{char}}");
} }
for await (const chunk of llama(prompt, llamaParams, { controller: controller.value })) { const runCompletion = async () => {
const data = chunk.data; if (controller.value) {
console.log('already running...');
if (data.stop) { return;
while (
currentMessages.length > 0 &&
currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
) {
currentMessages.pop();
} }
transcriptUpdate([...history, ["{{char}}", currentMessages]]) const {prompt} = session.value;
console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data); transcriptUpdate([...session.value.transcript, ["", prompt]]);
} else { await runLlama(prompt, {
currentMessages.push(data); ...params.value,
transcriptUpdate([...history, ["{{char}}", currentMessages]]) stop: [],
}, "");
} }
if (data.timings) {
llamaStats.value = data.timings;
}
}
controller.value = null;
}
function MessageInput() {
const message = useSignal("")
const stop = (e) => { const stop = (e) => {
e.preventDefault(); e.preventDefault();
if (controller.value) { if (controller.value) {
@ -440,6 +467,9 @@
transcriptUpdate([]); transcriptUpdate([]);
} }
function MessageInput() {
const message = useSignal("")
const submit = (e) => { const submit = (e) => {
stop(e); stop(e);
chat(message.value); chat(message.value);
@ -474,6 +504,19 @@
` `
} }
function CompletionControls() {
const submit = (e) => {
stop(e);
runCompletion();
}
return html`
<div>
<button onclick=${submit} type="button" disabled=${generating.value}>Start</button>
<button onclick=${stop} disabled=${!generating.value}>Stop</button>
<button onclick=${reset}>Reset</button>
</div>`;
}
const ChatLog = (props) => { const ChatLog = (props) => {
const messages = session.value.transcript; const messages = session.value.transcript;
const container = useRef(null) const container = useRef(null)
@ -497,7 +540,11 @@
data; data;
message = html`<${Markdownish} text=${template(text)} />` message = html`<${Markdownish} text=${template(text)} />`
} }
if(user) {
return html`<p key=${index}><strong>${template(user)}:</strong> ${message}</p>` return html`<p key=${index}><strong>${template(user)}:</strong> ${message}</p>`
} else {
return html`<p key=${index}>${message}</p>`
}
}; };
return html` return html`
@ -574,18 +621,31 @@
userTemplateAutosave() userTemplateAutosave()
}, [session.value, params.value]) }, [session.value, params.value])
return html` const GrammarControl = () => (
<form> html`
<fieldset> <div>
<${UserTemplateResetButton}/> <label for="template">Grammar</label>
</fieldset> <textarea id="grammar" name="grammar" placeholder="Use gbnf or JSON Schema+convert" value="${params.value.grammar}" rows=4 oninput=${updateParams}/>
<input type="text" name="prop-order" placeholder="order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
<button type="button" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
</div>
`
);
const PromptControlFieldSet = () => (
html`
<fieldset> <fieldset>
<div> <div>
<label for="prompt">Prompt</label> <label htmlFor="prompt">Prompt</label>
<textarea type="text" name="prompt" value="${session.value.prompt}" rows=4 oninput=${updateSession}/> <textarea type="text" name="prompt" value="${session.value.prompt}" oninput=${updateSession}/>
</div> </div>
</fieldset> </fieldset>
`
);
const ChatConfigForm = () => (
html`
${PromptControlFieldSet()}
<fieldset class="two"> <fieldset class="two">
<div> <div>
@ -609,15 +669,30 @@
<label for="template">Chat history template</label> <label for="template">Chat history template</label>
<textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/> <textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
</div> </div>
${GrammarControl()}
</fieldset>
`
);
const CompletionConfigForm = () => (
html`
${PromptControlFieldSet()}
<fieldset>${GrammarControl()}</fieldset>
`
);
return html`
<form>
<fieldset class="two">
<${UserTemplateResetButton}/>
<div> <div>
<label for="template">Grammar</label> <label class="slim"><input type="radio" name="type" value="chat" checked=${session.value.type === "chat"} oninput=${updateSession} /> Chat</label>
<textarea id="grammar" name="grammar" placeholder="Use gbnf or JSON Schema+convert" value="${params.value.grammar}" rows=4 oninput=${updateParams}/> <label class="slim"><input type="radio" name="type" value="completion" checked=${session.value.type === "completion"} oninput=${updateSession} /> Completion</label>
<input type="text" name="prop-order" placeholder="order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
<button type="button" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
</div> </div>
</fieldset> </fieldset>
${session.value.type === 'chat' ? ChatConfigForm() : CompletionConfigForm()}
<fieldset class="two"> <fieldset class="two">
${IntField({label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict})} ${IntField({label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict})}
${FloatField({label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature})} ${FloatField({label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature})}
@ -851,7 +926,7 @@
function App(props) { function App(props) {
return html` return html`
<div> <div class="mode-${session.value.type}">
<header> <header>
<h1>llama.cpp</h1> <h1>llama.cpp</h1>
</header> </header>
@ -861,7 +936,7 @@
</main> </main>
<section id="write"> <section id="write">
<${MessageInput} /> <${session.value.type === 'chat' ? MessageInput : CompletionControls} />
</section> </section>
<footer> <footer>