mirror of
https://github.com/ggerganov/llama.cpp.git
synced 2024-12-24 10:24:35 +00:00
server : add completion mode (no chat) (#3582)
This commit is contained in:
parent
6b3ae4da92
commit
b016596d90
File diff suppressed because it is too large
Load Diff
@ -136,6 +136,11 @@
|
|||||||
display: block;
|
display: block;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fieldset label.slim {
|
||||||
|
margin: 0 0.5em;
|
||||||
|
display: inline;
|
||||||
|
}
|
||||||
|
|
||||||
header, footer {
|
header, footer {
|
||||||
text-align: center;
|
text-align: center;
|
||||||
}
|
}
|
||||||
@ -145,6 +150,14 @@
|
|||||||
color: #888;
|
color: #888;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.mode-chat textarea[name=prompt] {
|
||||||
|
height: 4.5em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.mode-completion textarea[name=prompt] {
|
||||||
|
height: 10em;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
@keyframes loading-bg-wipe {
|
@keyframes loading-bg-wipe {
|
||||||
0% {
|
0% {
|
||||||
@ -187,7 +200,7 @@
|
|||||||
template: "{{prompt}}\n\n{{history}}\n{{char}}:",
|
template: "{{prompt}}\n\n{{history}}\n{{char}}:",
|
||||||
historyTemplate: "{{name}}: {{message}}",
|
historyTemplate: "{{name}}: {{message}}",
|
||||||
transcript: [],
|
transcript: [],
|
||||||
type: "chat",
|
type: "chat", // "chat" | "completion"
|
||||||
char: "Llama",
|
char: "Llama",
|
||||||
user: "User",
|
user: "User",
|
||||||
})
|
})
|
||||||
@ -365,13 +378,44 @@
|
|||||||
return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
|
return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function runLlama(prompt, llamaParams, char) {
|
||||||
|
const currentMessages = [];
|
||||||
|
const history = session.value.transcript;
|
||||||
|
if (controller.value) {
|
||||||
|
throw new Error("already running");
|
||||||
|
}
|
||||||
|
controller.value = new AbortController();
|
||||||
|
for await (const chunk of llama(prompt, llamaParams, {controller: controller.value})) {
|
||||||
|
const data = chunk.data;
|
||||||
|
|
||||||
|
if (data.stop) {
|
||||||
|
while (
|
||||||
|
currentMessages.length > 0 &&
|
||||||
|
currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
|
||||||
|
) {
|
||||||
|
currentMessages.pop();
|
||||||
|
}
|
||||||
|
transcriptUpdate([...history, [char, currentMessages]])
|
||||||
|
console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data);
|
||||||
|
} else {
|
||||||
|
currentMessages.push(data);
|
||||||
|
transcriptUpdate([...history, [char, currentMessages]])
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.timings) {
|
||||||
|
llamaStats.value = data.timings;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
controller.value = null;
|
||||||
|
}
|
||||||
|
|
||||||
// send message to server
|
// send message to server
|
||||||
const chat = async (msg) => {
|
const chat = async (msg) => {
|
||||||
if (controller.value) {
|
if (controller.value) {
|
||||||
console.log('already running...');
|
console.log('already running...');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
controller.value = new AbortController();
|
|
||||||
|
|
||||||
transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
|
transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
|
||||||
|
|
||||||
@ -391,55 +435,41 @@
|
|||||||
).join("\n"),
|
).join("\n"),
|
||||||
});
|
});
|
||||||
|
|
||||||
const currentMessages = [];
|
await runLlama(prompt, {
|
||||||
const history = session.value.transcript
|
|
||||||
|
|
||||||
const llamaParams = {
|
|
||||||
...params.value,
|
...params.value,
|
||||||
stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
|
stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
|
||||||
|
}, "{{char}}");
|
||||||
|
}
|
||||||
|
|
||||||
|
const runCompletion = async () => {
|
||||||
|
if (controller.value) {
|
||||||
|
console.log('already running...');
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
const {prompt} = session.value;
|
||||||
|
transcriptUpdate([...session.value.transcript, ["", prompt]]);
|
||||||
|
await runLlama(prompt, {
|
||||||
|
...params.value,
|
||||||
|
stop: [],
|
||||||
|
}, "");
|
||||||
|
}
|
||||||
|
|
||||||
for await (const chunk of llama(prompt, llamaParams, { controller: controller.value })) {
|
const stop = (e) => {
|
||||||
const data = chunk.data;
|
e.preventDefault();
|
||||||
|
if (controller.value) {
|
||||||
if (data.stop) {
|
controller.value.abort();
|
||||||
while (
|
controller.value = null;
|
||||||
currentMessages.length > 0 &&
|
|
||||||
currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
|
|
||||||
) {
|
|
||||||
currentMessages.pop();
|
|
||||||
}
|
|
||||||
transcriptUpdate([...history, ["{{char}}", currentMessages]])
|
|
||||||
console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data);
|
|
||||||
} else {
|
|
||||||
currentMessages.push(data);
|
|
||||||
transcriptUpdate([...history, ["{{char}}", currentMessages]])
|
|
||||||
}
|
|
||||||
|
|
||||||
if (data.timings) {
|
|
||||||
llamaStats.value = data.timings;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
controller.value = null;
|
const reset = (e) => {
|
||||||
|
stop(e);
|
||||||
|
transcriptUpdate([]);
|
||||||
}
|
}
|
||||||
|
|
||||||
function MessageInput() {
|
function MessageInput() {
|
||||||
const message = useSignal("")
|
const message = useSignal("")
|
||||||
|
|
||||||
const stop = (e) => {
|
|
||||||
e.preventDefault();
|
|
||||||
if (controller.value) {
|
|
||||||
controller.value.abort();
|
|
||||||
controller.value = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const reset = (e) => {
|
|
||||||
stop(e);
|
|
||||||
transcriptUpdate([]);
|
|
||||||
}
|
|
||||||
|
|
||||||
const submit = (e) => {
|
const submit = (e) => {
|
||||||
stop(e);
|
stop(e);
|
||||||
chat(message.value);
|
chat(message.value);
|
||||||
@ -474,6 +504,19 @@
|
|||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function CompletionControls() {
|
||||||
|
const submit = (e) => {
|
||||||
|
stop(e);
|
||||||
|
runCompletion();
|
||||||
|
}
|
||||||
|
return html`
|
||||||
|
<div>
|
||||||
|
<button onclick=${submit} type="button" disabled=${generating.value}>Start</button>
|
||||||
|
<button onclick=${stop} disabled=${!generating.value}>Stop</button>
|
||||||
|
<button onclick=${reset}>Reset</button>
|
||||||
|
</div>`;
|
||||||
|
}
|
||||||
|
|
||||||
const ChatLog = (props) => {
|
const ChatLog = (props) => {
|
||||||
const messages = session.value.transcript;
|
const messages = session.value.transcript;
|
||||||
const container = useRef(null)
|
const container = useRef(null)
|
||||||
@ -497,7 +540,11 @@
|
|||||||
data;
|
data;
|
||||||
message = html`<${Markdownish} text=${template(text)} />`
|
message = html`<${Markdownish} text=${template(text)} />`
|
||||||
}
|
}
|
||||||
return html`<p key=${index}><strong>${template(user)}:</strong> ${message}</p>`
|
if(user) {
|
||||||
|
return html`<p key=${index}><strong>${template(user)}:</strong> ${message}</p>`
|
||||||
|
} else {
|
||||||
|
return html`<p key=${index}>${message}</p>`
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
return html`
|
return html`
|
||||||
@ -574,18 +621,31 @@
|
|||||||
userTemplateAutosave()
|
userTemplateAutosave()
|
||||||
}, [session.value, params.value])
|
}, [session.value, params.value])
|
||||||
|
|
||||||
return html`
|
const GrammarControl = () => (
|
||||||
<form>
|
html`
|
||||||
<fieldset>
|
<div>
|
||||||
<${UserTemplateResetButton}/>
|
<label for="template">Grammar</label>
|
||||||
</fieldset>
|
<textarea id="grammar" name="grammar" placeholder="Use gbnf or JSON Schema+convert" value="${params.value.grammar}" rows=4 oninput=${updateParams}/>
|
||||||
|
<input type="text" name="prop-order" placeholder="order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
|
||||||
|
<button type="button" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
|
||||||
|
</div>
|
||||||
|
`
|
||||||
|
);
|
||||||
|
|
||||||
<fieldset>
|
const PromptControlFieldSet = () => (
|
||||||
<div>
|
html`
|
||||||
<label for="prompt">Prompt</label>
|
<fieldset>
|
||||||
<textarea type="text" name="prompt" value="${session.value.prompt}" rows=4 oninput=${updateSession}/>
|
<div>
|
||||||
</div>
|
<label htmlFor="prompt">Prompt</label>
|
||||||
</fieldset>
|
<textarea type="text" name="prompt" value="${session.value.prompt}" oninput=${updateSession}/>
|
||||||
|
</div>
|
||||||
|
</fieldset>
|
||||||
|
`
|
||||||
|
);
|
||||||
|
|
||||||
|
const ChatConfigForm = () => (
|
||||||
|
html`
|
||||||
|
${PromptControlFieldSet()}
|
||||||
|
|
||||||
<fieldset class="two">
|
<fieldset class="two">
|
||||||
<div>
|
<div>
|
||||||
@ -609,15 +669,30 @@
|
|||||||
<label for="template">Chat history template</label>
|
<label for="template">Chat history template</label>
|
||||||
<textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
|
<textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
|
||||||
</div>
|
</div>
|
||||||
|
${GrammarControl()}
|
||||||
|
</fieldset>
|
||||||
|
`
|
||||||
|
);
|
||||||
|
|
||||||
|
const CompletionConfigForm = () => (
|
||||||
|
html`
|
||||||
|
${PromptControlFieldSet()}
|
||||||
|
<fieldset>${GrammarControl()}</fieldset>
|
||||||
|
`
|
||||||
|
);
|
||||||
|
|
||||||
|
return html`
|
||||||
|
<form>
|
||||||
|
<fieldset class="two">
|
||||||
|
<${UserTemplateResetButton}/>
|
||||||
<div>
|
<div>
|
||||||
<label for="template">Grammar</label>
|
<label class="slim"><input type="radio" name="type" value="chat" checked=${session.value.type === "chat"} oninput=${updateSession} /> Chat</label>
|
||||||
<textarea id="grammar" name="grammar" placeholder="Use gbnf or JSON Schema+convert" value="${params.value.grammar}" rows=4 oninput=${updateParams}/>
|
<label class="slim"><input type="radio" name="type" value="completion" checked=${session.value.type === "completion"} oninput=${updateSession} /> Completion</label>
|
||||||
<input type="text" name="prop-order" placeholder="order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
|
|
||||||
<button type="button" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
|
|
||||||
</div>
|
</div>
|
||||||
</fieldset>
|
</fieldset>
|
||||||
|
|
||||||
|
${session.value.type === 'chat' ? ChatConfigForm() : CompletionConfigForm()}
|
||||||
|
|
||||||
<fieldset class="two">
|
<fieldset class="two">
|
||||||
${IntField({label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict})}
|
${IntField({label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict})}
|
||||||
${FloatField({label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature})}
|
${FloatField({label: "Temperature", max: 1.5, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature})}
|
||||||
@ -851,7 +926,7 @@
|
|||||||
function App(props) {
|
function App(props) {
|
||||||
|
|
||||||
return html`
|
return html`
|
||||||
<div>
|
<div class="mode-${session.value.type}">
|
||||||
<header>
|
<header>
|
||||||
<h1>llama.cpp</h1>
|
<h1>llama.cpp</h1>
|
||||||
</header>
|
</header>
|
||||||
@ -861,7 +936,7 @@
|
|||||||
</main>
|
</main>
|
||||||
|
|
||||||
<section id="write">
|
<section id="write">
|
||||||
<${MessageInput} />
|
<${session.value.type === 'chat' ? MessageInput : CompletionControls} />
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<footer>
|
<footer>
|
||||||
|
Loading…
Reference in New Issue
Block a user