|
@@ -115,56 +115,81 @@ export async function chatCompletion({ messages, options, format, model, signal
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
export async function chatCompletionStream({ messages, onChunk, signal, options }) {
|
|
export async function chatCompletionStream({ messages, onChunk, signal, options }) {
|
|
|
- const res = await fetch(`${config.ollama.url}/api/chat`, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- model: config.ollama.chatModel,
|
|
|
|
|
- messages,
|
|
|
|
|
- stream: true,
|
|
|
|
|
- ...(options && Object.keys(options).length > 0 && { options })
|
|
|
|
|
- }),
|
|
|
|
|
- signal
|
|
|
|
|
- });
|
|
|
|
|
-
|
|
|
|
|
- if (!res.ok) {
|
|
|
|
|
- const text = await res.text().catch(() => "");
|
|
|
|
|
- const err = new Error(`ollama_error:${res.status}:${text || res.statusText}`);
|
|
|
|
|
- err.statusCode = 502;
|
|
|
|
|
- throw err;
|
|
|
|
|
|
|
+ const idleTimeoutMs = config.ollama.streamIdleTimeoutMs;
|
|
|
|
|
+ const idleController = new AbortController();
|
|
|
|
|
+ let idleTimer = setTimeout(
|
|
|
|
|
+ () => idleController.abort(new Error(`ollama_stream_idle_timeout:${idleTimeoutMs}ms`)),
|
|
|
|
|
+ idleTimeoutMs
|
|
|
|
|
+ );
|
|
|
|
|
+ const resetIdleTimer = () => {
|
|
|
|
|
+ clearTimeout(idleTimer);
|
|
|
|
|
+ idleTimer = setTimeout(
|
|
|
|
|
+ () => idleController.abort(new Error(`ollama_stream_idle_timeout:${idleTimeoutMs}ms`)),
|
|
|
|
|
+ idleTimeoutMs
|
|
|
|
|
+ );
|
|
|
|
|
+ };
|
|
|
|
|
+ const onExternalAbort = () => idleController.abort(signal.reason);
|
|
|
|
|
+ if (signal) {
|
|
|
|
|
+ if (signal.aborted) idleController.abort(signal.reason);
|
|
|
|
|
+ else signal.addEventListener("abort", onExternalAbort, { once: true });
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- const reader = res.body.getReader();
|
|
|
|
|
- const decoder = new TextDecoder();
|
|
|
|
|
- let fullContent = "";
|
|
|
|
|
- let buffer = "";
|
|
|
|
|
-
|
|
|
|
|
- const processLine = (line) => {
|
|
|
|
|
- if (!line.trim()) return;
|
|
|
|
|
- try {
|
|
|
|
|
- const parsed = JSON.parse(line);
|
|
|
|
|
- const delta = parsed?.message?.content ?? "";
|
|
|
|
|
- if (delta) {
|
|
|
|
|
- fullContent += delta;
|
|
|
|
|
- onChunk(delta);
|
|
|
|
|
- }
|
|
|
|
|
- } catch {}
|
|
|
|
|
- };
|
|
|
|
|
|
|
+ try {
|
|
|
|
|
+ const res = await fetch(`${config.ollama.url}/api/chat`, {
|
|
|
|
|
+ method: "POST",
|
|
|
|
|
+ headers: { "content-type": "application/json" },
|
|
|
|
|
+ body: JSON.stringify({
|
|
|
|
|
+ model: config.ollama.chatModel,
|
|
|
|
|
+ messages,
|
|
|
|
|
+ stream: true,
|
|
|
|
|
+ ...(options && Object.keys(options).length > 0 && { options })
|
|
|
|
|
+ }),
|
|
|
|
|
+ signal: idleController.signal
|
|
|
|
|
+ });
|
|
|
|
|
|
|
|
- while (true) {
|
|
|
|
|
- const { done, value } = await reader.read();
|
|
|
|
|
- if (done) break;
|
|
|
|
|
-
|
|
|
|
|
- buffer += decoder.decode(value, { stream: true });
|
|
|
|
|
- const lines = buffer.split("\n");
|
|
|
|
|
- buffer = lines.pop() ?? "";
|
|
|
|
|
- for (const line of lines) processLine(line);
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ if (!res.ok) {
|
|
|
|
|
+ const text = await res.text().catch(() => "");
|
|
|
|
|
+ const err = new Error(`ollama_error:${res.status}:${text || res.statusText}`);
|
|
|
|
|
+ err.statusCode = 502;
|
|
|
|
|
+ throw err;
|
|
|
|
|
+ }
|
|
|
|
|
|
|
|
- buffer += decoder.decode();
|
|
|
|
|
- processLine(buffer);
|
|
|
|
|
|
|
+ const reader = res.body.getReader();
|
|
|
|
|
+ const decoder = new TextDecoder();
|
|
|
|
|
+ let fullContent = "";
|
|
|
|
|
+ let buffer = "";
|
|
|
|
|
+
|
|
|
|
|
+ const processLine = (line) => {
|
|
|
|
|
+ if (!line.trim()) return;
|
|
|
|
|
+ try {
|
|
|
|
|
+ const parsed = JSON.parse(line);
|
|
|
|
|
+ const delta = parsed?.message?.content ?? "";
|
|
|
|
|
+ if (delta) {
|
|
|
|
|
+ fullContent += delta;
|
|
|
|
|
+ onChunk(delta);
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch {}
|
|
|
|
|
+ };
|
|
|
|
|
|
|
|
- return { content: fullContent };
|
|
|
|
|
|
|
+ while (true) {
|
|
|
|
|
+ const { done, value } = await reader.read();
|
|
|
|
|
+ if (done) break;
|
|
|
|
|
+ resetIdleTimer();
|
|
|
|
|
+
|
|
|
|
|
+ buffer += decoder.decode(value, { stream: true });
|
|
|
|
|
+ const lines = buffer.split("\n");
|
|
|
|
|
+ buffer = lines.pop() ?? "";
|
|
|
|
|
+ for (const line of lines) processLine(line);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ buffer += decoder.decode();
|
|
|
|
|
+ processLine(buffer);
|
|
|
|
|
+
|
|
|
|
|
+ return { content: fullContent };
|
|
|
|
|
+ } finally {
|
|
|
|
|
+ clearTimeout(idleTimer);
|
|
|
|
|
+ signal?.removeEventListener("abort", onExternalAbort);
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
export async function visionExtractFromImage({ imageBase64, prompt }) {
|
|
export async function visionExtractFromImage({ imageBase64, prompt }) {
|