|
@@ -5,7 +5,6 @@ import { ensureCollection } from "../src/services/collectionService.js";
|
|
|
import { qdrant } from "../src/services/qdrantClient.js";
|
|
import { qdrant } from "../src/services/qdrantClient.js";
|
|
|
import { createHash } from "node:crypto";
|
|
import { createHash } from "node:crypto";
|
|
|
import { createRequire } from "node:module";
|
|
import { createRequire } from "node:module";
|
|
|
-import fs from "node:fs";
|
|
|
|
|
import mammoth from "mammoth";
|
|
import mammoth from "mammoth";
|
|
|
|
|
|
|
|
const require = createRequire(import.meta.url);
|
|
const require = createRequire(import.meta.url);
|
|
@@ -76,37 +75,6 @@ export async function ingestDocuments(documents) {
|
|
|
|
|
|
|
|
if (allChunks.length === 0) return { upserted: 0 };
|
|
if (allChunks.length === 0) return { upserted: 0 };
|
|
|
|
|
|
|
|
- // #region debug-point B:ingest-chunks
|
|
|
|
|
- (() => {
|
|
|
|
|
- let u = "http://127.0.0.1:7777/event";
|
|
|
|
|
- let s = "image-rag-miss";
|
|
|
|
|
- try {
|
|
|
|
|
- const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
|
|
|
|
|
- u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
|
|
|
|
|
- s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
|
|
|
|
|
- } catch {}
|
|
|
|
|
- fetch(u, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- sessionId: s,
|
|
|
|
|
- runId: "post",
|
|
|
|
|
- hypothesisId: "B",
|
|
|
|
|
- location: "api/chat/ingest.js",
|
|
|
|
|
- msg: "[DEBUG] ingestDocuments chunks built",
|
|
|
|
|
- data: {
|
|
|
|
|
- collection: collectionName,
|
|
|
|
|
- documents: Array.isArray(documents) ? documents.length : null,
|
|
|
|
|
- chunks: allChunks.length,
|
|
|
|
|
- avgChunkLen: allChunks.length ? Math.round(allChunks.reduce((a, c) => a + (c?.text?.length ?? 0), 0) / allChunks.length) : 0,
|
|
|
|
|
- sample0: allChunks[0]?.text ? String(allChunks[0].text).slice(0, 160) : ""
|
|
|
|
|
- },
|
|
|
|
|
- ts: Date.now()
|
|
|
|
|
- })
|
|
|
|
|
- }).catch(() => {});
|
|
|
|
|
- })();
|
|
|
|
|
- // #endregion
|
|
|
|
|
-
|
|
|
|
|
const vectors = await embedTexts(allChunks.map((c) => c.text));
|
|
const vectors = await embedTexts(allChunks.map((c) => c.text));
|
|
|
const vectorSize = vectors[0]?.length ?? 0;
|
|
const vectorSize = vectors[0]?.length ?? 0;
|
|
|
if (!vectorSize) {
|
|
if (!vectorSize) {
|
|
@@ -128,61 +96,11 @@ export async function ingestDocuments(documents) {
|
|
|
}
|
|
}
|
|
|
}));
|
|
}));
|
|
|
|
|
|
|
|
- // #region debug-point B:ingest-upsert
|
|
|
|
|
- (() => {
|
|
|
|
|
- let u = "http://127.0.0.1:7777/event";
|
|
|
|
|
- let s = "image-rag-miss";
|
|
|
|
|
- try {
|
|
|
|
|
- const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
|
|
|
|
|
- u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
|
|
|
|
|
- s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
|
|
|
|
|
- } catch {}
|
|
|
|
|
- fetch(u, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- sessionId: s,
|
|
|
|
|
- runId: "post",
|
|
|
|
|
- hypothesisId: "B",
|
|
|
|
|
- location: "api/chat/ingest.js",
|
|
|
|
|
- msg: "[DEBUG] ingestDocuments upsert start",
|
|
|
|
|
- data: { collection: collectionName, points: points.length, vectorSize },
|
|
|
|
|
- ts: Date.now()
|
|
|
|
|
- })
|
|
|
|
|
- }).catch(() => {});
|
|
|
|
|
- })();
|
|
|
|
|
- // #endregion
|
|
|
|
|
-
|
|
|
|
|
await qdrant.upsert(collectionName, {
|
|
await qdrant.upsert(collectionName, {
|
|
|
wait: true,
|
|
wait: true,
|
|
|
points
|
|
points
|
|
|
});
|
|
});
|
|
|
|
|
|
|
|
- // #region debug-point B:ingest-upsert-done
|
|
|
|
|
- (() => {
|
|
|
|
|
- let u = "http://127.0.0.1:7777/event";
|
|
|
|
|
- let s = "image-rag-miss";
|
|
|
|
|
- try {
|
|
|
|
|
- const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
|
|
|
|
|
- u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
|
|
|
|
|
- s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
|
|
|
|
|
- } catch {}
|
|
|
|
|
- fetch(u, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- sessionId: s,
|
|
|
|
|
- runId: "post",
|
|
|
|
|
- hypothesisId: "B",
|
|
|
|
|
- location: "api/chat/ingest.js",
|
|
|
|
|
- msg: "[DEBUG] ingestDocuments upsert done",
|
|
|
|
|
- data: { collection: collectionName, points: points.length },
|
|
|
|
|
- ts: Date.now()
|
|
|
|
|
- })
|
|
|
|
|
- }).catch(() => {});
|
|
|
|
|
- })();
|
|
|
|
|
- // #endregion
|
|
|
|
|
-
|
|
|
|
|
return { upserted: points.length };
|
|
return { upserted: points.length };
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -206,57 +124,8 @@ export async function extractDocumentsFromUpload({ buffer, filename, mimeType, s
|
|
|
const src = source ?? filename ?? "upload";
|
|
const src = source ?? filename ?? "upload";
|
|
|
const metadata = { filename: filename ?? null, mimeType: mimeType ?? null, kind };
|
|
const metadata = { filename: filename ?? null, mimeType: mimeType ?? null, kind };
|
|
|
|
|
|
|
|
- // #region debug-point A:extract-start
|
|
|
|
|
- (() => {
|
|
|
|
|
- let u = "http://127.0.0.1:7777/event";
|
|
|
|
|
- let s = "image-rag-miss";
|
|
|
|
|
- try {
|
|
|
|
|
- const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
|
|
|
|
|
- u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
|
|
|
|
|
- s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
|
|
|
|
|
- } catch {}
|
|
|
|
|
- fetch(u, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- sessionId: s,
|
|
|
|
|
- runId: "post",
|
|
|
|
|
- hypothesisId: "A",
|
|
|
|
|
- location: "api/chat/ingest.js",
|
|
|
|
|
- msg: "[DEBUG] extractDocumentsFromUpload start",
|
|
|
|
|
- data: { kind, filename: filename ?? null, mimeType: mimeType ?? null, bytes: buffer?.length ?? null, source: src },
|
|
|
|
|
- ts: Date.now()
|
|
|
|
|
- })
|
|
|
|
|
- }).catch(() => {});
|
|
|
|
|
- })();
|
|
|
|
|
- // #endregion
|
|
|
|
|
-
|
|
|
|
|
if (kind === "txt") {
|
|
if (kind === "txt") {
|
|
|
const text = buffer.toString("utf8").trim();
|
|
const text = buffer.toString("utf8").trim();
|
|
|
- // #region debug-point A:extract-txt
|
|
|
|
|
- (() => {
|
|
|
|
|
- let u = "http://127.0.0.1:7777/event";
|
|
|
|
|
- let s = "image-rag-miss";
|
|
|
|
|
- try {
|
|
|
|
|
- const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
|
|
|
|
|
- u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
|
|
|
|
|
- s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
|
|
|
|
|
- } catch {}
|
|
|
|
|
- fetch(u, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- sessionId: s,
|
|
|
|
|
- runId: "post",
|
|
|
|
|
- hypothesisId: "A",
|
|
|
|
|
- location: "api/chat/ingest.js",
|
|
|
|
|
- msg: "[DEBUG] extract txt done",
|
|
|
|
|
- data: { chars: text.length, head: text.slice(0, 160) },
|
|
|
|
|
- ts: Date.now()
|
|
|
|
|
- })
|
|
|
|
|
- }).catch(() => {});
|
|
|
|
|
- })();
|
|
|
|
|
- // #endregion
|
|
|
|
|
return [{ text, source: src, metadata }];
|
|
return [{ text, source: src, metadata }];
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -282,30 +151,6 @@ export async function extractDocumentsFromUpload({ buffer, filename, mimeType, s
|
|
|
|
|
|
|
|
const parsed = await pdfParse(buffer);
|
|
const parsed = await pdfParse(buffer);
|
|
|
const text = String(parsed?.text ?? "").trim();
|
|
const text = String(parsed?.text ?? "").trim();
|
|
|
- // #region debug-point A:extract-pdf
|
|
|
|
|
- (() => {
|
|
|
|
|
- let u = "http://127.0.0.1:7777/event";
|
|
|
|
|
- let s = "image-rag-miss";
|
|
|
|
|
- try {
|
|
|
|
|
- const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
|
|
|
|
|
- u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
|
|
|
|
|
- s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
|
|
|
|
|
- } catch {}
|
|
|
|
|
- fetch(u, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- sessionId: s,
|
|
|
|
|
- runId: "post",
|
|
|
|
|
- hypothesisId: "A",
|
|
|
|
|
- location: "api/chat/ingest.js",
|
|
|
|
|
- msg: "[DEBUG] extract pdf done",
|
|
|
|
|
- data: { chars: text.length, head: text.slice(0, 160) },
|
|
|
|
|
- ts: Date.now()
|
|
|
|
|
- })
|
|
|
|
|
- }).catch(() => {});
|
|
|
|
|
- })();
|
|
|
|
|
- // #endregion
|
|
|
|
|
return [{ text, source: src, metadata }];
|
|
return [{ text, source: src, metadata }];
|
|
|
}
|
|
}
|
|
|
|
|
|
|
@@ -373,38 +218,6 @@ export async function extractDocumentsFromUpload({ buffer, filename, mimeType, s
|
|
|
if (ocrTexts.length) textParts.push(ocrTexts.join("\n\n"));
|
|
if (ocrTexts.length) textParts.push(ocrTexts.join("\n\n"));
|
|
|
|
|
|
|
|
const text = textParts.join("\n\n").trim();
|
|
const text = textParts.join("\n\n").trim();
|
|
|
- // #region debug-point A:extract-docx
|
|
|
|
|
- (() => {
|
|
|
|
|
- let u = "http://127.0.0.1:7777/event";
|
|
|
|
|
- let s = "image-rag-miss";
|
|
|
|
|
- try {
|
|
|
|
|
- const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
|
|
|
|
|
- u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
|
|
|
|
|
- s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
|
|
|
|
|
- } catch {}
|
|
|
|
|
- fetch(u, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- sessionId: s,
|
|
|
|
|
- runId: "post",
|
|
|
|
|
- hypothesisId: "A",
|
|
|
|
|
- location: "api/chat/ingest.js",
|
|
|
|
|
- msg: "[DEBUG] extract docx done",
|
|
|
|
|
- data: {
|
|
|
|
|
- baseChars: baseText.length,
|
|
|
|
|
- totalChars: text.length,
|
|
|
|
|
- imagesTotal: images.length,
|
|
|
|
|
- imagesProcessed: images.length - visionSkipped,
|
|
|
|
|
- imagesSkipped: visionSkipped,
|
|
|
|
|
- visionError: visionError || null,
|
|
|
|
|
- head: text.slice(0, 160)
|
|
|
|
|
- },
|
|
|
|
|
- ts: Date.now()
|
|
|
|
|
- })
|
|
|
|
|
- }).catch(() => {});
|
|
|
|
|
- })();
|
|
|
|
|
- // #endregion
|
|
|
|
|
return [
|
|
return [
|
|
|
{
|
|
{
|
|
|
text,
|
|
text,
|
|
@@ -437,30 +250,6 @@ export async function extractDocumentsFromUpload({ buffer, filename, mimeType, s
|
|
|
});
|
|
});
|
|
|
text = String(r?.content ?? "").trim();
|
|
text = String(r?.content ?? "").trim();
|
|
|
}
|
|
}
|
|
|
- // #region debug-point A:extract-image
|
|
|
|
|
- (() => {
|
|
|
|
|
- let u = "http://127.0.0.1:7777/event";
|
|
|
|
|
- let s = "image-rag-miss";
|
|
|
|
|
- try {
|
|
|
|
|
- const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
|
|
|
|
|
- u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
|
|
|
|
|
- s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
|
|
|
|
|
- } catch {}
|
|
|
|
|
- fetch(u, {
|
|
|
|
|
- method: "POST",
|
|
|
|
|
- headers: { "content-type": "application/json" },
|
|
|
|
|
- body: JSON.stringify({
|
|
|
|
|
- sessionId: s,
|
|
|
|
|
- runId: "pre",
|
|
|
|
|
- hypothesisId: "A",
|
|
|
|
|
- location: "api/chat/ingest.js",
|
|
|
|
|
- msg: "[DEBUG] extract image done",
|
|
|
|
|
- data: { chars: text.length, head: text.slice(0, 160) },
|
|
|
|
|
- ts: Date.now()
|
|
|
|
|
- })
|
|
|
|
|
- }).catch(() => {});
|
|
|
|
|
- })();
|
|
|
|
|
- // #endregion
|
|
|
|
|
if (isVisionRefusal(text)) {
|
|
if (isVisionRefusal(text)) {
|
|
|
const err = new Error("vision_refused");
|
|
const err = new Error("vision_refused");
|
|
|
err.statusCode = 400;
|
|
err.statusCode = 400;
|