import { Router } from "express"; import { z } from "zod"; import multer from "multer"; import { extractDocumentsFromUpload, ingestDocuments } from "../chat/ingest.js"; import fs from "node:fs"; export const ingestRouter = Router(); const upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 25 * 1024 * 1024 } }); const docSchema = z.object({ id: z.string().min(1).optional(), text: z.string().min(1), source: z.string().min(1).optional(), metadata: z.record(z.any()).optional() }); const ingestBodySchema = z.object({ documents: z.array(docSchema).min(1) }); ingestRouter.post("/", async (req, res, next) => { try { const body = ingestBodySchema.parse(req.body); const result = await ingestDocuments(body.documents); res.json(result); } catch (err) { next(err); } }); ingestRouter.post("/file", upload.single("file"), async (req, res, next) => { try { const f = req.file; if (!f?.buffer) { res.status(400).json({ error: "file_required" }); return; } const source = typeof req.body?.source === "string" && req.body.source.trim() ? req.body.source.trim() : undefined; // #region debug-point A:ingest-file-start (() => { let u = "http://127.0.0.1:7777/event"; let s = "image-rag-miss"; try { const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8"); u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u; s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s; } catch {} fetch(u, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ sessionId: s, runId: "post", hypothesisId: "A", location: "api/routes/ingest.js", msg: "[DEBUG] /api/ingest/file received", data: { filename: f.originalname, mimeType: f.mimetype, bytes: f.size ?? null, source: source ?? null }, ts: Date.now() }) }).catch(() => {}); })(); // #endregion const docs = await extractDocumentsFromUpload({ buffer: f.buffer, filename: f.originalname, mimeType: f.mimetype, source }); const extractedChars = docs.reduce((acc, d) => acc + (d?.text?.length ?? 0), 0); // #region debug-point A:ingest-file-extracted (() => { let u = "http://127.0.0.1:7777/event"; let s = "image-rag-miss"; try { const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8"); u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u; s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s; } catch {} fetch(u, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ sessionId: s, runId: "post", hypothesisId: "A", location: "api/routes/ingest.js", msg: "[DEBUG] /api/ingest/file extracted", data: { documents: docs.length, extractedChars, head: String(docs?.[0]?.text ?? "").slice(0, 160) }, ts: Date.now() }) }).catch(() => {}); })(); // #endregion if (!extractedChars) { res.status(400).json({ error: "empty_extracted_text" }); return; } const result = await ingestDocuments(docs); res.json({ ...result, documents: docs.length, extractedChars }); } catch (err) { next(err); } });