backend/routes/ingest.js

import { Router } from "express";
import { z } from "zod";
import multer from "multer";
import { extractDocumentsFromUpload, ingestDocuments } from "../chat/ingest.js";
import fs from "node:fs";

export const ingestRouter = Router();

const upload = multer({
  storage: multer.memoryStorage(),
  limits: { fileSize: 25 * 1024 * 1024 }
});

const docSchema = z.object({
  id: z.string().min(1).optional(),
  text: z.string().min(1),
  source: z.string().min(1).optional(),
  metadata: z.record(z.any()).optional()
});

const ingestBodySchema = z.object({
  documents: z.array(docSchema).min(1)
});

ingestRouter.post("/", async (req, res, next) => {
  try {
    const body = ingestBodySchema.parse(req.body);
    const result = await ingestDocuments(body.documents);
    res.json(result);
  } catch (err) {
    next(err);
  }
});

ingestRouter.post("/file", upload.single("file"), async (req, res, next) => {
  try {
    const f = req.file;
    if (!f?.buffer) {
      res.status(400).json({ error: "file_required" });
      return;
    }

    const source = typeof req.body?.source === "string" && req.body.source.trim() ? req.body.source.trim() : undefined;
    // #region debug-point A:ingest-file-start
    (() => {
      let u = "http://127.0.0.1:7777/event";
      let s = "image-rag-miss";
      try {
        const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
        u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
        s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
      } catch {}
      fetch(u, {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify({
          sessionId: s,
          runId: "post",
          hypothesisId: "A",
          location: "api/routes/ingest.js",
          msg: "[DEBUG] /api/ingest/file received",
          data: { filename: f.originalname, mimeType: f.mimetype, bytes: f.size ?? null, source: source ?? null },
          ts: Date.now()
        })
      }).catch(() => {});
    })();
    // #endregion
    const docs = await extractDocumentsFromUpload({
      buffer: f.buffer,
      filename: f.originalname,
      mimeType: f.mimetype,
      source
    });

    const extractedChars = docs.reduce((acc, d) => acc + (d?.text?.length ?? 0), 0);
    // #region debug-point A:ingest-file-extracted
    (() => {
      let u = "http://127.0.0.1:7777/event";
      let s = "image-rag-miss";
      try {
        const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
        u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
        s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
      } catch {}
      fetch(u, {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify({
          sessionId: s,
          runId: "post",
          hypothesisId: "A",
          location: "api/routes/ingest.js",
          msg: "[DEBUG] /api/ingest/file extracted",
          data: { documents: docs.length, extractedChars, head: String(docs?.[0]?.text ?? "").slice(0, 160) },
          ts: Date.now()
        })
      }).catch(() => {});
    })();
    // #endregion
    if (!extractedChars) {
      res.status(400).json({ error: "empty_extracted_text" });
      return;
    }

    const result = await ingestDocuments(docs);
    res.json({ ...result, documents: docs.length, extractedChars });
  } catch (err) {
    next(err);
  }
});