backend/chat/ingest.js

import { config } from "../src/config/index.js";
import { chunkText } from "../src/services/textChunker.js";
import { embedTexts, visionExtractFromImage } from "../src/services/ollamaClient.js";
import { ensureCollection } from "../src/services/collectionService.js";
import { qdrant } from "../src/services/qdrantClient.js";
import { createHash } from "node:crypto";
import { createRequire } from "node:module";
import fs from "node:fs";
import mammoth from "mammoth";

const require = createRequire(import.meta.url);

function isVisionRefusal(text) {
  const t = String(text ?? "").toLowerCase();
  return (
    t.includes("desculpe") &&
    (t.includes("não posso") || t.includes("nao posso") || t.includes("não posso fornecer") || t.includes("nao posso fornecer"))
  );
}

function visionPrompt() {
  return [
    "Analise a imagem (print de tela / manual interno).",
    "Extraia apenas informações úteis para busca: nomes de campos, rótulos, valores exibidos, opções selecionadas (checkbox/radio/dropdown), botões, mensagens de erro e códigos.",
    "Não transcreva parágrafos longos; prefira listas curtas e objetivas.",
    "Responda em português, em texto puro."
  ].join("\n");
}

function stableUuid(seed) {
  const hex = createHash("sha1")
    .update(String(seed))
    .digest("hex")
    .slice(0, 32);

  const timeLow = hex.slice(0, 8);
  const timeMid = hex.slice(8, 12);
  let timeHiAndVersion = parseInt(hex.slice(12, 16), 16);
  timeHiAndVersion = (timeHiAndVersion & 0x0fff) | 0x5000;

  let clockSeqHi = parseInt(hex.slice(16, 18), 16);
  clockSeqHi = (clockSeqHi & 0x3f) | 0x80;
  const clockSeqLow = hex.slice(18, 20);

  const node = hex.slice(20, 32);

  return [
    timeLow,
    timeMid,
    timeHiAndVersion.toString(16).padStart(4, "0"),
    `${clockSeqHi.toString(16).padStart(2, "0")}${clockSeqLow}`,
    node
  ].join("-");
}

export async function ingestDocuments(documents) {
  const collectionName = config.qdrant.collection;
  const allChunks = [];

  for (const doc of documents) {
    const baseSeed = doc.id ?? `${doc.source ?? "doc"}:${doc.text.slice(0, 64)}`;
    const chunks = chunkText(doc.text, {
      chunkSize: config.rag.chunkSize,
      chunkOverlap: config.rag.chunkOverlap
    });
    chunks.forEach((chunk, idx) => {
      allChunks.push({
        id: stableUuid(`${baseSeed}:${idx}`),
        source: doc.source ?? null,
        metadata: doc.metadata ?? null,
        chunkIndex: idx,
        text: chunk
      });
    });
  }

  if (allChunks.length === 0) return { upserted: 0 };

  // #region debug-point B:ingest-chunks
  (() => {
    let u = "http://127.0.0.1:7777/event";
    let s = "image-rag-miss";
    try {
      const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
      u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
      s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
    } catch {}
    fetch(u, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({
        sessionId: s,
        runId: "post",
        hypothesisId: "B",
        location: "api/chat/ingest.js",
        msg: "[DEBUG] ingestDocuments chunks built",
        data: {
          collection: collectionName,
          documents: Array.isArray(documents) ? documents.length : null,
          chunks: allChunks.length,
          avgChunkLen: allChunks.length ? Math.round(allChunks.reduce((a, c) => a + (c?.text?.length ?? 0), 0) / allChunks.length) : 0,
          sample0: allChunks[0]?.text ? String(allChunks[0].text).slice(0, 160) : ""
        },
        ts: Date.now()
      })
    }).catch(() => {});
  })();
  // #endregion

  const vectors = await embedTexts(allChunks.map((c) => c.text));
  const vectorSize = vectors[0]?.length ?? 0;
  if (!vectorSize) {
    const err = new Error("embeddings_empty");
    err.statusCode = 502;
    throw err;
  }

  await ensureCollection({ vectorSize });

  const points = allChunks.map((c, idx) => ({
    id: c.id,
    vector: vectors[idx],
    payload: {
      source: c.source,
      chunkIndex: c.chunkIndex,
      text: c.text,
      metadata: c.metadata
    }
  }));

  // #region debug-point B:ingest-upsert
  (() => {
    let u = "http://127.0.0.1:7777/event";
    let s = "image-rag-miss";
    try {
      const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
      u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
      s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
    } catch {}
    fetch(u, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({
        sessionId: s,
        runId: "post",
        hypothesisId: "B",
        location: "api/chat/ingest.js",
        msg: "[DEBUG] ingestDocuments upsert start",
        data: { collection: collectionName, points: points.length, vectorSize },
        ts: Date.now()
      })
    }).catch(() => {});
  })();
  // #endregion

  await qdrant.upsert(collectionName, {
    wait: true,
    points
  });

  // #region debug-point B:ingest-upsert-done
  (() => {
    let u = "http://127.0.0.1:7777/event";
    let s = "image-rag-miss";
    try {
      const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
      u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
      s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
    } catch {}
    fetch(u, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({
        sessionId: s,
        runId: "post",
        hypothesisId: "B",
        location: "api/chat/ingest.js",
        msg: "[DEBUG] ingestDocuments upsert done",
        data: { collection: collectionName, points: points.length },
        ts: Date.now()
      })
    }).catch(() => {});
  })();
  // #endregion

  return { upserted: points.length };
}

function guessFileKind({ mimeType, filename }) {
  const name = String(filename ?? "").toLowerCase();
  const mt = String(mimeType ?? "").toLowerCase();

  if (mt === "text/plain" || name.endsWith(".txt")) return "txt";
  if (mt === "application/pdf" || name.endsWith(".pdf")) return "pdf";
  if (
    mt === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
    name.endsWith(".docx")
  )
    return "docx";
  if (mt.startsWith("image/") || /\.(png|jpe?g|webp)$/i.test(name)) return "image";
  return "unknown";
}

export async function extractDocumentsFromUpload({ buffer, filename, mimeType, source }) {
  const kind = guessFileKind({ mimeType, filename });
  const src = source ?? filename ?? "upload";
  const metadata = { filename: filename ?? null, mimeType: mimeType ?? null, kind };

  // #region debug-point A:extract-start
  (() => {
    let u = "http://127.0.0.1:7777/event";
    let s = "image-rag-miss";
    try {
      const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
      u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
      s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
    } catch {}
    fetch(u, {
      method: "POST",
      headers: { "content-type": "application/json" },
      body: JSON.stringify({
        sessionId: s,
        runId: "post",
        hypothesisId: "A",
        location: "api/chat/ingest.js",
        msg: "[DEBUG] extractDocumentsFromUpload start",
        data: { kind, filename: filename ?? null, mimeType: mimeType ?? null, bytes: buffer?.length ?? null, source: src },
        ts: Date.now()
      })
    }).catch(() => {});
  })();
  // #endregion

  if (kind === "txt") {
    const text = buffer.toString("utf8").trim();
    // #region debug-point A:extract-txt
    (() => {
      let u = "http://127.0.0.1:7777/event";
      let s = "image-rag-miss";
      try {
        const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
        u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
        s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
      } catch {}
      fetch(u, {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify({
          sessionId: s,
          runId: "post",
          hypothesisId: "A",
          location: "api/chat/ingest.js",
          msg: "[DEBUG] extract txt done",
          data: { chars: text.length, head: text.slice(0, 160) },
          ts: Date.now()
        })
      }).catch(() => {});
    })();
    // #endregion
    return [{ text, source: src, metadata }];
  }

  if (kind === "pdf") {
    let pdfParse;
    try {
      const mod = await import("pdf-parse");
      pdfParse = mod?.default ?? mod;
    } catch {
      try {
        pdfParse = require("pdf-parse");
      } catch {
        pdfParse = require("pdf-parse/lib/pdf-parse.js");
      }
      pdfParse = pdfParse?.default ?? pdfParse;
    }

    if (typeof pdfParse !== "function") {
      const err = new Error("pdf_parse_unavailable");
      err.statusCode = 500;
      throw err;
    }

    const parsed = await pdfParse(buffer);
    const text = String(parsed?.text ?? "").trim();
    // #region debug-point A:extract-pdf
    (() => {
      let u = "http://127.0.0.1:7777/event";
      let s = "image-rag-miss";
      try {
        const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
        u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
        s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
      } catch {}
      fetch(u, {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify({
          sessionId: s,
          runId: "post",
          hypothesisId: "A",
          location: "api/chat/ingest.js",
          msg: "[DEBUG] extract pdf done",
          data: { chars: text.length, head: text.slice(0, 160) },
          ts: Date.now()
        })
      }).catch(() => {});
    })();
    // #endregion
    return [{ text, source: src, metadata }];
  }

  if (kind === "docx") {
    const extracted = await mammoth.extractRawText({ buffer });
    const baseText = String(extracted?.value ?? "").trim();

    const images = [];
    await mammoth.convertToHtml(
      { buffer },
      {
        convertImage: mammoth.images.inline(async (image) => {
          const arr = await image.read();
          images.push(Buffer.from(arr));
          return { src: "" };
        })
      }
    );

    const ocrTexts = [];
    let visionSkipped = 0;
    let visionError = "";
    let visionUnavailable = false;
    for (let i = 0; i < images.length; i += 1) {
      if (visionUnavailable) {
        visionSkipped += 1;
        continue;
      }
      const imageBase64 = images[i].toString("base64");
      try {
        let r = await visionExtractFromImage({ imageBase64, prompt: visionPrompt() });
        let t = String(r?.content ?? "").trim();
        if (isVisionRefusal(t)) {
          r = await visionExtractFromImage({
            imageBase64,
            prompt: [
              "Analise a imagem e descreva somente os elementos de UI e estados selecionados.",
              "Liste itens curtos: campos/labels, opções marcadas, botões e mensagens de erro.",
              "Não faça transcrição literal de textos longos.",
              "Responda em português."
            ].join("\n")
          });
          t = String(r?.content ?? "").trim();
        }
        if (isVisionRefusal(t)) {
          visionError = "vision_refused";
          visionSkipped += 1;
          continue;
        }
        if (t) ocrTexts.push(`Imagem ${i + 1}:\n${t}`);
      } catch (e) {
        const msg = typeof e?.message === "string" ? e.message : "";
        visionError = msg || "vision_failed";
        if (msg.startsWith("ollama_model_not_found:")) {
          visionUnavailable = true;
          visionSkipped += images.length - i;
        } else {
          visionSkipped += 1;
        }
      }
    }

    const textParts = [];
    if (baseText) textParts.push(baseText);
    if (ocrTexts.length) textParts.push(ocrTexts.join("\n\n"));

    const text = textParts.join("\n\n").trim();
    // #region debug-point A:extract-docx
    (() => {
      let u = "http://127.0.0.1:7777/event";
      let s = "image-rag-miss";
      try {
        const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
        u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
        s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
      } catch {}
      fetch(u, {
        method: "POST",
        headers: { "content-type": "application/json" },
        body: JSON.stringify({
          sessionId: s,
          runId: "post",
          hypothesisId: "A",
          location: "api/chat/ingest.js",
          msg: "[DEBUG] extract docx done",
          data: {
            baseChars: baseText.length,
            totalChars: text.length,
            imagesTotal: images.length,
            imagesProcessed: images.length - visionSkipped,
            imagesSkipped: visionSkipped,
            visionError: visionError || null,
            head: text.slice(0, 160)
          },
          ts: Date.now()
        })
      }).catch(() => {});
    })();
    // #endregion
    return [
      {
        text,
        source: src,
        metadata: {
          ...metadata,
          imagesTotal: images.length,
          imagesProcessed: images.length - visionSkipped,
          imagesSkipped: visionSkipped,
          visionError: visionError || null
        }
      }
    ];
  }

  if (kind === "image") {
    const imageBase64 = buffer.toString("base64");
    try {
      let r = await visionExtractFromImage({ imageBase64, prompt: visionPrompt() });
      let text = String(r?.content ?? "").trim();
      if (isVisionRefusal(text)) {
        r = await visionExtractFromImage({
          imageBase64,
          prompt: [
            "Analise a imagem e descreva somente os elementos de UI e estados selecionados.",
            "Liste itens curtos: campos/labels, opções marcadas, botões e mensagens de erro.",
            "Não faça transcrição literal de textos longos.",
            "Responda em português."
          ].join("\n")
        });
        text = String(r?.content ?? "").trim();
      }
      // #region debug-point A:extract-image
      (() => {
        let u = "http://127.0.0.1:7777/event";
        let s = "image-rag-miss";
        try {
          const e = fs.readFileSync(".dbg/image-rag-miss.env", "utf8");
          u = e.match(/DEBUG_SERVER_URL=(.+)/)?.[1] || u;
          s = e.match(/DEBUG_SESSION_ID=(.+)/)?.[1] || s;
        } catch {}
        fetch(u, {
          method: "POST",
          headers: { "content-type": "application/json" },
          body: JSON.stringify({
            sessionId: s,
            runId: "pre",
            hypothesisId: "A",
            location: "api/chat/ingest.js",
            msg: "[DEBUG] extract image done",
            data: { chars: text.length, head: text.slice(0, 160) },
            ts: Date.now()
          })
        }).catch(() => {});
      })();
      // #endregion
      if (isVisionRefusal(text)) {
        const err = new Error("vision_refused");
        err.statusCode = 400;
        throw err;
      }
      return [{ text, source: src, metadata }];
    } catch (e) {
      const msg = typeof e?.message === "string" ? e.message : "";
      if (msg.startsWith("ollama_model_not_found:")) {
        const err = new Error(msg);
        err.statusCode = 400;
        throw err;
      }
      throw e;
    }
  }

  const err = new Error("unsupported_file_type");
  err.statusCode = 400;
  throw err;
}