backend/chat/ingest.js

import { config } from "../src/config/index.js";
import { chunkText } from "../src/services/textChunker.js";
import { embedTexts, visionExtractFromImage } from "../src/services/ollamaClient.js";
import { ensureCollection } from "../src/services/collectionService.js";
import { qdrant } from "../src/services/qdrantClient.js";
import { createHash } from "node:crypto";
import { createRequire } from "node:module";
import mammoth from "mammoth";

const require = createRequire(import.meta.url);

function isVisionRefusal(text) {
  const t = String(text ?? "").toLowerCase();
  return (
    t.includes("desculpe") &&
    (t.includes("não posso") || t.includes("nao posso") || t.includes("não posso fornecer") || t.includes("nao posso fornecer"))
  );
}

function visionPrompt() {
  return [
    "Analise a imagem (print de tela / manual interno).",
    "Extraia apenas informações úteis para busca: nomes de campos, rótulos, valores exibidos, opções selecionadas (checkbox/radio/dropdown), botões, mensagens de erro e códigos.",
    "Não transcreva parágrafos longos; prefira listas curtas e objetivas.",
    "Responda em português, em texto puro."
  ].join("\n");
}

function stableUuid(seed) {
  const hex = createHash("sha1")
    .update(String(seed))
    .digest("hex")
    .slice(0, 32);

  const timeLow = hex.slice(0, 8);
  const timeMid = hex.slice(8, 12);
  let timeHiAndVersion = parseInt(hex.slice(12, 16), 16);
  timeHiAndVersion = (timeHiAndVersion & 0x0fff) | 0x5000;

  let clockSeqHi = parseInt(hex.slice(16, 18), 16);
  clockSeqHi = (clockSeqHi & 0x3f) | 0x80;
  const clockSeqLow = hex.slice(18, 20);

  const node = hex.slice(20, 32);

  return [
    timeLow,
    timeMid,
    timeHiAndVersion.toString(16).padStart(4, "0"),
    `${clockSeqHi.toString(16).padStart(2, "0")}${clockSeqLow}`,
    node
  ].join("-");
}

export async function ingestDocuments(documents) {
  const collectionName = config.qdrant.collection;
  const allChunks = [];
  const ingestedAt = new Date().toISOString();

  for (const doc of documents) {
    const docHash = createHash("sha256").update(doc.text).digest("hex");
    const baseSeed = doc.id ?? docHash;
    const chunks = chunkText(doc.text, {
      chunkSize: config.rag.chunkSize,
      chunkOverlap: config.rag.chunkOverlap
    });
    chunks.forEach((chunk, idx) => {
      allChunks.push({
        id: stableUuid(`${baseSeed}:${idx}`),
        source: doc.source ?? null,
        title: doc.title ?? doc.source ?? null,
        metadata: doc.metadata ?? null,
        chunkIndex: idx,
        text: chunk,
        documentHash: docHash
      });
    });
  }

  if (allChunks.length === 0) return { upserted: 0 };

  const vectors = await embedTexts(allChunks.map((c) => c.text));
  const vectorSize = vectors[0]?.length ?? 0;
  if (!vectorSize) {
    const err = new Error("embeddings_empty");
    err.statusCode = 502;
    throw err;
  }

  await ensureCollection({ vectorSize });

  const points = allChunks.map((c, idx) => ({
    id: c.id,
    vector: vectors[idx],
    payload: {
      source: c.source,
      title: c.title,
      chunkIndex: c.chunkIndex,
      text: c.text,
      metadata: c.metadata,
      documentHash: c.documentHash,
      ingestedAt
    }
  }));

  await qdrant.upsert(collectionName, {
    wait: true,
    points
  });

  return { upserted: points.length };
}

function isPrivateUrl(urlStr) {
  try {
    const { hostname } = new URL(urlStr);
    return (
      hostname === "localhost" ||
      /^127\./.test(hostname) ||
      /^10\./.test(hostname) ||
      /^192\.168\./.test(hostname) ||
      /^172\.(1[6-9]|2\d|3[01])\./.test(hostname) ||
      hostname === "0.0.0.0" ||
      hostname.endsWith(".local")
    );
  } catch {
    return true;
  }
}

export async function fetchUrlText(urlStr) {
  if (!urlStr.startsWith("https://")) {
    const err = new Error("url_must_be_https");
    err.statusCode = 400;
    throw err;
  }
  if (isPrivateUrl(urlStr)) {
    const err = new Error("url_private_not_allowed");
    err.statusCode = 400;
    throw err;
  }

  const res = await fetch(urlStr, {
    headers: { "User-Agent": "Mozilla/5.0 star-oraculo/1.0" },
    redirect: "follow",
    signal: AbortSignal.timeout(15_000)
  });

  if (!res.ok) {
    const err = new Error(`url_fetch_error:${res.status}`);
    err.statusCode = 502;
    throw err;
  }

  const html = await res.text();
  const text = html
    .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, " ")
    .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, " ")
    .replace(/<[^>]+>/g, " ")
    .replace(/&nbsp;/g, " ")
    .replace(/&amp;/g, "&")
    .replace(/&lt;/g, "<")
    .replace(/&gt;/g, ">")
    .replace(/&quot;/g, '"')
    .replace(/&#39;/g, "'")
    .replace(/\s+/g, " ")
    .trim();

  return text;
}

function guessFileKind({ mimeType, filename }) {
  const name = String(filename ?? "").toLowerCase();
  const mt = String(mimeType ?? "").toLowerCase();

  if (mt === "text/plain" || name.endsWith(".txt")) return "txt";
  if (mt === "application/pdf" || name.endsWith(".pdf")) return "pdf";
  if (
    mt === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ||
    name.endsWith(".docx")
  )
    return "docx";
  if (mt.startsWith("image/") || /\.(png|jpe?g|webp)$/i.test(name)) return "image";
  return "unknown";
}

export async function extractDocumentsFromUpload({ buffer, filename, mimeType, source }) {
  const kind = guessFileKind({ mimeType, filename });
  const src = source ?? filename ?? "upload";
  const metadata = { filename: filename ?? null, mimeType: mimeType ?? null, kind };

  if (kind === "txt") {
    const text = buffer.toString("utf8").trim();
    return [{ text, source: src, metadata }];
  }

  if (kind === "pdf") {
    let pdfParse;
    try {
      const mod = await import("pdf-parse");
      pdfParse = mod?.default ?? mod;
    } catch {
      try {
        pdfParse = require("pdf-parse");
      } catch {
        pdfParse = require("pdf-parse/lib/pdf-parse.js");
      }
      pdfParse = pdfParse?.default ?? pdfParse;
    }

    if (typeof pdfParse !== "function") {
      const err = new Error("pdf_parse_unavailable");
      err.statusCode = 500;
      throw err;
    }

    const parsed = await pdfParse(buffer);
    const text = String(parsed?.text ?? "").trim();
    return [{ text, source: src, metadata }];
  }

  if (kind === "docx") {
    const extracted = await mammoth.extractRawText({ buffer });
    const baseText = String(extracted?.value ?? "").trim();

    const images = [];
    await mammoth.convertToHtml(
      { buffer },
      {
        convertImage: mammoth.images.inline(async (image) => {
          const arr = await image.read();
          images.push(Buffer.from(arr));
          return { src: "" };
        })
      }
    );

    const ocrTexts = [];
    let visionSkipped = 0;
    let visionError = "";
    let visionUnavailable = false;
    for (let i = 0; i < images.length; i += 1) {
      if (visionUnavailable) {
        visionSkipped += 1;
        continue;
      }
      const imageBase64 = images[i].toString("base64");
      try {
        let r = await visionExtractFromImage({ imageBase64, prompt: visionPrompt() });
        let t = String(r?.content ?? "").trim();
        if (isVisionRefusal(t)) {
          r = await visionExtractFromImage({
            imageBase64,
            prompt: [
              "Analise a imagem e descreva somente os elementos de UI e estados selecionados.",
              "Liste itens curtos: campos/labels, opções marcadas, botões e mensagens de erro.",
              "Não faça transcrição literal de textos longos.",
              "Responda em português."
            ].join("\n")
          });
          t = String(r?.content ?? "").trim();
        }
        if (isVisionRefusal(t)) {
          visionError = "vision_refused";
          visionSkipped += 1;
          continue;
        }
        if (t) ocrTexts.push(`Imagem ${i + 1}:\n${t}`);
      } catch (e) {
        const msg = typeof e?.message === "string" ? e.message : "";
        visionError = msg || "vision_failed";
        if (msg.startsWith("ollama_model_not_found:")) {
          visionUnavailable = true;
          visionSkipped += images.length - i;
        } else {
          visionSkipped += 1;
        }
      }
    }

    const textParts = [];
    if (baseText) textParts.push(baseText);
    if (ocrTexts.length) textParts.push(ocrTexts.join("\n\n"));

    const text = textParts.join("\n\n").trim();
    return [
      {
        text,
        source: src,
        metadata: {
          ...metadata,
          imagesTotal: images.length,
          imagesProcessed: images.length - visionSkipped,
          imagesSkipped: visionSkipped,
          visionError: visionError || null
        }
      }
    ];
  }

  if (kind === "image") {
    const imageBase64 = buffer.toString("base64");
    try {
      let r = await visionExtractFromImage({ imageBase64, prompt: visionPrompt() });
      let text = String(r?.content ?? "").trim();
      if (isVisionRefusal(text)) {
        r = await visionExtractFromImage({
          imageBase64,
          prompt: [
            "Analise a imagem e descreva somente os elementos de UI e estados selecionados.",
            "Liste itens curtos: campos/labels, opções marcadas, botões e mensagens de erro.",
            "Não faça transcrição literal de textos longos.",
            "Responda em português."
          ].join("\n")
        });
        text = String(r?.content ?? "").trim();
      }
      if (isVisionRefusal(text)) {
        const err = new Error("vision_refused");
        err.statusCode = 400;
        throw err;
      }
      return [{ text, source: src, metadata }];
    } catch (e) {
      const msg = typeof e?.message === "string" ? e.message : "";
      if (msg.startsWith("ollama_model_not_found:")) {
        const err = new Error(msg);
        err.statusCode = 400;
        throw err;
      }
      throw e;
    }
  }

  const err = new Error("unsupported_file_type");
  err.statusCode = 400;
  throw err;
}