ajuste documentaçao e permissoes

.env.example

@@ -0,0 +1,39 @@
+# Servidor
+PORT=3001
+CORS_ORIGIN=http://localhost:5173
+
+# Autenticação (jwt ou none)
+AUTH_MODE=jwt
+JWT_SECRET=coloque-um-segredo-longo-e-aleatorio-aqui
+JWT_ISSUER=oraculo-api
+JWT_ACCESS_TTL_SECONDS=900
+JWT_REFRESH_TTL_SECONDS=2592000
+
+# API Key alternativa (opcional, deixar vazio para usar só JWT)
+API_KEY=
+
+# Banco de dados MySQL
+DB_HOST=localhost
+DB_PORT=3306
+DB_USER=usuario
+DB_PASS=senha
+DB_SCHEMA=oraculo
+
+# Qdrant (vector DB)
+QDRANT_URL=http://localhost:6333
+QDRANT_COLLECTION=oraculo_docs
+QDRANT_API_KEY=
+
+# Ollama
+OLLAMA_URL=http://localhost:11434
+OLLAMA_EMBEDDINGS_MODEL=nomic-embed-text
+OLLAMA_CHAT_MODEL=llama3
+OLLAMA_VISION_MODEL=llava
+
+# RAG (valores padrão se não definidos)
+RAG_TOP_K=6
+RAG_CHUNK_SIZE=900
+RAG_CHUNK_OVERLAP=150
+
+# Rate limiting (0 = desativado)
+RATE_LIMIT_PER_MINUTE=0

chat/ingest.js

@@ -55,9 +55,11 @@ function stableUuid(seed) {
 export async function ingestDocuments(documents) {
   const collectionName = config.qdrant.collection;
   const allChunks = [];
+  const ingestedAt = new Date().toISOString();
 
   for (const doc of documents) {
-    const baseSeed = doc.id ?? `${doc.source ?? "doc"}:${doc.text.slice(0, 64)}`;
+    const docHash = createHash("sha256").update(doc.text).digest("hex");
+    const baseSeed = doc.id ?? docHash;
     const chunks = chunkText(doc.text, {
       chunkSize: config.rag.chunkSize,
       chunkOverlap: config.rag.chunkOverlap
@@ -66,9 +68,11 @@ export async function ingestDocuments(documents) {
       allChunks.push({
         id: stableUuid(`${baseSeed}:${idx}`),
         source: doc.source ?? null,
+        title: doc.title ?? doc.source ?? null,
         metadata: doc.metadata ?? null,
         chunkIndex: idx,
-        text: chunk
+        text: chunk,
+        documentHash: docHash
       });
     });
   }
@@ -90,9 +94,12 @@ export async function ingestDocuments(documents) {
     vector: vectors[idx],
     payload: {
       source: c.source,
+      title: c.title,
       chunkIndex: c.chunkIndex,
       text: c.text,
-      metadata: c.metadata
+      metadata: c.metadata,
+      documentHash: c.documentHash,
+      ingestedAt
     }
   }));
 
@@ -104,6 +111,64 @@ export async function ingestDocuments(documents) {
   return { upserted: points.length };
 }
 
+function isPrivateUrl(urlStr) {
+  try {
+    const { hostname } = new URL(urlStr);
+    return (
+      hostname === "localhost" ||
+      /^127\./.test(hostname) ||
+      /^10\./.test(hostname) ||
+      /^192\.168\./.test(hostname) ||
+      /^172\.(1[6-9]|2\d|3[01])\./.test(hostname) ||
+      hostname === "0.0.0.0" ||
+      hostname.endsWith(".local")
+    );
+  } catch {
+    return true;
+  }
+}
+
+export async function fetchUrlText(urlStr) {
+  if (!urlStr.startsWith("https://")) {
+    const err = new Error("url_must_be_https");
+    err.statusCode = 400;
+    throw err;
+  }
+  if (isPrivateUrl(urlStr)) {
+    const err = new Error("url_private_not_allowed");
+    err.statusCode = 400;
+    throw err;
+  }
+
+  const res = await fetch(urlStr, {
+    headers: { "User-Agent": "Mozilla/5.0 star-oraculo/1.0" },
+    redirect: "follow",
+    signal: AbortSignal.timeout(15_000)
+  });
+
+  if (!res.ok) {
+    const err = new Error(`url_fetch_error:${res.status}`);
+    err.statusCode = 502;
+    throw err;
+  }
+
+  const html = await res.text();
+  const text = html
+    .replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, " ")
+    .replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&nbsp;/g, " ")
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/\s+/g, " ")
+    .trim();
+
+  return text;
+}
+
 function guessFileKind({ mimeType, filename }) {
   const name = String(filename ?? "").toLowerCase();
   const mt = String(mimeType ?? "").toLowerCase();

knexfile.js

@@ -10,7 +10,6 @@ export default {
       password: process.env.DB_PASS,
       database: process.env.DB_SCHEMA,
       charset: "utf8mb4",
-      collation: "utf8mb4_bin"
     },
     migrations: {
       tableName: "knex_migrations",

src/middleware/auth.js

@@ -1,6 +1,10 @@
 import { config } from "../config/index.js";
 import { verifyAccessToken } from "../services/authTokens.js";
 
+if (config.auth.mode !== "none" && !config.jwt.secret && !config.apiKey) {
+  throw new Error("JWT_SECRET não configurado — defina a variável de ambiente antes de iniciar o servidor");
+}
+
 export function authMiddleware(req, res, next) {
   if (config.auth.mode === "none") {
     next();

src/middleware/errorHandler.js

@@ -1,8 +1,16 @@
+import { ZodError } from "zod";
+
 export function errorHandler(err, _req, res, _next) {
+  if (err instanceof ZodError) {
+    return res.status(400).json({ error: "validation_error", details: err.errors });
+  }
+
   const status = Number(err?.statusCode ?? err?.status ?? 500);
   const message = typeof err?.message === "string" ? err.message : "internal_error";
 
-  res.status(status).json({
-    error: message
-  });
+  if (status >= 500) {
+    console.error("[error]", err);
+  }
+
+  res.status(status).json({ error: message });
 }

src/middleware/rateLimit.js

@@ -1,5 +1,12 @@
 const buckets = new Map();
 
+setInterval(() => {
+  const cutoff = Date.now() - 60_000;
+  for (const [key, b] of buckets) {
+    if (b.resetAt < cutoff) buckets.delete(key);
+  }
+}, 60_000).unref();
+
 export function rateLimitMiddleware(req, res, next) {
   const limitPerMinute = Number(process.env.RATE_LIMIT_PER_MINUTE ?? 0);
   if (!limitPerMinute) return next();

src/routes/chat.js

@@ -31,7 +31,7 @@ chatRouter.post("/", async (req, res, next) => {
       Promise.all([
         addMessage(body.conversationId, { role: "user", content: body.message }),
         addMessage(body.conversationId, { role: "assistant", content: result.answer, sources: result.sources })
-      ]).catch(() => {});
+      ]).catch((err) => console.error("[chat] falha ao salvar mensagem:", err));
     }
 
     res.json(result);
@@ -72,7 +72,7 @@ chatRouter.post("/stream", async (req, res, next) => {
       Promise.all([
         addMessage(body.conversationId, { role: "user", content: body.message }),
         addMessage(body.conversationId, { role: "assistant", content: result.answer, sources: result.sources })
-      ]).catch(() => {});
+      ]).catch((err) => console.error("[chat] falha ao salvar mensagem:", err));
     }
   } catch (err) {
     if (!res.headersSent) {

src/routes/documents.js

@@ -6,9 +6,9 @@ export const documentsRouter = Router();
 
 documentsRouter.get("/", async (req, res, next) => {
   try {
+    if (!req.user?.sub) return res.status(401).json({ error: "unauthorized" });
     const limit = z.coerce.number().int().positive().max(200).catch(50).parse(req.query.limit);
-    const offsetRaw = z.string().optional().parse(req.query.offset);
-    const offset = offsetRaw && offsetRaw !== "0" ? offsetRaw : undefined;
+    const offset = z.coerce.number().int().nonnegative().optional().parse(req.query.offset);
     const result = await listDocuments({ limit, offset });
     res.json(result);
   } catch (err) {
@@ -18,6 +18,7 @@ documentsRouter.get("/", async (req, res, next) => {
 
 documentsRouter.delete("/source/:source", async (req, res, next) => {
   try {
+    if (!req.user?.sub) return res.status(401).json({ error: "unauthorized" });
     const source = req.params.source;
     if (!source) {
       return res.status(400).json({ error: "source_required" });

src/routes/ingest.js

@@ -1,7 +1,7 @@
 import { Router } from "express";
 import { z } from "zod";
 import multer from "multer";
-import { extractDocumentsFromUpload, ingestDocuments } from "../../chat/ingest.js";
+import { extractDocumentsFromUpload, ingestDocuments, fetchUrlText } from "../../chat/ingest.js";
 
 export const ingestRouter = Router();
 
@@ -21,8 +21,14 @@ const ingestBodySchema = z.object({
   documents: z.array(docSchema).min(1)
 });
 
+const ingestUrlSchema = z.object({
+  url: z.string().url().startsWith("https://"),
+  source: z.string().min(1).optional()
+});
+
 ingestRouter.post("/", async (req, res, next) => {
   try {
+    if (!req.user?.sub) return res.status(401).json({ error: "unauthorized" });
     const body = ingestBodySchema.parse(req.body);
     const result = await ingestDocuments(body.documents);
     res.json(result);
@@ -33,6 +39,7 @@ ingestRouter.post("/", async (req, res, next) => {
 
 ingestRouter.post("/file", upload.single("file"), async (req, res, next) => {
   try {
+    if (!req.user?.sub) return res.status(401).json({ error: "unauthorized" });
     const f = req.file;
     if (!f?.buffer) {
       res.status(400).json({ error: "file_required" });
@@ -59,3 +66,19 @@ ingestRouter.post("/file", upload.single("file"), async (req, res, next) => {
     next(err);
   }
 });
+
+ingestRouter.post("/url", async (req, res, next) => {
+  try {
+    if (!req.user?.sub) return res.status(401).json({ error: "unauthorized" });
+    const body = ingestUrlSchema.parse(req.body);
+    const text = await fetchUrlText(body.url);
+    if (!text) {
+      return res.status(400).json({ error: "empty_extracted_text" });
+    }
+    const source = body.source ?? body.url;
+    const result = await ingestDocuments([{ text, source, title: source }]);
+    res.json({ ...result, source, extractedChars: text.length });
+  } catch (err) {
+    next(err);
+  }
+});

src/routes/search.js

@@ -11,6 +11,7 @@ const searchBodySchema = z.object({
 
 searchRouter.post("/", async (req, res, next) => {
   try {
+    if (!req.user?.sub) return res.status(401).json({ error: "unauthorized" });
     const body = searchBodySchema.parse(req.body);
     const results = await searchDocs({
       query: body.query,

src/services/ollamaClient.js

@@ -34,18 +34,23 @@ async function ollamaFetch(path, body) {
   return res.json();
 }
 
-export async function embedTexts(texts) {
-  const embeddings = [];
+async function embedSingle(text) {
+  const data = await ollamaFetch("/api/embeddings", {
+    model: config.ollama.embeddingsModel,
+    prompt: text
+  });
+  return data.embedding;
+}
 
-  for (const text of texts) {
-    const data = await ollamaFetch("/api/embeddings", {
-      model: config.ollama.embeddingsModel,
-      prompt: text
-    });
-    embeddings.push(data.embedding);
+export async function embedTexts(texts) {
+  const BATCH = 10;
+  const results = [];
+  for (let i = 0; i < texts.length; i += BATCH) {
+    const batch = texts.slice(i, i + BATCH);
+    const embeddings = await Promise.all(batch.map(embedSingle));
+    results.push(...embeddings);
   }
-
-  return embeddings;
+  return results;
 }
 
 export async function chatCompletion({ messages }) {