// test/crawl-onpage.test.ts — handler `crawl-onpage` : batch-scrape
// Firecrawl (client mocké par DI, jamais mock.module — cf. mémoire projet),
// parse WASM, merge crawlStatus/champs on-page, totals, deadline de poll.
// Mirror de serp-semantics.test.ts (dispatch handler, DB réelle).
import { beforeAll, expect, test } from "bun:test";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { analyticsJobs, keywords } from "@/lib/db/domain-schema";
import type { SerpCompetitor, SerpSnapshot } from "@/lib/keywords/serp-types";
import { type CrawlOnpageDeps, processCrawlOnpage } from "../src/crawl-onpage";
import { db } from "../src/db";
import type { BatchState } from "../src/firecrawl";
import { ensureTestSite, TEST_SITE as SITE } from "./fixtures/site";

beforeAll(ensureTestSite);

const HTML_OK = `<html lang="en"><head><title>Ok Title</title><meta name="description" content="Ok description"></head><body><p>Some content words here for the count.</p></body></html>`;

function competitor(url: string, position: number): SerpCompetitor {
  return {
    position,
    domain: new URL(url).hostname,
    url,
    backlinks: null,
    refDomains: null,
    pageAuthority: null,
    searchTraffic: null,
    urlKeywords: null,
    title: null,
    metaDescription: null,
    lang: null,
    images: null,
    imagesWithoutAlt: null,
    imageAlts: null,
    missingAltSrcs: null,
    hasJsonLd: null,
    jsonLdTypes: null,
    crawledAt: null,
  };
}

function snapshot(competitors: SerpCompetitor[]): SerpSnapshot {
  return {
    source: "se-ranking",
    importedAt: "2026-01-01T00:00:00.000Z",
    features: [],
    competitors,
  };
}

async function insertKeyword(
  id: string,
  country: string,
  serp: Record<string, SerpSnapshot>,
): Promise<void> {
  await db.insert(keywords).values({
    id,
    siteId: SITE,
    term: id,
    lang: "en",
    country,
    status: "idea",
    intent: [],
    primarySource: "manual",
    sources: ["manual"],
    metrics: {},
    trend: [],
    serp,
  });
}

async function insertJob(
  id: string,
  keywordId: string | undefined,
): Promise<typeof analyticsJobs.$inferSelect> {
  await db.insert(analyticsJobs).values({
    id,
    siteId: SITE,
    type: "crawl-onpage",
    payload: keywordId ? { keywordId } : undefined,
    status: "running",
  });
  return (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, id))
  )[0];
}

async function cleanup(keywordId: string, jobId: string): Promise<void> {
  await db.delete(analyticsJobs).where(eq(analyticsJobs.id, jobId));
  await db.delete(keywords).where(eq(keywords.id, keywordId));
}

const alwaysConfigured = () => true;

test("mixte : ok/blocked/timeout → competitors mergés, totals, job done", async () => {
  const keywordId = `${SITE}-crawl-onpage-mixed`;
  const jobId = randomUUID();
  const urls = [
    "https://a.example.com/",
    "https://b.example.com/",
    "https://c.example.com/",
  ];
  await insertKeyword(keywordId, "FR", {
    FR: snapshot([
      competitor(urls[0]!, 1),
      competitor(urls[1]!, 2),
      competitor(urls[2]!, 3),
    ]),
  });
  const job = await insertJob(jobId, keywordId);

  let cancelCalls = 0;
  const deps: CrawlOnpageDeps = {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-1" }),
      getBatchScrape: async () => ({
        status: "completed",
        pages: [
          { url: urls[0]!, html: HTML_OK, status: "ok", httpStatus: 200 },
          { url: urls[1]!, html: null, status: "blocked", httpStatus: 403 },
          { url: urls[2]!, html: null, status: "timeout" },
        ],
      }),
      cancelBatchScrape: async () => {
        cancelCalls++;
      },
    },
  };

  await processCrawlOnpage(job, deps);
  expect(cancelCalls).toBe(0); // chemin nominal : jamais d'annulation

  const doneJob = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(doneJob.status).toBe("done");
  expect(doneJob.totals).toEqual({ crawled: 1, blocked: 1, failed: 1 });

  const kwRow = (
    await db.select().from(keywords).where(eq(keywords.id, keywordId))
  )[0];
  const competitors = kwRow.serp.FR!.competitors;
  expect(competitors[0]?.crawlStatus).toBe("ok");
  expect(competitors[0]?.crawlHttpStatus).toBe(200);
  expect(competitors[0]?.title).toBe("Ok Title");
  expect(competitors[0]?.metaDescription).toBe("Ok description");
  expect(competitors[0]?.lang).toBe("en");
  expect(competitors[1]?.crawlStatus).toBe("blocked");
  expect(competitors[1]?.crawlHttpStatus).toBe(403);
  expect(competitors[2]?.crawlStatus).toBe("timeout");

  await cleanup(keywordId, jobId);
});

test("payload manquant → failJob", async () => {
  const jobId = randomUUID();
  const job = await insertJob(jobId, undefined);

  await processCrawlOnpage(job, {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => {
        throw new Error("must not be called");
      },
      getBatchScrape: async () => {
        throw new Error("must not be called");
      },
      cancelBatchScrape: async () => {
        throw new Error("must not be called");
      },
    },
  });

  const failed = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(failed.status).toBe("failed");

  await db.delete(analyticsJobs).where(eq(analyticsJobs.id, jobId));
});

test("env Firecrawl non configurée → failJob avec message contenant FIRECRAWL", async () => {
  const jobId = randomUUID();
  const job = await insertJob(jobId, "some-keyword-id");

  await processCrawlOnpage(job, {
    firecrawl: {
      firecrawlConfigured: () => false,
      startBatchScrape: async () => {
        throw new Error("must not be called");
      },
      getBatchScrape: async () => {
        throw new Error("must not be called");
      },
      cancelBatchScrape: async () => {
        throw new Error("must not be called");
      },
    },
  });

  const failed = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(failed.status).toBe("failed");
  expect(failed.error).toContain("FIRECRAWL");

  await db.delete(analyticsJobs).where(eq(analyticsJobs.id, jobId));
});

test("merge non destructif : seule la locale du mot-clé est modifiée", async () => {
  const keywordId = `${SITE}-crawl-onpage-merge`;
  const jobId = randomUUID();
  const url = "https://fr.example.com/";
  const usSnapshot = snapshot([competitor("https://us.example.com/", 1)]);
  await insertKeyword(keywordId, "FR", {
    FR: snapshot([competitor(url, 1)]),
    US: usSnapshot,
  });
  const job = await insertJob(jobId, keywordId);

  await processCrawlOnpage(job, {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-2" }),
      getBatchScrape: async () => ({
        status: "completed",
        pages: [{ url, html: HTML_OK, status: "ok", httpStatus: 200 }],
      }),
      cancelBatchScrape: async () => {
        throw new Error("must not be called");
      },
    },
  });

  const kwRow = (
    await db.select().from(keywords).where(eq(keywords.id, keywordId))
  )[0];
  expect(kwRow.serp.FR!.competitors[0]?.crawlStatus).toBe("ok");
  expect(kwRow.serp.US).toEqual(usSnapshot);

  await cleanup(keywordId, jobId);
});

test("deadline globale (5 min) : getBatchScrape reste 'scraping' → failJob avec un message de timeout", async () => {
  const keywordId = `${SITE}-crawl-onpage-timeout`;
  const jobId = randomUUID();
  const url = "https://stuck.example.com/";
  await insertKeyword(keywordId, "FR", { FR: snapshot([competitor(url, 1)]) });
  const job = await insertJob(jobId, keywordId);

  const BASE = Date.parse("2026-01-01T00:00:00.000Z");
  let calls = 0;
  const now = () => {
    calls++;
    // 1er appel : calcul de la deadline ; 2e appel (et suivants) : bien
    // au-delà des 5 min — simule un poll qui traînerait indéfiniment sans
    // jamais réellement attendre (getBatchScrape répond "scraping" à l'infini,
    // cf. revue Tâche 1).
    return new Date(calls === 1 ? BASE : BASE + 6 * 60 * 1000);
  };

  let getBatchScrapeCalls = 0;
  const cancelledJobIds: string[] = [];
  await processCrawlOnpage(job, {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-3" }),
      getBatchScrape: async () => {
        getBatchScrapeCalls++;
        return { status: "scraping", completed: 0, total: 1 };
      },
      cancelBatchScrape: async (jobId: string) => {
        cancelledJobIds.push(jobId);
      },
    },
    now,
  });

  expect(getBatchScrapeCalls).toBe(1); // échoue dès le 1er poll, pas de sleep réel
  expect(cancelledJobIds).toEqual(["fc-job-3"]); // batch orphelin annulé (deadline)

  const failed = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(failed.status).toBe("failed");
  expect(failed.error).toContain("timeout");

  await cleanup(keywordId, jobId);
});

test("competitor absent des pages du batch (échec dur Firecrawl) : failed + fetch_error, champs intacts", async () => {
  const keywordId = `${SITE}-crawl-onpage-missing-page`;
  const jobId = randomUUID();
  await insertKeyword(keywordId, "FR", {
    FR: snapshot([
      competitor("https://a.com", 1),
      competitor("https://absente.com", 2),
    ]),
  });
  const job = await insertJob(jobId, keywordId);

  const deps: CrawlOnpageDeps = {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-5" }),
      // Constaté en E2E : la page en échec dur n'apparaît PAS dans data —
      // le batch ne renvoie que les pages abouties.
      getBatchScrape: async () => ({
        status: "completed",
        pages: [
          {
            url: "https://a.com",
            html: HTML_OK,
            status: "ok",
            httpStatus: 200,
          },
        ],
      }),
      cancelBatchScrape: async () => {
        throw new Error("must not be called");
      },
    },
  };

  await processCrawlOnpage(job, deps);

  const doneJob = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(doneJob.status).toBe("done");
  expect(doneJob.totals).toEqual({ crawled: 1, blocked: 0, failed: 1 });

  const kwRow = (
    await db.select().from(keywords).where(eq(keywords.id, keywordId))
  )[0];
  const competitors = kwRow.serp.FR!.competitors;
  expect(competitors[0]?.crawlStatus).toBe("ok");
  expect(competitors[1]?.crawlStatus).toBe("fetch_error");
  expect(competitors[1]?.crawlHttpStatus).toBeUndefined();
  // La fixture competitor() initialise les champs on-page à null — l'important
  // est qu'aucun crawl ne les ait REMPLIS.
  expect(competitors[1]?.title ?? null).toBeNull();
  expect(competitors[1]?.crawledAt ?? null).toBeNull();

  await cleanup(keywordId, jobId);
});

test("competitor absent du batch AVEC crawlStatus/crawlHttpStatus préexistants (run précédent) : le fetch_error de ce run remplace tout, pas de statut fantôme", async () => {
  // Reproduit l'incident constaté en prod (2026-07-23, « jewelry casting ») :
  // un competitor porte déjà crawlStatus "blocked"/crawlHttpStatus 200 d'un
  // run antérieur ; CE run échoue dur sur sa page (absente de `data`). Le
  // spread `{...c, crawlStatus: "fetch_error"}` écrase bien crawlStatus mais
  // laissait crawlHttpStatus (200) survivre — DB racontant alors "fetch_error
  // avec une réponse HTTP 200 reçue", contradiction avec les totals qui
  // comptent cette page en failed, jamais en blocked.
  const keywordId = `${SITE}-crawl-onpage-missing-page-stale-status`;
  const jobId = randomUUID();
  const staleCompetitor: SerpCompetitor = {
    ...competitor("https://absente.com", 2),
    crawlStatus: "blocked",
    crawlHttpStatus: 200,
  };
  await insertKeyword(keywordId, "FR", {
    FR: snapshot([competitor("https://a.com", 1), staleCompetitor]),
  });
  const job = await insertJob(jobId, keywordId);

  const deps: CrawlOnpageDeps = {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-6" }),
      // Cette fois, la page échoue dur côté Firecrawl (absente de data) —
      // peu importe qu'un run précédent l'ait vue "blocked".
      getBatchScrape: async () => ({
        status: "completed",
        pages: [
          {
            url: "https://a.com",
            html: HTML_OK,
            status: "ok",
            httpStatus: 200,
          },
        ],
      }),
      cancelBatchScrape: async () => {
        throw new Error("must not be called");
      },
    },
  };

  await processCrawlOnpage(job, deps);

  const doneJob = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(doneJob.status).toBe("done");
  // Un seul competitor absent du batch → failed, jamais blocked (le "blocked"
  // préexistant ne doit pas se refléter dans les totals de CE run).
  expect(doneJob.totals).toEqual({ crawled: 1, blocked: 0, failed: 1 });

  const kwRow = (
    await db.select().from(keywords).where(eq(keywords.id, keywordId))
  )[0];
  const competitors = kwRow.serp.FR!.competitors;
  expect(competitors[1]?.crawlStatus).toBe("fetch_error");
  // Le crawlHttpStatus (200) d'un run antérieur ne doit pas survivre sous un
  // crawlStatus "fetch_error" du run courant — DB et totals doivent raconter
  // la même histoire.
  expect(competitors[1]?.crawlHttpStatus).toBeUndefined();

  await cleanup(keywordId, jobId);
});

test("pages batch sans URL matchante : comptées en failed (plus d'ignore silencieux)", async () => {
  const keywordId = `${SITE}-crawl-onpage-unmatched-pages`;
  const jobId = randomUUID();
  const competitorUrl = "https://a.com";
  await insertKeyword(keywordId, "FR", {
    FR: snapshot([competitor(competitorUrl, 1)]),
  });
  const job = await insertJob(jobId, keywordId);

  const deps: CrawlOnpageDeps = {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-4" }),
      getBatchScrape: async () => ({
        status: "completed",
        pages: [
          {
            url: competitorUrl,
            html: HTML_OK,
            status: "ok",
            httpStatus: 200,
          },
          { url: "", html: null, status: "ok", httpStatus: 200 },
        ],
      }),
      cancelBatchScrape: async () => {
        throw new Error("must not be called");
      },
    },
  };

  await processCrawlOnpage(job, deps);

  const doneJob = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(doneJob.status).toBe("done");
  expect(doneJob.totals).toEqual({ crawled: 1, blocked: 0, failed: 1 });

  const kwRow = (
    await db.select().from(keywords).where(eq(keywords.id, keywordId))
  )[0];
  const competitors = kwRow.serp.FR!.competitors;
  expect(competitors[0]?.crawlStatus).toBe("ok");
  expect(competitors[0]?.crawlHttpStatus).toBe(200);
  expect(competitors[0]?.title).toBe("Ok Title");

  await cleanup(keywordId, jobId);
});

test("matching tolérant au slash final : competitor sans slash, page Firecrawl avec slash → matche", async () => {
  const keywordId = `${SITE}-crawl-onpage-match-trailing-slash`;
  const jobId = randomUUID();
  const competitorUrl = "https://x.com/a";
  await insertKeyword(keywordId, "FR", {
    FR: snapshot([competitor(competitorUrl, 1)]),
  });
  const job = await insertJob(jobId, keywordId);

  let cancelCalls = 0;
  const deps: CrawlOnpageDeps = {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-slash" }),
      getBatchScrape: async () => ({
        status: "completed",
        // Firecrawl renvoie la page sous une URL normalisée avec slash
        // final — le competitor n'en a pas.
        pages: [
          {
            url: "https://x.com/a/",
            html: HTML_OK,
            status: "ok",
            httpStatus: 200,
          },
        ],
      }),
      cancelBatchScrape: async () => {
        cancelCalls++;
      },
    },
  };

  await processCrawlOnpage(job, deps);
  expect(cancelCalls).toBe(0);

  const doneJob = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(doneJob.status).toBe("done");
  // Doit matcher : crawled:1, failed:0 (ni fetch_error, ni orpheline).
  expect(doneJob.totals).toEqual({ crawled: 1, blocked: 0, failed: 0 });

  const kwRow = (
    await db.select().from(keywords).where(eq(keywords.id, keywordId))
  )[0];
  const competitors = kwRow.serp.FR!.competitors;
  expect(competitors[0]?.crawlStatus).toBe("ok");
  expect(competitors[0]?.crawlHttpStatus).toBe(200);
  expect(competitors[0]?.title).toBe("Ok Title");

  await cleanup(keywordId, jobId);
});

test("matching tolérant au http→https (redirect) : competitor http, page Firecrawl https → matche", async () => {
  const keywordId = `${SITE}-crawl-onpage-match-http-https`;
  const jobId = randomUUID();
  const competitorUrl = "http://x.com/a";
  await insertKeyword(keywordId, "FR", {
    FR: snapshot([competitor(competitorUrl, 1)]),
  });
  const job = await insertJob(jobId, keywordId);

  const deps: CrawlOnpageDeps = {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-https" }),
      getBatchScrape: async () => ({
        status: "completed",
        // Redirect http → https côté Firecrawl : sourceURL revient en https.
        pages: [
          {
            url: "https://x.com/a",
            html: HTML_OK,
            status: "ok",
            httpStatus: 200,
          },
        ],
      }),
      cancelBatchScrape: async () => {
        throw new Error("must not be called");
      },
    },
  };

  await processCrawlOnpage(job, deps);

  const doneJob = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(doneJob.status).toBe("done");
  expect(doneJob.totals).toEqual({ crawled: 1, blocked: 0, failed: 0 });

  const kwRow = (
    await db.select().from(keywords).where(eq(keywords.id, keywordId))
  )[0];
  expect(kwRow.serp.FR!.competitors[0]?.crawlStatus).toBe("ok");

  await cleanup(keywordId, jobId);
});

const noRealWait = async () => {};

test("2 unreachable consécutifs puis completed → job done normal (blip réseau toléré)", async () => {
  const keywordId = `${SITE}-crawl-onpage-unreachable-tolerated`;
  const jobId = randomUUID();
  const url = "https://ok.example.com/";
  await insertKeyword(keywordId, "FR", { FR: snapshot([competitor(url, 1)]) });
  const job = await insertJob(jobId, keywordId);

  let calls = 0;
  let cancelCalls = 0;
  await processCrawlOnpage(job, {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-unreachable-1" }),
      getBatchScrape: async () => {
        calls++;
        if (calls <= 2) {
          return { status: "unreachable", error: "network blip" };
        }
        return {
          status: "completed",
          pages: [{ url, html: HTML_OK, status: "ok", httpStatus: 200 }],
        };
      },
      cancelBatchScrape: async () => {
        cancelCalls++;
      },
    },
    sleep: noRealWait,
  });

  expect(calls).toBe(3);
  expect(cancelCalls).toBe(0); // sous le seuil : jamais d'annulation

  const doneJob = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(doneJob.status).toBe("done");

  await cleanup(keywordId, jobId);
});

test("3 unreachable consécutifs → failJob avec message explicite + annulation du batch", async () => {
  const keywordId = `${SITE}-crawl-onpage-unreachable-dies`;
  const jobId = randomUUID();
  const url = "https://stuck-unreachable.example.com/";
  await insertKeyword(keywordId, "FR", { FR: snapshot([competitor(url, 1)]) });
  const job = await insertJob(jobId, keywordId);

  let calls = 0;
  const cancelledJobIds: string[] = [];
  await processCrawlOnpage(job, {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-unreachable-2" }),
      getBatchScrape: async () => {
        calls++;
        return { status: "unreachable", error: "connection refused" };
      },
      cancelBatchScrape: async (id: string) => {
        cancelledJobIds.push(id);
      },
    },
    sleep: noRealWait,
  });

  expect(calls).toBe(3);
  expect(cancelledJobIds).toEqual(["fc-job-unreachable-2"]);

  const failed = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(failed.status).toBe("failed");
  expect(failed.error).toContain("injoignable");
  expect(failed.error).toContain("3 polls consécutifs");
  expect(failed.error).toContain("connection refused");

  await cleanup(keywordId, jobId);
});

test("unreachable / sain en alternance (jamais 3 d'affilée) → ne meurt pas", async () => {
  const keywordId = `${SITE}-crawl-onpage-unreachable-alternating`;
  const jobId = randomUUID();
  const url = "https://alternating.example.com/";
  await insertKeyword(keywordId, "FR", { FR: snapshot([competitor(url, 1)]) });
  const job = await insertJob(jobId, keywordId);

  // Séquence : unreachable, scraping (reset), unreachable, unreachable
  // (2 d'affilée, sous le seuil de 3), scraping (reset), completed.
  const sequence: BatchState[] = [
    { status: "unreachable", error: "blip 1" },
    { status: "scraping", completed: 0, total: 1 },
    { status: "unreachable", error: "blip 2" },
    { status: "unreachable", error: "blip 3" },
    { status: "scraping", completed: 0, total: 1 },
    {
      status: "completed",
      pages: [{ url, html: HTML_OK, status: "ok", httpStatus: 200 }],
    },
  ];
  let calls = 0;
  let cancelCalls = 0;
  await processCrawlOnpage(job, {
    firecrawl: {
      firecrawlConfigured: alwaysConfigured,
      startBatchScrape: async () => ({ jobId: "fc-job-unreachable-3" }),
      getBatchScrape: async () => {
        const state = sequence[calls];
        calls++;
        return state!;
      },
      cancelBatchScrape: async () => {
        cancelCalls++;
      },
    },
    sleep: noRealWait,
  });

  expect(calls).toBe(sequence.length);
  expect(cancelCalls).toBe(0);

  const doneJob = (
    await db.select().from(analyticsJobs).where(eq(analyticsJobs.id, jobId))
  )[0];
  expect(doneJob.status).toBe("done");

  await cleanup(keywordId, jobId);
});