// src/lib/semantic/fetch.ts — mobile-first HTML fetch for the semantic pipeline.
// Mirrors src/lib/keywords/onpage-fetch.ts guardrails (AbortController 15s, streamed
// read capped at 3 MB, content-type filter). Any failure (network, timeout, non-2xx,
// non-HTML, body error) → null: never a thrown action.

const TIMEOUT_MS = 15_000;
/** Read cap: pages bigger than 3 MB are truncated. */
const MAX_BYTES = 3 * 1024 * 1024;

/** Mobile UA (CDC §3.0) — Chrome/Android, tagged as our bot. Overridable via config. */
export const MOBILE_UA =
  "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36 (compatible; SEOInsightBot/1.0)";

export interface SemanticHtmlResult {
  html: string;
  finalUrl: string;
}

/**
 * Fetches a page's HTML mobile-first. `fetcher` is injectable for tests (no
 * mock.module needed); `ua` defaults to `MOBILE_UA`. Returns `null` on any
 * failure — never throws.
 */
export async function fetchSemanticHtml(
  url: string,
  fetcher: typeof fetch = fetch,
  ua: string = MOBILE_UA,
): Promise<SemanticHtmlResult | null> {
  try {
    const res = await fetcher(url, {
      signal: AbortSignal.timeout(TIMEOUT_MS),
      redirect: "follow",
      headers: {
        "User-Agent": ua,
        Accept: "text/html,application/xhtml+xml",
      },
    });
    if (!res.ok) return null;
    const type = res.headers.get("content-type") ?? "";
    if (!type.includes("text/html") && !type.includes("xhtml")) return null;
    const reader = res.body?.getReader();
    if (!reader) return null;

    const chunks: Uint8Array[] = [];
    let total = 0;
    while (total < MAX_BYTES) {
      const { done, value } = await reader.read();
      if (done) break;
      chunks.push(value);
      total += value.byteLength;
    }
    if (total >= MAX_BYTES) {
      // over the cap: stop the stream, keep what was read (truncated below)
      await reader.cancel().catch(() => {});
    }

    const body = new Uint8Array(Math.min(total, MAX_BYTES));
    let offset = 0;
    for (const chunk of chunks) {
      const room = body.length - offset;
      if (room <= 0) break;
      body.set(
        room >= chunk.byteLength ? chunk : chunk.subarray(0, room),
        offset,
      );
      offset += Math.min(room, chunk.byteLength);
    }

    return {
      html: new TextDecoder().decode(body),
      finalUrl: res.url || url,
    };
  } catch {
    return null;
  }
}