import { createReadStream } from "node:fs";

/**
 * Streams trimmed, non-empty JSONL lines with bounded memory. A line longer than `maxLineBytes`
 * is discarded (reported as `tooLarge`) instead of being buffered, so a whole transcript is never
 * held in RAM and one oversized record does not invalidate the file. Parsing stays in each adapter.
 */
export async function* readJsonlLines(absolutePath: string, maxLineBytes: number): AsyncGenerator<{ line?: string; tooLarge?: true }> {
  const stream = createReadStream(absolutePath, { encoding: "utf8", highWaterMark: 64 * 1024 });
  let buffered = "";
  let discardingOversizedLine = false;
  for await (const chunk of stream) {
    let remaining = chunk as string;
    if (discardingOversizedLine) {
      const newlineIndex = remaining.indexOf("\n");
      if (newlineIndex < 0) continue;
      discardingOversizedLine = false;
      remaining = remaining.slice(newlineIndex + 1);
    }
    buffered += remaining;
    let newlineIndex = buffered.indexOf("\n");
    while (newlineIndex >= 0) {
      const line = buffered.slice(0, newlineIndex).trim();
      buffered = buffered.slice(newlineIndex + 1);
      if (Buffer.byteLength(line, "utf8") > maxLineBytes) yield { tooLarge: true };
      else if (line) yield { line };
      newlineIndex = buffered.indexOf("\n");
    }
    if (Buffer.byteLength(buffered, "utf8") > maxLineBytes) {
      buffered = "";
      discardingOversizedLine = true;
      yield { tooLarge: true };
    }
  }
  if (!discardingOversizedLine) {
    const line = buffered.trim();
    if (line) yield { line };
  }
}