Skip to content
Puccha Puccha KM

Chunker — headingless chunks, Thai anchors, TOC

Source: platform/packages/rag/src/ingest/chunker.ts · rendered from main on every deploy — edit in the repo, not here

platform/packages/rag/src/ingest/chunker.ts
import { isGenericHeading } from './headings.js';


export interface ChunkResult {
  ordinal: number;
  anchor: string;
  title: string;
  text: string;
  tokens: number;
}


export interface ChunkOptions {
  locale: string;
  maxTokens?: number;
  overlap?: number;
  /**
   * Document title. Heads the table-of-contents chunk (ADR-0187) so the
   * chunk answers "what <things> does <doc> have?" and cites as the
   * document itself. Optional — callers without a title still get a TOC
   * headed by a locale-appropriate "Contents".
   */
  title?: string;
}


const DEFAULT_MAX_TOKENS = 512;
const DEFAULT_OVERLAP = 64;


/**
 * Split markdown into chunks at heading boundaries, with a token budget per chunk.
 * Uses a rough 4-chars-per-token estimate; Thai tokenization is handled server-side
 * by the PyThaiNLP sidecar before FTS5 indexing.
 *
 * When the document has no markdown `#` headings (a plain `.txt` FAQ, a pasted
 * doc), we fall back to structure-aware splitting — Q&A markers and emoji
 * section headers — so a heading-less file doesn't collapse into a few giant
 * chunks all titled "Introduction" (ADR-0112 D2).
 */
export function chunkMarkdown(markdown: string, opts: ChunkOptions): ChunkResult[] {
  const maxTokens = opts.maxTokens ?? DEFAULT_MAX_TOKENS;
  const hasMarkdownHeadings = /^#{1,6}\s+.+/m.test(markdown);
  const sections = hasMarkdownHeadings ? splitByHeadings(markdown) : splitByStructure(markdown);
  const chunks: ChunkResult[] = [];


  // ADR-0187: a listing page (centres, products, branches…) is one name
  // per section or link, so a "what X are there?" question can only ever
  // retrieve topK of them. Prepend one chunk that lists every name.
  const toc = buildTocChunk(markdown, sections, opts);
  if (toc) chunks.push({ ordinal: 0, ...toc });


  for (const section of sections) {
    const estimatedTokens = Math.ceil(section.text.length / 4);


    if (estimatedTokens <= maxTokens) {
      chunks.push({
        ordinal: chunks.length,
        anchor: section.anchor,
        title: section.title,
        text: section.text,
        tokens: estimatedTokens,
      });
    } else {
      const subChunks = splitByTokenBudget(
        section.text,
        maxTokens,
        opts.overlap ?? DEFAULT_OVERLAP,
      );
      for (const sub of subChunks) {
        chunks.push({
          ordinal: chunks.length,
          anchor: section.anchor,
          title: section.title,
          text: sub.text,
          tokens: sub.tokens,
        });
      }
    }
  }


  return chunks;
}


interface Section {
  anchor: string;
  title: string;
  text: string;
}


/** Minimum distinct names before a document counts as a listing. */
const TOC_MIN_ITEMS = 8;
/** Names to keep at most — a TOC that needs more is a sitemap, not a list. */
const TOC_MAX_ITEMS = 150;
/** Link texts repeated this often are chrome (breadcrumbs, "Read more"). */
const TOC_REPEAT_LIMIT = 3;
const TOC_ANCHOR = 'toc';
const TOC_HEADING: Record<string, string> = { th: 'สารบัญ', en: 'Contents' };


/**
 * Build the table-of-contents chunk for a listing document, or null when the
 * document is not a listing. Names come from two places: real section
 * headings (letter-index and boilerplate headings excluded via
 * isGenericHeading) and markdown link texts — the centre cards on a hospital
 * site are `[ศูนย์ทันตกรรม](/th/...)`, not headings. Link texts that repeat
 * across the page are navigation, not names, and are dropped; so is the
 * document's own title. Output is trimmed to the chunk token budget so it
 * always embeds as one vector.
 */
function buildTocChunk(
  markdown: string,
  sections: Section[],
  opts: ChunkOptions,
): Omit<ChunkResult, 'ordinal'> | null {
  const title = (opts.title ?? '').trim();
  const seen = new Set<string>();
  const items: string[] = [];
  const push = (raw: string) => {
    const name = raw.replace(/\s+/g, ' ').trim();
    if (!name || name.length > 80) return;
    if (isGenericHeading(name)) return;
    if (title && name.toLowerCase() === title.toLowerCase()) return;
    const key = name.toLowerCase();
    if (seen.has(key)) return;
    seen.add(key);
    items.push(name);
  };


  for (const section of sections) push(section.title);


  const linkCounts = new Map<string, number>();
  const linkTexts: string[] = [];
  for (const m of markdown.matchAll(/\[([^\]\n]{1,80})\]\([^)\n]*\)/g)) {
    const text = m[1].replace(/\s+/g, ' ').trim();
    if (!text) continue;
    linkTexts.push(text);
    linkCounts.set(text, (linkCounts.get(text) ?? 0) + 1);
  }
  for (const text of linkTexts) {
    if ((linkCounts.get(text) ?? 0) >= TOC_REPEAT_LIMIT) continue;
    push(text);
  }


  if (items.length < TOC_MIN_ITEMS) return null;


  const heading = title || TOC_HEADING[opts.locale] || TOC_HEADING.en;
  const maxChars = (opts.maxTokens ?? DEFAULT_MAX_TOKENS) * 4;
  let kept = items.slice(0, TOC_MAX_ITEMS);
  let text = `${heading}\n\n${kept.map((i) => `- ${i}`).join('\n')}`;
  while (text.length > maxChars && kept.length > TOC_MIN_ITEMS) {
    kept = kept.slice(0, -1);
    text = `${heading}\n\n${kept.map((i) => `- ${i}`).join('\n')}`;
  }


  return {
    anchor: TOC_ANCHOR,
    title: heading,
    text,
    tokens: Math.ceil(text.length / 4),
  };
}


function splitByHeadings(markdown: string): Section[] {
  const lines = markdown.split('\n');
  const sections: Section[] = [];
  let current: Section = { anchor: '', title: 'Introduction', text: '' };


  for (const line of lines) {
    const match = line.match(/^(#{1,6})\s+(.+)/);
    if (match) {
      if (current.text.trim()) {
        sections.push(current);
      }
      const title = match[2].trim();
      current = {
        anchor: slugify(title),
        title,
        text: '',
      };
    } else {
      current.text += `${line}\n`;
    }
  }


  if (current.text.trim()) {
    sections.push(current);
  }


  return sections;
}


// FAQ-style item markers: "Q1:", "Q12.", Thai "ข้อ 3", "คำถามที่ 5". Capture
// group 1 is the marker (ASCII-safe for anchors), group 2 the question text.
const QA_MARKER_RE = /^(Q\d+|ข้อ\s*\d+|คำถาม(?:ที่)?\s*\d+)\s*[:.)\-–]?\s*(.*)$/i;
// An emoji-led short line is treated as a section header (e.g. "🔐 การสมัคร…").
const EMOJI_LEAD_RE = /^\p{Extended_Pictographic}/u;


/** True when the next non-empty line after index `i` is a Q&A marker. Used to
 * tell a section header (introduces questions) from an emoji list bullet
 * inside an answer ("📧 อีเมล / 🪪 บัตร / 📱 โทรศัพท์"). */
function nextMeaningfulIsQa(lines: string[], i: number): boolean {
  for (let j = i + 1; j < lines.length; j++) {
    const t = lines[j].trim();
    if (!t) continue;
    return QA_MARKER_RE.test(t);
  }
  return false;
}


/**
 * Structure-aware split for documents without markdown headings. Starts a new
 * section at each emoji section header and each Q&A marker, deriving a real
 * title (the section name + the question) instead of "Introduction". Plain
 * prose with neither marker yields a single "Introduction" section, matching
 * the previous behaviour.
 *
 * An emoji-led line is only a *section header* when it introduces questions
 * (the next non-empty line is a Q&A marker); otherwise it's a list bullet that
 * stays with its question's chunk — so a 3-option answer doesn't fragment into
 * three 8-char chunks. The header line itself is not emitted as a standalone
 * chunk: it sets the section context for the following Q blocks (whose titles
 * carry the section name, and whose embeddings include the title).
 */
function splitByStructure(markdown: string): Section[] {
  const lines = markdown.split('\n');
  const sections: Section[] = [];
  let sectionTitle = '';
  let current: Section = { anchor: '', title: 'Introduction', text: '' };


  const flush = () => {
    if (current.text.trim()) sections.push(current);
  };


  for (let i = 0; i < lines.length; i++) {
    const line = lines[i];
    const trimmed = line.trim();
    const qa = trimmed.match(QA_MARKER_RE);
    const emojiLed = EMOJI_LEAD_RE.test(trimmed) && trimmed.length <= 60 && !qa;
    const isHeader = emojiLed && nextMeaningfulIsQa(lines, i);


    if (isHeader) {
      flush();
      sectionTitle = stripLeadingEmoji(trimmed);
      current = {
        anchor: slugify(sectionTitle),
        title: sectionTitle || 'Introduction',
        text: '',
      };
    } else if (qa) {
      flush();
      const question = qa[2].trim();
      const title = question
        ? sectionTitle
          ? `${sectionTitle} — ${truncate(question, 80)}`
          : truncate(question, 80)
        : sectionTitle || 'Introduction';
      current = {
        anchor: slugify(`${sectionTitle}-${qa[1]}`),
        title,
        text: `${line}\n`,
      };
    } else {
      current.text += `${line}\n`;
    }
  }


  flush();


  if (sections.length === 0 && markdown.trim()) {
    return [{ anchor: '', title: 'Introduction', text: markdown }];
  }
  return sections;
}


function splitByTokenBudget(
  text: string,
  maxTokens: number,
  _overlap: number,
): { text: string; tokens: number }[] {
  // Break into units no larger than the budget — paragraphs first, then lines,
  // then a hard char slice — so a doc with single-newline structure (no blank
  // lines) can never produce an over-budget chunk.
  const units: string[] = [];
  for (const para of text.split(/\n\n+/)) {
    if (Math.ceil(para.length / 4) <= maxTokens) {
      units.push(para);
      continue;
    }
    for (const line of para.split('\n')) {
      if (Math.ceil(line.length / 4) <= maxTokens) {
        units.push(line);
        continue;
      }
      const maxChars = maxTokens * 4;
      for (let i = 0; i < line.length; i += maxChars) {
        units.push(line.slice(i, i + maxChars));
      }
    }
  }


  const results: { text: string; tokens: number }[] = [];
  let buf = '';
  let bufTokens = 0;


  for (const unit of units) {
    const unitTokens = Math.ceil(unit.length / 4);
    if (bufTokens + unitTokens > maxTokens && buf) {
      results.push({ text: buf.trim(), tokens: bufTokens });
      buf = '';
      bufTokens = 0;
    }
    buf += `${unit}\n`;
    bufTokens += unitTokens;
  }


  if (buf.trim()) {
    results.push({ text: buf.trim(), tokens: bufTokens });
  }


  return results;
}


function stripLeadingEmoji(text: string): string {
  return text.replace(/^(?:\p{Extended_Pictographic}|\uFE0F|\u200D|\s)+/u, '').trim();
}


function truncate(text: string, maxChars: number): string {
  return text.length <= maxChars ? text : `${text.slice(0, maxChars).trim()}…`;
}


function slugify(text: string): string {
  // Unicode-aware like $lib/docs/slug.ts (#451): `\w` is ASCII and would
  // strip every Thai letter; \p{M} keeps vowel signs and tone marks.
  return text
    .toLowerCase()
    .replace(/[^\p{L}\p{N}\p{M}\s-]/gu, '')
    .replace(/\s+/g, '-')
    .replace(/-+/g, '-')
    .trim();
}