Thai segmentation

Source: platform/apps/app/src/lib/rag/thai-segment.ts · rendered from main on every deploy — edit in the repo, not here
/**
 * Thai word segmentation via Intl.Segmenter (available in CF Workers).
 * Zero-dependency. Pluggable — swap to PyThaiNLP sidecar later via the same interface.
 */

export interface ThaiSegmenter {
  segment(text: string): string;
}

let segmenter: Intl.Segmenter | null = null;

function getSegmenter(): Intl.Segmenter {
  if (!segmenter) {
    segmenter = new Intl.Segmenter('th', { granularity: 'word' });
  }
  return segmenter;
}

/** Segment Thai text into space-separated words. Latin text passes through as-is. */
export function segmentThai(text: string): string {
  const seg = getSegmenter();
  const segments = seg.segment(text);
  return Array.from(segments)
    .filter((s) => s.isWordLike)
    .map((s) => s.segment)
    .join(' ');
}

/** Segment for FTS5 indexing — combines Thai segmentation with lowercase Latin. */
export function segmentForIndex(text: string): string {
  return segmentThai(text).toLowerCase();
}

/** Segment a search query — same as index but preserves original tokens too. */
export function segmentQuery(query: string): string {
  const segmented = segmentThai(query);
  // Include original query as well for exact-match boosting
  const original = query.trim();
  if (segmented === original) return segmented.toLowerCase();
  return `${segmented} ${original}`.toLowerCase();
}