Ingestion & Chunking

import {
  buildRAGUpsertInputFromDirectory,
  createInMemoryRAGStore,
  createRAGCollection,
  openaiEmbeddings
} from '@absolutejs/rag';

const collection = createRAGCollection({
  embedding: openaiEmbeddings({
    apiKey: process.env.OPENAI_API_KEY ?? '',
    defaultModel: 'text-embedding-3-small'
  }),
  store: createInMemoryRAGStore()
});

// Walks ./docs (recursive by default), picks an extractor per file,
// chunks every document, and flattens to upsert-ready chunks.
const upsert = await buildRAGUpsertInputFromDirectory({
  baseMetadata: { team: 'platform' },
  directory: './docs',
  includeExtensions: ['.md', '.mdx', '.pdf']
});

// Chunks without a precomputed embedding are embedded by the
// collection's provider at upsert time (kind: 'passage').
await collection.ingest(upsert);

import {
  createRAGChunkingRegistry,
  createRAGFileExtractorRegistry,
  createRAGImageOCRExtractor,
  createRAGPDFOCRExtractor,
  loadRAGDocumentsFromDirectory,
  openaiOCR,
  prepareRAGDocuments
} from '@absolutejs/rag';

// OCR and media extractors are opt-in : they need a provider, so they
// are not part of the built-in extractor chain.
const ocr = openaiOCR({ apiKey: process.env.OPENAI_API_KEY ?? '' });

const extractorRegistry = createRAGFileExtractorRegistry([
  {
    extensions: ['.png', '.jpg', '.webp'],
    extractor: createRAGImageOCRExtractor(ocr)
  },
  {
    extensions: ['.pdf'],
    // Uses the native text layer when it yields >= 80 characters;
    // falls back to OCR for scanned pages (or set alwaysOCR: true).
    extractor: createRAGPDFOCRExtractor({ provider: ocr })
  }
]);

// Chunking profiles match on format / source / document id; anything
// unmatched falls back to defaultChunking, then the built-in
// 900 / 120 / 80 paragraphs defaults.
const chunkingRegistry = createRAGChunkingRegistry([
  {
    formats: ['markdown'],
    profile: { maxChunkLength: 1200, strategy: 'source_aware' }
  }
]);

const loaded = await loadRAGDocumentsFromDirectory({
  chunkingRegistry,
  defaultChunking: { chunkOverlap: 120, maxChunkLength: 900 },
  directory: './scans',
  extractorRegistry
});

const prepared = prepareRAGDocuments(loaded);