Chunk Node

import { createFlow, parse, chunk } from '@doclo/flows';

const flow = createFlow()
  .step('parse', parse({ provider: ocrProvider }))
  .step('chunk', chunk({
    strategy: 'recursive',
    maxSize: 1000,
    overlap: 100
  }))
  .build();

const result = await flow.run({ base64: pdfDataUrl });
// result.output is ChunkOutput

chunk({
  strategy: 'recursive',     // Chunking strategy
  maxSize: 1000,             // Max characters per chunk
  minSize: 100,              // Min characters per chunk
  overlap: 100,              // Character overlap between chunks
  separators: ['\n\n', '\n', '. ', ' ']  // Hierarchical separators
})

chunk({
  strategy: 'recursive',
  maxSize: 1000,
  minSize: 100,
  overlap: 100,
  separators: ['\n\n', '\n', '. ', ' ']
})

chunk({
  strategy: 'section',
  maxSize: 2000,
  minSize: 100
})

chunk({
  strategy: 'page',
  pagesPerChunk: 1,
  combineShortPages: true,
  minPageContent: 100
})

chunk({
  strategy: 'fixed',
  size: 512,
  unit: 'characters',
  overlap: 50
})

interface ChunkOutput {
  chunks: ChunkMetadata[];
  totalChunks: number;
  averageChunkSize: number;
  sourceDocument?: DocumentIR;  // Original for citation mapping
}

interface ChunkMetadata {
  content: string;         // Chunk text content
  id: string;              // Unique chunk identifier
  index: number;           // Position in sequence
  startChar: number;       // Start position in original
  endChar: number;         // End position in original
  pageNumbers: number[];   // Pages this chunk spans
  section?: string;        // Section title if detected
  headers?: string[];      // Header hierarchy
  strategy: string;        // Which strategy created this
  wordCount: number;
  charCount: number;
}

const ragFlow = createFlow()
  .step('parse', parse({ provider: ocrProvider }))
  .step('chunk', chunk({
    strategy: 'recursive',
    maxSize: 500,
    overlap: 50
  }))
  .build();

const result = await ragFlow.run({ base64: pdf });

// Generate embeddings for each chunk
for (const chunkData of result.output.chunks) {
  const embedding = await generateEmbedding(chunkData.content);
  await saveToVectorStore(chunkData.id, embedding, chunkData);
}

const flow = createFlow()
  .step('parse', parse({ provider: ocrProvider }))
  .step('chunk', chunk({
    strategy: 'page',
    pagesPerChunk: 5
  }))
  .forEach('extract', (chunkOutput) =>
    // Process each page group
    createFlow()
      .step('extract', extract({
        provider: llmProvider,
        schema: schema
      }))
  )
  .step('combine', combine({ strategy: 'merge' }))
  .build();

chunk({
  strategy: 'recursive',
  maxSize: 1000,
  overlap: 200  // 200 char overlap
})

const result = await flow.run({ base64: pdf });

for (const chunkData of result.output.chunks) {
  console.log(`Chunk ${chunkData.index}:`);
  console.log(`  Pages: ${chunkData.pageNumbers.join(', ')}`);
  console.log(`  Position: ${chunkData.startChar}-${chunkData.endChar}`);
  console.log(`  Content: ${chunkData.content.substring(0, 100)}...`);
}