chunking.late_chunking

chunktuner.chunking.late_chunking

Late chunking placeholder: token pooling needs per-token embedding APIs.

LateChunkingStrategy 

LateChunkingStrategy(encoding_name='cl100k_base')

Placeholder: delegates to fixed token windows until true late pooling is wired.

Source code in src/chunktuner/chunking/late_chunking.py 
22
23
def __init__(self, encoding_name: str = "cl100k_base"):
    self._inner = FixedTokenStrategy(encoding_name=encoding_name)

chunk 

chunk(doc, config)

Chunk with inner FixedTokenStrategy using chunk_size_tokens / overlap.

Source code in src/chunktuner/chunking/late_chunking.py 
25
26
27
28
29
30
31
32
33
34
35
36
37
38
def chunk(self, doc: Document, config: ChunkConfig) -> list[Chunk]:
    """Chunk with inner `FixedTokenStrategy` using ``chunk_size_tokens`` / overlap."""
    validate_content_type(self.name, self.supported_content_types, doc.content_type)
    max_tokens = int(
        config.params.get("chunk_size_tokens", config.params.get("max_tokens", 256))
    )
    overlap = int(config.params.get("overlap_tokens", 0))
    return self._inner.chunk(
        doc,
        ChunkConfig(
            name="fixed_tokens",
            params={"max_tokens": max(16, max_tokens), "overlap_tokens": overlap},
        ),
    )