ingestion.repo_ingestor

chunktuner.ingestion.repo_ingestor

Walk a repository tree into per-file Document records.

RepoIngestor

Walk a repository tree into code or prose Document records (skips vendor dirs).

ingest_repo 

ingest_repo(root)

Recursively ingest supported extensions under root.

Source code in src/chunktuner/ingestion/repo_ingestor.py 
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
def ingest_repo(self, root: Path) -> list[Document]:
    """Recursively ingest supported extensions under ``root``."""
    root = root.resolve()
    docs: list[Document] = []
    exts = {".py", ".js", ".ts", ".tsx", ".go", ".java", ".rs", ".cpp", ".c", ".md", ".txt"}
    for path in sorted(root.rglob("*")):
        if not path.is_file():
            continue
        if path.suffix.lower() not in exts:
            continue
        if self._skip(path, root):
            continue
        try:
            raw = path.read_text(encoding="utf-8", errors="replace")
        except OSError:
            continue
        detected = detect_content_type(path, raw)
        if detected in ("markdown", "text"):
            ct: ContentType = "markdown" if detected == "markdown" else "text"
            content = preprocess(raw, detected)
        else:
            ct = "code"
            content = raw
        lang = _EXT_LANG.get(path.suffix.lower())
        docs.append(
            Document(
                id=str(uuid.uuid4()),
                content=content,
                content_type=ct,
                path=str(path),
                language=lang,
                metadata={"repo_root": str(root), "rel_path": str(path.relative_to(root))},
            )
        )
    return docs