Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# ccBitTorrent - High-Performance BitTorrent Client

[![codecov](https://codecov.io/gh/ccBittorrent/ccbt/branch/main/graph/badge.svg)](https://codecov.io/gh/ccBittorrent/ccbt)
[![codecov](https://codecov.io/gh/ccBitTorrent/ccbt/branch/main/graph/badge.svg)](https://app.codecov.io/gh/ccBitTorrent/ccbt)
[![🥷 Bandit](https://img.shields.io/badge/🥷-security-yellow.svg)](https://ccbittorrent.readthedocs.io/en/reports/bandit/)
[![🐍python 🟰](https://github.com/ccBitTorrent/ccbt/actions/workflows/test.yml/badge.svg)](https://github.com/ccBitTorrent/ccbt/actions/workflows/test.yml)
[![🐧Linux](https://github.com/ccBitTorrent/ccbt/actions/workflows/test.yml/badge.svg)](https://github.com/ccBitTorrent/ccbt/actions/workflows/test.yml)
Expand Down
52 changes: 39 additions & 13 deletions .github/workflows/build-documentation.yml
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,6 @@ jobs:
restore-keys: |
mkdocs-

- name: Cache pytest cache (for coverage report)
uses: actions/cache@v3
with:
path: .pytest_cache
key: pytest-docs-${{ github.sha }}
restore-keys: |
pytest-docs-

- name: Install dependencies
run: |
uv sync --dev
Expand Down Expand Up @@ -131,10 +123,41 @@ jobs:
mkdir -p docs/reports/bandit
mkdir -p docs/en/reports/bandit

- name: Generate coverage report
run: |
uv run pytest -c dev/pytest.ini tests/ --cov=ccbt --cov-report=html:site/reports/htmlcov
- name: Resolve Test workflow run for this commit
id: test-run
uses: actions/github-script@v7
with:
script: |
const headSha = context.eventName === 'pull_request'
? context.payload.pull_request.head.sha
: context.sha;
const { data } = await github.rest.actions.listWorkflowRuns({
owner: context.repo.owner,
repo: context.repo.repo,
workflow_id: 'test.yml',
head_sha: headSha,
per_page: 10,
});
const run = data.workflow_runs.find(
(r) => r.head_sha === headSha && r.status === 'completed' && r.conclusion === 'success'
);
if (run) {
core.info(`Using coverage from Test run ${run.id} for ${headSha}`);
core.setOutput('run_id', String(run.id));
} else {
core.warning(`No successful Test run found for ${headSha}; docs will use the coverage placeholder.`);
core.setOutput('run_id', '');
}

- name: Download combined coverage HTML from Test run
if: steps.test-run.outputs.run_id != ''
continue-on-error: true
uses: actions/download-artifact@v4
with:
name: coverage-htmlcov
path: site/reports/htmlcov
run-id: ${{ steps.test-run.outputs.run_id }}
github-token: ${{ secrets.GITHUB_TOKEN }}

- name: Generate Bandit report
run: |
Expand All @@ -144,12 +167,15 @@ jobs:

- name: Ensure report files exist in documentation location
run: |
# Ensure coverage directory has at least an index file (even if empty)
# Fallback when no coverage-htmlcov artifact was downloaded (e.g. manual
# dispatch on a ref with no successful Test run): use a placeholder so the
# docs build still succeeds without re-running the full test suite.
mkdir -p site/reports/htmlcov
if [ ! -f site/reports/htmlcov/index.html ]; then
echo "No coverage HTML downloaded; writing placeholder coverage page."
echo '<html><body><h1>Coverage Report</h1><p>Coverage report not available. Run tests to generate coverage data.</p></body></html>' > site/reports/htmlcov/index.html
fi

# Ensure bandit reports exist in documentation location (docs/en/reports/bandit/)
mkdir -p docs/en/reports/bandit

Expand Down
38 changes: 34 additions & 4 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -137,12 +137,23 @@ jobs:
--timeout-method=thread \
${COV_ARGS}

# Give each shard a unique data-file name so the coverage job's
# merge-multiple download does not overwrite shards sharing ".coverage".
- name: Stage shard coverage data with unique name
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
shell: bash
run: |
if [ -f .coverage ]; then
cp .coverage ".coverage.${{ matrix.shard }}"
fi

- name: Upload shard coverage data
if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
uses: actions/upload-artifact@v4
with:
name: coverage-data-py311-${{ matrix.shard }}
path: .coverage
path: .coverage.${{ matrix.shard }}
include-hidden-files: true
retention-days: 1
if-no-files-found: ignore

Expand Down Expand Up @@ -189,22 +200,41 @@ jobs:
pattern: coverage-data-py311-*
merge-multiple: true

- name: Combine coverage and upload to Codecov
- name: Combine coverage
id: combine
run: |
if ! ls .coverage* 1>/dev/null 2>&1; then
echo "No coverage data from test shards; skipping combine."
echo "has_coverage=false" >> "$GITHUB_OUTPUT"
exit 0
fi
uv run coverage combine || true
uv run coverage xml -o coverage.xml
uv run coverage report --fail-under=0
if [ -f coverage.xml ]; then
echo "has_coverage=true" >> "$GITHUB_OUTPUT"
else
echo "has_coverage=false" >> "$GITHUB_OUTPUT"
fi
# HTML is a docs convenience; never let it block the Codecov upload.
uv run coverage html -d site/reports/htmlcov || echo "::warning::coverage html failed; docs will use the placeholder page"

- name: Upload coverage to Codecov
if: hashFiles('.coverage', 'coverage.xml') != ''
if: steps.combine.outputs.has_coverage == 'true'
uses: codecov/codecov-action@v4
with:
file: ./coverage.xml
files: ./coverage.xml
flags: unittests
name: codecov-umbrella
codecov_yml_path: dev/.codecov.yml
token: ${{ secrets.CODECOV_TOKEN }}
fail_ci_if_error: true

- name: Upload combined coverage HTML for docs
if: steps.combine.outputs.has_coverage == 'true'
uses: actions/upload-artifact@v4
with:
name: coverage-htmlcov
path: site/reports/htmlcov/
retention-days: 7
if-no-files-found: ignore
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

The canonical changelog is maintained in [`dev/CHANGELOG.md`](dev/CHANGELOG.md).

## [0.3.0] - 2026-11-15

See [`dev/CHANGELOG.md`](dev/CHANGELOG.md) for the full 0.3.0 release notes.

## [0.2.0] - 2026-11-15

See [`dev/CHANGELOG.md`](dev/CHANGELOG.md) for the full 0.2.0 release notes.
Expand Down
2 changes: 1 addition & 1 deletion ccbt/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import importlib

__version__ = "0.2.0"
__version__ = "0.3.0"

# Ensure a default asyncio event loop exists on import for libraries/tests that
# construct futures outside of a running loop (e.g., asyncio.Future()).
Expand Down
16 changes: 12 additions & 4 deletions ccbt/cas_server/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,13 +160,21 @@ async def post_shard_v2(
body = await request.read()
resp = web.StreamResponse(headers={"Content-Type": "application/x-ndjson"})
await resp.prepare(request)
# NDJSON frames follow the upstream ShardUploadEvent serde shape
# (xet_client::cas_types): {"type": ...} tagged frames that stock
# hf_xet parses. A {"status": ...} frame is rejected by hf_xet.
try:
await resp.write(b'{"status": "validating"}\n')
result = _store(request).put_shard(body)
await resp.write(json.dumps({"result": result}).encode() + b"\n")
await resp.write(b'{"type": "validating", "verified": 0, "total": 1}\n')
_store(request).put_shard(body)
await resp.write(b'{"type": "validating", "verified": 1, "total": 1}\n')
await resp.write(b'{"type": "committing", "stage": "uploading"}\n')
await resp.write(b'{"type": "committing", "stage": "syncing"}\n')
await resp.write(b'{"type": "result"}\n')
except ValueError as e:
await resp.write(
json.dumps({"error": {"message": str(e), "retryable": False}}).encode()
json.dumps(
{"type": "error", "message": str(e), "retryable": False}
).encode()
+ b"\n"
)
await resp.write_eof()
Expand Down
11 changes: 9 additions & 2 deletions ccbt/discovery/xet_cas_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,14 @@ async def _upload_shard_v2(self, body: bytes) -> Optional[int]:
event = json.loads(line)
except json.JSONDecodeError:
continue
if "error" in event:
# Upstream ShardUploadEvent frames are {"type": ...} tagged;
# older servers used {"result": n} / {"error": {...}} shapes.
if event.get("type") == "error":
msg = f"shard upload (v2) error frame: {event.get('message', '')}"
raise CasHttpError(msg)
if event.get("type") == "result":
result = 1
elif "error" in event:
err = event["error"]
msg = (
err.get("message", str(err))
Expand All @@ -296,7 +303,7 @@ async def _upload_shard_v2(self, body: bytes) -> Optional[int]:
)
msg = f"shard upload (v2) error frame: {msg}"
raise CasHttpError(msg)
if "result" in event:
elif "result" in event:
result = int(event["result"])
return result if result is not None else 1
finally:
Expand Down
81 changes: 78 additions & 3 deletions ccbt/hub/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,13 +50,38 @@ def _repo_id(request: web.Request) -> str:


# -------------------------------------------------------------- hub repos
def _payload_repo_id(payload: dict) -> Optional[str]:
"""Resolve repo id from hub client payloads (name + organization split)."""
name = payload.get("name") or payload.get("repo_id")
organization = payload.get("organization") or payload.get("namespace")
if name and organization and "/" not in str(name):
return f"{organization}/{name}"
return str(name) if name else None


async def delete_repo(request: web.Request) -> web.Response:
"""Public function."""
try:
payload = await request.json()
except Exception:
return _err("expected JSON body", 400)
repo_id = _payload_repo_id(payload)
if not repo_id:
return _err("name is required", 400)
if not _repos(request).delete_repo(repo_id):
return _err("repo not found", 404)
return web.json_response({"id": repo_id, "deleted": True})


async def create_repo(request: web.Request) -> web.Response:
"""Public function."""
try:
payload = await request.json()
except Exception:
return _err("expected JSON body", 400)
name = payload.get("name") or payload.get("repo_id")
# Modern huggingface_hub sends {"name": ..., "organization": ...}
# separately; older clients send a single "namespace/name" string.
name = _payload_repo_id(payload)
if not name:
return _err("name is required", 400)
repo_type = payload.get("type", "model")
Expand All @@ -67,8 +92,11 @@ async def create_repo(request: web.Request) -> web.Response:
store = _repos(request)
created = store.create_repo(name, repo_type, bool(payload.get("private", False)))
base = str(request.url.origin())
# huggingface_hub parses the returned url as a repo page URL
# ({base}/{namespace}/{name}); an /api/... URL is rejected as an
# invalid HF URI by newer hub versions.
return web.json_response(
{"id": name, "url": f"{base}/api/{repo_type_path(repo_type)}/{name}"},
{"id": name, "url": f"{base}/{name}"},
status=201 if created else 200,
)

Expand All @@ -90,6 +118,37 @@ async def repo_info(request: web.Request) -> web.Response:
)


async def preupload(request: web.Request) -> web.Response:
"""huggingface_hub upload-mode probe (POST .../preupload/{revision}).

Every file is answered "regular": content then arrives inline in the
commit NDJSON, which `commit` already handles. The Xet/LFS transfer
paths are exercised separately (hf_xet against the CAS, git-lfs with
the git-xet agent), so they are not negotiated here.
"""
store = _repos(request)
repo_id = _repo_id(request)
if not store.repo_exists(repo_id):
return _err("repo not found", 404)
try:
payload = await request.json()
except Exception:
return _err("expected JSON body", 400)
files = payload.get("files") if isinstance(payload, dict) else None
if not isinstance(files, list):
return _err("files must be a list", 400)
out = [
{
"path": str(f.get("path", "")),
"uploadMode": "regular",
"shouldIgnore": False,
}
for f in files
if isinstance(f, dict)
]
return web.json_response({"files": out})


async def repo_tree(request: web.Request) -> web.Response:
"""Public function."""
store = _repos(request)
Expand Down Expand Up @@ -168,7 +227,16 @@ async def commit(request: web.Request) -> web.Response:
continue # deletions tracked in WS8-style tombstones; accept + ignore here
else:
return _err(f"unsupported commit operation: {key}", 400)
return web.json_response({"commitOid": "main", "changed": changed})
# huggingface_hub requires commitUrl + commitOid in the response.
commit_oid = hashlib.sha256(body.encode("utf-8")).hexdigest()
base = str(request.url.origin())
return web.json_response(
{
"commitOid": commit_oid,
"commitUrl": f"{base}/{repo_id}/commit/{commit_oid}",
"changed": changed,
}
)


async def resolve_file(request: web.Request) -> web.Response:
Expand Down Expand Up @@ -202,6 +270,7 @@ def fetch(xorb_hash: bytes) -> bytes:
body=data,
headers={
"X-Linked-Etag": entry.oid,
"X-Repo-Commit": entry.oid,
"ETag": entry.oid,
"Content-Length": str(len(data)),
},
Expand Down Expand Up @@ -314,6 +383,8 @@ async def xet_transfer_upload_register(request: web.Request) -> web.Response:
def register_hub_routes(app: web.Application) -> None:
"""Public function."""
app.router.add_post("/api/repos/create", create_repo)
app.router.add_delete("/api/repos/delete", delete_repo)
app.router.add_post("/api/repos/delete", delete_repo)
for prefix in ("models", "datasets", "spaces"):
app.router.add_get(f"/api/{prefix}/{{namespace}}/{{name}}", repo_info)
app.router.add_get(f"/api/{prefix}/{{name}}", repo_info)
Expand All @@ -332,6 +403,10 @@ def register_hub_routes(app: web.Application) -> None:
f"/api/{prefix}/{{namespace}}/{{name}}/commit/{{revision}}", commit
)
app.router.add_post(f"/api/{prefix}/{{name}}/commit/{{revision}}", commit)
app.router.add_post(
f"/api/{prefix}/{{namespace}}/{{name}}/preupload/{{revision}}", preupload
)
app.router.add_post(f"/api/{prefix}/{{name}}/preupload/{{revision}}", preupload)
app.router.add_get("/{namespace}/{name}/resolve/{revision}/{path:.*}", resolve_file)
app.router.add_get("/{name}/resolve/{revision}/{path:.*}", resolve_file)
# Git LFS (git-lfs derives several URL shapes depending on config)
Expand Down
8 changes: 8 additions & 0 deletions ccbt/hub/repos.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,14 @@ def ensure_repo(self, repo_id: str) -> None:
if not self.repo_exists(repo_id):
self.create_repo(repo_id)

def delete_repo(self, repo_id: str) -> bool:
"""Delete a repo and its file manifest (objects are content-addressed and kept)."""
with self._lock:
self.db.execute("DELETE FROM repo_files WHERE repo_id = ?", (repo_id,))
cur = self.db.execute("DELETE FROM repos WHERE repo_id = ?", (repo_id,))
self.db.commit()
return cur.rowcount > 0

# --------------------------------------------------------------- files
def put_file(self, repo_id: str, entry: RepoFile) -> None:
"""Public method."""
Expand Down
11 changes: 11 additions & 0 deletions dev/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [0.3.0] - 2026-11-15

### Exciting New Features 🎉

- XET is now **supported**: all 20 claims (C1–C20) in `dev/xet-claim-registry.yaml` flipped from experimental to supported after the G9 big-bang gate passed on local live evidence (real `hf_xet`/`huggingface_hub`/git-lfs against ccbt CAS/Hub on loopback, plus a cross-test against a real external OpenXet CAS server and the HF production CAS; harness: `dev/scripts/run_xet_live_gate.py`, never run in CI). Suite: `xet-blake3-gearhash-lz4:v2`; Byzantine consensus requires `n >= 3f + 1`

### Fixed 🐞

- Fix CAS shard-upload v2 stream to emit upstream `ShardUploadEvent` frames (`validating`/`committing`/`result` with `type` tags) so stock `hf_xet` accepts uploads against `ccbt cas serve`
- Fix Hub API compatibility with modern `huggingface_hub` (repo URL shape, `preupload`, `commitOid`, repo deletion)

## [0.2.0] - 2026-11-15

### Exciting New Features 🎉
Expand Down
Loading
Loading