From b2157301e5d0fd5652d8a1bd02fbf5e87735f9d3 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Fri, 14 Aug 2026 01:20:11 -0700 Subject: [PATCH] Add portable hardware evidence records --- .../ISSUE_TEMPLATE/benchmark_submission.yml | 24 +- .github/workflows/docs.yml | 1 + PROJECT_STATE.md | 2 + PYPI.md | 10 + README.md | 10 + README.zh-CN.md | 10 + docs/community-benchmarks.md | 28 + docs/generated/hardware-record-v1.schema.json | 555 ++++++++++++++++++ docs/release-checklist.md | 2 +- scripts/publish_hardware_record_schema.py | 43 ++ src/miniverl/cli.py | 2 + src/miniverl/commands/evidence.py | 80 +++ src/miniverl/evidence/hardware.py | 375 ++++++++++++ tests/cli/test_cli.py | 2 + tests/unit/test_hardware_records.py | 230 ++++++++ 15 files changed, 1362 insertions(+), 12 deletions(-) create mode 100644 docs/generated/hardware-record-v1.schema.json create mode 100644 scripts/publish_hardware_record_schema.py create mode 100644 src/miniverl/commands/evidence.py create mode 100644 src/miniverl/evidence/hardware.py create mode 100644 tests/unit/test_hardware_records.py diff --git a/.github/ISSUE_TEMPLATE/benchmark_submission.yml b/.github/ISSUE_TEMPLATE/benchmark_submission.yml index e407298..f7f5407 100644 --- a/.github/ISSUE_TEMPLATE/benchmark_submission.yml +++ b/.github/ISSUE_TEMPLATE/benchmark_submission.yml @@ -8,18 +8,18 @@ body: Thank you. Measured numbers from hardware we do not own are the most useful contribution this project can receive. - Produce a submission file first: + Produce and validate a private-by-default hardware record first: ```bash - miniverl train - miniverl export-benchmark runs/ --out my-result.json + miniverl hardware record --run runs/ --out hardware-record.json + miniverl hardware validate hardware-record.json ``` - `export-benchmark` strips absolute paths and keeps only the GPU model, - VRAM, OS family and library versions. Validate it against - `benchmarks/schema/benchmark-result.schema.json`, compute its SHA-256, - and read it before posting. Never include tokens, private paths or - proprietary data. + The record binds profile, plan, models, quantization, token bounds, + batches, updates, memory, timing and retained artifact hashes. The + command never uploads. Read the JSON before posting and never include + tokens, private paths or proprietary data. Schema-valid community + records remain unreviewed until a maintainer validates their evidence. - type: input id: hardware @@ -40,7 +40,7 @@ body: - type: textarea id: result attributes: - label: Contents of the exported result JSON + label: Contents of the validated hardware-record JSON render: json validations: required: true @@ -58,7 +58,7 @@ body: - type: input id: artifact_hash attributes: - label: Exported JSON SHA-256 + label: Hardware-record JSON SHA-256 description: Hash the exact JSON attached to this submission. placeholder: "64 lowercase hexadecimal characters" validations: @@ -81,9 +81,11 @@ body: required: true - label: I ran the recipe to completion and did not select the best of several runs without saying so. required: true - - label: I validated the JSON against the committed schema and recorded the exact artifact SHA-256. + - label: I ran `miniverl hardware validate` and recorded the exact JSON SHA-256. required: true - label: I reviewed the export and attachments for tokens, private paths and proprietary data. required: true + - label: I consent to publishing this sanitized record and its reported hardware/software metadata. + required: true - label: I am willing to open a pull request adding this file under `benchmarks/results/`. required: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 099931b..511bcdb 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -34,6 +34,7 @@ jobs: python scripts/publish_alignment_lab_artifacts.py --check python scripts/publish_verl_bridge_diagrams.py --check python scripts/publish_verl_opd_compatibility.py --check + python scripts/publish_hardware_record_schema.py --check - name: Build complete development site strictly run: mkdocs build --strict - name: Install deterministic Chromium diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index 46507d0..4ca3ca4 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -35,6 +35,8 @@ Development `0.10.0.dev0` has a closed typed profile registry and torch-free compatibility introspection. Profile-scoped plans, caches, checkpoints and exports bind an independent identity. The direct-GKD and sampled-k1 vanilla policy-loss profiles both have pinned conformance and measured RTX 4080 paths. +Portable hardware records preserve measured, estimated and unknown states; +schema-valid community submissions remain unreviewed until maintainer validation. Arbitrary verl YAML, other policy-gradient modes, rewards, PPO/GRPO, Ray, FSDP, Megatron, multi-GPU and distributed execution remain unsupported. The legacy diff --git a/PYPI.md b/PYPI.md index 6913715..f6bbf4c 100644 --- a/PYPI.md +++ b/PYPI.md @@ -190,6 +190,16 @@ updates. See [hardware planning](https://github.com/DaoyuanLi2816/mini-verl/blob automatic downgrade to a different model, teacher, context, top-k or loss when memory is tight. +Share a sanitized result without uploading it automatically: + +```bash +miniverl hardware record --run runs/my-opd --out hardware-record.json +miniverl hardware validate hardware-record.json +``` + +Community records remain unreviewed until their hashes and provenance pass the +[documented maintainer gate](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/community-benchmarks.md). + ## Data and artifact interoperability The profile consumes structured verl-style Parquet without substituting a toy diff --git a/README.md b/README.md index be1e009..40fc48c 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,16 @@ updates. See [hardware planning](docs/hardware-planning.md). There is no automatic downgrade to a different model, teacher, context, top-k or loss when memory is tight. +Share a sanitized result without uploading it automatically: + +```bash +miniverl hardware record --run runs/my-opd --out hardware-record.json +miniverl hardware validate hardware-record.json +``` + +Community records remain unreviewed until their hashes and provenance pass the +[documented maintainer gate](docs/community-benchmarks.md). + ## Data and artifact interoperability The profile consumes structured verl-style Parquet without substituting a toy diff --git a/README.zh-CN.md b/README.zh-CN.md index f8b5bb8..7d41ae7 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -161,6 +161,16 @@ BF16/FP16 自动选择取决于设备能力，而不是 3070、4080、5090 或 T [硬件规划](docs/hardware-planning.md)。 显存紧张时不会静默更换模型、teacher、context、top-k 或 loss。 +已完成的 run 可以生成经过清理、且不会自动上传的硬件记录： + +```bash +miniverl hardware record --run artifacts/runs/ --out hardware-record.json +miniverl hardware validate hardware-record.json +``` + +社区提交即使通过 schema 校验也仍标记为 `unreviewed`；只有维护者复核并确认发布许可后， +才会进入公开的“maintainer-validated measured”矩阵。 + ## 数据与产物互操作 profile 直接读取结构化 verl 风格 Parquet，不会在缺失数据时静默换成 calculator 环境。 diff --git a/docs/community-benchmarks.md b/docs/community-benchmarks.md index 69693da..240a1e6 100644 --- a/docs/community-benchmarks.md +++ b/docs/community-benchmarks.md @@ -27,3 +27,31 @@ To propose a real result, add the JSON plus any new versioned recipe record in one pull request. Describe unsupported hardware or software explicitly; a negative or failed run is useful evidence and should not be removed merely because it does not improve a benchmark. + +## OPD hardware records + +Completed OPD runs use a separate, stricter version-1 record: + +```bash +miniverl hardware record --run runs/my-opd --out hardware-record.json +miniverl hardware validate hardware-record.json +``` + +The record binds the profile identity, plan and config digests, GPU/VRAM and +software stack, pinned actor/teacher identities, quantization, prompt/response +bounds, teacher-target mode, logical and physical batches, update count, +measured memory and phase timing, resume state, and artifact hashes. Every +numeric evidence item is labelled `measured`, `estimated`, or `unknown`. +Generation and validation are torch-free and perform no upload. The committed +[JSON Schema](generated/hardware-record-v1.schema.json) is generated from the +same Pydantic contract used by the CLI. + +| Record origin | Public status | Required gate | +| --- | --- | --- | +| `maintainer_measured` | measured only after repository review | retained run, hashes, schema/privacy validation and maintainer approval | +| `community_submitted` | unreviewed candidate | validated JSON plus explicit publication consent | +| `not_measured` | documented gap | no numeric value may be presented as measured | + +`hardware validate` proves schema and privacy conformance only. It does not +change `review_status`, publish a file or add a row to the public matrix. An +issue comment or unreviewed JSON can never become a measured row automatically. diff --git a/docs/generated/hardware-record-v1.schema.json b/docs/generated/hardware-record-v1.schema.json new file mode 100644 index 0000000..a335433 --- /dev/null +++ b/docs/generated/hardware-record-v1.schema.json @@ -0,0 +1,555 @@ +{ + "$defs": { + "ArtifactEvidence": { + "additionalProperties": false, + "properties": { + "bytes": { + "minimum": 0, + "title": "Bytes", + "type": "integer" + }, + "name": { + "title": "Name", + "type": "string" + }, + "sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Sha256", + "type": "string" + } + }, + "required": [ + "name", + "sha256", + "bytes" + ], + "title": "ArtifactEvidence", + "type": "object" + }, + "BatchingEvidence": { + "additionalProperties": false, + "properties": { + "logical": { + "minimum": 1, + "title": "Logical", + "type": "integer" + }, + "rollout_physical": { + "minimum": 1, + "title": "Rollout Physical", + "type": "integer" + }, + "teacher_score_physical": { + "minimum": 1, + "title": "Teacher Score Physical", + "type": "integer" + }, + "update_trajectory_physical": { + "minimum": 1, + "title": "Update Trajectory Physical", + "type": "integer" + } + }, + "required": [ + "logical", + "rollout_physical", + "teacher_score_physical", + "update_trajectory_physical" + ], + "title": "BatchingEvidence", + "type": "object" + }, + "BoundsEvidence": { + "additionalProperties": false, + "properties": { + "prompt_tokens": { + "minimum": 1, + "title": "Prompt Tokens", + "type": "integer" + }, + "response_tokens": { + "minimum": 1, + "title": "Response Tokens", + "type": "integer" + } + }, + "required": [ + "prompt_tokens", + "response_tokens" + ], + "title": "BoundsEvidence", + "type": "object" + }, + "EvidenceValue": { + "additionalProperties": false, + "description": "One value with an explicit evidence state and source.", + "properties": { + "source": { + "title": "Source", + "type": "string" + }, + "status": { + "enum": [ + "measured", + "estimated", + "unknown" + ], + "title": "Status", + "type": "string" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Unit" + }, + "value": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "integer" + }, + { + "type": "string" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Value" + } + }, + "required": [ + "status", + "source" + ], + "title": "EvidenceValue", + "type": "object" + }, + "HardwareEvidence": { + "additionalProperties": false, + "properties": { + "cuda_runtime": { + "title": "Cuda Runtime", + "type": "string" + }, + "driver": { + "$ref": "#/$defs/EvidenceValue" + }, + "gpu_count": { + "minimum": 1, + "title": "Gpu Count", + "type": "integer" + }, + "gpu_name": { + "title": "Gpu Name", + "type": "string" + }, + "measurement_status": { + "enum": [ + "measured", + "estimated", + "unknown" + ], + "title": "Measurement Status", + "type": "string" + }, + "os": { + "title": "Os", + "type": "string" + }, + "python": { + "title": "Python", + "type": "string" + }, + "torch": { + "title": "Torch", + "type": "string" + }, + "vram_gib": { + "exclusiveMinimum": 0, + "title": "Vram Gib", + "type": "number" + } + }, + "required": [ + "measurement_status", + "gpu_name", + "gpu_count", + "vram_gib", + "driver", + "cuda_runtime", + "torch", + "os", + "python" + ], + "title": "HardwareEvidence", + "type": "object" + }, + "MeasurementsEvidence": { + "additionalProperties": false, + "properties": { + "peak_allocated_gib": { + "$ref": "#/$defs/EvidenceValue" + }, + "peak_reserved_gib": { + "$ref": "#/$defs/EvidenceValue" + }, + "phase_medians": { + "$ref": "#/$defs/TimingEvidence" + } + }, + "required": [ + "peak_allocated_gib", + "peak_reserved_gib", + "phase_medians" + ], + "title": "MeasurementsEvidence", + "type": "object" + }, + "ModelEvidence": { + "additionalProperties": false, + "properties": { + "dtype": { + "title": "Dtype", + "type": "string" + }, + "model_id": { + "title": "Model Id", + "type": "string" + }, + "quantization": { + "enum": [ + "none", + "nf4", + "int8" + ], + "title": "Quantization", + "type": "string" + }, + "revision": { + "pattern": "^[0-9a-f]{40}$", + "title": "Revision", + "type": "string" + }, + "role": { + "enum": [ + "student", + "teacher" + ], + "title": "Role", + "type": "string" + } + }, + "required": [ + "role", + "model_id", + "revision", + "quantization", + "dtype" + ], + "title": "ModelEvidence", + "type": "object" + }, + "ProfileEvidence": { + "additionalProperties": false, + "properties": { + "identity_digest": { + "pattern": "^[0-9a-f]{64}$", + "title": "Identity Digest", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "upstream_commit": { + "pattern": "^[0-9a-f]{40}$", + "title": "Upstream Commit", + "type": "string" + }, + "upstream_tag": { + "title": "Upstream Tag", + "type": "string" + } + }, + "required": [ + "name", + "identity_digest", + "upstream_tag", + "upstream_commit" + ], + "title": "ProfileEvidence", + "type": "object" + }, + "ResumeEvidence": { + "additionalProperties": false, + "properties": { + "checkpoint_digest": { + "anyOf": [ + { + "pattern": "^[0-9a-f]{64}$", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Checkpoint Digest" + }, + "outcome": { + "enum": [ + "resumed", + "not_exercised", + "unknown" + ], + "title": "Outcome", + "type": "string" + }, + "status": { + "enum": [ + "measured", + "estimated", + "unknown" + ], + "title": "Status", + "type": "string" + } + }, + "required": [ + "status", + "outcome" + ], + "title": "ResumeEvidence", + "type": "object" + }, + "RunEvidence": { + "additionalProperties": false, + "properties": { + "manifest_sha256": { + "pattern": "^[0-9a-f]{64}$", + "title": "Manifest Sha256", + "type": "string" + }, + "plan_digest": { + "pattern": "^[0-9a-f]{64}$", + "title": "Plan Digest", + "type": "string" + }, + "resolved_config_digest": { + "pattern": "^[0-9a-f]{64}$", + "title": "Resolved Config Digest", + "type": "string" + }, + "run_id": { + "title": "Run Id", + "type": "string" + }, + "status": { + "const": "completed", + "title": "Status", + "type": "string" + } + }, + "required": [ + "run_id", + "status", + "manifest_sha256", + "plan_digest", + "resolved_config_digest" + ], + "title": "RunEvidence", + "type": "object" + }, + "TargetEvidence": { + "additionalProperties": false, + "properties": { + "estimator": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Estimator" + }, + "mode": { + "title": "Mode", + "type": "string" + }, + "top_k": { + "anyOf": [ + { + "minimum": 1, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Top K" + } + }, + "required": [ + "mode" + ], + "title": "TargetEvidence", + "type": "object" + }, + "TimingEvidence": { + "additionalProperties": false, + "properties": { + "actor_update_seconds": { + "$ref": "#/$defs/EvidenceValue" + }, + "rollout_seconds": { + "$ref": "#/$defs/EvidenceValue" + }, + "teacher_scoring_seconds": { + "$ref": "#/$defs/EvidenceValue" + } + }, + "required": [ + "rollout_seconds", + "teacher_scoring_seconds", + "actor_update_seconds" + ], + "title": "TimingEvidence", + "type": "object" + } + }, + "additionalProperties": false, + "description": "Version-1 portable record; validation does not confer publication trust.", + "properties": { + "artifacts": { + "items": { + "$ref": "#/$defs/ArtifactEvidence" + }, + "minItems": 1, + "title": "Artifacts", + "type": "array" + }, + "batching": { + "$ref": "#/$defs/BatchingEvidence" + }, + "bounds": { + "$ref": "#/$defs/BoundsEvidence" + }, + "consent_to_publish": { + "default": false, + "title": "Consent To Publish", + "type": "boolean" + }, + "created_at": { + "title": "Created At", + "type": "string" + }, + "hardware": { + "$ref": "#/$defs/HardwareEvidence" + }, + "kind": { + "const": "miniverl_hardware_record", + "default": "miniverl_hardware_record", + "title": "Kind", + "type": "string" + }, + "measurements": { + "$ref": "#/$defs/MeasurementsEvidence" + }, + "miniverl_version": { + "title": "Miniverl Version", + "type": "string" + }, + "models": { + "items": { + "$ref": "#/$defs/ModelEvidence" + }, + "maxItems": 2, + "minItems": 2, + "title": "Models", + "type": "array" + }, + "optimizer_updates": { + "$ref": "#/$defs/EvidenceValue" + }, + "profile": { + "$ref": "#/$defs/ProfileEvidence" + }, + "record_classification": { + "enum": [ + "maintainer_measured", + "community_submitted", + "not_measured" + ], + "title": "Record Classification", + "type": "string" + }, + "resume": { + "$ref": "#/$defs/ResumeEvidence" + }, + "review_status": { + "default": "unreviewed", + "enum": [ + "unreviewed", + "maintainer_validated" + ], + "title": "Review Status", + "type": "string" + }, + "run": { + "$ref": "#/$defs/RunEvidence" + }, + "schema_version": { + "const": 1, + "default": 1, + "title": "Schema Version", + "type": "integer" + }, + "scientific_scope": { + "additionalProperties": { + "type": "boolean" + }, + "title": "Scientific Scope", + "type": "object" + }, + "target": { + "$ref": "#/$defs/TargetEvidence" + } + }, + "required": [ + "created_at", + "record_classification", + "miniverl_version", + "profile", + "run", + "hardware", + "models", + "bounds", + "target", + "batching", + "optimizer_updates", + "measurements", + "resume", + "artifacts", + "scientific_scope" + ], + "title": "HardwareRecord", + "type": "object" +} diff --git a/docs/release-checklist.md b/docs/release-checklist.md index 0ab5fec..c9a636f 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -13,7 +13,7 @@ after the exact release commit and its remote checks are green. freshness, resume and export conformance. - [x] Publish bounded RTX 4080 systems evidence for PG k1 and a full SmolLM2 direct-GKD developer recipe; do not add a task-quality benchmark. -- [ ] Add validated community hardware records and document the measured or +- [x] Add validated community hardware records and document the measured or explicitly unmeasured Linux/WSL state. ## v0.9.1 development diff --git a/scripts/publish_hardware_record_schema.py b/scripts/publish_hardware_record_schema.py new file mode 100644 index 0000000..46d3fd8 --- /dev/null +++ b/scripts/publish_hardware_record_schema.py @@ -0,0 +1,43 @@ +"""Publish the JSON Schema for portable hardware records.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from miniverl.evidence.hardware import HardwareRecord + +DEFAULT_OUT = Path("docs/generated/hardware-record-v1.schema.json") + + +def render() -> str: + """Return deterministic schema bytes.""" + return ( + json.dumps( + HardwareRecord.model_json_schema(), + indent=2, + sort_keys=True, + ensure_ascii=False, + allow_nan=False, + ) + + "\n" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + expected = render() + if args.check: + if not args.out.is_file() or args.out.read_text(encoding="utf-8") != expected: + parser.error(f"generated hardware schema is stale: {args.out}") + return + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(expected, encoding="utf-8", newline="\n") + + +if __name__ == "__main__": + main() diff --git a/src/miniverl/cli.py b/src/miniverl/cli.py index e0da093..65cf40a 100644 --- a/src/miniverl/cli.py +++ b/src/miniverl/cli.py @@ -25,6 +25,7 @@ from miniverl import __version__ from miniverl.commands.compat import compat_app, profiles_app +from miniverl.commands.evidence import hardware_app from miniverl.errors import ConfigError, MiniVerlError from miniverl.utils.runs import make_run_id @@ -61,6 +62,7 @@ app.add_typer(data_app, name="data") app.add_typer(profiles_app, name="profiles") app.add_typer(compat_app, name="compat") +app.add_typer(hardware_app, name="hardware") console = Console() err_console = Console(stderr=True) diff --git a/src/miniverl/commands/evidence.py b/src/miniverl/commands/evidence.py new file mode 100644 index 0000000..c078544 --- /dev/null +++ b/src/miniverl/commands/evidence.py @@ -0,0 +1,80 @@ +"""Portable hardware-evidence commands; intentionally torch-free.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import typer +from rich.console import Console +from rich.markup import escape + +hardware_app = typer.Typer( + help="Create and validate private-by-default hardware records.", + no_args_is_help=True, +) +console = Console() +err_console = Console(stderr=True) + + +def _fail(exc: Exception) -> None: + err_console.print(f"[red]error[/red] {escape(str(exc))}") + raise typer.Exit(1) + + +@hardware_app.command("record") +def hardware_record_command( + run: Path = typer.Option(..., "--run", help="Completed miniVERL run directory."), + out: Path = typer.Option(..., "--out", help="New portable JSON record."), + consent_to_publish: bool = typer.Option( + False, + "--consent-to-publish", + help="Record publication consent; no upload is performed.", + ), + as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Derive a schema-validated record without uploading or exposing paths.""" + from miniverl.evidence.hardware import build_hardware_record, write_hardware_record + + try: + record = build_hardware_record( + run, + consent_to_publish=consent_to_publish, + ) + destination = write_hardware_record(record, out) + except Exception as exc: + _fail(exc) + return + payload = record.model_dump(mode="json") + if as_json: + console.print_json(json.dumps({"written": str(destination), "record": payload})) + else: + console.print(f"[green]hardware record written[/green] {escape(str(destination))}") + console.print(" schema-valid and unreviewed; nothing was uploaded") + + +@hardware_app.command("validate") +def hardware_validate_command( + path: Path = typer.Argument(..., help="Standalone hardware-record JSON."), + as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), +) -> None: + """Validate schema and privacy without promoting the record to measured docs.""" + from miniverl.evidence.hardware import validate_hardware_record + + problems = validate_hardware_record(path) + payload = { + "path": str(path), + "valid": not problems, + "review_status": "unreviewed", + "problems": problems, + } + if as_json: + console.print_json(json.dumps(payload)) + elif not problems: + console.print(f"[green]valid[/green] {escape(str(path))}") + console.print(" schema validation does not confer maintainer review or publication") + else: + for problem in problems: + err_console.print(f"[red]invalid[/red] {escape(problem)}") + if problems: + raise typer.Exit(1) diff --git a/src/miniverl/evidence/hardware.py b/src/miniverl/evidence/hardware.py new file mode 100644 index 0000000..86cf5ee --- /dev/null +++ b/src/miniverl/evidence/hardware.py @@ -0,0 +1,375 @@ +"""Strict, portable community hardware records derived from completed runs.""" + +from __future__ import annotations + +import hashlib +import json +import statistics +from pathlib import Path +from typing import Any, Literal + +import yaml +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from miniverl.utils.privacy import portable_text +from miniverl.utils.runs import read_json, read_jsonl, utc_now, write_json_atomic + +__all__ = [ + "HardwareRecord", + "build_hardware_record", + "load_hardware_record", + "validate_hardware_record", + "write_hardware_record", +] + +_SHA256_PATTERN = r"^[0-9a-f]{64}$" + + +class _StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + + +class EvidenceValue(_StrictModel): + """One value with an explicit evidence state and source.""" + + status: Literal["measured", "estimated", "unknown"] + value: float | int | str | bool | None = None + unit: str | None = None + source: str + + @model_validator(mode="after") + def _status_matches_value(self) -> EvidenceValue: + if self.status in {"measured", "estimated"} and self.value is None: + raise ValueError(f"{self.status} evidence requires a value") + if self.status == "unknown" and self.value is not None: + raise ValueError("unknown evidence must not invent a value") + return self + + +class ProfileEvidence(_StrictModel): + name: str + identity_digest: str = Field(pattern=_SHA256_PATTERN) + upstream_tag: str + upstream_commit: str = Field(pattern=r"^[0-9a-f]{40}$") + + +class RunEvidence(_StrictModel): + run_id: str + status: Literal["completed"] + manifest_sha256: str = Field(pattern=_SHA256_PATTERN) + plan_digest: str = Field(pattern=_SHA256_PATTERN) + resolved_config_digest: str = Field(pattern=_SHA256_PATTERN) + + +class HardwareEvidence(_StrictModel): + measurement_status: Literal["measured", "estimated", "unknown"] + gpu_name: str + gpu_count: int = Field(ge=1) + vram_gib: float = Field(gt=0) + driver: EvidenceValue + cuda_runtime: str + torch: str + os: str + python: str + + +class ModelEvidence(_StrictModel): + role: Literal["student", "teacher"] + model_id: str + revision: str = Field(pattern=r"^[0-9a-f]{40}$") + quantization: Literal["none", "nf4", "int8"] + dtype: str + + +class BoundsEvidence(_StrictModel): + prompt_tokens: int = Field(ge=1) + response_tokens: int = Field(ge=1) + + +class TargetEvidence(_StrictModel): + mode: str + top_k: int | None = Field(default=None, ge=1) + estimator: str | None = None + + +class BatchingEvidence(_StrictModel): + logical: int = Field(ge=1) + rollout_physical: int = Field(ge=1) + teacher_score_physical: int = Field(ge=1) + update_trajectory_physical: int = Field(ge=1) + + +class TimingEvidence(_StrictModel): + rollout_seconds: EvidenceValue + teacher_scoring_seconds: EvidenceValue + actor_update_seconds: EvidenceValue + + +class MeasurementsEvidence(_StrictModel): + peak_allocated_gib: EvidenceValue + peak_reserved_gib: EvidenceValue + phase_medians: TimingEvidence + + +class ResumeEvidence(_StrictModel): + status: Literal["measured", "estimated", "unknown"] + outcome: Literal["resumed", "not_exercised", "unknown"] + checkpoint_digest: str | None = Field(default=None, pattern=_SHA256_PATTERN) + + +class ArtifactEvidence(_StrictModel): + name: str + sha256: str = Field(pattern=_SHA256_PATTERN) + bytes: int = Field(ge=0) + + +class HardwareRecord(_StrictModel): + """Version-1 portable record; validation does not confer publication trust.""" + + schema_version: Literal[1] = 1 + kind: Literal["miniverl_hardware_record"] = "miniverl_hardware_record" + created_at: str + record_classification: Literal["maintainer_measured", "community_submitted", "not_measured"] + review_status: Literal["unreviewed", "maintainer_validated"] = "unreviewed" + miniverl_version: str + profile: ProfileEvidence + run: RunEvidence + hardware: HardwareEvidence + models: list[ModelEvidence] = Field(min_length=2, max_length=2) + bounds: BoundsEvidence + target: TargetEvidence + batching: BatchingEvidence + optimizer_updates: EvidenceValue + measurements: MeasurementsEvidence + resume: ResumeEvidence + artifacts: list[ArtifactEvidence] = Field(min_length=1) + scientific_scope: dict[str, bool] + consent_to_publish: bool = False + + @model_validator(mode="after") + def _trust_and_roles_are_closed(self) -> HardwareRecord: + if [model.role for model in self.models] != ["student", "teacher"]: + raise ValueError("models must contain student then teacher") + if ( + self.record_classification == "maintainer_measured" + and self.review_status != "maintainer_validated" + ): + raise ValueError("maintainer_measured requires maintainer_validated review status") + if ( + self.review_status == "maintainer_validated" + and self.record_classification != "maintainer_measured" + ): + raise ValueError("maintainer_validated records must be maintainer_measured") + if self.review_status == "maintainer_validated" and not self.consent_to_publish: + raise ValueError("maintainer validation for publication requires consent") + if self.record_classification == "not_measured": + raise ValueError("run-derived hardware records cannot be labelled not_measured") + return self + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _measured(value: float | int, unit: str, source: str) -> EvidenceValue: + return EvidenceValue(status="measured", value=value, unit=unit, source=source) + + +def _median(rows: list[dict[str, Any]], field: str) -> EvidenceValue: + values = [float(row[field]) for row in rows if row.get(field) is not None] + if not values: + return EvidenceValue(status="unknown", source="metrics.jsonl") + return _measured(round(float(statistics.median(values)), 4), "seconds", "metrics.jsonl") + + +def _artifact_records(run: Path) -> list[ArtifactEvidence]: + candidates = { + "checkpoint_adapter": run / "checkpoints/final/adapter.safetensors", + "checkpoint_optimizer": run / "checkpoints/final/optimizer.safetensors", + "checkpoint_state": run / "checkpoints/final/state.json", + "trajectories": run / "trajectories.jsonl", + "peft_adapter": run / "final-peft-adapter/adapter_model.safetensors", + } + return [ + ArtifactEvidence(name=name, sha256=_sha256(path), bytes=path.stat().st_size) + for name, path in candidates.items() + if path.is_file() + ] + + +def build_hardware_record( + run: str | Path, + *, + classification: Literal["maintainer_measured", "community_submitted"] = ("community_submitted"), + review_status: Literal["unreviewed", "maintainer_validated"] = "unreviewed", + consent_to_publish: bool = False, +) -> HardwareRecord: + """Derive a portable candidate record without importing torch or uploading it.""" + root = Path(run).resolve(strict=True) + manifest_path = root / "manifest.json" + config_path = root / "config.validated.yaml" + plan_path = root / "local-execution-plan.json" + manifest = read_json(manifest_path) + plan = read_json(plan_path) + if manifest.get("status") != "completed": + raise ValueError("hardware record requires a completed run manifest") + try: + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, yaml.YAMLError) as exc: + raise ValueError(f"cannot read validated run config: {exc}") from exc + if not isinstance(config, dict): + raise ValueError("validated run config must contain a mapping") + profile = manifest.get("profile_identity") or {} + gpu = manifest.get("gpu") or {} + models = manifest.get("models") or {} + objective = manifest.get("objective") or {} + source = config.get("source") or {} + rollout = config.get("rollout") or {} + train = config.get("train") or {} + loss_config = config.get("loss") or {} + plan_batching = (plan.get("system_plan") or {}).get("batching") or {} + cycle_rows = [ + row for row in read_jsonl(root / "metrics.jsonl") if row.get("phase") == "opd_cycle" + ] + update_rows = [row for row in read_jsonl(root / "metrics.jsonl") if row.get("phase") == "opd"] + memory_rows = [ + row.get("memory") or {} for row in [*cycle_rows, *update_rows] if row.get("memory") + ] + cuda_status = (manifest.get("measurement_status") or {}).get("cuda_metrics") + if cuda_status != "measured": + raise ValueError("hardware record requires measured CUDA metrics") + artifacts = _artifact_records(root) + if not artifacts: + raise ValueError("hardware record requires checksummed run artifacts") + checkpoint = manifest.get("final_checkpoint") or {} + resumed = manifest.get("resumed_from") is not None + driver = gpu.get("driver_version") + record = HardwareRecord( + created_at=utc_now(), + record_classification=classification, + review_status=review_status, + miniverl_version=str(manifest["miniverl_version"]), + profile=ProfileEvidence( + name=str(profile["profile_name"]), + identity_digest=str(profile["digest"]), + upstream_tag=str(profile["upstream_tag"]), + upstream_commit=str(profile["upstream_commit"]), + ), + run=RunEvidence( + run_id=str(manifest["run_id"]), + status="completed", + manifest_sha256=_sha256(manifest_path), + plan_digest=str(manifest["execution_plan_digest"]), + resolved_config_digest=str(manifest["resolved_config_digest"]), + ), + hardware=HardwareEvidence( + measurement_status="measured", + gpu_name=str(gpu["name"]), + gpu_count=int(gpu["device_count"]), + vram_gib=float(gpu["total_memory_gib"]), + driver=( + EvidenceValue(status="unknown", source="manifest.json") + if driver is None + else EvidenceValue(status="measured", value=str(driver), source="manifest.json") + ), + cuda_runtime=str(gpu["torch_cuda_version"]), + torch=str(gpu["torch_version"]), + os=str(manifest["platform"]), + python=str(manifest["python_version"]), + ), + models=[ + ModelEvidence( + role=role, + model_id=str(models[role]["model_id"]), + revision=str(models[role]["revision"]), + quantization=str(models[role]["quantization"]), + dtype=str(models[role]["capabilities"]["dtype"]), + ) + for role in ("student", "teacher") + ], + bounds=BoundsEvidence( + prompt_tokens=int(source["max_prompt_length"]), + response_tokens=int(source["max_response_length"]), + ), + target=TargetEvidence( + mode=str(objective["loss_mode"]), + top_k=( + None + if objective["loss_mode"] == "verl_pg_k1" + else int(objective["top_k"]) + if objective.get("top_k") is not None + else None + ), + estimator=loss_config.get("estimator_implementation_version"), + ), + batching=BatchingEvidence( + logical=int(train["rollouts_per_cycle"]), + rollout_physical=int(rollout["prompt_batch_size"]), + teacher_score_physical=int(plan_batching["teacher_score"]), + update_trajectory_physical=int(train["trajectory_batch_size"]), + ), + optimizer_updates=_measured( + int(manifest["actual_optimizer_updates"]), "updates", "manifest.json" + ), + measurements=MeasurementsEvidence( + peak_allocated_gib=_measured( + max(float(row.get("peak_allocated_gib") or 0.0) for row in memory_rows), + "GiB", + "metrics.jsonl", + ), + peak_reserved_gib=_measured( + max(float(row.get("peak_reserved_gib") or 0.0) for row in memory_rows), + "GiB", + "metrics.jsonl", + ), + phase_medians=TimingEvidence( + rollout_seconds=_median(cycle_rows, "rollout_seconds"), + teacher_scoring_seconds=_median(cycle_rows, "teacher_scoring_seconds"), + actor_update_seconds=_median(update_rows, "seconds"), + ), + ), + resume=ResumeEvidence( + status="measured" if resumed else "unknown", + outcome="resumed" if resumed else "not_exercised", + checkpoint_digest=(str(checkpoint.get("digest")) if resumed else None), + ), + artifacts=artifacts, + scientific_scope={ + "runtime_correctness_only": True, + "task_quality_evaluated": False, + "alignment_quality_evaluated": False, + "distributed_execution_tested": False, + }, + consent_to_publish=consent_to_publish, + ) + serialized = json.dumps(record.model_dump(mode="json"), ensure_ascii=False, sort_keys=True) + if portable_text(serialized) != serialized: + raise ValueError("generated hardware record contains private or non-portable text") + return record + + +def load_hardware_record(path: str | Path) -> HardwareRecord: + """Read and validate one standalone record.""" + return HardwareRecord.model_validate(read_json(path)) + + +def validate_hardware_record(path: str | Path) -> list[str]: + """Return validation problems; an empty list means schema-valid, not trusted.""" + try: + payload = read_json(path) + HardwareRecord.model_validate(payload) + except Exception as exc: + return [f"schema: {exc}"] + serialized = json.dumps(payload, ensure_ascii=False, sort_keys=True) + if portable_text(serialized) != serialized: + return ["privacy: record contains a local path, credential, or environment reference"] + return [] + + +def write_hardware_record(record: HardwareRecord, path: str | Path) -> Path: + """Publish one record atomically; no network operation is performed.""" + return write_json_atomic(path, record.model_dump(mode="json")) diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py index f715c20..7aa7ed5 100644 --- a/tests/cli/test_cli.py +++ b/tests/cli/test_cli.py @@ -44,6 +44,8 @@ "profiles schema", "compat explain", "compat check", + "hardware record", + "hardware validate", "data sample", "align", "alignment-suite prepare", diff --git a/tests/unit/test_hardware_records.py b/tests/unit/test_hardware_records.py new file mode 100644 index 0000000..7550130 --- /dev/null +++ b/tests/unit/test_hardware_records.py @@ -0,0 +1,230 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml +from typer.testing import CliRunner + +from miniverl.cli import app + +_DIGEST = "a" * 64 +_REVISION = "b" * 40 + + +def _run(tmp_path: Path) -> Path: + run = tmp_path / "run" + (run / "checkpoints/final").mkdir(parents=True) + (run / "final-peft-adapter").mkdir() + manifest = { + "status": "completed", + "run_id": "portable-run", + "miniverl_version": "0.10.0.dev0", + "execution_plan_digest": _DIGEST, + "resolved_config_digest": "c" * 64, + "actual_optimizer_updates": 8, + "resumed_from": None, + "final_checkpoint": {"digest": "d" * 64}, + "measurement_status": {"cuda_metrics": "measured"}, + "platform": "Linux-6.8", + "python_version": "3.12.4", + "profile_identity": { + "profile_name": "verl-opd-v0.8-single-gpu-v1", + "digest": "e" * 64, + "upstream_tag": "v0.8.0", + "upstream_commit": _REVISION, + }, + "gpu": { + "name": "NVIDIA GeForce RTX 4080", + "device_count": 1, + "total_memory_gib": 15.992, + "driver_version": None, + "torch_cuda_version": "13.0", + "torch_version": "2.13.0+cu130", + }, + "models": { + "student": { + "model_id": "org/student", + "revision": _REVISION, + "quantization": "nf4", + "capabilities": {"dtype": "bfloat16"}, + }, + "teacher": { + "model_id": "org/teacher", + "revision": "c" * 40, + "quantization": "nf4", + "capabilities": {"dtype": "bfloat16"}, + }, + }, + "objective": {"loss_mode": "forward_kl_topk", "top_k": 32}, + } + (run / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + config = { + "source": {"max_prompt_length": 128, "max_response_length": 64}, + "rollout": {"prompt_batch_size": 4}, + "train": {"rollouts_per_cycle": 4, "trajectory_batch_size": 1}, + "loss": {"estimator_implementation_version": None}, + } + (run / "config.validated.yaml").write_text(yaml.safe_dump(config), encoding="utf-8") + (run / "local-execution-plan.json").write_text( + json.dumps({"system_plan": {"batching": {"teacher_score": 2}}}), + encoding="utf-8", + ) + rows = [ + { + "phase": "opd_cycle", + "rollout_seconds": 2.0, + "teacher_scoring_seconds": 0.5, + "memory": {"peak_allocated_gib": 1.1, "peak_reserved_gib": 1.3}, + }, + { + "phase": "opd", + "seconds": 1.0, + "memory": {"peak_allocated_gib": 1.2, "peak_reserved_gib": 1.4}, + }, + ] + (run / "metrics.jsonl").write_text( + "".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8" + ) + for relative in ( + "checkpoints/final/adapter.safetensors", + "checkpoints/final/optimizer.safetensors", + "checkpoints/final/state.json", + "trajectories.jsonl", + "final-peft-adapter/adapter_model.safetensors", + ): + (run / relative).write_bytes(relative.encode("utf-8")) + return run + + +def test_record_extracts_strict_portable_measured_evidence(tmp_path: Path) -> None: + from miniverl.evidence.hardware import build_hardware_record + + record = build_hardware_record(_run(tmp_path)) + payload = record.model_dump(mode="json") + + assert payload["record_classification"] == "community_submitted" + assert payload["review_status"] == "unreviewed" + assert payload["hardware"]["driver"]["status"] == "unknown" + assert payload["batching"] == { + "logical": 4, + "rollout_physical": 4, + "teacher_score_physical": 2, + "update_trajectory_physical": 1, + } + assert payload["optimizer_updates"]["value"] == 8 + assert payload["measurements"]["peak_reserved_gib"]["value"] == 1.4 + assert payload["resume"] == { + "status": "unknown", + "outcome": "not_exercised", + "checkpoint_digest": None, + } + assert len(payload["artifacts"]) == 5 + assert str(tmp_path) not in json.dumps(payload) + assert payload["consent_to_publish"] is False + + +def test_validation_rejects_status_invention_and_private_paths(tmp_path: Path) -> None: + from miniverl.evidence.hardware import ( + build_hardware_record, + validate_hardware_record, + write_hardware_record, + ) + + record = build_hardware_record(_run(tmp_path)) + path = write_hardware_record(record, tmp_path / "record.json") + assert validate_hardware_record(path) == [] + + payload = json.loads(path.read_text(encoding="utf-8")) + payload["hardware"]["driver"]["value"] = "invented" + path.write_text(json.dumps(payload), encoding="utf-8") + assert "unknown evidence must not invent a value" in validate_hardware_record(path)[0] + + payload["hardware"]["driver"] = { + "status": "measured", + "value": "C:\\Users\\someone\\driver", + "unit": None, + "source": "manifest.json", + } + path.write_text(json.dumps(payload), encoding="utf-8") + assert validate_hardware_record(path)[0].startswith("privacy:") + + +def test_pg_record_reports_estimator_instead_of_internal_top_k_sentinel(tmp_path: Path) -> None: + from miniverl.evidence.hardware import build_hardware_record + + run = _run(tmp_path) + manifest_path = run / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["objective"]["loss_mode"] = "verl_pg_k1" + manifest["objective"]["top_k"] = 1 + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + config_path = run / "config.validated.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) + config["loss"]["estimator_implementation_version"] = "verl-v0.8-pg-k1-v1" + config_path.write_text(yaml.safe_dump(config), encoding="utf-8") + + target = build_hardware_record(run).target + assert target.mode == "verl_pg_k1" + assert target.top_k is None + assert target.estimator == "verl-v0.8-pg-k1-v1" + + +def test_hardware_cli_round_trip_is_unreviewed_and_does_not_upload(tmp_path: Path) -> None: + run = _run(tmp_path) + out = tmp_path / "hardware-record.json" + runner = CliRunner() + + recorded = runner.invoke(app, ["hardware", "record", "--run", str(run), "--out", str(out)]) + assert recorded.exit_code == 0, recorded.output + assert "nothing was uploaded" in recorded.output + validated = runner.invoke(app, ["hardware", "validate", str(out), "--json"]) + assert validated.exit_code == 0, validated.output + payload = json.loads(validated.stdout) + assert payload["valid"] is True + assert payload["review_status"] == "unreviewed" + + +def test_maintainer_review_cannot_be_self_asserted_without_consent(tmp_path: Path) -> None: + from miniverl.evidence.hardware import build_hardware_record + + record = build_hardware_record(_run(tmp_path)) + payload = record.model_dump(mode="json") + payload["record_classification"] = "maintainer_measured" + payload["review_status"] = "maintainer_validated" + path = tmp_path / "record.json" + path.write_text(json.dumps(payload), encoding="utf-8") + + from miniverl.evidence.hardware import validate_hardware_record + + assert "requires consent" in validate_hardware_record(path)[0] + + +def test_maintainer_measurement_requires_review_and_consent(tmp_path: Path) -> None: + from pydantic import ValidationError + + from miniverl.evidence.hardware import build_hardware_record + + run = _run(tmp_path) + with pytest.raises(ValidationError, match="requires maintainer_validated"): + build_hardware_record( + run, + classification="maintainer_measured", + consent_to_publish=True, + ) + + record = build_hardware_record( + run, + classification="maintainer_measured", + review_status="maintainer_validated", + consent_to_publish=True, + ) + assert record.review_status == "maintainer_validated" + + +def test_committed_hardware_schema_is_exactly_generated() -> None: + from scripts.publish_hardware_record_schema import render + + path = Path("docs/generated/hardware-record-v1.schema.json") + assert path.read_text(encoding="utf-8") == render()