From 64f668e6ff86c9a2f6d42dcd978c58aecf1945d4 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:05:17 -0700 Subject: [PATCH] Reframe documentation around product workflows --- CHANGELOG.md | 12 + CONTRIBUTING.md | 25 +- PYPI.md | 351 +++++++----------- README.md | 351 +++++++----------- README.zh-CN.md | 270 ++++++-------- TODO.md | 4 +- .../benchmark-governance.md | 5 +- docs/alignment-lab/demo.md | 7 +- docs/alignment-lab/launch-copy.md | 4 +- docs/banner.svg | 2 +- docs/comparisons.md | 161 ++++---- docs/compatibility.md | 24 +- docs/config-overrides.md | 6 +- docs/consumer-runtime-v1.md | 5 +- docs/design.md | 7 +- docs/for-verl-users.md | 54 ++- docs/index.md | 153 ++++---- docs/limitations.md | 22 +- docs/maintainer-architecture.md | 3 +- docs/opd-quickstart.md | 36 +- docs/profiles/index.md | 28 +- docs/repository-settings.md | 4 +- docs/scaleout-materialization.md | 8 +- docs/security-review-v0.10.1.md | 4 +- docs/single-gpu-guide.md | 57 ++- docs/social-preview-v0.6.svg | 2 +- docs/upstream-support-policy.md | 5 +- docs/v1-readiness.md | 6 +- docs/verl-bridge-demo.md | 4 +- docs/verl-local-runtime-mobile.svg | 6 +- docs/verl-local-runtime.svg | 10 +- docs/verl-opd-runtime.md | 28 +- docs/verl-opd-scaleout.md | 15 +- mkdocs.yml | 5 +- pyproject.toml | 2 +- tests/unit/test_documentation_voice.py | 56 +++ 36 files changed, 793 insertions(+), 949 deletions(-) create mode 100644 tests/unit/test_documentation_voice.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9481bdc..3529339 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ All notable changes to miniVERL are recorded here. The format follows ## [Unreleased] +### Documentation narrative + +- Rebuilt the English, Chinese and PyPI landing narratives around the local + workflow, concrete artifacts, user paths and measured systems evidence. +- Reframed the documentation home, stack-selection guide, verl migration, + hardware planning and OPD guides around supported workflows, with complete + execution and scientific boundaries consolidated in compatibility and + limitations. +- Updated the main runtime diagram and contributor guidance so product pages + lead with capabilities while research, security and operational caveats stay + next to the decisions they qualify. + ### Release evidence polish - Corrected the v1-readiness record after the completed v0.10.1 attempt-1 RTX diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5f97ef6..42681af 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,15 +1,14 @@ # Contributing to miniVERL -Thank you for considering it. This document is short and specific, because the -project has a small number of rules that matter a lot and very few that do not. +Thank you for helping make one-GPU distillation easier to inspect, reproduce +and extend. This guide covers the checks that keep code and published evidence +aligned. -## The one rule +## Measurement provenance -**Never state a number you did not measure.** Not in code, not in a docstring, -not in the README, not in a pull-request description. If something was not run, -write "not run" and leave the exact command that would run it. A missing number -is fine; an invented one is not, and it is the fastest way to make a -distillation project worthless. +**Every reported number needs a measurement artifact or command.** When a check +was unavailable, write "not run" and include the exact command for a future +run. This convention keeps reviews useful across different hardware. ## Setup @@ -77,6 +76,16 @@ required hardware or credentials are unavailable. * Silent behaviour: truncating a sequence, changing a model, reusing a stale cache, or swallowing an exception. Fail loudly with a hint instead. +## Documentation voice + +Lead with what the workflow does, who it helps and which artifact it produces. +State a local constraint where it changes the reader's next action; link to +the [compatibility policy](docs/compatibility.md) or +[limitations](docs/limitations.md) for complete exclusion lists and scientific +boundaries. Avoid repeating a catalogue of absent features after every product +capability. Research reports and security contracts should keep their precise +caveats close to the evidence or decision they qualify. + ## Adding an environment See the "New environment" issue template for the checklist. In short: seeded and diff --git a/PYPI.md b/PYPI.md index 8c4700f..1c0441a 100644 --- a/PYPI.md +++ b/PYPI.md @@ -19,19 +19,17 @@ 中文

-**Run a documented subset of verl-style on-policy distillation on one consumer -GPU.** miniVERL takes typed verl-shaped YAML and Parquet prompts through actor -rollout → teacher scoring → actor update, records every local reinterpretation, -and exports standard PEFT, Parquet and config artifacts for a pinned scale-out -handoff. +**Run verl-style on-policy distillation on one NVIDIA GPU, with every config +mapping, teacher target and training artifact available for inspection.** +miniVERL turns typed YAML and structured Parquet prompts into a local actor +rollout → teacher scoring → actor update loop, then exports standard PEFT, +Parquet and config artifacts for scale-out work. -PyPI `v0.10.1` is stable; `main` is development. miniVERL is an independent -project with no upstream endorsement. It does not execute arbitrary verl YAML, -launch distributed jobs, or claim full algorithmic compatibility. +PyPI `v0.10.1` is stable; `main` is development. -## Pip-only quickstart +## Start in 60 seconds -Install the matching CUDA-enabled PyTorch build for your machine first, then: +Install the CUDA-enabled PyTorch build that matches your machine, then: ```bash python -m pip install "miniverl[train,cuda]" @@ -43,13 +41,29 @@ miniverl run --profile verl-opd-v0.8-single-gpu-v1 \ --plan plan.json --dry-run ``` -These commands need no Git checkout. `data sample` creates a real structured -Parquet file, `plan` compiles the complete field matrix without loading weights, -and `run --dry-run` validates the native execution contract. On one NVIDIA CUDA -GPU, remove `--dry-run` to run the pinned Qwen3-0.6B actor and Qwen3-1.7B teacher -recipe and produce an inspectable PEFT adapter. - -After reviewing the plan, the canonical continuation is: +This path works from the published wheel and loads no model weights while +planning. The generated `plan.json` binds the source config, ordered overrides, +profile version and input Parquet bytes. On a CUDA GPU, remove `--dry-run` to +run the pinned Qwen3-0.6B actor and Qwen3-1.7B teacher recipe. + +The `[train,cuda]` extra installs the ML and quantization dependencies. Select +the matching CUDA PyTorch wheel separately with the +[PyTorch installer](https://pytorch.org/get-started/locally/). The +[single-GPU guide](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md) covers memory planning from 8 GiB +cards upward and includes the maintainer-measured RTX 4080 stack. + +## What a run gives you + +- **A reviewable plan.** Every accepted verl field has a local effect, + classification and risk level before weights are loaded. +- **Strict current-policy trajectories.** The actor policy version, token + spans and teacher-supervised positions travel together. +- **Compact teacher targets.** Top-k targets and sampled-k1 signals use + checksummed, pickle-free caches. +- **Recoverable training.** Transactional manifests, checkpoints and cache + indexes support inspection and exact resume. +- **Standard outputs.** PEFT adapters, safetensors, structured Parquet, + resolved config and typed provenance remain portable. ```bash miniverl run --profile verl-opd-v0.8-single-gpu-v1 --plan plan.json \ @@ -60,213 +74,117 @@ miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout miniverl bridge doctor scaleout --json ``` -The `train` extra installs the ML runtime, but does not choose the correct CUDA -PyTorch wheel. The optional `cuda` extra adds bitsandbytes only. Follow the -[one-GPU installation and memory guide](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/single-gpu-guide.md) for both the -flexible install and the machine-readable RTX 4080 known-good stack. Hardware -other than that one measured RTX 4080 remains unmeasured. - -## Architecture +## How it works - verl-shaped YAML, overrides and Parquet prompts pass through a typed compiler; one CUDA GPU runs actor rollout, teacher scoring and actor update; inspectable artifacts can be handed to pinned verl while distributed execution remains outside miniVERL. + Typed verl-style configuration and Parquet prompts compile into sequential actor rollout, teacher scoring and actor update phases on one CUDA GPU, producing inspectable local artifacts and a pinned scale-out bundle. -miniVERL uses one ordinary process and schedules model roles in phases. It does -not emulate Ray resource pools or pretend local Hugging Face generation is -vLLM. Instead, the compatibility report retains the source value, explains the -local meaning, assigns a risk level, and fails closed when a field would change -the algorithm or distributed semantics. - -The native runtime also remains available for SFT, DPO, offline KD and -tool-aware OPD recipes. Those workflows share strict token provenance, -pickle-free teacher caches, transactional checkpoints and adapter export, but -the verl-shaped profile is the shortest route for an existing verl user. - -### One local scheduler, explicit roles - -The actor generates from the current adapter; the teacher scores exactly those -visited token positions; the actor then receives a padded, token-mean update. -The teacher is never treated as a reward model, tool output never becomes a -training label, and a stale actor-policy version cannot enter an on-policy -batch. Memory planning chooses resident phased roles for quantized models, -swap only for movable unquantized roles, or compatible shared-backbone -placement while keeping actor, teacher and reference identities distinct. - -Each phase writes evidence before the next boundary: structured trajectories -carry per-token provenance, top-k teacher targets are checksummed without -pickle, checkpoints publish transactionally, and the final adapter is verified -with the standard PEFT loader. A crash can therefore be inspected and resumed -without reconstructing intent from console text. - -## Coming from verl? - -| What you do in verl | miniVERL equivalent | -| --- | --- | -| pass Hydra-style overrides | repeat `--set`; v0.9 development also accepts `--overrides-file` and tokens after `--` | -| inspect the resolved config | `miniverl plan --json` | -| execute pure OPD | `miniverl run` | -| reuse prompt Parquet | point `data.train_files` at it directly | -| allocate rollout/teacher workers | compile resource intent into local phases | -| save an FSDP/Megatron checkpoint | unsupported | -| prepare a scale-out handoff | `miniverl export-verl` | - -The same field names stay visible: - -```bash -# familiar resolved intent -actor_rollout_ref.model.path=Qwen/Qwen3-0.6B -distillation.teacher_models.teacher_model.model_path=Qwen/Qwen3-1.7B - -# bounded local compilation -miniverl plan --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml \ - --set actor_rollout_ref.actor.optim.lr=1e-5 -``` - -External YAML must explicitly accept the high-risk local mappings printed by -`plan` before `run`; the packaged profile carries a value-bound reviewed -manifest. [Override precedence and safe input forms](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/config-overrides.md) -are documented without executing Hydra interpolation or shell text. +miniVERL schedules actor, teacher and optional reference roles in phases inside +one ordinary process. The actor generates with its current adapter, the teacher +scores the visited token positions, and the actor receives a padded token-mean +update. Resident, swap and shared-backbone placement strategies keep those role +identities explicit while adapting to available memory. -The public built-in profile deliberately uses upstream-shaped `name: vllm` -values. miniVERL classifies both rollout and teacher engine names as local -reinterpretations and executes them with sequential local HF phases; this is -not vLLM equivalence. See [For verl users](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/for-verl-users.md) for config, -data, role and error mappings. +Each phase publishes evidence before the next boundary: structured +trajectories carry per-token provenance, teacher targets are checksummed, +checkpoints publish transactionally, and the final adapter is reloaded through +standard PEFT. The result is a training run you can inspect without +reconstructing intent from console logs. -## Tested profile boundary +## Choose your path -Use `miniverl profiles list`, `profiles show/schema`, and `compat -explain/check` to inspect the closed built-in registry and distinguish an -accepted field from one with a demonstrated native effect. New plans, caches, -checkpoints and exports bind the complete independent profile identity. - -Both profiles pin official verl `v0.8.0` at commit -`7aed6b230776f963fa09509c10d9c3a767d1102c`: - -| Profile | Objective | Teacher target | Status | +| Goal | First command | Primary artifact | Next step | | --- | --- | --- | --- | -| direct GKD | `forward_kl_topk` | top-k IDs/log-probabilities | measured | -| PG OPD | sampled `k1` + vanilla policy loss | sampled-token teacher log-probability | measured | - -Their shared executable boundary is intentionally narrow: - -- one trainable actor and one teacher; -- one generated response per prompt (`n=1`); -- reward-free distillation with token-mean aggregation; -- LoRA or QLoRA adapter updates on one CUDA GPU; -- immutable model revisions and verl-style structured prompt Parquet. - -The PG path is not PPO: it uses a detached distillation-derived advantage with -the pinned vanilla policy-loss form. Task rewards, KL penalties, multi-teacher routing, -multimodal inputs, PPO, GRPO, critics, Ray, FSDP, Megatron, multi-GPU and -multi-node execution are unsupported. Known unsupported values receive a -machine-readable classification; unknown fields and unresolved `${...}` -interpolations are rejected. A resolved profile subset is input—not an -arbitrary launch script. - -## Measured RTX 4080 path - -The Qwen3-0.6B/1.7B developer workload consumed **32 distinct prompts**, each -with a 64-token response bound, and completed **8 current-policy updates** at -**3.1914 GiB peak reserved VRAM**. Median steady-state rollout, teacher-scoring -and update times were 9.7200, 0.4864 and 2.3260 seconds. A matched 4-update -interruption resumed to the same byte-identical trajectories, adapter and -optimizer tensors. See the [data-bound figure and full record](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-opd-reference-workload.md); -the original one-update [pip smoke](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/opd-quickstart.md) remains preserved. -A separate pinned SmolLM2-360M/1.7B recipe consumed 32 distinct prompts across -8 updates at 1.4961 GiB peak reserved VRAM. Interruption/resume was -byte-identical, PEFT reload passed, and the exact-snapshot scale-out bundle -materialized successfully. See the [full SmolLM2 systems record](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/smollm2-opd-workload.md). - -This is deliberately a runtime and artifact proof. It is not a throughput -benchmark, an alignment-quality endpoint, or evidence that OPD beats SFT, DPO -or KD. Other NVIDIA GPUs use the same device-name-agnostic CUDA path, but model -fit depends on VRAM, context length, quantization and installed kernels; they -remain unmeasured. The [release qualification contract](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/release-qualification.md) -builds one candidate wheel on a hosted runner and qualifies those exact bytes -on the RTX 4080 before any release can reuse them, while the -[upstream support policy](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/upstream-support-policy.md) keeps both published -profile identities fixed. Runner activation is not described as continuous CI. - -### Choose a path by hardware, not GPU branding - -| Situation | Recommended starting point | What remains constant | -| --- | --- | --- | -| inspect on CPU or a laptop | `plan` and `run --dry-run` | full config classification | -| one CUDA GPU with limited VRAM | QLoRA with resident local phases, or unquantized LoRA with swap | logical batch and loss semantics | -| same-base actor and teacher adapters | shared-backbone mode | explicit role provenance | -| more VRAM available | larger physical phase batches | source data and optimizer intent | - -Automatic BF16/FP16 selection follows device support; it is not inferred from -marketing names such as 3070, 4080, 5090 or Titan. `miniverl doctor` reports the -installed CUDA/PyTorch path. Normal planning is weight-free; explicit -`plan --probe` adds bounded, cached CUDA measurements with zero optimizer -updates. See [hardware planning](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/hardware-planning.md). There is no -automatic downgrade to a different model, -teacher, context, top-k or loss when memory is tight. - -Share a sanitized result without uploading it automatically: - -```bash -miniverl hardware record --run runs/my-opd --out hardware-record.json -miniverl hardware validate hardware-record.json +| **Run local OPD** | `miniverl plan --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml` | immutable execution plan | [OPD quickstart](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/opd-quickstart.md) | +| **Bring a verl profile** | `miniverl compat check --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml` | field-by-field compatibility report | [For verl users](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/for-verl-users.md) | +| **Fit your GPU** | `miniverl plan --config verl-opd.yaml --probe` | measured placement plan | [Hardware planning](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/hardware-planning.md) | +| **Hand off for scale-out** | `miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout` | PEFT + Parquet + config bundle | [Scale-out contract](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-opd-scaleout.md) | + +The native recipe system also supports SFT, DPO, offline KD and tool-aware OPD +over calculator, JSON navigation, read-only SQLite and custom environments. + +## Familiar verl inputs, local execution + +The current profiles pin official verl `v0.8.0` at +`7aed6b230776f963fa09509c10d9c3a767d1102c` and preserve recognizable fields: + +```yaml +actor_rollout_ref: + model: {path: Qwen/Qwen3-0.6B} + rollout: {name: vllm, n: 1} +distillation: + teacher_models: + teacher_model: + model_path: Qwen/Qwen3-1.7B + inference: {name: vllm} ``` -Community records remain unreviewed until their hashes and provenance pass the -[documented maintainer gate](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/community-benchmarks.md). - -## Data and artifact interoperability - -The profile consumes structured verl-style Parquet without substituting a toy -environment. Prompt roles and content remain structured; data source, ability -and extra metadata survive conversion. `miniverl convert-dataset` is available -when crossing the native trajectory boundary, and rejects lossy rows unless the -operator explicitly permits a partial conversion. - -A completed local run contains the resolved source config, compatibility -matrix, local execution plan, trajectories, selected teacher targets, -checkpoints, measurements and a PEFT adapter. Inspect before moving it: - -```bash -miniverl inspect runs/my-opd/trajectories.jsonl -miniverl cache stats runs/my-opd/teacher-cache -miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout -miniverl bridge materialize scaleout --download --offline -miniverl bridge doctor scaleout --json -``` +The compiler translates distributed resource intent into sequential local +Hugging Face phases and records that translation in the plan. Two measured +profiles are available: -The v0.9 export preserves student/teacher identities, Parquet bytes and pure -OPD overrides, but reports `launchable: false` until exact base snapshots are -materialized and validated against the installed pinned verl commit. Only then -does `bridge materialize` publish a checksummed `launch.sh`; distributed -execution remains untested. Review the [current scale-out contract](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-opd-scaleout.md), -[legacy bridge](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/legacy-verl-bridge.md) and [compatibility policy](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/compatibility.md). - -The intended operating loop is **plan → inspect → run → inspect → export**. -`plan --out` byte-binds the YAML, ordered overrides and scanned Parquet inputs -to the exact native config; `run --plan` rejects drift before loading weights. -Its digest follows the run manifest, teacher cache and checkpoints. Direct -`run --config` remains available for experiments. See [immutable execution -plans](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/immutable-plans.md). - -## Research and validation - -miniVERL keeps every measured study—including negative results, superseded -runs and preregistered early stops—public under the documentation. None is used -as a claim that OPD universally beats SFT, DPO or KD: see the -[v0.7 External Alignment Gate](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-external/alignment-external-v1.md), -[Alignment Lab](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md), -[RecoveryBench](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/recoverybench/recoverybench-v1.md), and the -[calculator study](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/benchmarking.md). - -New runs establish tokenizer compatibility through structural identity. The -legacy behavioral fingerprint is retained only for migration and is not an -identity proof. Scientific caveats and immutable source hashes remain in the -detailed reports and [limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/limitations.md). - -## Development, security and license +| Profile | Objective | Teacher target | +| --- | --- | --- | +| `verl-opd-v0.8-single-gpu-v1` | direct GKD `forward_kl_topk` | top-k token IDs and log-probabilities | +| `verl-opd-v0.8-single-gpu-pg-k1-v1` | sampled `k1` + vanilla policy loss | sampled-token teacher log-probability | + +Use `miniverl profiles show`, `compat explain` and `compat check` to inspect +the complete mapping. [Compatibility profiles](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/profiles/index.md) explains +how profile identity follows plans, caches, checkpoints and exports. + +## Measured systems evidence + +The Qwen3-0.6B/1.7B developer workload consumed **32 distinct prompts**, used a +64-token response bound, and completed **8 current-policy updates** at +**3.1914 GiB peak reserved VRAM** on an RTX 4080. Median steady-state rollout, +teacher-scoring and update times were 9.7200, 0.4864 and 2.3260 seconds. A +matched interruption after update four resumed to byte-identical trajectories, +adapter and optimizer tensors. + +A second SmolLM2-360M/1.7B workload completed the same 32-prompt, 8-update +shape at 1.4961 GiB peak reserved VRAM, including exact resume, PEFT reload and +scale-out materialization. Read the +[Qwen3](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/verl-opd-reference-workload.md) and +[SmolLM2](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/smollm2-opd-workload.md) systems records for configs, hashes and +phase-level measurements. + +Other NVIDIA GPUs use the same device-name-agnostic CUDA path. Model fit and +speed vary with VRAM, context, quantization, kernels and software versions; +`miniverl doctor` and `plan --probe` expose those machine-specific choices. + +## Research record + +The repository publishes positive, mixed and negative results with the same +resolved configs and source hashes. The calculator study found that a +protocol-qualified teacher prevented collapse but tied supervised +continuation. RecoveryBench found no fresh-state advantage in its scoped +SQLite setting. Alignment Lab began at a saturated SFT checkpoint and exposed +utility regressions that two sandbox safety checks missed. The preregistered +External Alignment Gate stopped before continuation because every candidate +failed the retained-utility threshold. + +- [Calculator protocol study](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/benchmarking.md) +- [RecoveryBench](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/recoverybench/recoverybench-v1.md) +- [Alignment Lab](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-lab/alignment-lab-v1.md) +- [External Alignment Gate](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/alignment-external/alignment-external-v1.md) + +These reports answer scoped experimental questions; the +[limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/limitations.md) page collects the measurement, architecture, +security and generalization boundaries in one place. + +## Scope + +miniVERL is designed for one local process, one NVIDIA CUDA GPU and the +documented OPD profiles above. Scale-out support produces and validates a +portable bundle against the pinned upstream source. The +[compatibility policy](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/compatibility.md) lists supported fields, profile +semantics and handoff readiness states; [limitations](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/limitations.md) +covers the broader execution and scientific boundaries. miniVERL is an +independent Apache-2.0 project. + +## Development ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git @@ -275,8 +193,7 @@ python -m pip install -e ".[dev]" pytest -q -m "not gpu and not network" ``` -Contributions should keep the one-GPU boundary explicit and include tests for -new failure modes. Report vulnerabilities privately through -[SECURITY.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/SECURITY.md). See [CONTRIBUTING.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CONTRIBUTING.md), the -[changelog](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CHANGELOG.md), [citation metadata](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CITATION.cff), -[reproducibility guide](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/reproducibility.md), and [Apache-2.0 license](https://github.com/DaoyuanLi2816/mini-verl/blob/main/LICENSE). +See [CONTRIBUTING.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CONTRIBUTING.md), [CHANGELOG.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CHANGELOG.md), +[CITATION.cff](https://github.com/DaoyuanLi2816/mini-verl/blob/main/CITATION.cff), the +[reproducibility guide](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/reproducibility.md), [SECURITY.md](https://github.com/DaoyuanLi2816/mini-verl/blob/main/SECURITY.md) +and the [Apache-2.0 license](https://github.com/DaoyuanLi2816/mini-verl/blob/main/LICENSE). diff --git a/README.md b/README.md index e1e3869..51b9f82 100644 --- a/README.md +++ b/README.md @@ -19,19 +19,17 @@ 中文

-**Run a documented subset of verl-style on-policy distillation on one consumer -GPU.** miniVERL takes typed verl-shaped YAML and Parquet prompts through actor -rollout → teacher scoring → actor update, records every local reinterpretation, -and exports standard PEFT, Parquet and config artifacts for a pinned scale-out -handoff. +**Run verl-style on-policy distillation on one NVIDIA GPU, with every config +mapping, teacher target and training artifact available for inspection.** +miniVERL turns typed YAML and structured Parquet prompts into a local actor +rollout → teacher scoring → actor update loop, then exports standard PEFT, +Parquet and config artifacts for scale-out work. -PyPI `v0.10.1` is stable; `main` is development. miniVERL is an independent -project with no upstream endorsement. It does not execute arbitrary verl YAML, -launch distributed jobs, or claim full algorithmic compatibility. +PyPI `v0.10.1` is stable; `main` is development. -## Pip-only quickstart +## Start in 60 seconds -Install the matching CUDA-enabled PyTorch build for your machine first, then: +Install the CUDA-enabled PyTorch build that matches your machine, then: ```bash python -m pip install "miniverl[train,cuda]" @@ -43,13 +41,29 @@ miniverl run --profile verl-opd-v0.8-single-gpu-v1 \ --plan plan.json --dry-run ``` -These commands need no Git checkout. `data sample` creates a real structured -Parquet file, `plan` compiles the complete field matrix without loading weights, -and `run --dry-run` validates the native execution contract. On one NVIDIA CUDA -GPU, remove `--dry-run` to run the pinned Qwen3-0.6B actor and Qwen3-1.7B teacher -recipe and produce an inspectable PEFT adapter. - -After reviewing the plan, the canonical continuation is: +This path works from the published wheel and loads no model weights while +planning. The generated `plan.json` binds the source config, ordered overrides, +profile version and input Parquet bytes. On a CUDA GPU, remove `--dry-run` to +run the pinned Qwen3-0.6B actor and Qwen3-1.7B teacher recipe. + +The `[train,cuda]` extra installs the ML and quantization dependencies. Select +the matching CUDA PyTorch wheel separately with the +[PyTorch installer](https://pytorch.org/get-started/locally/). The +[single-GPU guide](docs/single-gpu-guide.md) covers memory planning from 8 GiB +cards upward and includes the maintainer-measured RTX 4080 stack. + +## What a run gives you + +- **A reviewable plan.** Every accepted verl field has a local effect, + classification and risk level before weights are loaded. +- **Strict current-policy trajectories.** The actor policy version, token + spans and teacher-supervised positions travel together. +- **Compact teacher targets.** Top-k targets and sampled-k1 signals use + checksummed, pickle-free caches. +- **Recoverable training.** Transactional manifests, checkpoints and cache + indexes support inspection and exact resume. +- **Standard outputs.** PEFT adapters, safetensors, structured Parquet, + resolved config and typed provenance remain portable. ```bash miniverl run --profile verl-opd-v0.8-single-gpu-v1 --plan plan.json \ @@ -60,213 +74,117 @@ miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout miniverl bridge doctor scaleout --json ``` -The `train` extra installs the ML runtime, but does not choose the correct CUDA -PyTorch wheel. The optional `cuda` extra adds bitsandbytes only. Follow the -[one-GPU installation and memory guide](docs/single-gpu-guide.md) for both the -flexible install and the machine-readable RTX 4080 known-good stack. Hardware -other than that one measured RTX 4080 remains unmeasured. - -## Architecture +## How it works - verl-shaped YAML, overrides and Parquet prompts pass through a typed compiler; one CUDA GPU runs actor rollout, teacher scoring and actor update; inspectable artifacts can be handed to pinned verl while distributed execution remains outside miniVERL. + Typed verl-style configuration and Parquet prompts compile into sequential actor rollout, teacher scoring and actor update phases on one CUDA GPU, producing inspectable local artifacts and a pinned scale-out bundle. -miniVERL uses one ordinary process and schedules model roles in phases. It does -not emulate Ray resource pools or pretend local Hugging Face generation is -vLLM. Instead, the compatibility report retains the source value, explains the -local meaning, assigns a risk level, and fails closed when a field would change -the algorithm or distributed semantics. - -The native runtime also remains available for SFT, DPO, offline KD and -tool-aware OPD recipes. Those workflows share strict token provenance, -pickle-free teacher caches, transactional checkpoints and adapter export, but -the verl-shaped profile is the shortest route for an existing verl user. - -### One local scheduler, explicit roles - -The actor generates from the current adapter; the teacher scores exactly those -visited token positions; the actor then receives a padded, token-mean update. -The teacher is never treated as a reward model, tool output never becomes a -training label, and a stale actor-policy version cannot enter an on-policy -batch. Memory planning chooses resident phased roles for quantized models, -swap only for movable unquantized roles, or compatible shared-backbone -placement while keeping actor, teacher and reference identities distinct. - -Each phase writes evidence before the next boundary: structured trajectories -carry per-token provenance, top-k teacher targets are checksummed without -pickle, checkpoints publish transactionally, and the final adapter is verified -with the standard PEFT loader. A crash can therefore be inspected and resumed -without reconstructing intent from console text. - -## Coming from verl? - -| What you do in verl | miniVERL equivalent | -| --- | --- | -| pass Hydra-style overrides | repeat `--set`; v0.9 development also accepts `--overrides-file` and tokens after `--` | -| inspect the resolved config | `miniverl plan --json` | -| execute pure OPD | `miniverl run` | -| reuse prompt Parquet | point `data.train_files` at it directly | -| allocate rollout/teacher workers | compile resource intent into local phases | -| save an FSDP/Megatron checkpoint | unsupported | -| prepare a scale-out handoff | `miniverl export-verl` | - -The same field names stay visible: - -```bash -# familiar resolved intent -actor_rollout_ref.model.path=Qwen/Qwen3-0.6B -distillation.teacher_models.teacher_model.model_path=Qwen/Qwen3-1.7B - -# bounded local compilation -miniverl plan --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml \ - --set actor_rollout_ref.actor.optim.lr=1e-5 -``` - -External YAML must explicitly accept the high-risk local mappings printed by -`plan` before `run`; the packaged profile carries a value-bound reviewed -manifest. [Override precedence and safe input forms](docs/config-overrides.md) -are documented without executing Hydra interpolation or shell text. +miniVERL schedules actor, teacher and optional reference roles in phases inside +one ordinary process. The actor generates with its current adapter, the teacher +scores the visited token positions, and the actor receives a padded token-mean +update. Resident, swap and shared-backbone placement strategies keep those role +identities explicit while adapting to available memory. -The public built-in profile deliberately uses upstream-shaped `name: vllm` -values. miniVERL classifies both rollout and teacher engine names as local -reinterpretations and executes them with sequential local HF phases; this is -not vLLM equivalence. See [For verl users](docs/for-verl-users.md) for config, -data, role and error mappings. +Each phase publishes evidence before the next boundary: structured +trajectories carry per-token provenance, teacher targets are checksummed, +checkpoints publish transactionally, and the final adapter is reloaded through +standard PEFT. The result is a training run you can inspect without +reconstructing intent from console logs. -## Tested profile boundary +## Choose your path -Use `miniverl profiles list`, `profiles show/schema`, and `compat -explain/check` to inspect the closed built-in registry and distinguish an -accepted field from one with a demonstrated native effect. New plans, caches, -checkpoints and exports bind the complete independent profile identity. - -Both profiles pin official verl `v0.8.0` at commit -`7aed6b230776f963fa09509c10d9c3a767d1102c`: - -| Profile | Objective | Teacher target | Status | +| Goal | First command | Primary artifact | Next step | | --- | --- | --- | --- | -| direct GKD | `forward_kl_topk` | top-k IDs/log-probabilities | measured | -| PG OPD | sampled `k1` + vanilla policy loss | sampled-token teacher log-probability | measured | - -Their shared executable boundary is intentionally narrow: - -- one trainable actor and one teacher; -- one generated response per prompt (`n=1`); -- reward-free distillation with token-mean aggregation; -- LoRA or QLoRA adapter updates on one CUDA GPU; -- immutable model revisions and verl-style structured prompt Parquet. - -The PG path is not PPO: it uses a detached distillation-derived advantage with -the pinned vanilla policy-loss form. Task rewards, KL penalties, multi-teacher routing, -multimodal inputs, PPO, GRPO, critics, Ray, FSDP, Megatron, multi-GPU and -multi-node execution are unsupported. Known unsupported values receive a -machine-readable classification; unknown fields and unresolved `${...}` -interpolations are rejected. A resolved profile subset is input—not an -arbitrary launch script. - -## Measured RTX 4080 path - -The Qwen3-0.6B/1.7B developer workload consumed **32 distinct prompts**, each -with a 64-token response bound, and completed **8 current-policy updates** at -**3.1914 GiB peak reserved VRAM**. Median steady-state rollout, teacher-scoring -and update times were 9.7200, 0.4864 and 2.3260 seconds. A matched 4-update -interruption resumed to the same byte-identical trajectories, adapter and -optimizer tensors. See the [data-bound figure and full record](docs/verl-opd-reference-workload.md); -the original one-update [pip smoke](docs/opd-quickstart.md) remains preserved. -A separate pinned SmolLM2-360M/1.7B recipe consumed 32 distinct prompts across -8 updates at 1.4961 GiB peak reserved VRAM. Interruption/resume was -byte-identical, PEFT reload passed, and the exact-snapshot scale-out bundle -materialized successfully. See the [full SmolLM2 systems record](docs/smollm2-opd-workload.md). - -This is deliberately a runtime and artifact proof. It is not a throughput -benchmark, an alignment-quality endpoint, or evidence that OPD beats SFT, DPO -or KD. Other NVIDIA GPUs use the same device-name-agnostic CUDA path, but model -fit depends on VRAM, context length, quantization and installed kernels; they -remain unmeasured. The [release qualification contract](docs/release-qualification.md) -builds one candidate wheel on a hosted runner and qualifies those exact bytes -on the RTX 4080 before any release can reuse them, while the -[upstream support policy](docs/upstream-support-policy.md) keeps both published -profile identities fixed. Runner activation is not described as continuous CI. - -### Choose a path by hardware, not GPU branding - -| Situation | Recommended starting point | What remains constant | -| --- | --- | --- | -| inspect on CPU or a laptop | `plan` and `run --dry-run` | full config classification | -| one CUDA GPU with limited VRAM | QLoRA with resident local phases, or unquantized LoRA with swap | logical batch and loss semantics | -| same-base actor and teacher adapters | shared-backbone mode | explicit role provenance | -| more VRAM available | larger physical phase batches | source data and optimizer intent | - -Automatic BF16/FP16 selection follows device support; it is not inferred from -marketing names such as 3070, 4080, 5090 or Titan. `miniverl doctor` reports the -installed CUDA/PyTorch path. Normal planning is weight-free; explicit -`plan --probe` adds bounded, cached CUDA measurements with zero optimizer -updates. See [hardware planning](docs/hardware-planning.md). There is no -automatic downgrade to a different model, -teacher, context, top-k or loss when memory is tight. - -Share a sanitized result without uploading it automatically: - -```bash -miniverl hardware record --run runs/my-opd --out hardware-record.json -miniverl hardware validate hardware-record.json +| **Run local OPD** | `miniverl plan --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml` | immutable execution plan | [OPD quickstart](docs/opd-quickstart.md) | +| **Bring a verl profile** | `miniverl compat check --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml` | field-by-field compatibility report | [For verl users](docs/for-verl-users.md) | +| **Fit your GPU** | `miniverl plan --config verl-opd.yaml --probe` | measured placement plan | [Hardware planning](docs/hardware-planning.md) | +| **Hand off for scale-out** | `miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout` | PEFT + Parquet + config bundle | [Scale-out contract](docs/verl-opd-scaleout.md) | + +The native recipe system also supports SFT, DPO, offline KD and tool-aware OPD +over calculator, JSON navigation, read-only SQLite and custom environments. + +## Familiar verl inputs, local execution + +The current profiles pin official verl `v0.8.0` at +`7aed6b230776f963fa09509c10d9c3a767d1102c` and preserve recognizable fields: + +```yaml +actor_rollout_ref: + model: {path: Qwen/Qwen3-0.6B} + rollout: {name: vllm, n: 1} +distillation: + teacher_models: + teacher_model: + model_path: Qwen/Qwen3-1.7B + inference: {name: vllm} ``` -Community records remain unreviewed until their hashes and provenance pass the -[documented maintainer gate](docs/community-benchmarks.md). - -## Data and artifact interoperability - -The profile consumes structured verl-style Parquet without substituting a toy -environment. Prompt roles and content remain structured; data source, ability -and extra metadata survive conversion. `miniverl convert-dataset` is available -when crossing the native trajectory boundary, and rejects lossy rows unless the -operator explicitly permits a partial conversion. - -A completed local run contains the resolved source config, compatibility -matrix, local execution plan, trajectories, selected teacher targets, -checkpoints, measurements and a PEFT adapter. Inspect before moving it: - -```bash -miniverl inspect runs/my-opd/trajectories.jsonl -miniverl cache stats runs/my-opd/teacher-cache -miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout -miniverl bridge materialize scaleout --download --offline -miniverl bridge doctor scaleout --json -``` +The compiler translates distributed resource intent into sequential local +Hugging Face phases and records that translation in the plan. Two measured +profiles are available: -The v0.9 export preserves student/teacher identities, Parquet bytes and pure -OPD overrides, but reports `launchable: false` until exact base snapshots are -materialized and validated against the installed pinned verl commit. Only then -does `bridge materialize` publish a checksummed `launch.sh`; distributed -execution remains untested. Review the [current scale-out contract](docs/verl-opd-scaleout.md), -[legacy bridge](docs/legacy-verl-bridge.md) and [compatibility policy](docs/compatibility.md). - -The intended operating loop is **plan → inspect → run → inspect → export**. -`plan --out` byte-binds the YAML, ordered overrides and scanned Parquet inputs -to the exact native config; `run --plan` rejects drift before loading weights. -Its digest follows the run manifest, teacher cache and checkpoints. Direct -`run --config` remains available for experiments. See [immutable execution -plans](docs/immutable-plans.md). - -## Research and validation - -miniVERL keeps every measured study—including negative results, superseded -runs and preregistered early stops—public under the documentation. None is used -as a claim that OPD universally beats SFT, DPO or KD: see the -[v0.7 External Alignment Gate](docs/alignment-external/alignment-external-v1.md), -[Alignment Lab](docs/alignment-lab/alignment-lab-v1.md), -[RecoveryBench](docs/recoverybench/recoverybench-v1.md), and the -[calculator study](docs/benchmarking.md). - -New runs establish tokenizer compatibility through structural identity. The -legacy behavioral fingerprint is retained only for migration and is not an -identity proof. Scientific caveats and immutable source hashes remain in the -detailed reports and [limitations](docs/limitations.md). - -## Development, security and license +| Profile | Objective | Teacher target | +| --- | --- | --- | +| `verl-opd-v0.8-single-gpu-v1` | direct GKD `forward_kl_topk` | top-k token IDs and log-probabilities | +| `verl-opd-v0.8-single-gpu-pg-k1-v1` | sampled `k1` + vanilla policy loss | sampled-token teacher log-probability | + +Use `miniverl profiles show`, `compat explain` and `compat check` to inspect +the complete mapping. [Compatibility profiles](docs/profiles/index.md) explains +how profile identity follows plans, caches, checkpoints and exports. + +## Measured systems evidence + +The Qwen3-0.6B/1.7B developer workload consumed **32 distinct prompts**, used a +64-token response bound, and completed **8 current-policy updates** at +**3.1914 GiB peak reserved VRAM** on an RTX 4080. Median steady-state rollout, +teacher-scoring and update times were 9.7200, 0.4864 and 2.3260 seconds. A +matched interruption after update four resumed to byte-identical trajectories, +adapter and optimizer tensors. + +A second SmolLM2-360M/1.7B workload completed the same 32-prompt, 8-update +shape at 1.4961 GiB peak reserved VRAM, including exact resume, PEFT reload and +scale-out materialization. Read the +[Qwen3](docs/verl-opd-reference-workload.md) and +[SmolLM2](docs/smollm2-opd-workload.md) systems records for configs, hashes and +phase-level measurements. + +Other NVIDIA GPUs use the same device-name-agnostic CUDA path. Model fit and +speed vary with VRAM, context, quantization, kernels and software versions; +`miniverl doctor` and `plan --probe` expose those machine-specific choices. + +## Research record + +The repository publishes positive, mixed and negative results with the same +resolved configs and source hashes. The calculator study found that a +protocol-qualified teacher prevented collapse but tied supervised +continuation. RecoveryBench found no fresh-state advantage in its scoped +SQLite setting. Alignment Lab began at a saturated SFT checkpoint and exposed +utility regressions that two sandbox safety checks missed. The preregistered +External Alignment Gate stopped before continuation because every candidate +failed the retained-utility threshold. + +- [Calculator protocol study](docs/benchmarking.md) +- [RecoveryBench](docs/recoverybench/recoverybench-v1.md) +- [Alignment Lab](docs/alignment-lab/alignment-lab-v1.md) +- [External Alignment Gate](docs/alignment-external/alignment-external-v1.md) + +These reports answer scoped experimental questions; the +[limitations](docs/limitations.md) page collects the measurement, architecture, +security and generalization boundaries in one place. + +## Scope + +miniVERL is designed for one local process, one NVIDIA CUDA GPU and the +documented OPD profiles above. Scale-out support produces and validates a +portable bundle against the pinned upstream source. The +[compatibility policy](docs/compatibility.md) lists supported fields, profile +semantics and handoff readiness states; [limitations](docs/limitations.md) +covers the broader execution and scientific boundaries. miniVERL is an +independent Apache-2.0 project. + +## Development ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git @@ -275,8 +193,7 @@ python -m pip install -e ".[dev]" pytest -q -m "not gpu and not network" ``` -Contributions should keep the one-GPU boundary explicit and include tests for -new failure modes. Report vulnerabilities privately through -[SECURITY.md](SECURITY.md). See [CONTRIBUTING.md](CONTRIBUTING.md), the -[changelog](CHANGELOG.md), [citation metadata](CITATION.cff), -[reproducibility guide](docs/reproducibility.md), and [Apache-2.0 license](LICENSE). +See [CONTRIBUTING.md](CONTRIBUTING.md), [CHANGELOG.md](CHANGELOG.md), +[CITATION.cff](CITATION.cff), the +[reproducibility guide](docs/reproducibility.md), [SECURITY.md](SECURITY.md) +and the [Apache-2.0 license](LICENSE). diff --git a/README.zh-CN.md b/README.zh-CN.md index 40fd856..a3ea69a 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -1,5 +1,5 @@

- miniVERL — 在单张个人 GPU 上运行 verl 风格 OPD + miniVERL — 在一张个人 GPU 上运行 verl 风格 OPD

@@ -18,17 +18,16 @@ English

-**在一张个人 GPU 上运行边界明确的 verl 风格在线策略蒸馏。** miniVERL 读取类型化的 -verl 风格 YAML 与 Parquet prompt，在本地依次执行 actor rollout → teacher scoring → -actor update，记录每个本地语义重解释，并导出标准 PEFT、Parquet 与配置产物供固定版本 -的 verl 接手扩展。 +**在一张 NVIDIA GPU 上运行 verl 风格在线策略蒸馏，并能检查每一项配置映射、 +teacher target 与训练产物。** miniVERL 把类型化 YAML 和结构化 Parquet prompt +编译成本地 actor rollout → teacher scoring → actor update 循环，再导出标准 PEFT、 +Parquet 与配置产物，便于继续扩展。 -PyPI `v0.10.1` 是稳定版；`main` 是开发版。miniVERL 是独立项目，不代表上游背书；它 -不执行任意 verl YAML、不启动分布式任务，也不声称完整的算法兼容性。 +PyPI `v0.10.1` 是稳定版；`main` 是开发版。 -## 仅用 pip 的快速开始 +## 60 秒开始 -请先安装与本机 CUDA 匹配的 PyTorch wheel，然后运行： +先安装与本机匹配的 CUDA PyTorch，再运行： ```bash python -m pip install "miniverl[train,cuda]" @@ -40,12 +39,25 @@ miniverl run --profile verl-opd-v0.8-single-gpu-v1 \ --plan plan.json --dry-run ``` -这些命令不需要 Git checkout。`data sample` 生成真实的结构化 Parquet；`plan` 在不加载 -权重时编译完整字段矩阵；`run --dry-run` 验证本地执行契约。在一张 NVIDIA CUDA GPU -上移除 `--dry-run`，即可运行固定版本的 Qwen3-0.6B actor 与 Qwen3-1.7B teacher，并 -得到可检查、可重新加载的 PEFT adapter。 +这条路径直接使用 PyPI wheel。规划阶段无需加载模型权重；生成的 `plan.json` 会绑定 +源配置、顺序 override、profile 版本与输入 Parquet 字节。准备好 CUDA GPU 后移除 +`--dry-run`，即可运行固定版本的 Qwen3-0.6B actor 与 Qwen3-1.7B teacher recipe。 -检查 plan 后，标准后续路径为： +`[train,cuda]` extra 安装 ML 与量化依赖。CUDA PyTorch wheel 请通过 +[PyTorch 安装器](https://pytorch.org/get-started/locally/)单独选择。 +[单卡指南](docs/single-gpu-guide.md)介绍从 8 GiB 显卡开始的显存规划，并附有维护者实测的 +RTX 4080 环境。 + +## 一次运行会得到什么 + +- **可审阅的 plan。** 每个接受的 verl 字段在加载权重前就有本地作用、分类与风险等级。 +- **严格的 current-policy trajectory。** actor policy version、token span 与 teacher + 监督位置始终绑定在一起。 +- **紧凑的 teacher target。** top-k target 与 sampled-k1 signal 写入带校验和、无 pickle + 的 cache。 +- **可恢复训练。** 事务化 manifest、checkpoint 与 cache index 支持检查和精确续跑。 +- **标准产物。** PEFT adapter、safetensors、结构化 Parquet、resolved config 与类型化 + provenance 可直接携带。 ```bash miniverl run --profile verl-opd-v0.8-single-gpu-v1 --plan plan.json \ @@ -56,179 +68,105 @@ miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout miniverl bridge doctor scaleout --json ``` -`train` extra 安装训练运行时，但不会选择正确的 CUDA PyTorch；`cuda` extra 只额外安装 -bitsandbytes。[单卡安装与显存指南](docs/single-gpu-guide.md)同时提供灵活安装与机器可读的 -RTX 4080 known-good 环境；除这一张实测 RTX 4080 外，其余硬件仍是未测量状态。 - -## 架构 +## 工作方式 - verl 风格 YAML、override 与 Parquet prompt 经过类型化编译器；一张 CUDA GPU 依次执行 actor rollout、teacher scoring 与 actor update；可检查产物交给固定版本 verl，而分布式执行不属于 miniVERL。 + 类型化 verl 风格配置和 Parquet prompt 被编译为一张 CUDA GPU 上顺序执行的 actor rollout、teacher scoring 与 actor update，并产生可检查的本地产物和固定版本的 scale-out bundle。 -miniVERL 在一个普通进程里按阶段调度模型角色。它不模拟 Ray resource pool，也不把本地 -Hugging Face generation 伪装成 vLLM。兼容报告保留源字段、解释本地含义、标注风险， -并在字段改变算法或分布式语义时 fail closed。 - -原生 SFT、DPO、offline KD 与工具型 OPD recipe 仍然保留。它们共享严格 token provenance、 -无 pickle teacher cache、事务化 checkpoint 与 adapter 导出；但对于已有 verl 经验的用户， -类型化 verl profile 是最短入口。 - -### 单一本地 scheduler，角色始终明确 - -actor 使用当前 adapter 生成；teacher 只对 actor 实际访问的 token 位置评分；随后 actor 接收 -padded、token-mean update。teacher 不会被当作 reward model，tool output 不会成为训练标签， -过期 actor-policy version 也不能进入 on-policy batch。显存规划可选择 resident、swap 或兼容 -的 shared-backbone，同时始终区分 actor、teacher 与 reference 身份。 - -每个阶段都在跨越下一边界前写入证据：trajectory 带逐 token provenance，top-k teacher -target 以无 pickle 格式校验，checkpoint 事务化发布，最终 adapter 用标准 PEFT loader 验证。 -因此崩溃后的检查与恢复不需要依靠终端文本猜测原始意图。 - -## 从 verl 迁移？ - -| verl 中的动作 | miniVERL 对应方式 | -| --- | --- | -| 传入 Hydra 风格 override | 重复使用 `--set`；v0.9 开发版也接受 `--overrides-file` 与 `--` 后的参数 | -| 检查 resolved config | `miniverl plan --json` | -| 执行纯 OPD | `miniverl run` | -| 复用 prompt Parquet | 直接设置 `data.train_files` | -| 分配 rollout/teacher worker | 编译为单卡顺序阶段 | -| 保存 FSDP/Megatron checkpoint | 不支持 | -| 准备扩展交接 | `miniverl export-verl` | - -```bash -actor_rollout_ref.model.path=Qwen/Qwen3-0.6B -distillation.teacher_models.teacher_model.model_path=Qwen/Qwen3-1.7B - -miniverl plan --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml \ - --set actor_rollout_ref.actor.optim.lr=1e-5 -``` - -外部 YAML 在 `run` 前必须显式接受 `plan` 列出的高风险本地重解释；内置 -profile 使用与具体值绑定的审核清单。详见[覆盖优先级与安全输入格式](docs/config-overrides.md)， -miniVERL 不执行 Hydra interpolation 或 shell 文本。 - -公开内置 profile 刻意使用上游形状的 `name: vllm`。miniVERL 会把 rollout 与 teacher -engine 名分类为本地重解释，再通过顺序本地 HF 阶段执行；这不等于 vLLM 语义。完整 -config、data、role 与错误映射见[面向 verl 用户的指南](docs/for-verl-users.md)。 - -## 已测试的 profile 边界 +miniVERL 在一个普通进程中分阶段调度 actor、teacher 与可选 reference。actor 使用当前 +adapter 生成，teacher 对实际访问的 token 位置评分，随后 actor 接收 padded token-mean +update。resident、swap 与 shared-backbone placement 会适配可用显存，同时保持各角色身份 +清晰可追踪。 -可用 `miniverl profiles list`、`profiles show/schema` 和 `compat -explain/check` 查看封闭的内置 registry，并区分“字段可接受”和“字段确实对本地运行时 -生效”。新生成的 plan、cache、checkpoint 与 export 都绑定完整且独立版本化的 profile -identity。 +每个阶段都会先发布证据，再跨越下一边界：trajectory 携带逐 token provenance，teacher +target 带校验和，checkpoint 事务化发布，最终 adapter 通过标准 PEFT 重新加载。检查一次 +运行时不必再从终端日志反推原始意图。 -两个 profile 都固定官方 verl `v0.8.0` 与 commit -`7aed6b230776f963fa09509c10d9c3a767d1102c`： +## 选择你的路径 -| Profile | 目标 | Teacher target | 状态 | +| 目标 | 第一个命令 | 主要产物 | 下一步 | | --- | --- | --- | --- | -| direct GKD | `forward_kl_topk` | top-k ID/log-probability | 已实测 | -| PG OPD | sampled `k1` + vanilla policy loss | sampled-token teacher log-probability | 已实测 | - -两者的可执行边界有意保持狭窄： - -- 一个可训练 actor 与一个 teacher； -- 每个 prompt 只生成一个 response（`n=1`）； -- 无 reward 的 distillation 与 token-mean 聚合； -- 一张 CUDA GPU 上的 LoRA/QLoRA adapter 更新； -- 不可变模型 revision 与 verl 风格结构化 prompt Parquet。 - -PG 路径不是 PPO：它用 detach 的 distillation advantage 和固定 vanilla policy-loss 形式。 -Task reward、KL penalty、多 teacher、多模态、PPO、GRPO、critic、Ray、FSDP、 -Megatron、多 GPU 与多节点均不支持。已知不支持值会得到机器可读分类；未知字段和未解析 -的 `${...}` 会被拒绝。输入必须是 resolved profile 子集，而不是任意启动脚本。 +| **在本地运行 OPD** | `miniverl plan --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml` | 不可变执行 plan | [OPD quickstart](docs/opd-quickstart.md) | +| **迁移 verl profile** | `miniverl compat check --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml` | 逐字段兼容报告 | [面向 verl 用户](docs/for-verl-users.md) | +| **适配你的显卡** | `miniverl plan --config verl-opd.yaml --probe` | 实测 placement plan | [硬件规划](docs/hardware-planning.md) | +| **准备 scale-out 交接** | `miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout` | PEFT + Parquet + config bundle | [Scale-out 契约](docs/verl-opd-scaleout.md) | + +原生 recipe 系统还支持 SFT、DPO、offline KD，以及 calculator、JSON navigation、只读 +SQLite 和自定义环境上的 tool-aware OPD。 + +## 熟悉的 verl 输入，本地执行 + +当前 profile 固定官方 verl `v0.8.0` commit +`7aed6b230776f963fa09509c10d9c3a767d1102c`，并保留熟悉的字段： + +```yaml +actor_rollout_ref: + model: {path: Qwen/Qwen3-0.6B} + rollout: {name: vllm, n: 1} +distillation: + teacher_models: + teacher_model: + model_path: Qwen/Qwen3-1.7B + inference: {name: vllm} +``` -## RTX 4080 实测路径 +编译器把分布式 resource intent 转换为顺序本地 Hugging Face 阶段，并在 plan 中记录这次 +转换。目前有两个实测 profile： -Qwen3-0.6B/1.7B developer workload 实际消费 **32 个不同 prompt**，response 上限为 -64 token，完成 **8 次 current-policy update**，**peak reserved VRAM 为 3.1914 GiB**。 -稳态 rollout、teacher scoring 与 update 的中位耗时分别为 9.7200、0.4864 与 2.3260 秒。 -匹配的运行在第 4 次 update 后中断并恢复；最终 trajectory、adapter 与 optimizer tensor -均字节一致。详见[数据绑定图与完整记录](docs/verl-opd-reference-workload.md)；原始一次更新的 -[pip smoke](docs/opd-quickstart.md)仍完整保留。 -另一套固定版本的 SmolLM2-360M/1.7B recipe 消费 32 个不同 prompt 并完成 8 次 -update，peak reserved VRAM 为 1.4961 GiB；中断恢复保持字节一致，PEFT 重载与精确 -snapshot 的 scale-out 物化均通过。详见[完整 SmolLM2 系统记录](docs/smollm2-opd-workload.md)。 - -这只证明一个运行时与产物路径，不是吞吐 benchmark、对齐质量 endpoint，也不证明 OPD -优于 SFT、DPO 或 KD。其他 NVIDIA GPU 使用相同的 device-name-agnostic CUDA 路径，但 -能否装下仍取决于显存、上下文、量化与 kernel，并且仍未测量。 -[发布资格契约](docs/release-qualification.md)先在 GitHub-hosted runner 上构建唯一 candidate -wheel，再在实测 RTX 4080 上验证完全相同的字节；发布只能复用这份制品。 -[上游支持政策](docs/upstream-support-policy.md)保证已发布 profile identity 不原地漂移。 -runner 尚未激活时不会被描述成持续 GPU CI。 - -### 按硬件条件选择路径，而不是按显卡名称 - -| 情况 | 建议起点 | 保持不变的内容 | +| Profile | 目标 | Teacher target | | --- | --- | --- | -| CPU 或笔记本上先检查 | `plan` 与 `run --dry-run` | 完整配置分类 | -| 单张小显存 CUDA GPU | QLoRA 使用 resident 本地分阶段；或 unquantized LoRA 使用 swap | 逻辑 batch 与 loss 语义 | -| 同 base 的 actor/teacher adapter | shared-backbone | 明确角色 provenance | -| 显存更充足 | 增大各阶段 physical batch | 数据与 optimizer 意图 | - -BF16/FP16 自动选择取决于设备能力，而不是 3070、4080、5090 或 Titan 等市场名称。 -`miniverl doctor` 报告实际 CUDA/PyTorch 路径；普通 plan 不加载权重，显式 -`plan --probe` 才执行零 optimizer update 的有界 CUDA 测量。详见 -[硬件规划](docs/hardware-planning.md)。 -显存紧张时不会静默更换模型、teacher、context、top-k 或 loss。 +| `verl-opd-v0.8-single-gpu-v1` | direct GKD `forward_kl_topk` | top-k token ID 与 log-probability | +| `verl-opd-v0.8-single-gpu-pg-k1-v1` | sampled `k1` + vanilla policy loss | sampled-token teacher log-probability | -已完成的 run 可以生成经过清理、且不会自动上传的硬件记录： - -```bash -miniverl hardware record --run artifacts/runs/ --out hardware-record.json -miniverl hardware validate hardware-record.json -``` +使用 `miniverl profiles show`、`compat explain` 与 `compat check` 查看完整映射。 +[兼容 profile](docs/profiles/index.md)说明 profile identity 如何跟随 plan、cache、checkpoint +与 export。 -社区提交即使通过 schema 校验也仍标记为 `unreviewed`；只有维护者复核并确认发布许可后， -才会进入公开的“maintainer-validated measured”矩阵。 +## 实测系统证据 -## 数据与产物互操作 +Qwen3-0.6B/1.7B developer workload 消费 **32 个不同 prompt**，response 上限为 64 token， +在 RTX 4080 上完成 **8 次 current-policy update**，**peak reserved VRAM 为 3.1914 GiB**。 +稳态 rollout、teacher scoring 与 update 的中位耗时分别为 9.7200、0.4864 与 2.3260 秒。 +匹配运行在第 4 次 update 后中断并续跑，最终 trajectory、adapter 与 optimizer tensor +字节一致。 -profile 直接读取结构化 verl 风格 Parquet，不会在缺失数据时静默换成 calculator 环境。 -prompt 的 role/content 保持结构化，data source、ability 与 extra metadata 也会保留。跨越 -原生 trajectory 边界时可用 `miniverl convert-dataset`；除非明确允许部分转换，任何有损行 -都会被拒绝。 +第二套 SmolLM2-360M/1.7B workload 以 1.4961 GiB peak reserved VRAM 完成相同的 +32-prompt、8-update 形状，并通过精确续跑、PEFT 重载与 scale-out 物化。完整配置、哈希和 +分阶段测量见 [Qwen3](docs/verl-opd-reference-workload.md) 与 +[SmolLM2](docs/smollm2-opd-workload.md) 系统记录。 -本地 run 包含源配置、兼容矩阵、本地执行计划、trajectory、teacher target、checkpoint、 -实测记录与 PEFT adapter。导出前可执行： +其他 NVIDIA GPU 使用同一条 device-name-agnostic CUDA 路径。模型能否装下以及实际速度 +取决于显存、上下文、量化、kernel 与软件版本；`miniverl doctor` 和 `plan --probe` 会把 +这些机器相关选择显示出来。 -```bash -miniverl inspect runs/my-opd/trajectories.jsonl -miniverl cache stats runs/my-opd/teacher-cache -miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout -miniverl bridge materialize scaleout --download --offline -miniverl bridge doctor scaleout --json -``` +## 研究记录 -v0.9 export 保留 student/teacher 身份、Parquet 原始字节与纯 OPD override；在精确 base -snapshot 被物化并通过已安装的固定 verl commit 验证前，仍报告 `launchable: false`。只有此后 -`bridge materialize` 才会发布带校验和的 `launch.sh`；分布式执行仍是未测试。详见 -[物化契约](docs/scaleout-materialization.md)、[当前 scale-out 契约](docs/verl-opd-scaleout.md)与 -[兼容性政策](docs/compatibility.md)。 +仓库以相同的 resolved config 与源哈希发布正面、混合和负面结果。Calculator study 中， +protocol-qualified teacher 避免了 collapse，但只追平 supervised continuation。 +RecoveryBench 在限定的 SQLite 设置中没有观察到 fresh-state 优势。Alignment Lab 从饱和 +SFT checkpoint 出发，发现了两个 sandbox safety check 没有捕捉到的 utility regression。 +预注册 External Alignment Gate 则因所有候选都未通过 retained-utility 阈值，在 continuation +之前结束。 -建议操作闭环是 **plan → inspect → run → inspect → export**。`plan --out` 把 YAML、按顺序 -应用的 override、扫描后的 Parquet 与精确 native config 绑定；`run --plan` 在加载权重前拒绝 -任何漂移。plan digest 同时进入 run manifest、teacher cache 和 checkpoint；直接 -`run --config` 仍适合快速实验。详见[不可变执行计划](docs/immutable-plans.md)。 +- [Calculator protocol study](docs/benchmarking.md) +- [RecoveryBench](docs/recoverybench/recoverybench-v1.md) +- [Alignment Lab](docs/alignment-lab/alignment-lab-v1.md) +- [External Alignment Gate](docs/alignment-external/alignment-external-v1.md) -## 研究与验证 +这些报告回答的是各自限定的实验问题；[限制页面](docs/limitations.md)集中列出测量、架构、 +安全与泛化边界。 -miniVERL 将所有实测研究——包括负结果、被取代的运行与预注册 early stop——继续公开在 -文档中；它们都不会被用来宣称 OPD 普遍优于 SFT、DPO 或 KD。参见 -[v0.7 External Alignment Gate](docs/alignment-external/alignment-external-v1.md)、 -[Alignment Lab](docs/alignment-lab/alignment-lab-v1.md)、 -[RecoveryBench](docs/recoverybench/recoverybench-v1.md)与 -[calculator study](docs/benchmarking.md)。 +## 适用范围 -新运行通过结构身份确认 tokenizer 兼容性；旧 behavioral fingerprint 只用于迁移，不是身份 -证明。科学限制与不可变源哈希继续保留在详细报告和[限制页面](docs/limitations.md)。 +miniVERL 面向一个本地进程、一张 NVIDIA CUDA GPU 和上面的文档化 OPD profile。 +Scale-out 支持会生成 portable bundle，并对固定版本的上游源码执行验证。 +[兼容性政策](docs/compatibility.md)列出支持字段、profile 语义与 handoff readiness state； +[限制页面](docs/limitations.md)集中说明更广的执行和科学边界。miniVERL 是独立的 +Apache-2.0 项目。 -## 开发、安全与许可 +## 开发 ```bash git clone https://github.com/DaoyuanLi2816/mini-verl.git @@ -237,6 +175,6 @@ python -m pip install -e ".[dev]" pytest -q -m "not gpu and not network" ``` -贡献应保持单卡边界清晰，并为新失败模式添加测试。安全问题请按 [SECURITY.md](SECURITY.md) -私下报告。另见 [CONTRIBUTING.md](CONTRIBUTING.md)、[changelog](CHANGELOG.md)、 -[citation](CITATION.cff)、[复现指南](docs/reproducibility.md)与 [Apache-2.0 license](LICENSE)。 +另见 [CONTRIBUTING.md](CONTRIBUTING.md)、[CHANGELOG.md](CHANGELOG.md)、 +[CITATION.cff](CITATION.cff)、[复现指南](docs/reproducibility.md)、 +[SECURITY.md](SECURITY.md) 与 [Apache-2.0 license](LICENSE)。 diff --git a/TODO.md b/TODO.md index 759d9e9..e5d2f28 100644 --- a/TODO.md +++ b/TODO.md @@ -6,5 +6,5 @@ GitHub issues are the active roadmap: - [#64 — single-GPU verl-style OPD roadmap](https://github.com/DaoyuanLi2816/mini-verl/issues/64) Completed work and evidence are indexed in [PROJECT_STATE.md](PROJECT_STATE.md) -and [CHANGELOG.md](CHANGELOG.md). Multi-GPU training, Ray, FSDP, DeepSpeed, -vLLM and PPO/GRPO remain outside miniVERL's local runtime. +and [CHANGELOG.md](CHANGELOG.md). Current execution and research boundaries are +maintained in [docs/limitations.md](docs/limitations.md). diff --git a/docs/alignment-external/benchmark-governance.md b/docs/alignment-external/benchmark-governance.md index 4f406f3..1a1bdb7 100644 --- a/docs/alignment-external/benchmark-governance.md +++ b/docs/alignment-external/benchmark-governance.md @@ -1,8 +1,7 @@ # External benchmark governance -This page records where every external alignment endpoint comes from, what -licence it carries, what this project redistributes, and what each measurement -does *not* establish. It is the auditable half of +This page records the source, licence, redistribution policy and interpretation +scope for every external alignment endpoint. It is the auditable half of [`benchmarks/external-alignment/registry.yaml`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/external-alignment/registry.yaml), which is the machine-readable half. diff --git a/docs/alignment-lab/demo.md b/docs/alignment-lab/demo.md index cad6907..bffd20d 100644 --- a/docs/alignment-lab/demo.md +++ b/docs/alignment-lab/demo.md @@ -1,9 +1,8 @@ # Alignment Lab demo recording script -This is a reproducible recording plan, not a fabricated video. The short run -uses the CPU toy backend and proves the workflow and artifact surfaces; its 0% -task score is not capability evidence. The measured v0.5 result remains the -separate three-seed Qwen3 artifact. +This reproducible recording plan uses the CPU toy backend to show the workflow +and artifact surfaces. Its 0% task score belongs to the demo fixture; the +measured v0.5 result is the separate three-seed Qwen3 artifact. ## Install diff --git a/docs/alignment-lab/launch-copy.md b/docs/alignment-lab/launch-copy.md index 3166543..2147609 100644 --- a/docs/alignment-lab/launch-copy.md +++ b/docs/alignment-lab/launch-copy.md @@ -1,7 +1,7 @@ # miniVERL v0.5.0 launch copy -All text below is evidence-bounded. Update the release URL only after the exact -tag is public. +This archived launch copy is tied to the v0.5.0 evidence bundle. Update its +release URL only when republishing against the exact tag. ## GitHub Release summary diff --git a/docs/banner.svg b/docs/banner.svg index 0d21991..f058b63 100644 --- a/docs/banner.svg +++ b/docs/banner.svg @@ -1,7 +1,7 @@ - + diff --git a/docs/comparisons.md b/docs/comparisons.md index 8ee2ca6..6a588ef 100644 --- a/docs/comparisons.md +++ b/docs/comparisons.md @@ -1,95 +1,76 @@ -# How miniVERL compares +# Choose the right distillation stack -This page exists so that you can decide quickly whether miniVERL is the wrong -tool for your problem. For most serious distillation work it is. +miniVERL, verl, TRL GKD, KDFlow and research OPD harnesses serve different +workflows. Start from the execution environment and the artifact contract you +need, then choose the smallest stack that covers them. -Every cell below is either something read out of the relevant source tree, a -value returned by the GitHub or arXiv API, or the literal string `not verified`. -Nothing here is inferred from a project's marketing copy. Repository/API and -official-documentation checks were refreshed on 2026-07-29; upstream projects -move, so re-check the primary sources in [references](references.md) before -relying on a row. +The source snapshot behind this page was checked on 2026-07-29. Upstream +projects evolve, so follow the primary links in [references](references.md) +before making a long-lived infrastructure decision. -## Feature table +## Quick choice -| dimension | miniVERL | verl | TRL GKD | KDFlow | OPSD | plain SFT | +| Your priority | Best starting point | +| --- | --- | +| Read, modify and audit OPD on one NVIDIA GPU | **miniVERL** | +| Scale training across accelerators and nodes | **verl** | +| Add generalized-JSD distillation to a Transformers training workflow | **TRL GKD** | +| Explore cross-tokenizer or multimodal KD | **KDFlow** | +| Reproduce one paper's OPD experiments | the paper's **research harness** | +| Train a fixed supervised dataset | **plain SFT** | + +## Capability snapshot + +| Dimension | miniVERL | verl | TRL GKD | KDFlow | OPSD | plain SFT | | --- | --- | --- | --- | --- | --- | --- | -| **Primary scope** | Single-accelerator on-policy distillation of a tool-using agent, plus SFT and offline KD, over three built-in deterministic environments | General RL post-training framework (HybridFlow). PPO/GRPO family, plus a first-class distillation trainer in `verl/trainer/distillation/` | One trainer, `trl.experimental.gkd.GKDTrainer`, doing generalized-JSD distillation on chat datasets | Knowledge-distillation framework: off-policy, on-policy, cross-tokenizer and multimodal, per the repository description | Research harness for on-policy distillation experiments, built on verl | Next-token cross-entropy on a fixed dataset. miniVERL implements this itself as `run.mode: sft`, for use as a baseline arm | -| **Mandatory infrastructure** | None beyond a Python process. `doctor`, `validate`, `inspect`, `report`, `cache`, `schema` and `export-benchmark` run on the base install; torch, transformers and peft arrive only with the `train` extra | Ray. `ray[default]` is an unconditional line in `requirements.txt` | The `transformers` Trainer plus `accelerate`. No cluster runtime | Ray and SGLang, both unconditional in `requirements.txt` | Whatever verl needs, therefore Ray. Setup is a shell script (`scripts/opd/setup_opd.sh`), not a package install | None | -| **Target hardware** | One accelerator, or CPU for the toy backend. Every GPU number in this repository comes from a single RTX 4080 (16 GB). There is no multi-GPU code path | The published device-tuning table starts at 1xH100 and runs through 8xH100, 8xH800 and 32xH800 / 32xH20 | Not stated in the trainer documentation; scales with whatever `accelerate` is configured for | Examples assume 8 GPUs per node | not verified; inherits verl's requirements | Any | -| **On-policy rollouts** | Yes. `run.mode: opd` samples from the current student every cycle, and the config validator forces `cache.strict_policy_version: true`, so a target produced at policy version *v* cannot be consumed at *v+1* | Yes | Configurable, not the default. `lmbda` defaults to `0.5`, so on average half the batches use student-generated sequences | Yes | Yes | No. The trajectory set is fixed before training starts | -| **Multi-turn tool use with environment execution** | Yes. `RolloutRunner` interleaves real tool execution (calculator, JSON navigation, read-only SQLite) with generation, bounded by `max_turns`, `max_new_tokens_per_turn`, `max_total_tokens`, `max_parse_errors` and `max_repeated_calls` | Yes. `verl/experimental/agent_loop/` contains `tool_agent_loop.py` and `tool_parser.py` | No tool environment in the distillation trainers | None found. A code search over the repository for `tool_call` and for `agent` returns 0 results | Yes. The README describes multi-turn agent-loop rollouts with tool and environment tokens excluded from the loss | No | -| **Teacher-target compression / cache** | `bucketed_topk_tail` (teacher top-k log-probs plus one aggregated tail bucket) or `exact_full_vocab`. Bucketed targets persist to a sharded safetensors cache with a SHA-256 per shard and a recorded teacher revision, tokenizer fingerprint, temperature and `top_k` | not verified | `GKDTrainer` recomputes full-vocabulary teacher logits under `no_grad` every step and keeps no cache. Separately, `ServerDistillationTrainer` exposes `loss_top_k` (default `1`) and `loss_add_tail` (default `True`) | not verified | Chunked divergence computation rather than a persistent cache; the README states it processes tokens in chunks instead of materializing full-vocabulary tensors for the whole batch | Not applicable; the targets are the tokens | -| **Packaging and tests** | Published hatchling wheel plus self-testing sdist, `miniverl` console script, 1,000+ tests and 86%+ branch coverage; exact release evidence is in the [quality record](generated/quality.json) | Published on PyPI as `verl`; workflow inventory checked on the date above | Published on PyPI as `trl`; tests in-repo, including `tests/experimental/test_server_distillation_trainer.py` | Has a `pyproject.toml`; package/repository surfaces were checked on the date above | Repository package/test/workflow surfaces were checked on the date above | Not applicable | -| **License** | Apache-2.0 (`LICENSE` in the repository root) | Apache-2.0 | Apache-2.0 | MIT | No `LICENSE` file; the GitHub API reports no license, so the code is all-rights-reserved and cannot be reused | Not applicable | - -One more research codebase is worth naming even though it does not get a column: -**thunlp/OPD**, the official code for arXiv:2604.13016, also builds on verl -(v0.7.0) with LlamaFactory for the SFT stage, runs its experiments on 8xA800 -80 GB, and likewise ships **no LICENSE file**. - -## What is not novel here - -Two claims that a reader might expect this project to make, and which would be -false: - -- **Top-k plus a tail bucket is not a new idea.** TRL's - `ServerDistillationTrainer` already has `loss_top_k` (default `1`) and - `loss_add_tail` (default `True`), which is the same coarse-graining. miniVERL's - contribution on this axis is bookkeeping, not the objective: the bucket - parameters are recorded in the cache index and re-checked on load, and the - documentation is explicit that the result is a lower bound on the exact - divergence rather than the divergence itself. -- **verl is not missing on-policy distillation, and it is not missing tool - use.** verl has `verl/trainer/distillation/` with FSDP and Megatron backends - and a `DistillationConfig` in `verl.workers.config`, and it has an agent loop - with a tool parser in `verl/experimental/agent_loop/`. Any comparison that - presents miniVERL as filling those gaps is wrong. - -## What miniVERL is not - -miniVERL is not a scaling framework, and it is not a faster or better -implementation of anything listed above. It has no Ray integration, no FSDP or -Megatron backend, no vLLM or SGLang rollout engine, no tensor or pipeline -parallelism, and no multi-node story of any kind. Rollouts come from a custom -sampling loop that decodes one sequence at a time with a KV cache, projecting a -single position through the LM head per step. The update path supports local -mask-isolated padded batches, but has no continuous rollout batching or remote -workers. `train.gradient_accumulation_steps` is the optimizer-group size and -`train.trajectory_batch_size` is only the physical update-forward size. There is no -PPO, no GRPO and no reward model. The three environments are synthetic and -generated in-process; there is no dataset loader, no containerized or networked -tool sandbox, and no support for vision-language models. Capability results in -this repository come from one calculator task family on one consumer GPU at two -prespecified seeds, which is enough to show the pipeline runs and is not enough to establish -that any objective beats any other. - -It is a single-GPU teaching and experimentation lab: a readable implementation -of the exact and bucketed divergences with the orientation written into every -function name, a rollout loop that records token provenance as it generates -rather than reconstructing it afterwards, a teacher cache whose policy version -is enforced rather than assumed, and a CLI that refuses contradictory recipes -before anything is downloaded. - -## When you should use verl instead - -Use verl, not miniVERL, if any of the following is true: - -- You have more than one GPU, or more than one node. verl was built for this; - miniVERL has no code path for it at all. -- Your chosen student/teacher pair and token budget do not fit on one device. - miniVERL's `swap` strategy can help for unquantized pairs, but is unavailable - when either model is quantized. -- You need rollout throughput. verl drives vLLM or SGLang; miniVERL decodes one - sequence at a time, and on the machine used here that is kernel-launch bound - rather than compute bound. -- You want on-policy distillation combined with RL objectives, a reward model, - or an advantage estimator. verl has the PPO and GRPO machinery and a - distillation trainer that plugs into it. -- You need the broader maintainer community and release surface of an upstream - scaling framework. - -miniVERL is a reasonable choice when you have exactly one personal NVIDIA GPU, you want -to read and modify the loss, and you care more about being able to audit which -token was supervised by which teacher distribution than about throughput. -Reading `docs/limitations.md` before starting is strongly recommended. +| **Design center** | Inspectable single-GPU OPD, plus local SFT/DPO/KD baselines | General RL and distillation post-training at scale | A distillation trainer inside the Transformers ecosystem | Distributed KD across policy, tokenizer and modality choices | Paper-oriented OPD experiments built on verl | Next-token learning on a fixed dataset | +| **Runtime shape** | One Python process; optional CUDA training stack | Ray with distributed model and rollout backends | Transformers Trainer + Accelerate | Ray + SGLang | verl-based | Framework-dependent | +| **On-policy path** | Strict current-policy rollout → teacher score → actor update | First-class distributed distillation and agent-loop paths | Configurable student-generated sequences | Available | Available | Fixed dataset | +| **Tool trajectories** | Calculator, JSON navigation, read-only SQLite and custom typed environments | Agent loop and tool parser | Chat-dataset training | Project-dependent | Tool-oriented experiments | Dataset-defined | +| **Teacher artifacts** | Exact or top-k targets, sampled-k1 signals and sharded safetensors cache | Distributed trainer state | Teacher forward pass in the trainer; server distillation supports top-k + tail controls | Chunked/distributed target handling | Chunked divergence path | Target tokens | +| **Primary output** | PEFT adapter, trajectories, cache, plan and portable provenance bundle | Distributed checkpoints and rollout/training artifacts | Transformers model or adapter | Project-defined model artifacts | Experiment artifacts | Model or adapter | +| **Best fit** | One-GPU experiments where semantic traceability matters | Throughput, scale and RL integration | Existing Transformers/TRL workflows | Broader KD research space | Reproducing its published setup | A known supervised target dataset | + +The table summarizes each project's documented design center rather than +ranking quality or speed. Hardware numbers and algorithm outcomes are meaningful +only within their original model, data and runtime setup. + +## miniVERL's design center + +miniVERL concentrates on the parts of a local distillation run that benefit +from explicit evidence: + +- a typed compiler that records how each source field affects local execution; +- strict policy-version binding between rollouts and teacher targets; +- token-span provenance created during generation; +- exact, top-k and sampled-k1 teacher signals with checksummed cache identity; +- transactional plans, checkpoints and export publication; +- standard PEFT, safetensors and Parquet interchange. + +That design is especially useful when you have one personal NVIDIA GPU, want +to inspect or change the loss, and value semantic traceability over rollout +throughput. + +## When scale is the main requirement + +verl is the natural continuation when the workload needs multi-GPU execution, +high-throughput rollout engines, distributed checkpointing or RL objectives. +miniVERL's export path prepares a pinned bundle of local artifacts for that +workflow and reports artifact completeness, materialization and launchability +as separate states. + +Use the [scale-out contract](verl-opd-scaleout.md) for that handoff. The full +list of miniVERL's algorithm, architecture and evidence boundaries lives in +[limitations](limitations.md), while [compatibility](compatibility.md) defines +the exact versioned profile contract. + +## Evidence notes + +The public miniVERL results cover one-GPU systems behavior and several scoped +task studies. They include negative outcomes and preregistered early stops. +Read the detailed reports before treating a method result as portable to a new +teacher, task, model pair or budget. + +The external project names and descriptions above are attributed to their +primary repositories and papers. miniVERL is an independent project; the table +is an interoperability and workflow guide. diff --git a/docs/compatibility.md b/docs/compatibility.md index 67479d6..37d0acc 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -1,10 +1,9 @@ # Compatibility policy -miniVERL is a `0.x` research package: a minor release may add validated fields -or tighten unsafe input, but it must not silently reinterpret a supported -artifact. Release notes identify migrations, and the current reader either -loads an older artifact with its historical semantics or rejects it with a -regeneration hint. +miniVERL versions its executable profiles and artifact schemas independently. +A minor release may add validated fields or tighten input validation while +preserving the historical meaning of existing artifacts. Release notes identify +migrations, and readers retain the schema-specific behavior needed to load them. ## Stable surfaces through v0.9 @@ -21,7 +20,7 @@ regeneration hint. commit `7aed6b230776f963fa09509c10d9c3a767d1102c`; every bundle carries that exact contract in `REQUIRED_VERL.txt`. -## Pinned bridge, not generic compatibility +## Pinned OPD profiles v0.8 adds the separately versioned `verl-opd-v0.8-single-gpu-v1` executable profile: one local actor, one teacher, one generation, pure GKD @@ -50,12 +49,13 @@ bundle, reward scaffold, hashes and an exact-source smoke. Unknown, algorithm-changing or distributed-only verl fields are rejected instead of guessed. -These miniVERL-defined levels do not make miniVERL a verl runtime. Optimizer state, distributed -RNG, FSDP/Megatron native checkpoints, Ray runtime state and teacher-cache to -PPO-reference-cache conversion are unsupported. The release smoke validates -artifacts and configuration; distributed execution is recorded as not tested. -Current bundles contain a fail-closed reward scaffold and an absent base -snapshot, so they use `launch.template.sh` and report `launchable: false`. +These miniVERL-defined levels describe artifact and configuration exchange. +Optimizer state, distributed RNG, FSDP/Megatron native checkpoints, Ray runtime +state and teacher-cache conversion sit beyond that exchange contract. The +release smoke records artifact/config validation and distributed execution as +separate evidence fields. Legacy reward-scaffold bundles use +`launch.template.sh`; current pure-OPD bundles become launchable through the +documented materialization checks. See the [current local runtime](verl-opd-runtime.md), [current scale-out contract](verl-opd-scaleout.md), and [legacy bridge](legacy-verl-bridge.md). diff --git a/docs/config-overrides.md b/docs/config-overrides.md index 100e6ae..3a005b9 100644 --- a/docs/config-overrides.md +++ b/docs/config-overrides.md @@ -1,9 +1,9 @@ # Config overrides miniVERL accepts one resolved YAML mapping plus familiar dotted `key=value` -tokens. It does not execute a verl launcher, shell command, Hydra resolver or -OmegaConf interpolation. Resolve composition inside the trusted verl workflow, -then pass either the YAML, an argv JSON array, or a plain override file. +tokens. Resolve Hydra/OmegaConf composition inside the trusted verl workflow, +then pass either the YAML, an argv JSON array, or a plain override file to the +local compiler. ```bash miniverl plan \ diff --git a/docs/consumer-runtime-v1.md b/docs/consumer-runtime-v1.md index 792d806..17d93e1 100644 --- a/docs/consumer-runtime-v1.md +++ b/docs/consumer-runtime-v1.md @@ -4,9 +4,8 @@ > online policy update. v0.4 adds padded multi-trajectory update batches and a shared-backbone adapter -runtime. It does not add a batched rollout server, distributed workers or a new -training objective. The default remains the compatible `dual_model` runtime -with `train.trajectory_batch_size: 1`. +runtime while keeping the existing objective and rollout semantics. The default +remains `dual_model` with `train.trajectory_batch_size: 1`. ## Measured result diff --git a/docs/design.md b/docs/design.md index 562644d..75c1a74 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,9 +1,8 @@ # miniVERL design -This document explains what miniVERL is for, how it is layered, what happens -during one on-policy distillation (OPD) cycle, and which invariants are checked -where. Every module, function, field name and file name below was read out of -the source tree; every number was produced by a command that was run. +This document explains miniVERL's layers, one on-policy distillation (OPD) +cycle, and the invariants checked at each boundary. Module, function and field +names follow the source tree; measured numbers link to their result artifacts. For the mathematics of the objectives see [math.md](math.md). diff --git a/docs/for-verl-users.md b/docs/for-verl-users.md index 2e10257..2ccd5c3 100644 --- a/docs/for-verl-users.md +++ b/docs/for-verl-users.md @@ -1,9 +1,9 @@ # For verl users -miniVERL is a local runtime for two documented subsets of verl v0.8 OPD. It -keeps familiar field names and Parquet data, then compiles distributed resource -intent into sequential phases on one CUDA GPU. It is an independent project; -the mapping is explicit and does not imply endorsement or full compatibility. +miniVERL brings two documented verl v0.8 OPD profiles into a local, inspectable +workflow. Familiar field names and Parquet data flow through a typed compiler, +which turns resource intent into sequential phases on one CUDA GPU and records +every local mapping in the resulting plan. Inspect the closed, versioned registry with `miniverl profiles list`; use `miniverl compat explain` before assuming that an accepted field is effective. @@ -12,10 +12,10 @@ by plans, caches, checkpoints and exports. - verl-shaped YAML, overrides and Parquet prompts pass through a typed compiler; one CUDA GPU runs actor rollout, teacher scoring and actor update; portable artifacts can be handed to pinned verl while distributed execution remains outside miniVERL. + verl-shaped YAML, overrides and Parquet prompts pass through a typed compiler; one CUDA GPU runs actor rollout, teacher scoring and actor update; portable artifacts form a pinned scale-out handoff. -## What you can reuse +## Supported input shape - A resolved YAML using the `verl-opd-v0.8-single-gpu-v1` field subset. - Reward-free verl-style Parquet prompts with structured chat messages. @@ -26,11 +26,10 @@ by plans, caches, checkpoints and exports. `distillation.teacher_models.teacher_model.model_path`, response bounds, learning rate and LoRA configuration. -What is not reusable: arbitrary Hydra composition inside miniVERL, shell launch scripts, -resource pools, Ray actors, FSDP/Megatron checkpoints, PPO/GRPO, critics, -policy-gradient modes beyond the closed sampled-k1 profile, task-reward -mixtures, multiple teachers and multimodal workers. Unsupported semantics fail -closed instead of falling back silently. +The profile compiler accepts a resolved, documented subset and classifies every +field before execution. [Compatibility profiles](profiles/index.md) contains +the field contract; [limitations](limitations.md) collects the algorithms and +distributed runtime surfaces covered by other tools. ## Command mapping @@ -42,7 +41,6 @@ closed instead of falling back silently. | launch OPD | `miniverl run` | | read prompt Parquet | use the file directly | | allocate resource pools | compile to sequential local phases | -| save a distributed checkpoint | unsupported | | hand artifacts back | `miniverl export-verl` | ```bash @@ -53,9 +51,9 @@ miniverl plan --profile verl-opd-v0.8-single-gpu-v1 \ ``` Planning is weight-free and offline. Use `--json` to retain the complete field -matrix. The stable v0.9 CLI records repeated `--set`, override files and -trailing tokens with deterministic precedence; it does not execute `${...}` -interpolations or shell text. See [Config overrides](config-overrides.md). +matrix. The CLI records repeated `--set`, override files and +trailing tokens with deterministic precedence. Resolve `${...}` composition in +the trusted source workflow first; see [Config overrides](config-overrides.md). For a serious run, add `--accept-local-reinterpretations --out plan.json`, inspect the immutable artifact, then execute `miniverl run --plan plan.json`. @@ -77,10 +75,10 @@ distillation: inference: {name: vllm} ``` -miniVERL does not launch vLLM here. The compiler classifies both engine names -as local reinterpretations and runs local Hugging Face generation and teacher -scoring sequentially. The source value, local meaning and risk are preserved in -the compatibility report. +The `vllm` values remain visible as source intent. The compiler classifies them +as local reinterpretations, runs Hugging Face generation and teacher scoring +sequentially, and preserves the source value, local meaning and risk in the +compatibility report. ## Data mapping @@ -90,9 +88,9 @@ row preserves its structured messages and source metadata. Run `miniverl data sample --out prompts.parquet` for a valid small file, or use `miniverl convert-dataset` when crossing the native trajectory boundary. -The compiler never substitutes a calculator environment for missing Parquet -data. A missing file is an actionable error when execution begins. Dataset -bytes and schema are copied into export provenance. +Parquet paths are direct execution inputs, so a missing file produces an +actionable error when execution begins. Dataset bytes and schema are copied +into export provenance. ## Actor, teacher and reference roles @@ -101,10 +99,10 @@ produces top-k token IDs and log-probabilities on actor-generated tokens; top-k mass and overlap are diagnostics, not an explicit tail bucket. In the PG k1 profile the target is only the sampled token's teacher log-probability, bound to the rollout actor log-probability and policy version; current actor -log-probability is recomputed at update time. Neither profile has a reference -policy, critic or task reward. Local runtime strategies +log-probability is recomputed at update time. Both profiles use reward-free +distillation with one actor and one teacher. Local runtime strategies may keep quantized roles resident, swap movable unquantized roles, or share a -compatible backbone, but role identities never collapse in provenance. +compatible backbone while preserving distinct role identities in provenance. ## Export boundary @@ -114,12 +112,12 @@ miniverl bridge materialize scaleout --download --offline miniverl bridge doctor scaleout --json ``` -The stable v0.9 bundle carries PEFT, Parquet, config and provenance artifacts, +The export bundle carries PEFT, Parquet, config and provenance artifacts, but is reported as `launchable: false` until exact base snapshots are materialized and the pinned upstream checks pass. Read the [materialization workflow](scaleout-materialization.md). Upstream parse/load smoke, artifact completeness, launchability and distributed -execution are separate statuses. miniVERL never reports a distributed job as -tested. +execution are separate statuses, so the report shows exactly how far a bundle +has progressed. ## Common errors diff --git a/docs/index.md b/docs/index.md index 066141d..22ebd80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,134 +1,149 @@ # miniVERL -Run a documented subset of verl v0.8 on-policy distillation on one consumer -GPU. Bring a typed OPD profile and Parquet prompts, inspect rollout → teacher -scoring → update locally, then export standard artifacts. miniVERL is -independent; distributed execution and full verl compatibility are not claimed. +Run verl-style on-policy distillation on one NVIDIA GPU, inspect every local +mapping and teacher target, and carry standard artifacts into a scale-out +workflow. [Install and run locally](single-gpu-guide.md){ .md-button .md-button--primary } -[Read the compatibility boundary](verl-opd-runtime.md){ .md-button } +[Start with a verl profile](for-verl-users.md){ .md-button } -## Pip-only OPD quickstart +## From profile to adapter + +miniVERL compiles typed YAML and structured Parquet prompts into three local +phases: actor rollout, teacher scoring and actor update. The same execution plan +binds the profile version, overrides and input bytes to trajectories, +checkpoints, teacher caches and the final PEFT adapter. ```bash python -m pip install torch --index-url https://download.pytorch.org/whl/cu130 python -m pip install "miniverl[train,cuda]" -miniverl doctor miniverl data sample --format verl-parquet --out prompts.parquet miniverl plan --profile verl-opd-v0.8-single-gpu-v1 \ --config builtin:qwen3-0.6b-1.7b-opd \ - --set 'data.train_files=["prompts.parquet"]' + --set 'data.train_files=["prompts.parquet"]' --out plan.json miniverl run --profile verl-opd-v0.8-single-gpu-v1 \ - --config builtin:qwen3-0.6b-1.7b-opd \ - --set 'data.train_files=["prompts.parquet"]' --dry-run + --plan plan.json --dry-run ``` -Planning loads no weights. Remove `--dry-run` on one CUDA GPU to execute the -pinned Qwen3 recipe and export a standard PEFT adapter. Install the matching -CUDA-enabled PyTorch wheel first; the `[cuda]` extra does not select one. -For the exact maintainer-measured versions, use the -[machine-readable known-good stack](single-gpu-guide.md). - -After inspection, write `plan.json` with `plan --out`, run it without -`--dry-run`, then use `inspect`, `cache stats`, `export-verl` and `bridge -doctor`; the [quickstart](opd-quickstart.md) carries the complete copyable path. - -## Runtime and compatibility boundary +Planning is weight-free. Review `plan.json`, remove `--dry-run` on a CUDA GPU, +then inspect and export the result: -miniVERL runs one local CPU process or one NVIDIA CUDA GPU. Fit depends on the -model pair, context, kernels and VRAM; there is no GPU-name allowlist. Ray, -FSDP, Megatron, PPO, GRPO and distributed launch are outside the runtime. - -The executable profile pins verl `v0.8.0` at `7aed6b23`: one actor, one teacher, -`n=1`, GKD `forward_kl_topk`, token-mean and no reward/KL penalty. Unsupported -algorithm or distributed semantics fail closed. Exports remain unlaunchable -until exact base snapshots are materialized and checked under the pinned verl -commit. See [scale-out materialization](scaleout-materialization.md). - -## Measured runtime evidence - -The v0.9 RTX 4080 developer workload consumed 32 distinct prompts and completed -8 current-policy updates at 3.1914 GiB peak reserved VRAM. Median steady-state -rollout, teacher-scoring and update times were 9.7200, 0.4864 and 2.3260 seconds; -a matched interruption/resume reproduced byte-identical trajectories, adapter -and optimizer tensors. No quality endpoint or method comparison ran. +```bash +miniverl run --profile verl-opd-v0.8-single-gpu-v1 --plan plan.json \ + --output runs --run-id my-opd +miniverl inspect runs/my-opd/trajectories.jsonl +miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout +miniverl bridge doctor scaleout --json +``` -[Measured workload evidence](verl-opd-reference-workload.md){ .md-button } +The `[train,cuda]` extra supplies the training and quantization stack. Choose +the matching CUDA PyTorch wheel separately; the +[single-GPU installation guide](single-gpu-guide.md) includes flexible and +maintainer-measured setups. -## Choose a path +## Three ways to use miniVERL
-## Run OPD locally +### Run local OPD -Plan, execute, inspect and resume one local pure-OPD run. +Compile a pinned direct-GKD or sampled-k1 profile, execute it in local phases, +and inspect strict current-policy trajectories. ```bash miniverl plan --profile verl-opd-v0.8-single-gpu-v1 --config verl-opd.yaml ``` -**Artifact:** a checksummed local execution plan, trajectories and PEFT adapter. +**Artifact:** immutable plan, trajectories, teacher cache and PEFT adapter. -**Next:** [Plan and run](opd-quickstart.md) +**Next:** [OPD quickstart](opd-quickstart.md)
-## Bring a verl config +### Bring verl-shaped inputs -Import the resolved, documented OPD subset with field-by-field classifications. +Keep familiar field names and structured Parquet while receiving a +field-by-field account of each local effect. ```bash -miniverl import-verl --profile verl-opd-v0.8-single-gpu-v1 \ - --config verl-opd.yaml --out local-opd.yaml +miniverl compat check --profile verl-opd-v0.8-single-gpu-v1 \ + --config verl-opd.yaml ``` -**Artifact:** a round-trippable profile and `local-opd.import-report.json`. +**Artifact:** resolved compatibility matrix and local execution plan. -**Next:** [Supported field boundary](compatibility.md) +**Next:** [For verl users](for-verl-users.md)
-## Move data and artifacts +### Prepare scale-out artifacts -Convert Parquet, export standard artifacts and inspect the unsupported boundary. +Package the local adapter, source Parquet, resolved config and provenance, then +materialize exact model snapshots against the pinned upstream source. ```bash miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout ``` -**Artifact:** PEFT + Parquet + OPD overrides and separate readiness flags. +**Artifact:** checksummed PEFT + Parquet + config bundle with readiness states. -**Next:** [current scale-out contract](verl-opd-scaleout.md) +**Next:** [Scale-out contract](verl-opd-scaleout.md)
-## Research Notes +## Runtime design -The v0.7 external study stopped at its first preregistered gate: **0 selected -checkpoints, 0 qualified teachers, 0 continuation arms and 0 final-test tasks -accessed**. All eight candidates scored 0/64 retained JSONNav utility against -the unchanged 20% floor. +The local scheduler keeps actor, teacher and optional reference roles explicit +while choosing resident, swap or shared-backbone placement. Structured token +provenance, pickle-free caches and transactional publication make a run +inspectable and resumable across phase boundaries. -```bash -miniverl pilot --builtin-study alignment-external-v1 --json -``` +Two measured profiles pin official verl `v0.8.0` at `7aed6b23`: -[Read the early-stop study](alignment-external/alignment-external-v1.md){ .md-button .md-button--primary } +| Profile | Objective | Teacher signal | +| --- | --- | --- | +| direct GKD | `forward_kl_topk` | top-k token IDs and log-probabilities | +| sampled-k1 PG | sampled `k1` + vanilla policy loss | sampled-token teacher log-probability | -Alignment Lab starts from a saturated SFT checkpoint. No continuation method -improves it; measured regressions and unexecuted external safety endpoints stay -visible. RecoveryBench and the calculator study likewise preserve their -negative and mixed results rather than turning them into product claims. +[Compatibility profiles](profiles/index.md) describes the exact field and +semantic contract. [Hardware planning](hardware-planning.md) explains how the +same device-name-agnostic CUDA path adapts to different VRAM budgets. + +## Measured runtime evidence + +The RTX 4080 Qwen3 developer workload consumed 32 distinct prompts and +completed eight current-policy updates at 3.1914 GiB peak reserved VRAM. A +matched interruption/resume reproduced byte-identical trajectories, adapter +and optimizer tensors. The companion SmolLM2 workload completed the same run +shape at 1.4961 GiB peak reserved VRAM. + +[Qwen3 workload](verl-opd-reference-workload.md){ .md-button } +[SmolLM2 workload](smollm2-opd-workload.md){ .md-button } + +## Research record + +The studies section preserves the actual outcome of each scoped experiment. +It includes a protocol-qualified teacher that tied supervised continuation, a +negative RecoveryBench result, regressions from a saturated Alignment Lab +starting point, and a preregistered external-alignment early stop. -- [Alignment Lab](alignment-lab/alignment-lab-v1.md) -- [RecoveryBench](recoverybench/recoverybench-v1.md) - [Calculator protocol study](benchmarking.md) +- [RecoveryBench](recoverybench/recoverybench-v1.md) +- [Alignment Lab](alignment-lab/alignment-lab-v1.md) - [External Alignment Gate](alignment-external/alignment-external-v1.md) + +## Scope and boundaries + +miniVERL's execution scope is one local process, one NVIDIA CUDA GPU and its +versioned profiles. Its scale-out path ends in a validated artifact handoff. +Read [compatibility](compatibility.md) for field and handoff semantics, and +[limitations](limitations.md) for the consolidated architecture, measurement, +security and generalization boundaries. diff --git a/docs/limitations.md b/docs/limitations.md index d2cdc84..9620319 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -1,10 +1,9 @@ # Limitations -This page is the list of reasons not to trust miniVERL beyond what it has -actually been shown to do. It is deliberately longer than the feature list. - -Everything below is traceable to a file in this repository or to a command that -was run against it. Measurements come from one machine: Windows 11 Pro +This page collects miniVERL's scientific, runtime, compatibility and security +boundaries so the product guides can stay focused on workflows. Each item is +traceable to a repository file or an executed command. Measurements come from +one machine: Windows 11 Pro 10.0.22631, RTX 4080 (16376 MiB, driver 596.49), CPython 3.12.13, torch 2.13.0+cu130, transformers 5.14.1, peft 0.19.1, bitsandbytes 0.50.0. The GPU figures include the published calculator runs, hardware probes and @@ -183,7 +182,7 @@ automatically. Sequence length, batch size, models and objective are never changed behind your back, so a run that does not fit will fail rather than quietly train something different. -## Evidence quality +## Evidence scope ### The toy backend is a machinery harness, not a capability result @@ -425,7 +424,7 @@ Models with tied embeddings, weight-sharing tricks, or a non-`Linear` output projection have not been exercised beyond the pinned Qwen3 pair (which does have `tie_word_embeddings: true`) and a tiny offline `Qwen3ForCausalLM` fixture. -### Things this project does not contain +### Execution boundary miniVERL has no distributed training, multi-GPU or multi-node execution, high-throughput rollout engine, PPO/GRPO objective, trained reward model, @@ -439,7 +438,7 @@ node whitelist instead of calling `eval`, and the SQLite environment uses a `sqlite3` authorizer plus a function whitelist and permits one statement per call. -### The verified verl bridge does not execute verl +### Verl bridge boundary The v0.6 **miniVERL-defined compatibility Level 3** bridge targets official verl `v0.8.0` at one exact commit and one named profile. It validates standard @@ -463,7 +462,7 @@ See the [current OPD runtime](verl-opd-runtime.md), [scale-out contract](verl-op implement `privileged_context()`. All three built-in environments do; `miniverl validate` warns if you point it at one that does not. -### Portable redaction is not a secret store +### Portable redaction boundary Shareable views structurally redact credential-like keys, URL userinfo and embedded absolute paths across the supported report and export formats. That is @@ -506,10 +505,9 @@ commit identity, date and opt-in GPU/network status live in the single machine-readable [release-quality record](generated/quality.json), so this page does not drift when tests are added. -## Roadmap (not implemented) +## Roadmap -Nothing in this section exists in the code. It is recorded here so that the -sections above cannot be read as implying it does. +These are future design directions rather than current runtime surfaces. - **Cross-tokenizer distillation.** Referenced as a roadmap item by the error hints in `models/factory.py` and `trajectory/alignment.py`. Would require a diff --git a/docs/maintainer-architecture.md b/docs/maintainer-architecture.md index 6501afd..19e76b8 100644 --- a/docs/maintainer-architecture.md +++ b/docs/maintainer-architecture.md @@ -1,6 +1,6 @@ # Maintainer architecture -This page describes change boundaries, not product marketing. +This page maps the change boundaries maintainers use for review. ## Config to runtime @@ -70,4 +70,3 @@ python scripts/release_gate.py --qualification path/to/qualification.json The first command is CPU CI. Network tests verify pinned remote resources. GPU tests exercise local CUDA behavior. The qualification workflow additionally uses a built wheel and the exact-SHA release contract. - diff --git a/docs/opd-quickstart.md b/docs/opd-quickstart.md index 1b634f0..4fac3e3 100644 --- a/docs/opd-quickstart.md +++ b/docs/opd-quickstart.md @@ -1,9 +1,8 @@ # Run verl-style OPD locally -miniVERL v0.9 supports one pinned, fail-closed subset of verl v0.8: one actor, -one teacher, one generation per prompt, reward-free direct GKD with -`forward_kl_topk`, token-mean aggregation and a LoRA/QLoRA student. It is local -single-GPU execution, not Ray/FSDP or distributed verl execution. +miniVERL's direct-GKD profile brings a pinned verl v0.8 field subset into one +local CUDA process: one actor, one teacher, one generation per prompt, +`forward_kl_topk`, token-mean aggregation and a LoRA/QLoRA student. Install a CUDA build of PyTorch that matches your machine first, then: @@ -30,15 +29,26 @@ prompts, 64 response tokens and eight optimizer updates. See its | GPU | student / teacher | strategy | limits / top-k | peak reserved | first update | status | | --- | --- | --- | --- | ---: | ---: | --- | | RTX 4080 16 GiB | Qwen3-0.6B / Qwen3-1.7B, both NF4 | dual resident | 128 + 16 tokens / 32 | 3.176 GiB | 12.02 s | measured | -| 12 GiB CUDA GPU | same built-in recipe | planner-selected | same | — | — | not measured | -| 24 GiB CUDA GPU | same built-in recipe | planner-selected | same | — | — | not measured | The measured run completed one current-policy rollout/scoring/update cycle, exported a loadable PEFT adapter, and used one RTX 4080. It demonstrates -runtime and artifact correctness only; it did not evaluate alignment quality. +runtime and artifact correctness only; task-quality comparisons live in the +research reports. The checksummed record is [`rtx4080-verl-opd-runtime-v1.json`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/rtx4080-verl-opd-runtime-v1.json). +
+Evidence status for other VRAM classes + +| GPU | Recipe path | Evidence status | +| --- | --- | --- | +| 12 GiB CUDA GPU | same built-in recipe, planner-selected placement | not measured | +| 24 GiB CUDA GPU | same built-in recipe, planner-selected placement | not measured | + +Use the hardware record schema to turn either row into measured evidence. + +
+ ## What `plan` means `plan` reports estimates separately from measurements. Auto placement uses @@ -48,9 +58,10 @@ bitsandbytes weights cannot legally swap. `plan --probe` performs a bounded, cached CUDA calibration with zero optimizer updates; normal planning remains weight-free. -Unsupported settings—including policy-gradient OPD, task rewards, reference -KL, multiple teachers, multiple generations, multimodal inputs and every -distributed dimension—are rejected rather than reinterpreted. +The selected profile fixes the algorithm and role shape before the planner +chooses physical placement. See [compatibility profiles](profiles/index.md) for +the field matrix and [limitations](limitations.md) for the broader algorithm +and execution boundary. ## Import and export the same bounded profile @@ -74,5 +85,6 @@ The bundle stays `launchable: false` until the exact student and teacher base snapshots are materialized. A local teacher adapter adds an explicit merge requirement because the pinned upstream profile does not consume that adapter path directly. Materialization requires the exact installed verl pin and emits -`launch.sh` only after model/tokenizer/data/config checks pass. The bridge never -claims a distributed verl job ran. See the [materialization contract](scaleout-materialization.md). +`launch.sh` after model/tokenizer/data/config checks pass. Readiness remains +split into artifact, materialization, launch and execution states; see the +[materialization contract](scaleout-materialization.md). diff --git a/docs/profiles/index.md b/docs/profiles/index.md index 0ba0718..58a499a 100644 --- a/docs/profiles/index.md +++ b/docs/profiles/index.md @@ -1,9 +1,8 @@ # Compatibility profiles -A miniVERL compatibility profile is a closed, versioned contract—not a claim -that arbitrary verl YAML works locally. Each built-in profile binds its accepted -schema and field rules to the upstream repository, tag and commit, plus explicit -native-compiler, loss-conformance and export versions. +A miniVERL compatibility profile is a closed, versioned contract that binds an +accepted schema and its field rules to one upstream repository, tag and commit. +Native-compiler, loss-conformance and export versions travel with that identity. ```bash miniverl profiles list @@ -12,7 +11,7 @@ miniverl profiles schema verl-opd-v0.8-single-gpu-v1 --json ``` `profiles show` includes a copyable resolved YAML example and override command. -No entry points or third-party profile code are loaded. +The packaged registry is data-only, so profile inspection stays deterministic. ## Check a resolved profile @@ -27,17 +26,16 @@ miniverl compat check \ --accept-local-reinterpretations --json ``` -The report distinguishes the supported algorithm, accepted and effective -fields, local reinterpretations, informational fields, unsupported fields and -profiles that do not apply. A field being parsed is not evidence that it changes -the native runtime. +The report distinguishes the selected algorithm, effective fields, local +reinterpretations, informational fields and out-of-profile values. Each +effective field is also covered by mutation-based field-effect evidence. ## Independent artifact identity New immutable plans, native run manifests, teacher caches, checkpoints and scale-out reports carry the complete profile identity. Changing any version -axis produces a new digest. Existing profile names cannot silently acquire new -semantics; a later algorithm path is registered under a different name. +axis produces a new digest, and a later algorithm path receives its own profile +name. ## Which profile should I use? @@ -47,9 +45,9 @@ semantics; a later algorithm path is registered under a different name. | `verl-opd-v0.8-single-gpu-pg-k1-v1` | sampled `k1` + vanilla policy loss | sampled-token teacher log-probability | sampled-token signal; smaller target artifact | measured | Both are reward-free, strict current-policy, one-actor/one-teacher profiles on -one CUDA GPU. The PG profile is not PPO: it uses a policy-loss form with a -detached distillation-derived advantage and has no task reward, critic, -reference KL or replay. Neither profile is claimed to be more accurate or -better aligned. See [For verl users](../for-verl-users.md), the +one CUDA GPU. The PG profile uses a detached distillation-derived advantage +with the pinned vanilla policy-loss form. The measured records compare runtime +and semantic conformance; task-quality questions belong to a declared benchmark. +See [For verl users](../for-verl-users.md), the [PG-k1 ADR](../adr/0010-verl-v0.8-pg-k1-contract.md), and the runtime evidence for the measured boundary. diff --git a/docs/repository-settings.md b/docs/repository-settings.md index a7be759..1c19cf1 100644 --- a/docs/repository-settings.md +++ b/docs/repository-settings.md @@ -1,7 +1,7 @@ # Repository settings audit -This is a maintainer checklist, not a claim about settings that code cannot -inspect. +This maintainer checklist records the intended repository policy; verify each +item in GitHub settings during an audit. - Default branch: `main`. - Require a pull request and the current CPU/build/docs/pinned-verl checks. diff --git a/docs/scaleout-materialization.md b/docs/scaleout-materialization.md index fd771a3..5bbc6cb 100644 --- a/docs/scaleout-materialization.md +++ b/docs/scaleout-materialization.md @@ -1,9 +1,9 @@ # Materialize a pinned verl handoff -`export-verl` intentionally starts with identity-only student and teacher base -models. Its PEFT adapter, Parquet data, pure-OPD override and provenance are -complete, but `launchable` remains `false` until the exact model snapshots and -the pinned upstream validation are present. +`export-verl` first publishes the PEFT adapter, Parquet data, pure-OPD override, +provenance and exact student/teacher identities. Materialization then resolves +the model snapshots and runs the pinned upstream validation required for +`launchable: true`. ## Offline-first workflow diff --git a/docs/security-review-v0.10.1.md b/docs/security-review-v0.10.1.md index b70e425..a0d0926 100644 --- a/docs/security-review-v0.10.1.md +++ b/docs/security-review-v0.10.1.md @@ -1,8 +1,8 @@ # v0.10.1 focused security review This review covers input and artifact boundaries touched by the current local -runtime and release qualification. It does not claim sandboxing of model code -or third-party libraries. +runtime and release qualification. Model-code and third-party-library isolation +remain in the consolidated [execution limitations](limitations.md#execution-boundary). | surface | enforced boundary | regression evidence | | --- | --- | --- | diff --git a/docs/single-gpu-guide.md b/docs/single-gpu-guide.md index 8a97bfe..226c037 100644 --- a/docs/single-gpu-guide.md +++ b/docs/single-gpu-guide.md @@ -1,11 +1,11 @@ # Bring your own GPU -miniVERL is a **single-GPU CUDA LLM post-training** stack. It has no GPU model -allowlist and no multi-GPU launcher: one process uses one CUDA device, while -the model pair and sequence budget determine whether a recipe fits. +miniVERL is a **single-GPU CUDA LLM post-training** stack. One process uses one +CUDA device, and the model pair, sequence budget and runtime strategy determine +how the recipe fits. -The shipped Qwen3 recipe is measured on an RTX 4080, but it is not coded for an -RTX 4080. Its portable defaults are: +The shipped Qwen3 recipe follows a device-name-agnostic CUDA path and has a +measured RTX 4080 reference. Its portable defaults are: - `models.device: auto` selects CUDA when PyTorch can use it; - `dtype: auto` selects bfloat16 when the device supports it and float16 @@ -14,9 +14,9 @@ RTX 4080. Its portable defaults are: - `train.trajectory_batch_size: 1` is the conservative compatibility default; `2` or `4` can improve update throughput when measured headroom permits; - `memory.strategy: auto` resolves the supported resident/swap policy; -- an out-of-memory error may reduce only the vocabulary-loss chunk size and - retry the gradient phase. It never silently changes the objective, model, - rollout length, or optimizer update. +- an out-of-memory retry reduces the mathematically neutral vocabulary-loss + chunk size while keeping the objective, model, rollout length and optimizer + update fixed. ## Start here @@ -48,10 +48,10 @@ python scripts/check_known_good_environment.py ``` The ordinary dependency ranges remain the library contract. In the manifest, -Python and package pins are required reproducibility inputs; GPU name, VRAM, -driver and CUDA runtime are observed audit fields, not install requirements. -This is one maintainer-measured known-good stack, not a universal lock: other -GPUs are unmeasured, and other CUDA/PyTorch combinations remain user-selected. +Python and package pins are reproducibility inputs; GPU name, VRAM, driver and +CUDA runtime record the measured machine. In the schema, the driver and CUDA +runtime are observed audit fields; other GPUs are unmeasured by the maintained +reference and can be recorded with the same schema. Then watch free memory in another terminal before the real run: @@ -62,32 +62,31 @@ miniverl train recipes/qwen_consumer_gpu_calc.yaml `doctor` reports the device, CUDA availability and optional dependencies. `--dry-run` validates model identity, tokenizer compatibility, adapter -provenance and the resolved configuration. It does not prove that every phase -will fit; only an actual run can do that. +provenance and the resolved configuration. `plan --probe` adds a bounded +measurement before the full run establishes end-to-end fit. The default calculator recipe owns separate 0.6B student and 1.7B teacher models, so it uses `models.runtime: dual_model`. When student and teacher use the same base revision, `models.runtime: shared_backbone` can instead keep one base with separate adapters. The shipped [`qwen_consumer_gpu_shared.yaml`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/recipes/qwen_consumer_gpu_shared.yaml) -demonstrates the wiring and batch-4 update path; its frozen teacher is a systems -artifact, not a newly quality-qualified recipe. See the -[measured runtime report](consumer-runtime-v1.md) before adapting it. +demonstrates the wiring and batch-4 update path. See the +[measured runtime report](consumer-runtime-v1.md) for its systems evidence and +teacher provenance before adapting it. ## What different cards change -These are starting points, not benchmark claims: +Use these as starting points and record the resulting plan: | Your card | Sensible first move | Evidence status | | --- | --- | --- | -| 8–12 GiB, such as an RTX 3070 or Titan V | Try the shipped pair, but be ready to shorten token budgets or select a smaller teacher. Keep `dtype: auto`; Titan V-class hardware resolves to fp16 because it has no bf16 path. | Supported by the device-name-agnostic CUDA path; not measured in this repository | +| 8–12 GiB, such as an RTX 3070 or Titan V | Try the shipped pair, then shorten token budgets or select a smaller teacher if the probe requires it. Keep `dtype: auto`; Titan V-class hardware resolves to fp16. | Portable CUDA path; contribute a measured record | | 16–24 GiB | Start with the shipped recipe unchanged. Increase budgets only after recording a successful baseline. | The exact default pair is measured on one RTX 4080 16 GiB | -| 24–32+ GiB, such as RTX 3090/4090/5090-class cards | Use the same recipe first; extra headroom can support longer contexts, larger models, or less quantization. Change one variable at a time. | Expected from the same single-device path; not measured here | +| 24–32+ GiB, such as RTX 3090/4090/5090-class cards | Use the same recipe first; extra headroom can support longer contexts, larger models, or less quantization. Change one variable at a time. | Portable CUDA path; contribute a measured record | -The default measured run peaked below 5 GiB of CUDA allocated/reserved memory, -but allocator behavior, kernels, driver versions and generation length vary. -That number is evidence from one machine, **not** a promise that every 8 GiB -card or software stack will complete. +The default measured run peaked below 5 GiB of CUDA allocated/reserved memory. +Allocator behavior, kernels, driver versions and generation length all affect +the headroom on another machine, which is why the plan records them. ## If the recipe does not fit @@ -106,10 +105,9 @@ Work down this list and preserve the resulting YAML with the run: students remain resident because moving bitsandbytes modules between devices is not a supported lifecycle. -Do not infer correctness from a falling loss alone. Run the strict held-out -verifier and inspect the terminal manifest. The published negative controls -are included precisely because an incompatible teacher can optimize normally -while producing a 0% tool policy. +Use the strict held-out verifier and terminal manifest alongside the loss. The +published negative controls show why task behavior and teacher protocol +competence belong in the same review. ## Share a hardware result @@ -122,5 +120,4 @@ miniverl export-benchmark runs/ --notes "GPU, VRAM, driver, torch and CU Validate the exported JSON and open a pull request following [`benchmarks/README.md`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/README.md). Results from cards other than -the measured RTX 4080 are welcome; until they exist, the README labels those -cards as portable code paths rather than measured performance. +the measured RTX 4080 are welcome and extend the public hardware matrix. diff --git a/docs/social-preview-v0.6.svg b/docs/social-preview-v0.6.svg index 8ef77ff..3a5ea70 100644 --- a/docs/social-preview-v0.6.svg +++ b/docs/social-preview-v0.6.svg @@ -58,6 +58,6 @@ Parquet round trip checksummed bundle - independent project · no endorsement implied + typed profiles · portable artifacts · explicit readiness diff --git a/docs/upstream-support-policy.md b/docs/upstream-support-policy.md index 97e87b3..a7ffeff 100644 --- a/docs/upstream-support-policy.md +++ b/docs/upstream-support-policy.md @@ -1,7 +1,7 @@ # Upstream verl support policy -miniVERL supports closed compatibility profiles, not “latest verl” and not -arbitrary upstream configuration. The machine-readable matrix is +miniVERL supports versioned compatibility profiles tied to exact upstream +source states. The machine-readable matrix is [`generated/upstream-support-matrix.json`](generated/upstream-support-matrix.json). ## Lifecycle @@ -26,4 +26,3 @@ The active direct-GKD and sampled-k1 profiles both target official verl `v0.8.0` at `7aed6b230776f963fa09509c10d9c3a767d1102c`. No v0.9-or-newer profile is currently supported. Existing bridge artifacts remain independent-project handoffs and do not imply upstream endorsement. - diff --git a/docs/v1-readiness.md b/docs/v1-readiness.md index d15ab1f..b2b8a9b 100644 --- a/docs/v1-readiness.md +++ b/docs/v1-readiness.md @@ -1,8 +1,8 @@ # v1 readiness contract -Version 1 is a stability promise, not a feature-count milestone. A successful -release qualification is necessary evidence, but it does not by itself freeze -the public surface or make miniVERL a v1 candidate. +Version 1 marks a reviewed stability promise across the public surface. Release +qualification supplies the binary evidence; the contracts below define the +additional API, artifact and maintenance commitments. ## Stability contract diff --git a/docs/verl-bridge-demo.md b/docs/verl-bridge-demo.md index d503781..12ab339 100644 --- a/docs/verl-bridge-demo.md +++ b/docs/verl-bridge-demo.md @@ -1,7 +1,7 @@ # Verified-bridge demo recording script -This recording is an artifact-only CPU demo. It needs the `bridge` extra but -does not need CUDA, model downloads, Ray or a verl installation. +This CPU recording follows the artifact path with the `bridge` extra. It runs +from local fixtures and produces the same reports without a training setup. ## Before recording diff --git a/docs/verl-local-runtime-mobile.svg b/docs/verl-local-runtime-mobile.svg index ae37762..200cbe3 100644 --- a/docs/verl-local-runtime-mobile.svg +++ b/docs/verl-local-runtime-mobile.svg @@ -1,10 +1,10 @@ miniVERL local OPD product flow, mobile layout - A vertical view of typed verl-shaped inputs, local one-GPU OPD phases, inspectable artifacts and the bounded export to pinned verl. Distributed execution is outside miniVERL. + A vertical view of typed verl-shaped inputs, local one-GPU OPD phases, inspectable artifacts and a portable export bundle with explicit readiness states. - One-GPU OPD workflowtyped · inspectable · fail-closed + One-GPU OPD workflowtyped · inspectable · resumable INPUTSverl-shaped YAML + overridesParquet prompt data @@ -14,6 +14,6 @@ actor rolloutteacher scoringactor update INSPECTABLE OUTPUTScheckpoint · PEFT · targetstrajectories · metrics · provenance materialized export bundle→ pinned verl v0.8.0 - OUTSIDE miniVERLdistributed execution + READINESS REPORTbundle · launch · execution diff --git a/docs/verl-local-runtime.svg b/docs/verl-local-runtime.svg index 0c7301b..56787c9 100644 --- a/docs/verl-local-runtime.svg +++ b/docs/verl-local-runtime.svg @@ -1,6 +1,6 @@ miniVERL local OPD product flow - verl-shaped configuration and Parquet prompts enter a typed compiler, then one GPU runs actor rollout, teacher scoring, and actor update. Inspectable artifacts can be exported to pinned verl; distributed execution remains outside miniVERL. + verl-shaped configuration and Parquet prompts enter a typed compiler, then one GPU runs actor rollout, teacher scoring, and actor update. Inspectable artifacts form a portable bundle with explicit readiness states. @@ -10,7 +10,7 @@ One-GPU OPD, from familiar inputs to portable artifacts - Every boundary is typed, recorded, and fail-closed. + Every boundary is typed, recorded, and resumable. @@ -42,8 +42,8 @@ materialized export bundle → pinned verl v0.8.0 - - OUTSIDE miniVERL - distributed execution + + READINESS REPORT + bundle · launch · execution diff --git a/docs/verl-opd-runtime.md b/docs/verl-opd-runtime.md index 7ad7592..1ac0040 100644 --- a/docs/verl-opd-runtime.md +++ b/docs/verl-opd-runtime.md @@ -1,9 +1,8 @@ # Current verl-style OPD runtime -This is miniVERL's current executable path: two documented, resolved subsets of -official verl `v0.8.0`, pinned at `7aed6b23`, compiled into local phases on one -NVIDIA CUDA GPU. It is not a distributed verl runtime and does not accept -arbitrary Hydra YAML. +This is miniVERL's current executable path: two documented, resolved profiles +from official verl `v0.8.0`, pinned at `7aed6b23`, compiled into local phases +on one NVIDIA CUDA GPU. ## Start from the pinned profile @@ -35,26 +34,27 @@ reinterpretations; the built-in profile has a value-bound approval manifest. | `lora_adapter_path` plus pinned metadata | validated trainable student initialization | `forward_kl_topk` consumes teacher top-k token IDs and log-probabilities and -reports top-k mass/overlap diagnostics. It does not create the explicit K+1 -tail bucket used by miniVERL's separate native `bucketed_topk_tail` objective. +reports top-k mass/overlap diagnostics. miniVERL's separate native +`bucketed_topk_tail` objective adds an explicit K+1 tail bucket. The second profile, `verl-opd-v0.8-single-gpu-pg-k1-v1`, records the sampled token, rollout-time actor log-probability and teacher log-probability, then recomputes the current actor log-probability for pinned `k1` + vanilla -policy-loss semantics. It exports no top-k requirement. See [Which profile -should I use?](profiles/index.md) for the neutral choice boundary. +policy-loss semantics. Its export carries sampled-token signals in place of a +top-k target requirement. See [Which profile should I use?](profiles/index.md) +for the choice guide. -## Placement is fail-closed +## Placement strategy -- NF4/int8 roles use resident local phases; bitsandbytes parameters cannot swap. +- NF4/int8 roles use resident local phases. - Swap is available only for movable unquantized LoRA roles. - Shared backbone requires compatible same-base roles. -- Unknown-size quantized roles report `requires_probe` and are not executable - until feasibility is proven. +- Unknown-size quantized roles use `requires_probe` until a bounded measurement + establishes feasibility. Normal planning is weight-free. `plan --probe` is a bounded, cached CUDA -calibration that performs zero optimizer updates. An executable plan must not -select a placement the native runtime rejects for a known static reason. +calibration with zero optimizer updates. The finalized plan therefore contains +either a statically valid placement or the measurement needed to validate it. The generated [compatibility matrix](generated/verl-opd-v0.8-compatibility.json) and [field-effect evidence](generated/verl-opd-v0.8-field-effects.json) are diff --git a/docs/verl-opd-scaleout.md b/docs/verl-opd-scaleout.md index aa7cbe9..ab430f8 100644 --- a/docs/verl-opd-scaleout.md +++ b/docs/verl-opd-scaleout.md @@ -2,8 +2,8 @@ A completed direct-GKD or sampled-k1 PG profile run can export portable PEFT, Parquet, config and provenance artifacts for the exact pinned upstream profile. -This is an artifact handoff, not proof that a distributed job ran or that -miniVERL and verl have algorithmic parity. +The handoff advances through explicit bundle, materialization, launch and +execution states. ```bash miniverl export-verl --run runs/my-opd --target-verl v0.8.0 --out scaleout @@ -25,13 +25,14 @@ validation data remains missing rather than being replaced with training data. | upstream config parse passed | the exact pinned source parsed the bounded overrides | | model/data load smoke passed | local load checks completed for materialized inputs | | launchable | all required local inputs exist and the fail-closed checks passed | -| distributed execution tested | always false in the current evidence | -| algorithm semantic parity | not claimed | +| distributed execution tested | whether the recorded evidence includes the upstream distributed job | +| algorithm semantic parity | whether a separately declared parity study exists | -Fresh identity-only exports remain `launchable: false`. Materialization resolves +Fresh identity-only exports begin before model snapshots are materialized. +Materialization resolves the exact base snapshots, validates tokenizer/model/adapter/Parquet closure and -publishes a checksummed `launch.sh` transactionally. Even `launchable: true` -does not say Ray, FSDP, vLLM or a distributed optimizer step executed. +publishes a checksummed `launch.sh` transactionally. The later execution and +semantic-parity fields keep those evidence stages distinct from launchability. The current pure-OPD export has no reward scaffold. The older `single-gpu-online-distillation-v1` environment/PPO scaffold is retained only diff --git a/mkdocs.yml b/mkdocs.yml index cfb8d6d..7cbfbcc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,5 @@ site_name: miniVERL -site_description: Run a documented subset of verl-style OPD on one consumer GPU +site_description: Inspectable verl-style on-policy distillation on one NVIDIA GPU site_url: https://daoyuanli2816.github.io/mini-verl/ repo_url: https://github.com/DaoyuanLi2816/mini-verl repo_name: DaoyuanLi2816/mini-verl @@ -41,6 +41,7 @@ plugins: - search nav: - Home: index.md + - Choose a stack: comparisons.md - For verl users: for-verl-users.md - Compatibility profiles: profiles/index.md - PG-k1 semantic ADR: adr/0010-verl-v0.8-pg-k1-contract.md @@ -95,4 +96,4 @@ extra_css: - assets/stylesheets/extra.css extra_javascript: - assets/javascripts/versioning.js -copyright: miniVERL is an independent project; no endorsement by upstream projects is implied. +copyright: miniVERL · inspectable single-GPU post-training · Apache-2.0 diff --git a/pyproject.toml b/pyproject.toml index fe18d75..72bbc4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "hatchling.build" [project] name = "miniverl" dynamic = ["version"] -description = "Run a documented subset of verl-style OPD on one consumer GPU." +description = "Inspectable verl-style on-policy distillation on one NVIDIA GPU." readme = "PYPI.md" requires-python = ">=3.10" license = { text = "Apache-2.0" } diff --git a/tests/unit/test_documentation_voice.py b/tests/unit/test_documentation_voice.py new file mode 100644 index 0000000..2a44d37 --- /dev/null +++ b/tests/unit/test_documentation_voice.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def _read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_product_entry_points_lead_with_workflows() -> None: + entry_points = "\n".join( + _read(path) + for path in ( + "README.md", + "README.zh-CN.md", + "docs/index.md", + "docs/comparisons.md", + ) + ) + for old_framing in ( + "For most serious distillation work it is", + "What miniVERL is not", + "reasons not to trust miniVERL", + "full verl compatibility are not claimed", + "不执行任意 verl YAML", + ): + assert old_framing not in entry_points + + assert "## What a run gives you" in entry_points + assert "## Choose your path" in entry_points + assert "## 一次运行会得到什么" in entry_points + assert "## 选择你的路径" in entry_points + + +def test_complete_boundaries_have_canonical_destinations() -> None: + for path in ("README.md", "README.zh-CN.md", "docs/index.md", "docs/comparisons.md"): + text = _read(path) + assert "limitations" in text + assert "compatibility" in text or "兼容" in text + + limitations = _read("docs/limitations.md") + assert "one machine" in limitations + assert "multi-GPU" in limitations + assert "PPO/GRPO" in limitations + assert "## Evidence scope" in limitations + assert "### Execution boundary" in limitations + + +def test_main_runtime_visual_describes_the_product_flow() -> None: + visuals = _read("docs/verl-local-runtime.svg") + _read("docs/verl-local-runtime-mobile.svg") + assert "READINESS REPORT" in visuals + assert "typed · inspectable · resumable" in visuals + assert "OUTSIDE miniVERL" not in visuals + assert "fail-closed" not in visuals