From b5b5b3d29bc80cd36416952788bc82bdac1810bd Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Sat, 29 Aug 2026 07:37:14 -0700 Subject: [PATCH 1/6] Add the selected-backend benchmark harness --- .../benchmark_rollout_runtime_v2_selected.py | 618 ++++++++++++++++++ 1 file changed, 618 insertions(+) create mode 100644 scripts/benchmark_rollout_runtime_v2_selected.py diff --git a/scripts/benchmark_rollout_runtime_v2_selected.py b/scripts/benchmark_rollout_runtime_v2_selected.py new file mode 100644 index 0000000..ab5a411 --- /dev/null +++ b/scripts/benchmark_rollout_runtime_v2_selected.py @@ -0,0 +1,618 @@ +#!/usr/bin/env python3 +"""Measure the selected Rollout Runtime v2 backends on the frozen token workload. + +This is a rollout-only qualification. It runs one backend per process so CUDA +ownership and teardown remain observable. The output is raw machine evidence; +``publish_rollout_runtime_v2_evidence.py`` combines completed runs without +changing their measured fields. +""" + +from __future__ import annotations + +import argparse +import gc +import hashlib +import importlib.metadata +import json +import platform +import socket +import statistics +import subprocess +import threading +import time +from collections.abc import Sequence +from itertools import pairwise +from pathlib import Path +from typing import Any + +import torch +import yaml + +from miniverl import __version__ +from miniverl.config.models import RolloutEngineConfig, StudentModelConfig +from miniverl.models.hf import HFBackend +from miniverl.models.tokenizers import HFTokenizerAdapter +from miniverl.runtime.backends.hf_cached import HFCachedGenerationBackend +from miniverl.runtime.backends.vllm import VLLMGenerationBackend, parse_vllm_completion +from miniverl.runtime.generation import ( + GenerationRequest, + PolicySnapshot, + RolloutBackendKind, + RolloutGroupIdentity, + SamplingParameters, + derive_sample_seed, +) +from miniverl.runtime.policy_sync import build_rollout_policy_identity +from miniverl.utils.runs import canonical_json, write_json + +ROOT = Path(__file__).resolve().parents[1] +PREREGISTRATION = ROOT / "benchmarks/preregistration/rollout-runtime-v2.yaml" +FROZEN_CALCULATOR = ROOT / "benchmarks/results/gpu-calc-hard-equal-update-v2.json" +FROZEN_CALCULATOR_SHA256 = "53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc" +PROFILE_IDENTITY = hashlib.sha256(b"rollout-runtime-v2-selected-backends-v1").hexdigest() +EXECUTION_PLAN_DIGEST = hashlib.sha256(b"rollout-runtime-v2-measurement-plan-v1").hexdigest() + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def _git(*args: str) -> str: + return subprocess.check_output(["git", *args], cwd=ROOT, text=True).strip() + + +def _package_version(name: str) -> str | None: + try: + return importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + return None + + +def _gpu_used_mib() -> int: + raw = subprocess.check_output( + [ + "nvidia-smi", + "--query-gpu=memory.used", + "--format=csv,noheader,nounits", + ], + text=True, + ) + return int(raw.splitlines()[0].strip()) + + +def _process_max_rss_bytes() -> int | None: + try: + import resource + + usage = resource.getrusage(resource.RUSAGE_SELF) # type: ignore[attr-defined] + return int(usage.ru_maxrss * 1024) + except (ImportError, OSError): + return None + + +class _GpuSampler: + def __init__(self) -> None: + self.peak_mib = _gpu_used_mib() + self._stop = threading.Event() + self._thread = threading.Thread(target=self._run, daemon=True) + + def _run(self) -> None: + while not self._stop.wait(0.05): + try: + self.peak_mib = max(self.peak_mib, _gpu_used_mib()) + except (OSError, subprocess.SubprocessError, ValueError): + pass + + def __enter__(self) -> _GpuSampler: + self._thread.start() + return self + + def __exit__(self, *_: object) -> None: + self._stop.set() + self._thread.join(timeout=2) + self.peak_mib = max(self.peak_mib, _gpu_used_mib()) + + +def _environment() -> dict[str, Any]: + driver = subprocess.check_output( + [ + "nvidia-smi", + "--query-gpu=driver_version", + "--format=csv,noheader,nounits", + ], + text=True, + ).splitlines()[0] + return { + "platform": platform.platform(), + "python": platform.python_version(), + "gpu": torch.cuda.get_device_name(0), + "gpu_total_memory_bytes": int(torch.cuda.get_device_properties(0).total_memory), + "driver_version": driver.strip(), + "cuda_runtime": torch.version.cuda, + "packages": { + name: _package_version(name) + for name in ( + "torch", + "transformers", + "peft", + "bitsandbytes", + "accelerate", + "vllm", + ) + }, + } + + +def _fixed_prompt(tokenizer: Any, *, length: int, index: int) -> tuple[int, ...]: + prefix = list(tokenizer.encode(f"System benchmark prompt {index}. Continue with text.")) + filler = list(tokenizer.encode(" benchmark")) + if not filler: + raise RuntimeError("tokenizer produced no filler tokens") + while len(prefix) < length: + prefix.extend(filler) + prompt = prefix[:length] + if tokenizer.eos_token_id is not None and prompt[-1] == tokenizer.eos_token_id: + prompt[-1] = next(token for token in filler if token != tokenizer.eos_token_id) + return tuple(prompt) + + +def _requests( + tokenizer: Any, + *, + identity: Any, + prompt_length: int, + response_bound: int, + samples_per_prompt: int, + logical_prompts: int, + run_seed: int, +) -> list[GenerationRequest]: + requests: list[GenerationRequest] = [] + for prompt_index in range(logical_prompts): + prompt = _fixed_prompt(tokenizer, length=prompt_length, index=prompt_index) + prompt_digest = hashlib.sha256(canonical_json(prompt).encode("utf-8")).hexdigest() + group_id = f"benchmark-p{prompt_index:02d}-{prompt_digest[:12]}" + for sample_index in range(samples_per_prompt): + group = RolloutGroupIdentity( + prompt_group_id=group_id, + prompt_digest=prompt_digest, + sample_index=sample_index, + samples_per_prompt=samples_per_prompt, + ) + requests.append( + GenerationRequest( + request_id=f"{group_id}-s{sample_index}", + group=group, + deterministic_sample_seed=derive_sample_seed( + run_seed=run_seed, + prompt_digest=prompt_digest, + policy_version=identity.parameter_version, + sample_index=sample_index, + ), + prompt_token_ids=prompt, + max_new_tokens=response_bound, + sampling=SamplingParameters(temperature=0.0, top_p=1.0, top_k=0), + need_sampled_token_logprobs=False, + expected_policy_identity=identity, + ) + ) + return requests + + +def _generate_partitioned(backend: Any, requests: Sequence[GenerationRequest]) -> list[Any]: + results: list[Any] = [] + for start in range(0, len(requests), 4): + results.extend(backend.generate(requests[start : start + 4]).results) + return results + + +def _digest_results(results: Sequence[Any]) -> str: + payload = [list(result.output_token_ids) for result in results] + return hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + + +def _cell( + backend: Any, + tokenizer: Any, + *, + identity: Any, + prompt_length: int, + response_bound: int, + samples_per_prompt: int, + logical_prompts: int, + run_seed: int, + warmups: int, + repetitions: int, +) -> dict[str, Any]: + requests = _requests( + tokenizer, + identity=identity, + prompt_length=prompt_length, + response_bound=response_bound, + samples_per_prompt=samples_per_prompt, + logical_prompts=logical_prompts, + run_seed=run_seed, + ) + for _ in range(warmups): + _generate_partitioned(backend, requests) + torch.cuda.synchronize() + durations: list[float] = [] + digests: list[str] = [] + final: list[Any] = [] + with _GpuSampler() as memory: + for _ in range(repetitions): + started = time.perf_counter() + final = _generate_partitioned(backend, requests) + torch.cuda.synchronize() + durations.append(time.perf_counter() - started) + digests.append(_digest_results(final)) + if len(set(digests)) != 1: + raise RuntimeError("backend output changed across greedy repetitions") + generated_tokens = sum(len(result.output_token_ids) for result in final) + prompt_tokens = sum(len(request.prompt_token_ids) for request in requests) + median = float(statistics.median(durations)) + return { + "cell_id": f"p{prompt_length}-r{response_bound}-n{samples_per_prompt}-greedy", + "prompt_length": prompt_length, + "response_bound": response_bound, + "samples_per_prompt": samples_per_prompt, + "sampling": "greedy", + "logical_prompts": logical_prompts, + "generated_trajectories": len(final), + "prompt_tokens": prompt_tokens, + "generated_tokens": generated_tokens, + "physical_concurrency": 4, + "warmup_repetitions": warmups, + "measured_repetitions": repetitions, + "rollout_seconds": durations, + "median_rollout_seconds": median, + "output_tokens_per_second": generated_tokens / median, + "prompt_tokens_per_second": prompt_tokens / median, + "output_token_ids_sha256": digests[0], + "peak_total_gpu_memory_mib": memory.peak_mib, + } + + +def _conformance_probe( + actor: HFBackend, external: VLLMGenerationBackend, tokenizer: Any +) -> dict[str, Any]: + identity = external._active_identity + adapter_name = external._active_adapter_name + if identity is None or adapter_name is None: + raise RuntimeError("external backend is not synchronized for conformance") + prompt = _fixed_prompt(tokenizer, length=128, index=0) + local = actor.generate( + list(prompt), + max_new_tokens=32, + temperature=0.0, + top_p=1.0, + top_k=0, + seed=20260829, + record_logprobs=True, + ) + raw = external.manager.complete( + { + "model": adapter_name, + "prompt": list(prompt), + "max_tokens": 32, + "temperature": 0.0, + "top_p": 1.0, + "top_k": -1, + "seed": 20260829, + "logprobs": 1, + } + ) + remote = parse_vllm_completion(raw, need_logprobs=True) + compared = min(len(local.token_ids), len(remote.token_ids)) + agreement = sum( + left == right + for left, right in zip(local.token_ids[:compared], remote.token_ids[:compared], strict=True) + ) + differences = [ + abs(left - right) + for left, right in zip(local.logprobs[:compared], remote.logprobs[:compared], strict=True) + ] + ordered = sorted(differences) + p99 = ordered[min(len(ordered) - 1, int(0.99 * len(ordered)))] if ordered else None + return { + "prompt_token_ids_sha256": hashlib.sha256( + canonical_json(prompt).encode("utf-8") + ).hexdigest(), + "compared_output_tokens": compared, + "token_agreement_count": agreement, + "token_agreement_fraction": agreement / compared if compared else None, + "local_output_token_ids_sha256": hashlib.sha256( + canonical_json(local.token_ids).encode("utf-8") + ).hexdigest(), + "engine_output_token_ids_sha256": hashlib.sha256( + canonical_json(remote.token_ids).encode("utf-8") + ).hexdigest(), + "sampled_logprob_max_abs_difference": max(differences) if differences else None, + "sampled_logprob_mean_abs_difference": ( + statistics.fmean(differences) if differences else None + ), + "sampled_logprob_p99_abs_difference": p99, + "pg_nf4_threshold": 0.01, + "pg_threshold_passed": bool(differences) and max(differences) <= 0.01, + } + + +def _refresh_probe( + actor: HFBackend, + backend: VLLMGenerationBackend, + tokenizer: Any, + *, + run_seed: int, +) -> dict[str, Any]: + trainable = [parameter for parameter in actor.model.parameters() if parameter.requires_grad] + if not trainable: + raise RuntimeError("refresh probe requires a trainable adapter") + rows: list[dict[str, Any]] = [] + for version in range(1, 9): + with torch.no_grad(): + trainable[0].view(-1)[0].add_(1e-4) + identity = build_rollout_policy_identity( + backend=actor, + parameter_version=version, + generation_backend=RolloutBackendKind.VLLM, + backend_version=backend.backend_version, + profile_identity=PROFILE_IDENTITY, + execution_plan_digest=EXECUTION_PLAN_DIGEST, + ) + started = time.perf_counter() + synced = backend.synchronize(PolicySnapshot(identity)) + sync_seconds = time.perf_counter() - started + request = _requests( + tokenizer, + identity=identity, + prompt_length=128, + response_bound=8, + samples_per_prompt=1, + logical_prompts=1, + run_seed=run_seed, + ) + result = backend.generate(request).results[0] + rows.append( + { + "policy_version": version, + "policy_identity_digest": identity.digest, + "adapter_tensor_digest": identity.adapter_tensor_digest, + "sync_confirmed": synced.active_policy_digest == identity.digest, + "sync_seconds": sync_seconds, + "output_token_ids_sha256": hashlib.sha256( + canonical_json(result.output_token_ids).encode("utf-8") + ).hexdigest(), + "total_gpu_memory_mib": _gpu_used_mib(), + } + ) + memory = [row["total_gpu_memory_mib"] for row in rows] + return { + "cycles": rows, + "all_policy_identities_unique": len({row["policy_identity_digest"] for row in rows}) + == len(rows), + "all_syncs_confirmed": all(row["sync_confirmed"] for row in rows), + "strictly_monotonic_memory_growth": all(right > left for left, right in pairwise(memory)), + } + + +def run( + *, + backend_name: str, + wheel: Path, + expected_commit: str, + output: Path, + repetitions: int, +) -> dict[str, Any]: + if not torch.cuda.is_available(): + raise RuntimeError("selected-backend benchmark requires CUDA") + if platform.system() != "Linux" or "microsoft" not in platform.release().lower(): + raise RuntimeError("formal selected-backend benchmark requires WSL2/Linux") + if _sha256(FROZEN_CALCULATOR) != FROZEN_CALCULATOR_SHA256: + raise RuntimeError("frozen calculator benchmark hash changed") + if _git("rev-parse", "HEAD") != expected_commit: + raise RuntimeError("source commit does not match --expected-commit") + if _git("status", "--porcelain"): + raise RuntimeError("formal selected-backend benchmark requires a clean worktree") + if not wheel.is_file(): + raise FileNotFoundError(wheel) + + prereg = yaml.safe_load(PREREGISTRATION.read_text(encoding="utf-8")) + actor_spec = prereg["models"]["actor"] + spec = StudentModelConfig( + model_id=actor_spec["id"], + revision=actor_spec["revision"], + tokenizer_revision=actor_spec["tokenizer_revision"], + dtype=prereg["models"]["dtype"], + quantization=prereg["models"]["quantization"], + attn_implementation=prereg["models"]["attention_implementation"], + gradient_checkpointing=False, + ) + baseline_gpu_mib = _gpu_used_mib() + load_started = time.perf_counter() + tokenizer = HFTokenizerAdapter.load( + spec.tokenizer_id or spec.model_id, + revision=spec.tokenizer_revision or spec.revision, + trust_remote_code=spec.trust_remote_code, + local_files_only=True, + ) + actor = HFBackend.load( + spec, + device="cuda", + tokenizer=tokenizer, + trainable=True, + local_files_only=True, + ) + torch.cuda.synchronize() + actor_load_seconds = time.perf_counter() - load_started + if backend_name == "hf_cached": + backend: Any = HFCachedGenerationBackend(actor) + kind = RolloutBackendKind.HF_CACHED + elif backend_name == "vllm": + backend = VLLMGenerationBackend( + actor, + engine_config=RolloutEngineConfig( + managed=True, + host="127.0.0.1", + startup_timeout_seconds=120, + request_timeout_seconds=120, + memory_fraction=0.5, + ), + max_model_len=1024, + ) + kind = RolloutBackendKind.VLLM + else: + raise ValueError(f"unsupported backend {backend_name!r}") + + identity = build_rollout_policy_identity( + backend=actor, + parameter_version=0, + generation_backend=kind, + backend_version=backend.backend_version, + profile_identity=PROFILE_IDENTITY, + execution_plan_digest=EXECUTION_PLAN_DIGEST, + ) + sync_started = time.perf_counter() + sync = backend.synchronize(PolicySnapshot(identity)) + initial_sync_seconds = time.perf_counter() - sync_started + cells = [ + _cell( + backend, + tokenizer, + identity=identity, + prompt_length=128, + response_bound=response_bound, + samples_per_prompt=samples_per_prompt, + logical_prompts=4, + run_seed=int(prereg["workload"]["run_seed"]), + warmups=1, + repetitions=repetitions, + ) + for response_bound in (64, 256, 512) + for samples_per_prompt in (1, 4) + ] + conformance = _conformance_probe(actor, backend, tokenizer) if backend_name == "vllm" else None + refresh = ( + _refresh_probe( + actor, + backend, + tokenizer, + run_seed=int(prereg["workload"]["run_seed"]), + ) + if backend_name == "vllm" + else None + ) + lifecycle = backend.lifecycle_metrics() if backend_name == "vllm" else {} + port = getattr(getattr(backend, "manager", None), "port", None) + teardown_started = time.perf_counter() + backend.close() + actor.release() + del actor + del tokenizer + gc.collect() + torch.cuda.empty_cache() + torch.cuda.synchronize() + teardown_seconds = time.perf_counter() - teardown_started + time.sleep(1) + residual_gpu_mib = _gpu_used_mib() + port_closed = None + if isinstance(port, int): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.settimeout(0.5) + port_closed = probe.connect_ex(("127.0.0.1", port)) != 0 + + payload = { + "schema_version": 1, + "name": f"rollout-runtime-v2-{backend_name}-rtx4080-raw", + "measurement_status": "completed", + "source": { + "commit": expected_commit, + "dirty": False, + "miniverl_version": __version__, + "wheel_sha256": _sha256(wheel), + }, + "preregistration_sha256": _sha256(PREREGISTRATION), + "frozen_calculator_sha256": FROZEN_CALCULATOR_SHA256, + "environment": _environment(), + "models": prereg["models"], + "backend": { + "name": backend_name, + "version": backend.backend_version, + "capabilities": backend.inspect().to_dict(), + }, + "policy_identity": { + "parameter_version": identity.parameter_version, + "adapter_tensor_digest": identity.adapter_tensor_digest, + "identity_digest": identity.digest, + }, + "workload": { + "manifest_version": prereg["workload"]["manifest_version"], + "run_seed": prereg["workload"]["run_seed"], + "logical_prompts": 4, + "prompt_length": 128, + "response_bounds": [64, 256, 512], + "samples_per_prompt": [1, 4], + "sampling": "greedy", + "physical_concurrency": 4, + }, + "timing": { + "actor_load_seconds": actor_load_seconds, + "initial_sync_seconds": initial_sync_seconds, + "engine_lifecycle": lifecycle, + "teardown_seconds": teardown_seconds, + }, + "memory": { + "baseline_total_gpu_memory_mib": baseline_gpu_mib, + "peak_total_gpu_memory_mib": max(cell["peak_total_gpu_memory_mib"] for cell in cells), + "residual_total_gpu_memory_mib": residual_gpu_mib, + "process_max_rss_bytes": _process_max_rss_bytes(), + }, + "cells": cells, + "conformance": conformance, + "refresh_probe": refresh, + "teardown": { + "port": port, + "port_closed": port_closed, + "backend_state": backend.state.value, + }, + "sync_confirmed": sync.active_policy_digest == identity.digest, + } + write_json(output, payload) + return payload + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--backend", choices=("hf_cached", "vllm"), required=True) + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--expected-commit", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--repetitions", type=int, default=3) + args = parser.parse_args() + if args.repetitions < 1: + parser.error("--repetitions must be positive") + payload = run( + backend_name=args.backend, + wheel=args.wheel.resolve(), + expected_commit=args.expected_commit, + output=args.output.resolve(), + repetitions=args.repetitions, + ) + print( + json.dumps( + { + "status": payload["measurement_status"], + "backend": payload["backend"]["name"], + "cells": len(payload["cells"]), + "output": str(args.output), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 2ae447f9ffa15a570db1f2fd8cce70150f254b4d Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Sat, 29 Aug 2026 07:38:52 -0700 Subject: [PATCH 2/6] Resolve cached benchmark snapshots explicitly --- .../benchmark_rollout_runtime_v2_selected.py | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/benchmark_rollout_runtime_v2_selected.py b/scripts/benchmark_rollout_runtime_v2_selected.py index ab5a411..89d606c 100644 --- a/scripts/benchmark_rollout_runtime_v2_selected.py +++ b/scripts/benchmark_rollout_runtime_v2_selected.py @@ -72,6 +72,18 @@ def _package_version(name: str) -> str | None: return None +def _cached_snapshot(model_id: str, revision: str) -> Path: + from huggingface_hub import try_to_load_from_cache + + config = try_to_load_from_cache(model_id, "config.json", revision=revision) + if not isinstance(config, str): + raise RuntimeError(f"cached model {model_id}@{revision} has no config.json") + snapshot = Path(config).resolve().parent + if not any(snapshot.glob("*.safetensors")): + raise RuntimeError(f"cached model {model_id}@{revision} has no safetensors weights") + return snapshot + + def _gpu_used_mib() -> int: raw = subprocess.check_output( [ @@ -430,6 +442,12 @@ def run( attn_implementation=prereg["models"]["attention_implementation"], gradient_checkpointing=False, ) + local_spec = spec.model_copy( + update={ + "model_id": str(_cached_snapshot(spec.model_id, str(spec.revision))), + "revision": None, + } + ) baseline_gpu_mib = _gpu_used_mib() load_started = time.perf_counter() tokenizer = HFTokenizerAdapter.load( @@ -439,12 +457,16 @@ def run( local_files_only=True, ) actor = HFBackend.load( - spec, + local_spec, device="cuda", tokenizer=tokenizer, trainable=True, local_files_only=True, ) + # The concrete path avoids Hub completeness checks; public identity remains + # the exact repository and revision from the preregistration. + actor.model_id = spec.model_id + actor.model_revision = spec.revision torch.cuda.synchronize() actor_load_seconds = time.perf_counter() - load_started if backend_name == "hf_cached": From 4c2cc1366d3dcf7155f84ac7acaf8b35058c987f Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Sat, 29 Aug 2026 07:40:02 -0700 Subject: [PATCH 3/6] Preserve cached snapshot directories for vLLM --- .../benchmark_rollout_runtime_v2_selected.py | 2 +- src/miniverl/runtime/backends/vllm.py | 4 +++- tests/unit/test_vllm_rollout_backend.py | 24 +++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/scripts/benchmark_rollout_runtime_v2_selected.py b/scripts/benchmark_rollout_runtime_v2_selected.py index 89d606c..e8ca7f7 100644 --- a/scripts/benchmark_rollout_runtime_v2_selected.py +++ b/scripts/benchmark_rollout_runtime_v2_selected.py @@ -78,7 +78,7 @@ def _cached_snapshot(model_id: str, revision: str) -> Path: config = try_to_load_from_cache(model_id, "config.json", revision=revision) if not isinstance(config, str): raise RuntimeError(f"cached model {model_id}@{revision} has no config.json") - snapshot = Path(config).resolve().parent + snapshot = Path(config).parent.resolve() if not any(snapshot.glob("*.safetensors")): raise RuntimeError(f"cached model {model_id}@{revision} has no safetensors weights") return snapshot diff --git a/src/miniverl/runtime/backends/vllm.py b/src/miniverl/runtime/backends/vllm.py index 018fce2..10ce841 100644 --- a/src/miniverl/runtime/backends/vllm.py +++ b/src/miniverl/runtime/backends/vllm.py @@ -217,7 +217,9 @@ def _resolve_model_snapshot(model_id: str, revision: str | None) -> Path: f"base snapshot {model_id!r} is not cached; run `hf download {model_id}" f"{revision_hint}` before using rollout.backend=vllm" ) - snapshot = Path(cached).resolve().parent + # Hub snapshot entries are commonly symlinks into ``blobs/``. Resolve the + # containing snapshot directory, not the config symlink target. + snapshot = Path(cached).parent.resolve() if not (snapshot / "config.json").is_file(): raise RuntimeError("resolved vLLM base snapshot has no config.json") return snapshot diff --git a/tests/unit/test_vllm_rollout_backend.py b/tests/unit/test_vllm_rollout_backend.py index 0ce2e1f..269447f 100644 --- a/tests/unit/test_vllm_rollout_backend.py +++ b/tests/unit/test_vllm_rollout_backend.py @@ -77,6 +77,30 @@ def test_vllm_launch_is_local_managed_and_disables_persistent_prefix_cache() -> assert environment["VLLM_USE_FLASHINFER_SAMPLER"] == "0" +def test_vllm_snapshot_resolution_preserves_hub_snapshot_symlink_parent( + tmp_path: Path, +) -> None: + from miniverl.runtime.backends.vllm import _resolve_model_snapshot + + snapshot = tmp_path / "models--org--model" / "snapshots" / ("a" * 40) + blobs = tmp_path / "models--org--model" / "blobs" + snapshot.mkdir(parents=True) + blobs.mkdir(parents=True) + config_blob = blobs / "config-digest" + config_blob.write_text("{}", encoding="utf-8") + config = snapshot / "config.json" + try: + config.symlink_to(config_blob) + except OSError: + pytest.skip("symlinks are unavailable") + + with patch( + "huggingface_hub.try_to_load_from_cache", + return_value=str(config), + ): + assert _resolve_model_snapshot("org/model", "a" * 40) == snapshot.resolve() + + def test_vllm_completion_parser_requires_raw_ids_and_aligned_logprobs() -> None: from miniverl.runtime.backends.vllm import parse_vllm_completion From 690db9b079bfecfec14dc3bedc3aa0308cbacf60 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Sat, 29 Aug 2026 08:14:35 -0700 Subject: [PATCH 4/6] Measure the full rollout workload matrix --- .../benchmark_rollout_runtime_v2_selected.py | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/scripts/benchmark_rollout_runtime_v2_selected.py b/scripts/benchmark_rollout_runtime_v2_selected.py index e8ca7f7..cbe9d6e 100644 --- a/scripts/benchmark_rollout_runtime_v2_selected.py +++ b/scripts/benchmark_rollout_runtime_v2_selected.py @@ -181,7 +181,14 @@ def _requests( samples_per_prompt: int, logical_prompts: int, run_seed: int, + sampling_name: str, ) -> list[GenerationRequest]: + if sampling_name == "greedy": + sampling = SamplingParameters(temperature=0.0, top_p=1.0, top_k=0) + elif sampling_name == "seeded_stochastic": + sampling = SamplingParameters(temperature=0.8, top_p=0.95, top_k=50) + else: + raise ValueError(f"unsupported sampling mode {sampling_name!r}") requests: list[GenerationRequest] = [] for prompt_index in range(logical_prompts): prompt = _fixed_prompt(tokenizer, length=prompt_length, index=prompt_index) @@ -206,7 +213,7 @@ def _requests( ), prompt_token_ids=prompt, max_new_tokens=response_bound, - sampling=SamplingParameters(temperature=0.0, top_p=1.0, top_k=0), + sampling=sampling, need_sampled_token_logprobs=False, expected_policy_identity=identity, ) @@ -236,6 +243,7 @@ def _cell( samples_per_prompt: int, logical_prompts: int, run_seed: int, + sampling_name: str, warmups: int, repetitions: int, ) -> dict[str, Any]: @@ -247,6 +255,7 @@ def _cell( samples_per_prompt=samples_per_prompt, logical_prompts=logical_prompts, run_seed=run_seed, + sampling_name=sampling_name, ) for _ in range(warmups): _generate_partitioned(backend, requests) @@ -262,16 +271,16 @@ def _cell( durations.append(time.perf_counter() - started) digests.append(_digest_results(final)) if len(set(digests)) != 1: - raise RuntimeError("backend output changed across greedy repetitions") + raise RuntimeError(f"backend output changed across {sampling_name} repetitions") generated_tokens = sum(len(result.output_token_ids) for result in final) prompt_tokens = sum(len(request.prompt_token_ids) for request in requests) median = float(statistics.median(durations)) return { - "cell_id": f"p{prompt_length}-r{response_bound}-n{samples_per_prompt}-greedy", + "cell_id": (f"p{prompt_length}-r{response_bound}-n{samples_per_prompt}-{sampling_name}"), "prompt_length": prompt_length, "response_bound": response_bound, "samples_per_prompt": samples_per_prompt, - "sampling": "greedy", + "sampling": sampling_name, "logical_prompts": logical_prompts, "generated_trajectories": len(final), "prompt_tokens": prompt_tokens, @@ -385,6 +394,7 @@ def _refresh_probe( samples_per_prompt=1, logical_prompts=1, run_seed=run_seed, + sampling_name="greedy", ) result = backend.generate(request).results[0] rows.append( @@ -504,16 +514,19 @@ def run( backend, tokenizer, identity=identity, - prompt_length=128, + prompt_length=prompt_length, response_bound=response_bound, samples_per_prompt=samples_per_prompt, logical_prompts=4, run_seed=int(prereg["workload"]["run_seed"]), + sampling_name=sampling_name, warmups=1, repetitions=repetitions, ) + for prompt_length in (128, 512) for response_bound in (64, 256, 512) for samples_per_prompt in (1, 4) + for sampling_name in ("greedy", "seeded_stochastic") ] conformance = _conformance_probe(actor, backend, tokenizer) if backend_name == "vllm" else None refresh = ( @@ -573,10 +586,10 @@ def run( "manifest_version": prereg["workload"]["manifest_version"], "run_seed": prereg["workload"]["run_seed"], "logical_prompts": 4, - "prompt_length": 128, + "prompt_lengths": [128, 512], "response_bounds": [64, 256, 512], "samples_per_prompt": [1, 4], - "sampling": "greedy", + "sampling": ["greedy", "seeded_stochastic"], "physical_concurrency": 4, }, "timing": { From 8abf946d6eb0fcc603ba69f57063c240525c1661 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:38:53 -0700 Subject: [PATCH 5/6] Publish RTX 4080 rollout evidence --- CHANGELOG.md | 14 + PYPI.md | 15 + README.md | 15 + README.zh-CN.md | 12 + benchmarks/README.md | 1 + .../hf-cached-rtx4080-raw.json | 683 +++++++++++++++ .../rollout-runtime-v2/vllm-rtx4080-raw.json | 782 ++++++++++++++++++ .../results/rollout-runtime-v2-rtx4080.json | 468 +++++++++++ .../rollout-runtime-v2-result.schema.json | 31 + .../rollout-runtime-v2-selected.schema.json | 71 ++ docs/benchmarks/rollout-runtime-v2.md | 56 +- docs/competitive-landscape.md | 25 +- docs/release-checklist.md | 16 +- docs/rollout-runtime-v2.md | 38 +- .../publish_rollout_runtime_v2_evidence.py | 240 ++++++ src/miniverl/runtime/backends/vllm.py | 10 +- tests/unit/test_packaging.py | 1 + .../unit/test_rollout_runtime_v2_benchmark.py | 59 ++ tests/unit/test_vllm_rollout_backend.py | 4 + 19 files changed, 2519 insertions(+), 22 deletions(-) create mode 100644 benchmarks/evidence/rollout-runtime-v2/hf-cached-rtx4080-raw.json create mode 100644 benchmarks/evidence/rollout-runtime-v2/vllm-rtx4080-raw.json create mode 100644 benchmarks/results/rollout-runtime-v2-rtx4080.json create mode 100644 benchmarks/schema/rollout-runtime-v2-result.schema.json create mode 100644 benchmarks/schema/rollout-runtime-v2-selected.schema.json create mode 100644 scripts/publish_rollout_runtime_v2_evidence.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 970cd8c..e7b46bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,20 @@ All notable changes to miniVERL are recorded here. The format follows - Kept PG-k1 on `hf_cached`: the BF16 external engine and NF4 actor agreed on greedy tokens in the qualification probe, but their sampled-token log-probabilities did not satisfy the preregistered numerical threshold. +- Preserved the initial managed-server startup timing across later adapter + refreshes instead of overwriting it with no-op start checks. + +### Measured rollout evidence + +- Published the exact-wheel 24-cell RTX 4080 comparison for `hf_reference`, + `hf_cached` and managed vLLM across prompt lengths 128/512, response bounds + 64/256/512, `n=1/4`, greedy and seeded sampling. +- Selected vLLM 0.28.0 for the direct-GKD development path after a measured + 1.65–4.55× advantage over `hf_cached` at 256/512 tokens, eight confirmed + policy refreshes, 11,820 MiB peak total GPU memory and clean teardown. +- Recorded the blocking negative result: `hf_cached` did not reach the + preregistered 2× speedup over `hf_reference`. v0.11.0 remains unpublished + until the local backend is optimized and the frozen workload is rerun. ### Documentation narrative diff --git a/PYPI.md b/PYPI.md index 9266821..e875907 100644 --- a/PYPI.md +++ b/PYPI.md @@ -140,6 +140,21 @@ rewards and explicit group advantage composition; it has no task-quality claim. ## Measured systems evidence +Rollout Runtime v2 now has a complete 24-cell RTX 4080 measurement across +64/256/512-token responses, `n=1/4`, greedy and seeded sampling. Managed vLLM +0.28.0 delivered 108.5–117.7 output tokens/s and was 1.65–4.55× faster than +`hf_cached` in the 256/512-token cells, with 11.54 GiB peak total GPU memory, +eight confirmed policy refreshes and clean teardown. It is the measured +direct-GKD engine; PG-k1 stays on `hf_cached` because the engine log-probability +probe exceeded its numerical threshold. + +The same evidence found the next optimization target: `hf_cached` reached +about 1.8–1.9× on most seeded 256/512-token cells but did not meet the +preregistered 2× gate, while the greedy path was usually slower than the legacy +oracle. The development line therefore remains `0.11.0.dev0` rather than a +published v0.11.0 release. See the [full runtime result](https://github.com/DaoyuanLi2816/mini-verl/blob/main/docs/benchmarks/rollout-runtime-v2.md) +for every cell and artifact hash. + The Qwen3-0.6B/1.7B developer workload consumed **32 distinct prompts**, used a 64-token response bound, and completed **8 current-policy updates** at **3.1914 GiB peak reserved VRAM** on an RTX 4080. Median steady-state rollout, diff --git a/README.md b/README.md index a0543a7..ab7acc4 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,21 @@ rewards and explicit group advantage composition; it has no task-quality claim. ## Measured systems evidence +Rollout Runtime v2 now has a complete 24-cell RTX 4080 measurement across +64/256/512-token responses, `n=1/4`, greedy and seeded sampling. Managed vLLM +0.28.0 delivered 108.5–117.7 output tokens/s and was 1.65–4.55× faster than +`hf_cached` in the 256/512-token cells, with 11.54 GiB peak total GPU memory, +eight confirmed policy refreshes and clean teardown. It is the measured +direct-GKD engine; PG-k1 stays on `hf_cached` because the engine log-probability +probe exceeded its numerical threshold. + +The same evidence found the next optimization target: `hf_cached` reached +about 1.8–1.9× on most seeded 256/512-token cells but did not meet the +preregistered 2× gate, while the greedy path was usually slower than the legacy +oracle. The development line therefore remains `0.11.0.dev0` rather than a +published v0.11.0 release. See the [full runtime result](docs/benchmarks/rollout-runtime-v2.md) +for every cell and artifact hash. + The Qwen3-0.6B/1.7B developer workload consumed **32 distinct prompts**, used a 64-token response bound, and completed **8 current-policy updates** at **3.1914 GiB peak reserved VRAM** on an RTX 4080. Median steady-state rollout, diff --git a/README.zh-CN.md b/README.zh-CN.md index 81a702d..f30df9f 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -129,6 +129,18 @@ group advantage 组合;目前不对任务质量作结论。 ## 实测系统证据 +Rollout Runtime v2 已在 RTX 4080 上完成 24-cell 测量,覆盖 64/256/512-token +response、`n=1/4`、greedy 与 seeded sampling。Managed vLLM 0.28.0 达到 +108.5–117.7 output tokens/s;在 256/512-token cell 中比 `hf_cached` 快 +1.65–4.55×,peak total GPU memory 为 11.54 GiB,并通过 8 次 policy refresh 与 +完整 teardown。它是实测的 direct-GKD engine;engine log-probability probe 超出数值 +门槛,因此 PG-k1 继续使用 `hf_cached`。 + +同一份证据也给出了下一步优化目标:`hf_cached` 在多数 seeded 256/512-token cell 中 +约为旧路径的 1.8–1.9×,没有达到预注册的 2× gate;greedy 路径通常比旧 oracle 更慢。 +因此开发线保持 `0.11.0.dev0`,尚未发布 v0.11.0。每个 cell 与 artifact hash 见 +[完整 runtime 结果](docs/benchmarks/rollout-runtime-v2.md)。 + Qwen3-0.6B/1.7B developer workload 消费 **32 个不同 prompt**,response 上限为 64 token, 在 RTX 4080 上完成 **8 次 current-policy update**,**peak reserved VRAM 为 3.1914 GiB**。 稳态 rollout、teacher scoring 与 update 的中位耗时分别为 9.7200、0.4864 与 2.3260 秒。 diff --git a/benchmarks/README.md b/benchmarks/README.md index 9a11731..e926a8a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -80,6 +80,7 @@ Listed so you can check that this repository does not. | File | Hardware | What it shows | | --- | --- | --- | +| `results/rollout-runtime-v2-rtx4080.json` | WSL2 RTX 4080 16 GB | Exact-wheel 24-cell `hf_reference` / `hf_cached` / vLLM comparison. vLLM passed the direct-GKD engine gates; `hf_cached` missed the preregistered 2× speed gate, so v0.11.0 remains blocked. [Result, method and hashes.](../docs/benchmarks/rollout-runtime-v2.md) | | `results/rollout-runtime-v2-hf-reference.json` | WSL2 RTX 4080 16 GB | Pre-v0.11 actor-rollout systems baseline: all 24 preregistered prompt/response/n/sampling cells completed. It freezes throughput and memory for the future `hf_cached` comparison; teacher scoring, updates, full-cycle timing and typed grouped semantics are not measured here. [Method, table and boundaries.](../docs/benchmarks/rollout-runtime-v2.md) | | `results/alignment-external-v1.json` | RTX 4080 16 GB | **Preregistered early stop, not a method comparison.** Two declared lineages and eight candidates all measured 0/64 retained JSONNav utility against an unchanged 20% floor. No checkpoint, teacher, continuation arm or final-test result exists. [Study, correction provenance and evaluator boundary.](../docs/alignment-external/alignment-external-v1.md) | | `results/alignment-lab-v1.json` | RTX 4080 16 GB | Alignment Lab v1: six methods from one Qwen3-0.6B SFT checkpoint, three seeds and 48 paired policy tasks per arm. The SFT start saturated alignment and utility; no continuation method improved it, so the pilot recommends no online teacher querying. [Report, figures and caveats.](../docs/alignment-lab/alignment-lab-v1.md) | diff --git a/benchmarks/evidence/rollout-runtime-v2/hf-cached-rtx4080-raw.json b/benchmarks/evidence/rollout-runtime-v2/hf-cached-rtx4080-raw.json new file mode 100644 index 0000000..5a8e54d --- /dev/null +++ b/benchmarks/evidence/rollout-runtime-v2/hf-cached-rtx4080-raw.json @@ -0,0 +1,683 @@ +{ + "backend": { + "capabilities": { + "backend_version": "hf_cached-v1", + "kind": "hf_cached", + "reproducibility": "batch_invariant", + "supports_greedy": true, + "supports_sampled_token_logprobs": true, + "supports_seeded_sampling": true, + "supports_text_stops": true + }, + "name": "hf_cached", + "version": "hf_cached-v1" + }, + "cells": [ + { + "cell_id": "p128-r64-n1-greedy", + "generated_tokens": 256, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 3.9074172189999956, + "output_token_ids_sha256": "cd9f13d9e943573baeb2d4e1dd04b122c4faceda6069544c4a48df4a673d8643", + "output_tokens_per_second": 65.5164231644341, + "peak_total_gpu_memory_mib": 3385, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 131.0328463288682, + "response_bound": 64, + "rollout_seconds": [ + 3.888729779000002, + 3.9074172189999956, + 3.952097882000004 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r64-n1-seeded_stochastic", + "generated_tokens": 256, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 4.630542718000001, + "output_token_ids_sha256": "cd9f13d9e943573baeb2d4e1dd04b122c4faceda6069544c4a48df4a673d8643", + "output_tokens_per_second": 55.2850962814506, + "peak_total_gpu_memory_mib": 3385, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 110.5701925629012, + "response_bound": 64, + "rollout_seconds": [ + 4.630542718000001, + 4.722462727, + 4.567237745000007 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r64-n4-greedy", + "generated_tokens": 1024, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 16.11618772099999, + "output_token_ids_sha256": "3b4b83de35b1d649fe3f49cf2489de7a434a97304abb50ee7ef985da4ea3b328", + "output_tokens_per_second": 63.53859968171568, + "peak_total_gpu_memory_mib": 3405, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 127.07719936343136, + "response_bound": 64, + "rollout_seconds": [ + 16.428164232, + 15.945546788000001, + 16.11618772099999 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r64-n4-seeded_stochastic", + "generated_tokens": 1024, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 18.59308446900002, + "output_token_ids_sha256": "3b4b83de35b1d649fe3f49cf2489de7a434a97304abb50ee7ef985da4ea3b328", + "output_tokens_per_second": 55.07424019437444, + "peak_total_gpu_memory_mib": 3400, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 110.14848038874888, + "response_bound": 64, + "rollout_seconds": [ + 18.321573126000004, + 18.59308446900002, + 18.720864084 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r256-n1-greedy", + "generated_tokens": 1024, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 15.033431644000018, + "output_token_ids_sha256": "711427fff6e3d357944a8bc254ed4cae8b33627122172a8d8e0d7724dc22c321", + "output_tokens_per_second": 68.11485389689372, + "peak_total_gpu_memory_mib": 3518, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 34.05742694844686, + "response_bound": 256, + "rollout_seconds": [ + 15.033431644000018, + 15.010858850000005, + 15.034443231000012 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r256-n1-seeded_stochastic", + "generated_tokens": 1024, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 18.459553870000036, + "output_token_ids_sha256": "711427fff6e3d357944a8bc254ed4cae8b33627122172a8d8e0d7724dc22c321", + "output_tokens_per_second": 55.472629902728954, + "peak_total_gpu_memory_mib": 3516, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 27.736314951364477, + "response_bound": 256, + "rollout_seconds": [ + 18.468877176999968, + 18.459553870000036, + 18.35073380900002 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r256-n4-greedy", + "generated_tokens": 4096, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 62.81358740699994, + "output_token_ids_sha256": "eef961f1441414411acf4c676a6720a72986f1a635f4c62431adf91b9b288d13", + "output_tokens_per_second": 65.20882135675541, + "peak_total_gpu_memory_mib": 3521, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 32.604410678377704, + "response_bound": 256, + "rollout_seconds": [ + 63.32590217400002, + 62.81358740699994, + 62.355235934999996 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r256-n4-seeded_stochastic", + "generated_tokens": 4096, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 74.07784019300004, + "output_token_ids_sha256": "eef961f1441414411acf4c676a6720a72986f1a635f4c62431adf91b9b288d13", + "output_tokens_per_second": 55.2931887502175, + "peak_total_gpu_memory_mib": 3518, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 27.64659437510875, + "response_bound": 256, + "rollout_seconds": [ + 74.48019879900005, + 73.80075905199999, + 74.07784019300004 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r512-n1-greedy", + "generated_tokens": 2048, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 29.406566596999937, + "output_token_ids_sha256": "fdf6053d6a62183ea73d5a3e6da66538b71bcae75835bd826ecf81ddefa4a1b5", + "output_tokens_per_second": 69.6443086357772, + "peak_total_gpu_memory_mib": 3890, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 17.4110771589443, + "response_bound": 512, + "rollout_seconds": [ + 31.003336795999985, + 29.30517272399993, + 29.406566596999937 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r512-n1-seeded_stochastic", + "generated_tokens": 2048, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 36.802451045, + "output_token_ids_sha256": "fdf6053d6a62183ea73d5a3e6da66538b71bcae75835bd826ecf81ddefa4a1b5", + "output_tokens_per_second": 55.648467475598814, + "peak_total_gpu_memory_mib": 3882, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 13.912116868899703, + "response_bound": 512, + "rollout_seconds": [ + 36.802451045, + 36.433882607999976, + 36.929255971999964 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r512-n4-greedy", + "generated_tokens": 8192, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 131.81104275799999, + "output_token_ids_sha256": "9d36364e8c8bb02886b0de3de5708858843f570413918901df1afe2fcebaf47d", + "output_tokens_per_second": 62.149572817204685, + "peak_total_gpu_memory_mib": 3949, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 15.537393204301171, + "response_bound": 512, + "rollout_seconds": [ + 120.12635308600011, + 131.81104275799999, + 144.6125067060002 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r512-n4-seeded_stochastic", + "generated_tokens": 8192, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 147.09275044599985, + "output_token_ids_sha256": "9d36364e8c8bb02886b0de3de5708858843f570413918901df1afe2fcebaf47d", + "output_tokens_per_second": 55.69275151332096, + "peak_total_gpu_memory_mib": 3891, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 13.92318787833024, + "response_bound": 512, + "rollout_seconds": [ + 148.7805746439999, + 147.09275044599985, + 146.74338501400007 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r64-n1-greedy", + "generated_tokens": 256, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 3.9021043279999503, + "output_token_ids_sha256": "cd9f13d9e943573baeb2d4e1dd04b122c4faceda6069544c4a48df4a673d8643", + "output_tokens_per_second": 65.60562672890258, + "peak_total_gpu_memory_mib": 3892, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 524.8450138312206, + "response_bound": 64, + "rollout_seconds": [ + 3.9021043279999503, + 4.019779680999818, + 3.870203253999989 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r64-n1-seeded_stochastic", + "generated_tokens": 256, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 4.6236177370001315, + "output_token_ids_sha256": "cd9f13d9e943573baeb2d4e1dd04b122c4faceda6069544c4a48df4a673d8643", + "output_tokens_per_second": 55.367899026638916, + "peak_total_gpu_memory_mib": 3879, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 442.94319221311133, + "response_bound": 64, + "rollout_seconds": [ + 4.5715059699996345, + 4.6236177370001315, + 4.6512741629999255 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r64-n4-greedy", + "generated_tokens": 1024, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 16.069187178999982, + "output_token_ids_sha256": "3b4b83de35b1d649fe3f49cf2489de7a434a97304abb50ee7ef985da4ea3b328", + "output_tokens_per_second": 63.724442847875615, + "peak_total_gpu_memory_mib": 3885, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 509.7955427830049, + "response_bound": 64, + "rollout_seconds": [ + 16.069187178999982, + 16.372716328000024, + 15.885387186000116 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r64-n4-seeded_stochastic", + "generated_tokens": 1024, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 18.154042705999927, + "output_token_ids_sha256": "3b4b83de35b1d649fe3f49cf2489de7a434a97304abb50ee7ef985da4ea3b328", + "output_tokens_per_second": 56.4061689499919, + "peak_total_gpu_memory_mib": 3884, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 451.2493515999352, + "response_bound": 64, + "rollout_seconds": [ + 17.860801494000043, + 18.154042705999927, + 18.566430468999897 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r256-n1-greedy", + "generated_tokens": 1024, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 15.406603385999915, + "output_token_ids_sha256": "711427fff6e3d357944a8bc254ed4cae8b33627122172a8d8e0d7724dc22c321", + "output_tokens_per_second": 66.46500687688993, + "peak_total_gpu_memory_mib": 4038, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 132.93001375377986, + "response_bound": 256, + "rollout_seconds": [ + 15.504227704999721, + 15.406603385999915, + 15.336330732000079 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r256-n1-seeded_stochastic", + "generated_tokens": 1024, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 18.620330084000216, + "output_token_ids_sha256": "711427fff6e3d357944a8bc254ed4cae8b33627122172a8d8e0d7724dc22c321", + "output_tokens_per_second": 54.99365453676284, + "peak_total_gpu_memory_mib": 4044, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 109.98730907352568, + "response_bound": 256, + "rollout_seconds": [ + 18.80326953299982, + 18.620330084000216, + 18.267822254000293 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r256-n4-greedy", + "generated_tokens": 4096, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 62.470829560999846, + "output_token_ids_sha256": "eef961f1441414411acf4c676a6720a72986f1a635f4c62431adf91b9b288d13", + "output_tokens_per_second": 65.56660170488767, + "peak_total_gpu_memory_mib": 4136, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 131.13320340977535, + "response_bound": 256, + "rollout_seconds": [ + 62.470829560999846, + 62.70202690499991, + 59.443766433999826 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r256-n4-seeded_stochastic", + "generated_tokens": 4096, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 162.64140629199983, + "output_token_ids_sha256": "eef961f1441414411acf4c676a6720a72986f1a635f4c62431adf91b9b288d13", + "output_tokens_per_second": 25.184238708845193, + "peak_total_gpu_memory_mib": 4130, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 50.368477417690386, + "response_bound": 256, + "rollout_seconds": [ + 118.80729817299971, + 267.9298800730003, + 162.64140629199983 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r512-n1-greedy", + "generated_tokens": 2048, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 29.397636921999947, + "output_token_ids_sha256": "fdf6053d6a62183ea73d5a3e6da66538b71bcae75835bd826ecf81ddefa4a1b5", + "output_tokens_per_second": 69.66546343278917, + "peak_total_gpu_memory_mib": 4588, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 69.66546343278917, + "response_bound": 512, + "rollout_seconds": [ + 29.397636921999947, + 29.269759253000302, + 30.989748956999847 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r512-n1-seeded_stochastic", + "generated_tokens": 2048, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 35.93708506699977, + "output_token_ids_sha256": "fdf6053d6a62183ea73d5a3e6da66538b71bcae75835bd826ecf81ddefa4a1b5", + "output_tokens_per_second": 56.988484073813574, + "peak_total_gpu_memory_mib": 4598, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 56.988484073813574, + "response_bound": 512, + "rollout_seconds": [ + 36.054504858999735, + 35.93708506699977, + 35.91486711599964 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r512-n4-greedy", + "generated_tokens": 8192, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 121.44754048100003, + "output_token_ids_sha256": "9d36364e8c8bb02886b0de3de5708858843f570413918901df1afe2fcebaf47d", + "output_tokens_per_second": 67.45299219362623, + "peak_total_gpu_memory_mib": 4595, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 67.45299219362623, + "response_bound": 512, + "rollout_seconds": [ + 120.52083667099987, + 121.44754048100003, + 124.26085396300005 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r512-n4-seeded_stochastic", + "generated_tokens": 8192, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 144.01206198199998, + "output_token_ids_sha256": "9d36364e8c8bb02886b0de3de5708858843f570413918901df1afe2fcebaf47d", + "output_tokens_per_second": 56.88412406055206, + "peak_total_gpu_memory_mib": 4595, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 56.88412406055206, + "response_bound": 512, + "rollout_seconds": [ + 144.01206198199998, + 143.02474348499982, + 144.75823360599952 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + } + ], + "conformance": null, + "environment": { + "cuda_runtime": "13.0", + "driver_version": "596.49", + "gpu": "NVIDIA GeForce RTX 4080", + "gpu_total_memory_bytes": 17170956288, + "packages": { + "accelerate": "1.14.0", + "bitsandbytes": "0.50.2", + "peft": "0.20.0", + "torch": "2.13.0+cu130", + "transformers": "5.14.1", + "vllm": "0.28.0" + }, + "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", + "python": "3.12.13" + }, + "frozen_calculator_sha256": "53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc", + "measurement_status": "completed", + "memory": { + "baseline_total_gpu_memory_mib": 1867, + "peak_total_gpu_memory_mib": 4598, + "process_max_rss_bytes": 2607476736, + "residual_total_gpu_memory_mib": 3381 + }, + "models": { + "actor": { + "id": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "tokenizer_revision": "c1899de289a04d12100db370d81485cdf75e47ca" + }, + "attention_implementation": "sdpa", + "dtype": "bfloat16", + "quantization": "nf4", + "teacher": { + "id": "Qwen/Qwen3-1.7B", + "revision": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e", + "tokenizer_revision": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e" + } + }, + "name": "rollout-runtime-v2-hf_cached-rtx4080-raw", + "policy_identity": { + "adapter_tensor_digest": "dbdf11478b1b99be19b1e8c9b3007194625f6e108c6b89c5a3aa4d09cb77de6d", + "identity_digest": "b9f054c7027d005fef98fb5d3a222ad6e33f8cabf01bd051ee6969e220390882", + "parameter_version": 0 + }, + "preregistration_sha256": "8cc3ba738c69b59ed19c22c1de874fd00249404198a3e05983477dc8899bb7e5", + "refresh_probe": null, + "schema_version": 1, + "source": { + "commit": "690db9b079bfecfec14dc3bedc3aa0308cbacf60", + "dirty": false, + "miniverl_version": "0.11.0.dev0", + "wheel_sha256": "266fcd59bb3e85b02974a92b26c9a4e59c7c9b9205853d17baadcfffdb898e27" + }, + "sync_confirmed": true, + "teardown": { + "backend_state": "closed", + "port": null, + "port_closed": null + }, + "timing": { + "actor_load_seconds": 4.5306232269999995, + "engine_lifecycle": {}, + "initial_sync_seconds": 7.161399999944251e-05, + "teardown_seconds": 0.27761382999960915 + }, + "workload": { + "logical_prompts": 4, + "manifest_version": "rollout-runtime-v2-workload-v1", + "physical_concurrency": 4, + "prompt_lengths": [ + 128, + 512 + ], + "response_bounds": [ + 64, + 256, + 512 + ], + "run_seed": 20260829, + "samples_per_prompt": [ + 1, + 4 + ], + "sampling": [ + "greedy", + "seeded_stochastic" + ] + } +} diff --git a/benchmarks/evidence/rollout-runtime-v2/vllm-rtx4080-raw.json b/benchmarks/evidence/rollout-runtime-v2/vllm-rtx4080-raw.json new file mode 100644 index 0000000..667594e --- /dev/null +++ b/benchmarks/evidence/rollout-runtime-v2/vllm-rtx4080-raw.json @@ -0,0 +1,782 @@ +{ + "backend": { + "capabilities": { + "backend_version": "vllm-0.28.0-direct-gkd-v1", + "kind": "vllm", + "reproducibility": "deterministic_greedy", + "supports_greedy": true, + "supports_sampled_token_logprobs": false, + "supports_seeded_sampling": true, + "supports_text_stops": true + }, + "name": "vllm", + "version": "vllm-0.28.0-direct-gkd-v1" + }, + "cells": [ + { + "cell_id": "p128-r64-n1-greedy", + "generated_tokens": 256, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 2.2531619830006093, + "output_token_ids_sha256": "cd9f13d9e943573baeb2d4e1dd04b122c4faceda6069544c4a48df4a673d8643", + "output_tokens_per_second": 113.61810732270409, + "peak_total_gpu_memory_mib": 11813, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 227.23621464540818, + "response_bound": 64, + "rollout_seconds": [ + 2.62165723399994, + 2.2414963499995793, + 2.2531619830006093 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r64-n1-seeded_stochastic", + "generated_tokens": 256, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 2.3414881290000267, + "output_token_ids_sha256": "cd9f13d9e943573baeb2d4e1dd04b122c4faceda6069544c4a48df4a673d8643", + "output_tokens_per_second": 109.33217932192946, + "peak_total_gpu_memory_mib": 11798, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 218.66435864385892, + "response_bound": 64, + "rollout_seconds": [ + 3.1992711439997947, + 2.3414881290000267, + 2.3064274689995727 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r64-n4-greedy", + "generated_tokens": 1024, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 9.086888663000536, + "output_token_ids_sha256": "3b4b83de35b1d649fe3f49cf2489de7a434a97304abb50ee7ef985da4ea3b328", + "output_tokens_per_second": 112.68983675011486, + "peak_total_gpu_memory_mib": 11798, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 225.37967350022973, + "response_bound": 64, + "rollout_seconds": [ + 9.022614702999817, + 9.086888663000536, + 9.135586309000246 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r64-n4-seeded_stochastic", + "generated_tokens": 1024, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 9.016678904999935, + "output_token_ids_sha256": "3b4b83de35b1d649fe3f49cf2489de7a434a97304abb50ee7ef985da4ea3b328", + "output_tokens_per_second": 113.56731350743463, + "peak_total_gpu_memory_mib": 11816, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 227.13462701486927, + "response_bound": 64, + "rollout_seconds": [ + 9.016678904999935, + 9.037268824999956, + 8.896722251000028 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r256-n1-greedy", + "generated_tokens": 1024, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 8.723777497999436, + "output_token_ids_sha256": "711427fff6e3d357944a8bc254ed4cae8b33627122172a8d8e0d7724dc22c321", + "output_tokens_per_second": 117.38034357648701, + "peak_total_gpu_memory_mib": 11820, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 58.690171788243504, + "response_bound": 256, + "rollout_seconds": [ + 8.617909939000128, + 8.723777497999436, + 8.76598722899962 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r256-n1-seeded_stochastic", + "generated_tokens": 1024, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 9.038505589000124, + "output_token_ids_sha256": "711427fff6e3d357944a8bc254ed4cae8b33627122172a8d8e0d7724dc22c321", + "output_tokens_per_second": 113.29306486751634, + "peak_total_gpu_memory_mib": 11809, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 56.64653243375817, + "response_bound": 256, + "rollout_seconds": [ + 9.038505589000124, + 9.011646335000478, + 9.04828194899983 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r256-n4-greedy", + "generated_tokens": 4096, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 35.510011456999564, + "output_token_ids_sha256": "eef961f1441414411acf4c676a6720a72986f1a635f4c62431adf91b9b288d13", + "output_tokens_per_second": 115.34775213913979, + "peak_total_gpu_memory_mib": 11815, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 57.67387606956989, + "response_bound": 256, + "rollout_seconds": [ + 35.310575405000236, + 35.510011456999564, + 36.533883660000356 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r256-n4-seeded_stochastic", + "generated_tokens": 4096, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 36.230648508999366, + "output_token_ids_sha256": "eef961f1441414411acf4c676a6720a72986f1a635f4c62431adf91b9b288d13", + "output_tokens_per_second": 113.05345525301847, + "peak_total_gpu_memory_mib": 11816, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 56.526727626509235, + "response_bound": 256, + "rollout_seconds": [ + 36.58195353100018, + 36.230648508999366, + 35.53464139799962 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r512-n1-greedy", + "generated_tokens": 2048, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 17.843044302000635, + "output_token_ids_sha256": "fdf6053d6a62183ea73d5a3e6da66538b71bcae75835bd826ecf81ddefa4a1b5", + "output_tokens_per_second": 114.77861991131019, + "peak_total_gpu_memory_mib": 11815, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 28.694654977827547, + "response_bound": 512, + "rollout_seconds": [ + 17.37090838899985, + 17.843044302000635, + 17.955705602000307 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r512-n1-seeded_stochastic", + "generated_tokens": 2048, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 17.924843521000184, + "output_token_ids_sha256": "fdf6053d6a62183ea73d5a3e6da66538b71bcae75835bd826ecf81ddefa4a1b5", + "output_tokens_per_second": 114.25483283023517, + "peak_total_gpu_memory_mib": 11814, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 512, + "prompt_tokens_per_second": 28.563708207558793, + "response_bound": 512, + "rollout_seconds": [ + 18.171731839999666, + 17.690389328000492, + 17.924843521000184 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r512-n4-greedy", + "generated_tokens": 8192, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 70.66215166100028, + "output_token_ids_sha256": "9d36364e8c8bb02886b0de3de5708858843f570413918901df1afe2fcebaf47d", + "output_tokens_per_second": 115.93193537752563, + "peak_total_gpu_memory_mib": 11817, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 28.98298384438141, + "response_bound": 512, + "rollout_seconds": [ + 72.95830444900002, + 70.66215166100028, + 69.57042592499965 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p128-r512-n4-seeded_stochastic", + "generated_tokens": 8192, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 73.21603285499987, + "output_token_ids_sha256": "9d36364e8c8bb02886b0de3de5708858843f570413918901df1afe2fcebaf47d", + "output_tokens_per_second": 111.88806167938358, + "peak_total_gpu_memory_mib": 11815, + "physical_concurrency": 4, + "prompt_length": 128, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 27.972015419845896, + "response_bound": 512, + "rollout_seconds": [ + 73.21603285499987, + 74.58704273000058, + 71.42158291899977 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r64-n1-greedy", + "generated_tokens": 256, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 2.2277635929995085, + "output_token_ids_sha256": "cd9f13d9e943573baeb2d4e1dd04b122c4faceda6069544c4a48df4a673d8643", + "output_tokens_per_second": 114.91344988510029, + "peak_total_gpu_memory_mib": 11803, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 919.3075990808023, + "response_bound": 64, + "rollout_seconds": [ + 2.2277635929995085, + 2.187645269999848, + 2.2364577660000577 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r64-n1-seeded_stochastic", + "generated_tokens": 256, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 2.2409465799992176, + "output_token_ids_sha256": "cd9f13d9e943573baeb2d4e1dd04b122c4faceda6069544c4a48df4a673d8643", + "output_tokens_per_second": 114.23743978764963, + "peak_total_gpu_memory_mib": 11805, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 913.8995183011971, + "response_bound": 64, + "rollout_seconds": [ + 2.2409465799992176, + 2.2403018139993947, + 2.241125968999768 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r64-n4-greedy", + "generated_tokens": 1024, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 8.94467353500022, + "output_token_ids_sha256": "3b4b83de35b1d649fe3f49cf2489de7a434a97304abb50ee7ef985da4ea3b328", + "output_tokens_per_second": 114.48153987880283, + "peak_total_gpu_memory_mib": 11813, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 915.8523190304227, + "response_bound": 64, + "rollout_seconds": [ + 8.876637038999434, + 8.94467353500022, + 9.114220225000281 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r64-n4-seeded_stochastic", + "generated_tokens": 1024, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 9.369612710000183, + "output_token_ids_sha256": "3b4b83de35b1d649fe3f49cf2489de7a434a97304abb50ee7ef985da4ea3b328", + "output_tokens_per_second": 109.28946923356665, + "peak_total_gpu_memory_mib": 11800, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 874.3157538685332, + "response_bound": 64, + "rollout_seconds": [ + 9.369612710000183, + 9.37310400899969, + 9.20746241300003 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r256-n1-greedy", + "generated_tokens": 1024, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 8.729656229999819, + "output_token_ids_sha256": "711427fff6e3d357944a8bc254ed4cae8b33627122172a8d8e0d7724dc22c321", + "output_tokens_per_second": 117.30129721271067, + "peak_total_gpu_memory_mib": 11812, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 234.60259442542133, + "response_bound": 256, + "rollout_seconds": [ + 8.752310851999937, + 8.729656229999819, + 8.674441957999989 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r256-n1-seeded_stochastic", + "generated_tokens": 1024, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 9.005479987999934, + "output_token_ids_sha256": "711427fff6e3d357944a8bc254ed4cae8b33627122172a8d8e0d7724dc22c321", + "output_tokens_per_second": 113.70854206155697, + "peak_total_gpu_memory_mib": 11810, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 227.41708412311394, + "response_bound": 256, + "rollout_seconds": [ + 9.005479987999934, + 8.991832864000571, + 9.038849573999869 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r256-n4-greedy", + "generated_tokens": 4096, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 36.63942431099986, + "output_token_ids_sha256": "eef961f1441414411acf4c676a6720a72986f1a635f4c62431adf91b9b288d13", + "output_tokens_per_second": 111.7921494953812, + "peak_total_gpu_memory_mib": 11818, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 223.5842989907624, + "response_bound": 256, + "rollout_seconds": [ + 37.70672317699973, + 36.63942431099986, + 36.13123115799954 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r256-n4-seeded_stochastic", + "generated_tokens": 4096, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 35.777886221999324, + "output_token_ids_sha256": "eef961f1441414411acf4c676a6720a72986f1a635f4c62431adf91b9b288d13", + "output_tokens_per_second": 114.48412504261995, + "peak_total_gpu_memory_mib": 11817, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 228.9682500852399, + "response_bound": 256, + "rollout_seconds": [ + 35.1045437089997, + 35.777886221999324, + 36.53902506199938 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r512-n1-greedy", + "generated_tokens": 2048, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 17.399170167999728, + "output_token_ids_sha256": "fdf6053d6a62183ea73d5a3e6da66538b71bcae75835bd826ecf81ddefa4a1b5", + "output_tokens_per_second": 117.70676303670209, + "peak_total_gpu_memory_mib": 11811, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 117.70676303670209, + "response_bound": 512, + "rollout_seconds": [ + 17.399170167999728, + 17.36422732400024, + 17.612558808999893 + ], + "samples_per_prompt": 1, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r512-n1-seeded_stochastic", + "generated_tokens": 2048, + "generated_trajectories": 4, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 18.870895087999997, + "output_token_ids_sha256": "fdf6053d6a62183ea73d5a3e6da66538b71bcae75835bd826ecf81ddefa4a1b5", + "output_tokens_per_second": 108.5269135591943, + "peak_total_gpu_memory_mib": 11817, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 2048, + "prompt_tokens_per_second": 108.5269135591943, + "response_bound": 512, + "rollout_seconds": [ + 18.35272308399999, + 18.892191573000673, + 18.870895087999997 + ], + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r512-n4-greedy", + "generated_tokens": 8192, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 70.58616557799996, + "output_token_ids_sha256": "9d36364e8c8bb02886b0de3de5708858843f570413918901df1afe2fcebaf47d", + "output_tokens_per_second": 116.05673623038184, + "peak_total_gpu_memory_mib": 11819, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 116.05673623038184, + "response_bound": 512, + "rollout_seconds": [ + 69.94117840200033, + 70.62990214100046, + 70.58616557799996 + ], + "samples_per_prompt": 4, + "sampling": "greedy", + "warmup_repetitions": 1 + }, + { + "cell_id": "p512-r512-n4-seeded_stochastic", + "generated_tokens": 8192, + "generated_trajectories": 16, + "logical_prompts": 4, + "measured_repetitions": 3, + "median_rollout_seconds": 71.60341306300052, + "output_token_ids_sha256": "9d36364e8c8bb02886b0de3de5708858843f570413918901df1afe2fcebaf47d", + "output_tokens_per_second": 114.40795416822156, + "peak_total_gpu_memory_mib": 11817, + "physical_concurrency": 4, + "prompt_length": 512, + "prompt_tokens": 8192, + "prompt_tokens_per_second": 114.40795416822156, + "response_bound": 512, + "rollout_seconds": [ + 73.05197539500023, + 71.05781802199999, + 71.60341306300052 + ], + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "warmup_repetitions": 1 + } + ], + "conformance": { + "compared_output_tokens": 32, + "engine_output_token_ids_sha256": "ea866878696501cedd0c811bcf5621af9e6c30dad8d4d059bcd9b385063b8e60", + "local_output_token_ids_sha256": "ea866878696501cedd0c811bcf5621af9e6c30dad8d4d059bcd9b385063b8e60", + "pg_nf4_threshold": 0.01, + "pg_threshold_passed": false, + "prompt_token_ids_sha256": "ec3d9233b7496cc409374c6754e679be90bc20f97bfb5ffbb0c588dad51bebfe", + "sampled_logprob_max_abs_difference": 0.01942583080381155, + "sampled_logprob_mean_abs_difference": 0.014255227812100202, + "sampled_logprob_p99_abs_difference": 0.01942583080381155, + "token_agreement_count": 32, + "token_agreement_fraction": 1.0 + }, + "environment": { + "cuda_runtime": "13.0", + "driver_version": "596.49", + "gpu": "NVIDIA GeForce RTX 4080", + "gpu_total_memory_bytes": 17170956288, + "packages": { + "accelerate": "1.14.0", + "bitsandbytes": "0.50.2", + "peft": "0.20.0", + "torch": "2.13.0+cu130", + "transformers": "5.14.1", + "vllm": "0.28.0" + }, + "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", + "python": "3.12.13" + }, + "frozen_calculator_sha256": "53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc", + "measurement_status": "completed", + "memory": { + "baseline_total_gpu_memory_mib": 1860, + "peak_total_gpu_memory_mib": 11820, + "process_max_rss_bytes": 2603896832, + "residual_total_gpu_memory_mib": 3382 + }, + "models": { + "actor": { + "id": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "tokenizer_revision": "c1899de289a04d12100db370d81485cdf75e47ca" + }, + "attention_implementation": "sdpa", + "dtype": "bfloat16", + "quantization": "nf4", + "teacher": { + "id": "Qwen/Qwen3-1.7B", + "revision": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e", + "tokenizer_revision": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e" + } + }, + "name": "rollout-runtime-v2-vllm-rtx4080-raw", + "policy_identity": { + "adapter_tensor_digest": "94c4863b7f73cf662b3d9a5e80338cf0f4d835a1faf24aa3bfe73915ac069f0b", + "identity_digest": "7dcbd2b85354cb39dc7b317a465a9b6201c220350bc03fb016c8d1d6dd2302e5", + "parameter_version": 0 + }, + "preregistration_sha256": "8cc3ba738c69b59ed19c22c1de874fd00249404198a3e05983477dc8899bb7e5", + "refresh_probe": { + "all_policy_identities_unique": true, + "all_syncs_confirmed": true, + "cycles": [ + { + "adapter_tensor_digest": "4c7478fb4b789d32683bf89bff5425262a84abb1bb8168196be613392297d32a", + "output_token_ids_sha256": "03374731498e5cd9cdfe41557372b9e219f2bc073096bd5570c4c5e14e1ee9c0", + "policy_identity_digest": "f597fcad13b3c8115a82b432bc39690f552b6a8c052ff8747c20b3074fcb5f28", + "policy_version": 1, + "sync_confirmed": true, + "sync_seconds": 0.1319834339992667, + "total_gpu_memory_mib": 11869 + }, + { + "adapter_tensor_digest": "44b206d9cca72015adafd721a8a6cdabf41cf24af5eb2f6778f379980f8a16d5", + "output_token_ids_sha256": "03374731498e5cd9cdfe41557372b9e219f2bc073096bd5570c4c5e14e1ee9c0", + "policy_identity_digest": "32b09a2fc6b6b3f724063e0163e920b0dfadb717183547dc16360ed22f56a87b", + "policy_version": 2, + "sync_confirmed": true, + "sync_seconds": 0.09738474000005226, + "total_gpu_memory_mib": 11869 + }, + { + "adapter_tensor_digest": "ba986c87fffea8d4d72b2f54b042c0f12514f97baee84b49d7f7378b63dc6ed4", + "output_token_ids_sha256": "03374731498e5cd9cdfe41557372b9e219f2bc073096bd5570c4c5e14e1ee9c0", + "policy_identity_digest": "f6b9be801e274c4d4a8364a861ea3f69d7b065057ca0388d046fcf5bb4ee503f", + "policy_version": 3, + "sync_confirmed": true, + "sync_seconds": 0.10946140099986224, + "total_gpu_memory_mib": 11869 + }, + { + "adapter_tensor_digest": "b115fa9ffc1d9026553797d6a26928bfdb207f5e0548361ab89a9607c6abae75", + "output_token_ids_sha256": "03374731498e5cd9cdfe41557372b9e219f2bc073096bd5570c4c5e14e1ee9c0", + "policy_identity_digest": "2d7228509d1485b873891a30b32ef01e19d3ebe049a2ffffed52b7217f8679f2", + "policy_version": 4, + "sync_confirmed": true, + "sync_seconds": 0.0904563309995865, + "total_gpu_memory_mib": 11869 + }, + { + "adapter_tensor_digest": "5b491948443dcb5e005a7c1f20068c0ef83d289408f4fcdbe14f86b03ca0c348", + "output_token_ids_sha256": "03374731498e5cd9cdfe41557372b9e219f2bc073096bd5570c4c5e14e1ee9c0", + "policy_identity_digest": "e38dae4927c13af247bc94d0025d394108d53c915b3df51396000e1bdfcb7134", + "policy_version": 5, + "sync_confirmed": true, + "sync_seconds": 0.09008129599988024, + "total_gpu_memory_mib": 11869 + }, + { + "adapter_tensor_digest": "8f294852d2370a4ad15bb1c9ab8c3169c422bf376c2be1f32fadf67897175526", + "output_token_ids_sha256": "03374731498e5cd9cdfe41557372b9e219f2bc073096bd5570c4c5e14e1ee9c0", + "policy_identity_digest": "9a58338fe264b43fc70d4647481659ab15acc622f7ff890dca1323cfb60f9448", + "policy_version": 6, + "sync_confirmed": true, + "sync_seconds": 0.098372207000466, + "total_gpu_memory_mib": 11869 + }, + { + "adapter_tensor_digest": "71c5b25ed0dae2429cd157f1c60e1f3595abbe1029e8183f91285bcc59532420", + "output_token_ids_sha256": "03374731498e5cd9cdfe41557372b9e219f2bc073096bd5570c4c5e14e1ee9c0", + "policy_identity_digest": "53f33d14ebf5ce978416180b0d73ba464cbad330affd1bac896ac49894b22641", + "policy_version": 7, + "sync_confirmed": true, + "sync_seconds": 0.09035330500046257, + "total_gpu_memory_mib": 11869 + }, + { + "adapter_tensor_digest": "2eb19589bfeb3b85e25899501b3c9ecc13cccc76b4bb6969e3b1c39ccf927be6", + "output_token_ids_sha256": "03374731498e5cd9cdfe41557372b9e219f2bc073096bd5570c4c5e14e1ee9c0", + "policy_identity_digest": "ca260a65ac58ac02aadb294b75dc34e410078f96a7cb97e99d851e11d8f3c812", + "policy_version": 8, + "sync_confirmed": true, + "sync_seconds": 0.09096788599981664, + "total_gpu_memory_mib": 11869 + } + ], + "strictly_monotonic_memory_growth": false + }, + "schema_version": 1, + "source": { + "commit": "690db9b079bfecfec14dc3bedc3aa0308cbacf60", + "dirty": false, + "miniverl_version": "0.11.0.dev0", + "wheel_sha256": "266fcd59bb3e85b02974a92b26c9a4e59c7c9b9205853d17baadcfffdb898e27" + }, + "sync_confirmed": true, + "teardown": { + "backend_state": "closed", + "port": 35829, + "port_closed": true + }, + "timing": { + "actor_load_seconds": 6.018847035000363, + "engine_lifecycle": { + "adapter_name": "miniverl-p00000008-2eb19589bfeb", + "adapter_sync_seconds": 0.02843916799974977, + "numerical_equivalence_class": "bf16-external-vs-nf4-actor-direct-gkd", + "policy_identity_digest": "ca260a65ac58ac02aadb294b75dc34e410078f96a7cb97e99d851e11d8f3c812", + "policy_version": 8, + "prefix_cache_enabled": false, + "startup_seconds": 8.508000064466614e-06, + "sync_total_seconds": 0.09089016699999775 + }, + "initial_sync_seconds": 17.573423056000138, + "teardown_seconds": 0.7304750389994297 + }, + "workload": { + "logical_prompts": 4, + "manifest_version": "rollout-runtime-v2-workload-v1", + "physical_concurrency": 4, + "prompt_lengths": [ + 128, + 512 + ], + "response_bounds": [ + 64, + 256, + 512 + ], + "run_seed": 20260829, + "samples_per_prompt": [ + 1, + 4 + ], + "sampling": [ + "greedy", + "seeded_stochastic" + ] + } +} diff --git a/benchmarks/results/rollout-runtime-v2-rtx4080.json b/benchmarks/results/rollout-runtime-v2-rtx4080.json new file mode 100644 index 0000000..dcd8978 --- /dev/null +++ b/benchmarks/results/rollout-runtime-v2-rtx4080.json @@ -0,0 +1,468 @@ +{ + "artifacts": { + "hf_cached_raw_sha256": "a9f8f0b0275d940f30497a0d88d76da0e112ffbe2bc1b37a42c6ed313b852242", + "hf_reference_sha256": "b25daee7ee726b7a7be18d7dbe26590fb61325bf212cfd9e7110b69f5fa8889c", + "vllm_raw_sha256": "abf14d73a7289f9619515943b4c8d7dafe7cd187fefe928655739ac7ed1ab16c" + }, + "backend_selection": { + "hf_cached": { + "initial_sync_seconds": 7.161399999944251e-05, + "peak_total_gpu_memory_mib": 4598, + "status": "supported_local_backend_performance_gate_failed" + }, + "selected": "vllm", + "sglang": { + "blockers": [ + "FlashInfer and system CUDA compatibility failed in the WSL2 spike", + "the Triton fallback expected /usr/local/cuda/bin/nvcc" + ], + "status": "not_selected_install_blocked_before_measurement", + "throughput": null, + "version_attempted": "0.5.18" + }, + "supported_scope": "managed direct GKD on the qualified WSL2 RTX 4080 stack", + "vllm": { + "initial_managed_sync_seconds": 17.573423056000138, + "isolated_startup_seconds": null, + "peak_total_gpu_memory_mib": 11820, + "pg_k1_supported": false, + "port_closed": true, + "startup_measurement_note": "The raw lifecycle startup field was overwritten by later no-op start checks. Initial managed sync is valid; isolated startup is not reported.", + "status": "selected_direct_gkd_backend", + "version": "0.28.0" + } + }, + "cells": [ + { + "cell_id": "p128-r256-n1-greedy", + "hf_cached_output_tokens_per_second": 68.11485389689372, + "hf_cached_peak_total_gpu_memory_mib": 3518, + "hf_cached_speedup_over_hf_reference": 0.7105217066831933, + "hf_reference_output_tokens_per_second": 95.86597180100608, + "prompt_length": 128, + "response_bound": 256, + "samples_per_prompt": 1, + "sampling": "greedy", + "vllm_output_tokens_per_second": 117.38034357648701, + "vllm_peak_total_gpu_memory_mib": 11820, + "vllm_speedup_over_hf_cached": 1.723270870611674 + }, + { + "cell_id": "p128-r256-n1-seeded_stochastic", + "hf_cached_output_tokens_per_second": 55.472629902728954, + "hf_cached_peak_total_gpu_memory_mib": 3516, + "hf_cached_speedup_over_hf_reference": 1.8522331122295925, + "hf_reference_output_tokens_per_second": 29.949054218102585, + "prompt_length": 128, + "response_bound": 256, + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 113.29306486751634, + "vllm_peak_total_gpu_memory_mib": 11809, + "vllm_speedup_over_hf_cached": 2.0423236660344983 + }, + { + "cell_id": "p128-r256-n4-greedy", + "hf_cached_output_tokens_per_second": 65.20882135675541, + "hf_cached_peak_total_gpu_memory_mib": 3521, + "hf_cached_speedup_over_hf_reference": 0.7060930743951971, + "hf_reference_output_tokens_per_second": 92.3515946004851, + "prompt_length": 128, + "response_bound": 256, + "samples_per_prompt": 4, + "sampling": "greedy", + "vllm_output_tokens_per_second": 115.34775213913979, + "vllm_peak_total_gpu_memory_mib": 11815, + "vllm_speedup_over_hf_cached": 1.7688979763654347 + }, + { + "cell_id": "p128-r256-n4-seeded_stochastic", + "hf_cached_output_tokens_per_second": 55.2931887502175, + "hf_cached_peak_total_gpu_memory_mib": 3518, + "hf_cached_speedup_over_hf_reference": 1.8387231471804049, + "hf_reference_output_tokens_per_second": 30.07151393890211, + "prompt_length": 128, + "response_bound": 256, + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 113.05345525301847, + "vllm_peak_total_gpu_memory_mib": 11816, + "vllm_speedup_over_hf_cached": 2.044618113159078 + }, + { + "cell_id": "p128-r512-n1-greedy", + "hf_cached_output_tokens_per_second": 69.6443086357772, + "hf_cached_peak_total_gpu_memory_mib": 3890, + "hf_cached_speedup_over_hf_reference": 0.7475940861876353, + "hf_reference_output_tokens_per_second": 93.15791807681244, + "prompt_length": 128, + "response_bound": 512, + "samples_per_prompt": 1, + "sampling": "greedy", + "vllm_output_tokens_per_second": 114.77861991131019, + "vllm_peak_total_gpu_memory_mib": 11815, + "vllm_speedup_over_hf_cached": 1.6480689112957452 + }, + { + "cell_id": "p128-r512-n1-seeded_stochastic", + "hf_cached_output_tokens_per_second": 55.648467475598814, + "hf_cached_peak_total_gpu_memory_mib": 3882, + "hf_cached_speedup_over_hf_reference": 1.8410290799695297, + "hf_reference_output_tokens_per_second": 30.226826985546495, + "prompt_length": 128, + "response_bound": 512, + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 114.25483283023517, + "vllm_peak_total_gpu_memory_mib": 11814, + "vllm_speedup_over_hf_cached": 2.0531532675241153 + }, + { + "cell_id": "p128-r512-n4-greedy", + "hf_cached_output_tokens_per_second": 62.149572817204685, + "hf_cached_peak_total_gpu_memory_mib": 3949, + "hf_cached_speedup_over_hf_reference": 0.6689355114418016, + "hf_reference_output_tokens_per_second": 92.90816790881611, + "prompt_length": 128, + "response_bound": 512, + "samples_per_prompt": 4, + "sampling": "greedy", + "vllm_output_tokens_per_second": 115.93193537752563, + "vllm_peak_total_gpu_memory_mib": 11817, + "vllm_speedup_over_hf_cached": 1.865369786506924 + }, + { + "cell_id": "p128-r512-n4-seeded_stochastic", + "hf_cached_output_tokens_per_second": 55.69275151332096, + "hf_cached_peak_total_gpu_memory_mib": 3891, + "hf_cached_speedup_over_hf_reference": 1.8382802161434009, + "hf_reference_output_tokens_per_second": 30.296116459416037, + "prompt_length": 128, + "response_bound": 512, + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 111.88806167938358, + "vllm_peak_total_gpu_memory_mib": 11815, + "vllm_speedup_over_hf_cached": 2.0090237713003183 + }, + { + "cell_id": "p128-r64-n1-greedy", + "hf_cached_output_tokens_per_second": 65.5164231644341, + "hf_cached_peak_total_gpu_memory_mib": 3385, + "hf_cached_speedup_over_hf_reference": 0.6865895392370194, + "hf_reference_output_tokens_per_second": 95.42298479705937, + "prompt_length": 128, + "response_bound": 64, + "samples_per_prompt": 1, + "sampling": "greedy", + "vllm_output_tokens_per_second": 113.61810732270409, + "vllm_peak_total_gpu_memory_mib": 11813, + "vllm_speedup_over_hf_cached": 1.7341927693082948 + }, + { + "cell_id": "p128-r64-n1-seeded_stochastic", + "hf_cached_output_tokens_per_second": 55.2850962814506, + "hf_cached_peak_total_gpu_memory_mib": 3385, + "hf_cached_speedup_over_hf_reference": 1.8470802331123204, + "hf_reference_output_tokens_per_second": 29.931074617314, + "prompt_length": 128, + "response_bound": 64, + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 109.33217932192946, + "vllm_peak_total_gpu_memory_mib": 11798, + "vllm_speedup_over_hf_cached": 1.9776067453212138 + }, + { + "cell_id": "p128-r64-n4-greedy", + "hf_cached_output_tokens_per_second": 63.53859968171568, + "hf_cached_peak_total_gpu_memory_mib": 3405, + "hf_cached_speedup_over_hf_reference": 0.6545762180626561, + "hf_reference_output_tokens_per_second": 97.06829843248867, + "prompt_length": 128, + "response_bound": 64, + "samples_per_prompt": 4, + "sampling": "greedy", + "vllm_output_tokens_per_second": 112.68983675011486, + "vllm_peak_total_gpu_memory_mib": 11798, + "vllm_speedup_over_hf_cached": 1.7735650032360297 + }, + { + "cell_id": "p128-r64-n4-seeded_stochastic", + "hf_cached_output_tokens_per_second": 55.07424019437444, + "hf_cached_peak_total_gpu_memory_mib": 3400, + "hf_cached_speedup_over_hf_reference": 1.8390902322318687, + "hf_reference_output_tokens_per_second": 29.94645897691375, + "prompt_length": 128, + "response_bound": 64, + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 113.56731350743463, + "vllm_peak_total_gpu_memory_mib": 11816, + "vllm_speedup_over_hf_cached": 2.0620768095323623 + }, + { + "cell_id": "p512-r256-n1-greedy", + "hf_cached_output_tokens_per_second": 66.46500687688993, + "hf_cached_peak_total_gpu_memory_mib": 4038, + "hf_cached_speedup_over_hf_reference": 0.8297837938514568, + "hf_reference_output_tokens_per_second": 80.09918652230043, + "prompt_length": 512, + "response_bound": 256, + "samples_per_prompt": 1, + "sampling": "greedy", + "vllm_output_tokens_per_second": 117.30129721271067, + "vllm_peak_total_gpu_memory_mib": 11812, + "vllm_speedup_over_hf_cached": 1.7648579715034478 + }, + { + "cell_id": "p512-r256-n1-seeded_stochastic", + "hf_cached_output_tokens_per_second": 54.99365453676284, + "hf_cached_peak_total_gpu_memory_mib": 4044, + "hf_cached_speedup_over_hf_reference": 1.8142708817513173, + "hf_reference_output_tokens_per_second": 30.31171094124457, + "prompt_length": 512, + "response_bound": 256, + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 113.70854206155697, + "vllm_peak_total_gpu_memory_mib": 11810, + "vllm_speedup_over_hf_cached": 2.0676665884341925 + }, + { + "cell_id": "p512-r256-n4-greedy", + "hf_cached_output_tokens_per_second": 65.56660170488767, + "hf_cached_peak_total_gpu_memory_mib": 4136, + "hf_cached_speedup_over_hf_reference": 0.8488332510651431, + "hf_reference_output_tokens_per_second": 77.24320603912795, + "prompt_length": 512, + "response_bound": 256, + "samples_per_prompt": 4, + "sampling": "greedy", + "vllm_output_tokens_per_second": 111.7921494953812, + "vllm_peak_total_gpu_memory_mib": 11818, + "vllm_speedup_over_hf_cached": 1.7050166790487729 + }, + { + "cell_id": "p512-r256-n4-seeded_stochastic", + "hf_cached_output_tokens_per_second": 25.184238708845193, + "hf_cached_peak_total_gpu_memory_mib": 4130, + "hf_cached_speedup_over_hf_reference": 0.8377758608430231, + "hf_reference_output_tokens_per_second": 30.060831167304364, + "prompt_length": 512, + "response_bound": 256, + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 114.48412504261995, + "vllm_peak_total_gpu_memory_mib": 11817, + "vllm_speedup_over_hf_cached": 4.54586403687521 + }, + { + "cell_id": "p512-r512-n1-greedy", + "hf_cached_output_tokens_per_second": 69.66546343278917, + "hf_cached_peak_total_gpu_memory_mib": 4588, + "hf_cached_speedup_over_hf_reference": 1.0510089818096247, + "hf_reference_output_tokens_per_second": 66.28436544171045, + "prompt_length": 512, + "response_bound": 512, + "samples_per_prompt": 1, + "sampling": "greedy", + "vllm_output_tokens_per_second": 117.70676303670209, + "vllm_peak_total_gpu_memory_mib": 11811, + "vllm_speedup_over_hf_cached": 1.6895999428793222 + }, + { + "cell_id": "p512-r512-n1-seeded_stochastic", + "hf_cached_output_tokens_per_second": 56.988484073813574, + "hf_cached_peak_total_gpu_memory_mib": 4598, + "hf_cached_speedup_over_hf_reference": 1.9047093935244002, + "hf_reference_output_tokens_per_second": 29.919778979177657, + "prompt_length": 512, + "response_bound": 512, + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 108.5269135591943, + "vllm_peak_total_gpu_memory_mib": 11817, + "vllm_speedup_over_hf_cached": 1.9043656858572735 + }, + { + "cell_id": "p512-r512-n4-greedy", + "hf_cached_output_tokens_per_second": 67.45299219362623, + "hf_cached_peak_total_gpu_memory_mib": 4595, + "hf_cached_speedup_over_hf_reference": 1.0179514827749148, + "hf_reference_output_tokens_per_second": 66.26346474760345, + "prompt_length": 512, + "response_bound": 512, + "samples_per_prompt": 4, + "sampling": "greedy", + "vllm_output_tokens_per_second": 116.05673623038184, + "vllm_peak_total_gpu_memory_mib": 11819, + "vllm_speedup_over_hf_cached": 1.7205572719033253 + }, + { + "cell_id": "p512-r512-n4-seeded_stochastic", + "hf_cached_output_tokens_per_second": 56.88412406055206, + "hf_cached_peak_total_gpu_memory_mib": 4595, + "hf_cached_speedup_over_hf_reference": 1.8764463166201748, + "hf_reference_output_tokens_per_second": 30.31481559409109, + "prompt_length": 512, + "response_bound": 512, + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 114.40795416822156, + "vllm_peak_total_gpu_memory_mib": 11817, + "vllm_speedup_over_hf_cached": 2.011245774769011 + }, + { + "cell_id": "p512-r64-n1-greedy", + "hf_cached_output_tokens_per_second": 65.60562672890258, + "hf_cached_peak_total_gpu_memory_mib": 3892, + "hf_cached_speedup_over_hf_reference": 0.7078862746387611, + "hf_reference_output_tokens_per_second": 92.67820139948546, + "prompt_length": 512, + "response_bound": 64, + "samples_per_prompt": 1, + "sampling": "greedy", + "vllm_output_tokens_per_second": 114.91344988510029, + "vllm_peak_total_gpu_memory_mib": 11803, + "vllm_speedup_over_hf_cached": 1.7515791802424034 + }, + { + "cell_id": "p512-r64-n1-seeded_stochastic", + "hf_cached_output_tokens_per_second": 55.367899026638916, + "hf_cached_peak_total_gpu_memory_mib": 3879, + "hf_cached_speedup_over_hf_reference": 1.8288352683944704, + "hf_reference_output_tokens_per_second": 30.27495148606044, + "prompt_length": 512, + "response_bound": 64, + "samples_per_prompt": 1, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 114.23743978764963, + "vllm_peak_total_gpu_memory_mib": 11805, + "vllm_speedup_over_hf_cached": 2.0632431751236773 + }, + { + "cell_id": "p512-r64-n4-greedy", + "hf_cached_output_tokens_per_second": 63.724442847875615, + "hf_cached_peak_total_gpu_memory_mib": 3885, + "hf_cached_speedup_over_hf_reference": 0.7603399854578152, + "hf_reference_output_tokens_per_second": 83.81045856677643, + "prompt_length": 512, + "response_bound": 64, + "samples_per_prompt": 4, + "sampling": "greedy", + "vllm_output_tokens_per_second": 114.48153987880283, + "vllm_peak_total_gpu_memory_mib": 11813, + "vllm_speedup_over_hf_cached": 1.7965090750514001 + }, + { + "cell_id": "p512-r64-n4-seeded_stochastic", + "hf_cached_output_tokens_per_second": 56.4061689499919, + "hf_cached_peak_total_gpu_memory_mib": 3884, + "hf_cached_speedup_over_hf_reference": 1.86698066165715, + "hf_reference_output_tokens_per_second": 30.212508414482045, + "prompt_length": 512, + "response_bound": 64, + "samples_per_prompt": 4, + "sampling": "seeded_stochastic", + "vllm_output_tokens_per_second": 109.28946923356665, + "vllm_peak_total_gpu_memory_mib": 11800, + "vllm_speedup_over_hf_cached": 1.9375446208810878 + } + ], + "environment": { + "cuda_runtime": "13.0", + "driver_version": "596.49", + "gpu": "NVIDIA GeForce RTX 4080", + "gpu_total_memory_bytes": 17170956288, + "packages": { + "accelerate": "1.14.0", + "bitsandbytes": "0.50.2", + "peft": "0.20.0", + "torch": "2.13.0+cu130", + "transformers": "5.14.1", + "vllm": "0.28.0" + }, + "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", + "python": "3.12.13" + }, + "frozen_calculator_sha256": "53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc", + "gate_interpretation": "A response-length gate passes only when every paired prompt-length, samples-per-prompt and sampling cell reaches the preregistered threshold.", + "gates": { + "hf_cached_speedup_over_hf_reference": { + "passed": false, + "responses": { + "256": { + "observed_maximum_speedup": 1.8522331122295925, + "observed_minimum_speedup": 0.7060930743951971, + "passed": false, + "required_minimum_speedup": 2.0 + }, + "512": { + "observed_maximum_speedup": 1.9047093935244002, + "observed_minimum_speedup": 0.6689355114418016, + "passed": false, + "required_minimum_speedup": 2.0 + } + } + }, + "peak_total_gpu_memory": { + "hf_cached_peak_mib": 4598, + "limit_mib": 14848, + "passed": true, + "vllm_peak_mib": 11820 + }, + "selected_external_engine_advantage_over_hf_cached": { + "observed_maximum_speedup": 4.54586403687521, + "observed_minimum_speedup": 1.6480689112957452, + "passed": true, + "required_minimum_speedup": 1.2 + }, + "vllm_pg_logprob_conformance": { + "effect": "vllm remains limited to direct GKD; PG-k1 fails closed", + "maximum_absolute_difference": 0.01942583080381155, + "mean_absolute_difference": 0.014255227812100202, + "p99_absolute_difference": 0.01942583080381155, + "passed": false, + "threshold": 0.01, + "token_agreement_fraction": 1.0 + }, + "vllm_policy_refresh": { + "all_policy_identities_unique": true, + "all_syncs_confirmed": true, + "cycles": 8, + "passed": true, + "strictly_monotonic_memory_growth": false + } + }, + "measurement_status": "completed_release_blocked", + "models": { + "actor": { + "id": "Qwen/Qwen3-0.6B", + "revision": "c1899de289a04d12100db370d81485cdf75e47ca", + "tokenizer_revision": "c1899de289a04d12100db370d81485cdf75e47ca" + }, + "attention_implementation": "sdpa", + "dtype": "bfloat16", + "quantization": "nf4", + "teacher": { + "id": "Qwen/Qwen3-1.7B", + "revision": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e", + "tokenizer_revision": "70d244cc86ccca08cf5af4e1e306ecf908b1ad5e" + } + }, + "name": "rollout-runtime-v2-rtx4080", + "preregistration_sha256": "8cc3ba738c69b59ed19c22c1de874fd00249404198a3e05983477dc8899bb7e5", + "release_decision": { + "action": "publish evidence, optimize hf_cached, rerun the frozen workload", + "blocking_gate": "hf_cached_speedup_over_hf_reference", + "v0_11_0_publishable": false + }, + "schema_version": 1, + "source": { + "commit": "690db9b079bfecfec14dc3bedc3aa0308cbacf60", + "dirty": false, + "miniverl_version": "0.11.0.dev0", + "wheel_sha256": "266fcd59bb3e85b02974a92b26c9a4e59c7c9b9205853d17baadcfffdb898e27" + } +} diff --git a/benchmarks/schema/rollout-runtime-v2-result.schema.json b/benchmarks/schema/rollout-runtime-v2-result.schema.json new file mode 100644 index 0000000..2393c0a --- /dev/null +++ b/benchmarks/schema/rollout-runtime-v2-result.schema.json @@ -0,0 +1,31 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://daoyuanli2816.github.io/mini-verl/schemas/rollout-runtime-v2-result.schema.json", + "title": "Rollout Runtime v2 RTX 4080 aggregate", + "type": "object", + "required": [ + "schema_version", + "name", + "measurement_status", + "source", + "artifacts", + "gates", + "backend_selection", + "release_decision", + "cells" + ], + "properties": { + "schema_version": {"const": 1}, + "name": {"const": "rollout-runtime-v2-rtx4080"}, + "measurement_status": {"const": "completed_release_blocked"}, + "cells": {"type": "array", "minItems": 24, "maxItems": 24}, + "release_decision": { + "type": "object", + "required": ["v0_11_0_publishable", "blocking_gate", "action"], + "properties": { + "v0_11_0_publishable": {"const": false}, + "blocking_gate": {"const": "hf_cached_speedup_over_hf_reference"} + } + } + } +} diff --git a/benchmarks/schema/rollout-runtime-v2-selected.schema.json b/benchmarks/schema/rollout-runtime-v2-selected.schema.json new file mode 100644 index 0000000..fb68577 --- /dev/null +++ b/benchmarks/schema/rollout-runtime-v2-selected.schema.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://daoyuanli2816.github.io/mini-verl/schemas/rollout-runtime-v2-selected.schema.json", + "title": "Rollout Runtime v2 selected-backend raw evidence", + "type": "object", + "required": [ + "schema_version", + "measurement_status", + "source", + "environment", + "backend", + "workload", + "cells", + "memory", + "teardown", + "sync_confirmed" + ], + "properties": { + "schema_version": {"const": 1}, + "measurement_status": {"const": "completed"}, + "source": { + "type": "object", + "required": ["commit", "dirty", "miniverl_version", "wheel_sha256"], + "properties": { + "commit": {"type": "string", "pattern": "^[0-9a-f]{40}$"}, + "dirty": {"const": false}, + "miniverl_version": {"const": "0.11.0.dev0"}, + "wheel_sha256": {"type": "string", "pattern": "^[0-9a-f]{64}$"} + } + }, + "cells": { + "type": "array", + "minItems": 24, + "maxItems": 24, + "items": { + "type": "object", + "required": [ + "cell_id", + "prompt_length", + "response_bound", + "samples_per_prompt", + "sampling", + "rollout_seconds", + "output_tokens_per_second", + "output_token_ids_sha256", + "peak_total_gpu_memory_mib" + ], + "properties": { + "cell_id": {"type": "string"}, + "prompt_length": {"enum": [128, 512]}, + "response_bound": {"enum": [64, 256, 512]}, + "samples_per_prompt": {"enum": [1, 4]}, + "sampling": {"enum": ["greedy", "seeded_stochastic"]}, + "rollout_seconds": { + "type": "array", + "minItems": 3, + "maxItems": 3, + "items": {"type": "number", "exclusiveMinimum": 0} + }, + "output_tokens_per_second": {"type": "number", "exclusiveMinimum": 0}, + "output_token_ids_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "peak_total_gpu_memory_mib": {"type": "integer", "minimum": 0} + } + } + }, + "sync_confirmed": {"const": true} + } +} diff --git a/docs/benchmarks/rollout-runtime-v2.md b/docs/benchmarks/rollout-runtime-v2.md index 69321fe..4922d33 100644 --- a/docs/benchmarks/rollout-runtime-v2.md +++ b/docs/benchmarks/rollout-runtime-v2.md @@ -1,4 +1,4 @@ -# Rollout Runtime v2: frozen HF reference baseline +# Rollout Runtime v2: baseline and selected-backend evidence The pre-v0.11 Hugging Face path is now frozen as the compatibility and performance baseline for Rollout Runtime v2. All 24 preregistered cells @@ -70,3 +70,57 @@ baseline and cannot be inferred from it. See the [method](rollout-runtime-v2-method.md) for timing and invalidation rules. The next backend must use this workload unchanged and pass the preregistered correctness, memory and 256/512-token speedup gates. + +## Development-line backend measurement + +The exact `0.11.0.dev0` wheel at source `690db9b079bfecfec14dc3bedc3aa0308cbacf60` +ran the same 24 cells with `hf_cached` and managed vLLM 0.28.0. Both runs used +wheel SHA-256 `266fcd59bb3e85b02974a92b26c9a4e59c7c9b9205853d17baadcfffdb898e27` +and the baseline software stack: Python 3.12.13, Torch 2.13.0+cu130, +Transformers 5.14.1, PEFT 0.20.0 and bitsandbytes 0.50.2. + +| Response | Sampling | `hf_cached` / reference | vLLM / `hf_cached` | vLLM tokens/s | +| ---: | --- | ---: | ---: | ---: | +| 64 | greedy | 0.65–0.76× | 1.73–1.80× | 112.7–114.9 | +| 64 | seeded stochastic | 1.83–1.87× | 1.94–2.06× | 109.3–114.2 | +| 256 | greedy | 0.71–0.85× | 1.71–1.77× | 111.8–117.4 | +| 256 | seeded stochastic | 0.84–1.85× | 2.04–4.55× | 113.1–114.5 | +| 512 | greedy | 0.67–1.05× | 1.65–1.87× | 114.8–117.7 | +| 512 | seeded stochastic | 1.84–1.90× | 1.90–2.05× | 108.5–114.4 | + +Ranges contain both prompt lengths and both `n` values. One `hf_cached` +p512/r256/n4 stochastic cell had measured runs of 118.807, 267.930 and +162.641 seconds; the median and resulting 0.84× speedup remain in the result. + +### Gate outcome + +- `hf_cached >= 2× hf_reference` at response 256 and 512: **failed**. The + conservative preregistration interpretation requires every paired cell to + reach the threshold. +- vLLM `>= 1.2× hf_cached` at response 256 and 512: **passed** in every cell; + the observed range was 1.65–4.55×. +- Peak total GPU memory below 14.5 GiB: **passed**. `hf_cached` peaked at + 4,598 MiB and vLLM at 11,820 MiB. +- Eight vLLM refreshes: **passed**. Every sync and identity was unique, with no + monotonic memory growth; the localhost port closed at teardown. +- vLLM PG-k1 log-probability conformance: **failed**. Greedy token agreement + was 32/32, while maximum absolute log-probability difference was 0.0194 + against the 0.01 NF4 threshold. Direct GKD remains the supported scope. + +The failed local-backend speed gate blocks v0.11.0 publication. It does not +invalidate the completed runtime, grouped-sample, reward or managed-engine +implementation; it requires profiling `hf_cached` and rerunning this frozen +workload before release. + +## Candidate artifact integrity + +| Artifact | SHA-256 | +| --- | --- | +| `hf_cached` raw | `a9f8f0b0275d940f30497a0d88d76da0e112ffbe2bc1b37a42c6ed313b852242` | +| vLLM raw | `abf14d73a7289f9619515943b4c8d7dafe7cd187fefe928655739ac7ed1ab16c` | +| Aggregate result | `32be86856263ccdf787986c4ec54570d323af8c238dc1664e00a9ce41ca393c4` | +| Frozen calculator | `53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc` | + +The [aggregate JSON](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/rollout-runtime-v2-rtx4080.json) +contains every cell, exact raw hashes, gate calculation and backend-selection +state. `scripts/publish_rollout_runtime_v2_evidence.py --check` reproduces it. diff --git a/docs/competitive-landscape.md b/docs/competitive-landscape.md index 4c00787..7797cdc 100644 --- a/docs/competitive-landscape.md +++ b/docs/competitive-landscape.md @@ -10,19 +10,24 @@ Inspected: **2026-08-29**. | Project | Exact source inspected | Capability observed in source or stable documentation | miniVERL state at the start of v0.11 | Locally measured | Caveat | | --- | --- | --- | --- | --- | --- | -| [mini-opd](https://github.com/thu-nics/mini-opd) | `b47eaa1728fc3a7ff0b0b627ec2c93d60f07aa16` (`main`; no GitHub release) | SGLang HTTP rollouts, `n_rollouts` (default 4), task rewards, group reward processing, macro-batching, optional one-step-stale background rollout, and checkpoint weight reload | Local HF prompt rollouts; no typed grouped-sample contract and no external generation engine | **No** — source audit only | Its README starts SGLang in a separate terminal. Async measurements would not be compared with miniVERL's strict synchronous path without a separate staleness label. | -| [NVIDIA NeMo RL](https://github.com/NVIDIA-NeMo/RL) | stable `v0.7.0`, commit `81aa43dda4765b0429cf31dab44441e4e4383911`; current `main` also inspected at `46dfb92fb5ceed5b2d593d652361b10d0acc6ef6` | A documented single-GPU OPD example, vLLM and SGLang generation, refit/weight synchronization, multi-teacher MOPD, and broader Ray/distributed scope | One-GPU local trainer and a fail-closed portable verl bridge; no NeMo RL runtime integration | **No** — install and one-step feasibility remain to be measured | “Single GPU” is not evidence that the published example fits this repository's 16 GiB RTX 4080 envelope. The stable MOPD objective and runtime differ from miniVERL's closed profiles. | -| [SGLang](https://github.com/sgl-project/sglang) | stable `v0.5.18`; current `main` `0a585d5bb108cab8f0922b483d7f55812f05e245` | Generation server and engine APIs considered for a managed rollout backend | Not integrated | **No** | Public support requires a local lifecycle, raw-token, synchronization, cache-invalidation and teardown qualification; an API being present is not that qualification. | -| [vLLM](https://github.com/vllm-project/vllm) | stable `v0.28.0`; current `main` `cacc429f62c3738c9c95093e9bd410e96103221a` | Batched generation and model-serving APIs considered for a managed rollout backend | The verl-shaped config vocabulary may mention vLLM, but miniVERL currently executes local HF generation | **No** | No vLLM process or distributed verl job has been run by the v0.11 line yet. Support depends on measured policy refresh, cache invalidation, memory and teardown behavior. | +| [mini-opd](https://github.com/thu-nics/mini-opd) | `b47eaa1728fc3a7ff0b0b627ec2c93d60f07aa16` (`main`; no GitHub release) | SGLang HTTP rollouts, `n_rollouts` (default 4), task rewards, group reward processing, macro-batching, optional one-step-stale background rollout, and checkpoint weight reload | Typed grouped samples, deterministic rewards, local `hf_cached`, and managed vLLM direct-GKD generation | **No** — source audit only | Its documented separate-server and optional stale-async paths do not match miniVERL's managed strict-sync workload. No performance comparison is claimed. | +| [NVIDIA NeMo RL](https://github.com/NVIDIA-NeMo/RL) | stable `v0.7.0`, commit `81aa43dda4765b0429cf31dab44441e4e4383911`; current `main` `46dfb92fb5ceed5b2d593d652361b10d0acc6ef6` | A documented single-GPU OPD example, vLLM and SGLang generation, refit/weight synchronization, multi-teacher MOPD, and broader Ray/distributed scope | One-process consumer-GPU runtime with a portable, fail-closed verl bridge; no NeMo RL integration | **No** — setup and one-step run were not attempted after the source audit | Runtime, objective and distributed dependencies differ. There is no miniVERL-versus-NeMo performance claim. | +| [SGLang](https://github.com/sgl-project/sglang) | stable `v0.5.18` at `71de97b264b04dcd514cf904003028aefe9775c8`; current `main` `cdbfe90b4a6c728e03e6520862d792501b3a97bb` | Raw-token generation, batching and adapter lifecycle APIs suitable for a spike | Not selected | **Attempted; no throughput measurement** | The WSL2 spike stopped at FlashInfer/system-CUDA compatibility and a Triton fallback expecting `/usr/local/cuda/bin/nvcc`. Missing throughput is not represented as zero. | +| [vLLM](https://github.com/vllm-project/vllm) | stable `v0.28.0` at `2cf0a6915ce544dc493a0990f2ea38d81601128a`; current `main` `fd5d3aea9470bb92376eb2d9f5d64cd8f23de31b` | Batched raw-token generation and dynamic LoRA load/unload used by the managed backend | Selected external engine for the development-line direct-GKD path | **Yes** — full 24-cell WSL2 RTX 4080 workload | 108.5–117.7 tokens/s and 1.65–4.55× over `hf_cached`; 11,820 MiB peak total GPU memory. PG-k1 remains disabled because log-probability conformance failed. | ## Selection rule -`hf_reference` remains the compatibility oracle. A batched local `hf_cached` -backend is the first implementation target. SGLang and vLLM will be spiked on -the same WSL2 RTX 4080 workload; only one becomes a supported v0.11 backend, -and only if it has a measured advantage over `hf_cached` together with strict -policy synchronization and lifecycle cleanup. Until then, both are -**not measured** rather than recommended. +`hf_reference` remains the compatibility oracle and `hf_cached` remains the +service-free local backend. vLLM 0.28.0 passed the external-engine value, +strict-sync, memory and teardown gates for direct GKD, so it is the selected +engine. Eight policy refreshes used unique identities and showed no monotonic +memory growth. Its sampled-token log-probabilities exceeded the NF4 tolerance, +so PG-k1 continues to use `hf_cached`. + +The internal release gate is stricter than external-engine selection. +`hf_cached` did not reach the preregistered 2× speedup over `hf_reference` at +256/512 tokens, so v0.11.0 remains unpublished. mini-opd and NeMo RL were not +run locally, and no relative performance conclusion is made about either. The comparison workload, metrics and invalidation rules are preregistered in [`rollout-runtime-v2.yaml`](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/preregistration/rollout-runtime-v2.yaml). diff --git a/docs/release-checklist.md b/docs/release-checklist.md index caabe78..238adf4 100644 --- a/docs/release-checklist.md +++ b/docs/release-checklist.md @@ -15,16 +15,22 @@ after the exact release commit and its remote checks are green. `0256bd9e63ca6ed52999a5073a3577a008581a8d4418062314750d86f21cd5fe` before implementing `hf_cached`. - [x] Add typed policy identity, lifecycle and strict synchronization contracts. -- [x] Qualify `hf_cached` at 64/256/512 response bounds and require at least 2x +- [ ] Qualify `hf_cached` at 64/256/512 response bounds and require at least 2x speedup at 256 and 512 without exceeding 14.5 GiB peak reserved VRAM. + The exact 24-cell run at `690db9b` stayed below the memory limit but + failed the speed gate: greedy ranged from 0.67–1.05× and most seeded + 256/512-token cells reached about 1.8–1.9×. - [x] Add transactional `samples_per_prompt > 1` and exact no-skip/no-duplicate resume semantics without changing the published `n=1` profiles. -- [ ] Add the new deterministic task-reward plus distillation-advantage profile; +- [x] Add the new deterministic task-reward plus distillation-advantage profile; keep direct GKD and existing PG-k1 identities unchanged. -- [ ] Measure SGLang and vLLM on the same WSL2 RTX 4080 workload and support at - most one external engine after policy-sync, cache and teardown gates pass. +- [x] Attempt SGLang and measure vLLM on the WSL2 RTX 4080 workload. SGLang + stopped at CUDA-toolchain compatibility before throughput measurement; + vLLM 0.28.0 passed the direct-GKD value, sync, memory and teardown gates + and remains fail-closed for PG log-probabilities. - [ ] Run exact-candidate-wheel full qualification, dry-run and publication - only after every v0.11 acceptance gate passes. + only after every v0.11 acceptance gate passes. Publication is blocked on + the failed `hf_cached` performance gate. ## v0.10.2 development (superseded by v0.11 scope) diff --git a/docs/rollout-runtime-v2.md b/docs/rollout-runtime-v2.md index 5dc38d3..7e27744 100644 --- a/docs/rollout-runtime-v2.md +++ b/docs/rollout-runtime-v2.md @@ -89,8 +89,36 @@ token IDs, and terminates the complete process group before teacher scoring or the actor update. Prefix caching is disabled, and a failed policy refresh closes the server before another request can run. -This backend is qualified for direct GKD. Use `hf_cached` for PG-k1: vLLM's -BF16 rollout and the NF4 training actor are a recorded numerical-equivalence -class, not sampled-logprob identity. The release-chain qualification measured -throughput separately from the conformance probes; the frozen pre-change -`hf_reference` result remains the compatibility baseline. +The development-line evidence supports this backend for direct GKD. Use +`hf_cached` for PG-k1: vLLM's BF16 rollout and the NF4 training actor agreed on +all 32 greedy probe tokens, but the maximum sampled-token log-probability +difference was 0.0194 against the preregistered 0.01 limit. + +## Measured RTX 4080 envelope + +The exact `0.11.0.dev0` wheel ran the frozen 24-cell workload on WSL2 with +Qwen3-0.6B NF4/BF16, Transformers 5.14.1 and vLLM 0.28.0. Each cell has one +warmup and three measured repetitions. + +| Response | Sampling | `hf_cached` / `hf_reference` | vLLM / `hf_cached` | vLLM tokens/s | +| ---: | --- | ---: | ---: | ---: | +| 64 | greedy | 0.65–0.76× | 1.73–1.80× | 112.7–114.9 | +| 64 | seeded stochastic | 1.83–1.87× | 1.94–2.06× | 109.3–114.2 | +| 256 | greedy | 0.71–0.85× | 1.71–1.77× | 111.8–117.4 | +| 256 | seeded stochastic | 0.84–1.85× | 2.04–4.55× | 113.1–114.5 | +| 512 | greedy | 0.67–1.05× | 1.65–1.87× | 114.8–117.7 | +| 512 | seeded stochastic | 1.84–1.90× | 1.90–2.05× | 108.5–114.4 | + +vLLM peaked at 11,820 MiB of total GPU memory, confirmed eight unique policy +refreshes without monotonic growth, and closed its localhost port. Initial +managed synchronization, including server startup and first adapter load, took +17.57 seconds. The isolated startup field was overwritten by later no-op start +checks in this measurement, so no standalone startup number is reported. + +The external-engine value gate passed in every 256/512-token cell. The separate +`hf_cached >= 2× hf_reference` gate did not: the best seeded-stochastic cells +reached about 1.9× and the greedy path was usually slower than the legacy +oracle. That failed gate blocks v0.11.0 publication while `hf_cached` is +profiled and optimized. The [benchmark report](benchmarks/rollout-runtime-v2.md) +and [machine-readable result](https://github.com/DaoyuanLi2816/mini-verl/blob/main/benchmarks/results/rollout-runtime-v2-rtx4080.json) +contain every cell and raw artifact hash. diff --git a/scripts/publish_rollout_runtime_v2_evidence.py b/scripts/publish_rollout_runtime_v2_evidence.py new file mode 100644 index 0000000..91fed4a --- /dev/null +++ b/scripts/publish_rollout_runtime_v2_evidence.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Validate and aggregate the measured Rollout Runtime v2 evidence.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +PREREGISTRATION = ROOT / "benchmarks/preregistration/rollout-runtime-v2.yaml" +BASELINE = ROOT / "benchmarks/results/rollout-runtime-v2-hf-reference.json" +HF_CACHED = ROOT / "benchmarks/evidence/rollout-runtime-v2/hf-cached-rtx4080-raw.json" +VLLM = ROOT / "benchmarks/evidence/rollout-runtime-v2/vllm-rtx4080-raw.json" +OUTPUT = ROOT / "benchmarks/results/rollout-runtime-v2-rtx4080.json" +FROZEN_CALCULATOR = ROOT / "benchmarks/results/gpu-calc-hard-equal-update-v2.json" +FROZEN_CALCULATOR_SHA256 = "53fc1d4d5b7adee09618d77ad62d4086ba56b78569832d6fc7c3bcd5c2695bbc" + + +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _load(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"expected a JSON object: {path}") + return payload + + +def _bytes(payload: dict[str, Any]) -> bytes: + return ( + json.dumps(payload, indent=2, sort_keys=True, ensure_ascii=False, allow_nan=False) + "\n" + ).encode("utf-8") + + +def _expected_cells(preregistration: dict[str, Any]) -> set[str]: + workload = preregistration["workload"] + return { + f"p{prompt}-r{response}-n{samples}-{sampling['name']}" + for prompt in workload["prompt_lengths"] + for response in workload["response_bounds"] + for samples in workload["samples_per_prompt"] + for sampling in workload["sampling"] + } + + +def _cells(payload: dict[str, Any], expected: set[str]) -> dict[str, dict[str, Any]]: + rows = payload.get("cells") + if not isinstance(rows, list): + raise ValueError("benchmark evidence has no cell list") + by_id = {str(row["cell_id"]): row for row in rows} + if len(by_id) != len(rows) or set(by_id) != expected: + raise ValueError("benchmark evidence does not match the preregistered 24-cell matrix") + return by_id + + +def build_result() -> dict[str, Any]: + preregistration = yaml.safe_load(PREREGISTRATION.read_text(encoding="utf-8")) + baseline = _load(BASELINE) + hf_cached = _load(HF_CACHED) + vllm = _load(VLLM) + expected = _expected_cells(preregistration) + if len(expected) != 24: + raise ValueError("the preregistered workload is no longer the frozen 24-cell matrix") + baseline_cells = _cells(baseline, expected) + hf_cells = _cells(hf_cached, expected) + vllm_cells = _cells(vllm, expected) + if hf_cached["source"] != vllm["source"]: + raise ValueError("selected backends were not measured from the same source and wheel") + if hf_cached["environment"] != vllm["environment"]: + raise ValueError("selected backends were not measured in the same software environment") + if _sha256(FROZEN_CALCULATOR) != FROZEN_CALCULATOR_SHA256: + raise ValueError("the immutable calculator benchmark changed") + + cells: list[dict[str, Any]] = [] + for cell_id in sorted(expected): + baseline_cell = baseline_cells[cell_id] + hf_cell = hf_cells[cell_id] + vllm_cell = vllm_cells[cell_id] + baseline_rate = float(baseline_cell["rates"]["output_tokens_per_second"]) + hf_rate = float(hf_cell["output_tokens_per_second"]) + vllm_rate = float(vllm_cell["output_tokens_per_second"]) + cells.append( + { + "cell_id": cell_id, + "prompt_length": hf_cell["prompt_length"], + "response_bound": hf_cell["response_bound"], + "samples_per_prompt": hf_cell["samples_per_prompt"], + "sampling": hf_cell["sampling"], + "hf_reference_output_tokens_per_second": baseline_rate, + "hf_cached_output_tokens_per_second": hf_rate, + "vllm_output_tokens_per_second": vllm_rate, + "hf_cached_speedup_over_hf_reference": hf_rate / baseline_rate, + "vllm_speedup_over_hf_cached": vllm_rate / hf_rate, + "hf_cached_peak_total_gpu_memory_mib": hf_cell["peak_total_gpu_memory_mib"], + "vllm_peak_total_gpu_memory_mib": vllm_cell["peak_total_gpu_memory_mib"], + } + ) + + hf_gate_rows: dict[str, dict[str, Any]] = {} + for response in (256, 512): + speedups = [ + row["hf_cached_speedup_over_hf_reference"] + for row in cells + if row["response_bound"] == response + ] + hf_gate_rows[str(response)] = { + "required_minimum_speedup": 2.0, + "observed_minimum_speedup": min(speedups), + "observed_maximum_speedup": max(speedups), + "passed": all(value >= 2.0 for value in speedups), + } + external_speedups = [ + row["vllm_speedup_over_hf_cached"] for row in cells if row["response_bound"] in (256, 512) + ] + vllm_conformance = vllm["conformance"] + refresh = vllm["refresh_probe"] + hf_peak = int(hf_cached["memory"]["peak_total_gpu_memory_mib"]) + vllm_peak = int(vllm["memory"]["peak_total_gpu_memory_mib"]) + memory_limit_mib = int(float(preregistration["hardware"]["peak_reserved_limit_gib"]) * 1024) + hf_passed = all(row["passed"] for row in hf_gate_rows.values()) + external_passed = all(value >= 1.2 for value in external_speedups) + return { + "schema_version": 1, + "name": "rollout-runtime-v2-rtx4080", + "measurement_status": "completed_release_blocked", + "source": hf_cached["source"], + "environment": hf_cached["environment"], + "models": hf_cached["models"], + "preregistration_sha256": _sha256(PREREGISTRATION), + "frozen_calculator_sha256": FROZEN_CALCULATOR_SHA256, + "artifacts": { + "hf_reference_sha256": _sha256(BASELINE), + "hf_cached_raw_sha256": _sha256(HF_CACHED), + "vllm_raw_sha256": _sha256(VLLM), + }, + "gate_interpretation": ( + "A response-length gate passes only when every paired prompt-length, " + "samples-per-prompt and sampling cell reaches the preregistered threshold." + ), + "gates": { + "hf_cached_speedup_over_hf_reference": { + "responses": hf_gate_rows, + "passed": hf_passed, + }, + "selected_external_engine_advantage_over_hf_cached": { + "required_minimum_speedup": 1.2, + "observed_minimum_speedup": min(external_speedups), + "observed_maximum_speedup": max(external_speedups), + "passed": external_passed, + }, + "peak_total_gpu_memory": { + "limit_mib": memory_limit_mib, + "hf_cached_peak_mib": hf_peak, + "vllm_peak_mib": vllm_peak, + "passed": max(hf_peak, vllm_peak) <= memory_limit_mib, + }, + "vllm_policy_refresh": { + "cycles": len(refresh["cycles"]), + "all_syncs_confirmed": refresh["all_syncs_confirmed"], + "all_policy_identities_unique": refresh["all_policy_identities_unique"], + "strictly_monotonic_memory_growth": refresh["strictly_monotonic_memory_growth"], + "passed": bool( + refresh["all_syncs_confirmed"] + and refresh["all_policy_identities_unique"] + and not refresh["strictly_monotonic_memory_growth"] + ), + }, + "vllm_pg_logprob_conformance": { + "threshold": vllm_conformance["pg_nf4_threshold"], + "maximum_absolute_difference": vllm_conformance[ + "sampled_logprob_max_abs_difference" + ], + "mean_absolute_difference": vllm_conformance["sampled_logprob_mean_abs_difference"], + "p99_absolute_difference": vllm_conformance["sampled_logprob_p99_abs_difference"], + "token_agreement_fraction": vllm_conformance["token_agreement_fraction"], + "passed": vllm_conformance["pg_threshold_passed"], + "effect": "vllm remains limited to direct GKD; PG-k1 fails closed", + }, + }, + "backend_selection": { + "selected": "vllm", + "supported_scope": "managed direct GKD on the qualified WSL2 RTX 4080 stack", + "hf_cached": { + "status": "supported_local_backend_performance_gate_failed", + "initial_sync_seconds": hf_cached["timing"]["initial_sync_seconds"], + "peak_total_gpu_memory_mib": hf_peak, + }, + "vllm": { + "status": "selected_direct_gkd_backend", + "version": vllm["environment"]["packages"]["vllm"], + "initial_managed_sync_seconds": vllm["timing"]["initial_sync_seconds"], + "isolated_startup_seconds": None, + "startup_measurement_note": ( + "The raw lifecycle startup field was overwritten by later no-op start " + "checks. Initial managed sync is valid; isolated startup is not reported." + ), + "peak_total_gpu_memory_mib": vllm_peak, + "port_closed": vllm["teardown"]["port_closed"], + "pg_k1_supported": False, + }, + "sglang": { + "status": "not_selected_install_blocked_before_measurement", + "version_attempted": "0.5.18", + "throughput": None, + "blockers": [ + "FlashInfer and system CUDA compatibility failed in the WSL2 spike", + "the Triton fallback expected /usr/local/cuda/bin/nvcc", + ], + }, + }, + "release_decision": { + "v0_11_0_publishable": False, + "blocking_gate": "hf_cached_speedup_over_hf_reference", + "action": "publish evidence, optimize hf_cached, rerun the frozen workload", + }, + "cells": cells, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + rendered = _bytes(build_result()) + if args.check: + if not OUTPUT.is_file() or OUTPUT.read_bytes() != rendered: + raise SystemExit(f"generated artifact differs: {OUTPUT}") + else: + OUTPUT.write_bytes(rendered) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/miniverl/runtime/backends/vllm.py b/src/miniverl/runtime/backends/vllm.py index 10ce841..8e926ad 100644 --- a/src/miniverl/runtime/backends/vllm.py +++ b/src/miniverl/runtime/backends/vllm.py @@ -509,6 +509,9 @@ def synchronize(self, snapshot: PolicySnapshot) -> PolicySyncResult: self.manager.close() raise old_active_name = self._active_adapter_name + initial_startup_seconds = self._lifecycle.get("initial_startup_seconds") + if initial_startup_seconds is None: + initial_startup_seconds = startup_seconds self._active_identity = identity self._active_adapter_name = name self._state = BackendLifecycleState.SYNCHRONIZED @@ -518,7 +521,12 @@ def synchronize(self, snapshot: PolicySnapshot) -> PolicySyncResult: "policy_identity_digest": identity.digest, "policy_version": identity.parameter_version, "adapter_name": name, - "startup_seconds": startup_seconds, + # Keep the first managed-server startup separate from later no-op + # health checks. Replacing it on every adapter refresh made the + # reported startup time collapse to a few microseconds. + "startup_seconds": initial_startup_seconds, + "initial_startup_seconds": initial_startup_seconds, + "latest_start_check_seconds": startup_seconds, "adapter_sync_seconds": adapter_sync_seconds, "sync_total_seconds": time.perf_counter() - sync_started, "prefix_cache_enabled": False, diff --git a/tests/unit/test_packaging.py b/tests/unit/test_packaging.py index 2f366b0..1ae78b1 100644 --- a/tests/unit/test_packaging.py +++ b/tests/unit/test_packaging.py @@ -306,6 +306,7 @@ def test_every_published_benchmark_result_validates_against_the_schema(): "alignment-lab-v1-state-supervision.json", "alignment-external-v1.json", "rollout-runtime-v2-hf-reference.json", + "rollout-runtime-v2-rtx4080.json", "rtx4080-verl-opd-runtime-v1.json", "rtx4080-verl-opd-developer-v1.json", "rtx4080-smollm2-opd-family-smoke-v1.json", diff --git a/tests/unit/test_rollout_runtime_v2_benchmark.py b/tests/unit/test_rollout_runtime_v2_benchmark.py index 1585c0f..8dd2a9e 100644 --- a/tests/unit/test_rollout_runtime_v2_benchmark.py +++ b/tests/unit/test_rollout_runtime_v2_benchmark.py @@ -4,6 +4,8 @@ import hashlib import json +import subprocess +import sys from pathlib import Path import jsonschema @@ -13,6 +15,11 @@ PREREGISTRATION = ROOT / "benchmarks/preregistration/rollout-runtime-v2.yaml" SCHEMA = ROOT / "benchmarks/schema/rollout-runtime-v2.schema.json" FROZEN = ROOT / "benchmarks/results/gpu-calc-hard-equal-update-v2.json" +SELECTED_SCHEMA = ROOT / "benchmarks/schema/rollout-runtime-v2-selected.schema.json" +RESULT_SCHEMA = ROOT / "benchmarks/schema/rollout-runtime-v2-result.schema.json" +HF_CACHED = ROOT / "benchmarks/evidence/rollout-runtime-v2/hf-cached-rtx4080-raw.json" +VLLM = ROOT / "benchmarks/evidence/rollout-runtime-v2/vllm-rtx4080-raw.json" +SELECTED_RESULT = ROOT / "benchmarks/results/rollout-runtime-v2-rtx4080.json" def test_rollout_runtime_v2_workload_is_preregistered_before_baseline() -> None: @@ -199,3 +206,55 @@ def test_rollout_runtime_v2_schema_keeps_unavailable_phases_distinct_from_zero() jsonschema.validate(result, schema) for name in ("prefill", "decode", "teacher_scoring", "actor_update", "full_cycle"): assert "median_seconds" not in phases[name] + + +def test_selected_backend_evidence_is_exact_private_path_free_and_schema_valid() -> None: + expected = { + HF_CACHED: "a9f8f0b0275d940f30497a0d88d76da0e112ffbe2bc1b37a42c6ed313b852242", + VLLM: "abf14d73a7289f9619515943b4c8d7dafe7cd187fefe928655739ac7ed1ab16c", + SELECTED_RESULT: "32be86856263ccdf787986c4ec54570d323af8c238dc1664e00a9ce41ca393c4", + } + raw_schema = json.loads(SELECTED_SCHEMA.read_text(encoding="utf-8")) + result_schema = json.loads(RESULT_SCHEMA.read_text(encoding="utf-8")) + for path, digest in expected.items(): + assert hashlib.sha256(path.read_bytes()).hexdigest() == digest + text = path.read_text(encoding="utf-8") + for private_fragment in ( + "14191", + "daoyuanli/.venvs", + "OneDrive", + "AppData", + "C:\\\\Users", + ): + assert private_fragment not in text + for path in (HF_CACHED, VLLM): + jsonschema.validate(json.loads(path.read_text(encoding="utf-8")), raw_schema) + jsonschema.validate(json.loads(SELECTED_RESULT.read_text(encoding="utf-8")), result_schema) + + +def test_selected_backend_result_preserves_failed_release_gate() -> None: + result = json.loads(SELECTED_RESULT.read_text(encoding="utf-8")) + assert len(result["cells"]) == 24 + assert result["source"] == { + "commit": "690db9b079bfecfec14dc3bedc3aa0308cbacf60", + "dirty": False, + "miniverl_version": "0.11.0.dev0", + "wheel_sha256": "266fcd59bb3e85b02974a92b26c9a4e59c7c9b9205853d17baadcfffdb898e27", + } + assert result["gates"]["hf_cached_speedup_over_hf_reference"]["passed"] is False + assert result["gates"]["selected_external_engine_advantage_over_hf_cached"]["passed"] is True + assert result["gates"]["vllm_policy_refresh"]["passed"] is True + assert result["gates"]["vllm_pg_logprob_conformance"]["passed"] is False + assert result["release_decision"] == { + "action": "publish evidence, optimize hf_cached, rerun the frozen workload", + "blocking_gate": "hf_cached_speedup_over_hf_reference", + "v0_11_0_publishable": False, + } + + +def test_selected_backend_aggregate_is_reproducible() -> None: + subprocess.run( + [sys.executable, "scripts/publish_rollout_runtime_v2_evidence.py", "--check"], + cwd=ROOT, + check=True, + ) diff --git a/tests/unit/test_vllm_rollout_backend.py b/tests/unit/test_vllm_rollout_backend.py index 269447f..bcdcd38 100644 --- a/tests/unit/test_vllm_rollout_backend.py +++ b/tests/unit/test_vllm_rollout_backend.py @@ -213,6 +213,7 @@ def test_vllm_backend_uses_unique_content_bound_adapter_names_and_tears_down( second = _identity(2) backend.synchronize(PolicySnapshot(first)) + first_lifecycle = backend.lifecycle_metrics() first_name = manager.loaded[-1][0] assert "p00000001" in first_name assert first.adapter_tensor_digest[:12] in first_name @@ -229,6 +230,9 @@ def test_vllm_backend_uses_unique_content_bound_adapter_names_and_tears_down( assert lifecycle["adapter_name"] == second_name assert lifecycle["prefix_cache_enabled"] is False assert lifecycle["numerical_equivalence_class"] == "bf16-external-vs-nf4-actor-direct-gkd" + assert lifecycle["startup_seconds"] == first_lifecycle["startup_seconds"] + assert lifecycle["initial_startup_seconds"] == first_lifecycle["startup_seconds"] + assert lifecycle["latest_start_check_seconds"] >= 0.0 backend.quiesce() backend.release_generation_memory() From 578077504b9b547d1992bddcfc97c778596cdca5 Mon Sep 17 00:00:00 2001 From: Daoyuan Li <94409450+DaoyuanLi2816@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:44:43 -0700 Subject: [PATCH 6/6] Keep vLLM snapshot test core-only --- tests/unit/test_vllm_rollout_backend.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/test_vllm_rollout_backend.py b/tests/unit/test_vllm_rollout_backend.py index bcdcd38..3a1ea59 100644 --- a/tests/unit/test_vllm_rollout_backend.py +++ b/tests/unit/test_vllm_rollout_backend.py @@ -2,8 +2,9 @@ import json import math +import sys from pathlib import Path -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace from unittest.mock import patch import pytest @@ -94,10 +95,9 @@ def test_vllm_snapshot_resolution_preserves_hub_snapshot_symlink_parent( except OSError: pytest.skip("symlinks are unavailable") - with patch( - "huggingface_hub.try_to_load_from_cache", - return_value=str(config), - ): + hub = ModuleType("huggingface_hub") + hub.try_to_load_from_cache = lambda *_args, **_kwargs: str(config) # type: ignore[attr-defined] + with patch.dict(sys.modules, {"huggingface_hub": hub}): assert _resolve_model_snapshot("org/model", "a" * 40) == snapshot.resolve()