From 024d284f39f2c010f1f9dc800de8032288dabef6 Mon Sep 17 00:00:00 2001 From: kierandidi Date: Fri, 30 Oct 2026 12:33:56 +0000 Subject: [PATCH 01/11] Add an RFD4-Proteina LoRA training example Trains PEFT adapters on the denoiser with the released recipe, saved in the format RFD4ProteinaPipeline.load_lora_weights reads. --- examples/rfd4_proteina/README.md | 137 +++++ examples/rfd4_proteina/requirements.txt | 5 + .../rfd4_proteina/train_rfd4_proteina_lora.py | 557 ++++++++++++++++++ 3 files changed, 699 insertions(+) create mode 100644 examples/rfd4_proteina/README.md create mode 100644 examples/rfd4_proteina/requirements.txt create mode 100644 examples/rfd4_proteina/train_rfd4_proteina_lora.py diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md new file mode 100644 index 000000000000..0e3e2214c663 --- /dev/null +++ b/examples/rfd4_proteina/README.md @@ -0,0 +1,137 @@ +# LoRA fine-tuning for RFD4-Proteina + +[RFD4-Proteína](https://huggingface.co/docs/diffusers/main/en/api/pipelines/rfd4_proteina) generates protein backbones +and sequences by flow matching in a product space of backbone coordinates and per-token local latents. This example +trains LoRA adapters on the denoiser so it follows a set of structures you care about, and writes them in the format +`RFD4ProteinaPipeline.load_lora_weights` reads. + +It follows the fine-tuning recipe the model was released with: the autoencoder stays frozen and paired with the +denoiser, adapters go on the attention projections, and the loss scores the predicted clean sample of both modalities. + +## Setup + +Install Diffusers from source, then this example's requirements: + +```bash +pip install -e . +pip install -r examples/rfd4_proteina/requirements.txt +``` + +Training reads structures as annotated CIF files, which are turned into model features by the +[`rfd4-proteina-prep`](https://huggingface.co/nvidia/RFD4-Proteina-prep) package — the same preprocessing the pipeline's +inference examples use. It builds on AtomWorks and is not part of Diffusers: + +```bash +pip install git+https://huggingface.co/nvidia/RFD4-Proteina-prep@c9959efdebd75ac7fe9129fbbf0e291a6e0487d5 +``` + +Initialize an [🤗 Accelerate](https://github.com/huggingface/accelerate) environment with: + +```bash +accelerate config +``` + +Or write the default configuration without answering questions: + +```bash +accelerate config default +``` + +## Training + +Point `--instance_data_dir` at a directory of CIF files; it is searched recursively, so a nested layout such as +PLINDER's `systems///.cif` works as it is. + +The model repository ships the small PLINDER subset the released fine-tuning tutorial uses, fifty systems in about a +megabyte. Download and unpack it: + +```python +import tarfile + +from huggingface_hub import hf_hub_download + +archive = hf_hub_download("nvidia/RFD4-Proteina", "test_dataset/test_dataset.tar.gz") +with tarfile.open(archive) as tar: + tar.extractall("test_dataset") +``` + +That writes the structures under `test_dataset/cifs`. Train on them with: + +```bash +accelerate launch examples/rfd4_proteina/train_rfd4_proteina_lora.py \ + --pretrained_model_name_or_path nvidia/RFD4-Proteina-Diffusers \ + --instance_data_dir test_dataset/cifs \ + --features_cache_dir feature-cache \ + --output_dir rfd4-proteina-lora \ + --mixed_precision bf16 \ + --rank 32 --lora_alpha 64 \ + --learning_rate 5e-5 \ + --gradient_accumulation_steps 2 \ + --max_train_steps 500 \ + --seed 0 +``` + +Notes on the defaults: + +- **Rank, alpha and target modules** are the released fine-tune's: rank 32, alpha 64, adapters on the attention + projections (`to_q`, `to_k`, `to_v`, `to_out`, which are the native recipe's `to_qkv` and `to_out_node`). +- **One structure per batch.** Token counts differ per structure and the model has no padding-free path, so the batch + size is fixed at one; raise the effective batch size with `--gradient_accumulation_steps`. +- **`--max_tokens`** skips structures longer than 384 tokens, as the released fine-tune does. Memory and step time grow + quickly with token count. +- **Preparation is cached.** Parsing and featurizing a CIF costs far more than a training step, so pass + `--features_cache_dir` to prepare each structure once and reuse it across epochs. +- **`--gradient_checkpointing`** trades compute for memory if a long structure does not fit. +- **`--recycle_depth`** sets the trunk passes per training step, one by default. Sampling recycles more deeply, up to + six passes early in the schedule, so a deeper training step matches inference more closely and costs proportionally + more time and memory. + +- **Structures preparation rejects are skipped** with a warning — an experimental file whose coordinates the model's + own input checks refuse, for instance. A run where nothing can be prepared fails rather than writing an untrained + adapter. +- **Missing atoms are dropped from the mask.** A deposited structure does not always hold every atom its residues + should have; those slots are removed before encoding, and a token is supervised only where its CA is resolved. + +Pass `--validation_cif` to generate from one input every `--validation_steps`, with the sampler settings the +checkpoint was released with, and write the result next to the adapter. Generation uses the checkpoint's own recycle +schedule, which also fixes the number of integration steps. + +The reference noise is drawn with the pipeline's own helpers, so the noise end of the training path is the same +distribution the sampler starts from: masked to the emitted tokens, and for the backbone with zero centre of mass and +the per-molecule stochastic offset. + +The loss is flow matching on the two modalities. The native recipe adds auxiliary sequence and insertion terms, which +this example leaves out: adapters trained here move the structure distribution, and the sequence readout follows only +indirectly. + +## Inference + +The saved adapter loads into the pipeline: + +```python +import torch +from diffusers import RFD4ProteinaPipeline +from rfd4_proteina_prep import prepare_cif, write_cif + +pipe = RFD4ProteinaPipeline.from_pretrained("nvidia/RFD4-Proteina-Diffusers", dtype=torch.bfloat16).to("cuda") +pipe.load_lora_weights("rfd4-proteina-lora") + +features, atom_array = prepare_cif("input.cif", seed=0, device=pipe.device) +named = ( + "padding_mask", + "is_in_context_conditioning", + "supports_local_latent", + "token_type", + "atom37_mask", + "coords_nm", +) +output = pipe( + **{name: features[name] for name in named}, + conditioning_features={name: value for name, value in features.items() if name not in named}, + generator=torch.Generator(device=pipe.device).manual_seed(0), +) +write_cif(output, features, atom_array, "design.cif") +``` + +Adapters trained with the model's own trainer convert to this format with +`scripts/convert_rfd4_proteina_to_diffusers.py`. diff --git a/examples/rfd4_proteina/requirements.txt b/examples/rfd4_proteina/requirements.txt new file mode 100644 index 000000000000..4c1463fe8956 --- /dev/null +++ b/examples/rfd4_proteina/requirements.txt @@ -0,0 +1,5 @@ +accelerate>=1.0.0 +peft>=0.14.0 +safetensors>=0.4.5 +torch>=2.4.0 +tensorboard diff --git a/examples/rfd4_proteina/train_rfd4_proteina_lora.py b/examples/rfd4_proteina/train_rfd4_proteina_lora.py new file mode 100644 index 000000000000..163b2b79040d --- /dev/null +++ b/examples/rfd4_proteina/train_rfd4_proteina_lora.py @@ -0,0 +1,557 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LoRA fine-tuning of the RFD4-Proteina denoiser on a folder of structures.""" + +import argparse +import logging +import os +import shutil +from pathlib import Path + +import torch +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import ProjectConfiguration, set_seed +from peft import LoraConfig +from peft.utils import get_peft_model_state_dict, set_peft_model_state_dict +from torch.utils.data import DataLoader, Dataset +from tqdm.auto import tqdm + +import diffusers +from diffusers import RFD4ProteinaAutoencoder, RFD4ProteinaPipeline, RFD4ProteinaScheduler, RFD4ProteinaTransformer +from diffusers.optimization import get_scheduler +from diffusers.training_utils import cast_training_params +from diffusers.utils import convert_state_dict_to_diffusers, convert_unet_state_dict_to_peft + + +logger = get_logger(__name__) + +# The pipeline takes the molecular tensors by name and the rest of the prepared features as conditioning. +PIPELINE_INPUTS = ( + "padding_mask", + "is_in_context_conditioning", + "supports_local_latent", + "token_type", + "atom37_mask", + "coords_nm", +) + +# Training times per modality, as the released recipe samples them: a Beta distribution mixed with a uniform one. +# See `loss.t_distribution` in the RFD4-Proteina `configs/training_proteina.yaml`. +TIME_DISTRIBUTION = { + "bb_ca": {"concentration1": 1.9, "concentration0": 1.0, "uniform_fraction": 0.02}, + "local_latents": {"concentration1": 1.0, "concentration0": 1.5, "uniform_fraction": 0.2}, +} + +# The backbone is the modality the sampler draws with zero centre of mass and a per-molecule stochastic offset. +BACKBONE = "bb_ca" + + +def parse_args(): + parser = argparse.ArgumentParser(description="LoRA fine-tuning for RFD4-Proteina.") + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + required=True, + help="Repository id or local path of the RFD4-Proteina pipeline to adapt.", + ) + parser.add_argument("--revision", type=str, default=None, help="Model revision to load.") + parser.add_argument("--variant", type=str, default=None, help="Weight variant to load, such as `fp16`.") + parser.add_argument( + "--instance_data_dir", + type=str, + required=True, + help="Directory searched recursively for the annotated CIF files to fine-tune on.", + ) + parser.add_argument( + "--features_cache_dir", + type=str, + default=None, + help="Directory for the prepared tensors. Preparation runs once per file and is reused across epochs.", + ) + parser.add_argument( + "--max_tokens", + type=int, + default=384, + help="Skip structures with more tokens than this, as the released fine-tune does.", + ) + parser.add_argument("--prep_seed", type=int, default=0, help="Seed for the AtomWorks preparation pipeline.") + parser.add_argument("--output_dir", type=str, default="rfd4-proteina-lora", help="Where the adapter is written.") + parser.add_argument("--cache_dir", type=str, default=None, help="Hugging Face cache directory.") + parser.add_argument("--seed", type=int, default=None, help="Seed for training.") + parser.add_argument("--rank", type=int, default=32, help="LoRA rank.") + parser.add_argument("--lora_alpha", type=float, default=64.0, help="LoRA scaling factor.") + parser.add_argument("--lora_dropout", type=float, default=0.0, help="Dropout on the LoRA layers.") + parser.add_argument( + "--target_modules", + type=str, + nargs="+", + default=["to_q", "to_k", "to_v", "to_out"], + help="Attention projections to adapt. The default is the released recipe's `to_qkv` and `to_out_node`.", + ) + parser.add_argument("--learning_rate", type=float, default=5e-5, help="Initial learning rate.") + parser.add_argument("--lr_scheduler", type=str, default="constant", help="Learning rate schedule.") + parser.add_argument("--lr_warmup_steps", type=int, default=0, help="Warmup steps for the learning rate.") + parser.add_argument("--adam_beta1", type=float, default=0.9, help="Adam beta1.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="Adam beta2.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-4, help="Weight decay for the adapters.") + parser.add_argument("--adam_epsilon", type=float, default=1e-8, help="Adam epsilon.") + parser.add_argument("--max_grad_norm", type=float, default=1.0, help="Gradient clipping norm.") + parser.add_argument("--max_train_steps", type=int, default=500, help="Optimizer steps to run.") + parser.add_argument("--gradient_accumulation_steps", type=int, default=2, help="Batches per optimizer step.") + parser.add_argument("--gradient_checkpointing", action="store_true", help="Trade compute for memory.") + parser.add_argument( + "--recycle_depth", + type=int, + default=1, + help="Trunk passes per training step. Sampling recycles more deeply and more slowly; raising this costs " + "proportionally more time and memory per step.", + ) + parser.add_argument("--dataloader_num_workers", type=int, default=0, help="Worker processes for the loader.") + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help="Precision of the frozen base model. The adapters always train in float32.", + ) + parser.add_argument("--checkpointing_steps", type=int, default=250, help="Steps between training checkpoints.") + parser.add_argument("--checkpoints_total_limit", type=int, default=None, help="Checkpoints to keep.") + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help="A checkpoint directory under `output_dir`, or `latest` for the most recent one.", + ) + parser.add_argument("--logging_dir", type=str, default="logs", help="Log directory under `output_dir`.") + parser.add_argument("--report_to", type=str, default="tensorboard", help="Tracker to log to.") + parser.add_argument( + "--validation_cif", + type=str, + default=None, + help="Annotated CIF generated from after every `validation_steps` to watch the adapter's effect.", + ) + parser.add_argument("--validation_steps", type=int, default=250, help="Steps between validation generations.") + return parser.parse_args() + + +class PreparedStructureDataset(Dataset): + """The CIF files under a directory, prepared into the feature tensors the model consumes. + + Preparation parses the file with AtomWorks and runs the model's own datapipe, which is slow enough to be worth + caching: each file is prepared once and reloaded from `cache_dir` afterwards. + """ + + def __init__(self, data_dir: str, seed: int, max_tokens: int, cache_dir: str | None = None): + from rfd4_proteina_prep import prepare_cif + + self._prepare_cif = prepare_cif + self.seed = seed + self.max_tokens = max_tokens + self.cache_dir = None if cache_dir is None else Path(cache_dir) + if self.cache_dir is not None: + self.cache_dir.mkdir(parents=True, exist_ok=True) + root = Path(data_dir) + if not root.is_dir(): + raise ValueError(f"{data_dir} is not a directory.") + self.paths = sorted(path for pattern in ("*.cif", "*.cif.gz", "*.cif.zst") for path in root.rglob(pattern)) + if not self.paths: + raise ValueError(f"No CIF files found under {data_dir}.") + + def __len__(self) -> int: + return len(self.paths) + + def _features(self, path: Path) -> dict[str, torch.Tensor]: + cached = None if self.cache_dir is None else self.cache_dir / f"{path.stem}-{self.seed}.pt" + if cached is not None and cached.is_file(): + return torch.load(cached, map_location="cpu", weights_only=True) + features, _ = self._prepare_cif(path, seed=self.seed, device=torch.device("cpu")) + features = {name: value for name, value in features.items() if isinstance(value, torch.Tensor)} + if cached is not None: + torch.save(features, cached) + return features + + def __getitem__(self, index: int) -> dict[str, torch.Tensor] | None: + path = self.paths[index] + try: + features = self._features(path) + except ValueError as error: + # Preparation rejects a structure the model cannot be trained on, such as one with unusable coordinates. + logger.warning(f"Skipping {path.name}: {error}") + return None + # A structure that does not fit is dropped from its batch rather than truncated. + if features["padding_mask"].shape[1] > self.max_tokens: + return None + return features + + +def collate_fn(examples): + """Keep one structure per batch: token counts differ per structure, and the model has no padding-free path. + + Raise the effective batch size with `--gradient_accumulation_steps` instead. + """ + examples = [example for example in examples if example is not None] + return examples[0] if examples else None + + +def sample_times(batch_size: int, device: torch.device) -> dict[str, torch.Tensor]: + """Draw one time per modality and batch row from the released training distribution.""" + times = {} + for name, parameters in TIME_DISTRIBUTION.items(): + beta = torch.distributions.Beta(parameters["concentration1"], parameters["concentration0"]) + sampled = beta.sample((batch_size,)).to(device) + uniform = torch.rand((batch_size,), device=device) + take_uniform = torch.rand((batch_size,), device=device) < parameters["uniform_fraction"] + # `add_noise` rejects a time of exactly 1, where the state is clean and the loss carries no signal. + times[name] = torch.where(take_uniform, uniform, sampled).clamp(max=1.0 - 1e-6) + return times + + +def draw_noise( + name: str, shape: tuple[int, ...], emit_mask: torch.Tensor, conditioning: dict[str, torch.Tensor] +) -> torch.Tensor: + """Draw one modality's reference noise the way the sampler draws its starting state. + + Training and sampling have to agree on what the noise end of the path looks like, so the draw reuses the + pipeline's own helpers: masked to the emitted tokens, and for the backbone with zero centre of mass and the + per-molecule stochastic offset the released recipe applies. + """ + noise = RFD4ProteinaPipeline._sample_noise(shape, emit_mask, None, zero_com=name == BACKBONE) + if name != BACKBONE: + return noise + return RFD4ProteinaPipeline._center_noise(noise, emit_mask, None, conditioning) + + +def flow_matching_loss(prediction, state, times, clean, supervision_masks) -> torch.Tensor: + """Average the per-modality error of the predicted clean sample over the supervised tokens. + + The model predicts a velocity; `state + (1 - t) * velocity` is its estimate of the clean sample, which is what the + released recipe scores. Each modality is normalised by its own token count and the two terms are summed. + """ + velocities = {"bb_ca": prediction.backbone.float(), "local_latents": prediction.local_latents.float()} + loss = 0.0 + for name, velocity in velocities.items(): + mask = supervision_masks[name] + estimate = state[name] + (1 - times[name][..., None, None]) * velocity + error = (clean[name] - estimate) * mask[..., None] + tokens = mask.sum(dim=-1).clamp(min=1) + loss = loss + ((error**2).sum(dim=(-1, -2)) / tokens).mean() + return loss + + +def main(): + args = parse_args() + logging_dir = Path(args.output_dir, args.logging_dir) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=ProjectConfiguration(project_dir=args.output_dir, logging_dir=str(logging_dir)), + ) + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + if args.seed is not None: + set_seed(args.seed) + if accelerator.is_main_process: + os.makedirs(args.output_dir, exist_ok=True) + + weight_dtype = {"fp16": torch.float16, "bf16": torch.bfloat16}.get(accelerator.mixed_precision, torch.float32) + load_kwargs = {"revision": args.revision, "variant": args.variant, "cache_dir": args.cache_dir} + transformer = RFD4ProteinaTransformer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, **load_kwargs + ) + vae = RFD4ProteinaAutoencoder.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", torch_dtype=weight_dtype, **load_kwargs + ) + # One scheduler per modality, as the pipeline holds them; training only uses their shared `add_noise`. + schedulers = { + name: RFD4ProteinaScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder=subfolder, + cache_dir=args.cache_dir, + revision=args.revision, + ) + for name, subfolder in (("bb_ca", "bb_ca_scheduler"), ("local_latents", "local_latents_scheduler")) + } + + # The autoencoder is paired with the denoiser and stays frozen; only the adapters on the denoiser train. + vae.requires_grad_(False) + transformer.requires_grad_(False) + vae.to(accelerator.device) + transformer.to(accelerator.device) + if args.gradient_checkpointing: + transformer.enable_gradient_checkpointing() + + transformer.add_adapter( + LoraConfig( + r=args.rank, + lora_alpha=args.lora_alpha, + lora_dropout=args.lora_dropout, + bias="none", + init_lora_weights="gaussian", + target_modules=args.target_modules, + ) + ) + if weight_dtype != torch.float32: + cast_training_params([transformer], dtype=torch.float32) + + def unwrap_model(model): + return accelerator.unwrap_model(model) + + def save_model_hook(models, weights, output_dir): + if not accelerator.is_main_process: + return + for model in models: + if isinstance(unwrap_model(model), type(unwrap_model(transformer))): + layers = convert_state_dict_to_diffusers(get_peft_model_state_dict(unwrap_model(model))) + else: + raise ValueError(f"Unexpected model to save: {model.__class__}") + weights.pop() + RFD4ProteinaPipeline.save_lora_weights(output_dir, transformer_lora_layers=layers) + + def load_model_hook(models, input_dir): + transformer_ = None + while models: + model = models.pop() + if isinstance(unwrap_model(model), type(unwrap_model(transformer))): + transformer_ = unwrap_model(model) + else: + raise ValueError(f"Unexpected model to load: {model.__class__}") + state_dict = RFD4ProteinaPipeline.lora_state_dict(input_dir) + state_dict = { + key.removeprefix("transformer."): value + for key, value in state_dict.items() + if key.startswith("transformer.") + } + incompatible = set_peft_model_state_dict( + transformer_, convert_unet_state_dict_to_peft(state_dict), adapter_name="default" + ) + if getattr(incompatible, "unexpected_keys", None): + logger.warning(f"Unexpected keys in the checkpoint: {incompatible.unexpected_keys}") + if accelerator.mixed_precision == "fp16": + cast_training_params([transformer_]) + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + parameters = [parameter for parameter in transformer.parameters() if parameter.requires_grad] + optimizer = torch.optim.AdamW( + parameters, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + dataset = PreparedStructureDataset( + args.instance_data_dir, seed=args.prep_seed, max_tokens=args.max_tokens, cache_dir=args.features_cache_dir + ) + dataloader = DataLoader( + dataset, + batch_size=1, + shuffle=True, + collate_fn=collate_fn, + num_workers=args.dataloader_num_workers, + ) + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + transformer, optimizer, dataloader, lr_scheduler = accelerator.prepare( + transformer, optimizer, dataloader, lr_scheduler + ) + if accelerator.is_main_process: + # Trackers take scalars, so the list of adapted modules is logged as one string. + accelerator.init_trackers( + "rfd4-proteina-lora", config={**vars(args), "target_modules": ",".join(args.target_modules)} + ) + + logger.info("***** Running training *****") + logger.info(f" Structures = {len(dataset)}") + logger.info(f" Optimizer steps = {args.max_train_steps}") + logger.info(f" Trainable parameters = {sum(parameter.numel() for parameter in parameters)}") + + global_step = 0 + if args.resume_from_checkpoint: + path = args.resume_from_checkpoint + if path == "latest": + checkpoints = sorted( + (entry for entry in os.listdir(args.output_dir) if entry.startswith("checkpoint")), + key=lambda entry: int(entry.split("-")[1]), + ) + path = os.path.join(args.output_dir, checkpoints[-1]) if checkpoints else None + if path is None: + logger.info("No checkpoint found; starting a new run.") + else: + accelerator.load_state(path) + global_step = int(Path(path).name.split("-")[1]) + logger.info(f"Resuming from {path} at step {global_step}.") + + progress_bar = tqdm( + range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process, desc="Steps" + ) + # Structures the preparation rejects drop out of their batch, so the data is cycled until the step count is met. + while global_step < args.max_train_steps: + transformer.train() + for batch in dataloader: + if batch is None: # Every structure in this batch was skipped. + continue + with accelerator.accumulate(transformer): + features = {name: value.to(accelerator.device) for name, value in batch.items()} + padding_mask = features["padding_mask"].bool() + conditioning = {name: value for name, value in features.items() if name not in PIPELINE_INPUTS} + conditioning["is_in_context_conditioning"] = features["is_in_context_conditioning"] + conditioning["supports_local_latent"] = features["supports_local_latent"] + emitted = padding_mask & ~features["is_in_context_conditioning"].bool() + emit_masks = { + "bb_ca": emitted, + "local_latents": emitted & features["supports_local_latent"].bool(), + } + # An experimental structure is missing atoms the residue should have: their slots are marked present + # but hold no coordinate. Drop them from the mask and zero them, or the encoder returns NaN latents + # for the whole structure. + resolved = torch.isfinite(features["coords_nm"]).all(dim=-1) + atom37_mask = features["atom37_mask"].bool() & resolved + coords_nm = torch.nan_to_num(features["coords_nm"]) + # Only tokens whose CA atom is resolved carry a target. + supervision_masks = { + "bb_ca": emit_masks["bb_ca"] & atom37_mask[..., 1], + "local_latents": emit_masks["local_latents"] & atom37_mask[..., 1], + } + if not supervision_masks["bb_ca"].any(): + continue + + with torch.no_grad(): + encoded = vae.encode( + coords_nm=coords_nm.to(weight_dtype), + padding_mask=padding_mask, + token_type=features["token_type"], + atom37_mask=atom37_mask, + conditioning=conditioning, + ) + clean = {"bb_ca": coords_nm[..., 1, :].float(), "local_latents": encoded.latent.float()} + clean = {name: value * emit_masks[name][..., None] for name, value in clean.items()} + noise = { + name: draw_noise(name, tuple(clean[name].shape), emit_masks[name], conditioning) for name in clean + } + times = sample_times(padding_mask.shape[0], accelerator.device) + state = { + name: schedulers[name].add_noise(clean[name], noise[name], times[name]) + * emit_masks[name][..., None] + for name in clean + } + + prediction = transformer( + bb_ca=state["bb_ca"].to(weight_dtype), + local_latents=state["local_latents"].to(weight_dtype), + bb_ca_time=times["bb_ca"], + local_latents_time=times["local_latents"], + padding_mask=padding_mask, + token_type=features["token_type"], + atom37_mask=atom37_mask, + conditioning=conditioning, + recycle_depth=args.recycle_depth, + ) + loss = flow_matching_loss(prediction, state, times, clean, supervision_masks) + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(parameters, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + logs = {"loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + accelerator.log(logs, step=global_step) + + if accelerator.is_main_process and global_step % args.checkpointing_steps == 0: + save_checkpoint(accelerator, args, global_step) + if args.validation_cif is not None and global_step % args.validation_steps == 0: + run_validation(accelerator, args, unwrap_model(transformer), vae, schedulers, global_step) + transformer.train() + if global_step >= args.max_train_steps: + break + if global_step == 0: + raise ValueError( + f"No structure under {args.instance_data_dir} could be trained on; see the warnings above." + ) + + accelerator.wait_for_everyone() + if accelerator.is_main_process: + layers = convert_state_dict_to_diffusers(get_peft_model_state_dict(unwrap_model(transformer))) + RFD4ProteinaPipeline.save_lora_weights(args.output_dir, transformer_lora_layers=layers) + logger.info(f"Saved the adapter to {args.output_dir}.") + accelerator.end_training() + + +def save_checkpoint(accelerator, args, global_step: int) -> None: + if args.checkpoints_total_limit is not None: + checkpoints = sorted( + (entry for entry in os.listdir(args.output_dir) if entry.startswith("checkpoint")), + key=lambda entry: int(entry.split("-")[1]), + ) + for checkpoint in checkpoints[: max(len(checkpoints) - args.checkpoints_total_limit + 1, 0)]: + shutil.rmtree(os.path.join(args.output_dir, checkpoint)) + path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(path) + logger.info(f"Saved state to {path}.") + + +def run_validation(accelerator, args, transformer, vae, schedulers, global_step: int) -> None: + """Generate one structure from the validation input with the adapter attached.""" + from rfd4_proteina_prep import prepare_cif, write_cif + + pipeline = RFD4ProteinaPipeline( + transformer=transformer, + vae=vae, + bb_ca_scheduler=schedulers["bb_ca"], + local_latents_scheduler=schedulers["local_latents"], + ) + pipeline.set_progress_bar_config(disable=True) + features, atom_array = prepare_cif(args.validation_cif, seed=args.prep_seed, device=accelerator.device) + inputs = {name: features[name] for name in PIPELINE_INPUTS} + conditioning = {name: value for name, value in features.items() if name not in PIPELINE_INPUTS} + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed or 0) + with torch.no_grad(): + output = pipeline( + **inputs, + conditioning_features=conditioning, + generator=generator, + ) + path = os.path.join(args.output_dir, f"validation-{global_step}.cif") + write_cif(output, features, atom_array, path) + logger.info(f"Wrote {path}.") + + +if __name__ == "__main__": + main() From 2c0be9a8a4e09e8466f671503f78d908db82405f Mon Sep 17 00:00:00 2001 From: Kieran Didi Date: Fri, 6 Nov 2026 12:07:55 +0000 Subject: [PATCH 02/11] Follow the established LoRA save pattern Save the adapter the way the current LoRA examples do: the PEFT state dict goes out unconverted, and `_collate_lora_metadata` records rank, alpha and target modules alongside it. Resolve `--resume_from_checkpoint` against `--output_dir`, so naming a checkpoint works as well as passing its path. --- examples/rfd4_proteina/README.md | 11 +++++-- .../rfd4_proteina/train_rfd4_proteina_lora.py | 33 ++++++++++++------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index 0e3e2214c663..e4e2d2336ed1 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -133,5 +133,12 @@ output = pipe( write_cif(output, features, atom_array, "design.cif") ``` -Adapters trained with the model's own trainer convert to this format with -`scripts/convert_rfd4_proteina_to_diffusers.py`. +An adapter trained with the model's own trainer loads too, once converted — the native recipe adapts a fused +`to_qkv`, which the conversion splits into the `to_q`, `to_k` and `to_v` this transformer exposes: + +```bash +python scripts/convert_rfd4_proteina_to_diffusers.py \ + --adapter \ + --source native-adapter \ + --destination rfd4-proteina-lora-converted +``` diff --git a/examples/rfd4_proteina/train_rfd4_proteina_lora.py b/examples/rfd4_proteina/train_rfd4_proteina_lora.py index 163b2b79040d..1ed9d19807f1 100644 --- a/examples/rfd4_proteina/train_rfd4_proteina_lora.py +++ b/examples/rfd4_proteina/train_rfd4_proteina_lora.py @@ -35,8 +35,8 @@ import diffusers from diffusers import RFD4ProteinaAutoencoder, RFD4ProteinaPipeline, RFD4ProteinaScheduler, RFD4ProteinaTransformer from diffusers.optimization import get_scheduler -from diffusers.training_utils import cast_training_params -from diffusers.utils import convert_state_dict_to_diffusers, convert_unet_state_dict_to_peft +from diffusers.training_utils import _collate_lora_metadata, cast_training_params +from diffusers.utils import convert_unet_state_dict_to_peft logger = get_logger(__name__) @@ -327,13 +327,17 @@ def unwrap_model(model): def save_model_hook(models, weights, output_dir): if not accelerator.is_main_process: return + modules_to_save = {} for model in models: if isinstance(unwrap_model(model), type(unwrap_model(transformer))): - layers = convert_state_dict_to_diffusers(get_peft_model_state_dict(unwrap_model(model))) + modules_to_save["transformer"] = unwrap_model(model) + layers = get_peft_model_state_dict(unwrap_model(model)) else: raise ValueError(f"Unexpected model to save: {model.__class__}") weights.pop() - RFD4ProteinaPipeline.save_lora_weights(output_dir, transformer_lora_layers=layers) + RFD4ProteinaPipeline.save_lora_weights( + output_dir, transformer_lora_layers=layers, **_collate_lora_metadata(modules_to_save) + ) def load_model_hook(models, input_dir): transformer_ = None @@ -400,18 +404,21 @@ def load_model_hook(models, input_dir): global_step = 0 if args.resume_from_checkpoint: - path = args.resume_from_checkpoint - if path == "latest": + if args.resume_from_checkpoint == "latest": checkpoints = sorted( (entry for entry in os.listdir(args.output_dir) if entry.startswith("checkpoint")), key=lambda entry: int(entry.split("-")[1]), ) - path = os.path.join(args.output_dir, checkpoints[-1]) if checkpoints else None - if path is None: + name = checkpoints[-1] if checkpoints else None + else: + # A checkpoint is named by its directory under `--output_dir`, whether or not a path was passed. + name = os.path.basename(args.resume_from_checkpoint) + if name is None: logger.info("No checkpoint found; starting a new run.") else: + path = os.path.join(args.output_dir, name) accelerator.load_state(path) - global_step = int(Path(path).name.split("-")[1]) + global_step = int(name.split("-")[1]) logger.info(f"Resuming from {path} at step {global_step}.") progress_bar = tqdm( @@ -508,8 +515,12 @@ def load_model_hook(models, input_dir): accelerator.wait_for_everyone() if accelerator.is_main_process: - layers = convert_state_dict_to_diffusers(get_peft_model_state_dict(unwrap_model(transformer))) - RFD4ProteinaPipeline.save_lora_weights(args.output_dir, transformer_lora_layers=layers) + trained = unwrap_model(transformer) + RFD4ProteinaPipeline.save_lora_weights( + save_directory=args.output_dir, + transformer_lora_layers=get_peft_model_state_dict(trained), + **_collate_lora_metadata({"transformer": trained}), + ) logger.info(f"Saved the adapter to {args.output_dir}.") accelerator.end_training() From 1f0f41e9160d578c049364ea68935f9322067a25 Mon Sep 17 00:00:00 2001 From: kierandidi Date: Sun, 8 Nov 2026 20:26:14 +0000 Subject: [PATCH 03/11] Make RFD4 LoRA data preparation and checkpointing distributed-safe --- examples/rfd4_proteina/README.md | 16 +- .../rfd4_proteina/train_rfd4_proteina_lora.py | 119 +++++---- tests/others/test_rfd4_proteina_training.py | 230 ++++++++++++++++++ 3 files changed, 321 insertions(+), 44 deletions(-) create mode 100644 tests/others/test_rfd4_proteina_training.py diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index e4e2d2336ed1..a4add0d98746 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -80,7 +80,9 @@ Notes on the defaults: - **`--max_tokens`** skips structures longer than 384 tokens, as the released fine-tune does. Memory and step time grow quickly with token count. - **Preparation is cached.** Parsing and featurizing a CIF costs far more than a training step, so pass - `--features_cache_dir` to prepare each structure once and reuse it across epochs. + `--features_cache_dir` to prepare each structure once and reuse it across epochs. Cache keys include the full source + path, file size, modification time and preparation seed. Writes are locked and published atomically so distributed + workers and concurrent runs can share the cache. Without a cache directory, prepared tensors stay in CPU memory. - **`--gradient_checkpointing`** trades compute for memory if a long structure does not fit. - **`--recycle_depth`** sets the trunk passes per training step, one by default. Sampling recycles more deeply, up to six passes early in the schedule, so a deeper training step matches inference more closely and costs proportionally @@ -88,13 +90,21 @@ Notes on the defaults: - **Structures preparation rejects are skipped** with a warning — an experimental file whose coordinates the model's own input checks refuse, for instance. A run where nothing can be prepared fails rather than writing an untrained - adapter. + adapter. Filtering happens before distributed sharding, including the token limit and structures without any resolved + emitted CA atoms, so all ranks execute the same number of training steps. - **Missing atoms are dropped from the mask.** A deposited structure does not always hold every atom its residues should have; those slots are removed before encoding, and a token is supervised only where its CA is resolved. Pass `--validation_cif` to generate from one input every `--validation_steps`, with the sampler settings the checkpoint was released with, and write the result next to the adapter. Generation uses the checkpoint's own recycle -schedule, which also fixes the number of integration steps. +schedule, which also fixes the number of integration steps. In distributed training, only the main process generates +and writes validation output; the other ranks wait. Validation temporarily switches the transformer to evaluation +mode, disabling LoRA dropout, then restores its previous mode. + +For multi-GPU training, add `--multi_gpu --num_processes 2` to `accelerate launch`. All ranks participate in +checkpoint saving, including their RNG states; only the main process prunes old checkpoints and writes adapter +weights. Resume with `--resume_from_checkpoint latest` and the same LoRA configuration. Both checkpoint and final +adapter files include the PEFT metadata, preserving the rank and alpha when loaded for inference. The reference noise is drawn with the pipeline's own helpers, so the noise end of the training path is the same distribution the sampler starts from: masked to the emitted tokens, and for the backbone with zero centre of mass and diff --git a/examples/rfd4_proteina/train_rfd4_proteina_lora.py b/examples/rfd4_proteina/train_rfd4_proteina_lora.py index 1ed9d19807f1..f6e600856601 100644 --- a/examples/rfd4_proteina/train_rfd4_proteina_lora.py +++ b/examples/rfd4_proteina/train_rfd4_proteina_lora.py @@ -17,9 +17,11 @@ """LoRA fine-tuning of the RFD4-Proteina denoiser on a folder of structures.""" import argparse +import hashlib import logging import os import shutil +import tempfile from pathlib import Path import torch @@ -27,6 +29,7 @@ from accelerate import Accelerator from accelerate.logging import get_logger from accelerate.utils import ProjectConfiguration, set_seed +from filelock import FileLock from peft import LoraConfig from peft.utils import get_peft_model_state_dict, set_peft_model_state_dict from torch.utils.data import DataLoader, Dataset @@ -161,6 +164,7 @@ def __init__(self, data_dir: str, seed: int, max_tokens: int, cache_dir: str | N from rfd4_proteina_prep import prepare_cif self._prepare_cif = prepare_cif + self._memory_cache = {} self.seed = seed self.max_tokens = max_tokens self.cache_dir = None if cache_dir is None else Path(cache_dir) @@ -172,32 +176,61 @@ def __init__(self, data_dir: str, seed: int, max_tokens: int, cache_dir: str | N self.paths = sorted(path for pattern in ("*.cif", "*.cif.gz", "*.cif.zst") for path in root.rglob(pattern)) if not self.paths: raise ValueError(f"No CIF files found under {data_dir}.") + # Filter before Accelerate shards the loader: every rank must execute the same number of backwards. + self.paths = [path for path in self.paths if self._is_trainable(path)] + if not self.paths: + raise ValueError(f"No trainable structures found under {data_dir}.") def __len__(self) -> int: return len(self.paths) def _features(self, path: Path) -> dict[str, torch.Tensor]: - cached = None if self.cache_dir is None else self.cache_dir / f"{path.stem}-{self.seed}.pt" - if cached is not None and cached.is_file(): - return torch.load(cached, map_location="cpu", weights_only=True) + if self.cache_dir is None: + if path not in self._memory_cache: + self._memory_cache[path] = self._prepare_features(path) + return self._memory_cache[path] + # Include the full source path and file version, so nested names and edited inputs cannot alias. + stat = path.stat() + key = hashlib.sha256(f"{path.resolve()}:{stat.st_size}:{stat.st_mtime_ns}:{self.seed}".encode()).hexdigest() + cached = self.cache_dir / f"{key}.pt" + with FileLock(str(cached) + ".lock"): + if cached.is_file(): + return torch.load(cached, map_location="cpu", weights_only=True) + features = self._prepare_features(path) + # Publish only complete files, including when different jobs share this cache. + temporary = None + try: + with tempfile.NamedTemporaryFile(dir=self.cache_dir, suffix=".pt", delete=False) as handle: + temporary = Path(handle.name) + torch.save(features, handle) + os.replace(temporary, cached) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + return features + + def _prepare_features(self, path: Path) -> dict[str, torch.Tensor]: features, _ = self._prepare_cif(path, seed=self.seed, device=torch.device("cpu")) - features = {name: value for name, value in features.items() if isinstance(value, torch.Tensor)} - if cached is not None: - torch.save(features, cached) - return features + return {name: value for name, value in features.items() if isinstance(value, torch.Tensor)} - def __getitem__(self, index: int) -> dict[str, torch.Tensor] | None: - path = self.paths[index] + def _is_trainable(self, path: Path) -> bool: try: features = self._features(path) except ValueError as error: - # Preparation rejects a structure the model cannot be trained on, such as one with unusable coordinates. - logger.warning(f"Skipping {path.name}: {error}") - return None - # A structure that does not fit is dropped from its batch rather than truncated. - if features["padding_mask"].shape[1] > self.max_tokens: - return None - return features + logger.warning(f"Skipping {path}: {error}") + return False + emitted = features["padding_mask"].bool() & ~features["is_in_context_conditioning"].bool() + resolved_ca = features["atom37_mask"][..., 1].bool() & torch.isfinite(features["coords_nm"][..., 1, :]).all( + dim=-1 + ) + valid = features["padding_mask"].shape[1] <= self.max_tokens and bool((emitted & resolved_ca).any()) + if not valid: + logger.warning(f"Skipping {path}: too many tokens or no resolved emitted CA atoms.") + self._memory_cache.pop(path, None) + return valid + + def __getitem__(self, index: int) -> dict[str, torch.Tensor]: + return self._features(self.paths[index]) def collate_fn(examples): @@ -205,8 +238,7 @@ def collate_fn(examples): Raise the effective batch size with `--gradient_accumulation_steps` instead. """ - examples = [example for example in examples if example is not None] - return examples[0] if examples else None + return examples[0] def sample_times(batch_size: int, device: torch.device) -> dict[str, torch.Tensor]: @@ -325,8 +357,6 @@ def unwrap_model(model): return accelerator.unwrap_model(model) def save_model_hook(models, weights, output_dir): - if not accelerator.is_main_process: - return modules_to_save = {} for model in models: if isinstance(unwrap_model(model), type(unwrap_model(transformer))): @@ -335,9 +365,10 @@ def save_model_hook(models, weights, output_dir): else: raise ValueError(f"Unexpected model to save: {model.__class__}") weights.pop() - RFD4ProteinaPipeline.save_lora_weights( - output_dir, transformer_lora_layers=layers, **_collate_lora_metadata(modules_to_save) - ) + if accelerator.is_main_process: + RFD4ProteinaPipeline.save_lora_weights( + output_dir, transformer_lora_layers=layers, **_collate_lora_metadata(modules_to_save) + ) def load_model_hook(models, input_dir): transformer_ = None @@ -372,9 +403,10 @@ def load_model_hook(models, input_dir): weight_decay=args.adam_weight_decay, eps=args.adam_epsilon, ) - dataset = PreparedStructureDataset( - args.instance_data_dir, seed=args.prep_seed, max_tokens=args.max_tokens, cache_dir=args.features_cache_dir - ) + with accelerator.main_process_first(): + dataset = PreparedStructureDataset( + args.instance_data_dir, seed=args.prep_seed, max_tokens=args.max_tokens, cache_dir=args.features_cache_dir + ) dataloader = DataLoader( dataset, batch_size=1, @@ -424,12 +456,10 @@ def load_model_hook(models, input_dir): progress_bar = tqdm( range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process, desc="Steps" ) - # Structures the preparation rejects drop out of their batch, so the data is cycled until the step count is met. + # All entries are trainable before sharding; cycle the data until the step count is met. while global_step < args.max_train_steps: transformer.train() for batch in dataloader: - if batch is None: # Every structure in this batch was skipped. - continue with accelerator.accumulate(transformer): features = {name: value.to(accelerator.device) for name, value in batch.items()} padding_mask = features["padding_mask"].bool() @@ -452,8 +482,6 @@ def load_model_hook(models, input_dir): "bb_ca": emit_masks["bb_ca"] & atom37_mask[..., 1], "local_latents": emit_masks["local_latents"] & atom37_mask[..., 1], } - if not supervision_masks["bb_ca"].any(): - continue with torch.no_grad(): encoded = vae.encode( @@ -501,11 +529,13 @@ def load_model_hook(models, input_dir): progress_bar.set_postfix(**logs) accelerator.log(logs, step=global_step) - if accelerator.is_main_process and global_step % args.checkpointing_steps == 0: + if global_step % args.checkpointing_steps == 0: save_checkpoint(accelerator, args, global_step) if args.validation_cif is not None and global_step % args.validation_steps == 0: - run_validation(accelerator, args, unwrap_model(transformer), vae, schedulers, global_step) - transformer.train() + accelerator.wait_for_everyone() + if accelerator.is_main_process: + run_validation(accelerator, args, unwrap_model(transformer), vae, schedulers, global_step) + accelerator.wait_for_everyone() if global_step >= args.max_train_steps: break if global_step == 0: @@ -526,7 +556,7 @@ def load_model_hook(models, input_dir): def save_checkpoint(accelerator, args, global_step: int) -> None: - if args.checkpoints_total_limit is not None: + if accelerator.is_main_process and args.checkpoints_total_limit is not None: checkpoints = sorted( (entry for entry in os.listdir(args.output_dir) if entry.startswith("checkpoint")), key=lambda entry: int(entry.split("-")[1]), @@ -534,7 +564,9 @@ def save_checkpoint(accelerator, args, global_step: int) -> None: for checkpoint in checkpoints[: max(len(checkpoints) - args.checkpoints_total_limit + 1, 0)]: shutil.rmtree(os.path.join(args.output_dir, checkpoint)) path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.wait_for_everyone() accelerator.save_state(path) + accelerator.wait_for_everyone() logger.info(f"Saved state to {path}.") @@ -553,12 +585,17 @@ def run_validation(accelerator, args, transformer, vae, schedulers, global_step: inputs = {name: features[name] for name in PIPELINE_INPUTS} conditioning = {name: value for name, value in features.items() if name not in PIPELINE_INPUTS} generator = torch.Generator(device=accelerator.device).manual_seed(args.seed or 0) - with torch.no_grad(): - output = pipeline( - **inputs, - conditioning_features=conditioning, - generator=generator, - ) + was_training = transformer.training + transformer.eval() + try: + with torch.no_grad(): + output = pipeline( + **inputs, + conditioning_features=conditioning, + generator=generator, + ) + finally: + transformer.train(was_training) path = os.path.join(args.output_dir, f"validation-{global_step}.cif") write_cif(output, features, atom_array, path) logger.info(f"Wrote {path}.") diff --git a/tests/others/test_rfd4_proteina_training.py b/tests/others/test_rfd4_proteina_training.py new file mode 100644 index 000000000000..5daaa7087a7d --- /dev/null +++ b/tests/others/test_rfd4_proteina_training.py @@ -0,0 +1,230 @@ +# Copyright 2026 The HuggingFace Team. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy at http://www.apache.org/licenses/LICENSE-2.0 + +import importlib.util +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch + + +@pytest.fixture +def training(monkeypatch): + path = Path(__file__).resolve().parents[2] / "examples/rfd4_proteina/train_rfd4_proteina_lora.py" + spec = importlib.util.spec_from_file_location("rfd4_training", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + monkeypatch.setattr(module, "logger", Mock()) + monkeypatch.setitem(sys.modules, "rfd4_proteina_prep", SimpleNamespace(prepare_cif=Mock())) + return module + + +def features(tokens=2, resolved=True, emitted=True): + return { + "padding_mask": torch.ones(1, tokens, dtype=torch.bool), + "is_in_context_conditioning": torch.full((1, tokens), not emitted, dtype=torch.bool), + "atom37_mask": torch.ones(1, tokens, 37, dtype=torch.bool), + "coords_nm": torch.full((1, tokens, 37, 3), 0.0 if resolved else float("nan")), + } + + +def test_nested_cache_keys_and_file_changes(training, tmp_path): + for subdir in ("a", "b"): + (tmp_path / subdir).mkdir() + (tmp_path / subdir / "model.cif").touch() + prep = sys.modules["rfd4_proteina_prep"].prepare_cif + prep.side_effect = lambda path, **kwargs: (features(2 if path.parent.name == "a" else 3), None) + dataset = training.PreparedStructureDataset(tmp_path, 0, 4, tmp_path / "cache") + assert [dataset[i]["padding_mask"].shape[1] for i in range(2)] == [2, 3] + assert prep.call_count == 2 + assert len(list((tmp_path / "cache").glob("*.pt"))) == 2 + dataset.paths[0].write_text("changed source") + dataset[0] + assert prep.call_count == 3 + + +def test_filter_before_sharding_and_cache_in_memory(training, tmp_path): + for name in ("good", "invalid", "long", "unresolved", "context"): + (tmp_path / f"{name}.cif").touch() + + def prepare(path, **kwargs): + if path.stem == "invalid": + raise ValueError("unusable structure") + return features(5 if path.stem == "long" else 2, path.stem != "unresolved", path.stem != "context"), None + + prep = sys.modules["rfd4_proteina_prep"].prepare_cif + prep.side_effect = prepare + dataset = training.PreparedStructureDataset(tmp_path, 0, 4) + assert [path.stem for path in dataset.paths] == ["good"] + for _ in range(3): + assert dataset[0]["padding_mask"].shape == (1, 2) + assert prep.call_count == 5 + assert len(dataset._memory_cache) == 1 + (tmp_path / "good.cif").unlink() + with pytest.raises(ValueError, match="No trainable structures"): + training.PreparedStructureDataset(tmp_path, 0, 4) + + +def test_concurrent_cache_preparation_is_atomic(training, tmp_path, monkeypatch): + (tmp_path / "model.cif").touch() + cache = tmp_path / "cache" + prep = sys.modules["rfd4_proteina_prep"].prepare_cif + + def prepare(*args, **kwargs): + time.sleep(0.05) + return features(), None + + prep.side_effect = prepare + save = torch.save + + def slow_save(value, handle): + assert not list(cache.glob("*.pt.lock"))[0].with_suffix("").exists() + time.sleep(0.05) + save(value, handle) + + monkeypatch.setattr(torch, "save", slow_save) + with ThreadPoolExecutor(max_workers=4) as pool: + datasets = list(pool.map(lambda _: training.PreparedStructureDataset(tmp_path, 0, 4, cache), range(4))) + assert prep.call_count == 1 + assert len(list(cache.glob("*.pt"))) == 1 + for dataset in datasets: + torch.testing.assert_close(dataset[0]["coords_nm"], features()["coords_nm"]) + + +@pytest.mark.parametrize("main", [True, False]) +def test_checkpoint_all_ranks_save_only_main_prunes(training, tmp_path, main): + (tmp_path / "checkpoint-1").mkdir() + accelerator = Mock(is_main_process=main) + args = SimpleNamespace(output_dir=str(tmp_path), checkpoints_total_limit=1) + training.save_checkpoint(accelerator, args, 2) + accelerator.save_state.assert_called_once_with(str(tmp_path / "checkpoint-2")) + assert accelerator.wait_for_everyone.call_count == 2 + assert (tmp_path / "checkpoint-1").exists() is not main + + +@pytest.mark.parametrize("fail", [True, False]) +@pytest.mark.parametrize("initial_training", [True, False]) +def test_validation_eval_and_restore(training, tmp_path, monkeypatch, fail, initial_training): + transformer = torch.nn.Sequential(torch.nn.Dropout(0.5)).train(initial_training) + prep_features = {key: torch.zeros(1) for key in training.PIPELINE_INPUTS} + writer = Mock() + monkeypatch.setitem( + sys.modules, + "rfd4_proteina_prep", + SimpleNamespace(prepare_cif=lambda *a, **k: (prep_features, None), write_cif=writer), + ) + + class Pipeline: + def __init__(self, **kwargs): + pass + + def set_progress_bar_config(self, **kwargs): + pass + + def __call__(self, **kwargs): + assert not transformer.training + assert not torch.is_grad_enabled() + if fail: + raise RuntimeError("generation failed") + return object() + + monkeypatch.setattr(training, "RFD4ProteinaPipeline", Pipeline) + args = SimpleNamespace(validation_cif="input.cif", prep_seed=0, seed=0, output_dir=str(tmp_path)) + + def call(): + training.run_validation( + SimpleNamespace(device=torch.device("cpu")), + args, + transformer, + None, + {"bb_ca": None, "local_latents": None}, + 1, + ) + + if fail: + with pytest.raises(RuntimeError, match="generation failed"): + call() + else: + call() + writer.assert_called_once() + assert transformer.training is initial_training + + +def test_lora_metadata_preserves_nondefault_scaling(training, tmp_path): + from peft import LoraConfig + from peft.utils import get_peft_model_state_dict + + from diffusers import RFD4ProteinaTransformer + + config = { + "nlayers": 1, + "token_dim": 8, + "pair_repr_dim": 4, + "nheads": 2, + "feats_seq": ["xt_bb_ca", "xt_local_latents"], + "feats_cond_seq": [], + "feats_pair_repr": [], + "feats_pair_cond": [], + "dim_cond": 8, + "latent_dim": 4, + } + model = RFD4ProteinaTransformer(**config) + model.add_adapter( + LoraConfig(r=2, lora_alpha=7, lora_dropout=0.1, target_modules=["to_q", "to_k", "to_v", "to_out"]) + ) + with torch.no_grad(): + for name, param in model.named_parameters(): + if "lora_B" in name: + param.normal_() + training.RFD4ProteinaPipeline.save_lora_weights( + tmp_path, + transformer_lora_layers=get_peft_model_state_dict(model), + **training._collate_lora_metadata({"transformer": model}), + ) + state, metadata = training.RFD4ProteinaPipeline.lora_state_dict(tmp_path, return_lora_metadata=True) + restored = RFD4ProteinaTransformer(**config) + training.RFD4ProteinaPipeline.load_lora_into_transformer( + state, transformer=restored, metadata=metadata, adapter_name="default" + ) + assert restored.peft_config["default"].lora_alpha == 7 + assert restored.peft_config["default"].lora_dropout == 0.1 + torch.testing.assert_close(get_peft_model_state_dict(model), get_peft_model_state_dict(restored)) + for name, module in model.named_modules(): + if hasattr(module, "scaling"): + other = restored.get_submodule(name) + assert other.scaling["default"] == module.scaling["default"] == 3.5 + value = torch.randn(1, 2, module.in_features) + module.eval() + other.eval() + + # Compare the adapter contribution without depending on the randomly initialized base layer. + def adapter(layer): + return layer.lora_B["default"](layer.lora_A["default"](value)) * layer.scaling["default"] + + torch.testing.assert_close(adapter(module), adapter(other), rtol=0, atol=0) + + +def test_failed_cache_write_does_not_publish_partial_file(training, tmp_path, monkeypatch): + (tmp_path / "model.cif").touch() + cache = tmp_path / "cache" + sys.modules["rfd4_proteina_prep"].prepare_cif.return_value = (features(), None) + save = torch.save + + def fail_save(value, handle): + handle.write(b"partial") + raise OSError("disk full") + + monkeypatch.setattr(torch, "save", fail_save) + with pytest.raises(OSError, match="disk full"): + training.PreparedStructureDataset(tmp_path, 0, 4, cache) + assert not list(cache.glob("*.pt")) + monkeypatch.setattr(torch, "save", save) + dataset = training.PreparedStructureDataset(tmp_path, 0, 4, cache) + torch.testing.assert_close(dataset[0]["coords_nm"], features()["coords_nm"]) From f1efe9a533ea7a3bff14557260946914dbf7cc72 Mon Sep 17 00:00:00 2001 From: kierandidi Date: Sun, 8 Nov 2026 20:53:37 +0000 Subject: [PATCH 04/11] Trim RFD4 example docs and keep focused dataset tests --- examples/rfd4_proteina/README.md | 71 ++---- .../rfd4_proteina/test_rfd4_proteina_lora.py | 79 ++++++ .../rfd4_proteina/train_rfd4_proteina_lora.py | 46 ++-- tests/others/test_rfd4_proteina_training.py | 230 ------------------ 4 files changed, 113 insertions(+), 313 deletions(-) create mode 100644 examples/rfd4_proteina/test_rfd4_proteina_lora.py delete mode 100644 tests/others/test_rfd4_proteina_training.py diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index a4add0d98746..fadfb908bf47 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -1,12 +1,8 @@ # LoRA fine-tuning for RFD4-Proteina -[RFD4-Proteína](https://huggingface.co/docs/diffusers/main/en/api/pipelines/rfd4_proteina) generates protein backbones -and sequences by flow matching in a product space of backbone coordinates and per-token local latents. This example -trains LoRA adapters on the denoiser so it follows a set of structures you care about, and writes them in the format -`RFD4ProteinaPipeline.load_lora_weights` reads. - -It follows the fine-tuning recipe the model was released with: the autoencoder stays frozen and paired with the -denoiser, adapters go on the attention projections, and the loss scores the predicted clean sample of both modalities. +This example fine-tunes [RFD4-Proteína](https://huggingface.co/docs/diffusers/main/en/api/pipelines/rfd4_proteina) +with LoRA on a folder of annotated CIF files. The autoencoder stays frozen; only the denoiser adapters train. +Saved adapters load with `RFD4ProteinaPipeline.load_lora_weights`. ## Setup @@ -71,48 +67,22 @@ accelerate launch examples/rfd4_proteina/train_rfd4_proteina_lora.py \ --seed 0 ``` -Notes on the defaults: - -- **Rank, alpha and target modules** are the released fine-tune's: rank 32, alpha 64, adapters on the attention - projections (`to_q`, `to_k`, `to_v`, `to_out`, which are the native recipe's `to_qkv` and `to_out_node`). -- **One structure per batch.** Token counts differ per structure and the model has no padding-free path, so the batch - size is fixed at one; raise the effective batch size with `--gradient_accumulation_steps`. -- **`--max_tokens`** skips structures longer than 384 tokens, as the released fine-tune does. Memory and step time grow - quickly with token count. -- **Preparation is cached.** Parsing and featurizing a CIF costs far more than a training step, so pass - `--features_cache_dir` to prepare each structure once and reuse it across epochs. Cache keys include the full source - path, file size, modification time and preparation seed. Writes are locked and published atomically so distributed - workers and concurrent runs can share the cache. Without a cache directory, prepared tensors stay in CPU memory. -- **`--gradient_checkpointing`** trades compute for memory if a long structure does not fit. -- **`--recycle_depth`** sets the trunk passes per training step, one by default. Sampling recycles more deeply, up to - six passes early in the schedule, so a deeper training step matches inference more closely and costs proportionally - more time and memory. - -- **Structures preparation rejects are skipped** with a warning — an experimental file whose coordinates the model's - own input checks refuse, for instance. A run where nothing can be prepared fails rather than writing an untrained - adapter. Filtering happens before distributed sharding, including the token limit and structures without any resolved - emitted CA atoms, so all ranks execute the same number of training steps. -- **Missing atoms are dropped from the mask.** A deposited structure does not always hold every atom its residues - should have; those slots are removed before encoding, and a token is supervised only where its CA is resolved. - -Pass `--validation_cif` to generate from one input every `--validation_steps`, with the sampler settings the -checkpoint was released with, and write the result next to the adapter. Generation uses the checkpoint's own recycle -schedule, which also fixes the number of integration steps. In distributed training, only the main process generates -and writes validation output; the other ranks wait. Validation temporarily switches the transformer to evaluation -mode, disabling LoRA dropout, then restores its previous mode. - -For multi-GPU training, add `--multi_gpu --num_processes 2` to `accelerate launch`. All ranks participate in -checkpoint saving, including their RNG states; only the main process prunes old checkpoints and writes adapter -weights. Resume with `--resume_from_checkpoint latest` and the same LoRA configuration. Both checkpoint and final -adapter files include the PEFT metadata, preserving the rank and alpha when loaded for inference. - -The reference noise is drawn with the pipeline's own helpers, so the noise end of the training path is the same -distribution the sampler starts from: masked to the emitted tokens, and for the backbone with zero centre of mass and -the per-molecule stochastic offset. - -The loss is flow matching on the two modalities. The native recipe adds auxiliary sequence and insertion terms, which -this example leaves out: adapters trained here move the structure distribution, and the sequence readout follows only -indirectly. +Training uses rank 32 / alpha 64 adapters on `to_q`, `to_k`, `to_v` and `to_out`, following the released recipe. +The flow-matching loss scores the predicted clean backbone and local latents. It does not include the native +trainer's auxiliary sequence and insertion losses. + +- Each batch holds one structure. Use `--gradient_accumulation_steps` to increase the effective batch size. +- Structures above `--max_tokens` (384 by default), rejected inputs, and structures without resolved target CA atoms + are filtered before training. Missing atoms are masked before encoding. +- `--features_cache_dir` avoids preparing the CIFs again on later runs. Without it, features stay in CPU memory. +- `--gradient_checkpointing` saves GPU memory. `--recycle_depth` controls trunk passes per training step (default 1). + +For two GPUs, add `--multi_gpu --num_processes 2` to `accelerate launch`. +Resume with `--resume_from_checkpoint latest`, keeping the same output directory and LoRA configuration. +Use `--checkpointing_steps` and `--checkpoints_total_limit` to control checkpoint frequency and retention. + +Pass `--validation_cif` to generate a structure every `--validation_steps`. Validation runs on the main process, +with dropout disabled, and writes a CIF next to the adapter. It uses the checkpoint's full sampling schedule. ## Inference @@ -143,8 +113,7 @@ output = pipe( write_cif(output, features, atom_array, "design.cif") ``` -An adapter trained with the model's own trainer loads too, once converted — the native recipe adapts a fused -`to_qkv`, which the conversion splits into the `to_q`, `to_k` and `to_v` this transformer exposes: +Convert native adapters before loading them. This splits the fused `to_qkv` weights into `to_q`, `to_k` and `to_v`: ```bash python scripts/convert_rfd4_proteina_to_diffusers.py \ diff --git a/examples/rfd4_proteina/test_rfd4_proteina_lora.py b/examples/rfd4_proteina/test_rfd4_proteina_lora.py new file mode 100644 index 000000000000..8a861eecbaab --- /dev/null +++ b/examples/rfd4_proteina/test_rfd4_proteina_lora.py @@ -0,0 +1,79 @@ +# coding=utf-8 +# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import torch +import train_rfd4_proteina_lora as training + + +class RFD4ProteinaDatasetTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + self.prepare = Mock() + for patcher in ( + patch.dict(sys.modules, {"rfd4_proteina_prep": SimpleNamespace(prepare_cif=self.prepare)}), + patch.object(training, "logger", logging.getLogger(__name__)), + ): + patcher.start() + self.addCleanup(patcher.stop) + + @staticmethod + def features(tokens=2, resolved=True): + return { + "padding_mask": torch.ones(1, tokens, dtype=torch.bool), + "is_in_context_conditioning": torch.zeros(1, tokens, dtype=torch.bool), + "atom37_mask": torch.ones(1, tokens, 37, dtype=torch.bool), + "coords_nm": torch.full((1, tokens, 37, 3), 0.0 if resolved else float("nan")), + } + + def test_cache_distinguishes_nested_filenames(self): + for name in ("a", "b"): + (self.root / name).mkdir() + (self.root / name / "model.cif").touch() + self.prepare.side_effect = lambda path, **kwargs: (self.features(2 if path.parent.name == "a" else 3), None) + cache = self.root / "cache" + for _ in range(2): + dataset = training.PreparedStructureDataset(self.root, 0, 4, cache) + self.assertEqual([dataset[i]["padding_mask"].shape[1] for i in range(2)], [2, 3]) + self.assertEqual(self.prepare.call_count, 2) + + def test_filters_untrainable_structures(self): + for name in ("good", "invalid", "long", "unresolved", "context"): + (self.root / f"{name}.cif").touch() + + def prepare(path, **kwargs): + if path.stem == "invalid": + raise ValueError("unusable structure") + features = self.features(5 if path.stem == "long" else 2, path.stem != "unresolved") + if path.stem == "context": + features["is_in_context_conditioning"].fill_(True) + return features, None + + self.prepare.side_effect = prepare + dataset = training.PreparedStructureDataset(self.root, 0, 4) + self.assertEqual([path.stem for path in dataset.paths], ["good"]) + self.assertEqual(dataset[0]["padding_mask"].shape, (1, 2)) + (self.root / "good.cif").unlink() + with self.assertRaisesRegex(ValueError, "No trainable structures"): + training.PreparedStructureDataset(self.root, 0, 4) diff --git a/examples/rfd4_proteina/train_rfd4_proteina_lora.py b/examples/rfd4_proteina/train_rfd4_proteina_lora.py index f6e600856601..fbfd518a789a 100644 --- a/examples/rfd4_proteina/train_rfd4_proteina_lora.py +++ b/examples/rfd4_proteina/train_rfd4_proteina_lora.py @@ -44,7 +44,7 @@ logger = get_logger(__name__) -# The pipeline takes the molecular tensors by name and the rest of the prepared features as conditioning. +# Other prepared features are passed as conditioning. PIPELINE_INPUTS = ( "padding_mask", "is_in_context_conditioning", @@ -54,14 +54,14 @@ "coords_nm", ) -# Training times per modality, as the released recipe samples them: a Beta distribution mixed with a uniform one. +# Match the released recipe: a mixture of Beta and uniform training times. # See `loss.t_distribution` in the RFD4-Proteina `configs/training_proteina.yaml`. TIME_DISTRIBUTION = { "bb_ca": {"concentration1": 1.9, "concentration0": 1.0, "uniform_fraction": 0.02}, "local_latents": {"concentration1": 1.0, "concentration0": 1.5, "uniform_fraction": 0.2}, } -# The backbone is the modality the sampler draws with zero centre of mass and a per-molecule stochastic offset. +# Backbone noise uses zero centre of mass and a per-molecule offset. BACKBONE = "bb_ca" @@ -154,11 +154,7 @@ def parse_args(): class PreparedStructureDataset(Dataset): - """The CIF files under a directory, prepared into the feature tensors the model consumes. - - Preparation parses the file with AtomWorks and runs the model's own datapipe, which is slow enough to be worth - caching: each file is prepared once and reloaded from `cache_dir` afterwards. - """ + """Prepare CIF features once, filter unusable structures, and cache the results.""" def __init__(self, data_dir: str, seed: int, max_tokens: int, cache_dir: str | None = None): from rfd4_proteina_prep import prepare_cif @@ -176,7 +172,7 @@ def __init__(self, data_dir: str, seed: int, max_tokens: int, cache_dir: str | N self.paths = sorted(path for pattern in ("*.cif", "*.cif.gz", "*.cif.zst") for path in root.rglob(pattern)) if not self.paths: raise ValueError(f"No CIF files found under {data_dir}.") - # Filter before Accelerate shards the loader: every rank must execute the same number of backwards. + # Filter before sharding so every rank runs the same number of backward passes. self.paths = [path for path in self.paths if self._is_trainable(path)] if not self.paths: raise ValueError(f"No trainable structures found under {data_dir}.") @@ -189,7 +185,7 @@ def _features(self, path: Path) -> dict[str, torch.Tensor]: if path not in self._memory_cache: self._memory_cache[path] = self._prepare_features(path) return self._memory_cache[path] - # Include the full source path and file version, so nested names and edited inputs cannot alias. + # Distinguish duplicate filenames and invalidate edited inputs. stat = path.stat() key = hashlib.sha256(f"{path.resolve()}:{stat.st_size}:{stat.st_mtime_ns}:{self.seed}".encode()).hexdigest() cached = self.cache_dir / f"{key}.pt" @@ -197,7 +193,7 @@ def _features(self, path: Path) -> dict[str, torch.Tensor]: if cached.is_file(): return torch.load(cached, map_location="cpu", weights_only=True) features = self._prepare_features(path) - # Publish only complete files, including when different jobs share this cache. + # Readers must never see a partially written cache file. temporary = None try: with tempfile.NamedTemporaryFile(dir=self.cache_dir, suffix=".pt", delete=False) as handle: @@ -234,10 +230,7 @@ def __getitem__(self, index: int) -> dict[str, torch.Tensor]: def collate_fn(examples): - """Keep one structure per batch: token counts differ per structure, and the model has no padding-free path. - - Raise the effective batch size with `--gradient_accumulation_steps` instead. - """ + """Use one structure per batch because token counts vary.""" return examples[0] @@ -257,12 +250,7 @@ def sample_times(batch_size: int, device: torch.device) -> dict[str, torch.Tenso def draw_noise( name: str, shape: tuple[int, ...], emit_mask: torch.Tensor, conditioning: dict[str, torch.Tensor] ) -> torch.Tensor: - """Draw one modality's reference noise the way the sampler draws its starting state. - - Training and sampling have to agree on what the noise end of the path looks like, so the draw reuses the - pipeline's own helpers: masked to the emitted tokens, and for the backbone with zero centre of mass and the - per-molecule stochastic offset the released recipe applies. - """ + """Use the pipeline's noise distribution for training.""" noise = RFD4ProteinaPipeline._sample_noise(shape, emit_mask, None, zero_com=name == BACKBONE) if name != BACKBONE: return noise @@ -270,11 +258,7 @@ def draw_noise( def flow_matching_loss(prediction, state, times, clean, supervision_masks) -> torch.Tensor: - """Average the per-modality error of the predicted clean sample over the supervised tokens. - - The model predicts a velocity; `state + (1 - t) * velocity` is its estimate of the clean sample, which is what the - released recipe scores. Each modality is normalised by its own token count and the two terms are summed. - """ + """Score the clean estimate `state + (1 - t) * velocity`, normalized by supervised token count.""" velocities = {"bb_ca": prediction.backbone.float(), "local_latents": prediction.local_latents.float()} loss = 0.0 for name, velocity in velocities.items(): @@ -321,7 +305,7 @@ def main(): vae = RFD4ProteinaAutoencoder.from_pretrained( args.pretrained_model_name_or_path, subfolder="vae", torch_dtype=weight_dtype, **load_kwargs ) - # One scheduler per modality, as the pipeline holds them; training only uses their shared `add_noise`. + # Each modality has its own scheduler, as in the pipeline. schedulers = { name: RFD4ProteinaScheduler.from_pretrained( args.pretrained_model_name_or_path, @@ -332,7 +316,7 @@ def main(): for name, subfolder in (("bb_ca", "bb_ca_scheduler"), ("local_latents", "local_latents_scheduler")) } - # The autoencoder is paired with the denoiser and stays frozen; only the adapters on the denoiser train. + # Freeze the base model and train only the denoiser adapters. vae.requires_grad_(False) transformer.requires_grad_(False) vae.to(accelerator.device) @@ -456,7 +440,7 @@ def load_model_hook(models, input_dir): progress_bar = tqdm( range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process, desc="Steps" ) - # All entries are trainable before sharding; cycle the data until the step count is met. + # Repeat the dataset until max_train_steps is reached. while global_step < args.max_train_steps: transformer.train() for batch in dataloader: @@ -471,9 +455,7 @@ def load_model_hook(models, input_dir): "bb_ca": emitted, "local_latents": emitted & features["supports_local_latent"].bool(), } - # An experimental structure is missing atoms the residue should have: their slots are marked present - # but hold no coordinate. Drop them from the mask and zero them, or the encoder returns NaN latents - # for the whole structure. + # Mask missing coordinates before encoding to avoid NaN latents. resolved = torch.isfinite(features["coords_nm"]).all(dim=-1) atom37_mask = features["atom37_mask"].bool() & resolved coords_nm = torch.nan_to_num(features["coords_nm"]) diff --git a/tests/others/test_rfd4_proteina_training.py b/tests/others/test_rfd4_proteina_training.py deleted file mode 100644 index 5daaa7087a7d..000000000000 --- a/tests/others/test_rfd4_proteina_training.py +++ /dev/null @@ -1,230 +0,0 @@ -# Copyright 2026 The HuggingFace Team. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy at http://www.apache.org/licenses/LICENSE-2.0 - -import importlib.util -import sys -import time -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest -import torch - - -@pytest.fixture -def training(monkeypatch): - path = Path(__file__).resolve().parents[2] / "examples/rfd4_proteina/train_rfd4_proteina_lora.py" - spec = importlib.util.spec_from_file_location("rfd4_training", path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - monkeypatch.setattr(module, "logger", Mock()) - monkeypatch.setitem(sys.modules, "rfd4_proteina_prep", SimpleNamespace(prepare_cif=Mock())) - return module - - -def features(tokens=2, resolved=True, emitted=True): - return { - "padding_mask": torch.ones(1, tokens, dtype=torch.bool), - "is_in_context_conditioning": torch.full((1, tokens), not emitted, dtype=torch.bool), - "atom37_mask": torch.ones(1, tokens, 37, dtype=torch.bool), - "coords_nm": torch.full((1, tokens, 37, 3), 0.0 if resolved else float("nan")), - } - - -def test_nested_cache_keys_and_file_changes(training, tmp_path): - for subdir in ("a", "b"): - (tmp_path / subdir).mkdir() - (tmp_path / subdir / "model.cif").touch() - prep = sys.modules["rfd4_proteina_prep"].prepare_cif - prep.side_effect = lambda path, **kwargs: (features(2 if path.parent.name == "a" else 3), None) - dataset = training.PreparedStructureDataset(tmp_path, 0, 4, tmp_path / "cache") - assert [dataset[i]["padding_mask"].shape[1] for i in range(2)] == [2, 3] - assert prep.call_count == 2 - assert len(list((tmp_path / "cache").glob("*.pt"))) == 2 - dataset.paths[0].write_text("changed source") - dataset[0] - assert prep.call_count == 3 - - -def test_filter_before_sharding_and_cache_in_memory(training, tmp_path): - for name in ("good", "invalid", "long", "unresolved", "context"): - (tmp_path / f"{name}.cif").touch() - - def prepare(path, **kwargs): - if path.stem == "invalid": - raise ValueError("unusable structure") - return features(5 if path.stem == "long" else 2, path.stem != "unresolved", path.stem != "context"), None - - prep = sys.modules["rfd4_proteina_prep"].prepare_cif - prep.side_effect = prepare - dataset = training.PreparedStructureDataset(tmp_path, 0, 4) - assert [path.stem for path in dataset.paths] == ["good"] - for _ in range(3): - assert dataset[0]["padding_mask"].shape == (1, 2) - assert prep.call_count == 5 - assert len(dataset._memory_cache) == 1 - (tmp_path / "good.cif").unlink() - with pytest.raises(ValueError, match="No trainable structures"): - training.PreparedStructureDataset(tmp_path, 0, 4) - - -def test_concurrent_cache_preparation_is_atomic(training, tmp_path, monkeypatch): - (tmp_path / "model.cif").touch() - cache = tmp_path / "cache" - prep = sys.modules["rfd4_proteina_prep"].prepare_cif - - def prepare(*args, **kwargs): - time.sleep(0.05) - return features(), None - - prep.side_effect = prepare - save = torch.save - - def slow_save(value, handle): - assert not list(cache.glob("*.pt.lock"))[0].with_suffix("").exists() - time.sleep(0.05) - save(value, handle) - - monkeypatch.setattr(torch, "save", slow_save) - with ThreadPoolExecutor(max_workers=4) as pool: - datasets = list(pool.map(lambda _: training.PreparedStructureDataset(tmp_path, 0, 4, cache), range(4))) - assert prep.call_count == 1 - assert len(list(cache.glob("*.pt"))) == 1 - for dataset in datasets: - torch.testing.assert_close(dataset[0]["coords_nm"], features()["coords_nm"]) - - -@pytest.mark.parametrize("main", [True, False]) -def test_checkpoint_all_ranks_save_only_main_prunes(training, tmp_path, main): - (tmp_path / "checkpoint-1").mkdir() - accelerator = Mock(is_main_process=main) - args = SimpleNamespace(output_dir=str(tmp_path), checkpoints_total_limit=1) - training.save_checkpoint(accelerator, args, 2) - accelerator.save_state.assert_called_once_with(str(tmp_path / "checkpoint-2")) - assert accelerator.wait_for_everyone.call_count == 2 - assert (tmp_path / "checkpoint-1").exists() is not main - - -@pytest.mark.parametrize("fail", [True, False]) -@pytest.mark.parametrize("initial_training", [True, False]) -def test_validation_eval_and_restore(training, tmp_path, monkeypatch, fail, initial_training): - transformer = torch.nn.Sequential(torch.nn.Dropout(0.5)).train(initial_training) - prep_features = {key: torch.zeros(1) for key in training.PIPELINE_INPUTS} - writer = Mock() - monkeypatch.setitem( - sys.modules, - "rfd4_proteina_prep", - SimpleNamespace(prepare_cif=lambda *a, **k: (prep_features, None), write_cif=writer), - ) - - class Pipeline: - def __init__(self, **kwargs): - pass - - def set_progress_bar_config(self, **kwargs): - pass - - def __call__(self, **kwargs): - assert not transformer.training - assert not torch.is_grad_enabled() - if fail: - raise RuntimeError("generation failed") - return object() - - monkeypatch.setattr(training, "RFD4ProteinaPipeline", Pipeline) - args = SimpleNamespace(validation_cif="input.cif", prep_seed=0, seed=0, output_dir=str(tmp_path)) - - def call(): - training.run_validation( - SimpleNamespace(device=torch.device("cpu")), - args, - transformer, - None, - {"bb_ca": None, "local_latents": None}, - 1, - ) - - if fail: - with pytest.raises(RuntimeError, match="generation failed"): - call() - else: - call() - writer.assert_called_once() - assert transformer.training is initial_training - - -def test_lora_metadata_preserves_nondefault_scaling(training, tmp_path): - from peft import LoraConfig - from peft.utils import get_peft_model_state_dict - - from diffusers import RFD4ProteinaTransformer - - config = { - "nlayers": 1, - "token_dim": 8, - "pair_repr_dim": 4, - "nheads": 2, - "feats_seq": ["xt_bb_ca", "xt_local_latents"], - "feats_cond_seq": [], - "feats_pair_repr": [], - "feats_pair_cond": [], - "dim_cond": 8, - "latent_dim": 4, - } - model = RFD4ProteinaTransformer(**config) - model.add_adapter( - LoraConfig(r=2, lora_alpha=7, lora_dropout=0.1, target_modules=["to_q", "to_k", "to_v", "to_out"]) - ) - with torch.no_grad(): - for name, param in model.named_parameters(): - if "lora_B" in name: - param.normal_() - training.RFD4ProteinaPipeline.save_lora_weights( - tmp_path, - transformer_lora_layers=get_peft_model_state_dict(model), - **training._collate_lora_metadata({"transformer": model}), - ) - state, metadata = training.RFD4ProteinaPipeline.lora_state_dict(tmp_path, return_lora_metadata=True) - restored = RFD4ProteinaTransformer(**config) - training.RFD4ProteinaPipeline.load_lora_into_transformer( - state, transformer=restored, metadata=metadata, adapter_name="default" - ) - assert restored.peft_config["default"].lora_alpha == 7 - assert restored.peft_config["default"].lora_dropout == 0.1 - torch.testing.assert_close(get_peft_model_state_dict(model), get_peft_model_state_dict(restored)) - for name, module in model.named_modules(): - if hasattr(module, "scaling"): - other = restored.get_submodule(name) - assert other.scaling["default"] == module.scaling["default"] == 3.5 - value = torch.randn(1, 2, module.in_features) - module.eval() - other.eval() - - # Compare the adapter contribution without depending on the randomly initialized base layer. - def adapter(layer): - return layer.lora_B["default"](layer.lora_A["default"](value)) * layer.scaling["default"] - - torch.testing.assert_close(adapter(module), adapter(other), rtol=0, atol=0) - - -def test_failed_cache_write_does_not_publish_partial_file(training, tmp_path, monkeypatch): - (tmp_path / "model.cif").touch() - cache = tmp_path / "cache" - sys.modules["rfd4_proteina_prep"].prepare_cif.return_value = (features(), None) - save = torch.save - - def fail_save(value, handle): - handle.write(b"partial") - raise OSError("disk full") - - monkeypatch.setattr(torch, "save", fail_save) - with pytest.raises(OSError, match="disk full"): - training.PreparedStructureDataset(tmp_path, 0, 4, cache) - assert not list(cache.glob("*.pt")) - monkeypatch.setattr(torch, "save", save) - dataset = training.PreparedStructureDataset(tmp_path, 0, 4, cache) - torch.testing.assert_close(dataset[0]["coords_nm"], features()["coords_nm"]) From ca024cf21aece0571700280da0feddc37a0fcd66 Mon Sep 17 00:00:00 2001 From: kierandidi Date: Mon, 9 Nov 2026 11:01:51 +0000 Subject: [PATCH 05/11] Address RFD4 example follow-up review --- examples/rfd4_proteina/README.md | 22 +++++- .../rfd4_proteina/test_rfd4_proteina_lora.py | 79 ------------------- .../rfd4_proteina/train_rfd4_proteina_lora.py | 2 +- 3 files changed, 21 insertions(+), 82 deletions(-) delete mode 100644 examples/rfd4_proteina/test_rfd4_proteina_lora.py diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index fadfb908bf47..9b81d650c168 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -113,11 +113,29 @@ output = pipe( write_cif(output, features, atom_array, "design.cif") ``` -Convert native adapters before loading them. This splits the fused `to_qkv` weights into `to_q`, `to_k` and `to_v`: +The [released native adapter](https://huggingface.co/nvidia/RFD4-Proteina/tree/main/adapter) is one example you can +convert. Download it first: + +```python +from huggingface_hub import snapshot_download + +snapshot_download("nvidia/RFD4-Proteina", allow_patterns=["adapter/*"], local_dir="rfd4-native") +``` + +Conversion splits the fused `to_qkv` weights into `to_q`, `to_k` and `to_v`: ```bash python scripts/convert_rfd4_proteina_to_diffusers.py \ --adapter \ - --source native-adapter \ + --source rfd4-native/adapter \ --destination rfd4-proteina-lora-converted ``` + +This adapter goes with the `pretrain-base` revision. The default revision already has it merged: + +```python +pipe = RFD4ProteinaPipeline.from_pretrained( + "nvidia/RFD4-Proteina-Diffusers", revision="pretrain-base", dtype=torch.bfloat16 +).to("cuda") +pipe.load_lora_weights("rfd4-proteina-lora-converted") +``` diff --git a/examples/rfd4_proteina/test_rfd4_proteina_lora.py b/examples/rfd4_proteina/test_rfd4_proteina_lora.py deleted file mode 100644 index 8a861eecbaab..000000000000 --- a/examples/rfd4_proteina/test_rfd4_proteina_lora.py +++ /dev/null @@ -1,79 +0,0 @@ -# coding=utf-8 -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -import sys -import tempfile -import unittest -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import Mock, patch - -import torch -import train_rfd4_proteina_lora as training - - -class RFD4ProteinaDatasetTests(unittest.TestCase): - def setUp(self): - directory = tempfile.TemporaryDirectory() - self.addCleanup(directory.cleanup) - self.root = Path(directory.name) - self.prepare = Mock() - for patcher in ( - patch.dict(sys.modules, {"rfd4_proteina_prep": SimpleNamespace(prepare_cif=self.prepare)}), - patch.object(training, "logger", logging.getLogger(__name__)), - ): - patcher.start() - self.addCleanup(patcher.stop) - - @staticmethod - def features(tokens=2, resolved=True): - return { - "padding_mask": torch.ones(1, tokens, dtype=torch.bool), - "is_in_context_conditioning": torch.zeros(1, tokens, dtype=torch.bool), - "atom37_mask": torch.ones(1, tokens, 37, dtype=torch.bool), - "coords_nm": torch.full((1, tokens, 37, 3), 0.0 if resolved else float("nan")), - } - - def test_cache_distinguishes_nested_filenames(self): - for name in ("a", "b"): - (self.root / name).mkdir() - (self.root / name / "model.cif").touch() - self.prepare.side_effect = lambda path, **kwargs: (self.features(2 if path.parent.name == "a" else 3), None) - cache = self.root / "cache" - for _ in range(2): - dataset = training.PreparedStructureDataset(self.root, 0, 4, cache) - self.assertEqual([dataset[i]["padding_mask"].shape[1] for i in range(2)], [2, 3]) - self.assertEqual(self.prepare.call_count, 2) - - def test_filters_untrainable_structures(self): - for name in ("good", "invalid", "long", "unresolved", "context"): - (self.root / f"{name}.cif").touch() - - def prepare(path, **kwargs): - if path.stem == "invalid": - raise ValueError("unusable structure") - features = self.features(5 if path.stem == "long" else 2, path.stem != "unresolved") - if path.stem == "context": - features["is_in_context_conditioning"].fill_(True) - return features, None - - self.prepare.side_effect = prepare - dataset = training.PreparedStructureDataset(self.root, 0, 4) - self.assertEqual([path.stem for path in dataset.paths], ["good"]) - self.assertEqual(dataset[0]["padding_mask"].shape, (1, 2)) - (self.root / "good.cif").unlink() - with self.assertRaisesRegex(ValueError, "No trainable structures"): - training.PreparedStructureDataset(self.root, 0, 4) diff --git a/examples/rfd4_proteina/train_rfd4_proteina_lora.py b/examples/rfd4_proteina/train_rfd4_proteina_lora.py index fbfd518a789a..4714f84e6cc1 100644 --- a/examples/rfd4_proteina/train_rfd4_proteina_lora.py +++ b/examples/rfd4_proteina/train_rfd4_proteina_lora.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding=utf-8 -# Copyright 2026 The HuggingFace Inc. team. All rights reserved. +# Copyright 2026 The NVIDIA Team and The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 33764084acc4fd27f7671a17b7043511c39e4087 Mon Sep 17 00:00:00 2001 From: Kieran Didi Date: Mon, 9 Nov 2026 15:01:49 +0000 Subject: [PATCH 06/11] Show held-out validation for the RFD4 LoRA example Train on 40 of the fifty published structures and score the remaining ten with the adapter toggled on and off over identical noise draws. Held-out flow-matching loss falls 11%, on every held-out structure. --- examples/rfd4_proteina/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index 9b81d650c168..baea2dd890b6 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -84,6 +84,23 @@ Use `--checkpointing_steps` and `--checkpoints_total_limit` to control checkpoin Pass `--validation_cif` to generate a structure every `--validation_steps`. Validation runs on the main process, with dropout disabled, and writes a CIF next to the adapter. It uses the checkpoint's full sampling schedule. +### Does the adapter learn anything? + +Splitting the fifty published structures into 40 for training and 10 held out, and training on the 40 with the +settings above, held-out flow-matching loss drops on every held-out structure: + +| | mean held-out loss | +| --- | --- | +| base model | 19.41 | +| after 500 LoRA steps | 17.28 | + +That is 11% lower, on 9 of 9 held-out structures — one of the ten is dropped by preparation for NaN coordinates. +Both numbers come from the same loaded model with the adapter toggled on and off, scored over identical times and +noise draws per structure (sixteen each), so the gap is the adapter rather than sampling spread. + +Training and held-out structures come from the same PLINDER subset, so this shows the adapter fits the distribution +it was trained on. It is a check that the recipe trains, not a claim about protein families it has not seen. + ## Inference The saved adapter loads into the pipeline: From c096196aab8e43ba5c0d4666c4b1fdf58b9d3082 Mon Sep 17 00:00:00 2001 From: kierandidi Date: Mon, 9 Nov 2026 15:14:37 +0000 Subject: [PATCH 07/11] Remove benchmark results from the example README --- examples/rfd4_proteina/README.md | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index baea2dd890b6..9b81d650c168 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -84,23 +84,6 @@ Use `--checkpointing_steps` and `--checkpoints_total_limit` to control checkpoin Pass `--validation_cif` to generate a structure every `--validation_steps`. Validation runs on the main process, with dropout disabled, and writes a CIF next to the adapter. It uses the checkpoint's full sampling schedule. -### Does the adapter learn anything? - -Splitting the fifty published structures into 40 for training and 10 held out, and training on the 40 with the -settings above, held-out flow-matching loss drops on every held-out structure: - -| | mean held-out loss | -| --- | --- | -| base model | 19.41 | -| after 500 LoRA steps | 17.28 | - -That is 11% lower, on 9 of 9 held-out structures — one of the ten is dropped by preparation for NaN coordinates. -Both numbers come from the same loaded model with the adapter toggled on and off, scored over identical times and -noise draws per structure (sixteen each), so the gap is the adapter rather than sampling spread. - -Training and held-out structures come from the same PLINDER subset, so this shows the adapter fits the distribution -it was trained on. It is a check that the recipe trains, not a claim about protein families it has not seen. - ## Inference The saved adapter loads into the pipeline: From d6edacae3ba2033fd437a3ee789aab40b2d2b2d5 Mon Sep 17 00:00:00 2001 From: kierandidi Date: Tue, 10 Nov 2026 15:25:53 +0000 Subject: [PATCH 08/11] Cyclic peptide check --- examples/rfd4_proteina/README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index 9b81d650c168..7a2964c3a5b1 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -84,6 +84,21 @@ Use `--checkpointing_steps` and `--checkpoints_total_limit` to control checkpoin Pass `--validation_cif` to generate a structure every `--validation_steps`. Validation runs on the main process, with dropout disabled, and writes a CIF next to the adapter. It uses the checkpoint's full sampling schedule. +### Cyclic peptide check + +Trained on 255 CPSea complexes for 1,000 steps on two H200s, starting from weights matched to the updated native +release. We tested the [PD-1 cyclic peptide inputs](https://github.com/RosettaCommons/RFD4-Proteina/pull/82) with the +same 16 seeds per chemistry at steps 0, 500 and 1,000. PD-1 was excluded from fine-tuning. + +![PD-1 contacts and training loss](https://raw.githubusercontent.com/kierandidi/diffusers/ec4c9150455cf160a68caae75e9bf968cd9edee8/pr-assets/contacts-and-loss.png) + +Median contacts (heavy-atom pairs within 5 Å) rose from 187.5→229.5, 150.5→262 and 160.5→220 for head-to-tail, +disulfide and isopeptide designs. Contacts per peptide atom also increased. Shading shows 95% seed-bootstrap intervals; +loss uses EMA α=0.02 over faint raw values. + +This is one target and one training run. Head-to-tail closure stayed at 16/16, but disulfide fell from 12/16 to 6/16 +and isopeptide from 5/16 to 1/16. More contacts alone don't establish better binding or overall design quality. + ## Inference The saved adapter loads into the pipeline: From 552328aa9b24ac2e5e499daeef893c21497e85ff Mon Sep 17 00:00:00 2001 From: Kieran Didi Date: Wed, 11 Nov 2026 14:11:07 +0000 Subject: [PATCH 09/11] Write a model card and support pushing to the Hub Every run now writes a model card next to the adapter, naming the base checkpoint and the structures trained on, with a snippet that generates from the result. `--push_to_hub`, `--hub_model_id` and `--hub_token` upload the adapter and that card at the end of training, as the other LoRA examples do. --- examples/rfd4_proteina/README.md | 5 + .../rfd4_proteina/train_rfd4_proteina_lora.py | 93 +++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index 7a2964c3a5b1..e598f1ebe7fc 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -84,6 +84,11 @@ Use `--checkpointing_steps` and `--checkpoints_total_limit` to control checkpoin Pass `--validation_cif` to generate a structure every `--validation_steps`. Validation runs on the main process, with dropout disabled, and writes a CIF next to the adapter. It uses the checkpoint's full sampling schedule. +A run writes a model card next to the adapter, naming the base checkpoint and the structures trained on, with a +snippet that generates from the result. Add `--push_to_hub` to upload the adapter and that card at the end of +training, `--hub_model_id` to name the repository (the output directory's name by default) and `--hub_token` to +authenticate as someone other than the logged-in user. Checkpoint directories and logs are left behind. + ### Cyclic peptide check Trained on 255 CPSea complexes for 1,000 steps on two H200s, starting from weights matched to the updated native diff --git a/examples/rfd4_proteina/train_rfd4_proteina_lora.py b/examples/rfd4_proteina/train_rfd4_proteina_lora.py index 4714f84e6cc1..982f15ebe0d3 100644 --- a/examples/rfd4_proteina/train_rfd4_proteina_lora.py +++ b/examples/rfd4_proteina/train_rfd4_proteina_lora.py @@ -30,6 +30,7 @@ from accelerate.logging import get_logger from accelerate.utils import ProjectConfiguration, set_seed from filelock import FileLock +from huggingface_hub import create_repo, upload_folder from peft import LoraConfig from peft.utils import get_peft_model_state_dict, set_peft_model_state_dict from torch.utils.data import DataLoader, Dataset @@ -40,6 +41,7 @@ from diffusers.optimization import get_scheduler from diffusers.training_utils import _collate_lora_metadata, cast_training_params from diffusers.utils import convert_unet_state_dict_to_peft +from diffusers.utils.hub_utils import load_or_create_model_card, populate_model_card logger = get_logger(__name__) @@ -150,6 +152,14 @@ def parse_args(): help="Annotated CIF generated from after every `validation_steps` to watch the adapter's effect.", ) parser.add_argument("--validation_steps", type=int, default=250, help="Steps between validation generations.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) return parser.parse_args() @@ -229,6 +239,68 @@ def __getitem__(self, index: int) -> dict[str, torch.Tensor]: return self._features(self.paths[index]) +def save_model_card(repo_id: str, base_model: str, instance_data_dir: str, repo_folder: str) -> None: + """Write a model card describing the adapter and how to generate with it.""" + model_description = f""" +# RFD4-Proteina LoRA - {repo_id} + +These are LoRA adapter weights for [{base_model}](https://huggingface.co/{base_model}), trained on structures from +`{Path(instance_data_dir).name}`. + +The denoiser was adapted with [the RFD4-Proteina LoRA example](https://github.com/huggingface/diffusers/blob/main/examples/rfd4_proteina/README.md); +the autoencoder is paired with the checkpoint and stays frozen. + +## Generating with the adapter + +Structures are read and written by +[`rfd4-proteina-prep`](https://huggingface.co/nvidia/RFD4-Proteina-prep), which is not part of Diffusers. + +```py +import torch +from diffusers import RFD4ProteinaPipeline +from rfd4_proteina_prep import prepare_cif, write_cif + +pipe = RFD4ProteinaPipeline.from_pretrained("{base_model}", dtype=torch.bfloat16).to("cuda") +pipe.load_lora_weights("{repo_id}") + +features, atom_array = prepare_cif("input.cif", seed=0, device=pipe.device) +named = ( + "padding_mask", + "is_in_context_conditioning", + "supports_local_latent", + "token_type", + "atom37_mask", + "coords_nm", +) +output = pipe( + **{{name: features[name] for name in named}}, + conditioning_features={{name: value for name, value in features.items() if name not in named}}, + generator=torch.Generator(device=pipe.device).manual_seed(0), +) +write_cif(output, features, atom_array, "design.cif") +``` + +For weighting, merging and fusing adapters, see the +[documentation on loading LoRAs](https://huggingface.co/docs/diffusers/main/en/using-diffusers/loading_adapters). + +## License + +Please adhere to the licensing terms as described in the model repository. +""" + model_card = load_or_create_model_card( + repo_id_or_path=repo_id, + from_training=True, + license="other", + base_model=base_model, + model_description=model_description, + ) + model_card = populate_model_card( + model_card, + tags=["rfd4-proteina", "protein-structure", "diffusers-training", "diffusers", "lora", "template:sd-lora"], + ) + model_card.save(os.path.join(repo_folder, "README.md")) + + def collate_fn(examples): """Use one structure per batch because token counts vary.""" return examples[0] @@ -294,8 +366,15 @@ def main(): if args.seed is not None: set_seed(args.seed) + repo_id = None if accelerator.is_main_process: os.makedirs(args.output_dir, exist_ok=True) + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, + exist_ok=True, + token=args.hub_token, + ).repo_id weight_dtype = {"fp16": torch.float16, "bf16": torch.bfloat16}.get(accelerator.mixed_precision, torch.float32) load_kwargs = {"revision": args.revision, "variant": args.variant, "cache_dir": args.cache_dir} @@ -534,6 +613,20 @@ def load_model_hook(models, input_dir): **_collate_lora_metadata({"transformer": trained}), ) logger.info(f"Saved the adapter to {args.output_dir}.") + save_model_card( + repo_id or args.hub_model_id or Path(args.output_dir).name, + base_model=args.pretrained_model_name_or_path, + instance_data_dir=args.instance_data_dir, + repo_folder=args.output_dir, + ) + if args.push_to_hub: + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["checkpoint-*", "logs/*"], + token=args.hub_token, + ) accelerator.end_training() From d0a695f7368f444e21a70a8b1fe91b635b200b3c Mon Sep 17 00:00:00 2001 From: kierandidi Date: Wed, 11 Nov 2026 14:23:35 +0000 Subject: [PATCH 10/11] Add Hub support to the RFD4 LoRA example --- examples/rfd4_proteina/README.md | 22 +- examples/rfd4_proteina/requirements.txt | 1 + .../rfd4_proteina/train_rfd4_proteina_lora.py | 228 +++++++++++------- 3 files changed, 156 insertions(+), 95 deletions(-) diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index e598f1ebe7fc..d403a3e7c25f 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -67,6 +67,23 @@ accelerate launch examples/rfd4_proteina/train_rfd4_proteina_lora.py \ --seed 0 ``` +To upload the final adapter and its model card, log in with `hf auth login` and add +`--push_to_hub --hub_model_id your-username/rfd4-proteina-lora`. Authentication also supports `HF_TOKEN` or `--hub_token` to override the logged-in account. +Only the final adapter and model card are uploaded. Every run writes a model card to `--output_dir`, even without upload. + +For a Hub dataset, replace `--instance_data_dir` with `--dataset_name your-username/annotated-structures`. +Use `--dataset_config_name`, `--dataset_split` (default `train`), `--dataset_revision` and `--structure_column` +(default `structure`) to select the data. The column must contain annotated `.cif`, `.cif.gz` or `.cif.zst` files +stored with `AtomicStructure`. Original file bytes are cached without decoding so conditioning annotations are preserved; +ordinary unannotated structures still need the preparation described above. + +`AtomicStructure` is currently in [Datasets PR #8895](https://github.com/huggingface/datasets/pull/8895). +Until it is released, `--dataset_name` requires this revision; local CIF training does not require Datasets: + +```bash +pip install git+https://github.com/huggingface/datasets.git@9ddf85d4fbd0aacba0b04911b12ee8527103c23a +``` + Training uses rank 32 / alpha 64 adapters on `to_q`, `to_k`, `to_v` and `to_out`, following the released recipe. The flow-matching loss scores the predicted clean backbone and local latents. It does not include the native trainer's auxiliary sequence and insertion losses. @@ -84,11 +101,6 @@ Use `--checkpointing_steps` and `--checkpoints_total_limit` to control checkpoin Pass `--validation_cif` to generate a structure every `--validation_steps`. Validation runs on the main process, with dropout disabled, and writes a CIF next to the adapter. It uses the checkpoint's full sampling schedule. -A run writes a model card next to the adapter, naming the base checkpoint and the structures trained on, with a -snippet that generates from the result. Add `--push_to_hub` to upload the adapter and that card at the end of -training, `--hub_model_id` to name the repository (the output directory's name by default) and `--hub_token` to -authenticate as someone other than the logged-in user. Checkpoint directories and logs are left behind. - ### Cyclic peptide check Trained on 255 CPSea complexes for 1,000 steps on two H200s, starting from weights matched to the updated native diff --git a/examples/rfd4_proteina/requirements.txt b/examples/rfd4_proteina/requirements.txt index 4c1463fe8956..cc3914b29d9e 100644 --- a/examples/rfd4_proteina/requirements.txt +++ b/examples/rfd4_proteina/requirements.txt @@ -3,3 +3,4 @@ peft>=0.14.0 safetensors>=0.4.5 torch>=2.4.0 tensorboard +Jinja2 diff --git a/examples/rfd4_proteina/train_rfd4_proteina_lora.py b/examples/rfd4_proteina/train_rfd4_proteina_lora.py index 982f15ebe0d3..adb77b5a3ef4 100644 --- a/examples/rfd4_proteina/train_rfd4_proteina_lora.py +++ b/examples/rfd4_proteina/train_rfd4_proteina_lora.py @@ -30,7 +30,7 @@ from accelerate.logging import get_logger from accelerate.utils import ProjectConfiguration, set_seed from filelock import FileLock -from huggingface_hub import create_repo, upload_folder +from huggingface_hub import ModelCard, ModelCardData, create_repo, upload_folder from peft import LoraConfig from peft.utils import get_peft_model_state_dict, set_peft_model_state_dict from torch.utils.data import DataLoader, Dataset @@ -41,7 +41,7 @@ from diffusers.optimization import get_scheduler from diffusers.training_utils import _collate_lora_metadata, cast_training_params from diffusers.utils import convert_unet_state_dict_to_peft -from diffusers.utils.hub_utils import load_or_create_model_card, populate_model_card +from diffusers.utils.hub_utils import MODEL_CARD_TEMPLATE_PATH, populate_model_card logger = get_logger(__name__) @@ -80,9 +80,23 @@ def parse_args(): parser.add_argument( "--instance_data_dir", type=str, - required=True, + default=None, help="Directory searched recursively for the annotated CIF files to fine-tune on.", ) + parser.add_argument("--dataset_name", type=str, default=None, help="Dataset id or local dataset path.") + parser.add_argument("--dataset_config_name", type=str, default=None, help="Dataset configuration.") + parser.add_argument("--dataset_revision", type=str, default=None, help="Dataset revision to load.") + parser.add_argument("--dataset_split", type=str, default="train", help="Dataset split to train on.") + parser.add_argument( + "--structure_column", type=str, default="structure", help="AtomicStructure column of annotated CIFs." + ) + parser.add_argument( + "--push_to_hub", action="store_true", help="Upload the final adapter and model card to the Hub." + ) + parser.add_argument( + "--hub_token", type=str, default=None, help="Token to use instead of the logged-in Hub account." + ) + parser.add_argument("--hub_model_id", type=str, default=None, help="Hub repository for the trained adapter.") parser.add_argument( "--features_cache_dir", type=str, @@ -152,15 +166,116 @@ def parse_args(): help="Annotated CIF generated from after every `validation_steps` to watch the adapter's effect.", ) parser.add_argument("--validation_steps", type=int, default=250, help="Steps between validation generations.") - parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") - parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") - parser.add_argument( - "--hub_model_id", - type=str, - default=None, - help="The name of the repository to keep in sync with the local `output_dir`.", + args = parser.parse_args() + if (args.instance_data_dir is None) == (args.dataset_name is None): + parser.error("Specify exactly one of --instance_data_dir or --dataset_name.") + return args + + +def prepare_dataset(args): + """Keep the original CIF bytes so conditioning annotations survive loading.""" + try: + from datasets import AtomicStructure, DownloadManager, config, load_dataset + except ImportError as error: + raise ImportError( + "--dataset_name requires Datasets with AtomicStructure support; see the example README." + ) from error + + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + split=args.dataset_split, + revision=args.dataset_revision, + cache_dir=args.cache_dir, + ) + if args.structure_column not in dataset.column_names: + raise ValueError(f"Dataset has no {args.structure_column!r} column. Choose it with --structure_column.") + dataset = dataset.select_columns([args.structure_column]).cast_column( + args.structure_column, AtomicStructure(decode=False) + ) + root = Path(args.cache_dir or config.HF_DATASETS_CACHE) / "rfd4_proteina" / dataset._fingerprint + root.mkdir(parents=True, exist_ok=True) + downloader = DownloadManager() + with FileLock(str(root) + ".lock"): + for index, row in enumerate(dataset): + source = row[args.structure_column] + suffix = next( + (ext for ext in (".cif", ".cif.gz", ".cif.zst") if (source["path"] or "structure.cif").endswith(ext)), + None, + ) + if suffix is None: + raise ValueError( + f"Row {index}: expected an annotated .cif, .cif.gz or .cif.zst, got {source['path']!r}." + ) + path = root / f"{index}{suffix}" + if path.is_file(): + continue + content = source["bytes"] + if content is None: + content = Path(downloader.download(source["path"])).read_bytes() + temporary = path.with_name(path.name + ".tmp") + temporary.write_bytes(content) + os.replace(temporary, path) + return root + + +def save_model_card(args, adapter_id, num_structures, global_step): + dataset_description = ( + f"{args.dataset_name} (config: {args.dataset_config_name or 'default'}, split: {args.dataset_split}, " + f"revision: {args.dataset_revision or 'default'})" + if args.dataset_name + else f"Local annotated CIFs ({Path(args.instance_data_dir).name})" + ) + model_description = f""" +# RFD4-Proteina LoRA + +Adapter for `{args.pretrained_model_name_or_path}` (revision: `{args.revision or "default"}`). +The autoencoder and base denoiser were frozen. + +## Training + +- Dataset: {dataset_description} +- Structures after filtering: {num_structures} +- Optimizer steps: {global_step}; learning rate: {args.learning_rate} +- Rank: {args.rank}; alpha: {args.lora_alpha}; dropout: {args.lora_dropout} +- Target modules: {", ".join(args.target_modules)} +- Seed: {args.seed}; precision: {args.mixed_precision or "Accelerate configuration"} + +Training uses backbone and local-latent flow-matching losses, without the native auxiliary sequence and insertion losses. + +## Inference + +```python +import torch +from diffusers import RFD4ProteinaPipeline +from rfd4_proteina_prep import prepare_cif, write_cif + +pipe = RFD4ProteinaPipeline.from_pretrained( + {args.pretrained_model_name_or_path!r}, revision={args.revision!r}, variant={args.variant!r}, dtype=torch.bfloat16 +).to("cuda") +pipe.load_lora_weights({adapter_id!r}) +features, atom_array = prepare_cif("input.cif", seed=0, device=pipe.device) +named = {PIPELINE_INPUTS!r} +inputs = {{name: features[name] for name in named}} +conditioning = {{name: value for name, value in features.items() if name not in named}} +output = pipe( + **inputs, + conditioning_features=conditioning, + generator=torch.Generator(device=pipe.device).manual_seed(0), +) +write_cif(output, features, atom_array, "generated.cif") +``` + +See the [training example](https://github.com/huggingface/diffusers/tree/main/examples/rfd4_proteina#inference) +for CIF preparation and generation. Follow the base model and dataset licenses when using or sharing this adapter. +""" + card = ModelCard.from_template( + ModelCardData(library_name="diffusers", base_model=args.pretrained_model_name_or_path, inference=False), + template_path=MODEL_CARD_TEMPLATE_PATH, + model_description=model_description, ) - return parser.parse_args() + card = populate_model_card(card, tags=["diffusers-training", "lora", "rfd4-proteina"]) + card.save(Path(args.output_dir) / "README.md") class PreparedStructureDataset(Dataset): @@ -239,68 +354,6 @@ def __getitem__(self, index: int) -> dict[str, torch.Tensor]: return self._features(self.paths[index]) -def save_model_card(repo_id: str, base_model: str, instance_data_dir: str, repo_folder: str) -> None: - """Write a model card describing the adapter and how to generate with it.""" - model_description = f""" -# RFD4-Proteina LoRA - {repo_id} - -These are LoRA adapter weights for [{base_model}](https://huggingface.co/{base_model}), trained on structures from -`{Path(instance_data_dir).name}`. - -The denoiser was adapted with [the RFD4-Proteina LoRA example](https://github.com/huggingface/diffusers/blob/main/examples/rfd4_proteina/README.md); -the autoencoder is paired with the checkpoint and stays frozen. - -## Generating with the adapter - -Structures are read and written by -[`rfd4-proteina-prep`](https://huggingface.co/nvidia/RFD4-Proteina-prep), which is not part of Diffusers. - -```py -import torch -from diffusers import RFD4ProteinaPipeline -from rfd4_proteina_prep import prepare_cif, write_cif - -pipe = RFD4ProteinaPipeline.from_pretrained("{base_model}", dtype=torch.bfloat16).to("cuda") -pipe.load_lora_weights("{repo_id}") - -features, atom_array = prepare_cif("input.cif", seed=0, device=pipe.device) -named = ( - "padding_mask", - "is_in_context_conditioning", - "supports_local_latent", - "token_type", - "atom37_mask", - "coords_nm", -) -output = pipe( - **{{name: features[name] for name in named}}, - conditioning_features={{name: value for name, value in features.items() if name not in named}}, - generator=torch.Generator(device=pipe.device).manual_seed(0), -) -write_cif(output, features, atom_array, "design.cif") -``` - -For weighting, merging and fusing adapters, see the -[documentation on loading LoRAs](https://huggingface.co/docs/diffusers/main/en/using-diffusers/loading_adapters). - -## License - -Please adhere to the licensing terms as described in the model repository. -""" - model_card = load_or_create_model_card( - repo_id_or_path=repo_id, - from_training=True, - license="other", - base_model=base_model, - model_description=model_description, - ) - model_card = populate_model_card( - model_card, - tags=["rfd4-proteina", "protein-structure", "diffusers-training", "diffusers", "lora", "template:sd-lora"], - ) - model_card.save(os.path.join(repo_folder, "README.md")) - - def collate_fn(examples): """Use one structure per batch because token counts vary.""" return examples[0] @@ -366,14 +419,11 @@ def main(): if args.seed is not None: set_seed(args.seed) - repo_id = None if accelerator.is_main_process: os.makedirs(args.output_dir, exist_ok=True) if args.push_to_hub: repo_id = create_repo( - repo_id=args.hub_model_id or Path(args.output_dir).name, - exist_ok=True, - token=args.hub_token, + args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token ).repo_id weight_dtype = {"fp16": torch.float16, "bf16": torch.bfloat16}.get(accelerator.mixed_precision, torch.float32) @@ -467,8 +517,9 @@ def load_model_hook(models, input_dir): eps=args.adam_epsilon, ) with accelerator.main_process_first(): + data_dir = prepare_dataset(args) if args.dataset_name else args.instance_data_dir dataset = PreparedStructureDataset( - args.instance_data_dir, seed=args.prep_seed, max_tokens=args.max_tokens, cache_dir=args.features_cache_dir + data_dir, seed=args.prep_seed, max_tokens=args.max_tokens, cache_dir=args.features_cache_dir ) dataloader = DataLoader( dataset, @@ -489,7 +540,11 @@ def load_model_hook(models, input_dir): if accelerator.is_main_process: # Trackers take scalars, so the list of adapted modules is logged as one string. accelerator.init_trackers( - "rfd4-proteina-lora", config={**vars(args), "target_modules": ",".join(args.target_modules)} + "rfd4-proteina-lora", + config={ + **{key: value for key, value in vars(args).items() if key != "hub_token"}, + "target_modules": ",".join(args.target_modules), + }, ) logger.info("***** Running training *****") @@ -600,9 +655,7 @@ def load_model_hook(models, input_dir): if global_step >= args.max_train_steps: break if global_step == 0: - raise ValueError( - f"No structure under {args.instance_data_dir} could be trained on; see the warnings above." - ) + raise ValueError(f"No structure under {data_dir} could be trained on; see the warnings above.") accelerator.wait_for_everyone() if accelerator.is_main_process: @@ -612,21 +665,16 @@ def load_model_hook(models, input_dir): transformer_lora_layers=get_peft_model_state_dict(trained), **_collate_lora_metadata({"transformer": trained}), ) - logger.info(f"Saved the adapter to {args.output_dir}.") - save_model_card( - repo_id or args.hub_model_id or Path(args.output_dir).name, - base_model=args.pretrained_model_name_or_path, - instance_data_dir=args.instance_data_dir, - repo_folder=args.output_dir, - ) + save_model_card(args, repo_id if args.push_to_hub else args.output_dir, len(dataset), global_step) if args.push_to_hub: upload_folder( repo_id=repo_id, folder_path=args.output_dir, + allow_patterns=["README.md", "pytorch_lora_weights.safetensors"], commit_message="End of training", - ignore_patterns=["checkpoint-*", "logs/*"], token=args.hub_token, ) + logger.info(f"Saved the adapter to {args.output_dir}.") accelerator.end_training() From 82d34b52304bcbd857134cac5bfb7aeb7f94b4fe Mon Sep 17 00:00:00 2001 From: kierandidi Date: Wed, 11 Nov 2026 14:32:16 +0000 Subject: [PATCH 11/11] Align RFD4 LoRA example conventions --- examples/rfd4_proteina/README.md | 3 ++- examples/rfd4_proteina/train_rfd4_proteina_lora.py | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/rfd4_proteina/README.md b/examples/rfd4_proteina/README.md index d403a3e7c25f..5fba260e590c 100644 --- a/examples/rfd4_proteina/README.md +++ b/examples/rfd4_proteina/README.md @@ -1,7 +1,7 @@ # LoRA fine-tuning for RFD4-Proteina This example fine-tunes [RFD4-Proteína](https://huggingface.co/docs/diffusers/main/en/api/pipelines/rfd4_proteina) -with LoRA on a folder of annotated CIF files. The autoencoder stays frozen; only the denoiser adapters train. +with LoRA on annotated CIFs from a local folder or a Hub dataset. The autoencoder stays frozen; only the denoiser adapters train. Saved adapters load with `RFD4ProteinaPipeline.load_lora_weights`. ## Setup @@ -69,6 +69,7 @@ accelerate launch examples/rfd4_proteina/train_rfd4_proteina_lora.py \ To upload the final adapter and its model card, log in with `hf auth login` and add `--push_to_hub --hub_model_id your-username/rfd4-proteina-lora`. Authentication also supports `HF_TOKEN` or `--hub_token` to override the logged-in account. +For W&B logging, use `hf auth login` or `HF_TOKEN` rather than passing `--hub_token`. Only the final adapter and model card are uploaded. Every run writes a model card to `--output_dir`, even without upload. For a Hub dataset, replace `--instance_data_dir` with `--dataset_name your-username/annotated-structures`. diff --git a/examples/rfd4_proteina/train_rfd4_proteina_lora.py b/examples/rfd4_proteina/train_rfd4_proteina_lora.py index adb77b5a3ef4..1eeebf1bc1d7 100644 --- a/examples/rfd4_proteina/train_rfd4_proteina_lora.py +++ b/examples/rfd4_proteina/train_rfd4_proteina_lora.py @@ -40,10 +40,12 @@ from diffusers import RFD4ProteinaAutoencoder, RFD4ProteinaPipeline, RFD4ProteinaScheduler, RFD4ProteinaTransformer from diffusers.optimization import get_scheduler from diffusers.training_utils import _collate_lora_metadata, cast_training_params -from diffusers.utils import convert_unet_state_dict_to_peft +from diffusers.utils import check_min_version, convert_unet_state_dict_to_peft from diffusers.utils.hub_utils import MODEL_CARD_TEMPLATE_PATH, populate_model_card +check_min_version("0.42.0.dev0") + logger = get_logger(__name__) # Other prepared features are passed as conditioning. @@ -397,6 +399,8 @@ def flow_matching_loss(prediction, state, times, clean, supervision_masks) -> to def main(): args = parse_args() + if args.hub_token is not None and args.report_to in ("wandb", "all"): + raise ValueError("Use `hf auth login` or HF_TOKEN instead of --hub_token when reporting to W&B.") logging_dir = Path(args.output_dir, args.logging_dir) accelerator = Accelerator( gradient_accumulation_steps=args.gradient_accumulation_steps,