test vLLM 0.30 support

.github/workflows/ci.yaml

@@ -77,8 +77,16 @@ jobs:
             -v -s --tb=short -x \
             --cov=granite_switch --cov-report=xml
 
+      # COVERAGE REPORTING MUST NOT GATE A PR. `fail_ci_if_error: false` already
+      # declared that intent, but v4 can fail the step *before* it consults that
+      # flag -- it exits nonzero on a token/rate-limit problem of its own, which
+      # is what turned a fully green CPU suite red (the "Run CPU tests" step
+      # passed on both 3.11 and 3.12; only this step failed). `continue-on-error`
+      # enforces the intent at the workflow level, where the action cannot
+      # override it. v5 is the supported line; v4 is deprecated.
       - name: Upload coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
+        continue-on-error: true
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: coverage.xml

.github/workflows/gpu-tests.yaml

@@ -105,19 +105,29 @@ jobs:
     # The suite is in the job name because it cannot be read from env.LEG here --
     # this is evaluated before any step runs.
     name: GPU Tests (${{ matrix.label }} · ${{ inputs.suite || 'full' }})
-    # granite-switch supports two mutually-exclusive vLLM lines and both must be
-    # tested. They cannot share a venv (pyproject declares vllm26/vllm27 as
-    # conflicting groups), so each gets its own cluster pod. Only the dev* groups
-    # used because the bare vllm26/vllm27 groups omit pytest. The label encodes
-    # the vLLM minor the leg installs (the GPU harness asserts it matches).
+    # granite-switch supports ONE tree across vLLM 0.26-0.30, and these two legs are
+    # the ENDS of that range. They cannot share a venv (pyproject declares
+    # vllm26/vllm30 as conflicting groups), so each gets its own cluster pod. Only
+    # the dev* groups are used because the bare vllm26/vllm30 groups omit pytest.
+    # The label encodes the vLLM minor the leg installs (the GPU harness asserts it
+    # matches).
+    #
+    # WHY THE ENDS AND NOT A MIDDLE LINE. The breaks this range actually contains
+    # are all at a boundary: APIs deleted at 0.29 (AttentionBackend.get_kv_cache_shape,
+    # BaseMultiModalProcessor._call_hf_processor) only fail at the ceiling, and an API
+    # that does not exist yet (ForwardContext.is_padding, added at 0.27) only fails at
+    # the floor. A middle leg is bracketed by two tested lines and would have caught
+    # neither. 0.27/0.28/0.29 were verified green out of band by a five-leg matrix,
+    # every suite, so this is a sampling decision about per-run cost -- not a claim
+    # that the middle is untested.
     strategy:
-      fail-fast: false   # a vllm26 failure must not hide the vllm27 result
+      fail-fast: false   # a vllm26 failure must not hide the vllm30 result
       matrix:
         include:
           - label: vllm26
             dep_group: dev
-          - label: vllm27
-            dep_group: dev-vllm27
+          - label: vllm30
+            dep_group: dev-vllm30
     runs-on: [self-hosted, gpu]
 
     # MUST be set explicitly. GitHub's default is 360 minutes, which is exactly the

README.md

@@ -47,16 +47,18 @@ Other install options depending on your use case:
 ```bash
 pip install "granite-switch[compose]"   # Compose modular models
 pip install "granite-switch[hf]"        # HuggingFace inference
-pip install "granite-switch[vllm27]"    # newer vLLM line (0.27.x)
 pip install "granite-switch[dev]"       # Everything
 ```
 
 Requires Python 3.11+ and PyTorch 2.11+.
 
-> **vLLM version note:** This project requires `transformers>=5.16`, so it pins vLLM to the
-> `0.26.x` line (the earliest vLLM whose model code handles the transformers-5.13+
-> `full_attention` layer-type rename) via the default `[vllm]` extra. `[vllm27]` selects the
-> newer `0.27.x` line. Both require PyTorch 2.11+ (CUDA 13+).
+> **vLLM version note:** `[vllm]` accepts any line from `0.26.x` to `0.30.x`. The floor is
+> `transformers>=5.16`: `0.26` is the earliest vLLM whose model code handles the
+> transformers-5.13+ `full_attention` layer-type rename. One tree serves all five lines —
+> where their APIs differ, the backend checks for the capability rather than the version. All
+> of them require PyTorch 2.11+ (CUDA 13+). To pin a single line for development, use the
+> `vllm26` / `vllm30` dependency *groups* — the two ends of the range, which is also what CI
+> tests on every GPU run.
 
 ### Compose a Model
 

docs/AUDIO.md

@@ -27,7 +27,7 @@ with an `--asr-model` your installed transformers supports (for example
 
 ```bash
 # Serving an audio-enabled checkpoint
-uv sync --extra vllm --extra audio     # or --extra vllm27 --extra audio
+uv sync --extra vllm --extra audio     # any vLLM 0.26.x - 0.30.x
 
 # Development / running the test suite (the dev groups include audio already)
 uv sync --group dev                    # vLLM 0.26.x

pyproject.toml

@@ -20,8 +20,10 @@ Documentation = "https://github.com/generative-computing/granite-switch/tree/mai
 
 [project.optional-dependencies]
 hf = ["accelerate>=0.20.0"]
-vllm = ["vllm>=0.26.0,<0.27.0"]
-vllm27 = ["vllm>=0.27.0,<0.28.0"]
+# Every line from 0.26 through 0.30 is supported by one tree; the backend uses
+# capability checks (hasattr) rather than version comparisons where the two
+# differ. Pin a specific line with the per-line dependency groups below.
+vllm = ["vllm>=0.26.0,<0.31.0"]
 compose = ["huggingface_hub", "pyyaml", "tqdm", "safetensors"]
 build = ["huggingface_hub", "pyyaml", "tqdm", "safetensors"]  # Backward compatibility alias for compose
 # Audio (ASR) decode + resample. Reuse vLLM's own audio deps (unversioned, so it
@@ -56,33 +58,63 @@ markers = [
 ]
 
 [dependency-groups]
+# vllm26 AND vllm30 ARE THE TWO CI LEGS -- the ends of the supported range, which
+# is why these two and not two others. The `vllm` extra above claims 0.26-0.30,
+# and the two bounds are where a break actually lands: a deleted API shows up at
+# the ceiling and an API that does not exist yet shows up at the floor.
+# 0.27/0.28/0.29 sit strictly between two tested lines, so they add install time
+# without adding a failure mode the ends would miss. All five lines were verified
+# together out of band (five-leg GPU matrix, every suite green); these two are
+# what CI re-checks on every run.
 vllm26     = ["vllm>=0.26.0,<0.27.0"]
+vllm30     = ["vllm>=0.30.0,<0.31.0"]
+# vllm27 IS NOT A CI LEG -- it exists so the GPU workflow stays dispatchable
+# during this transition. `workflow_dispatch` always executes the workflow file
+# from the DEFAULT BRANCH, so until the 0.26/0.30 matrix lands on main, a
+# `/gpu-test` run uses main's matrix (legs vllm26 -> dev and vllm27 ->
+# dev-vllm27) while checking out the PR's commit. Deleting dev-vllm27 here made
+# that leg unresolvable against this tree -- `uv sync --group dev-vllm27` has no
+# such group -- and the leg died before pytest started. Keeping the group costs
+# nothing and is not a fiction: the `vllm` extra claims 0.26-0.30, so 0.27 is a
+# supported line that simply isn't worth its own per-run leg. Removable once the
+# matrix rename is on main.
 vllm27     = ["vllm>=0.27.0,<0.28.0"]
 # `audio` is included so the audio tests can actually run: the ASR path needs
 # vLLM's audio deps (av/soundfile/resampy) at runtime, and no group pulled them
 # in before (integration tests failed with ModuleNotFoundError on a synced pod).
 dev        = ["pytest", "pytest-cov", { include-group = "vllm26" }, "granite-switch[hf,compose,audio]"]
 dev-vllm27 = ["pytest", "pytest-cov", { include-group = "vllm27" }, "granite-switch[hf,compose,audio]"]
+dev-vllm30 = ["pytest", "pytest-cov", { include-group = "vllm30" }, "granite-switch[hf,compose,audio]"]
 test       = ["pytest", "pytest-cov", "bitsandbytes", "optimum-quanto", { include-group = "dev" }]
 
 [tool.uv]
 default-groups = ["vllm26"]
+# Only the per-line GROUPS are mutually exclusive now. The `vllm` extra spans
+# 0.26-0.30, so it overlaps every group and no longer conflicts with any of them --
+# which is what made `uv sync` unsatisfiable while the extra said 0.30-only and the
+# default group said 0.26.
+# Every pair that would pull two different vLLM lines into one venv. Three lines
+# are declared (0.26 / 0.27 / 0.30) with a bare and a dev- flavour each, so the
+# 26-side {vllm26, dev}, the 27-side {vllm27, dev-vllm27} and the 30-side
+# {vllm30, dev-vllm30} give 3 x (2 x 2) = 12 cross-line pairs. `test` is absent
+# deliberately: it reaches vllm26 through `include-group = "dev"` and uv derives
+# that transitively -- main's two-line version listed it nowhere either.
 conflicts = [
-    # group-vs-group
+    # 0.26 vs 0.27
     [{ group = "vllm26" },     { group = "vllm27" }],
+    [{ group = "vllm26" },     { group = "dev-vllm27" }],
     [{ group = "dev" },        { group = "vllm27" }],
     [{ group = "dev" },        { group = "dev-vllm27" }],
-    [{ group = "dev-vllm27" }, { group = "vllm26" }],
-    # group-vs-extra
-    [{ group = "vllm26" },     { extra = "vllm27" }],
-    [{ group = "vllm27" },     { extra = "vllm" }],
-    [{ group = "vllm27" },     { extra = "tutorials" }],
-    [{ group = "dev" },        { extra = "vllm27" }],
-    [{ group = "dev-vllm27" }, { extra = "vllm" }],
-    [{ group = "dev-vllm27" }, { extra = "tutorials" }],
-    # extra-vs-extra
-    [{ extra = "vllm" },       { extra = "vllm27" }],
-    [{ extra = "tutorials" },  { extra = "vllm27" }],
+    # 0.26 vs 0.30
+    [{ group = "vllm26" },     { group = "vllm30" }],
+    [{ group = "vllm26" },     { group = "dev-vllm30" }],
+    [{ group = "dev" },        { group = "vllm30" }],
+    [{ group = "dev" },        { group = "dev-vllm30" }],
+    # 0.27 vs 0.30
+    [{ group = "vllm27" },     { group = "vllm30" }],
+    [{ group = "vllm27" },     { group = "dev-vllm30" }],
+    [{ group = "dev-vllm27" }, { group = "vllm30" }],
+    [{ group = "dev-vllm27" }, { group = "dev-vllm30" }],
 ]
 
 [tool.coverage.run]

src/granite_switch/vllm/audio/processor.py

@@ -1,10 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 """vLLM multimodal processor for the audio cascade.
 
-``_call_hf_processor`` runs ASR and tokenizes the transcript;
-``_get_prompt_updates`` then replaces the ``<|audio|>`` marker with the real
-transcript token ids via ``PromptReplacement``, so the scheduler sizes KV for the
-runtime-determined length rather than a fixed audio window.
+ASR runs in the HF-processor hook (``_call_hf_processor`` on vLLM 0.26 - 0.28,
+``_apply_hf_processor_main`` on 0.29+, which deleted the former) and tokenizes the
+transcript; ``_get_prompt_updates`` then replaces the ``<|audio|>`` marker with the
+real transcript token ids via ``PromptReplacement``, so the scheduler sizes KV for
+the runtime-determined length rather than a fixed audio window.
 
 Modeled on vLLM 0.19.1's ``ultravox.py``. Audio is answered by the base model —
 no adapter control tokens are placed, so the switch is not involved.
@@ -338,19 +339,17 @@ def _reject_reserved_ids(self, ids: Sequence[int]) -> None:
             f"tokens must not originate from audio content."
         )
 
-    def _call_hf_processor(
+    def _audio_features(
         self,
-        prompt: str,
-        mm_data: Mapping[str, object],
+        audios: Sequence[object],
         mm_kwargs: Mapping[str, object],
-        tok_kwargs: Mapping[str, object],
-    ) -> BatchFeature:
-        tokenizer = self.info.get_tokenizer()
-        audios = mm_data.get("audios", []) or []
+    ) -> dict[str, torch.Tensor]:
+        """Transcribe every clip into the out-of-band audio tensors.
 
+        Empty when there is no audio, so the caller can splat it unconditionally.
+        """
         if not audios:
-            input_ids = tokenizer.encode(prompt, add_special_tokens=False)
-            return BatchFeature(dict(input_ids=[input_ids]), tensor_type="pt")
+            return {}
 
         # Resolved once, then applied to every audio item in this request.
         generate_kwargs = resolve_generate_kwargs(
@@ -359,19 +358,72 @@ def _call_hf_processor(
             DEFAULT_ALLOWED_REQUEST_GENERATE_KEYS,
         )
 
-        input_ids = tokenizer.encode(prompt, add_special_tokens=False)
-
         # Concatenated flat, with per-item sizes to split them back.
         per_item_ids = [self._transcribe(a, generate_kwargs) for a in audios]
         sizes = [len(ids) for ids in per_item_ids]
         flat_ids = [tid for ids in per_item_ids for tid in ids]
 
+        return dict(
+            audio_token_ids=torch.tensor(flat_ids, dtype=torch.long),
+            audio_num_tokens=torch.tensor(sizes, dtype=torch.long),
+        )
+
+    def _call_hf_processor(
+        self,
+        prompt: str,
+        mm_data: Mapping[str, object],
+        mm_kwargs: Mapping[str, object],
+        tok_kwargs: Mapping[str, object],
+    ) -> BatchFeature:
+        """vLLM 0.26 - 0.28 only; the base class dropped this hook at 0.29.
+
+        Here the processor owns both halves of the job, so it returns the
+        tokenized prompt alongside the audio tensors. See
+        ``_apply_hf_processor_main`` for the 0.29+ split.
+        """
+        tokenizer = self.info.get_tokenizer()
+        input_ids = tokenizer.encode(prompt, add_special_tokens=False)
+        features = self._audio_features(mm_data.get("audios", []) or [], mm_kwargs)
         return BatchFeature(
-            dict(
-                input_ids=[input_ids],
-                audio_token_ids=torch.tensor(flat_ids, dtype=torch.long),
-                audio_num_tokens=torch.tensor(sizes, dtype=torch.long),
-            ),
+            dict(input_ids=[input_ids], **features),
+            tensor_type="pt",
+        )
+
+    def _apply_hf_processor_main(self, **kwargs):
+        """Produce the audio tensors on whichever hook this vLLM actually calls.
+
+        vLLM 0.29 deleted ``BaseMultiModalProcessor._call_hf_processor`` (and
+        ``_apply_hf_processor_text_mm``, which was its only caller). Text and
+        multi-modal data are now processed independently: ``apply()`` tokenizes the
+        prompt itself via ``_postprocess_prompt`` and ``_apply_hf_processor_main``
+        was narrowed to ``(mm_items, hf_processor_mm_kwargs) -> BatchFeature``,
+        returning the MM tensors ONLY. Left alone, our ``_call_hf_processor`` above
+        is simply never called on 0.29+ and no transcript is ever produced.
+
+        The two signatures are mutually exclusive, so this dispatches on whether the
+        base class still has ``_call_hf_processor``: that is the hook whose removal
+        is the actual problem, and it was removed in the same commit that narrowed
+        this signature. Both vLLM call sites pass keywords only (0.28 adds
+        ``prompt`` / ``tokenization_kwargs`` / ``enable_hf_prompt_update``), hence
+        ``**kwargs`` rather than a signature that could only match one line.
+        """
+        if hasattr(super(), "_call_hf_processor"):  # 0.26 - 0.28
+            return super()._apply_hf_processor_main(**kwargs)
+
+        mm_items = kwargs["mm_items"]
+        hf_processor_mm_kwargs = kwargs["hf_processor_mm_kwargs"]
+
+        valid_mm_items = mm_items.select(
+            {k for k, c in mm_items.get_all_counts().items() if c > 0}
+        )
+        processor_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
+        features = self._audio_features(
+            processor_data.get("audios", []) or [],
+            hf_processor_mm_kwargs,
+        )
+        # No input_ids: on 0.29+ the prompt is vLLM's business, not ours.
+        return BatchFeature(
+            dict(**features, **passthrough_data),
             tensor_type="pt",
         )
 
@@ -397,6 +449,11 @@ def _hf_processor_applies_updates(
         Only the uncached path consults this hook; the cached path already
         hardcodes False, which is why audio works with the default
         ``mm_processor_cache_gb=4`` and breaks under ``--mm-processor-cache-gb 0``.
+
+        Required on vLLM 0.26 - 0.28 and inert on 0.29+, where the base class
+        deleted the hook along with the ``is_update_applied`` branch it fed:
+        ``_apply_prompt_updates`` now always runs, which is exactly the False
+        behaviour asked for here. Kept because one tree has to serve both.
         """
         return False
 
@@ -428,10 +485,31 @@ def replacement(item_idx: int):
             e = int(starts[item_idx + 1])
             return [int(t) for t in all_ids[s:e]]
 
+        # TARGET IS A TOKEN-ID LIST, NOT THE MARKER STRING.  vLLM narrowed the
+        # accepted target type at 0.29:
+        #
+        #     0.26 - 0.28   UpdateTarget = PromptSeq | PromptIndex   # str allowed
+        #     0.29 - 0.30   UpdateTarget = list[int] | PromptIndex   # str dropped
+        #
+        # A ``list[int]`` is legal on every one of those lines, so this needs no
+        # version branch -- only ``str`` stopped being legal.
+        #
+        # Passing the string on 0.29+ does not fail loudly where it is given.
+        # Token matching simply finds nothing, which sends vLLM into
+        # ``_apply_prompt_updates_via_text`` -- a fallback that only exists from
+        # 0.29 -- and that calls ``tokenizer.decode(target)`` on the raw string.
+        # The resulting error surfaces far away and looks like a tokenizer bug:
+        # ``TypeError: Can't extract `str` to `Vec``` from the Rust tokenizer,
+        # killing EngineCore during init rather than rejecting the request.
+        #
+        # ``_marker_id()`` cannot answer -1 here: ``_validate_marker_count``
+        # already requires one marker per audio item, and ``_count_markers``
+        # reports 0 for an unregistered marker, so a prompt that reached this
+        # point with audio items has a registered marker.
         return [
             PromptReplacement(
                 modality="audio",
-                target=AUDIO_MARKER,
+                target=[self._marker_id()],
                 replacement=replacement,
             )
         ]