From 88286d0dde8135a08662bc62ace7118c99008906 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 29 Oct 2026 14:22:50 +0100 Subject: [PATCH 1/8] Add U2NetP foreground segmentation model presets --- .cspell-wordlist.txt | 1 + .../app/segmentation/index.tsx | 16 +++++++- .../06-semantic-segmentation.md | 13 +++--- .../react-native-executorch/src/models.ts | 40 +++++++++++++++++++ 4 files changed, 63 insertions(+), 7 deletions(-) diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 56e970131f..e8a0412c75 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -447,3 +447,4 @@ grayscale Premul eurollm doclayout +netp diff --git a/apps/computer-vision/app/segmentation/index.tsx b/apps/computer-vision/app/segmentation/index.tsx index 8222bfa60a..57205b766f 100644 --- a/apps/computer-vision/app/segmentation/index.tsx +++ b/apps/computer-vision/app/segmentation/index.tsx @@ -1,5 +1,5 @@ import React, { useState, useRef } from 'react'; -import { View, Text, ScrollView } from 'react-native'; +import { View, Text, ScrollView, Platform } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { commonStyles, theme } from '../../theme'; import { @@ -23,6 +23,20 @@ const SEGMENTATION_OPTIONS: ModelOption[] = [ label: 'Selfie Seg (FP32)', value: models.semanticSegmentation.SELFIE_SEGMENTATION.XNNPACK_FP32, }, + { + label: 'U²-Netp (XNNPACK FP32)', + value: models.semanticSegmentation.U2NETP.XNNPACK_FP32, + }, + { + label: 'U²-Netp (CoreML FP16)', + value: models.semanticSegmentation.U2NETP.COREML_FP16, + disabled: Platform.OS !== 'ios' || Number.parseInt(String(Platform.Version), 10) < 18, + }, + { + label: 'U²-Netp (CoreML FP32)', + value: models.semanticSegmentation.U2NETP.COREML_FP32, + disabled: Platform.OS !== 'ios' || Number.parseInt(String(Platform.Version), 10) < 18, + }, { label: 'LRASPP MobileNet V3 (INT8)', value: models.semanticSegmentation.LRASPP_MOBILENET_V3_LARGE.XNNPACK_INT8, diff --git a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md index e8f92ccb3c..e5e85aeb5d 100644 --- a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md +++ b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md @@ -165,12 +165,13 @@ See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading. The library provides ready-to-use segmentation models from the [Software Mansion HuggingFace Semantic Segmentation Collection](https://huggingface.co/collections/software-mansion/semantic-segmentation), accessible via [`models.semanticSegmentation`](../../06-api-reference/variables/models.md#semanticsegmentation): -| Model Family | Variants | Classes / Labels | Size Range | Supported Backends | Notes | -| :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------- | :----------------- | :----------------------------- | :----------------------------------------------------------------------- | -| **Selfie Segmentation** | [`Portrait`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation), [`Landscape`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation_landscape) | Person / Background | 0.5 MB – 0.6 MB | XNNPACK (CPU), Core ML (Apple) | Real-time front-camera portrait background replacement and blur effects. | -| **LRASPP MobileNetV3** | [See](../../06-api-reference/variables/models.md#semanticsegmentationlraspp_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 3.4 MB – 12.3 MB | XNNPACK (CPU), Core ML (Apple) | Lightweight multi-class scene segmentation with low CPU overhead. | -| **DeepLabV3** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet101), [`MobileNetV3`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 40.4 MB – 223.6 MB | XNNPACK (CPU), Core ML (Apple) | High-fidelity dense pixel classification for complex scenes. | -| **FCN** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet101) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 34.0 MB – 198.1 MB | XNNPACK (CPU), Core ML (Apple) | Fully Convolutional Networks baseline for dense multi-class parsing. | +| Model Family | Variants | Classes / Labels | Size Range | Supported Backends | Notes | +| :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------- | :----------------- | :----------------------------- | :----------------------------------------------------------------------------- | +| **Selfie Segmentation** | [`Portrait`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation), [`Landscape`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation_landscape) | Person / Background | 0.5 MB – 0.6 MB | XNNPACK (CPU), Core ML (Apple) | Real-time front-camera portrait background replacement and blur effects. | +| **LRASPP MobileNetV3** | [See](../../06-api-reference/variables/models.md#semanticsegmentationlraspp_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 3.4 MB – 12.3 MB | XNNPACK (CPU), Core ML (Apple) | Lightweight multi-class scene segmentation with low CPU overhead. | +| **DeepLabV3** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet101), [`MobileNetV3`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 40.4 MB – 223.6 MB | XNNPACK (CPU), Core ML (Apple) | High-fidelity dense pixel classification for complex scenes. | +| **FCN** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet101) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 34.0 MB – 198.1 MB | XNNPACK (CPU), Core ML (Apple) | Fully Convolutional Networks baseline for dense multi-class parsing. | +| **U²-Netp** | [Small](../../06-api-reference/variables/models.md#semanticsegmentationu2netp) | Foreground probability (1 output channel) | 3.3 MB – 5.5 MB | XNNPACK (CPU), Core ML (Apple) | General foreground object masks; FP16 favors speed, FP32 preserves boundaries. | :::tip Using Custom Models To use your own fine-tuned semantic segmentation `.pte` model, pass a [`SemanticSegmenterModel`](../../06-api-reference/type-aliases/SemanticSegmenterModel.md) configuration object to [`useSemanticSegmenter`](../../06-api-reference/functions/useSemanticSegmenter.md) or [`createSemanticSegmenter`](../../06-api-reference/functions/createSemanticSegmenter.md): diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index f6298b2c04..8be6be5d49 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -482,6 +482,25 @@ const SELFIE_SEGMENTATION_LANDSCAPE_COREML_FP16: SemanticSegmenterModel<'backgro modelOpts: SELFIE_SEGMENTATION_XNNPACK_FP32.modelOpts, }; +const U2NETP_XNNPACK_FP32: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/xnnpack/u2netp_xnnpack_fp32.pte`, + modelOpts: { + labels: ['background', 'foreground'] as const, + resizeMode: 'stretch', + interpolation: 'linear', + normalizeOpts: { alpha: 1 / 255, beta: 0 }, + outInterpolation: 'linear', + }, +}; +const U2NETP_COREML_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp16.pte`, + modelOpts: U2NETP_XNNPACK_FP32.modelOpts, +}; +const U2NETP_COREML_FP32: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp32.pte`, + modelOpts: U2NETP_XNNPACK_FP32.modelOpts, +}; + const LRASPP_MOBILENET_V3_LARGE_OPTS = { labels: PASCAL_VOC_LABELS, resizeMode: 'stretch' as const, @@ -2753,6 +2772,27 @@ export const models = { XNNPACK_FP32: SELFIE_SEGMENTATION_LANDSCAPE_XNNPACK_FP32, COREML_FP16: SELFIE_SEGMENTATION_LANDSCAPE_COREML_FP16, }), + /** + * U²-Netp salient foreground segmentation at 320x320, for objects as well + * as people. Produces one foreground logit channel rather than object + * categories or separate instances. The export includes image-maximum + * scaling and ImageNet normalization; the configured preprocessor only + * scales RGB bytes to [0, 1]. + * + * The segmenter applies sigmoid and returns foreground intensity in each + * RGB channel of an RGBA mask (0 = background, 255 = foreground), with + * opaque alpha and no colormap. Use an RGB channel as the alpha mask when + * compositing the original image. Thin structures can be missed. + * + * CoreML requires iOS 18+. FP16 favors speed and may shift mask boundaries; + * choose COREML_FP32 or XNNPACK_FP32 when boundary precision matters. + * Host measurements and CoreML placement still need on-device validation. + */ + U2NETP: variants({ + XNNPACK_FP32: U2NETP_XNNPACK_FP32, + COREML_FP16: U2NETP_COREML_FP16, + COREML_FP32: U2NETP_COREML_FP32, + }), /** * Lite R-ASPP semantic segmentation model with MobileNetV3-Large backbone * (21 classes, see {@link PASCAL_VOC_LABELS}). Optimized for low-latency, From ed0b8c5ef09ccd4970fddd1255397fdf26276fa0 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 29 Oct 2026 14:26:10 +0100 Subject: [PATCH 2/8] Cache download sizes for U2NetP benchmark variants --- apps/benchmarks/scripts/variant-sizes.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index c4dca6a1bf..64df8e1652 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -493,5 +493,8 @@ "https://huggingface.co/software-mansion/all-mpnet-base-v2-ExecuTorch/resolve/v0.11.0/vulkan/all_mpnet_base_v2_vulkan_fp16.pte": 217937154, "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/vulkan/all_minilm_l6_v2_vulkan_fp16.pte": 45208066, "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte": 90391296, - "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_fp16.pte": 45460244 + "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_fp16.pte": 45460244, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_fp32.pte": 4610948, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp16.pte": 3272420, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp32.pte": 5476362 } From 9c337aff5e480b392b519915c4783f316dccaa48 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 29 Oct 2026 14:30:39 +0100 Subject: [PATCH 3/8] Shorten U2NetP model description --- packages/react-native-executorch/src/models.ts | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 8be6be5d49..996b358009 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2773,20 +2773,8 @@ export const models = { COREML_FP16: SELFIE_SEGMENTATION_LANDSCAPE_COREML_FP16, }), /** - * U²-Netp salient foreground segmentation at 320x320, for objects as well - * as people. Produces one foreground logit channel rather than object - * categories or separate instances. The export includes image-maximum - * scaling and ImageNet normalization; the configured preprocessor only - * scales RGB bytes to [0, 1]. - * - * The segmenter applies sigmoid and returns foreground intensity in each - * RGB channel of an RGBA mask (0 = background, 255 = foreground), with - * opaque alpha and no colormap. Use an RGB channel as the alpha mask when - * compositing the original image. Thin structures can be missed. - * - * CoreML requires iOS 18+. FP16 favors speed and may shift mask boundaries; - * choose COREML_FP32 or XNNPACK_FP32 when boundary precision matters. - * Host measurements and CoreML placement still need on-device validation. + * Lightweight U²-Netp foreground segmentation at 320x320 for objects and + * people. Returns a soft grayscale foreground mask for background removal. */ U2NETP: variants({ XNNPACK_FP32: U2NETP_XNNPACK_FP32, From 3d9cf77f7338b993db5cd6abcc63a666d3afeac1 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 29 Oct 2026 15:01:56 +0100 Subject: [PATCH 4/8] Add selective XNNPACK INT8 variant for U2NetP --- apps/benchmarks/scripts/variant-sizes.json | 3 ++- apps/computer-vision/app/segmentation/index.tsx | 4 ++++ .../computer-vision/06-semantic-segmentation.md | 14 +++++++------- packages/react-native-executorch/src/models.ts | 6 ++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index 64df8e1652..93e648ec8d 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -496,5 +496,6 @@ "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_fp16.pte": 45460244, "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_fp32.pte": 4610948, "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp16.pte": 3272420, - "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp32.pte": 5476362 + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp32.pte": 5476362, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_int8.pte": 4007684 } diff --git a/apps/computer-vision/app/segmentation/index.tsx b/apps/computer-vision/app/segmentation/index.tsx index 57205b766f..1ab6cbb08f 100644 --- a/apps/computer-vision/app/segmentation/index.tsx +++ b/apps/computer-vision/app/segmentation/index.tsx @@ -27,6 +27,10 @@ const SEGMENTATION_OPTIONS: ModelOption[] = [ label: 'U²-Netp (XNNPACK FP32)', value: models.semanticSegmentation.U2NETP.XNNPACK_FP32, }, + { + label: 'U²-Netp (XNNPACK INT8)', + value: models.semanticSegmentation.U2NETP.XNNPACK_INT8, + }, { label: 'U²-Netp (CoreML FP16)', value: models.semanticSegmentation.U2NETP.COREML_FP16, diff --git a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md index e5e85aeb5d..fbcff130ec 100644 --- a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md +++ b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md @@ -165,13 +165,13 @@ See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading. The library provides ready-to-use segmentation models from the [Software Mansion HuggingFace Semantic Segmentation Collection](https://huggingface.co/collections/software-mansion/semantic-segmentation), accessible via [`models.semanticSegmentation`](../../06-api-reference/variables/models.md#semanticsegmentation): -| Model Family | Variants | Classes / Labels | Size Range | Supported Backends | Notes | -| :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------- | :----------------- | :----------------------------- | :----------------------------------------------------------------------------- | -| **Selfie Segmentation** | [`Portrait`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation), [`Landscape`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation_landscape) | Person / Background | 0.5 MB – 0.6 MB | XNNPACK (CPU), Core ML (Apple) | Real-time front-camera portrait background replacement and blur effects. | -| **LRASPP MobileNetV3** | [See](../../06-api-reference/variables/models.md#semanticsegmentationlraspp_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 3.4 MB – 12.3 MB | XNNPACK (CPU), Core ML (Apple) | Lightweight multi-class scene segmentation with low CPU overhead. | -| **DeepLabV3** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet101), [`MobileNetV3`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 40.4 MB – 223.6 MB | XNNPACK (CPU), Core ML (Apple) | High-fidelity dense pixel classification for complex scenes. | -| **FCN** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet101) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 34.0 MB – 198.1 MB | XNNPACK (CPU), Core ML (Apple) | Fully Convolutional Networks baseline for dense multi-class parsing. | -| **U²-Netp** | [Small](../../06-api-reference/variables/models.md#semanticsegmentationu2netp) | Foreground probability (1 output channel) | 3.3 MB – 5.5 MB | XNNPACK (CPU), Core ML (Apple) | General foreground object masks; FP16 favors speed, FP32 preserves boundaries. | +| Model Family | Variants | Classes / Labels | Size Range | Supported Backends | Notes | +| :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------- | :----------------- | :----------------------------- | :----------------------------------------------------------------------------------------------------------------------------- | +| **Selfie Segmentation** | [`Portrait`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation), [`Landscape`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation_landscape) | Person / Background | 0.5 MB – 0.6 MB | XNNPACK (CPU), Core ML (Apple) | Real-time front-camera portrait background replacement and blur effects. | +| **LRASPP MobileNetV3** | [See](../../06-api-reference/variables/models.md#semanticsegmentationlraspp_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 3.4 MB – 12.3 MB | XNNPACK (CPU), Core ML (Apple) | Lightweight multi-class scene segmentation with low CPU overhead. | +| **DeepLabV3** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet101), [`MobileNetV3`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 40.4 MB – 223.6 MB | XNNPACK (CPU), Core ML (Apple) | High-fidelity dense pixel classification for complex scenes. | +| **FCN** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet101) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 34.0 MB – 198.1 MB | XNNPACK (CPU), Core ML (Apple) | Fully Convolutional Networks baseline for dense multi-class parsing. | +| **U²-Netp** | [Small](../../06-api-reference/variables/models.md#semanticsegmentationu2netp) | Foreground probability (1 output channel) | 3.3 MB – 5.5 MB | XNNPACK (CPU), Core ML (Apple) | General foreground object masks; selective INT8 for faster CPU inference, FP16 for Apple devices, FP32 for boundary precision. | :::tip Using Custom Models To use your own fine-tuned semantic segmentation `.pte` model, pass a [`SemanticSegmenterModel`](../../06-api-reference/type-aliases/SemanticSegmenterModel.md) configuration object to [`useSemanticSegmenter`](../../06-api-reference/functions/useSemanticSegmenter.md) or [`createSemanticSegmenter`](../../06-api-reference/functions/createSemanticSegmenter.md): diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index 996b358009..f5058b5a87 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -492,6 +492,10 @@ const U2NETP_XNNPACK_FP32: SemanticSegmenterModel<'background' | 'foreground'> = outInterpolation: 'linear', }, }; +const U2NETP_XNNPACK_INT8: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/xnnpack/u2netp_xnnpack_int8.pte`, + modelOpts: U2NETP_XNNPACK_FP32.modelOpts, +}; const U2NETP_COREML_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp16.pte`, modelOpts: U2NETP_XNNPACK_FP32.modelOpts, @@ -2775,9 +2779,11 @@ export const models = { /** * Lightweight U²-Netp foreground segmentation at 320x320 for objects and * people. Returns a soft grayscale foreground mask for background removal. + * INT8 selectively quantizes convolutions for faster CPU inference. */ U2NETP: variants({ XNNPACK_FP32: U2NETP_XNNPACK_FP32, + XNNPACK_INT8: U2NETP_XNNPACK_INT8, COREML_FP16: U2NETP_COREML_FP16, COREML_FP32: U2NETP_COREML_FP32, }), From 7b508c0dd080a4c6f474c63dfe6cecb1192be016 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 29 Oct 2026 15:54:39 +0100 Subject: [PATCH 5/8] Add device-validated U2NetP Vulkan variants --- apps/benchmarks/scripts/variant-sizes.json | 4 ++- .../app/segmentation/index.tsx | 10 ++++++ .../06-semantic-segmentation.md | 14 ++++---- .../08-native-libraries.md | 2 +- .../__tests__/api/modelVariants.test.ts | 28 +++++++++++---- .../scripts/download-libs.js | 4 +-- .../react-native-executorch/src/models.ts | 35 +++++++++++++------ 7 files changed, 69 insertions(+), 28 deletions(-) diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index 93e648ec8d..eb99e83ca1 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -497,5 +497,7 @@ "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_fp32.pte": 4610948, "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp16.pte": 3272420, "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp32.pte": 5476362, - "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_int8.pte": 4007684 + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_int8.pte": 4007684, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/vulkan/u2netp_vulkan_fp16.pte": 2437516, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/vulkan/u2netp_vulkan_fp32.pte": 4646540 } diff --git a/apps/computer-vision/app/segmentation/index.tsx b/apps/computer-vision/app/segmentation/index.tsx index 1ab6cbb08f..99c6b15514 100644 --- a/apps/computer-vision/app/segmentation/index.tsx +++ b/apps/computer-vision/app/segmentation/index.tsx @@ -41,6 +41,16 @@ const SEGMENTATION_OPTIONS: ModelOption[] = [ value: models.semanticSegmentation.U2NETP.COREML_FP32, disabled: Platform.OS !== 'ios' || Number.parseInt(String(Platform.Version), 10) < 18, }, + { + label: 'U²-Netp (Vulkan FP16)', + value: models.semanticSegmentation.U2NETP.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, + { + label: 'U²-Netp (Vulkan FP32)', + value: models.semanticSegmentation.U2NETP.VULKAN_FP32, + disabled: Platform.OS !== 'android', + }, { label: 'LRASPP MobileNet V3 (INT8)', value: models.semanticSegmentation.LRASPP_MOBILENET_V3_LARGE.XNNPACK_INT8, diff --git a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md index fbcff130ec..a768bdafec 100644 --- a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md +++ b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md @@ -165,13 +165,13 @@ See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading. The library provides ready-to-use segmentation models from the [Software Mansion HuggingFace Semantic Segmentation Collection](https://huggingface.co/collections/software-mansion/semantic-segmentation), accessible via [`models.semanticSegmentation`](../../06-api-reference/variables/models.md#semanticsegmentation): -| Model Family | Variants | Classes / Labels | Size Range | Supported Backends | Notes | -| :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------- | :----------------- | :----------------------------- | :----------------------------------------------------------------------------------------------------------------------------- | -| **Selfie Segmentation** | [`Portrait`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation), [`Landscape`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation_landscape) | Person / Background | 0.5 MB – 0.6 MB | XNNPACK (CPU), Core ML (Apple) | Real-time front-camera portrait background replacement and blur effects. | -| **LRASPP MobileNetV3** | [See](../../06-api-reference/variables/models.md#semanticsegmentationlraspp_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 3.4 MB – 12.3 MB | XNNPACK (CPU), Core ML (Apple) | Lightweight multi-class scene segmentation with low CPU overhead. | -| **DeepLabV3** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet101), [`MobileNetV3`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 40.4 MB – 223.6 MB | XNNPACK (CPU), Core ML (Apple) | High-fidelity dense pixel classification for complex scenes. | -| **FCN** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet101) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 34.0 MB – 198.1 MB | XNNPACK (CPU), Core ML (Apple) | Fully Convolutional Networks baseline for dense multi-class parsing. | -| **U²-Netp** | [Small](../../06-api-reference/variables/models.md#semanticsegmentationu2netp) | Foreground probability (1 output channel) | 3.3 MB – 5.5 MB | XNNPACK (CPU), Core ML (Apple) | General foreground object masks; selective INT8 for faster CPU inference, FP16 for Apple devices, FP32 for boundary precision. | +| Model Family | Variants | Classes / Labels | Size Range | Supported Backends | Notes | +| :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------- | :----------------- | :----------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Selfie Segmentation** | [`Portrait`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation), [`Landscape`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation_landscape) | Person / Background | 0.5 MB – 0.6 MB | XNNPACK (CPU), Core ML (Apple) | Real-time front-camera portrait background replacement and blur effects. | +| **LRASPP MobileNetV3** | [See](../../06-api-reference/variables/models.md#semanticsegmentationlraspp_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 3.4 MB – 12.3 MB | XNNPACK (CPU), Core ML (Apple) | Lightweight multi-class scene segmentation with low CPU overhead. | +| **DeepLabV3** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet101), [`MobileNetV3`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 40.4 MB – 223.6 MB | XNNPACK (CPU), Core ML (Apple) | High-fidelity dense pixel classification for complex scenes. | +| **FCN** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet101) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 34.0 MB – 198.1 MB | XNNPACK (CPU), Core ML (Apple) | Fully Convolutional Networks baseline for dense multi-class parsing. | +| **U²-Netp** | [Small](../../06-api-reference/variables/models.md#semanticsegmentationu2netp) | Foreground probability (1 output channel) | 2.4 MB – 5.5 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (Android) | General foreground object masks; selective INT8 for CPU, Core ML/Vulkan acceleration, FP32 for boundary precision. Vulkan FP16 retains one convolution in FP32. | :::tip Using Custom Models To use your own fine-tuned semantic segmentation `.pte` model, pass a [`SemanticSegmenterModel`](../../06-api-reference/type-aliases/SemanticSegmenterModel.md) configuration object to [`useSemanticSegmenter`](../../06-api-reference/functions/useSemanticSegmenter.md) or [`createSemanticSegmenter`](../../06-api-reference/functions/createSemanticSegmenter.md): diff --git a/docs/docs/03-core-and-advanced/08-native-libraries.md b/docs/docs/03-core-and-advanced/08-native-libraries.md index 74fc42bb54..ae56a31275 100644 --- a/docs/docs/03-core-and-advanced/08-native-libraries.md +++ b/docs/docs/03-core-and-advanced/08-native-libraries.md @@ -96,7 +96,7 @@ Specifying a task under `features` is shorthand: it automatically expands to the | `depthEstimation` | xnnpack, coreml, vulkan | opencv | | `objectDetection` | xnnpack, coreml, vulkan | opencv | | `keypointDetection` | xnnpack, coreml, mlx, vulkan | opencv | -| `semanticSegmentation` | xnnpack, coreml | opencv | +| `semanticSegmentation` | xnnpack, coreml, vulkan | opencv | | `instanceSegmentation` | xnnpack, coreml, vulkan | opencv | | `ocr` | xnnpack, coreml, vulkan | opencv | | `verticalOCR` | xnnpack | opencv | diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index cb6eef3dc1..26b932ba6f 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -97,15 +97,20 @@ function registryFor(options: { /** * Every `variants(...)` call in the registry source, with the variant keys it - * lists and whether it pins one for iOS. + * lists and whether it pins one for either platform. * * Prettier gives the two call shapes distinct first lines — `variants({` when * the group takes no pins, `variants(` when it does — which is what this reads. * @returns One entry per call, in source order. */ -function variantsCalls(): { name: string; keys: string[]; pinsIos: boolean }[] { +function variantsCalls(): { + name: string; + keys: string[]; + pinsIos: boolean; + pinsAndroid: boolean; +}[] { const source = readFileSync(join(__dirname, '../../src/models.ts'), 'utf8').split('\n'); - const calls: { name: string; keys: string[]; pinsIos: boolean }[] = []; + const calls: { name: string; keys: string[]; pinsIos: boolean; pinsAndroid: boolean }[] = []; for (let line = 0; line < source.length; line++) { const opened = source[line]!.match(/^(\s*)([A-Z][A-Z0-9_]*): variants\((\{?)$/); @@ -118,7 +123,12 @@ function variantsCalls(): { name: string; keys: string[]; pinsIos: boolean }[] { const body = source.slice(line + 1, end); const keys = body.flatMap((entry) => entry.match(/^\s*([A-Z][A-Z0-9_]*):/)?.slice(1) ?? []); - calls.push({ name: name!, keys, pinsIos: body.some((entry) => /\bios:/.test(entry)) }); + calls.push({ + name: name!, + keys, + pinsIos: body.some((entry) => /\bios:/.test(entry)), + pinsAndroid: body.some((entry) => /\bandroid:/.test(entry)), + }); line = end; } @@ -191,8 +201,14 @@ describe('DEFAULT variant resolution', () => { expect(offenders).toEqual([]); }); - it('prefers Vulkan over XNNPACK on Android wherever a Vulkan export exists', () => { + it('prefers Vulkan over XNNPACK on Android wherever a Vulkan export exists and nothing is pinned', () => { + const pinned = new Set( + variantsCalls() + .filter(({ pinsAndroid }) => pinsAndroid) + .map(({ name }) => name) + ); const offenders = defaultsOf(registryFor({ os: 'android' })) + .filter(({ label }) => !pinned.has(label.split('.').pop()!)) .filter(({ offers }) => offers.some((key) => key.startsWith('VULKAN'))) .filter(({ key }) => key !== undefined && !key.startsWith('VULKAN')) .map(({ label, key }) => `${label}: ${key}`); @@ -300,7 +316,7 @@ describe('DEFAULT variant resolution', () => { }); describe('variant selection rules', () => { - /** The distiluse group, the one place a pin overrides the backend order. */ + /** The distiluse group, which pins MLX over Core ML on iOS. */ const distiluse = (registry: Node): Node => (registry.textEmbeddings as Node).DISTILUSE_BASE_MULTILINGUAL_CASED_V2 as Node; diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 38b990f29c..873246a7f0 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -166,8 +166,8 @@ const FEATURE_MAP = { }, // Depth Anything V2, V3 and ZipDepth ship xnnpack, coreml and vulkan. depthEstimation: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, - // DeepLab/FCN/LR-ASPP/selfie all ship xnnpack + coreml. - semanticSegmentation: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, + // DeepLab/FCN/LR-ASPP/selfie ship xnnpack + coreml; U2NetP adds vulkan. + semanticSegmentation: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, // YOLO-seg xnnpack-only, rf_detr-seg/fastsam add coreml, and all three ship // vulkan Android exports → union. instanceSegmentation: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index f5058b5a87..a0cf5b6f9a 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -82,10 +82,10 @@ type TargetPlatform = 'ios' | 'android'; const PLATFORM: TargetPlatform = Platform.OS === 'ios' ? 'ios' : 'android'; -// Accelerators lead and XNNPACK trails: a model is only exported to Core ML, -// MLX or Vulkan once it has been shown to run better there, and XNNPACK is the -// one backend every model exports to. Core ML sits above MLX only to make the -// order deterministic; every group publishing both pins its winner explicitly. +// Accelerators lead and XNNPACK trails by default. Groups whose measured +// winner differs from this order pin a variant explicitly. Core ML sits above +// MLX only to make the order deterministic; every group publishing both pins +// its winner explicitly. // // The iOS simulator links the Core ML backend but cannot run it: no Neural // Engine, and MPSGraph refuses the compiled models. MLX only ever ships a @@ -496,6 +496,14 @@ const U2NETP_XNNPACK_INT8: SemanticSegmenterModel<'background' | 'foreground'> = modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/xnnpack/u2netp_xnnpack_int8.pte`, modelOpts: U2NETP_XNNPACK_FP32.modelOpts, }; +const U2NETP_VULKAN_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/vulkan/u2netp_vulkan_fp16.pte`, + modelOpts: U2NETP_XNNPACK_FP32.modelOpts, +}; +const U2NETP_VULKAN_FP32: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/vulkan/u2netp_vulkan_fp32.pte`, + modelOpts: U2NETP_XNNPACK_FP32.modelOpts, +}; const U2NETP_COREML_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp16.pte`, modelOpts: U2NETP_XNNPACK_FP32.modelOpts, @@ -2779,14 +2787,19 @@ export const models = { /** * Lightweight U²-Netp foreground segmentation at 320x320 for objects and * people. Returns a soft grayscale foreground mask for background removal. - * INT8 selectively quantizes convolutions for faster CPU inference. + * INT8 and Vulkan FP16 retain sensitive operations in FP32. */ - U2NETP: variants({ - XNNPACK_FP32: U2NETP_XNNPACK_FP32, - XNNPACK_INT8: U2NETP_XNNPACK_INT8, - COREML_FP16: U2NETP_COREML_FP16, - COREML_FP32: U2NETP_COREML_FP32, - }), + U2NETP: variants( + { + XNNPACK_FP32: U2NETP_XNNPACK_FP32, + XNNPACK_INT8: U2NETP_XNNPACK_INT8, + COREML_FP16: U2NETP_COREML_FP16, + COREML_FP32: U2NETP_COREML_FP32, + VULKAN_FP16: U2NETP_VULKAN_FP16, + VULKAN_FP32: U2NETP_VULKAN_FP32, + }, + { android: 'XNNPACK_FP32' } + ), /** * Lite R-ASPP semantic segmentation model with MobileNetV3-Large backbone * (21 classes, see {@link PASCAL_VOC_LABELS}). Optimized for low-latency, From 0d0af0397eb09a183c3709be0c955b46764ff8b6 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 29 Oct 2026 16:20:58 +0100 Subject: [PATCH 6/8] Keep only the U2NetP Vulkan FP16 variant --- apps/benchmarks/scripts/variant-sizes.json | 3 +-- apps/computer-vision/app/segmentation/index.tsx | 5 ----- packages/react-native-executorch/src/models.ts | 13 ++++--------- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index eb99e83ca1..9abd6a12c7 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -498,6 +498,5 @@ "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp16.pte": 3272420, "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp32.pte": 5476362, "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_int8.pte": 4007684, - "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/vulkan/u2netp_vulkan_fp16.pte": 2437516, - "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/vulkan/u2netp_vulkan_fp32.pte": 4646540 + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/vulkan/u2netp_vulkan_fp16.pte": 2437516 } diff --git a/apps/computer-vision/app/segmentation/index.tsx b/apps/computer-vision/app/segmentation/index.tsx index 99c6b15514..e95e3164a4 100644 --- a/apps/computer-vision/app/segmentation/index.tsx +++ b/apps/computer-vision/app/segmentation/index.tsx @@ -46,11 +46,6 @@ const SEGMENTATION_OPTIONS: ModelOption[] = [ value: models.semanticSegmentation.U2NETP.VULKAN_FP16, disabled: Platform.OS !== 'android', }, - { - label: 'U²-Netp (Vulkan FP32)', - value: models.semanticSegmentation.U2NETP.VULKAN_FP32, - disabled: Platform.OS !== 'android', - }, { label: 'LRASPP MobileNet V3 (INT8)', value: models.semanticSegmentation.LRASPP_MOBILENET_V3_LARGE.XNNPACK_INT8, diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index a0cf5b6f9a..d879576a5e 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -82,10 +82,10 @@ type TargetPlatform = 'ios' | 'android'; const PLATFORM: TargetPlatform = Platform.OS === 'ios' ? 'ios' : 'android'; -// Accelerators lead and XNNPACK trails by default. Groups whose measured -// winner differs from this order pin a variant explicitly. Core ML sits above -// MLX only to make the order deterministic; every group publishing both pins -// its winner explicitly. +// Accelerators lead and XNNPACK trails: a model is only exported to Core ML, +// MLX or Vulkan once it has been shown to run better there, and XNNPACK is the +// one backend every model exports to. Core ML sits above MLX only to make the +// order deterministic; every group publishing both pins its winner explicitly. // // The iOS simulator links the Core ML backend but cannot run it: no Neural // Engine, and MPSGraph refuses the compiled models. MLX only ever ships a @@ -500,10 +500,6 @@ const U2NETP_VULKAN_FP16: SemanticSegmenterModel<'background' | 'foreground'> = modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/vulkan/u2netp_vulkan_fp16.pte`, modelOpts: U2NETP_XNNPACK_FP32.modelOpts, }; -const U2NETP_VULKAN_FP32: SemanticSegmenterModel<'background' | 'foreground'> = { - modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/vulkan/u2netp_vulkan_fp32.pte`, - modelOpts: U2NETP_XNNPACK_FP32.modelOpts, -}; const U2NETP_COREML_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp16.pte`, modelOpts: U2NETP_XNNPACK_FP32.modelOpts, @@ -2796,7 +2792,6 @@ export const models = { COREML_FP16: U2NETP_COREML_FP16, COREML_FP32: U2NETP_COREML_FP32, VULKAN_FP16: U2NETP_VULKAN_FP16, - VULKAN_FP32: U2NETP_VULKAN_FP32, }, { android: 'XNNPACK_FP32' } ), From 0a8e005c955fa6a18e08c3a1e077b2fad9bf467f Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 29 Oct 2026 16:25:46 +0100 Subject: [PATCH 7/8] Restore standard Vulkan-first selection for U2NetP --- .../__tests__/api/modelVariants.test.ts | 28 ++++--------------- .../react-native-executorch/src/models.ts | 17 +++++------ 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts index 26b932ba6f..cb6eef3dc1 100644 --- a/packages/react-native-executorch/__tests__/api/modelVariants.test.ts +++ b/packages/react-native-executorch/__tests__/api/modelVariants.test.ts @@ -97,20 +97,15 @@ function registryFor(options: { /** * Every `variants(...)` call in the registry source, with the variant keys it - * lists and whether it pins one for either platform. + * lists and whether it pins one for iOS. * * Prettier gives the two call shapes distinct first lines — `variants({` when * the group takes no pins, `variants(` when it does — which is what this reads. * @returns One entry per call, in source order. */ -function variantsCalls(): { - name: string; - keys: string[]; - pinsIos: boolean; - pinsAndroid: boolean; -}[] { +function variantsCalls(): { name: string; keys: string[]; pinsIos: boolean }[] { const source = readFileSync(join(__dirname, '../../src/models.ts'), 'utf8').split('\n'); - const calls: { name: string; keys: string[]; pinsIos: boolean; pinsAndroid: boolean }[] = []; + const calls: { name: string; keys: string[]; pinsIos: boolean }[] = []; for (let line = 0; line < source.length; line++) { const opened = source[line]!.match(/^(\s*)([A-Z][A-Z0-9_]*): variants\((\{?)$/); @@ -123,12 +118,7 @@ function variantsCalls(): { const body = source.slice(line + 1, end); const keys = body.flatMap((entry) => entry.match(/^\s*([A-Z][A-Z0-9_]*):/)?.slice(1) ?? []); - calls.push({ - name: name!, - keys, - pinsIos: body.some((entry) => /\bios:/.test(entry)), - pinsAndroid: body.some((entry) => /\bandroid:/.test(entry)), - }); + calls.push({ name: name!, keys, pinsIos: body.some((entry) => /\bios:/.test(entry)) }); line = end; } @@ -201,14 +191,8 @@ describe('DEFAULT variant resolution', () => { expect(offenders).toEqual([]); }); - it('prefers Vulkan over XNNPACK on Android wherever a Vulkan export exists and nothing is pinned', () => { - const pinned = new Set( - variantsCalls() - .filter(({ pinsAndroid }) => pinsAndroid) - .map(({ name }) => name) - ); + it('prefers Vulkan over XNNPACK on Android wherever a Vulkan export exists', () => { const offenders = defaultsOf(registryFor({ os: 'android' })) - .filter(({ label }) => !pinned.has(label.split('.').pop()!)) .filter(({ offers }) => offers.some((key) => key.startsWith('VULKAN'))) .filter(({ key }) => key !== undefined && !key.startsWith('VULKAN')) .map(({ label, key }) => `${label}: ${key}`); @@ -316,7 +300,7 @@ describe('DEFAULT variant resolution', () => { }); describe('variant selection rules', () => { - /** The distiluse group, which pins MLX over Core ML on iOS. */ + /** The distiluse group, the one place a pin overrides the backend order. */ const distiluse = (registry: Node): Node => (registry.textEmbeddings as Node).DISTILUSE_BASE_MULTILINGUAL_CASED_V2 as Node; diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index d879576a5e..bb1ff59e18 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -2785,16 +2785,13 @@ export const models = { * people. Returns a soft grayscale foreground mask for background removal. * INT8 and Vulkan FP16 retain sensitive operations in FP32. */ - U2NETP: variants( - { - XNNPACK_FP32: U2NETP_XNNPACK_FP32, - XNNPACK_INT8: U2NETP_XNNPACK_INT8, - COREML_FP16: U2NETP_COREML_FP16, - COREML_FP32: U2NETP_COREML_FP32, - VULKAN_FP16: U2NETP_VULKAN_FP16, - }, - { android: 'XNNPACK_FP32' } - ), + U2NETP: variants({ + XNNPACK_FP32: U2NETP_XNNPACK_FP32, + XNNPACK_INT8: U2NETP_XNNPACK_INT8, + COREML_FP16: U2NETP_COREML_FP16, + COREML_FP32: U2NETP_COREML_FP32, + VULKAN_FP16: U2NETP_VULKAN_FP16, + }), /** * Lite R-ASPP semantic segmentation model with MobileNetV3-Large backbone * (21 classes, see {@link PASCAL_VOC_LABELS}). Optimized for low-latency, From fa7a6ddc11702c982480d08812a0a82eca1b4f07 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 29 Oct 2026 16:31:11 +0100 Subject: [PATCH 8/8] Share U2NetP configuration through a dedicated options constant --- .../react-native-executorch/src/models.ts | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index bb1ff59e18..073ce5564c 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -482,31 +482,32 @@ const SELFIE_SEGMENTATION_LANDSCAPE_COREML_FP16: SemanticSegmenterModel<'backgro modelOpts: SELFIE_SEGMENTATION_XNNPACK_FP32.modelOpts, }; +const U2NETP_OPTS = { + labels: ['background', 'foreground'] as const, + resizeMode: 'stretch' as const, + interpolation: 'linear' as const, + normalizeOpts: { alpha: 1 / 255, beta: 0 }, + outInterpolation: 'linear' as const, +}; const U2NETP_XNNPACK_FP32: SemanticSegmenterModel<'background' | 'foreground'> = { modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/xnnpack/u2netp_xnnpack_fp32.pte`, - modelOpts: { - labels: ['background', 'foreground'] as const, - resizeMode: 'stretch', - interpolation: 'linear', - normalizeOpts: { alpha: 1 / 255, beta: 0 }, - outInterpolation: 'linear', - }, + modelOpts: U2NETP_OPTS, }; const U2NETP_XNNPACK_INT8: SemanticSegmenterModel<'background' | 'foreground'> = { modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/xnnpack/u2netp_xnnpack_int8.pte`, - modelOpts: U2NETP_XNNPACK_FP32.modelOpts, + modelOpts: U2NETP_OPTS, }; const U2NETP_VULKAN_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/vulkan/u2netp_vulkan_fp16.pte`, - modelOpts: U2NETP_XNNPACK_FP32.modelOpts, + modelOpts: U2NETP_OPTS, }; const U2NETP_COREML_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp16.pte`, - modelOpts: U2NETP_XNNPACK_FP32.modelOpts, + modelOpts: U2NETP_OPTS, }; const U2NETP_COREML_FP32: SemanticSegmenterModel<'background' | 'foreground'> = { modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp32.pte`, - modelOpts: U2NETP_XNNPACK_FP32.modelOpts, + modelOpts: U2NETP_OPTS, }; const LRASPP_MOBILENET_V3_LARGE_OPTS = {