diff --git a/.cspell-wordlist.txt b/.cspell-wordlist.txt index 56e970131f..e8a0412c75 100644 --- a/.cspell-wordlist.txt +++ b/.cspell-wordlist.txt @@ -447,3 +447,4 @@ grayscale Premul eurollm doclayout +netp diff --git a/apps/benchmarks/scripts/variant-sizes.json b/apps/benchmarks/scripts/variant-sizes.json index c4dca6a1bf..9abd6a12c7 100644 --- a/apps/benchmarks/scripts/variant-sizes.json +++ b/apps/benchmarks/scripts/variant-sizes.json @@ -493,5 +493,10 @@ "https://huggingface.co/software-mansion/all-mpnet-base-v2-ExecuTorch/resolve/v0.11.0/vulkan/all_mpnet_base_v2_vulkan_fp16.pte": 217937154, "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/vulkan/all_minilm_l6_v2_vulkan_fp16.pte": 45208066, "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/xnnpack/all_minilm_l6_v2_xnnpack_fp32.pte": 90391296, - "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_fp16.pte": 45460244 + "https://huggingface.co/software-mansion/all-MiniLM-L6-v2-ExecuTorch/resolve/v0.11.0/coreml/all_minilm_l6_v2_coreml_fp16.pte": 45460244, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_fp32.pte": 4610948, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp16.pte": 3272420, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/coreml/u2netp_coreml_fp32.pte": 5476362, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/xnnpack/u2netp_xnnpack_int8.pte": 4007684, + "https://huggingface.co/software-mansion/U2NetP-ExecuTorch/resolve/v1.0.0/vulkan/u2netp_vulkan_fp16.pte": 2437516 } diff --git a/apps/computer-vision/app/segmentation/index.tsx b/apps/computer-vision/app/segmentation/index.tsx index 8222bfa60a..e95e3164a4 100644 --- a/apps/computer-vision/app/segmentation/index.tsx +++ b/apps/computer-vision/app/segmentation/index.tsx @@ -1,5 +1,5 @@ import React, { useState, useRef } from 'react'; -import { View, Text, ScrollView } from 'react-native'; +import { View, Text, ScrollView, Platform } from 'react-native'; import { useSafeAreaInsets } from 'react-native-safe-area-context'; import { commonStyles, theme } from '../../theme'; import { @@ -23,6 +23,29 @@ const SEGMENTATION_OPTIONS: ModelOption[] = [ label: 'Selfie Seg (FP32)', value: models.semanticSegmentation.SELFIE_SEGMENTATION.XNNPACK_FP32, }, + { + label: 'U²-Netp (XNNPACK FP32)', + value: models.semanticSegmentation.U2NETP.XNNPACK_FP32, + }, + { + label: 'U²-Netp (XNNPACK INT8)', + value: models.semanticSegmentation.U2NETP.XNNPACK_INT8, + }, + { + label: 'U²-Netp (CoreML FP16)', + value: models.semanticSegmentation.U2NETP.COREML_FP16, + disabled: Platform.OS !== 'ios' || Number.parseInt(String(Platform.Version), 10) < 18, + }, + { + label: 'U²-Netp (CoreML FP32)', + value: models.semanticSegmentation.U2NETP.COREML_FP32, + disabled: Platform.OS !== 'ios' || Number.parseInt(String(Platform.Version), 10) < 18, + }, + { + label: 'U²-Netp (Vulkan FP16)', + value: models.semanticSegmentation.U2NETP.VULKAN_FP16, + disabled: Platform.OS !== 'android', + }, { label: 'LRASPP MobileNet V3 (INT8)', value: models.semanticSegmentation.LRASPP_MOBILENET_V3_LARGE.XNNPACK_INT8, diff --git a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md index e8f92ccb3c..a768bdafec 100644 --- a/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md +++ b/docs/docs/02-extensions/computer-vision/06-semantic-segmentation.md @@ -165,12 +165,13 @@ See [Worklets & Threading](../../03-core-and-advanced/06-worklets-and-threading. The library provides ready-to-use segmentation models from the [Software Mansion HuggingFace Semantic Segmentation Collection](https://huggingface.co/collections/software-mansion/semantic-segmentation), accessible via [`models.semanticSegmentation`](../../06-api-reference/variables/models.md#semanticsegmentation): -| Model Family | Variants | Classes / Labels | Size Range | Supported Backends | Notes | -| :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------- | :----------------- | :----------------------------- | :----------------------------------------------------------------------- | -| **Selfie Segmentation** | [`Portrait`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation), [`Landscape`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation_landscape) | Person / Background | 0.5 MB – 0.6 MB | XNNPACK (CPU), Core ML (Apple) | Real-time front-camera portrait background replacement and blur effects. | -| **LRASPP MobileNetV3** | [See](../../06-api-reference/variables/models.md#semanticsegmentationlraspp_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 3.4 MB – 12.3 MB | XNNPACK (CPU), Core ML (Apple) | Lightweight multi-class scene segmentation with low CPU overhead. | -| **DeepLabV3** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet101), [`MobileNetV3`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 40.4 MB – 223.6 MB | XNNPACK (CPU), Core ML (Apple) | High-fidelity dense pixel classification for complex scenes. | -| **FCN** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet101) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 34.0 MB – 198.1 MB | XNNPACK (CPU), Core ML (Apple) | Fully Convolutional Networks baseline for dense multi-class parsing. | +| Model Family | Variants | Classes / Labels | Size Range | Supported Backends | Notes | +| :---------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :---------------------------------------------------------------------------------------- | :----------------- | :----------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Selfie Segmentation** | [`Portrait`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation), [`Landscape`](../../06-api-reference/variables/models.md#semanticsegmentationselfie_segmentation_landscape) | Person / Background | 0.5 MB – 0.6 MB | XNNPACK (CPU), Core ML (Apple) | Real-time front-camera portrait background replacement and blur effects. | +| **LRASPP MobileNetV3** | [See](../../06-api-reference/variables/models.md#semanticsegmentationlraspp_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 3.4 MB – 12.3 MB | XNNPACK (CPU), Core ML (Apple) | Lightweight multi-class scene segmentation with low CPU overhead. | +| **DeepLabV3** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_resnet101), [`MobileNetV3`](../../06-api-reference/variables/models.md#semanticsegmentationdeeplab_v3_mobilenet_v3_large) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 40.4 MB – 223.6 MB | XNNPACK (CPU), Core ML (Apple) | High-fidelity dense pixel classification for complex scenes. | +| **FCN** | [`ResNet50`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet50), [`ResNet101`](../../06-api-reference/variables/models.md#semanticsegmentationfcn_resnet101) | [`PASCAL_VOC_LABELS`](../../06-api-reference/variables/PASCAL_VOC_LABELS.md) (21 classes) | 34.0 MB – 198.1 MB | XNNPACK (CPU), Core ML (Apple) | Fully Convolutional Networks baseline for dense multi-class parsing. | +| **U²-Netp** | [Small](../../06-api-reference/variables/models.md#semanticsegmentationu2netp) | Foreground probability (1 output channel) | 2.4 MB – 5.5 MB | XNNPACK (CPU), Core ML (Apple), Vulkan (Android) | General foreground object masks; selective INT8 for CPU, Core ML/Vulkan acceleration, FP32 for boundary precision. Vulkan FP16 retains one convolution in FP32. | :::tip Using Custom Models To use your own fine-tuned semantic segmentation `.pte` model, pass a [`SemanticSegmenterModel`](../../06-api-reference/type-aliases/SemanticSegmenterModel.md) configuration object to [`useSemanticSegmenter`](../../06-api-reference/functions/useSemanticSegmenter.md) or [`createSemanticSegmenter`](../../06-api-reference/functions/createSemanticSegmenter.md): diff --git a/docs/docs/03-core-and-advanced/08-native-libraries.md b/docs/docs/03-core-and-advanced/08-native-libraries.md index 74fc42bb54..ae56a31275 100644 --- a/docs/docs/03-core-and-advanced/08-native-libraries.md +++ b/docs/docs/03-core-and-advanced/08-native-libraries.md @@ -96,7 +96,7 @@ Specifying a task under `features` is shorthand: it automatically expands to the | `depthEstimation` | xnnpack, coreml, vulkan | opencv | | `objectDetection` | xnnpack, coreml, vulkan | opencv | | `keypointDetection` | xnnpack, coreml, mlx, vulkan | opencv | -| `semanticSegmentation` | xnnpack, coreml | opencv | +| `semanticSegmentation` | xnnpack, coreml, vulkan | opencv | | `instanceSegmentation` | xnnpack, coreml, vulkan | opencv | | `ocr` | xnnpack, coreml, vulkan | opencv | | `verticalOCR` | xnnpack | opencv | diff --git a/packages/react-native-executorch/scripts/download-libs.js b/packages/react-native-executorch/scripts/download-libs.js index 38b990f29c..873246a7f0 100644 --- a/packages/react-native-executorch/scripts/download-libs.js +++ b/packages/react-native-executorch/scripts/download-libs.js @@ -166,8 +166,8 @@ const FEATURE_MAP = { }, // Depth Anything V2, V3 and ZipDepth ship xnnpack, coreml and vulkan. depthEstimation: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, - // DeepLab/FCN/LR-ASPP/selfie all ship xnnpack + coreml. - semanticSegmentation: { backends: ['xnnpack', 'coreml'], libs: ['opencv'] }, + // DeepLab/FCN/LR-ASPP/selfie ship xnnpack + coreml; U2NetP adds vulkan. + semanticSegmentation: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, // YOLO-seg xnnpack-only, rf_detr-seg/fastsam add coreml, and all three ship // vulkan Android exports → union. instanceSegmentation: { backends: ['xnnpack', 'coreml', 'vulkan'], libs: ['opencv'] }, diff --git a/packages/react-native-executorch/src/models.ts b/packages/react-native-executorch/src/models.ts index f6298b2c04..073ce5564c 100644 --- a/packages/react-native-executorch/src/models.ts +++ b/packages/react-native-executorch/src/models.ts @@ -482,6 +482,34 @@ const SELFIE_SEGMENTATION_LANDSCAPE_COREML_FP16: SemanticSegmenterModel<'backgro modelOpts: SELFIE_SEGMENTATION_XNNPACK_FP32.modelOpts, }; +const U2NETP_OPTS = { + labels: ['background', 'foreground'] as const, + resizeMode: 'stretch' as const, + interpolation: 'linear' as const, + normalizeOpts: { alpha: 1 / 255, beta: 0 }, + outInterpolation: 'linear' as const, +}; +const U2NETP_XNNPACK_FP32: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/xnnpack/u2netp_xnnpack_fp32.pte`, + modelOpts: U2NETP_OPTS, +}; +const U2NETP_XNNPACK_INT8: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/xnnpack/u2netp_xnnpack_int8.pte`, + modelOpts: U2NETP_OPTS, +}; +const U2NETP_VULKAN_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/vulkan/u2netp_vulkan_fp16.pte`, + modelOpts: U2NETP_OPTS, +}; +const U2NETP_COREML_FP16: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp16.pte`, + modelOpts: U2NETP_OPTS, +}; +const U2NETP_COREML_FP32: SemanticSegmenterModel<'background' | 'foreground'> = { + modelPath: `${BASE_URL}/U2NetP-ExecuTorch/${NEXT_VERSION_TAG}/coreml/u2netp_coreml_fp32.pte`, + modelOpts: U2NETP_OPTS, +}; + const LRASPP_MOBILENET_V3_LARGE_OPTS = { labels: PASCAL_VOC_LABELS, resizeMode: 'stretch' as const, @@ -2753,6 +2781,18 @@ export const models = { XNNPACK_FP32: SELFIE_SEGMENTATION_LANDSCAPE_XNNPACK_FP32, COREML_FP16: SELFIE_SEGMENTATION_LANDSCAPE_COREML_FP16, }), + /** + * Lightweight U²-Netp foreground segmentation at 320x320 for objects and + * people. Returns a soft grayscale foreground mask for background removal. + * INT8 and Vulkan FP16 retain sensitive operations in FP32. + */ + U2NETP: variants({ + XNNPACK_FP32: U2NETP_XNNPACK_FP32, + XNNPACK_INT8: U2NETP_XNNPACK_INT8, + COREML_FP16: U2NETP_COREML_FP16, + COREML_FP32: U2NETP_COREML_FP32, + VULKAN_FP16: U2NETP_VULKAN_FP16, + }), /** * Lite R-ASPP semantic segmentation model with MobileNetV3-Large backbone * (21 classes, see {@link PASCAL_VOC_LABELS}). Optimized for low-latency,