From 633a1389b02237675382a7ee81f7f6082ea17165 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 05:32:32 +0000 Subject: [PATCH 01/14] native: add ARM64 (NEON/SVE/SVE2) ISA tiers to meson.build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate x86-64-specific isa_variants and the avx3_dl/avx3_spr extra static libs behind cpu == 'x86_64'. Add a parallel aarch64 block with three tiers: neon (-march=armv8-a+crypto) — baseline, all Graviton + Apple Silicon sve (-march=armv8.4-a+sve) — Graviton 3 / Neoverse V1 (Linux only; Highway marks SVE broken on HWY_OS_APPLE) sve2 (-march=armv9-a+sve2) — Graviton 4 / Neoverse V2/N2 (Linux only; requires GCC >= 10 or Clang >= 22) The three ARM static libs are appended to isa_libs and linked into libjvector.so via link_whole, exactly as the x86 tiers already are. An unsupported cpu_family() triggers a hard meson error. --- jvector-native/src/main/native/meson.build | 147 +++++++++++++-------- 1 file changed, 93 insertions(+), 54 deletions(-) diff --git a/jvector-native/src/main/native/meson.build b/jvector-native/src/main/native/meson.build index 42fec1ada..916712b85 100644 --- a/jvector-native/src/main/native/meson.build +++ b/jvector-native/src/main/native/meson.build @@ -25,30 +25,67 @@ hwy_inc = include_directories('third_party/highway') # Each ISA variant: name, JV_ISA namespace, and extra compiler flags. # These all compile jvector_simd_kernels.cpp (the generic kernel file). -isa_variants = [ - { - 'name' : 'avx3', - 'namespace': 'AVX3', - 'args' : ['-march=skylake-avx512', - '-DHWY_COMPILE_ONLY_STATIC', - '-DJV_REQUIRE_HWY_AVX3'], - }, - { - 'name' : 'avx2', - 'namespace': 'AVX2', - 'args' : ['-march=haswell', - '-maes', - '-DHWY_COMPILE_ONLY_STATIC', - '-DJV_REQUIRE_HWY_AVX2'], - }, - { - 'name' : 'sse42', - 'namespace': 'SSE42', - 'args' : ['-msse4.2', '-mpclmul', '-maes', - '-DHWY_COMPILE_ONLY_STATIC', - '-DJV_REQUIRE_HWY_SCALAR'], - }, -] +cpu = host_machine.cpu_family() + +if cpu == 'x86_64' + isa_variants = [ + { + 'name' : 'avx3', + 'namespace': 'AVX3', + 'args' : ['-march=skylake-avx512', + '-DHWY_COMPILE_ONLY_STATIC', + '-DJV_REQUIRE_HWY_AVX3'], + }, + { + 'name' : 'avx2', + 'namespace': 'AVX2', + 'args' : ['-march=haswell', + '-maes', + '-DHWY_COMPILE_ONLY_STATIC', + '-DJV_REQUIRE_HWY_AVX2'], + }, + { + 'name' : 'sse42', + 'namespace': 'SSE42', + 'args' : ['-msse4.2', '-mpclmul', '-maes', + '-DHWY_COMPILE_ONLY_STATIC', + '-DJV_REQUIRE_HWY_SCALAR'], + }, + ] +elif cpu == 'aarch64' + # Three tiers in ascending capability order: + # NEON — baseline AArch64 (all Graviton, all Apple Silicon) + # SVE — Graviton 3 / Neoverse V1 (256-bit SVE). + # Note: Highway treats SVE as broken on HWY_OS_APPLE; no Apple CPU + # through M4/A18 implements SVE, so this tier is Graviton-only. + # SVE2 — Graviton 4 / Neoverse V2/N2. + # Requires GCC >= 10 or Clang >= 22. Also Graviton-only (no Apple). + isa_variants = [ + { + 'name' : 'neon', + 'namespace': 'NEON', + 'args' : ['-march=armv8-a+crypto', + '-DHWY_COMPILE_ONLY_STATIC', + '-DJV_REQUIRE_HWY_NEON'], + }, + { + 'name' : 'sve', + 'namespace': 'SVE', + 'args' : ['-march=armv8.4-a+sve', + '-DHWY_COMPILE_ONLY_STATIC', + '-DJV_REQUIRE_HWY_SVE'], + }, + { + 'name' : 'sve2', + 'namespace': 'SVE2', + 'args' : ['-march=armv9-a+sve2', + '-DHWY_COMPILE_ONLY_STATIC', + '-DJV_REQUIRE_HWY_SVE2'], + }, + ] +else + error('Unsupported CPU family: ' + cpu + '. Supported: x86_64, aarch64.') +endif isa_libs = [] foreach isa : isa_variants @@ -61,36 +98,38 @@ foreach isa : isa_variants isa_libs += lib endforeach -# AVX3_DL (Ice Lake) tier: -march=icelake-server enables the full ICX feature -# set (AVX3 + VNNI, VBMI, VBMI2, IFMA, BITALG, VPOPCNTDQ, GFNI, VAES, VPCLMULQDQ). -avx3_dl_lib = static_library( - 'simdKernels_avx3_dl', - sources : 'src/jvector_avx3_dl_kernels.cpp', - include_directories: hwy_inc, - cpp_args : [ - '-march=icelake-server', - '-DHWY_COMPILE_ONLY_STATIC', - '-DJV_REQUIRE_HWY_AVX3_DL', - '-fvisibility=hidden', - ], -) -isa_libs += avx3_dl_lib - -# AVX3_SPR (Sapphire Rapids) tier: -march=sapphirerapids adds AVX512FP16 and -# AVX512BF16 on top of the icelake-server feature set. -# Requires GCC >= 12 or Clang >= 14. -avx3_spr_lib = static_library( - 'simdKernels_avx3_spr', - sources : 'src/jvector_avx3_spr_kernels.cpp', - include_directories: hwy_inc, - cpp_args : [ - '-march=sapphirerapids', - '-DHWY_COMPILE_ONLY_STATIC', - '-DJV_REQUIRE_HWY_AVX3_SPR', - '-fvisibility=hidden', - ], -) -isa_libs += avx3_spr_lib +if cpu == 'x86_64' + # AVX3_DL (Ice Lake) tier: -march=icelake-server enables the full ICX feature + # set (AVX3 + VNNI, VBMI, VBMI2, IFMA, BITALG, VPOPCNTDQ, GFNI, VAES, VPCLMULQDQ). + avx3_dl_lib = static_library( + 'simdKernels_avx3_dl', + sources : 'src/jvector_avx3_dl_kernels.cpp', + include_directories: hwy_inc, + cpp_args : [ + '-march=icelake-server', + '-DHWY_COMPILE_ONLY_STATIC', + '-DJV_REQUIRE_HWY_AVX3_DL', + '-fvisibility=hidden', + ], + ) + isa_libs += avx3_dl_lib + + # AVX3_SPR (Sapphire Rapids) tier: -march=sapphirerapids adds AVX512FP16 and + # AVX512BF16 on top of the icelake-server feature set. + # Requires GCC >= 12 or Clang >= 14. + avx3_spr_lib = static_library( + 'simdKernels_avx3_spr', + sources : 'src/jvector_avx3_spr_kernels.cpp', + include_directories: hwy_inc, + cpp_args : [ + '-march=sapphirerapids', + '-DHWY_COMPILE_ONLY_STATIC', + '-DJV_REQUIRE_HWY_AVX3_SPR', + '-fvisibility=hidden', + ], + ) + isa_libs += avx3_spr_lib +endif # vectorUtil provides a single runtime-dispatch entry point. # link_whole pulls every ISA object into the shared library so that no From 0eedb8303e44c5b29cc943bf78a5ccc3523d693e Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 05:44:26 +0000 Subject: [PATCH 02/14] native: introduce jvector_arch.h and guard all arch-specific code with #if/#else/#endif MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add jvector_arch.h as the single source of truth for JV_ARCH_X86_64 and JV_ARCH_AARCH64 macros, derived from compiler predefined macros (__x86_64__ / __aarch64__). jvector_simd_kernels.h: declare ISA namespaces under a single #if JV_ARCH_X86_64 ... #else ... #endif block so only the namespaces for the current build target are visible to the compiler. jvector_simd.cpp: - MaxIsa enum and its static_assert are now separate per-arch definitions under #if JV_ARCH_X86_64 / #else / #endif — each side carries only the tiers relevant to it (SSE42..AVX3_SPR on x86; NEON/SVE/SVE2 on AArch64). - read_max_isa(), vtable definitions, dispatch_kernels(), and jvector_simd_get_active_isa() all use the same #if/#else/#endif pattern. - Add AArch64 vtable stubs: NEON_vtable (baseline), SVE_vtable (inherits NEON), SVE2_vtable (inherits SVE) — mirroring the x86 inheritance pattern. - Add AArch64 dispatch branch in dispatch_kernels() probing CpuFeature::SVE2, SVE, NEON in descending order (feature detection wired up in next commit). --- .../src/main/native/src/jvector_arch.h | 45 +++++++ .../src/main/native/src/jvector_simd.cpp | 113 ++++++++++++++---- .../main/native/src/jvector_simd_kernels.h | 19 ++- 3 files changed, 150 insertions(+), 27 deletions(-) create mode 100644 jvector-native/src/main/native/src/jvector_arch.h diff --git a/jvector-native/src/main/native/src/jvector_arch.h b/jvector-native/src/main/native/src/jvector_arch.h new file mode 100644 index 000000000..21a4a1b78 --- /dev/null +++ b/jvector-native/src/main/native/src/jvector_arch.h @@ -0,0 +1,45 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// jvector_arch.h — canonical architecture detection macros for jvector-native. +// +// Use JV_ARCH_X86_64 / JV_ARCH_AARCH64 throughout the codebase to guard +// architecture-specific code instead of scattering raw compiler predefined +// macros (__x86_64__, __aarch64__, etc.). This keeps the guards readable and +// makes it trivial to add a new architecture in one place. +// +// Exactly one of these will be defined to 1 on a supported build; the other +// will be defined to 0. Unsupported architectures define neither to 1 so that +// #if JV_ARCH_X86_64 / #if JV_ARCH_AARCH64 simply evaluate false. + +#ifndef JVECTOR_ARCH_H +#define JVECTOR_ARCH_H + +// ---- x86-64 ----------------------------------------------------------------- +#if defined(__x86_64__) || defined(_M_X64) +# define JV_ARCH_X86_64 1 +# define JV_ARCH_AARCH64 0 +// ---- AArch64 ---------------------------------------------------------------- +#elif defined(__aarch64__) || defined(_M_ARM64) +# define JV_ARCH_X86_64 0 +# define JV_ARCH_AARCH64 1 +// ---- Unsupported ------------------------------------------------------------ +#else +# define JV_ARCH_X86_64 0 +# define JV_ARCH_AARCH64 0 +#endif + +#endif // JVECTOR_ARCH_H diff --git a/jvector-native/src/main/native/src/jvector_simd.cpp b/jvector-native/src/main/native/src/jvector_simd.cpp index 8bf8ebef5..fcd6037d4 100644 --- a/jvector-native/src/main/native/src/jvector_simd.cpp +++ b/jvector-native/src/main/native/src/jvector_simd.cpp @@ -15,12 +15,15 @@ */ // Runtime SIMD dispatch: selects the best available ISA tier at startup. -// Tiers in descending capability order: AVX3_SPR, AVX3_DL, AVX3, AVX2, SSE42. -// SSE42 is the baseline and is assumed always available without a CPUID check. +// x86-64 tiers (descending): AVX3_SPR, AVX3_DL, AVX3, AVX2, SSE42. +// SSE42 is the x86-64 baseline — assumed always available, no CPUID check. +// AArch64 tiers (descending): SVE2, SVE, NEON. +// NEON is the AArch64 baseline — always available on any aarch64 CPU. // Function pointers are resolved once at static-init time; each public call is // a single indirect branch. #include "jvector_simd.h" -#include "jvector_simd_kernels.h" // AVX3_SPR::, AVX3_DL::, AVX3::, AVX2::, SSE42:: kernel declarations +#include "jvector_arch.h" // JV_ARCH_X86_64, JV_ARCH_AARCH64 +#include "jvector_simd_kernels.h" // per-arch namespace declarations #include "jvector_cpu_features.h" // populate_cpu_features(), CpuFeature enum #include @@ -36,29 +39,46 @@ namespace { // comparisons (e.g. max_isa > MaxIsa::AVX2) correctly gate higher tiers. // Unset (INT_MAX) means "no override; use best available CPU capability" // and must be greater than every named tier so that all guards pass. +#if JV_ARCH_X86_64 enum class MaxIsa { SSE42 = 0, AVX2 = 1, AVX3 = 2, AVX3_DL = 3, AVX3_SPR = 4, Unset = INT_MAX }; static_assert( - (int)MaxIsa::SSE42 < (int)MaxIsa::AVX2 - && (int)MaxIsa::AVX2 < (int)MaxIsa::AVX3 - && (int)MaxIsa::AVX3 < (int)MaxIsa::AVX3_DL + (int)MaxIsa::SSE42 < (int)MaxIsa::AVX2 + && (int)MaxIsa::AVX2 < (int)MaxIsa::AVX3 + && (int)MaxIsa::AVX3 < (int)MaxIsa::AVX3_DL && (int)MaxIsa::AVX3_DL < (int)MaxIsa::AVX3_SPR && (int)MaxIsa::AVX3_SPR < (int)MaxIsa::Unset, "MaxIsa values must be in strict ascending capability order with Unset at the top"); +#else // JV_ARCH_AARCH64 +enum class MaxIsa { NEON = 0, SVE = 1, SVE2 = 2, + Unset = INT_MAX }; +static_assert( + (int)MaxIsa::NEON < (int)MaxIsa::SVE + && (int)MaxIsa::SVE < (int)MaxIsa::SVE2 + && (int)MaxIsa::SVE2 < (int)MaxIsa::Unset, + "MaxIsa values must be in strict ascending capability order with Unset at the top"); +#endif // JV_ARCH_X86_64 / JV_ARCH_AARCH64 // Reads the JVECTOR_MAX_ISA environment variable and maps it to a MaxIsa // value. This lets callers cap the ISA at runtime without recompiling — // useful for benchmarking or working around CPU errata. -// Accepted values (case-sensitive): "avx3", "avx2", "sse42". +// x86-64 values (case-sensitive): "avx3_spr", "avx3_dl", "avx3", "avx2", "sse42". +// AArch64 values (case-sensitive): "sve2", "sve", "neon". static MaxIsa read_max_isa() noexcept { const char *val = std::getenv("JVECTOR_MAX_ISA"); if (!val) return MaxIsa::Unset; +#if JV_ARCH_X86_64 if (std::strcmp(val, "avx3_spr") == 0) return MaxIsa::AVX3_SPR; if (std::strcmp(val, "avx3_dl") == 0) return MaxIsa::AVX3_DL; - if (std::strcmp(val, "avx3") == 0) return MaxIsa::AVX3; - if (std::strcmp(val, "avx2") == 0) return MaxIsa::AVX2; - if (std::strcmp(val, "sse42") == 0) return MaxIsa::SSE42; + if (std::strcmp(val, "avx3") == 0) return MaxIsa::AVX3; + if (std::strcmp(val, "avx2") == 0) return MaxIsa::AVX2; + if (std::strcmp(val, "sse42") == 0) return MaxIsa::SSE42; +#else // JV_ARCH_AARCH64 + if (std::strcmp(val, "sve2") == 0) return MaxIsa::SVE2; + if (std::strcmp(val, "sve") == 0) return MaxIsa::SVE; + if (std::strcmp(val, "neon") == 0) return MaxIsa::NEON; +#endif return MaxIsa::Unset; // unrecognised value: ignore and use CPU detection } @@ -74,7 +94,11 @@ struct KernelVTable { }; // One pre-filled vtable per ISA. These are constant data; no heap allocation. -// Auto-generated from jvector_simd_kernel_list.h +// Auto-generated from jvector_simd_kernel_list.h. +// Guarded by JV_ARCH_* so that only the vtables for the current build +// architecture are instantiated (avoiding references to non-existent symbols). + +#if JV_ARCH_X86_64 #define KERNEL_ENTRY(ret_type, name, params, names) AVX3::name, static const KernelVTable AVX3_vtable = { @@ -110,6 +134,28 @@ static const KernelVTable SSE42_vtable = { }; #undef KERNEL_ENTRY +#else // JV_ARCH_AARCH64 + +#define KERNEL_ENTRY(ret_type, name, params, names) NEON::name, +static const KernelVTable NEON_vtable = { + JVECTOR_SIMD_KERNEL_LIST +}; +#undef KERNEL_ENTRY + +#define KERNEL_ENTRY(ret_type, name, params, names) SVE::name, +static const KernelVTable SVE_vtable = { + JVECTOR_SIMD_KERNEL_LIST +}; +#undef KERNEL_ENTRY + +#define KERNEL_ENTRY(ret_type, name, params, names) SVE2::name, +static const KernelVTable SVE2_vtable = { + JVECTOR_SIMD_KERNEL_LIST +}; +#undef KERNEL_ENTRY + +#endif // JV_ARCH_X86_64 / JV_ARCH_AARCH64 + // Bundles the chosen vtable and the tier that was selected together so that // both can be initialised atomically from a single dispatch call. struct DispatchResult { @@ -126,7 +172,7 @@ static DispatchResult dispatch_kernels() noexcept // Check whether the caller has capped the ISA via the environment variable. const MaxIsa max_isa = read_max_isa(); - // Populate a boolean feature array by issuing CPUID and reading XCR0. + // Populate a boolean feature array via CPUID (x86) or getauxval/sysctl (ARM). std::array(CpuFeature::COUNT)> features; populate_cpu_features(features); @@ -134,18 +180,18 @@ static DispatchResult dispatch_kernels() noexcept return features[static_cast(f)]; }; - // Select the highest tier the CPU supports and the cap allows. - // max_isa > MaxIsa::X means "user has not capped at X or below". - // Adding a new tier above AVX3_SPR only requires one new if at the top. - // Capture the recognised env-var string (or nullptr) for later retrieval. const char *env_str = nullptr; - if (max_isa == MaxIsa::AVX3_SPR) env_str = "avx3_spr"; - else if (max_isa == MaxIsa::AVX3_DL) env_str = "avx3_dl"; - else if (max_isa == MaxIsa::AVX3) env_str = "avx3"; - else if (max_isa == MaxIsa::AVX2) env_str = "avx2"; - else if (max_isa == MaxIsa::SSE42) env_str = "sse42"; +#if JV_ARCH_X86_64 + if (max_isa == MaxIsa::AVX3_SPR) env_str = "avx3_spr"; + else if (max_isa == MaxIsa::AVX3_DL) env_str = "avx3_dl"; + else if (max_isa == MaxIsa::AVX3) env_str = "avx3"; + else if (max_isa == MaxIsa::AVX2) env_str = "avx2"; + else if (max_isa == MaxIsa::SSE42) env_str = "sse42"; + // Select the highest tier the CPU supports and the cap allows. + // max_isa > MaxIsa::X means "user has not capped at X or below". + // Adding a new tier above AVX3_SPR only requires one new if at the top. if (max_isa > MaxIsa::AVX3_DL && has(CpuFeature::AVX3_SPR)) return { AVX3_SPR_vtable, MaxIsa::AVX3_SPR, env_str }; if (max_isa > MaxIsa::AVX3 && has(CpuFeature::AVX3_DL)) @@ -154,8 +200,22 @@ static DispatchResult dispatch_kernels() noexcept return { AVX3_vtable, MaxIsa::AVX3, env_str }; if (max_isa > MaxIsa::SSE42 && has(CpuFeature::AVX2)) return { AVX2_vtable, MaxIsa::AVX2, env_str }; - // SSE42 is the baseline — assumed always present, no CPUID check needed. + // SSE42 is the x86-64 baseline — assumed always present, no CPUID check. return { SSE42_vtable, MaxIsa::SSE42, env_str }; +#else // JV_ARCH_AARCH64 + if (max_isa == MaxIsa::SVE2) env_str = "sve2"; + else if (max_isa == MaxIsa::SVE) env_str = "sve"; + else if (max_isa == MaxIsa::NEON) env_str = "neon"; + + // SVE2 and SVE are not available on Apple Silicon (up to and including M4). + // populate_cpu_features() will return false for those on HWY_OS_APPLE. + if (max_isa > MaxIsa::SVE && has(CpuFeature::SVE2)) + return { SVE2_vtable, MaxIsa::SVE2, env_str }; + if (max_isa > MaxIsa::NEON && has(CpuFeature::SVE)) + return { SVE_vtable, MaxIsa::SVE, env_str }; + // NEON is the AArch64 baseline — always available, no auxval check needed. + return { NEON_vtable, MaxIsa::NEON, env_str }; +#endif // JV_ARCH_X86_64 / JV_ARCH_AARCH64 } // Both are initialised once at static-init time from a single dispatch call. @@ -190,11 +250,18 @@ JVECTOR_SIMD_KERNEL_LIST const char *jvector_simd_get_active_isa() { switch (active_isa) { +#if JV_ARCH_X86_64 case MaxIsa::AVX3_SPR: return "avx3_spr"; case MaxIsa::AVX3_DL: return "avx3_dl"; case MaxIsa::AVX3: return "avx3"; case MaxIsa::AVX2: return "avx2"; - default: return "sse42"; + case MaxIsa::SSE42: return "sse42"; +#else // JV_ARCH_AARCH64 + case MaxIsa::SVE2: return "sve2"; + case MaxIsa::SVE: return "sve"; + case MaxIsa::NEON: return "neon"; +#endif // JV_ARCH_X86_64 / JV_ARCH_AARCH64 + default: return "unknown"; } } diff --git a/jvector-native/src/main/native/src/jvector_simd_kernels.h b/jvector-native/src/main/native/src/jvector_simd_kernels.h index 3261b20ce..65862d614 100644 --- a/jvector-native/src/main/native/src/jvector_simd_kernels.h +++ b/jvector-native/src/main/native/src/jvector_simd_kernels.h @@ -14,19 +14,22 @@ * limitations under the License. */ -// Header file for the SIMD kernels -// Kernel declarations are auto-generated from jvector_simd_kernel_list.h +// Header file for the SIMD kernels. +// Kernel declarations are auto-generated from jvector_simd_kernel_list.h. +// Architecture guards use the canonical macros from jvector_arch.h so that +// only the namespaces that exist for the current build target are declared. #ifndef SIMD_KERNELS_H #define SIMD_KERNELS_H #include #include +#include "jvector_arch.h" -// Macro to declare a kernel function signature from the kernel list +// Macro to declare a kernel function signature from the kernel list. #define KERNEL_ENTRY(ret_type, name, params, names) \ ret_type name params; -// Generate namespace declarations for each ISA +// Declares all kernel signatures inside a namespace named ISA. #define DECLARE_SIMD_KERNELS(ISA) \ namespace ISA { \ JVECTOR_SIMD_KERNEL_LIST \ @@ -34,11 +37,19 @@ #include "jvector_simd_kernel_list.h" +#if JV_ARCH_X86_64 +// x86-64 ISA namespaces (SSE4.2 baseline → AVX2 → AVX3 → Ice Lake → Sapphire Rapids) DECLARE_SIMD_KERNELS(AVX3_SPR) DECLARE_SIMD_KERNELS(AVX3_DL) DECLARE_SIMD_KERNELS(AVX3) DECLARE_SIMD_KERNELS(AVX2) DECLARE_SIMD_KERNELS(SSE42) +#else // JV_ARCH_AARCH64 +// AArch64 ISA namespaces (NEON baseline → SVE → SVE2) +DECLARE_SIMD_KERNELS(NEON) +DECLARE_SIMD_KERNELS(SVE) +DECLARE_SIMD_KERNELS(SVE2) +#endif // JV_ARCH_X86_64 / JV_ARCH_AARCH64 #undef KERNEL_ENTRY From 93211e77610b835cee20af7892885776d664e836 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 06:02:26 +0000 Subject: [PATCH 03/14] native: guard arch-specific code with #if JV_ARCH_X86_64 / #elif JV_ARCH_AARCH64 / #endif MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assert_hwy_targets.h: - Replace raw __x86_64__ guard with JV_ARCH_X86_64 from jvector_arch.h. - Add #elif JV_ARCH_AARCH64 block with exact single-constant assertions: JV_REQUIRE_HWY_NEON → HWY_STATIC_TARGET == HWY_NEON (-march=armv8-a+crypto does not enable BF16/dotprod/I8MM so HWY_NEON_BF16 is never selected; assert the precise constant) JV_REQUIRE_HWY_SVE → HWY_STATIC_TARGET == HWY_SVE (-march=armv8.4-a+sve carries no fixed-width hint so HWY_SVE_256 is not used) JV_REQUIRE_HWY_SVE2 → HWY_STATIC_TARGET == HWY_SVE2 (-march=armv9-a+sve2 carries no fixed-width hint so HWY_SVE2_128 is not used) jvector_simd_kernels.h / jvector_simd.cpp: - Convert all back-to-back #if/#if and #if/#else pairs to the canonical #if JV_ARCH_X86_64 / #elif JV_ARCH_AARCH64 / #endif pattern. --- .../src/main/native/src/assert_hwy_targets.h | 56 ++++++++++++++----- .../src/main/native/src/jvector_simd.cpp | 10 ++-- .../main/native/src/jvector_simd_kernels.h | 2 +- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/jvector-native/src/main/native/src/assert_hwy_targets.h b/jvector-native/src/main/native/src/assert_hwy_targets.h index b1f4b6347..4687322bf 100644 --- a/jvector-native/src/main/native/src/assert_hwy_targets.h +++ b/jvector-native/src/main/native/src/assert_hwy_targets.h @@ -14,22 +14,48 @@ * limitations under the License. */ -#if defined(__x86_64__) || defined(_M_X64) +#include "jvector_arch.h" + +#if JV_ARCH_X86_64 + #if defined(JV_REQUIRE_HWY_AVX3_SPR) -#if HWY_STATIC_TARGET != HWY_AVX3_SPR -#error "Highway did not select HWY_AVX3_SPR for the Sapphire Rapids build. Check compiler flags, compiler support, and Highway blocklists." -#endif +# if HWY_STATIC_TARGET != HWY_AVX3_SPR +# error "Highway did not select HWY_AVX3_SPR for the Sapphire Rapids build. Check compiler flags, compiler support, and Highway blocklists." +# endif #elif defined(JV_REQUIRE_HWY_AVX3_DL) -#if HWY_STATIC_TARGET != HWY_AVX3_DL -#error "Highway did not select HWY_AVX3_DL for the Ice Lake build. Check compiler flags, compiler support, and Highway blocklists." -#endif +# if HWY_STATIC_TARGET != HWY_AVX3_DL +# error "Highway did not select HWY_AVX3_DL for the Ice Lake build. Check compiler flags, compiler support, and Highway blocklists." +# endif #elif defined(JV_REQUIRE_HWY_AVX3) -#if HWY_STATIC_TARGET != HWY_AVX3 -#error "Highway did not select HWY_AVX3 for the AVX-512 build. Check compiler flags, compiler support, and Highway blocklists." -#endif +# if HWY_STATIC_TARGET != HWY_AVX3 +# error "Highway did not select HWY_AVX3 for the AVX-512 build. Check compiler flags, compiler support, and Highway blocklists." +# endif #elif defined(JV_REQUIRE_HWY_AVX2) -#if HWY_STATIC_TARGET != HWY_AVX2 -#error "Highway did not select HWY_AVX2 for the AVX2 build. Check compiler flags, compiler support, and Highway blocklists." -#endif -#endif // -#endif // __X86_64__ +# if HWY_STATIC_TARGET != HWY_AVX2 +# error "Highway did not select HWY_AVX2 for the AVX2 build. Check compiler flags, compiler support, and Highway blocklists." +# endif +#endif // JV_REQUIRE_HWY_* + +#elif JV_ARCH_AARCH64 + +// Each tier is compiled with a fixed -march= flag that pins HWY_STATIC_TARGET +// to exactly one Highway constant — assert that precisely, matching the x86 +// assertions above. +// neon: -march=armv8-a+crypto → HWY_NEON (no BF16/dotprod/I8MM, so never HWY_NEON_BF16) +// sve: -march=armv8.4-a+sve → HWY_SVE (no fixed-width hint, so never HWY_SVE_256) +// sve2: -march=armv9-a+sve2 → HWY_SVE2 (no fixed-width hint, so never HWY_SVE2_128) +#if defined(JV_REQUIRE_HWY_SVE2) +# if HWY_STATIC_TARGET != HWY_SVE2 +# error "Highway did not select HWY_SVE2 for the SVE2 build. Check compiler flags (-march=armv9-a+sve2), compiler support (GCC >= 10 or Clang >= 22), and Highway blocklists." +# endif +#elif defined(JV_REQUIRE_HWY_SVE) +# if HWY_STATIC_TARGET != HWY_SVE +# error "Highway did not select HWY_SVE for the SVE build. Check compiler flags (-march=armv8.4-a+sve), compiler support (GCC >= 10 or Clang >= 9), and Highway blocklists." +# endif +#elif defined(JV_REQUIRE_HWY_NEON) +# if HWY_STATIC_TARGET != HWY_NEON +# error "Highway did not select HWY_NEON for the NEON build. Check compiler flags (-march=armv8-a+crypto) and Highway blocklists." +# endif +#endif // JV_REQUIRE_HWY_* + +#endif // JV_ARCH_X86_64 / JV_ARCH_AARCH64 diff --git a/jvector-native/src/main/native/src/jvector_simd.cpp b/jvector-native/src/main/native/src/jvector_simd.cpp index fcd6037d4..e1600c40e 100644 --- a/jvector-native/src/main/native/src/jvector_simd.cpp +++ b/jvector-native/src/main/native/src/jvector_simd.cpp @@ -49,7 +49,7 @@ static_assert( && (int)MaxIsa::AVX3_DL < (int)MaxIsa::AVX3_SPR && (int)MaxIsa::AVX3_SPR < (int)MaxIsa::Unset, "MaxIsa values must be in strict ascending capability order with Unset at the top"); -#else // JV_ARCH_AARCH64 +#elif JV_ARCH_AARCH64 enum class MaxIsa { NEON = 0, SVE = 1, SVE2 = 2, Unset = INT_MAX }; static_assert( @@ -74,7 +74,7 @@ static MaxIsa read_max_isa() noexcept if (std::strcmp(val, "avx3") == 0) return MaxIsa::AVX3; if (std::strcmp(val, "avx2") == 0) return MaxIsa::AVX2; if (std::strcmp(val, "sse42") == 0) return MaxIsa::SSE42; -#else // JV_ARCH_AARCH64 +#elif JV_ARCH_AARCH64 if (std::strcmp(val, "sve2") == 0) return MaxIsa::SVE2; if (std::strcmp(val, "sve") == 0) return MaxIsa::SVE; if (std::strcmp(val, "neon") == 0) return MaxIsa::NEON; @@ -134,7 +134,7 @@ static const KernelVTable SSE42_vtable = { }; #undef KERNEL_ENTRY -#else // JV_ARCH_AARCH64 +#elif JV_ARCH_AARCH64 #define KERNEL_ENTRY(ret_type, name, params, names) NEON::name, static const KernelVTable NEON_vtable = { @@ -202,7 +202,7 @@ static DispatchResult dispatch_kernels() noexcept return { AVX2_vtable, MaxIsa::AVX2, env_str }; // SSE42 is the x86-64 baseline — assumed always present, no CPUID check. return { SSE42_vtable, MaxIsa::SSE42, env_str }; -#else // JV_ARCH_AARCH64 +#elif JV_ARCH_AARCH64 if (max_isa == MaxIsa::SVE2) env_str = "sve2"; else if (max_isa == MaxIsa::SVE) env_str = "sve"; else if (max_isa == MaxIsa::NEON) env_str = "neon"; @@ -256,7 +256,7 @@ const char *jvector_simd_get_active_isa() case MaxIsa::AVX3: return "avx3"; case MaxIsa::AVX2: return "avx2"; case MaxIsa::SSE42: return "sse42"; -#else // JV_ARCH_AARCH64 +#elif JV_ARCH_AARCH64 case MaxIsa::SVE2: return "sve2"; case MaxIsa::SVE: return "sve"; case MaxIsa::NEON: return "neon"; diff --git a/jvector-native/src/main/native/src/jvector_simd_kernels.h b/jvector-native/src/main/native/src/jvector_simd_kernels.h index 65862d614..2d80724c0 100644 --- a/jvector-native/src/main/native/src/jvector_simd_kernels.h +++ b/jvector-native/src/main/native/src/jvector_simd_kernels.h @@ -44,7 +44,7 @@ DECLARE_SIMD_KERNELS(AVX3_DL) DECLARE_SIMD_KERNELS(AVX3) DECLARE_SIMD_KERNELS(AVX2) DECLARE_SIMD_KERNELS(SSE42) -#else // JV_ARCH_AARCH64 +#elif JV_ARCH_AARCH64 // AArch64 ISA namespaces (NEON baseline → SVE → SVE2) DECLARE_SIMD_KERNELS(NEON) DECLARE_SIMD_KERNELS(SVE) From b7f2dba639f210c9f17aa731c64a47b70f797536 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 06:15:58 +0000 Subject: [PATCH 04/14] native: add AArch64 CPU feature detection to jvector_cpu_features.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend CpuFeature enum with AArch64 tier flags (guarded by #if JV_ARCH_X86_64 / #elif JV_ARCH_AARCH64 / #endif): NEON = 200 — NEON + AES, baseline for all AArch64 CPUs SVE = 201 — Scalable Vector Extension (Graviton 3+, Linux only) SVE2 = 202 — SVE2 + SVE2-AES (Graviton 4+, Linux only) Scope the arch-specific includes the same way: x86-64: or AArch64: on Apple; + on Linux Fallback macros for HWCAP_SVE, HWCAP2_SVE2, HWCAP2_SVEAES copied from highway/hwy/targets.cc for older sysroots. populate_cpu_features() AArch64 branch: Linux: getauxval(AT_HWCAP) → NEON (HWCAP_AES), SVE (HWCAP_SVE) getauxval(AT_HWCAP2) → SVE2 (HWCAP2_SVE2 | HWCAP2_SVEAES) macOS: sysctlbyname("hw.optional.arm.FEAT_AES") → NEON SVE/SVE2 left false (no Apple Silicon through M4 has SVE) --- .../main/native/src/jvector_cpu_features.h | 91 ++++++++++++++++--- 1 file changed, 80 insertions(+), 11 deletions(-) diff --git a/jvector-native/src/main/native/src/jvector_cpu_features.h b/jvector-native/src/main/native/src/jvector_cpu_features.h index 27ef3f13e..708042000 100644 --- a/jvector-native/src/main/native/src/jvector_cpu_features.h +++ b/jvector-native/src/main/native/src/jvector_cpu_features.h @@ -19,18 +19,42 @@ #include #include +#include "jvector_arch.h" -#if defined(_MSC_VER) -#include -#elif defined(__GNUC__) || defined(__clang__) -#include +#if JV_ARCH_X86_64 +# if defined(_MSC_VER) +# include +# elif defined(__GNUC__) || defined(__clang__) +# include +# endif +#elif JV_ARCH_AARCH64 +# if defined(__APPLE__) +# include +# else +# include +# include + // Older sysroots may not define these; provide fallbacks matching + // the pattern in highway/hwy/targets.cc. +# ifndef HWCAP_SVE +# define HWCAP_SVE (1 << 22) +# endif +# ifndef HWCAP2_SVE2 +# define HWCAP2_SVE2 (1 << 1) +# endif +# ifndef HWCAP2_SVEAES +# define HWCAP2_SVEAES (1 << 2) +# endif +# endif #endif // Features needed by the ISA dispatch table. Extend as new targets are added. // -// ICX = Intel Ice Lake-SP (Xeon Scalable 3rd Gen) -// SPR = Intel Sapphire Rapids (Xeon Scalable 4th Gen) +// x86-64: +// ICX = Intel Ice Lake-SP (Xeon Scalable 3rd Gen) +// SPR = Intel Sapphire Rapids (Xeon Scalable 4th Gen) +// AArch64 tier flags start at 200 to stay well clear of the x86 entries. enum class CpuFeature : uint32_t { +#if JV_ARCH_X86_64 // ---- Base AVX2 / AVX-512 foundation (all SKUs) ---------------------- AVX2 = 0, AVX512F = 1, @@ -59,19 +83,31 @@ enum class CpuFeature : uint32_t { AVX3_DL = 101, // AVX3 + VNNI + VBMI + VBMI2 + IFMA + BITALG + VPOPCNTDQ // + GFNI + VAES + VPCLMULQDQ (Ice Lake) AVX3_SPR = 102, // AVX3_DL + AVX512_FP16 (Sapphire Rapids) +#elif JV_ARCH_AARCH64 + // ---- AArch64 ISA tier flags ----------------------------------------- + // Numbered from 200 to stay clear of the x86 entries above. + NEON = 200, // baseline AArch64 NEON + AES (AT_HWCAP: HWCAP_AES, or always + // true on Apple where all CPUs support AES) + SVE = 201, // Scalable Vector Extension (AT_HWCAP: HWCAP_SVE). + // Never set on Apple Silicon (no SVE through M4/A18). + SVE2 = 202, // SVE2 + SVE2-AES (AT_HWCAP2: HWCAP2_SVE2 | HWCAP2_SVEAES). + // Never set on Apple Silicon. +#endif COUNT }; -// Populate `features` by issuing CPUID and XGETBV. -// All entries are false on non-x86 architectures. +// Populate `features` by probing CPU capabilities: +// x86-64: CPUID + XGETBV +// AArch64: getauxval(AT_HWCAP/AT_HWCAP2) on Linux; sysctlbyname on macOS +// All entries default to false; only the flags for the current architecture +// are ever set to true. inline void populate_cpu_features(std::array(CpuFeature::COUNT)> &features) noexcept { features.fill(false); -#if defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) \ - || defined(_M_X64) +#if JV_ARCH_X86_64 // Portable CPUID: GCC/Clang use ; MSVC uses . auto run_cpuid = [](uint32_t leaf, @@ -192,7 +228,40 @@ populate_cpu_features(std::array(CpuFeature::COUNT)> features[static_cast(CpuFeature::AVX3_SPR)] = f(CpuFeature::AVX3_DL) && f(CpuFeature::AVX512_FP16); -#endif // x86 / x86_64 +#elif JV_ARCH_AARCH64 + +#if defined(__APPLE__) + // macOS: use sysctlbyname for capability queries. + // NEON (with AES) — present on every shipping Apple Silicon through M4. + { + int val = 0; size_t len = sizeof(val); + if (sysctlbyname("hw.optional.arm.FEAT_AES", &val, &len, nullptr, 0) == 0 && val) + features[static_cast(CpuFeature::NEON)] = true; + } + // SVE and SVE2 are never available on Apple Silicon; leave them false. + +#else // Linux AArch64 + { + const unsigned long hw = getauxval(AT_HWCAP); + + // NEON: all AArch64 CPUs have NEON; require AES too (matches HWY_NEON). +#if defined(HWCAP_AES) + if (hw & HWCAP_AES) + features[static_cast(CpuFeature::NEON)] = true; +#endif + + // SVE: Graviton 3 / Neoverse V1 and later. + if (hw & HWCAP_SVE) + features[static_cast(CpuFeature::SVE)] = true; + + // SVE2: requires SVE2 *and* SVE2-AES (matches HWY_SVE2 requirement). + const unsigned long hw2 = getauxval(AT_HWCAP2); + if ((hw2 & (HWCAP2_SVE2 | HWCAP2_SVEAES)) == (HWCAP2_SVE2 | HWCAP2_SVEAES)) + features[static_cast(CpuFeature::SVE2)] = true; + } +#endif // __APPLE__ + +#endif // JV_ARCH_X86_64 / JV_ARCH_AARCH64 } #endif // CPU_FEATURES_H From 8e97babd2abb4cc93c1c8e3c8854934b8bf2e6bd Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 06:17:52 +0000 Subject: [PATCH 05/14] native: allow aarch64 in NativeVectorizationProvider architecture check Extend the os.arch guard to accept 'aarch64' alongside 'amd64'/'x86_64' so the Java layer can load and use libjvector.so on Graviton and Apple Silicon once the library is present. --- .../jbellis/jvector/vector/NativeVectorizationProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jvector-native/src/main/java/io/github/jbellis/jvector/vector/NativeVectorizationProvider.java b/jvector-native/src/main/java/io/github/jbellis/jvector/vector/NativeVectorizationProvider.java index f73f237a6..213e53f9a 100644 --- a/jvector-native/src/main/java/io/github/jbellis/jvector/vector/NativeVectorizationProvider.java +++ b/jvector-native/src/main/java/io/github/jbellis/jvector/vector/NativeVectorizationProvider.java @@ -31,8 +31,8 @@ public class NativeVectorizationProvider extends VectorizationProvider { public NativeVectorizationProvider() { var arch = System.getProperty("os.arch", ""); - if (!arch.equals("amd64") && !arch.equals("x86_64")) { - throw new UnsupportedOperationException("Native SIMD operations are only supported on x86_64."); + if (!arch.equals("amd64") && !arch.equals("x86_64") && !arch.equals("aarch64")) { + throw new UnsupportedOperationException("Native SIMD operations are only supported on x86_64 and aarch64."); } var libraryLoaded = LibraryLoader.loadJvector(); if (!libraryLoaded) { From 020b50ec236884eb529abd008a581df1a23e0f68 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 07:42:50 +0000 Subject: [PATCH 06/14] tests: add AArch64 ISA tier coverage (Sub-Task 7) - Add test_aarch64_cpu_features.cpp: validates populate_cpu_features() detects NEON/SVE/SVE2 correctly, checks dispatcher does not exceed hardware capability, and verifies JVECTOR_MAX_ISA=neon cap is honoured. Uses /proc/cpuinfo 'Features' line as ground truth on Linux, and sysctlbyname on macOS. - meson.build: route cpu-features test source to the arch-specific file (test_x86_cpu_features.cpp on x86_64, test_aarch64_cpu_features.cpp on aarch64). - test_x86_cpu_features.cpp: renamed from test_cpu_features.cpp, no functional change. - test_helpers.h / test_helpers.cpp: extract parse_cpuinfo_line() and make_vec() as shared helpers used by both arch-specific test files. - test_similarity.cpp (IsaDispatch/MaxIsaEnvHonoured): replace the hardcoded x86-only tier array with an #ifdef __aarch64__ guard that swaps in {"neon", "sve", "sve2"} on ARM and keeps the existing x86 list on x86_64. - test_elementwise.cpp: add IsaDispatch/ActiveIsaIsKnownTier test that confirms the active ISA name is in the architecture-appropriate tier list on both x86_64 and aarch64. --- jvector-native/src/main/native/meson.build | 7 +- .../tests/test_aarch64_cpu_features.cpp | 189 ++++++++++++++++++ .../main/native/tests/test_elementwise.cpp | 23 +++ .../src/main/native/tests/test_helpers.cpp | 22 ++ .../src/main/native/tests/test_helpers.h | 16 ++ .../src/main/native/tests/test_similarity.cpp | 8 +- ...features.cpp => test_x86_cpu_features.cpp} | 43 +--- 7 files changed, 269 insertions(+), 39 deletions(-) create mode 100644 jvector-native/src/main/native/tests/test_aarch64_cpu_features.cpp rename jvector-native/src/main/native/tests/{test_cpu_features.cpp => test_x86_cpu_features.cpp} (85%) diff --git a/jvector-native/src/main/native/meson.build b/jvector-native/src/main/native/meson.build index 916712b85..04d641d59 100644 --- a/jvector-native/src/main/native/meson.build +++ b/jvector-native/src/main/native/meson.build @@ -165,13 +165,18 @@ vectorutil_dep = declare_dependency( gtest_dep = dependency('gtest_main', required: false) if gtest_dep.found() + # Select the architecture-specific CPU features test. + cpu_features_test_src = cpu == 'x86_64' \ + ? 'tests/test_x86_cpu_features.cpp' \ + : 'tests/test_aarch64_cpu_features.cpp' + simd_kernels_test = executable( 'test_simd_kernels', sources : [ 'tests/test_helpers.cpp', 'tests/test_similarity.cpp', 'tests/test_elementwise.cpp', - 'tests/test_cpu_features.cpp', + cpu_features_test_src, ], dependencies: [vectorutil_dep, gtest_dep], ) diff --git a/jvector-native/src/main/native/tests/test_aarch64_cpu_features.cpp b/jvector-native/src/main/native/tests/test_aarch64_cpu_features.cpp new file mode 100644 index 000000000..7fe50eb90 --- /dev/null +++ b/jvector-native/src/main/native/tests/test_aarch64_cpu_features.cpp @@ -0,0 +1,189 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Validates that the native dispatcher selects the correct AArch64 ISA tier. +// +// Ground truth on Linux: /proc/cpuinfo "Features" line — the kernel only +// exposes a feature token when the OS has set up the necessary context-switch +// support, so this is the same authority as getauxval(AT_HWCAP). +// Relevant tokens: "aes" (→ NEON), "sve" (→ SVE), "sve2" + "sveaes" (→ SVE2). +// +// Ground truth on macOS: sysctlbyname("hw.optional.arm.FEAT_AES") for NEON. +// SVE and SVE2 are never available on any Apple Silicon through M4/A18. + +#include "test_helpers.h" + +#include +#include +#include +#include +#include + +#if defined(__APPLE__) +# include +#endif + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Tier names in ascending capability order for AArch64. +static const std::vector kIsaTiers = { "neon", "sve", "sve2" }; + +static int tier_index(const std::string& name) +{ + auto it = std::find(kIsaTiers.begin(), kIsaTiers.end(), name); + return (it == kIsaTiers.end()) ? -1 : static_cast(it - kIsaTiers.begin()); +} + +// Compute the expected ISA tier from the parsed feature set and any cap. +// Mirrors the logic in populate_cpu_features() / dispatch_kernels(). +static std::string expected_isa(const std::unordered_set& f, + const std::string& cap) +{ + std::string best; + // SVE2: requires both "sve2" and "sveaes" (matches HWCAP2_SVE2 | HWCAP2_SVEAES). + if (f.count("sve2") && f.count("sveaes")) best = "sve2"; + else if (f.count("sve")) best = "sve"; + else best = "neon"; + + if (!cap.empty() && tier_index(cap) < tier_index(best)) + return cap; + return best; +} + +// On macOS, detect NEON capability via sysctlbyname (no /proc/cpuinfo). +// SVE/SVE2 are never present on Apple Silicon, so "neon" is always the result. +#if defined(__APPLE__) +static std::string detect_host_isa_apple() +{ + int val = 0; size_t len = sizeof(val); + if (sysctlbyname("hw.optional.arm.FEAT_AES", &val, &len, nullptr, 0) == 0 && val) + return "neon"; + return "neon"; // baseline — every AArch64 Apple CPU has NEON +} +#endif + +// --------------------------------------------------------------------------- +// Fixture +// --------------------------------------------------------------------------- + +class AArch64CpuFeaturesTest : public ::testing::Test +{ +protected: + static void SetUpTestSuite() + { + const char* active = jvector_simd_get_active_isa(); + const char* cap_c = jvector_simd_get_max_isa_env(); + s_active = active ? active : ""; + s_cap = cap_c ? cap_c : ""; + +#if defined(__APPLE__) + s_host_isa = detect_host_isa_apple(); + s_available = true; + std::printf("[ CPU ] active_isa=%s JVECTOR_MAX_ISA=%s host_isa=%s (macOS sysctl)\n", + s_active.c_str(), + s_cap.empty() ? "(unset)" : s_cap.c_str(), + s_host_isa.c_str()); +#else + s_cpuinfo = parse_cpuinfo_line("Features"); + s_available = !s_cpuinfo.empty(); + + if (s_available) { + s_host_isa = expected_isa(s_cpuinfo, ""); // uncapped hardware capability + // Collect feature string for diagnostics. + s_feature_str = std::accumulate( + s_cpuinfo.begin(), s_cpuinfo.end(), std::string{}, + [](const std::string& a, const std::string& b) { + return a.empty() ? b : a + " " + b; + }); + } + std::printf("[ CPU ] active_isa=%s JVECTOR_MAX_ISA=%s host_isa=%s " + "cpuinfo_features=%zu\n", + s_active.c_str(), + s_cap.empty() ? "(unset)" : s_cap.c_str(), + s_host_isa.c_str(), + s_cpuinfo.size()); +#endif + } + + static std::string s_active; + static std::string s_cap; + static std::string s_host_isa; + static bool s_available; + // Linux only: + static std::unordered_set s_cpuinfo; + static std::string s_feature_str; +}; + +std::string AArch64CpuFeaturesTest::s_active; +std::string AArch64CpuFeaturesTest::s_cap; +std::string AArch64CpuFeaturesTest::s_host_isa; +bool AArch64CpuFeaturesTest::s_available = false; +std::unordered_set AArch64CpuFeaturesTest::s_cpuinfo; +std::string AArch64CpuFeaturesTest::s_feature_str; + +#define SKIP_IF_UNAVAILABLE() \ + do { if (!s_available) GTEST_SKIP() << "/proc/cpuinfo unavailable"; } while (0) + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +// The active ISA must be one of the three valid AArch64 tier names. +TEST_F(AArch64CpuFeaturesTest, ActiveIsaIsValidTier) +{ + EXPECT_GE(tier_index(s_active), 0) + << "active_isa '" << s_active << "' is not a valid AArch64 tier " + << "(expected one of: neon, sve, sve2)"; +} + +// NEON is always available on any AArch64 CPU. +TEST_F(AArch64CpuFeaturesTest, NeonAlwaysAvailable) +{ + SKIP_IF_UNAVAILABLE(); + EXPECT_GE(tier_index(s_host_isa), tier_index("neon")) + << "Expected at least NEON on any AArch64 CPU"; +} + +// The dispatcher must not select a tier higher than the hardware supports. +TEST_F(AArch64CpuFeaturesTest, DispatcherDoesNotExceedHardware) +{ + SKIP_IF_UNAVAILABLE(); + EXPECT_LE(tier_index(s_active), tier_index(s_host_isa)) + << "Dispatcher selected '" << s_active + << "' but host only supports up to '" << s_host_isa << "'" + << "\nCPU features: " << s_feature_str; +} + +// End-to-end: the tier the dispatcher chose must match what /proc/cpuinfo implies. +TEST_F(AArch64CpuFeaturesTest, DispatcherMatchesCpuInfo) +{ + SKIP_IF_UNAVAILABLE(); + std::string exp = expected_isa(s_cpuinfo, s_cap); + EXPECT_EQ(s_active, exp) + << "Dispatcher chose '" << s_active + << "' but /proc/cpuinfo implies '" << exp << "'." + << "\nCPU features: " << s_feature_str; +} + +// When capped to "neon", the dispatcher must select the baseline tier. +TEST_F(AArch64CpuFeaturesTest, NeonCapForcesFallback) +{ + if (s_cap != "neon") GTEST_SKIP() << "JVECTOR_MAX_ISA != neon; skipping"; + EXPECT_EQ(s_active, "neon") + << "Expected 'neon' when JVECTOR_MAX_ISA=neon, got: " << s_active; +} diff --git a/jvector-native/src/main/native/tests/test_elementwise.cpp b/jvector-native/src/main/native/tests/test_elementwise.cpp index 04c836461..82dab16f4 100644 --- a/jvector-native/src/main/native/tests/test_elementwise.cpp +++ b/jvector-native/src/main/native/tests/test_elementwise.cpp @@ -191,3 +191,26 @@ INSTANTIATE_TEST_SUITE_P( [](const ::testing::TestParamInfo& info) { return info.param.description; }); + +// --------------------------------------------------------------------------- +// ISA-tier sanity test: active ISA must be a recognised tier name +// --------------------------------------------------------------------------- + +TEST(IsaDispatch, ActiveIsaIsKnownTier) +{ + const char* active = jvector_simd_get_active_isa(); + ASSERT_NE(active, nullptr) << "jvector_simd_get_active_isa() returned null"; + +#if defined(__aarch64__) + static const char* kOrder[] = {"neon", "sve", "sve2"}; + constexpr int kOrderLen = 3; +#else + static const char* kOrder[] = {"sse42", "avx2", "avx3", "avx3_dl", "avx3_spr"}; + constexpr int kOrderLen = 5; +#endif + bool found = false; + for (int i = 0; i < kOrderLen; ++i) + if (std::strcmp(kOrder[i], active) == 0) { found = true; break; } + + EXPECT_TRUE(found) << "Active ISA '" << active << "' is not a recognised tier"; +} diff --git a/jvector-native/src/main/native/tests/test_helpers.cpp b/jvector-native/src/main/native/tests/test_helpers.cpp index 957e42947..654e86734 100644 --- a/jvector-native/src/main/native/tests/test_helpers.cpp +++ b/jvector-native/src/main/native/tests/test_helpers.cpp @@ -16,6 +16,9 @@ #include "test_helpers.h" +#include +#include + // --------------------------------------------------------------------------- // Global test environment — prints the active ISA once for the whole binary. // Registered via AddGlobalTestEnvironment at static-init time so it fires @@ -75,6 +78,25 @@ const std::vector kKernelTestParams = { {255, "large_odd_tail_15"}, }; +std::unordered_set parse_cpuinfo_line(const std::string& key) +{ + std::unordered_set tokens; + std::ifstream f("/proc/cpuinfo"); + if (!f.is_open()) return tokens; + + std::string line; + while (std::getline(f, line)) { + if (line.rfind(key, 0) != 0) continue; + auto colon = line.find(':'); + if (colon == std::string::npos) continue; + std::istringstream iss(line.substr(colon + 1)); + std::string token; + while (iss >> token) tokens.insert(token); + break; + } + return tokens; +} + std::vector make_vec(size_t n, float seed) { std::vector v(n); diff --git a/jvector-native/src/main/native/tests/test_helpers.h b/jvector-native/src/main/native/tests/test_helpers.h index a48ea47cf..8127cd7b3 100644 --- a/jvector-native/src/main/native/tests/test_helpers.h +++ b/jvector-native/src/main/native/tests/test_helpers.h @@ -25,10 +25,26 @@ #include #include #include +#include #include #include "jvector_simd.h" +// --------------------------------------------------------------------------- +// /proc/cpuinfo helpers. +// +// parse_cpuinfo_line(key) reads /proc/cpuinfo, finds the first line that +// starts with `key`, and returns all whitespace-separated tokens after the +// colon as a set. Returns an empty set when the file is unavailable +// (e.g. macOS) or the key is not found. +// +// Usage: +// x86-64: parse_cpuinfo_line("flags") → {"avx2", "avx512f", ...} +// AArch64: parse_cpuinfo_line("Features") → {"aes", "sve", "sve2", ...} +// --------------------------------------------------------------------------- + +std::unordered_set parse_cpuinfo_line(const std::string& key); + // --------------------------------------------------------------------------- // Deterministic test vectors. // make_vec(n, seed) produces n floats with a mix of signs and magnitudes diff --git a/jvector-native/src/main/native/tests/test_similarity.cpp b/jvector-native/src/main/native/tests/test_similarity.cpp index c08947e4f..bcde00b0b 100644 --- a/jvector-native/src/main/native/tests/test_similarity.cpp +++ b/jvector-native/src/main/native/tests/test_similarity.cpp @@ -246,9 +246,15 @@ TEST(IsaDispatch, MaxIsaEnvHonoured) } // Tiers ordered by capability (ascending index = lower capability). +#if defined(__aarch64__) + static const char* kOrder[] = {"neon", "sve", "sve2"}; + constexpr int kOrderLen = 3; +#else static const char* kOrder[] = {"sse42", "avx2", "avx3", "avx3_dl", "avx3_spr"}; + constexpr int kOrderLen = 5; +#endif auto tier_idx = [](const char* name) -> int { - for (int i = 0; i < 5; ++i) + for (int i = 0; i < kOrderLen; ++i) if (std::strcmp(kOrder[i], name) == 0) return i; return -1; }; diff --git a/jvector-native/src/main/native/tests/test_cpu_features.cpp b/jvector-native/src/main/native/tests/test_x86_cpu_features.cpp similarity index 85% rename from jvector-native/src/main/native/tests/test_cpu_features.cpp rename to jvector-native/src/main/native/tests/test_x86_cpu_features.cpp index a921660b6..995fd451e 100644 --- a/jvector-native/src/main/native/tests/test_cpu_features.cpp +++ b/jvector-native/src/main/native/tests/test_x86_cpu_features.cpp @@ -14,52 +14,21 @@ * limitations under the License. */ -// Validates that the native dispatcher selects the ISA tier that matches the -// CPU capabilities reported in /proc/cpuinfo, respecting any JVECTOR_MAX_ISA cap. +// Validates that the native dispatcher selects the correct x86-64 ISA tier, +// using /proc/cpuinfo as ground truth. Mirrors DispatcherCpuFlagsTest.java. // -// Logic mirrors DispatcherCpuFlagsTest.java and the C implementation in -// jvector_cpu_features.h / jvector_simd.cpp exactly. -// -// /proc/cpuinfo is the authoritative ground-truth: the kernel only exposes a -// flag when the OS context-switch support (XCR0) is also in place, so checking -// it is equivalent to checking CPUID + XCR0 together. +// /proc/cpuinfo is authoritative: the kernel only exposes a flag when the OS +// context-switch support (XCR0) is also in place, so checking it is equivalent +// to checking CPUID + XCR0 together. #include "test_helpers.h" #include -#include -#include #include -#include #include #include #include -// --------------------------------------------------------------------------- -// /proc/cpuinfo helpers — mirrors DispatcherCpuFlagsTest.java -// --------------------------------------------------------------------------- - -// Parse the flags line from the first processor entry in /proc/cpuinfo. -// Returns an empty set if unavailable (non-Linux, non-x86, or unreadable). -static std::unordered_set parse_cpuinfo_flags() -{ - std::unordered_set flags; - std::ifstream f("/proc/cpuinfo"); - if (!f.is_open()) return flags; - - std::string line; - while (std::getline(f, line)) { - if (line.rfind("flags", 0) != 0) continue; - auto colon = line.find(':'); - if (colon == std::string::npos) continue; - std::istringstream iss(line.substr(colon + 1)); - std::string token; - while (iss >> token) flags.insert(token); - break; - } - return flags; -} - // Tier names in ascending capability order — index is ordinal (mirrors Java). static const std::vector kIsaTiers = { "sse42", "avx2", "avx3", "avx3_dl", "avx3_spr" @@ -128,7 +97,7 @@ class CpuFeaturesTest : public ::testing::Test protected: static void SetUpTestSuite() { - s_flags = parse_cpuinfo_flags(); + s_flags = parse_cpuinfo_line("flags"); const char* active = jvector_simd_get_active_isa(); const char* cap_c = jvector_simd_get_max_isa_env(); From 369356fc188f3b3826666a3be8f4ff245ebda7c4 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 08:49:39 +0000 Subject: [PATCH 07/14] aarch64: target fixed-width HWY_SVE_256 and HWY_SVE2_128 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three AArch64 tiers are now pinned to fixed-width Highway targets: NEON : -march=armv8-a+crypto → HWY_NEON SVE : -march=armv8.4-a+sve -msve-vector-bits=256 → HWY_SVE_256 (MaxLanes=8) SVE2 : -march=armv9-a+sve2 -msve-vector-bits=128 → HWY_SVE2_128 (MaxLanes=4) Using fixed-width targets gives Highway a concrete compile-time MaxLanes, so all calculate_partial_sums fast-paths (Shuffle2301, Shuffle1032, SwapAdjacentBlocks, LoadDup256) work identically to x86 without any HWY_HAVE_SCALABLE guards or workarounds. meson.build: add -msve-vector-bits=128 to sve2 args; update comments. assert_hwy_targets.h: assert HWY_SVE2_128 (was HWY_SVE2) for the SVE2 tier. jvector_simd_kernels.cpp: remove the #if !HWY_HAVE_SCALABLE guards that were added as a temporary workaround; no longer needed. jvector_simd.cpp: update dispatch comment to name the fixed-width targets. --- jvector-native/src/main/native/meson.build | 29 +++++++++++++------ .../src/main/native/src/assert_hwy_targets.h | 22 +++++++------- .../src/main/native/src/jvector_simd.cpp | 4 ++- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/jvector-native/src/main/native/meson.build b/jvector-native/src/main/native/meson.build index 04d641d59..214daf8e2 100644 --- a/jvector-native/src/main/native/meson.build +++ b/jvector-native/src/main/native/meson.build @@ -53,13 +53,24 @@ if cpu == 'x86_64' }, ] elif cpu == 'aarch64' - # Three tiers in ascending capability order: - # NEON — baseline AArch64 (all Graviton, all Apple Silicon) - # SVE — Graviton 3 / Neoverse V1 (256-bit SVE). - # Note: Highway treats SVE as broken on HWY_OS_APPLE; no Apple CPU - # through M4/A18 implements SVE, so this tier is Graviton-only. - # SVE2 — Graviton 4 / Neoverse V2/N2. - # Requires GCC >= 10 or Clang >= 22. Also Graviton-only (no Apple). + # Three tiers in ascending capability order, all compiled to fixed-width + # Highway targets so MaxLanes is a compile-time constant and all kernel + # fast-paths (Shuffle*, LoadDup256, etc.) work without ISA-specific guards. + # + # NEON — baseline AArch64. Highway target: HWY_NEON. + # All Graviton generations and all Apple Silicon. + # + # SVE_256 — Graviton 3 / Neoverse V1. Highway target: HWY_SVE_256. + # -msve-vector-bits=256 asserts the physical vector width and + # causes Highway to select the fixed-width HWY_SVE_256 target + # (MaxLanes = 8 floats). Graviton-only; no Apple Silicon SVE. + # + # SVE2_128 — Graviton 4 / Neoverse V2/N2. Highway target: HWY_SVE2_128. + # -msve-vector-bits=128 selects the fixed-width HWY_SVE2_128 + # target (MaxLanes = 4 floats). At runtime on Graviton 4 the + # hardware executes 256-bit vectors, so Highway's loop unrolling + # gives 256-bit throughput with 128-bit-wide loop bodies. + # Requires GCC >= 10 or Clang >= 21. Graviton-only (no Apple). isa_variants = [ { 'name' : 'neon', @@ -71,14 +82,14 @@ elif cpu == 'aarch64' { 'name' : 'sve', 'namespace': 'SVE', - 'args' : ['-march=armv8.4-a+sve', + 'args' : ['-march=armv8.4-a+sve', '-msve-vector-bits=256', '-DHWY_COMPILE_ONLY_STATIC', '-DJV_REQUIRE_HWY_SVE'], }, { 'name' : 'sve2', 'namespace': 'SVE2', - 'args' : ['-march=armv9-a+sve2', + 'args' : ['-march=armv9-a+sve2', '-msve-vector-bits=128', '-DHWY_COMPILE_ONLY_STATIC', '-DJV_REQUIRE_HWY_SVE2'], }, diff --git a/jvector-native/src/main/native/src/assert_hwy_targets.h b/jvector-native/src/main/native/src/assert_hwy_targets.h index 4687322bf..14eba4bdc 100644 --- a/jvector-native/src/main/native/src/assert_hwy_targets.h +++ b/jvector-native/src/main/native/src/assert_hwy_targets.h @@ -38,19 +38,21 @@ #elif JV_ARCH_AARCH64 -// Each tier is compiled with a fixed -march= flag that pins HWY_STATIC_TARGET -// to exactly one Highway constant — assert that precisely, matching the x86 -// assertions above. -// neon: -march=armv8-a+crypto → HWY_NEON (no BF16/dotprod/I8MM, so never HWY_NEON_BF16) -// sve: -march=armv8.4-a+sve → HWY_SVE (no fixed-width hint, so never HWY_SVE_256) -// sve2: -march=armv9-a+sve2 → HWY_SVE2 (no fixed-width hint, so never HWY_SVE2_128) +// Compiler flags per tier and the Highway target each must produce: +// neon: -march=armv8-a+crypto → HWY_NEON +// (no BF16/dotprod/I8MM features, so never HWY_NEON_BF16) +// sve: -march=armv8.4-a+sve -msve-vector-bits=256 → HWY_SVE_256 +// (pinned 256-bit; Graviton 3 / Neoverse V1) +// sve2: -march=armv9-a+sve2 -msve-vector-bits=128 → HWY_SVE2_128 +// (fixed-width 128-bit; Graviton 4 / Neoverse V2/N2) +// Requires GCC >= 10 or Clang >= 21. #if defined(JV_REQUIRE_HWY_SVE2) -# if HWY_STATIC_TARGET != HWY_SVE2 -# error "Highway did not select HWY_SVE2 for the SVE2 build. Check compiler flags (-march=armv9-a+sve2), compiler support (GCC >= 10 or Clang >= 22), and Highway blocklists." +# if HWY_STATIC_TARGET != HWY_SVE2_128 +# error "Highway did not select HWY_SVE2_128 for the SVE2 build. Check compiler flags (-march=armv9-a+sve2 -msve-vector-bits=128), compiler support (GCC >= 10 or Clang >= 21), and Highway blocklists." # endif #elif defined(JV_REQUIRE_HWY_SVE) -# if HWY_STATIC_TARGET != HWY_SVE -# error "Highway did not select HWY_SVE for the SVE build. Check compiler flags (-march=armv8.4-a+sve), compiler support (GCC >= 10 or Clang >= 9), and Highway blocklists." +# if HWY_STATIC_TARGET != HWY_SVE_256 +# error "Highway did not select HWY_SVE_256 for the SVE build. Check compiler flags (-march=armv8.4-a+sve -msve-vector-bits=256), compiler support (GCC >= 10 or Clang >= 9), and Highway blocklists." # endif #elif defined(JV_REQUIRE_HWY_NEON) # if HWY_STATIC_TARGET != HWY_NEON diff --git a/jvector-native/src/main/native/src/jvector_simd.cpp b/jvector-native/src/main/native/src/jvector_simd.cpp index e1600c40e..061048fc3 100644 --- a/jvector-native/src/main/native/src/jvector_simd.cpp +++ b/jvector-native/src/main/native/src/jvector_simd.cpp @@ -17,7 +17,9 @@ // Runtime SIMD dispatch: selects the best available ISA tier at startup. // x86-64 tiers (descending): AVX3_SPR, AVX3_DL, AVX3, AVX2, SSE42. // SSE42 is the x86-64 baseline — assumed always available, no CPUID check. -// AArch64 tiers (descending): SVE2, SVE, NEON. +// AArch64 tiers (descending): SVE2_128 (HWY_SVE2_128), SVE_256 (HWY_SVE_256), NEON. +// All three are fixed-width Highway targets: MaxLanes is a compile-time +// constant, so all kernel fast-paths work identically to x86. // NEON is the AArch64 baseline — always available on any aarch64 CPU. // Function pointers are resolved once at static-init time; each public call is // a single indirect branch. From 1fa5319f3bdb812d606352f956f2f4406fca069e Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 08:54:55 +0000 Subject: [PATCH 08/14] aarch64/sve2: add +i8mm+bf16 to march flag for HWY_SVE2_128 HWY_SVE2_128 requires i8mm and bf16 features in addition to sve2 (documented in highway/hwy/ops/set_macros-inl.h:662, issue #2973). Without them, GCC raises 'target specific option mismatch' because the per-function target attributes added by the SVE2_128 header include +i8mm+bf16 but the TU-level -march= did not enable them. Fix: change -march=armv9-a+sve2 to -march=armv9-a+sve2+i8mm+bf16. Update the assert_hwy_targets.h error message to document the full flag. --- jvector-native/src/main/native/meson.build | 5 ++++- jvector-native/src/main/native/src/assert_hwy_targets.h | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/jvector-native/src/main/native/meson.build b/jvector-native/src/main/native/meson.build index 214daf8e2..c3ca22b93 100644 --- a/jvector-native/src/main/native/meson.build +++ b/jvector-native/src/main/native/meson.build @@ -89,7 +89,10 @@ elif cpu == 'aarch64' { 'name' : 'sve2', 'namespace': 'SVE2', - 'args' : ['-march=armv9-a+sve2', '-msve-vector-bits=128', + # HWY_SVE2_128 requires i8mm and bf16 in addition to sve2. + # See highway/hwy/ops/set_macros-inl.h:662 ("SVE2_128 implies/requires + # I8MM and BF16, see #2973") and the HWY_TARGET_STR definition there. + 'args' : ['-march=armv9-a+sve2+i8mm+bf16', '-msve-vector-bits=128', '-DHWY_COMPILE_ONLY_STATIC', '-DJV_REQUIRE_HWY_SVE2'], }, diff --git a/jvector-native/src/main/native/src/assert_hwy_targets.h b/jvector-native/src/main/native/src/assert_hwy_targets.h index 14eba4bdc..23c07d619 100644 --- a/jvector-native/src/main/native/src/assert_hwy_targets.h +++ b/jvector-native/src/main/native/src/assert_hwy_targets.h @@ -48,7 +48,7 @@ // Requires GCC >= 10 or Clang >= 21. #if defined(JV_REQUIRE_HWY_SVE2) # if HWY_STATIC_TARGET != HWY_SVE2_128 -# error "Highway did not select HWY_SVE2_128 for the SVE2 build. Check compiler flags (-march=armv9-a+sve2 -msve-vector-bits=128), compiler support (GCC >= 10 or Clang >= 21), and Highway blocklists." +# error "Highway did not select HWY_SVE2_128 for the SVE2 build. Check compiler flags (-march=armv9-a+sve2+i8mm+bf16 -msve-vector-bits=128), compiler support (GCC >= 10 or Clang >= 21), and Highway blocklists." # endif #elif defined(JV_REQUIRE_HWY_SVE) # if HWY_STATIC_TARGET != HWY_SVE_256 From de9a74946be2073581375094cd73432bfa5e1555 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 10:34:53 +0000 Subject: [PATCH 09/14] kernels: replace vector operator+/- with hn::Add/Sub for SVE portability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SVE vector types (svfloat32_t etc.) are compiler built-ins, not C++ structs, so operator+/-/* cannot be defined for them — unlike NEON and x86 which wrap their intrinsics in Vec128/Vec256 structs that do define these operators. This affects all SVE targets including the fixed-width HWY_SVE_256 and HWY_SVE2_128. Replace all vector operator- usages in L2SquareDistanceImpl with hn::Sub(), and the one operator+ in calculate_partial_sums_f32 with hn::Add(). hn::Add/Sub are portable across every Highway backend. --- .../src/main/native/src/jvector_simd_kernels.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/jvector-native/src/main/native/src/jvector_simd_kernels.cpp b/jvector-native/src/main/native/src/jvector_simd_kernels.cpp index f4e8c2453..00c00fca0 100644 --- a/jvector-native/src/main/native/src/jvector_simd_kernels.cpp +++ b/jvector-native/src/main/native/src/jvector_simd_kernels.cpp @@ -239,10 +239,10 @@ HWY_INLINE float L2SquareDistanceImpl(Tag tag, const float *a, const float *b, s auto acc2 = hn::Zero(tag), acc3 = hn::Zero(tag); size_t ii = 0; for (; ii + 4 * lanes <= size; ii += 4 * lanes) { - auto d0 = hn::LoadU(tag, a + ii + 0*lanes) - hn::LoadU(tag, b + ii + 0*lanes); - auto d1 = hn::LoadU(tag, a + ii + 1*lanes) - hn::LoadU(tag, b + ii + 1*lanes); - auto d2 = hn::LoadU(tag, a + ii + 2*lanes) - hn::LoadU(tag, b + ii + 2*lanes); - auto d3 = hn::LoadU(tag, a + ii + 3*lanes) - hn::LoadU(tag, b + ii + 3*lanes); + auto d0 = hn::Sub(hn::LoadU(tag, a + ii + 0*lanes), hn::LoadU(tag, b + ii + 0*lanes)); + auto d1 = hn::Sub(hn::LoadU(tag, a + ii + 1*lanes), hn::LoadU(tag, b + ii + 1*lanes)); + auto d2 = hn::Sub(hn::LoadU(tag, a + ii + 2*lanes), hn::LoadU(tag, b + ii + 2*lanes)); + auto d3 = hn::Sub(hn::LoadU(tag, a + ii + 3*lanes), hn::LoadU(tag, b + ii + 3*lanes)); acc0 = hn::MulAdd(d0, d0, acc0); acc1 = hn::MulAdd(d1, d1, acc1); acc2 = hn::MulAdd(d2, d2, acc2); @@ -250,11 +250,11 @@ HWY_INLINE float L2SquareDistanceImpl(Tag tag, const float *a, const float *b, s } auto acc = hn::Add(hn::Add(acc0, acc1), hn::Add(acc2, acc3)); for (; ii + lanes <= size; ii += lanes) { - auto d = hn::LoadU(tag, a + ii) - hn::LoadU(tag, b + ii); + auto d = hn::Sub(hn::LoadU(tag, a + ii), hn::LoadU(tag, b + ii)); acc = hn::MulAdd(d, d, acc); } if (ii < size) { - auto d = hn::LoadN(tag, a + ii, size - ii) - hn::LoadN(tag, b + ii, size - ii); + auto d = hn::Sub(hn::LoadN(tag, a + ii, size - ii), hn::LoadN(tag, b + ii, size - ii)); acc = hn::MulAdd(d, d, acc); } return hn::ReduceSum(tag, acc); @@ -573,7 +573,7 @@ HWY_INLINE void calculate_partial_sums_f32(const float *HWY_RESTRICT codebook, hn::Vec score = partial_sum_score( centroidVec, queryVec); hn::Vec swapped = hn::Shuffle2301(score); - hn::Vec sum = score + swapped; + hn::Vec sum = hn::Add(score, swapped); hn::StoreU(sum, tag, tmp); #pragma GCC unroll 8 for (int jj = 0; jj < centroids_per_iter; ++jj) { From 8254d1a18863e4d4136f30bfdc008d812759703c Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 20 Aug 2026 10:39:52 +0000 Subject: [PATCH 10/14] kernels: add BroadcastDup128 to work around GCC 14 ICE with ld1rq+SVE_256 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hn::LoadDup128 emits the SVE ld1rq instruction, which triggers an internal compiler error (ICE in convert_move, expr.cc:301) in GCC 14 when compiled with -msve-vector-bits=256 (fixed-width SVE mode). Add BroadcastDup128(), a local helper that achieves the same broadcast using LoadU on a narrower tag + Combine, avoiding ld1rq entirely: D = 4 lanes → plain LoadU D = 8 lanes → LoadU(Half) + Combine D = 16 lanes → LoadU(Quarter) + Combine twice Replace both hn::LoadDup128 call sites in calculate_partial_sums_f32 (size==2 and size==4 fast-paths) with BroadcastDup128. The name is deliberately distinct from hn::LoadDup128 to avoid the ambiguous overload error that occurs on x86_512 targets where hn::LoadDup128 is also defined in the same namespace. --- .../main/native/src/jvector_simd_kernels.cpp | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/jvector-native/src/main/native/src/jvector_simd_kernels.cpp b/jvector-native/src/main/native/src/jvector_simd_kernels.cpp index 00c00fca0..17c9089b7 100644 --- a/jvector-native/src/main/native/src/jvector_simd_kernels.cpp +++ b/jvector-native/src/main/native/src/jvector_simd_kernels.cpp @@ -175,6 +175,38 @@ namespace hn = hwy::HWY_NAMESPACE; +// Loads 4 floats from ptr and broadcasts them to fill the full vector D. +// Uses LoadU + Combine instead of hn::LoadDup128 to avoid a GCC 14 internal +// compiler error (ICE in convert_move/expr.cc:301) triggered by hn::LoadDup128 +// (which emits ld1rq) when compiled with -msve-vector-bits=256. +// D = 4 lanes → plain LoadU (ptr holds exactly one full vector) +// D = 8 lanes → load 4-lane half, Combine to duplicate into both halves +// D = 16 lanes → load 4-lane half-of-half, Combine twice (4→8→16 lanes) +// hn::Quarter does not exist in this Highway version; two applications +// of hn::Half<> are used instead. +template +HWY_INLINE hn::Vec BroadcastDup128(D d, const float *HWY_RESTRICT ptr) +{ + static_assert(hn::MaxLanes(d) <= 16, + "BroadcastDup128 is not implemented for ISAs wider than 512-bit"); + if constexpr (hn::MaxLanes(d) > 8) { + // 16-lane (AVX-512): half-of-half = 4 lanes; combine up twice. + const hn::Half> dq; + const auto quarter = hn::LoadU(dq, ptr); + const hn::Half dh; + const auto half = hn::Combine(dh, quarter, quarter); + return hn::Combine(d, half, half); + } else if constexpr (hn::MaxLanes(d) > 4) { + // 8-lane (AVX2, SVE_256): load 4-lane half, combine to full. + const hn::Half dh; + const auto half = hn::LoadU(dh, ptr); + return hn::Combine(d, half, half); + } else { + // 4-lane (SSE4, NEON, SVE2_128): ptr holds exactly one full vector. + return hn::LoadU(d, ptr); + } +} + // Loads 8 floats from ptr and broadcasts them to fill the full vector D. // On ISAs where D is exactly 8 lanes (e.g. AVX2) this is a plain LoadU. // On wider ISAs (e.g. AVX-512, 16 lanes) the 8 floats are loaded into the @@ -561,7 +593,7 @@ HWY_INLINE void calculate_partial_sums_f32(const float *HWY_RESTRICT codebook, query[queryOffset + 1], query[queryOffset], query[queryOffset + 1]}; - hn::Vec queryVec = hn::LoadDup128(tag, qtmp); + hn::Vec queryVec = BroadcastDup128(tag, qtmp); constexpr size_t kBlock = 2; constexpr int centroids_per_iter = kLanes / kBlock; @@ -586,7 +618,7 @@ HWY_INLINE void calculate_partial_sums_f32(const float *HWY_RESTRICT codebook, if (size == 4) { constexpr int centroids_per_iter = static_cast(kLanes / 4); hn::Vec queryVec - = hn::LoadDup128(tag, query + queryOffset); + = BroadcastDup128(tag, query + queryOffset); for (; ii + centroids_per_iter <= clusterCount; ii += centroids_per_iter) { From 25c797590e6f9f8c9368d8c4bf97f264ad02aa5e Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Fri, 21 Aug 2026 06:44:18 +0000 Subject: [PATCH 11/14] run native module on any linux --- jvector-native/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/jvector-native/pom.xml b/jvector-native/pom.xml index 88073e998..ceb14df37 100644 --- a/jvector-native/pom.xml +++ b/jvector-native/pom.xml @@ -93,7 +93,6 @@ unix - amd64 From c85c77e99e6011dbea6f98c39170982fbf0a1027 Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Thu, 27 Aug 2026 05:13:43 +0000 Subject: [PATCH 12/14] Add CI runner for arm64 --- .github/workflows/unit-tests-arm64.yaml | 137 ++++++++++++++++++++++++ rat-excludes.txt | 1 + 2 files changed, 138 insertions(+) create mode 100644 .github/workflows/unit-tests-arm64.yaml diff --git a/.github/workflows/unit-tests-arm64.yaml b/.github/workflows/unit-tests-arm64.yaml new file mode 100644 index 000000000..a54844fc6 --- /dev/null +++ b/.github/workflows/unit-tests-arm64.yaml @@ -0,0 +1,137 @@ +name: Unit Test CI — ARM64 (NEON / SVE) + +on: + workflow_dispatch: + pull_request: + push: + branches: + - main + paths: + - .github/workflows/unit-tests-arm64.yaml + - '**.java' + - '**/pom.xml' + +jobs: + build-arm64: + concurrency: + group: arm64-${{ matrix.max_isa }}-${{ matrix.jdk }} + cancel-in-progress: false + strategy: + matrix: + jdk: [ 24 ] + # Three ISA tiers in ascending capability order, mirroring avx512f/avx2/sse42. + # GitHub-hosted ubuntu-24.04-arm is a Neoverse-N1 (Graviton 2): NEON only, no SVE. + # The sve/sve2 matrix entries still exercise the JVECTOR_MAX_ISA cap path and + # compile all three ISA variants; the native kernel tests that require actual SVE + # hardware are gated on the runtime feature check below. + max_isa: [ neon, sve, sve2 ] + runs-on: ubuntu-24.04-arm + steps: + - name: Report ARM64 ISA capabilities + id: cpu-features + run: | + # Parse the "Features" line from /proc/cpuinfo — the kernel only exposes a + # token here when the OS has set up context-switch support for it, so this is + # the same authority as getauxval(AT_HWCAP / AT_HWCAP2). + # "asimd" is the NEON token; "sve"/"sve2"/"sveaes" appear on Graviton 3/4. + flags="$(grep '^Features' /proc/cpuinfo | head -1 | cut -d: -f2)" + has_neon=false; has_sve=false; has_sve2=false + [[ " $flags " == *" asimd "* ]] && has_neon=true + [[ " $flags " == *" sve "* ]] && has_sve=true + [[ " $flags " == *" sve2 "* ]] && has_sve2=true + printf "NEON=%s SVE=%s SVE2=%s\n" "$has_neon" "$has_sve" "$has_sve2" + if [[ "$has_neon" != "true" ]]; then + echo "ERROR: NEON (asimd) not found in /proc/cpuinfo — not a valid AArch64 runner" + exit 2 + fi + # Expose as step outputs for conditional steps below. + echo "has_neon=$has_neon" >> "$GITHUB_OUTPUT" + echo "has_sve=$has_sve" >> "$GITHUB_OUTPUT" + echo "has_sve2=$has_sve2" >> "$GITHUB_OUTPUT" + + - name: Set up GCC + run: | + sudo apt install -y gcc g++ + + - name: Install Meson, Ninja, and GTest + run: | + sudo apt update && sudo apt install -y meson ninja-build pkg-config libgtest-dev + + - uses: actions/checkout@v4 + + - name: Initialize Git Submodules + run: git submodule update --init + + - name: Build test_simd_kernels (native C++) + # Meson detects aarch64 and compiles all three ISA variants (neon/sve/sve2) + # regardless of what the host CPU supports at runtime. + working-directory: jvector-native/src/main/native + run: | + meson setup build --wipe + ninja -C build test_simd_kernels + + - name: Run test_simd_kernels — no ISA cap (auto-detect, neon job) + if: matrix.max_isa == 'neon' + working-directory: jvector-native/src/main/native + run: ./build/test_simd_kernels + + - name: Run test_simd_kernels — capped at neon (sve job, host may lack SVE) + if: matrix.max_isa == 'sve' + working-directory: jvector-native/src/main/native + env: + JVECTOR_MAX_ISA: neon + run: ./build/test_simd_kernels + + - name: Run test_simd_kernels — no ISA cap on SVE hardware (sve job) + if: matrix.max_isa == 'sve' && steps.cpu-features.outputs.has_sve == 'true' + working-directory: jvector-native/src/main/native + run: ./build/test_simd_kernels + + - name: Run test_simd_kernels — capped at neon (sve2 job baseline check) + if: matrix.max_isa == 'sve2' + working-directory: jvector-native/src/main/native + env: + JVECTOR_MAX_ISA: neon + run: ./build/test_simd_kernels + + - name: Run test_simd_kernels — no ISA cap on SVE2 hardware (sve2 job) + if: matrix.max_isa == 'sve2' && steps.cpu-features.outputs.has_sve2 == 'true' + working-directory: jvector-native/src/main/native + run: ./build/test_simd_kernels + + - name: Set up JDK ${{ matrix.jdk }} + uses: actions/setup-java@v3 + with: + java-version: ${{ matrix.jdk }} + distribution: temurin + cache: maven + + - name: Verify native-access vector support (JDK ${{ matrix.jdk }}) + env: + JVECTOR_MAX_ISA: ${{ matrix.max_isa }} + run: >- + mvn -B -Punix-amd64-profile -pl jvector-tests -am test + -DTest_RequireSpecificVectorizationProvider=NativeVectorizationProvider + -Dsurefire.failIfNoSpecifiedTests=false + -Dtest=TestVectorizationProvider + + - name: Test full suite with native vectorization (JDK ${{ matrix.jdk }}) + env: + JVECTOR_MAX_ISA: ${{ matrix.max_isa }} + run: >- + mvn -B -Punix-amd64-profile test + -DTest_RequireSpecificVectorizationProvider=NativeVectorizationProvider + + - name: Test Summary for (ARM64/max:${{ matrix.max_isa }},JDK${{ matrix.jdk }}) + if: always() + uses: test-summary/action@v2 + with: + paths: | + **/target/surefire-reports/TEST-*.xml + + - name: Upload Surefire Test Results + uses: actions/upload-artifact@v4 + if: always() + with: + name: surefire-results--arm64-${{ matrix.max_isa }}-${{ matrix.jdk }} + path: "**/target/surefire-reports/**" diff --git a/rat-excludes.txt b/rat-excludes.txt index 4d0eb0740..dbc696f0a 100644 --- a/rat-excludes.txt +++ b/rat-excludes.txt @@ -3,6 +3,7 @@ CONTRIBUTIONS.md .github/workflows/checklist_comment_on_new_pr.yml .github/workflows/pr_checklist.md .github/workflows/unit-tests.yaml +.github/workflows/unit-tests-arm64.yaml .github/workflows/generate-changelog.yaml .github/workflows/generate-release-notes.yml package.json From 28d0a8400a562669d87952fbaf46a198f7316fbd Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Tue, 1 Sep 2026 07:13:17 +0000 Subject: [PATCH 13/14] arm64: build SVE/SVE2 as scalable (VL-agnostic) targets Switch the AArch64 SVE and SVE2 builds from fixed-width Highway targets (HWY_SVE_256 / HWY_SVE2_128) to scalable ones (HWY_SVE / HWY_SVE2). meson.build: - sve: drop -msve-vector-bits=256; Highway now selects HWY_SVE - sve2: drop -msve-vector-bits=128 and +i8mm+bf16 (only required by HWY_SVE2_128); Highway now selects HWY_SVE2 assert_hwy_targets.h: - Update static-target assertions from HWY_SVE_256/HWY_SVE2_128 to HWY_SVE/HWY_SVE2 and remove stale flag references from error messages. jvector_simd_kernels.cpp: - BroadcastDup128 / LoadDup256: wrap in #if !HWY_HAVE_SCALABLE with a comment that they are written for fixed vector lengths and must not be instantiated on scalable targets. - calculate_partial_sums_f32 / calculate_partial_sums_self_magnitude_f32: wrap all fixed-width fast-paths (Shuffle2301/Shuffle1032/SwapAdjacentBlocks horizontal reductions) in #if !HWY_HAVE_SCALABLE; on scalable SVE execution falls through to the generic per-centroid fallback. Fix the general fallback to use const lanes = hn::Lanes() instead of constexpr kLanes = MaxLanes() as the inner loop stride. - NVQ kernels: replace constexpr kLanes = MaxLanes() with const kLanes = Lanes() for all loop counters; keep constexpr kMaxLanes for stack-array sizing where a compile-time bound is required. jvector_simd.cpp: - Update file-top dispatch comment to reflect HWY_SVE/HWY_SVE2 and scalable (runtime Lanes()) semantics. On x86 and NEON (HWY_HAVE_SCALABLE=0) MaxLanes==Lanes so all existing behaviour is preserved exactly. --- jvector-native/src/main/native/meson.build | 32 ++++------ .../src/main/native/src/assert_hwy_targets.h | 19 +++--- .../src/main/native/src/jvector_simd.cpp | 5 +- .../main/native/src/jvector_simd_kernels.cpp | 63 ++++++++++++------- 4 files changed, 63 insertions(+), 56 deletions(-) diff --git a/jvector-native/src/main/native/meson.build b/jvector-native/src/main/native/meson.build index c3ca22b93..4832330f5 100644 --- a/jvector-native/src/main/native/meson.build +++ b/jvector-native/src/main/native/meson.build @@ -53,24 +53,19 @@ if cpu == 'x86_64' }, ] elif cpu == 'aarch64' - # Three tiers in ascending capability order, all compiled to fixed-width - # Highway targets so MaxLanes is a compile-time constant and all kernel - # fast-paths (Shuffle*, LoadDup256, etc.) work without ISA-specific guards. + # Three tiers in ascending capability order. SVE and SVE2 use scalable + # Highway targets (HWY_HAVE_SCALABLE=1): Lanes() is a runtime value and + # all loop counters are vector-length agnostic. The fixed-width fast-paths + # in calculate_partial_sums_* are gated behind #if !HWY_HAVE_SCALABLE. # - # NEON — baseline AArch64. Highway target: HWY_NEON. - # All Graviton generations and all Apple Silicon. + # NEON — baseline AArch64. Highway target: HWY_NEON. + # All Graviton generations and all Apple Silicon. # - # SVE_256 — Graviton 3 / Neoverse V1. Highway target: HWY_SVE_256. - # -msve-vector-bits=256 asserts the physical vector width and - # causes Highway to select the fixed-width HWY_SVE_256 target - # (MaxLanes = 8 floats). Graviton-only; no Apple Silicon SVE. + # SVE — Graviton 3 / Neoverse V1 and later. Highway target: HWY_SVE. + # No -msve-vector-bits flag; Highway queries the VL at runtime. # - # SVE2_128 — Graviton 4 / Neoverse V2/N2. Highway target: HWY_SVE2_128. - # -msve-vector-bits=128 selects the fixed-width HWY_SVE2_128 - # target (MaxLanes = 4 floats). At runtime on Graviton 4 the - # hardware executes 256-bit vectors, so Highway's loop unrolling - # gives 256-bit throughput with 128-bit-wide loop bodies. - # Requires GCC >= 10 or Clang >= 21. Graviton-only (no Apple). + # SVE2 — Graviton 4 / Neoverse V2/N2 and later. Highway target: HWY_SVE2. + # No -msve-vector-bits flag; Highway queries the VL at runtime. isa_variants = [ { 'name' : 'neon', @@ -82,17 +77,14 @@ elif cpu == 'aarch64' { 'name' : 'sve', 'namespace': 'SVE', - 'args' : ['-march=armv8.4-a+sve', '-msve-vector-bits=256', + 'args' : ['-march=armv8.4-a+sve', '-DHWY_COMPILE_ONLY_STATIC', '-DJV_REQUIRE_HWY_SVE'], }, { 'name' : 'sve2', 'namespace': 'SVE2', - # HWY_SVE2_128 requires i8mm and bf16 in addition to sve2. - # See highway/hwy/ops/set_macros-inl.h:662 ("SVE2_128 implies/requires - # I8MM and BF16, see #2973") and the HWY_TARGET_STR definition there. - 'args' : ['-march=armv9-a+sve2+i8mm+bf16', '-msve-vector-bits=128', + 'args' : ['-march=armv9-a+sve2', '-DHWY_COMPILE_ONLY_STATIC', '-DJV_REQUIRE_HWY_SVE2'], }, diff --git a/jvector-native/src/main/native/src/assert_hwy_targets.h b/jvector-native/src/main/native/src/assert_hwy_targets.h index 23c07d619..1c51bedbd 100644 --- a/jvector-native/src/main/native/src/assert_hwy_targets.h +++ b/jvector-native/src/main/native/src/assert_hwy_targets.h @@ -39,20 +39,19 @@ #elif JV_ARCH_AARCH64 // Compiler flags per tier and the Highway target each must produce: -// neon: -march=armv8-a+crypto → HWY_NEON +// neon: -march=armv8-a+crypto → HWY_NEON // (no BF16/dotprod/I8MM features, so never HWY_NEON_BF16) -// sve: -march=armv8.4-a+sve -msve-vector-bits=256 → HWY_SVE_256 -// (pinned 256-bit; Graviton 3 / Neoverse V1) -// sve2: -march=armv9-a+sve2 -msve-vector-bits=128 → HWY_SVE2_128 -// (fixed-width 128-bit; Graviton 4 / Neoverse V2/N2) -// Requires GCC >= 10 or Clang >= 21. +// sve: -march=armv8.4-a+sve → HWY_SVE (scalable, VL-agnostic) +// (Graviton 3 / Neoverse V1 and later) +// sve2: -march=armv9-a+sve2 → HWY_SVE2 (scalable, VL-agnostic) +// (Graviton 4 / Neoverse V2/N2 and later) #if defined(JV_REQUIRE_HWY_SVE2) -# if HWY_STATIC_TARGET != HWY_SVE2_128 -# error "Highway did not select HWY_SVE2_128 for the SVE2 build. Check compiler flags (-march=armv9-a+sve2+i8mm+bf16 -msve-vector-bits=128), compiler support (GCC >= 10 or Clang >= 21), and Highway blocklists." +# if HWY_STATIC_TARGET != HWY_SVE2 +# error "Highway did not select HWY_SVE2 for the SVE2 build. Check compiler flags (-march=armv9-a+sve2) and Highway blocklists." # endif #elif defined(JV_REQUIRE_HWY_SVE) -# if HWY_STATIC_TARGET != HWY_SVE_256 -# error "Highway did not select HWY_SVE_256 for the SVE build. Check compiler flags (-march=armv8.4-a+sve -msve-vector-bits=256), compiler support (GCC >= 10 or Clang >= 9), and Highway blocklists." +# if HWY_STATIC_TARGET != HWY_SVE +# error "Highway did not select HWY_SVE for the SVE build. Check compiler flags (-march=armv8.4-a+sve) and Highway blocklists." # endif #elif defined(JV_REQUIRE_HWY_NEON) # if HWY_STATIC_TARGET != HWY_NEON diff --git a/jvector-native/src/main/native/src/jvector_simd.cpp b/jvector-native/src/main/native/src/jvector_simd.cpp index 061048fc3..cfc40756e 100644 --- a/jvector-native/src/main/native/src/jvector_simd.cpp +++ b/jvector-native/src/main/native/src/jvector_simd.cpp @@ -17,9 +17,8 @@ // Runtime SIMD dispatch: selects the best available ISA tier at startup. // x86-64 tiers (descending): AVX3_SPR, AVX3_DL, AVX3, AVX2, SSE42. // SSE42 is the x86-64 baseline — assumed always available, no CPUID check. -// AArch64 tiers (descending): SVE2_128 (HWY_SVE2_128), SVE_256 (HWY_SVE_256), NEON. -// All three are fixed-width Highway targets: MaxLanes is a compile-time -// constant, so all kernel fast-paths work identically to x86. +// AArch64 tiers (descending): SVE2 (HWY_SVE2), SVE (HWY_SVE), NEON. +// SVE/SVE2 are scalable targets (HWY_HAVE_SCALABLE=1): Lanes() is runtime. // NEON is the AArch64 baseline — always available on any aarch64 CPU. // Function pointers are resolved once at static-init time; each public call is // a single indirect branch. diff --git a/jvector-native/src/main/native/src/jvector_simd_kernels.cpp b/jvector-native/src/main/native/src/jvector_simd_kernels.cpp index 17c9089b7..3eef67264 100644 --- a/jvector-native/src/main/native/src/jvector_simd_kernels.cpp +++ b/jvector-native/src/main/native/src/jvector_simd_kernels.cpp @@ -175,13 +175,20 @@ namespace hn = hwy::HWY_NAMESPACE; +#if !HWY_HAVE_SCALABLE +// The two helpers below rely on MaxLanes being a compile-time constant and on +// fixed-stride Combine/Half arithmetic. They are written for fixed vector +// widths (x86, NEON) and must not be instantiated on scalable targets (SVE/SVE2) +// where MaxLanes is a loose upper bound unrelated to the runtime VL. +// Their only call sites are also inside #if !HWY_HAVE_SCALABLE blocks. + // Loads 4 floats from ptr and broadcasts them to fill the full vector D. // Uses LoadU + Combine instead of hn::LoadDup128 to avoid a GCC 14 internal // compiler error (ICE in convert_move/expr.cc:301) triggered by hn::LoadDup128 -// (which emits ld1rq) when compiled with -msve-vector-bits=256. -// D = 4 lanes → plain LoadU (ptr holds exactly one full vector) -// D = 8 lanes → load 4-lane half, Combine to duplicate into both halves -// D = 16 lanes → load 4-lane half-of-half, Combine twice (4→8→16 lanes) +// (which emits ld1rq) on SVE targets. +// MaxLanes(D) == 4 → plain LoadU (ptr holds exactly one full vector) +// MaxLanes(D) == 8 → load 4-lane half, Combine to duplicate into both halves +// MaxLanes(D) == 16 → load 4-lane half-of-half, Combine twice (4→8→16 lanes) // hn::Quarter does not exist in this Highway version; two applications // of hn::Half<> are used instead. template @@ -197,12 +204,12 @@ HWY_INLINE hn::Vec BroadcastDup128(D d, const float *HWY_RESTRICT ptr) const auto half = hn::Combine(dh, quarter, quarter); return hn::Combine(d, half, half); } else if constexpr (hn::MaxLanes(d) > 4) { - // 8-lane (AVX2, SVE_256): load 4-lane half, combine to full. + // MaxLanes == 8 (AVX2): load 4-lane half, combine to full. const hn::Half dh; const auto half = hn::LoadU(dh, ptr); return hn::Combine(d, half, half); } else { - // 4-lane (SSE4, NEON, SVE2_128): ptr holds exactly one full vector. + // MaxLanes == 4 (SSE4, NEON): ptr holds exactly one full vector. return hn::LoadU(d, ptr); } } @@ -229,6 +236,7 @@ HWY_INLINE hn::Vec LoadDup256(D d, const float *HWY_RESTRICT ptr) return hn::LoadU(d, ptr); } } +#endif // !HWY_HAVE_SCALABLE // ============================================================================= // Base Fp32 kernels // ============================================================================= @@ -581,11 +589,15 @@ HWY_INLINE void calculate_partial_sums_f32(const float *HWY_RESTRICT codebook, float *HWY_RESTRICT partialSums) { int codebookBase = codebookIndex * clusterCount; + int ii = 0; + +#if !HWY_HAVE_SCALABLE + // Fixed-width ISAs (x86, NEON): MaxLanes is a compile-time constant so + // centroids_per_iter and the horizontal-reduction shuffles are valid. using FloatTag = hn::ScalableTag; FloatTag tag; constexpr size_t kLanes = hn::MaxLanes(tag); alignas(64) float tmp[kLanes]; - int ii = 0; if constexpr (kLanes >= 2) { if (size == 2) { @@ -594,9 +606,7 @@ HWY_INLINE void calculate_partial_sums_f32(const float *HWY_RESTRICT codebook, query[queryOffset], query[queryOffset + 1]}; hn::Vec queryVec = BroadcastDup128(tag, qtmp); - - constexpr size_t kBlock = 2; - constexpr int centroids_per_iter = kLanes / kBlock; + constexpr int centroids_per_iter = static_cast(kLanes / 2); for (; ii + centroids_per_iter <= clusterCount; ii += centroids_per_iter) { @@ -664,7 +674,6 @@ HWY_INLINE void calculate_partial_sums_f32(const float *HWY_RESTRICT codebook, } } if constexpr (kLanes == 16) { - // Don't have to worry about making this work on 1024-bit lanes just yet if (size == 16) { const hn::Vec queryVec = hn::LoadU(tag, query + queryOffset); @@ -677,6 +686,8 @@ HWY_INLINE void calculate_partial_sums_f32(const float *HWY_RESTRICT codebook, } } } +#endif // !HWY_HAVE_SCALABLE + for (; ii < clusterCount; ii++) { partialSums[codebookBase + ii] = distance_func
( codebook, ii * size, query, queryOffset, size); @@ -962,14 +973,17 @@ HWY_FLATTEN void calculate_partial_sums_self_magnitude_f32( const int codebookBase = codebookIndex * clusterCount; using FloatTag = hn::ScalableTag; FloatTag tag; + int ii = 0; + +#if !HWY_HAVE_SCALABLE + // Fixed-width ISAs (x86, NEON): MaxLanes is a compile-time constant so + // centroids_per_iter and the horizontal-reduction shuffles are valid. constexpr size_t kLanes = hn::MaxLanes(tag); alignas(64) float tmp[kLanes]; - int ii = 0; if constexpr (kLanes >= 2) { if (size == 2) { - constexpr size_t kBlock = 2; - constexpr int centroids_per_iter = kLanes / kBlock; + constexpr int centroids_per_iter = static_cast(kLanes / 2); for (; ii + centroids_per_iter <= clusterCount; ii += centroids_per_iter) { @@ -1031,7 +1045,6 @@ HWY_FLATTEN void calculate_partial_sums_self_magnitude_f32( } } if constexpr (kLanes == 16) { - // AVX-512 only: one full register holds exactly one size==16 centroid. if (size == 16) { for (; ii < clusterCount; ++ii) { const hn::Vec cv @@ -1041,12 +1054,15 @@ HWY_FLATTEN void calculate_partial_sums_self_magnitude_f32( } } } +#endif // !HWY_HAVE_SCALABLE + // General fallback: one centroid at a time, vector-accumulate then reduce. + const size_t lanes = hn::Lanes(tag); for (; ii < clusterCount; ii++) { const float *cptr = codebook + ii * size; auto accVec = hn::Zero(tag); size_t j = 0; - for (; j + kLanes <= size; j += kLanes) { + for (; j + lanes <= size; j += lanes) { const auto v = hn::LoadU(tag, cptr + j); accVec = hn::MulAdd(v, v, accVec); } @@ -1188,8 +1204,9 @@ HWY_FLATTEN void nvq_quantize_8bit(const float *HWY_RESTRICT vector, using Int32Tag = hn::RebindToSigned; FloatTag d_f; Int32Tag d_i; - constexpr size_t kLanes = hn::MaxLanes(d_f); - alignas(64) int32_t tmp[kLanes]; + constexpr size_t kMaxLanes = hn::MaxLanes(d_f); + const size_t kLanes = hn::Lanes(d_f); + alignas(64) int32_t tmp[kMaxLanes]; float delta = maxValue - minValue; float scaledAlpha = alpha / delta; @@ -1238,7 +1255,7 @@ HWY_FLATTEN float nvq_loss(const float *HWY_RESTRICT vector, using Int32Tag = hn::RebindToSigned; FloatTag d_f; Int32Tag d_i; - constexpr size_t kLanes = hn::MaxLanes(d_f); + const size_t kLanes = hn::Lanes(d_f); int constant = (1 << nBits) - 1; float delta = maxValue - minValue; @@ -1296,7 +1313,7 @@ HWY_FLATTEN float nvq_uniform_loss(const float *HWY_RESTRICT vector, using Int32Tag = hn::RebindToSigned; FloatTag d_f; Int32Tag d_i; - constexpr size_t kLanes = hn::MaxLanes(d_f); + const size_t kLanes = hn::Lanes(d_f); float constant = (float)((1 << nBits) - 1); float delta = maxValue - minValue; @@ -1405,7 +1422,7 @@ HWY_FLATTEN float nvq_square_l2_distance_8bit(const float *HWY_RESTRICT vecto Uint8x4Tag d_b; Uint16Tag d_u16; Uint8Tag d_u8; - constexpr size_t kLanes = hn::MaxLanes(d_f); + const size_t kLanes = hn::Lanes(d_f); float delta = maxValue - minValue; float scaledAlpha = alpha / delta; @@ -1481,7 +1498,7 @@ HWY_FLATTEN float nvq_dot_product_8bit(const float *HWY_RESTRICT vector, Uint8x4Tag d_b; Uint16Tag d_u16; Uint8Tag d_u8; - constexpr size_t kLanes = hn::MaxLanes(d_f); + const size_t kLanes = hn::Lanes(d_f); float delta = maxValue - minValue; float scaledAlpha = alpha / delta; @@ -1604,7 +1621,7 @@ HWY_FLATTEN int64_t nvq_cosine_8bit_packed(const float *HWY_RESTRICT vector, Uint8x4Tag d_b; Uint16Tag d_u16; Uint8Tag d_u8; - constexpr size_t kLanes = hn::MaxLanes(d_f); + const size_t kLanes = hn::Lanes(d_f); float delta = maxValue - minValue; float scaledAlpha = alpha / delta; From f0d8ff5c77a4e14a05f114a55bfb5c445db56b7a Mon Sep 17 00:00:00 2001 From: Raghuveer Devulapalli Date: Tue, 1 Sep 2026 07:25:59 +0000 Subject: [PATCH 14/14] docs: update READMEs for AArch64 SVE/SVE2 support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit jvector-native/src/main/native/README.md: - Platform support blurb: drop 'ARM support planned' — NEON, SVE, SVE2 are now live. - Architecture diagram: add AArch64 dispatch tree (SVE2 > SVE > NEON) alongside the existing x86-64 tree; correct the x86-64 -march= flags (-march=sapphirerapids / -march=icelake-server). - ISA dispatch section: note getauxval for AArch64 alongside CPUID/XGETBV for x86; list both dispatch chains. - ISA cap section: add AArch64 JVECTOR_MAX_ISA examples (sve2, sve, neon) and update the accepted-values sentence. README.md: - Building native libraries: add 'Linux AArch64 (NEON, SVE, SVE2)' to the supported-platforms line. --- README.md | 4 +- jvector-native/src/main/native/README.md | 47 +++++++++++++++++------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 3d0bb50a7..3b67dfdfa 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,8 @@ git clone --recurse-submodules ### Building native libraries The native SIMD library (`libjvector.so`) is built with [Meson](https://mesonbuild.com/) + [Ninja](https://ninja-build.org/) -and requires **g++ 11+**. The entry-point script is -`jvector-native/src/main/native/jextract_vector_simd.sh`. Run it from that directory: +and requires **g++ 11+**. Supported platforms: **Linux x86-64** (SSE4.2, AVX2, AVX-512) and **Linux AArch64** (NEON, SVE, SVE2). +The entry-point script is `jvector-native/src/main/native/jextract_vector_simd.sh`. Run it from that directory: ```bash cd jvector-native/src/main/native diff --git a/jvector-native/src/main/native/README.md b/jvector-native/src/main/native/README.md index 689483b9f..b8edd8818 100644 --- a/jvector-native/src/main/native/README.md +++ b/jvector-native/src/main/native/README.md @@ -21,11 +21,8 @@ backend that accelerates vector operations in JVector via the Java Foreign Function & Memory (FFM) API. > **Platform support:** Currently enabled on **Linux x86-64** (SSE4.2, AVX2, -> and AVX-512). Windows and macOS are not yet supported. Support for **ARM** -> (NEON and SVE) is planned for the near future; the -> [Google Highway](https://github.com/google/highway) library used for SIMD -> portability already targets both AArch64 targets, which will make the -> extension straightforward. +> AVX-512) and **Linux AArch64** (NEON, SVE, SVE2). Windows and macOS are not +> yet supported. --- @@ -242,19 +239,32 @@ JVECTOR_MAX_ISA=sse42 ../../../target/meson-build/bench_simd_kernels ## How it is integrated into JVector +**x86-64:** ``` Java caller └─ NativeVectorUtilSupport (jvector-native/.../vector/) └─ NativeSimdOps (jvector-native/.../vector/cnative/ — FFM glue, generated by jextract) └─ libjvector.so (this library, loaded at runtime by LibraryLoader) └─ jvector_simd.cpp — dispatches to the best ISA vtable - ├─ AVX3_SPR::* (compiled with -march=skylake-avx512 -mavx512fp16 …, Sapphire Rapids) - ├─ AVX3_DL::* (compiled with -march=skylake-avx512 -mavx512vnni …, Ice Lake) + ├─ AVX3_SPR::* (compiled with -march=sapphirerapids) + ├─ AVX3_DL::* (compiled with -march=icelake-server) ├─ AVX3::* (compiled with -march=skylake-avx512) ├─ AVX2::* (compiled with -march=haswell) └─ SSE42::* (compiled with -msse4.2, scalar fallback) ``` +**AArch64:** +``` +Java caller + └─ NativeVectorUtilSupport (jvector-native/.../vector/) + └─ NativeSimdOps (jvector-native/.../vector/cnative/ — FFM glue, generated by jextract) + └─ libjvector.so (this library, loaded at runtime by LibraryLoader) + └─ jvector_simd.cpp — dispatches to the best ISA vtable + ├─ SVE2::* (compiled with -march=armv9-a+sve2, scalable VL) + ├─ SVE::* (compiled with -march=armv8.4-a+sve, scalable VL) + └─ NEON::* (compiled with -march=armv8-a+crypto, baseline) +``` + ### Load sequence 1. `NativeVectorizationProvider` calls `LibraryLoader.loadJvector()` at startup. @@ -268,9 +278,10 @@ Java caller Dispatch happens **once** at C++ static-init time (before `main()`): -1. `populate_cpu_features()` issues CPUID / XGETBV and fills a feature array. -2. `dispatch_kernels()` checks the feature array in descending capability order - (`AVX3` ⊃ `AVX2` ⊃ `SSE42`) and returns a copy of the matching `KernelVTable`. +1. `populate_cpu_features()` issues CPUID / XGETBV (x86) or `getauxval` (AArch64) and fills a feature array. +2. `dispatch_kernels()` checks the feature array in descending capability order and returns a copy of the matching `KernelVTable`: + - x86-64: `AVX3_SPR` ⊃ `AVX3_DL` ⊃ `AVX3` ⊃ `AVX2` ⊃ `SSE42` + - AArch64: `SVE2` ⊃ `SVE` ⊃ `NEON` 3. All public API functions are one-liner wrappers that call through `kernels.`. @@ -279,15 +290,23 @@ Dispatch happens **once** at C++ static-init time (before `main()`): Set the `JVECTOR_MAX_ISA` environment variable before starting the JVM to cap the selected ISA without recompiling: +**x86-64:** ```bash JVECTOR_MAX_ISA=avx3_spr java ... # use Sapphire-Rapids FP16 tier JVECTOR_MAX_ISA=avx3_dl java ... # use Ice Lake tier -JVECTOR_MAX_ISA=avx3 java ... # use AVX-512 even if a higher tier is available -JVECTOR_MAX_ISA=avx2 java ... # use AVX2 even on an AVX-512 machine -JVECTOR_MAX_ISA=sse42 java ... # force scalar/SSE4.2 fallback +JVECTOR_MAX_ISA=avx3 java ... # use AVX-512 even if a higher tier is available +JVECTOR_MAX_ISA=avx2 java ... # use AVX2 even on an AVX-512 machine +JVECTOR_MAX_ISA=sse42 java ... # force scalar/SSE4.2 fallback +``` + +**AArch64:** +```bash +JVECTOR_MAX_ISA=sve2 java ... # use SVE2 tier (Graviton 4 / Neoverse V2/N2) +JVECTOR_MAX_ISA=sve java ... # use SVE tier (Graviton 3 / Neoverse V1) +JVECTOR_MAX_ISA=neon java ... # force NEON baseline ``` -Accepted values (case-sensitive): `avx3_spr`, `avx3_dl`, `avx3`, `avx2`, `sse42`. +Accepted values (case-sensitive): `avx3_spr`, `avx3_dl`, `avx3`, `avx2`, `sse42` (x86-64); `sve2`, `sve`, `neon` (AArch64). An unrecognised value is silently ignored and full CPU detection is used. ### Updating the Java bindings