diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp index 5ecfb087995..356f5797846 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022-2024 Arm Limited. + * Copyright (c) 2022-2024, 2026 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -81,8 +81,10 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "ldr x15, [%x[args], %[offsetof_Args_current_channel]]\n" "ld1rw { z8.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n" "whilelt p1.s, XZR, x16\n" - "whilelt p9.s, XZR, x20\n" - "whilelt p8.s, XZR, x17\n" + // Nine input rows must fit in the row mask even at the minimum SME vector length. + // Byte predicates avoid wrapping row indices when PSEL selects the padding mask. + "whilelt p9.b, XZR, x20\n" + "whilelt p8.b, XZR, x17\n" "eor p8.b, p2/Z, p8.b, p9.b\n" "1:" // Channel loop "ldr x20, [%x[args], %[offsetof_Args_bias]]\n" @@ -305,33 +307,33 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "11:" // Padded: 2 priming loads "mov x12, #0x0\n" "add x20, x13, %x[ld_in_row], LSL #2\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "ld1w { z23.s }, p0/Z, [x13]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x13, x13, %x[ld_in_col], LSL #2\n" "ld1w { z20.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z24.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x4\n" "ld1w { z21.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z25.s }, p0/Z, [x20]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z22.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z26.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x8\n" ".inst 0xc1361ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z6.s\n" "ld1w { z23.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z27.s }, p0/Z, [x20]\n" ".inst 0xc1321a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z2.s\n" @@ -339,33 +341,33 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "12:" // Padded: 1 priming loads "mov x12, #0x0\n" "add x20, x13, %x[ld_in_row], LSL #2\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "ld1w { z29.s }, p0/Z, [x13]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x13, x13, %x[ld_in_col], LSL #2\n" "ld1w { z20.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z30.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x4\n" "ld1w { z21.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z31.s }, p0/Z, [x20]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z22.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z0.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x8\n" ".inst 0xc1371ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z7.s\n" "ld1w { z23.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z1.s }, p0/Z, [x20]\n" ".inst 0xc1331a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z3.s\n" @@ -375,39 +377,39 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "blt 16f\n" "mov x12, #0x0\n" "add x21, x13, %x[ld_in_row], LSL #2\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "sub x14, x14, #0x2\n" "sub x11, x11, #0x1\n" "lsr x20, x14, #0x1\n" "cmp x20, x11\n" "and x14, x14, #0x1\n" "ld1w { z29.s }, p0/Z, [x13]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "csel x22, x20, x11, LT\n" "add x13, x13, %x[ld_in_col], LSL #2\n" "sub x11, x11, x22\n" "ld1w { z22.s }, p0/Z, [x21]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "ld1w { z30.s }, p0/Z, [x21]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "mov x12, #0x4\n" "ld1w { z23.s }, p0/Z, [x21]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "ld1w { z31.s }, p0/Z, [x21]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "ld1w { z24.s }, p0/Z, [x21]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "ld1w { z0.s }, p0/Z, [x21]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "mov x12, #0x8\n" "ld1w { z25.s }, p0/Z, [x21]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "ld1w { z1.s }, p0/Z, [x21]\n" "cbz x22, 15f\n" @@ -415,43 +417,43 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "mov x12, #0x0\n" ".inst 0xc13a1ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z10.s\n" "add x21, x13, %x[ld_in_row], LSL #2\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" ".inst 0xc1361ba1 // fmla za.s[x8, 1], { z29.s-z0.s }, z6.s\n" "subs x22, x22, #0x1\n" "ld1w { z18.s }, p0/Z, [x13]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" ".inst 0xc13b1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z11.s\n" "add x13, x13, %x[ld_in_col], LSL #2\n" ".inst 0xc1321ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z2.s\n" "add x20, x13, %x[ld_in_row], LSL #2\n" "ld1w { z29.s }, p0/Z, [x21]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0xc1391bc0 // fmla za.s[x8, 0], { z30.s-z1.s }, z9.s\n" ".inst 0xc1341bc1 // fmla za.s[x8, 1], { z30.s-z1.s }, z4.s\n" "ld1w { z19.s }, p0/Z, [x21]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "mov x12, #0x4\n" ".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" "ld1w { z30.s }, p0/Z, [x21]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" "ld1w { z20.s }, p0/Z, [x21]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0xc1a8cb98 // fclamp { z24.s-z27.s }, z28.s, z8.s\n" "ld1w { z31.s }, p0/Z, [x21]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "st1w { z24.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" "st1w { z25.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" "ld1w { z21.s }, p0/Z, [x21]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" "st1w { z26.s }, p1, [x26]\n" "mov x12, #0x8\n" @@ -459,39 +461,39 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "st1w { z27.s }, p1, [x25]\n" "add x25, x25, x23, LSL #2\n" "ld1w { z0.s }, p0/Z, [x21]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x21, x21, %x[ld_in_row], LSL #2\n" ".inst 0xc1371a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z7.s\n" "mov x12, #0x0\n" "ld1w { z22.s }, p0/Z, [x21]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" ".inst 0xc1331ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z3.s\n" "ld1w { z29.s }, p0/Z, [x13]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x13, x13, %x[ld_in_col], LSL #2\n" ".inst 0xc1351a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z5.s\n" "ld1w { z22.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z30.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x4\n" "ld1w { z23.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z31.s }, p0/Z, [x20]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z24.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z0.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x8\n" "ld1w { z25.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z1.s }, p0/Z, [x20]\n" "bgt 14b\n" @@ -499,41 +501,41 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "mov x12, #0x0\n" ".inst 0xc13a1ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z10.s\n" "add x20, x13, %x[ld_in_row], LSL #2\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" ".inst 0xc1361ba1 // fmla za.s[x8, 1], { z29.s-z0.s }, z6.s\n" "ld1w { z29.s }, p0/Z, [x13]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" ".inst 0xc13b1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z11.s\n" "add x13, x13, %x[ld_in_col], LSL #2\n" ".inst 0xc1321ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z2.s\n" "ld1w { z22.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0xc1391bc0 // fmla za.s[x8, 0], { z30.s-z1.s }, z9.s\n" ".inst 0xc1341bc1 // fmla za.s[x8, 1], { z30.s-z1.s }, z4.s\n" "ld1w { z30.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x4\n" ".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n" "add x8, x8, #0x1\n" "ld1w { z23.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n" "ld1w { z31.s }, p0/Z, [x20]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0xc1a8cb90 // fclamp { z16.s-z19.s }, z28.s, z8.s\n" "ld1w { z24.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "st1w { z16.s }, p1, [x10]\n" "add x10, x10, x28, LSL #2\n" "st1w { z17.s }, p1, [x9]\n" "add x9, x9, x27, LSL #2\n" "ld1w { z0.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "st1w { z18.s }, p1, [x26]\n" "mov x12, #0x8\n" @@ -542,7 +544,7 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "add x25, x25, x23, LSL #2\n" ".inst 0xc1371ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z7.s\n" "ld1w { z25.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z1.s }, p0/Z, [x20]\n" ".inst 0xc1331ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z3.s\n" @@ -551,33 +553,33 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl( "cbz x14, 17f\n" // Skip remainder inputs "mov x12, #0x0\n" "add x20, x13, %x[ld_in_row], LSL #2\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "sub x11, x11, #0x1\n" "ld1w { z23.s }, p0/Z, [x13]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "ld1w { z16.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z24.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x4\n" "ld1w { z17.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z25.s }, p0/Z, [x20]\n" - ".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n" + ".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z18.s }, p0/Z, [x20]\n" - ".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n" + ".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "ld1w { z26.s }, p0/Z, [x20]\n" - ".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n" + ".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" "mov x12, #0x8\n" ".inst 0xc13a1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z10.s\n" "ld1w { z19.s }, p0/Z, [x20]\n" - ".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n" + ".inst 0x25244500 // psel p0, p1, p8.b[w12]\n" "add x20, x20, %x[ld_in_row], LSL #2\n" ".inst 0xc1361ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z6.s\n" "ld1w { z27.s }, p0/Z, [x20]\n" diff --git a/tests/validation/NEON/DepthwiseConvolutionLayer.cpp b/tests/validation/NEON/DepthwiseConvolutionLayer.cpp index 55673ff8b0d..04515c21dbd 100644 --- a/tests/validation/NEON/DepthwiseConvolutionLayer.cpp +++ b/tests/validation/NEON/DepthwiseConvolutionLayer.cpp @@ -361,6 +361,25 @@ using NEDepthwiseConvolutionLayerVariableWeightsFixture = TEST_SUITE(Float) TEST_SUITE(F32) +// Exercise the ninth input row in the SME2 planar 3x3/stride-2 padding path. +FIXTURE_DATA_TEST_CASE_NEW( + Run3x3Stride2RowPadding, + NEDepthwiseConvolutionLayerFixture, + framework::DatasetMode::PRECOMMIT, + combine(make("In", {TensorShape(8U, 8U, 64U), TensorShape(9U, 9U, 17U), TensorShape(112U, 112U, 64U)}), + make("Weights", Size2D(3U, 3U)), + make("Info", + {PadStrideInfo(2, 2, 0, 1, 0, 1, DimensionRoundingType::FLOOR), + PadStrideInfo(2, 2, 1, 1, 1, 1, DimensionRoundingType::FLOOR)}), + make("Dilation", Size2D(1U, 1U)), + make("DepthMultiplier", {1}), + make("DataType", DataType::F32), + make("DataLayout", {DataLayout::NHWC}), + ActivationFunctionsDataset)) +{ + validate(Accessor(_target), _reference, tolerance_f32); +} + FIXTURE_DATA_TEST_CASE_NEW(RunActivations, NEDepthwiseConvolutionLayerFixture, framework::DatasetMode::NIGHTLY,