Skip to content

fix: Fix SME2 FP32 depthwise row masking at short vector lengths #1316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
morgolock wants to merge 1 commit into
base: main
Choose a base branch
from
+87 −66
Open

fix: Fix SME2 FP32 depthwise row masking at short vector lengths #1316

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 68 additions & 66 deletions .../NEON/kernels/arm_conv/depthwise/kernels/sme2_fp32_planar_3x3_s2_4rows_mla_za/generic.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2022-2024 Arm Limited.
* Copyright (c) 2022-2024, 2026 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
Expand Down Expand Up @@ -81,8 +81,10 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl(
"ldr x15, [%x[args], %[offsetof_Args_current_channel]]\n"
"ld1rw { z8.s }, p2/Z, [%x[args], %[offsetof_Args_clamp_max]]\n"
"whilelt p1.s, XZR, x16\n"
"whilelt p9.s, XZR, x20\n"
"whilelt p8.s, XZR, x17\n"
// Nine input rows must fit in the row mask even at the minimum SME vector length.
// Byte predicates avoid wrapping row indices when PSEL selects the padding mask.
"whilelt p9.b, XZR, x20\n"
"whilelt p8.b, XZR, x17\n"
"eor p8.b, p2/Z, p8.b, p9.b\n"
"1:" // Channel loop
"ldr x20, [%x[args], %[offsetof_Args_bias]]\n"
Expand Down Expand Up @@ -305,67 +307,67 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl(
"11:" // Padded: 2 priming loads
"mov x12, #0x0\n"
"add x20, x13, %x[ld_in_row], LSL #2\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"ld1w { z23.s }, p0/Z, [x13]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x13, x13, %x[ld_in_col], LSL #2\n"
"ld1w { z20.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z24.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x4\n"
"ld1w { z21.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z25.s }, p0/Z, [x20]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z22.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z26.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x8\n"
".inst 0xc1361ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z6.s\n"
"ld1w { z23.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z27.s }, p0/Z, [x20]\n"
".inst 0xc1321a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z2.s\n"
".inst 0xc1341b00 // fmla za.s[x8, 0], { z24.s-z27.s }, z4.s\n"
"12:" // Padded: 1 priming loads
"mov x12, #0x0\n"
"add x20, x13, %x[ld_in_row], LSL #2\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"ld1w { z29.s }, p0/Z, [x13]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x13, x13, %x[ld_in_col], LSL #2\n"
"ld1w { z20.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z30.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x4\n"
"ld1w { z21.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z31.s }, p0/Z, [x20]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z22.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z0.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x8\n"
".inst 0xc1371ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z7.s\n"
"ld1w { z23.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z1.s }, p0/Z, [x20]\n"
".inst 0xc1331a80 // fmla za.s[x8, 0], { z20.s-z23.s }, z3.s\n"
Expand All @@ -375,165 +377,165 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl(
"blt 16f\n"
"mov x12, #0x0\n"
"add x21, x13, %x[ld_in_row], LSL #2\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"sub x14, x14, #0x2\n"
"sub x11, x11, #0x1\n"
"lsr x20, x14, #0x1\n"
"cmp x20, x11\n"
"and x14, x14, #0x1\n"
"ld1w { z29.s }, p0/Z, [x13]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"csel x22, x20, x11, LT\n"
"add x13, x13, %x[ld_in_col], LSL #2\n"
"sub x11, x11, x22\n"
"ld1w { z22.s }, p0/Z, [x21]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"ld1w { z30.s }, p0/Z, [x21]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"mov x12, #0x4\n"
"ld1w { z23.s }, p0/Z, [x21]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"ld1w { z31.s }, p0/Z, [x21]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"ld1w { z24.s }, p0/Z, [x21]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"ld1w { z0.s }, p0/Z, [x21]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"mov x12, #0x8\n"
"ld1w { z25.s }, p0/Z, [x21]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"ld1w { z1.s }, p0/Z, [x21]\n"
"cbz x22, 15f\n"
"14:" // Padded: Main loop
"mov x12, #0x0\n"
".inst 0xc13a1ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z10.s\n"
"add x21, x13, %x[ld_in_row], LSL #2\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
".inst 0xc1361ba1 // fmla za.s[x8, 1], { z29.s-z0.s }, z6.s\n"
"subs x22, x22, #0x1\n"
"ld1w { z18.s }, p0/Z, [x13]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
".inst 0xc13b1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z11.s\n"
"add x13, x13, %x[ld_in_col], LSL #2\n"
".inst 0xc1321ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z2.s\n"
"add x20, x13, %x[ld_in_row], LSL #2\n"
"ld1w { z29.s }, p0/Z, [x21]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
".inst 0xc1391bc0 // fmla za.s[x8, 0], { z30.s-z1.s }, z9.s\n"
".inst 0xc1341bc1 // fmla za.s[x8, 1], { z30.s-z1.s }, z4.s\n"
"ld1w { z19.s }, p0/Z, [x21]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"mov x12, #0x4\n"
".inst 0xc0060c18 // mova { z24.d-z27.d }, za.d[x8, #0]\n"
"add x8, x8, #0x1\n"
"ld1w { z30.s }, p0/Z, [x21]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n"
"ld1w { z20.s }, p0/Z, [x21]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
".inst 0xc1a8cb98 // fclamp { z24.s-z27.s }, z28.s, z8.s\n"
"ld1w { z31.s }, p0/Z, [x21]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"st1w { z24.s }, p1, [x10]\n"
"add x10, x10, x28, LSL #2\n"
"st1w { z25.s }, p1, [x9]\n"
"add x9, x9, x27, LSL #2\n"
"ld1w { z21.s }, p0/Z, [x21]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
"st1w { z26.s }, p1, [x26]\n"
"mov x12, #0x8\n"
"add x26, x26, x24, LSL #2\n"
"st1w { z27.s }, p1, [x25]\n"
"add x25, x25, x23, LSL #2\n"
"ld1w { z0.s }, p0/Z, [x21]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x21, x21, %x[ld_in_row], LSL #2\n"
".inst 0xc1371a40 // fmla za.s[x8, 0], { z18.s-z21.s }, z7.s\n"
"mov x12, #0x0\n"
"ld1w { z22.s }, p0/Z, [x21]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
".inst 0xc1331ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z3.s\n"
"ld1w { z29.s }, p0/Z, [x13]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x13, x13, %x[ld_in_col], LSL #2\n"
".inst 0xc1351a60 // fmla za.s[x8, 0], { z19.s-z22.s }, z5.s\n"
"ld1w { z22.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z30.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x4\n"
"ld1w { z23.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z31.s }, p0/Z, [x20]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z24.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z0.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x8\n"
"ld1w { z25.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z1.s }, p0/Z, [x20]\n"
"bgt 14b\n"
"15:" // Main loop tail
"mov x12, #0x0\n"
".inst 0xc13a1ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z10.s\n"
"add x20, x13, %x[ld_in_row], LSL #2\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
".inst 0xc1361ba1 // fmla za.s[x8, 1], { z29.s-z0.s }, z6.s\n"
"ld1w { z29.s }, p0/Z, [x13]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
".inst 0xc13b1ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z11.s\n"
"add x13, x13, %x[ld_in_col], LSL #2\n"
".inst 0xc1321ac1 // fmla za.s[x8, 1], { z22.s-z25.s }, z2.s\n"
"ld1w { z22.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
".inst 0xc1391bc0 // fmla za.s[x8, 0], { z30.s-z1.s }, z9.s\n"
".inst 0xc1341bc1 // fmla za.s[x8, 1], { z30.s-z1.s }, z4.s\n"
"ld1w { z30.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x4\n"
".inst 0xc0060c10 // mova { z16.d-z19.d }, za.d[x8, #0]\n"
"add x8, x8, #0x1\n"
"ld1w { z23.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
".inst 0xc0040d82 // mova za.d[x8, #2], { z12.d-z15.d }\n"
"ld1w { z31.s }, p0/Z, [x20]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
".inst 0xc1a8cb90 // fclamp { z16.s-z19.s }, z28.s, z8.s\n"
"ld1w { z24.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"st1w { z16.s }, p1, [x10]\n"
"add x10, x10, x28, LSL #2\n"
"st1w { z17.s }, p1, [x9]\n"
"add x9, x9, x27, LSL #2\n"
"ld1w { z0.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"st1w { z18.s }, p1, [x26]\n"
"mov x12, #0x8\n"
Expand All @@ -542,7 +544,7 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl(
"add x25, x25, x23, LSL #2\n"
".inst 0xc1371ba0 // fmla za.s[x8, 0], { z29.s-z0.s }, z7.s\n"
"ld1w { z25.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z1.s }, p0/Z, [x20]\n"
".inst 0xc1331ac0 // fmla za.s[x8, 0], { z22.s-z25.s }, z3.s\n"
Expand All @@ -551,33 +553,33 @@ void sme2_fp32_planar_3x3_s2_4rows_mla_za_impl(
"cbz x14, 17f\n" // Skip remainder inputs
"mov x12, #0x0\n"
"add x20, x13, %x[ld_in_row], LSL #2\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"sub x11, x11, #0x1\n"
"ld1w { z23.s }, p0/Z, [x13]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"ld1w { z16.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z24.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x4\n"
"ld1w { z17.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z25.s }, p0/Z, [x20]\n"
".inst 0x25704500 // psel p0.s, p1.s/Z, p8.s[w12, #1]\n"
".inst 0x252c4500 // psel p0, p1, p8.b[w12, #1]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z18.s }, p0/Z, [x20]\n"
".inst 0x25b04500 // psel p0.s, p1.s/Z, p8.s[w12, #2]\n"
".inst 0x25344500 // psel p0, p1, p8.b[w12, #2]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"ld1w { z26.s }, p0/Z, [x20]\n"
".inst 0x25f04500 // psel p0.s, p1.s/Z, p8.s[w12, #3]\n"
".inst 0x253c4500 // psel p0, p1, p8.b[w12, #3]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
"mov x12, #0x8\n"
".inst 0xc13a1ae0 // fmla za.s[x8, 0], { z23.s-z26.s }, z10.s\n"
"ld1w { z19.s }, p0/Z, [x20]\n"
".inst 0x25304500 // psel p0.s, p1.s/Z, p8.s[w12]\n"
".inst 0x25244500 // psel p0, p1, p8.b[w12]\n"
"add x20, x20, %x[ld_in_row], LSL #2\n"
".inst 0xc1361ae1 // fmla za.s[x8, 1], { z23.s-z26.s }, z6.s\n"
"ld1w { z27.s }, p0/Z, [x20]\n"
Expand Down
19 changes: 19 additions & 0 deletions tests/validation/NEON/DepthwiseConvolutionLayer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,25 @@ using NEDepthwiseConvolutionLayerVariableWeightsFixture =
TEST_SUITE(Float)
TEST_SUITE(F32)

// Exercise the ninth input row in the SME2 planar 3x3/stride-2 padding path.
FIXTURE_DATA_TEST_CASE_NEW(
Run3x3Stride2RowPadding,
NEDepthwiseConvolutionLayerFixture<float>,
framework::DatasetMode::PRECOMMIT,
combine(make("In", {TensorShape(8U, 8U, 64U), TensorShape(9U, 9U, 17U), TensorShape(112U, 112U, 64U)}),
make("Weights", Size2D(3U, 3U)),
make("Info",
{PadStrideInfo(2, 2, 0, 1, 0, 1, DimensionRoundingType::FLOOR),
PadStrideInfo(2, 2, 1, 1, 1, 1, DimensionRoundingType::FLOOR)}),
make("Dilation", Size2D(1U, 1U)),
make("DepthMultiplier", {1}),
make("DataType", DataType::F32),
make("DataLayout", {DataLayout::NHWC}),
ActivationFunctionsDataset))
{
validate(Accessor(_target), _reference, tolerance_f32);
}

FIXTURE_DATA_TEST_CASE_NEW(RunActivations,
NEDepthwiseConvolutionLayerFixture<float>,
framework::DatasetMode::NIGHTLY,
Expand Down