Merge pull request #3672 from imzhuhl/neoversen2_bf16

sbgemm support for ARM Neoverse N2

Author: martin-frbg

Makefile.arm64

@@ -121,9 +121,9 @@ ifeq ($(CORE), NEOVERSEN2)
 ifeq (1, $(filter 1,$(GCCVERSIONGTEQ7) $(ISCLANG)))
 ifeq ($(GCCVERSIONGTEQ9), 1)
 ifeq (1, $(filter 1,$(GCCMINORVERSIONGTEQ4) $(GCCVERSIONGTEQ10)))
-CCOMMON_OPT += -march=armv8.5-a -mtune=neoverse-n2
+CCOMMON_OPT += -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2
 ifneq ($(F_COMPILER), NAG)
-FCOMMON_OPT += -march=armv8.5-a -mtune=neoverse-n2
+FCOMMON_OPT += -march=armv8.5-a+sve+sve2+bf16 -mtune=neoverse-n2
 endif
 else
 CCOMMON_OPT += -march=armv8.5-a -mtune=native

kernel/arm64/KERNEL.NEOVERSEN2

@@ -187,3 +187,14 @@ ZGEMMONCOPY    =  ../generic/zgemm_ncopy_$(ZGEMM_UNROLL_N).c
 ZGEMMOTCOPY    =  ../generic/zgemm_tcopy_$(ZGEMM_UNROLL_N).c
 ZGEMMONCOPYOBJ =  zgemm_oncopy$(TSUFFIX).$(SUFFIX)
 ZGEMMOTCOPYOBJ =  zgemm_otcopy$(TSUFFIX).$(SUFFIX)
+
+SBGEMM_BETA    =  sbgemm_beta_neoversen2.c
+SBGEMMKERNEL    = sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_neoversen2.c
+SBGEMMINCOPY    = sbgemm_ncopy_neoversen2.c
+SBGEMMITCOPY    = sbgemm_tcopy_neoversen2.c
+SBGEMMONCOPY    = sbgemm_ncopy_neoversen2.c
+SBGEMMOTCOPY    = sbgemm_tcopy_neoversen2.c
+SBGEMMINCOPYOBJ =  sbgemm_incopy$(TSUFFIX).$(SUFFIX)
+SBGEMMITCOPYOBJ =  sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
+SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
+SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)

kernel/arm64/sbgemm_beta_neoversen2.c

@@ -0,0 +1,83 @@
+/***************************************************************************
+ * Copyright (c) 2022, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#include "common.h"
+
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG dummy1, FLOAT beta, IFLOAT *dummy2,
+          BLASLONG dummy3, IFLOAT *dummy4, BLASLONG dummy5, FLOAT *c,
+          BLASLONG ldc) {
+
+    BLASLONG i, j;
+    BLASLONG chunk, remain;
+    FLOAT *c_offset1, *c_offset;
+    c_offset = c;
+    chunk = m >> 3;
+    remain = m & 7;
+    if (beta == ZERO) {
+        for (j = n; j > 0; j--) {
+            c_offset1 = c_offset;
+            c_offset += ldc;
+            for (i = chunk; i > 0; i--) {
+                *(c_offset1 + 0) = ZERO;
+                *(c_offset1 + 1) = ZERO;
+                *(c_offset1 + 2) = ZERO;
+                *(c_offset1 + 3) = ZERO;
+                *(c_offset1 + 4) = ZERO;
+                *(c_offset1 + 5) = ZERO;
+                *(c_offset1 + 6) = ZERO;
+                *(c_offset1 + 7) = ZERO;
+                c_offset1 += 8;
+            }
+            for (i = remain; i > 0; i--) {
+                *c_offset1 = ZERO;
+                c_offset1++;
+            }
+        }
+    } else {
+        for (j = n; j > 0; j--) {
+            c_offset1 = c_offset;
+            c_offset += ldc;
+            for (i = chunk; i > 0; i--) {
+                *(c_offset1 + 0) *= beta;
+                *(c_offset1 + 1) *= beta;
+                *(c_offset1 + 2) *= beta;
+                *(c_offset1 + 3) *= beta;
+                *(c_offset1 + 4) *= beta;
+                *(c_offset1 + 5) *= beta;
+                *(c_offset1 + 6) *= beta;
+                *(c_offset1 + 7) *= beta;
+                c_offset1 += 8;
+            }
+            for (i = remain; i > 0; i--) {
+                *c_offset1 *= beta;
+                c_offset1++;
+            }
+        }
+    }
+    return 0;
+};

kernel/arm64/sbgemm_kernel_8x4_neoversen2.c

@@ -0,0 +1,45 @@
+/***************************************************************************
+ * Copyright (c) 2022, The OpenBLAS Project
+ * All rights reserved.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * 3. Neither the name of the OpenBLAS project nor the names of
+ * its contributors may be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ * *****************************************************************************/
+
+#include <arm_sve.h>
+
+#include "common.h"
+
+#define ALPHA_ONE
+#include "sbgemm_kernel_8x4_neoversen2_impl.c"
+#undef ALPHA_ONE
+#include "sbgemm_kernel_8x4_neoversen2_impl.c"
+
+int CNAME(BLASLONG m, BLASLONG n, BLASLONG k, FLOAT alpha, IFLOAT *A, IFLOAT *B,
+          FLOAT *C, BLASLONG ldc) {
+    if (alpha == 1.0f)
+        return sbgemm_kernel_neoversen2_alpha_one(m, n, k, alpha, A, B, C, ldc);
+    else
+        return sbgemm_kernel_neoversen2_alpha(m, n, k, alpha, A, B, C, ldc);
+    return 0;
+}