Merge pull request #94 from xianyi/develop

rebase

Author: martin-frbg

Makefile.rule

@@ -279,7 +279,22 @@ COMMON_PROF = -pg
 
 # If you want to enable the experimental BFLOAT16 support
 # BUILD_HALF = 1
-#
+
+
+# Set the thread number threshold beyond which the job array for the threaded level3 BLAS
+# will be allocated on the heap rather than the stack. (This array alone requires 
+# NUM_THREADS*NUM_THREADS*128 bytes of memory so should not pose a problem at low cpu
+# counts, but obviously it is not the only item that ends up on the stack.
+# The default value of 32 ensures that the overall requirement is compatible
+# with the default 1MB stacksize imposed by having the Java VM loaded without use
+# of its -Xss parameter.
+# The value of 160 formerly used from about version 0.2.7 until 0.3.10 is easily compatible
+# with the common Linux stacksize of 8MB but will cause crashes with unwary use of the java
+# VM e.g. in Octave or with the java-based libhdfs in numpy or scipy code
+# BLAS3_MEM_ALLOC_THRESHOLD = 160
+
+
+
 # the below is not yet configurable, use cmake if you need to build only select types
 BUILD_SINGLE = 1
 BUILD_DOUBLE = 1

Makefile.x86_64

@@ -8,6 +8,11 @@ endif
 endif
 endif
 
+ifdef HAVE_SSE3
+CCOMMON_OPT += -msse3
+FCOMMON_OPT += -msse3
+endif
+
 ifeq ($(CORE), SKYLAKEX)
 ifndef DYNAMIC_ARCH
 ifndef NO_AVX512

cmake/system.cmake

@@ -70,6 +70,9 @@ if (DEFINED TARGET)
       set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -mavx2")
     endif()
   endif()
+  if (DEFINED HAVE_SSE3)
+    set (KERNEL_DEFINITIONS "${KERNEL_DEFINITIONS} -msse3")
+  endif()
 endif()
 
 if (DEFINED TARGET)
@@ -323,7 +326,13 @@ else ()
 set(CCOMMON_OPT "${CCOMMON_OPT} -DMAX_STACK_ALLOC=2048")
 endif ()
 endif ()
-
+if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows")
+if (DEFINED BLAS3_MEM_ALLOC_THRESHOLD)
+if (NOT ${BLAS3_MEM_ALLOC_THRESHOLD} EQUAL 32)
+set(CCOMMON_OPT "${CCOMMON_OPT} -DBLAS3_MEM_ALLOC_THRESHOLD=${BLAS3_MEM_ALLOC_THRESHOLD}")
+endif()
+endif()
+endif()
 if (DEFINED LIBNAMESUFFIX)
   set(LIBPREFIX "libopenblas_${LIBNAMESUFFIX}")
 else ()

common.h

@@ -402,7 +402,7 @@ please https://github.com/xianyi/OpenBLAS/issues/246
 #endif
 
 #ifndef BLAS3_MEM_ALLOC_THRESHOLD
-#define BLAS3_MEM_ALLOC_THRESHOLD 160
+#define BLAS3_MEM_ALLOC_THRESHOLD 32 
 #endif
 
 #ifdef QUAD_PRECISION

kernel/Makefile

@@ -5,6 +5,9 @@ endif
 TOPDIR	= ..
 include $(TOPDIR)/Makefile.system
 
+ifdef HAVE_SSE3
+CFLAGS += -msse3
+endif
 
 ifeq ($(C_COMPILER), GCC)
 GCCVERSIONGTEQ9 := $(shell expr `$(CC) -dumpversion | cut -f1 -d.` \>= 9)

kernel/arm64/KERNEL.ARMV8

@@ -97,7 +97,7 @@ CNRM2KERNEL    = znrm2.S
 ZNRM2KERNEL    = znrm2.S
 
 DDOTKERNEL     = dot.S
-SDOTKERNEL     = dot.S
+SDOTKERNEL     = ../generic/dot.c
 CDOTKERNEL     = zdot.S
 ZDOTKERNEL     = zdot.S
 DSDOTKERNEL    = dot.S

kernel/arm64/KERNEL.CORTEXA53

@@ -97,7 +97,7 @@ CNRM2KERNEL    = znrm2.S
 ZNRM2KERNEL    = znrm2.S
 
 DDOTKERNEL     = dot.S
-SDOTKERNEL     = dot.S
+SDOTKERNEL     = ../generic/dot.c
 CDOTKERNEL     = zdot.S
 ZDOTKERNEL     = zdot.S
 DSDOTKERNEL    = dot.S

kernel/arm64/KERNEL.CORTEXA57

@@ -70,7 +70,7 @@ DCOPYKERNEL  = copy.S
 CCOPYKERNEL  = copy.S
 ZCOPYKERNEL  = copy.S
 
-SDOTKERNEL   = dot.S
+SDOTKERNEL   = ../generic/dot.c
 DDOTKERNEL   = dot.S
 CDOTKERNEL   = zdot.S
 ZDOTKERNEL   = zdot.S

kernel/generic/dot.c

@@ -27,7 +27,7 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 #include "common.h"
-
+#include "../simd/intrin.h"
 #if defined(DSDOT)
 double CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 #else
@@ -47,27 +47,59 @@ FLOAT CNAME(BLASLONG n, FLOAT *x, BLASLONG inc_x, FLOAT *y, BLASLONG inc_y)
 
 	if ( (inc_x == 1) && (inc_y == 1) )
 	{
-
-		int n1 = n & -4;
-
-		while(i < n1)
+        int n1 = n & -4;
+#if V_SIMD && !defined(DSDOT)
+        const int vstep = v_nlanes_f32;
+        const int unrollx4 = n & (-vstep * 4);
+        const int unrollx  = n &  -vstep;
+		v_f32 vsum0 = v_zero_f32();
+        v_f32 vsum1 = v_zero_f32();
+        v_f32 vsum2 = v_zero_f32();
+        v_f32 vsum3 = v_zero_f32();
+		while(i < unrollx4)
+        {
+            vsum0 = v_muladd_f32(
+                v_loadu_f32(x + i),           v_loadu_f32(y + i),           vsum0
+            );
+            vsum1 = v_muladd_f32(
+                v_loadu_f32(x + i + vstep),   v_loadu_f32(y + i + vstep),   vsum1
+            );
+            vsum2 = v_muladd_f32(
+                v_loadu_f32(x + i + vstep*2), v_loadu_f32(y + i + vstep*2), vsum2
+            );
+            vsum3 = v_muladd_f32(
+                v_loadu_f32(x + i + vstep*3), v_loadu_f32(y + i + vstep*3), vsum3
+            );
+            i += vstep*4;
+        }
+        vsum0 = v_add_f32(
+            v_add_f32(vsum0, vsum1), v_add_f32(vsum2 , vsum3)
+        );
+		while(i < unrollx)
+        {
+            vsum0 = v_muladd_f32(
+                v_loadu_f32(x + i), v_loadu_f32(y + i), vsum0
+            );
+            i += vstep;
+        }
+        dot = v_sum_f32(vsum0);
+#elif defined(DSDOT)
+		for (; i < n1; i += 4)
 		{
-
-#if defined(DSDOT)
 			dot += (double) y[i] * (double) x[i]
 			    + (double) y[i+1] * (double) x[i+1]
 			    + (double) y[i+2] * (double) x[i+2]
 			    + (double) y[i+3] * (double) x[i+3] ;
+		}
 #else
+		for (; i < n1; i += 4)
+		{
 			dot += y[i] * x[i]
 			    + y[i+1] * x[i+1]
 			    + y[i+2] * x[i+2]
 			    + y[i+3] * x[i+3] ;
-#endif
-			i+=4 ;
-
 		}
-
+#endif
 		while(i < n)
 		{
 

kernel/simd/intrin.h

@@ -51,6 +51,11 @@ extern "C" {
 #include <immintrin.h>
 #endif
 
+/** NEON **/
+#ifdef HAVE_NEON
+#include <arm_neon.h>
+#endif
+
 // distribute
 #if defined(HAVE_AVX512VL) || defined(HAVE_AVX512BF16)
 #include "intrin_avx512.h"
@@ -60,6 +65,10 @@ extern "C" {
 #include "intrin_sse.h"
 #endif
 
+#ifdef HAVE_NEON
+#include "intrin_neon.h"
+#endif
+
 #ifndef V_SIMD
     #define V_SIMD 0
     #define V_SIMD_F64 0