Skip to content

Commit 13182b2

Browse files
martin-frbgmartin-frbg
authored
Merge pull request #3378 from martin-frbg/issue3368-2
Rework generation of BFLOAT16 objects in CMAKE builds and fix missing CBLAS_XERBLA
2 parents af88438 + ddf106f commit 13182b2

7 files changed

Lines changed: 110 additions & 26 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ endif ()
132132

133133
if (BUILD_BFLOAT16)
134134
message(STATUS "Building Half Precision")
135-
list(APPEND FLOAT_TYPES "BFLOAT16") # defines nothing
135+
# list(APPEND FLOAT_TYPES "BFLOAT16") # defines nothing
136136
endif ()
137137

138138
if (NOT DEFINED CORE OR "${CORE}" STREQUAL "UNKNOWN")

‎cmake/kernel.cmake‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ if (BUILD_BFLOAT16)
134134
set(SHSWAPKERNEL ../arm/swap.c)
135135
set(TOBF16KERNEL ../x86_64/tobf16.c)
136136
set(BF16TOKERNEL ../x86_64/bf16to.c)
137+
set(SBGEMVNKERNEL ../x86_64/sbgemv_n.c)
138+
set(SBGEMVTKERNEL ../x86_64/sbgemv_t.c)
137139
endif ()
138140
endmacro ()
139141

‎cmake/system.cmake‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,9 @@ endif()
469469
if (BUILD_COMPLEX16)
470470
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBUILD_COMPLEX16")
471471
endif()
472+
if (BUILD_BFLOAT16)
473+
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DBUILD_BFLOAT16")
474+
endif()
472475
if(NOT MSVC)
473476
set(CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${CCOMMON_OPT}")
474477
endif()

‎driver/level2/CMakeLists.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ foreach (float_type ${FLOAT_TYPES})
8181
GenerateNamedObjects("gbmv_thread.c" "TRANSA" "gbmv_thread_t" false "" "" false ${float_type})
8282
endif ()
8383

84+
# special defines for complex
8485
if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX")
8586

8687
foreach (u_source ${U_SOURCES})
@@ -197,6 +198,13 @@ foreach (float_type ${FLOAT_TYPES})
197198
endif ()
198199
endforeach ()
199200

201+
if (BUILD_BFLOAT16)
202+
if (USE_THREAD)
203+
GenerateNamedObjects("sbgemv_thread.c" "" "gemv_thread_n" false "" "" false "BFLOAT16")
204+
GenerateNamedObjects("sbgemv_thread.c" "TRANSA" "gemv_thread_t" false "" "" false "BFLOAT16")
205+
endif ()
206+
endif ()
207+
200208
if ( BUILD_COMPLEX AND NOT BUILD_SINGLE)
201209
if (USE_THREAD)
202210
GenerateNamedObjects("gemv_thread.c" "" "gemv_thread_n" false "" "" false "SINGLE")

‎driver/level3/CMakeLists.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,12 @@ foreach (GEMM_DEFINE ${GEMM_DEFINES})
1212
if (USE_THREAD AND NOT USE_SIMPLE_THREADED_LEVEL3)
1313
GenerateNamedObjects("gemm.c" "${GEMM_DEFINE};THREADED_LEVEL3" "gemm_thread_${GEMM_DEFINE_LC}" 0)
1414
endif ()
15+
if (BUILD_BFLOAT16)
16+
GenerateNamedObjects("gemm.c" "${GEMM_DEFINE}" "gemm_${GEMM_DEFINE_LC}" 0 "" "" false "BFLOAT16")
17+
if (USE_THREAD AND NOT USE_SIMPLE_THREADED_LEVEL3)
18+
GenerateNamedObjects("gemm.c" "${GEMM_DEFINE};THREADED_LEVEL3" "gemm_thread_${GEMM_DEFINE_LC}" 0 "" "" false "BFLOAT16")
19+
endif ()
20+
endif ()
1521
endforeach ()
1622

1723
if ( BUILD_COMPLEX16 AND NOT BUILD_DOUBLE)

‎interface/CMakeLists.txt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ foreach (CBLAS_FLAG ${CBLAS_FLAGS})
8282
GenerateNamedObjects("${BLAS3_SOURCES}" "" "" ${CBLAS_FLAG} "" "" false ${DISABLE_COMPLEX})
8383
GenerateNamedObjects("${BLAS3_MANGLED_SOURCES}" "" "" ${CBLAS_FLAG} "" "" false ${MANGLE_COMPLEX})
8484

85+
GenerateNamedObjects("xerbla.c" "" "xerbla" ${CBLAS_FLAG} "" "" true)
8586
#sdsdot, dsdot
8687
if (BUILD_SINGLE OR BUILD_DOUBLE)
8788
GenerateNamedObjects("sdsdot.c" "" "sdsdot" ${CBLAS_FLAG} "" "" true "SINGLE")
@@ -104,6 +105,15 @@ endif ()
104105
GenerateNamedObjects("imax.c" "USE_ABS;USE_MIN" "i*amin" ${CBLAS_FLAG})
105106
GenerateNamedObjects("imax.c" "USE_MIN" "i*min" ${CBLAS_FLAG})
106107

108+
if (BUILD_BFLOAT16)
109+
GenerateNamedObjects("bf16dot.c" "" "sbdot" ${CBLAS_FLAG} "" "" true "BFLOAT16")
110+
GenerateNamedObjects("gemm.c" "" "sbgemm" ${CBLAS_FLAG} "" "" true "BFLOAT16")
111+
GenerateNamedObjects("sbgemv.c" "" "sbgemv" ${CBLAS_FLAG} "" "" true "BFLOAT16")
112+
GenerateNamedObjects("tobf16.c" "SINGLE_PREC" "sbstobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16")
113+
GenerateNamedObjects("tobf16.c" "DOUBLE_PREC" "sbdtobf16" ${CBLAS_FLAG} "" "" true "BFLOAT16")
114+
GenerateNamedObjects("bf16to.c" "SINGLE_PREC" "sbf16tos" ${CBLAS_FLAG} "" "" true "BFLOAT16")
115+
GenerateNamedObjects("bf16to.c" "DOUBLE_PREC" "dbf16tod" ${CBLAS_FLAG} "" "" true "BFLOAT16")
116+
endif ()
107117

108118
# complex-specific sources
109119
foreach (float_type ${FLOAT_TYPES})

‎kernel/CMakeLists.txt‎

Lines changed: 80 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,15 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
9191
GenerateNamedObjects("${KERNELDIR}/${DSDOTKERNEL}" "DSDOT" "d*dot_k" false "" "" false "SINGLE")
9292
GenerateNamedObjects("${KERNELDIR}/${DSDOTKERNEL}" "DSDOT" "dsdot_k" false "" "" false "SINGLE")
9393

94+
# sbdot
95+
if (BUILD_BFLOAT16)
96+
GenerateNamedObjects("${KERNELDIR}/${SBDOTKERNEL}" "SBDOT" "dot_k" false "" "" false "BFLOAT16")
97+
GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "SINGLE" "f16tos_k" false "" "" false "BFLOAT16")
98+
GenerateNamedObjects("${KERNELDIR}/${BF16TOKERNEL}" "DOUBLE" "bf16tod_k" false "" "" false "DOUBLE")
99+
GenerateNamedObjects("${KERNELDIR}/${TOBF16KERNEL}" "SINGLE" "stobf16_k" false "" "" false "BFLOAT16")
100+
GenerateNamedObjects("${KERNELDIR}/${TOBF16KERNEL}" "DOUBLE" "dtobf16_k" false "" "" false "BFLOAT16")
101+
endif()
102+
94103
if ((BUILD_COMPLEX OR BUILD_DOUBLE) AND NOT BUILD_SINGLE)
95104
GenerateNamedObjects("${KERNELDIR}/${SAMAXKERNEL}" "USE_ABS" "amax_k" false "" "" false "SINGLE")
96105
GenerateNamedObjects("${KERNELDIR}/${SAMINKERNEL}" "USE_ABS;USE_MIN" "amin_k" false "" "" false "SINGLE")
@@ -149,9 +158,6 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
149158
GenerateNamedObjects("generic/ger.c" "" "ger_k" false "" "" "" 3)
150159
foreach (float_type ${FLOAT_TYPES})
151160
string(SUBSTRING ${float_type} 0 1 float_char)
152-
if (${float_type} STREQUAL "BFLOAT16")
153-
set (float_char "SB")
154-
endif ()
155161
if (${float_type} STREQUAL "COMPLEX" OR ${float_type} STREQUAL "ZCOMPLEX")
156162
GenerateNamedObjects("${KERNELDIR}/${${float_char}GERUKERNEL}" "" "geru_k" false "" "" false ${float_type})
157163
GenerateNamedObjects("${KERNELDIR}/${${float_char}GERCKERNEL}" "CONJ" "gerc_k" false "" "" false ${float_type})
@@ -185,6 +191,10 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
185191
GenerateNamedObjects("${KERNELDIR}/${SGEMVNKERNEL}" "" "gemv_n" false "" "" false "SINGLE")
186192
GenerateNamedObjects("${KERNELDIR}/${SGEMVTKERNEL}" "TRANS" "gemv_t" false "" "" false "SINGLE")
187193
endif ()
194+
if (BUILD_BFLOAT16)
195+
GenerateNamedObjects("${KERNELDIR}/${SBGEMVNKERNEL}" "" "gemv_n" false "" "" false "BFLOAT16")
196+
GenerateNamedObjects("${KERNELDIR}/${SBGEMVTKERNEL}" "" "gemv_t" false "" "" false "BFLOAT16")
197+
endif ()
188198
# Makefile.L3
189199
set(USE_TRMM false)
190200
string(TOUPPER ${TARGET_CORE} UC_TARGET_CORE)
@@ -209,15 +219,8 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
209219
GenerateNamedObjects("${KERNELDIR}/${SGEMMDIRECTPERFORMANT}" "" "gemm_direct_performant" false "" "" false SINGLE)
210220
endif()
211221

212-
foreach (float_type SINGLE DOUBLE BFLOAT16)
222+
foreach (float_type SINGLE DOUBLE)
213223
string(SUBSTRING ${float_type} 0 1 float_char)
214-
if (${float_type} STREQUAL "BFLOAT16")
215-
if (NOT ${BUILD_BFLOAT16})
216-
continue ()
217-
else ()
218-
set (float_char "SB")
219-
endif ()
220-
endif ()
221224
GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMKERNEL}" "" "gemm_kernel" false "" "" false ${float_type})
222225
endforeach()
223226
if (BUILD_COMPLEX16 AND NOT BUILD_DOUBLE)
@@ -253,11 +256,24 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
253256
GenerateNamedObjects("${KERNELDIR}/${SGEMM_BETA}" "" "gemm_beta" false "" "" false "SINGLE")
254257
endif ()
255258

259+
if (BUILD_BFLOAT16)
260+
if (SBGEMMINCOPY)
261+
GenerateNamedObjects("${KERNELDIR}/${SBGEMMINCOPY}" "" "${SBGEMMINCOPYOBJ}" false "" "" true "BFLOAT16")
262+
endif ()
263+
if (SBGEMMITCOPY)
264+
GenerateNamedObjects("${KERNELDIR}/${SBGEMMITCOPY}" "" "${SBGEMMITCOPYOBJ}" false "" "" true "BFLOAT16")
265+
endif ()
266+
if (SBGEMMONCOPY)
267+
GenerateNamedObjects("${KERNELDIR}/${SBGEMMONCOPY}" "" "${SBGEMMONCOPYOBJ}" false "" "" true "BFLOAT16")
268+
endif ()
269+
if (SBGEMMOTCOPY)
270+
GenerateNamedObjects("${KERNELDIR}/${SBGEMMOTCOPY}" "" "${SBGEMMOTCOPYOBJ}" false "" "" true "BFLOAT16")
271+
endif ()
272+
GenerateNamedObjects("${KERNELDIR}/${SBGEMMKERNEL}" "" "gemm_kernel" false "" "" false "BFLOAT16")
273+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_BETA}" "" "gemm_beta" false "" "" false "BFLOAT16")
274+
endif ()
256275
foreach (float_type ${FLOAT_TYPES})
257276
string(SUBSTRING ${float_type} 0 1 float_char)
258-
if (${float_type} STREQUAL "BFLOAT16")
259-
set (float_char "SB")
260-
endif ()
261277
if (${float_char}GEMMINCOPY)
262278
GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMMINCOPY}" "${float_type}" "${${float_char}GEMMINCOPYOBJ}" false "" "" true ${float_type})
263279
endif ()
@@ -568,6 +584,44 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
568584
GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_TN}" "B0" "gemm_small_kernel_b0_tn" false "" "" false ${float_type})
569585
GenerateNamedObjects("${KERNELDIR}/${${float_char}GEMM_SMALL_K_B0_NT}" "B0" "gemm_small_kernel_b0_tt" false "" "" false ${float_type})
570586
endif ()
587+
if (BUILD_BFLOAT16)
588+
if (NOT DEFINED SBGEMM_SMALL_M_PERMIT)
589+
set(SBGEMM_SMALL_M_PERMIT ../generic/gemm_small_matrix_permit.c)
590+
endif ()
591+
if (NOT DEFINED SBGEMM_SMALL_K_NN)
592+
set(SBGEMM_SMALL_K_NN ../generic/gemm_small_matrix_kernel_nn.c)
593+
endif ()
594+
if (NOT DEFINED SBGEMM_SMALL_K_NT)
595+
set(SBGEMM_SMALL_K_NT ../generic/gemm_small_matrix_kernel_nt.c)
596+
endif ()
597+
if (NOT DEFINED SBGEMM_SMALL_K_TN)
598+
set(SBGEMM_SMALL_K_TN ../generic/gemm_small_matrix_kernel_tn.c)
599+
endif ()
600+
if (NOT DEFINED SBGEMM_SMALL_K_TT)
601+
set(SBGEMM_SMALL_K_TT ../generic/gemm_small_matrix_kernel_tt.c)
602+
endif ()
603+
if (NOT DEFINED SBGEMM_SMALL_K_B0_NN)
604+
set(SBGEMM_SMALL_K_B0_NN ../generic/gemm_small_matrix_kernel_nn.c)
605+
endif ()
606+
if (NOT DEFINED SBGEMM_SMALL_K_B0_NT)
607+
set(SBGEMM_SMALL_K_B0_NT ../generic/gemm_small_matrix_kernel_nt.c)
608+
endif ()
609+
if (NOT DEFINED SBGEMM_SMALL_K_B0_TN)
610+
set(SBGEMM_SMALL_K_B0_TN ../generic/gemm_small_matrix_kernel_tn.c)
611+
endif ()
612+
if (NOT DEFINED SBGEMM_SMALL_K_B0_TT)
613+
set($SBGEMM_SMALL_K_B0_TT ../generic/gemm_small_matrix_kernel_tt.c)
614+
endif ()
615+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_M_PERMIT}" "" "gemm_small_matrix_permit" false "" "" false "BFLOAT16")
616+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_NN}" "" "gemm_small_kernel_nn" false "" "" false "BFLOAT16")
617+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_NT}" "" "gemm_small_kernel_nt" false "" "" false "BFLOAT16")
618+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_TN}" "" "gemm_small_kernel_tn" false "" "" false "BFLOAT16")
619+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_NT}" "" "gemm_small_kernel_tt" false "" "" false "BFLOAT16")
620+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_B0_NN}" "B0" "gemm_small_kernel_b0_nn" false "" "" false "BFLOAT16")
621+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_B0_NT}" "B0" "gemm_small_kernel_b0_nt" false "" "" false "BFLOAT16")
622+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_B0_TN}" "B0" "gemm_small_kernel_b0_tn" false "" "" false "BFLOAT16")
623+
GenerateNamedObjects("${KERNELDIR}/${SBGEMM_SMALL_K_B0_NT}" "B0" "gemm_small_kernel_b0_tt" false "" "" false "BFLOAT16")
624+
endif ()
571625
endif ()
572626

573627
if (NOT DEFINED ${float_char}OMATCOPY_CN)
@@ -702,6 +756,7 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
702756
#geadd
703757
GenerateNamedObjects("${KERNELDIR}/${${float_char}GEADD_KERNEL}" "" "geadd_k" false "" "" false ${float_type})
704758
endforeach ()
759+
705760
if (BUILD_DOUBLE AND NOT BUILD_SINGLE)
706761
GenerateNamedObjects("${KERNELDIR}/${STRSMKERNEL_LN}" "UPPER;LN;TRSMKERNEL" "trsm_kernel_LN" false "" "" false "SINGLE")
707762
GenerateNamedObjects("${KERNELDIR}/${STRSMKERNEL_LT}" "LT;TRSMKERNEL" "trsm_kernel_LT" false "" "" false "SINGLE")
@@ -840,22 +895,22 @@ function (build_core TARGET_CORE KDIR TSUFFIX KERNEL_DEFINITIONS)
840895
GenerateNamedObjects("generic/trsm_ltcopy_${SGEMM_UNROLL_N}.c" "OUTER;LOWER" "trsm_oltncopy" false "" ${TSUFFIX} false "SINGLE")
841896

842897
if (SGEMMINCOPY)
843-
GenerateNamedObjects("${KERNELDIR}/${SGEMMINCOPY}" "SINGLE" "${SGEMMINCOPYOBJ}" false "" "" true "SINGLE")
898+
GenerateNamedObjects("${KERNELDIR}/${SGEMMINCOPY}" "SINGLE" "${SGEMMINCOPYOBJ}" false "" "" true "SINGLE")
844899
endif ()
845-
if (SGEMMITCOPY)
846-
GenerateNamedObjects("${KERNELDIR}/${SGEMMITCOPY}" "SINGLE" "${SGEMMITCOPYOBJ}" false "" "" true "SINGLE")
847-
endif ()
848-
if (SGEMMONCOPY)
849-
GenerateNamedObjects("${KERNELDIR}/${SGEMMONCOPY}" "SINGLE" "${SGEMMONCOPYOBJ}" false "" "" true "SINGLE")
850-
endif ()
851-
if (SGEMMOTCOPY)
852-
GenerateNamedObjects("${KERNELDIR}/${SGEMMOTCOPY}" "SINGLE" "${SGEMMOTCOPYOBJ}" false "" "" true "SINGLE")
900+
if (SGEMMITCOPY)
901+
GenerateNamedObjects("${KERNELDIR}/${SGEMMITCOPY}" "SINGLE" "${SGEMMITCOPYOBJ}" false "" "" true "SINGLE")
902+
endif ()
903+
if (SGEMMONCOPY)
904+
GenerateNamedObjects("${KERNELDIR}/${SGEMMONCOPY}" "SINGLE" "${SGEMMONCOPYOBJ}" false "" "" true "SINGLE")
905+
endif ()
906+
if (SGEMMOTCOPY)
907+
GenerateNamedObjects("${KERNELDIR}/${SGEMMOTCOPY}" "SINGLE" "${SGEMMOTCOPYOBJ}" false "" "" true "SINGLE")
853908
endif ()
854909
GenerateNamedObjects("${KERNELDIR}/${SGEMVNKERNEL}" "" "gemv_n" false "" "" false "SINGLE")
855910
GenerateNamedObjects("${KERNELDIR}/${SGEMVTKERNEL}" "TRANS" "gemv_t" false "" "" false "SINGLE")
856911
endif ()
857-
858-
if (BUILD_COMPLEX16 AND NOT BUILD_DOUBLE)
912+
913+
if (BUILD_COMPLEX16 AND NOT BUILD_DOUBLE)
859914
GenerateNamedObjects("generic/neg_tcopy_${DGEMM_UNROLL_M}.c" "" "neg_tcopy" false "" ${TSUFFIX} false "DOUBLE")
860915
GenerateNamedObjects("generic/laswp_ncopy_${DGEMM_UNROLL_N}.c" "" "laswp_ncopy" false "" ${TSUFFIX} false "DOUBLE")
861916
endif ()

0 commit comments

Comments
 (0)