Skip to content

Commit 5d6209e

Browse files
martin-frbgmartin-frbg
authored
Merge pull request #3055 from RajalakshmiSR/swapp10
Optimize swap function for POWER10
2 parents 7870275 + 601b711 commit 5d6209e

6 files changed

Lines changed: 280 additions & 4 deletions

File tree

‎kernel/power/cswap.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3636
#include "common.h"
3737

3838

39-
#if defined(POWER8) || defined(POWER9) || defined(POWER10)
4039
#if defined(__VEC__) || defined(__ALTIVEC__)
40+
#if defined(POWER8) || defined(POWER9)
4141
#include "cswap_microk_power8.c"
42+
#elif defined(POWER10)
43+
#include "cswap_microk_power10.c"
4244
#endif
4345
#endif
4446

‎kernel/power/cswap_microk_power10.c‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
/***************************************************************************
2+
Copyright (c) 2021, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
28+
#if defined(DOUBLE)
29+
#define HAVE_KERNEL_16 1
30+
static void zswap_kernel_16 (long n, double *x, double *y)
31+
#else
32+
#define HAVE_KERNEL_32 1
33+
static void cswap_kernel_32 (long n, float *x, float *y)
34+
#endif
35+
{
36+
__asm__
37+
(
38+
".align 5 \n"
39+
"one%=: \n\t"
40+
"lxvp 32, 0(%4) \n\t"
41+
"lxvp 34, 32(%4) \n\t"
42+
"lxvp 36, 64(%4) \n\t"
43+
"lxvp 38, 96(%4) \n\t"
44+
45+
"lxvp 40, 128(%4) \n\t"
46+
"lxvp 42, 160(%4) \n\t"
47+
"lxvp 44, 192(%4) \n\t"
48+
"lxvp 46, 224(%4) \n\t"
49+
50+
"lxvp 48, 0(%3) \n\t"
51+
"lxvp 50, 32(%3) \n\t"
52+
"lxvp 52, 64(%3) \n\t"
53+
"lxvp 54, 96(%3) \n\t"
54+
55+
"lxvp 56, 128(%3) \n\t"
56+
"lxvp 58, 160(%3) \n\t"
57+
"lxvp 60, 192(%3) \n\t"
58+
"lxvp 62, 224(%3) \n\t"
59+
60+
61+
"stxv 33, 0(%3) \n\t"
62+
"stxv 32, 16(%3) \n\t"
63+
"stxv 35, 32(%3) \n\t"
64+
"stxv 34, 48(%3) \n\t"
65+
"stxv 37, 64(%3) \n\t"
66+
"stxv 36, 80(%3) \n\t"
67+
"stxv 39, 96(%3) \n\t"
68+
"stxv 38, 112(%3) \n\t"
69+
70+
"addi %3, %3, 128 \n\t"
71+
72+
"stxv 41, 0(%3) \n\t"
73+
"stxv 40, 16(%3) \n\t"
74+
"stxv 43, 32(%3) \n\t"
75+
"stxv 42, 48(%3) \n\t"
76+
"stxv 45, 64(%3) \n\t"
77+
"stxv 44, 80(%3) \n\t"
78+
"stxv 47, 96(%3) \n\t"
79+
"stxv 46, 112(%3) \n\t"
80+
81+
"addi %3, %3, 128 \n\t"
82+
83+
"stxv 49, 0(%4) \n\t"
84+
"stxv 48, 16(%4) \n\t"
85+
"stxv 51, 32(%4) \n\t"
86+
"stxv 50, 48(%4) \n\t"
87+
"stxv 53, 64(%4) \n\t"
88+
"stxv 52, 80(%4) \n\t"
89+
"stxv 55, 96(%4) \n\t"
90+
"stxv 54, 112(%4) \n\t"
91+
92+
"addi %4, %4, 128 \n\t"
93+
94+
"stxv 57, 0(%4) \n\t"
95+
"stxv 56, 16(%4) \n\t"
96+
"stxv 59, 32(%4) \n\t"
97+
"stxv 58, 48(%4) \n\t"
98+
"stxv 61, 64(%4) \n\t"
99+
"stxv 60, 80(%4) \n\t"
100+
"stxv 63, 96(%4) \n\t"
101+
"stxv 62, 112(%4) \n\t"
102+
103+
"addi %4, %4, 128 \n\t"
104+
105+
#if defined(DOUBLE)
106+
"addic. %2, %2, -16 \n\t"
107+
#else
108+
"addic. %2, %2, -32 \n\t"
109+
#endif
110+
"bgt one%= \n"
111+
112+
"#n=%2 x=%0=%3 y=%1=%4"
113+
:
114+
"+m" (*x),
115+
"+m" (*y),
116+
"+r" (n), // 2
117+
"+b" (x), // 3
118+
"+b" (y) // 4
119+
:
120+
:
121+
"cr0",
122+
"vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",
123+
"vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47",
124+
"vs48","vs49","vs50","vs51","vs52","vs53","vs54","vs55",
125+
"vs56","vs57","vs58","vs59","vs60","vs61","vs62","vs63"
126+
);
127+
}

‎kernel/power/dswap.c‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3535

3636
#include "common.h"
3737

38-
#if defined(POWER8) || defined(POWER9) || defined(POWER10)
3938
#if defined(__VEC__) || defined(__ALTIVEC__)
39+
#if defined(POWER8) || defined(POWER9)
4040
#include "dswap_microk_power8.c"
41+
#elif defined(POWER10)
42+
#include "swap_microk_power10.c"
4143
#endif
4244
#endif
4345

@@ -115,12 +117,30 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT dummy3, FLOAT *x,
115117
if ( (inc_x == 1) && (inc_y == 1 ))
116118
{
117119

120+
#if defined(POWER10)
121+
if ( n >= 32 )
122+
{
123+
BLASLONG align = ((32 - ((uintptr_t)y & (uintptr_t)0x1F)) >> 3) & 0x3;
124+
for (i = 0; i < align; i++) {
125+
temp = y[i];
126+
y[i] = x[i];
127+
x[i] = temp;
128+
}
129+
}
130+
BLASLONG n1 = (n-i) & -32;
131+
if ( n1 > 0 )
132+
{
133+
dswap_kernel_32(n1,&x[i], &y[i]);
134+
i+=n1;
135+
}
136+
#else
118137
BLASLONG n1 = n & -32;
119138
if ( n1 > 0 )
120139
{
121140
dswap_kernel_32(n1, x, y);
122141
i=n1;
123142
}
143+
#endif
124144

125145
while(i < n)
126146
{

‎kernel/power/sswap.c‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3535

3636
#include "common.h"
3737

38-
#if defined(POWER8) || defined(POWER9) || defined(POWER10)
3938
#if defined(__VEC__) || defined(__ALTIVEC__)
39+
#if defined(POWER8) || defined(POWER9)
4040
#include "sswap_microk_power8.c"
41+
#elif defined(POWER10)
42+
#include "swap_microk_power10.c"
4143
#endif
4244
#endif
4345

@@ -115,12 +117,30 @@ int CNAME(BLASLONG n, BLASLONG dummy0, BLASLONG dummy1, FLOAT dummy3, FLOAT *x,
115117
if ( (inc_x == 1) && (inc_y == 1 ))
116118
{
117119

120+
#if defined(POWER10)
121+
if ( n >= 64 )
122+
{
123+
BLASLONG align = ((32 - ((uintptr_t)y & (uintptr_t)0x1F)) >> 2) & 0x7;
124+
for (i = 0; i < align; i++) {
125+
temp = y[i];
126+
y[i] = x[i];
127+
x[i] = temp;
128+
}
129+
}
130+
BLASLONG n1 = (n-i) & -64;
131+
if ( n1 > 0 )
132+
{
133+
sswap_kernel_32(n1,&x[i], &y[i]);
134+
i+=n1;
135+
}
136+
#else
118137
BLASLONG n1 = n & -32;
119138
if ( n1 > 0 )
120139
{
121140
sswap_kernel_32(n1, x, y);
122141
i=n1;
123142
}
143+
#endif
124144

125145
while(i < n)
126146
{

‎kernel/power/swap_microk_power10.c‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/***************************************************************************
2+
Copyright (c) 2021, The OpenBLAS Project
3+
All rights reserved.
4+
Redistribution and use in source and binary forms, with or without
5+
modification, are permitted provided that the following conditions are
6+
met:
7+
1. Redistributions of source code must retain the above copyright
8+
notice, this list of conditions and the following disclaimer.
9+
2. Redistributions in binary form must reproduce the above copyright
10+
notice, this list of conditions and the following disclaimer in
11+
the documentation and/or other materials provided with the
12+
distribution.
13+
3. Neither the name of the OpenBLAS project nor the names of
14+
its contributors may be used to endorse or promote products
15+
derived from this software without specific prior written permission.
16+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
17+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19+
ARE DISCLAIMED. IN NO EVENT SHALL THE OPENBLAS PROJECT OR CONTRIBUTORS BE
20+
LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
23+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
25+
USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
*****************************************************************************/
27+
#define HAVE_KERNEL_32 1
28+
29+
#if defined(DOUBLE)
30+
static void dswap_kernel_32 (long n, double *x, double *y)
31+
#else
32+
static void sswap_kernel_32 (long n, float *x, float *y)
33+
#endif
34+
{
35+
__asm__
36+
(
37+
".align 5 \n"
38+
"one%=: \n\t"
39+
40+
"lxvp 32, 0(%4) \n\t"
41+
"lxvp 34, 32(%4) \n\t"
42+
"lxvp 36, 64(%4) \n\t"
43+
"lxvp 38, 96(%4) \n\t"
44+
45+
"lxvp 40, 128(%4) \n\t"
46+
"lxvp 42, 160(%4) \n\t"
47+
"lxvp 44, 192(%4) \n\t"
48+
"lxvp 46, 224(%4) \n\t"
49+
50+
"lxvp 48, 0(%3) \n\t"
51+
"lxvp 50, 32(%3) \n\t"
52+
"lxvp 52, 64(%3) \n\t"
53+
"lxvp 54, 96(%3) \n\t"
54+
55+
"lxvp 56, 128(%3) \n\t"
56+
"lxvp 58, 160(%3) \n\t"
57+
"lxvp 60, 192(%3) \n\t"
58+
"lxvp 62, 224(%3) \n\t"
59+
60+
"stxvp 32, 0(%3) \n\t"
61+
"stxvp 34, 32(%3) \n\t"
62+
"stxvp 36, 64(%3) \n\t"
63+
"stxvp 38, 96(%3) \n\t"
64+
65+
"stxvp 40, 128(%3) \n\t"
66+
"stxvp 42, 160(%3) \n\t"
67+
"stxvp 44, 192(%3) \n\t"
68+
"stxvp 46, 224(%3) \n\t"
69+
70+
"stxvp 48, 0(%4) \n\t"
71+
"stxvp 50, 32(%4) \n\t"
72+
"stxvp 52, 64(%4) \n\t"
73+
"stxvp 54, 96(%4) \n\t"
74+
75+
"stxvp 56, 128(%4) \n\t"
76+
"stxvp 58, 160(%4) \n\t"
77+
"stxvp 60, 192(%4) \n\t"
78+
"stxvp 62, 224(%4) \n\t"
79+
80+
"addi %4, %4, 256 \n\t"
81+
"addi %3, %3, 256 \n\t"
82+
83+
#if defined(DOUBLE)
84+
"addic. %2, %2, -32 \n\t"
85+
#else
86+
"addic. %2, %2, -64 \n\t"
87+
#endif
88+
"bgt one%= \n"
89+
90+
"#n=%2 x=%0=%3 y=%1=%4"
91+
:
92+
"+m" (*x),
93+
"+m" (*y),
94+
"+r" (n), // 2
95+
"+b" (x), // 3
96+
"+b" (y) // 4
97+
:
98+
:
99+
"cr0",
100+
"vs32","vs33","vs34","vs35","vs36","vs37","vs38","vs39",
101+
"vs40","vs41","vs42","vs43","vs44","vs45","vs46","vs47",
102+
"vs48","vs49","vs50","vs51","vs52","vs53","vs54","vs55",
103+
"vs56","vs57","vs58","vs59","vs60","vs61","vs62","vs63"
104+
);
105+
}

‎kernel/power/zswap.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,11 @@ USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3636
#include "common.h"
3737

3838

39-
#if defined(POWER8) || defined(POWER9) || defined(POWER10)
4039
#if defined(__VEC__) || defined(__ALTIVEC__)
40+
#if defined(POWER8) || defined(POWER9)
4141
#include "zswap_microk_power8.c"
42+
#elif defined(POWER10)
43+
#include "cswap_microk_power10.c"
4244
#endif
4345
#endif
4446

0 commit comments

Comments
 (0)