1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7#include "funcdata.h"
8
9// bool armcas(int32 *val, int32 old, int32 new)
10// Atomically:
11// if(*val == old){
12// *val = new;
13// return 1;
14// }else
15// return 0;
16//
17// To implement ·cas in sys_$GOOS_arm.s
18// using the native instructions, use:
19//
20// TEXT ·cas(SB),NOSPLIT,$0
21// B ·armcas(SB)
22//
23TEXT ·armcas(SB),NOSPLIT,$0-13
24 MOVW ptr+0(FP), R1
25 MOVW old+4(FP), R2
26 MOVW new+8(FP), R3
27casl:
28 LDREX (R1), R0
29 CMP R0, R2
30 BNE casfail
31
32#ifndef GOARM_7
33 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
34 CMP $0, R11
35 BEQ 2(PC)
36#endif
37 DMB MB_ISHST
38
39 STREX R3, (R1), R0
40 CMP $0, R0
41 BNE casl
42 MOVW $1, R0
43
44#ifndef GOARM_7
45 CMP $0, R11
46 BEQ 2(PC)
47#endif
48 DMB MB_ISH
49
50 MOVB R0, ret+12(FP)
51 RET
52casfail:
53 MOVW $0, R0
54 MOVB R0, ret+12(FP)
55 RET
56
57// stubs
58
59TEXT ·Loadp(SB),NOSPLIT|NOFRAME,$0-8
60 B ·Load(SB)
61
62TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-8
63 B ·Load(SB)
64
65TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-8
66 B ·Load(SB)
67
68TEXT ·Casint32(SB),NOSPLIT,$0-13
69 B ·Cas(SB)
70
71TEXT ·Casint64(SB),NOSPLIT,$-4-21
72 B ·Cas64(SB)
73
74TEXT ·Casuintptr(SB),NOSPLIT,$0-13
75 B ·Cas(SB)
76
77TEXT ·Casp1(SB),NOSPLIT,$0-13
78 B ·Cas(SB)
79
80TEXT ·CasRel(SB),NOSPLIT,$0-13
81 B ·Cas(SB)
82
83TEXT ·Loadint32(SB),NOSPLIT,$0-8
84 B ·Load(SB)
85
86TEXT ·Loadint64(SB),NOSPLIT,$-4-12
87 B ·Load64(SB)
88
89TEXT ·Loaduintptr(SB),NOSPLIT,$0-8
90 B ·Load(SB)
91
92TEXT ·Loaduint(SB),NOSPLIT,$0-8
93 B ·Load(SB)
94
95TEXT ·Storeint32(SB),NOSPLIT,$0-8
96 B ·Store(SB)
97
98TEXT ·Storeint64(SB),NOSPLIT,$0-12
99 B ·Store64(SB)
100
101TEXT ·Storeuintptr(SB),NOSPLIT,$0-8
102 B ·Store(SB)
103
104TEXT ·StorepNoWB(SB),NOSPLIT,$0-8
105 B ·Store(SB)
106
107TEXT ·StoreRel(SB),NOSPLIT,$0-8
108 B ·Store(SB)
109
110TEXT ·StoreReluintptr(SB),NOSPLIT,$0-8
111 B ·Store(SB)
112
113TEXT ·Xaddint32(SB),NOSPLIT,$0-12
114 B ·Xadd(SB)
115
116TEXT ·Xaddint64(SB),NOSPLIT,$-4-20
117 B ·Xadd64(SB)
118
119TEXT ·Xadduintptr(SB),NOSPLIT,$0-12
120 B ·Xadd(SB)
121
122TEXT ·Xchgint32(SB),NOSPLIT,$0-12
123 B ·Xchg(SB)
124
125TEXT ·Xchgint64(SB),NOSPLIT,$-4-20
126 B ·Xchg64(SB)
127
128// 64-bit atomics
129// The native ARM implementations use LDREXD/STREXD, which are
130// available on ARMv6k or later. We use them only on ARMv7.
131// On older ARM, we use Go implementations which simulate 64-bit
132// atomics with locks.
133TEXT armCas64<>(SB),NOSPLIT,$0-21
134 // addr is already in R1
135 MOVW old_lo+4(FP), R2
136 MOVW old_hi+8(FP), R3
137 MOVW new_lo+12(FP), R4
138 MOVW new_hi+16(FP), R5
139cas64loop:
140 LDREXD (R1), R6 // loads R6 and R7
141 CMP R2, R6
142 BNE cas64fail
143 CMP R3, R7
144 BNE cas64fail
145
146 DMB MB_ISHST
147
148 STREXD R4, (R1), R0 // stores R4 and R5
149 CMP $0, R0
150 BNE cas64loop
151 MOVW $1, R0
152
153 DMB MB_ISH
154
155 MOVBU R0, swapped+20(FP)
156 RET
157cas64fail:
158 MOVW $0, R0
159 MOVBU R0, swapped+20(FP)
160 RET
161
162TEXT armXadd64<>(SB),NOSPLIT,$0-20
163 // addr is already in R1
164 MOVW delta_lo+4(FP), R2
165 MOVW delta_hi+8(FP), R3
166
167add64loop:
168 LDREXD (R1), R4 // loads R4 and R5
169 ADD.S R2, R4
170 ADC R3, R5
171
172 DMB MB_ISHST
173
174 STREXD R4, (R1), R0 // stores R4 and R5
175 CMP $0, R0
176 BNE add64loop
177
178 DMB MB_ISH
179
180 MOVW R4, new_lo+12(FP)
181 MOVW R5, new_hi+16(FP)
182 RET
183
184TEXT armXchg64<>(SB),NOSPLIT,$0-20
185 // addr is already in R1
186 MOVW new_lo+4(FP), R2
187 MOVW new_hi+8(FP), R3
188
189swap64loop:
190 LDREXD (R1), R4 // loads R4 and R5
191
192 DMB MB_ISHST
193
194 STREXD R2, (R1), R0 // stores R2 and R3
195 CMP $0, R0
196 BNE swap64loop
197
198 DMB MB_ISH
199
200 MOVW R4, old_lo+12(FP)
201 MOVW R5, old_hi+16(FP)
202 RET
203
204TEXT armLoad64<>(SB),NOSPLIT,$0-12
205 // addr is already in R1
206
207 LDREXD (R1), R2 // loads R2 and R3
208 DMB MB_ISH
209
210 MOVW R2, val_lo+4(FP)
211 MOVW R3, val_hi+8(FP)
212 RET
213
214TEXT armStore64<>(SB),NOSPLIT,$0-12
215 // addr is already in R1
216 MOVW val_lo+4(FP), R2
217 MOVW val_hi+8(FP), R3
218
219store64loop:
220 LDREXD (R1), R4 // loads R4 and R5
221
222 DMB MB_ISHST
223
224 STREXD R2, (R1), R0 // stores R2 and R3
225 CMP $0, R0
226 BNE store64loop
227
228 DMB MB_ISH
229 RET
230
231// The following functions all panic if their address argument isn't
232// 8-byte aligned. Since we're calling back into Go code to do this,
233// we have to cooperate with stack unwinding. In the normal case, the
234// functions tail-call into the appropriate implementation, which
235// means they must not open a frame. Hence, when they go down the
236// panic path, at that point they push the LR to create a real frame
237// (they don't need to pop it because panic won't return; however, we
238// do need to set the SP delta back).
239
240// Check if R1 is 8-byte aligned, panic if not.
241// Clobbers R2.
242#define CHECK_ALIGN \
243 AND.S $7, R1, R2 \
244 BEQ 4(PC) \
245 MOVW.W R14, -4(R13) /* prepare a real frame */ \
246 BL ·panicUnaligned(SB) \
247 ADD $4, R13 /* compensate SP delta */
248
249TEXT ·Cas64(SB),NOSPLIT,$-4-21
250 NO_LOCAL_POINTERS
251 MOVW addr+0(FP), R1
252 CHECK_ALIGN
253
254#ifndef GOARM_7
255 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
256 CMP $1, R11
257 BEQ 2(PC)
258 JMP ·goCas64(SB)
259#endif
260 JMP armCas64<>(SB)
261
262TEXT ·Xadd64(SB),NOSPLIT,$-4-20
263 NO_LOCAL_POINTERS
264 MOVW addr+0(FP), R1
265 CHECK_ALIGN
266
267#ifndef GOARM_7
268 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
269 CMP $1, R11
270 BEQ 2(PC)
271 JMP ·goXadd64(SB)
272#endif
273 JMP armXadd64<>(SB)
274
275TEXT ·Xchg64(SB),NOSPLIT,$-4-20
276 NO_LOCAL_POINTERS
277 MOVW addr+0(FP), R1
278 CHECK_ALIGN
279
280#ifndef GOARM_7
281 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
282 CMP $1, R11
283 BEQ 2(PC)
284 JMP ·goXchg64(SB)
285#endif
286 JMP armXchg64<>(SB)
287
288TEXT ·Load64(SB),NOSPLIT,$-4-12
289 NO_LOCAL_POINTERS
290 MOVW addr+0(FP), R1
291 CHECK_ALIGN
292
293#ifndef GOARM_7
294 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
295 CMP $1, R11
296 BEQ 2(PC)
297 JMP ·goLoad64(SB)
298#endif
299 JMP armLoad64<>(SB)
300
301TEXT ·Store64(SB),NOSPLIT,$-4-12
302 NO_LOCAL_POINTERS
303 MOVW addr+0(FP), R1
304 CHECK_ALIGN
305
306#ifndef GOARM_7
307 MOVB internal∕cpu·ARM+const_offsetARMHasV7Atomics(SB), R11
308 CMP $1, R11
309 BEQ 2(PC)
310 JMP ·goStore64(SB)
311#endif
312 JMP armStore64<>(SB)
View as plain text