1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// RISC-V's atomic operations have two bits, aq ("acquire") and rl ("release"),
6// which may be toggled on and off. Their precise semantics are defined in
7// section 6.3 of the specification, but the basic idea is as follows:
8//
9// - If neither aq nor rl is set, the CPU may reorder the atomic arbitrarily.
10// It guarantees only that it will execute atomically.
11//
12// - If aq is set, the CPU may move the instruction backward, but not forward.
13//
14// - If rl is set, the CPU may move the instruction forward, but not backward.
15//
16// - If both are set, the CPU may not reorder the instruction at all.
17//
18// These four modes correspond to other well-known memory models on other CPUs.
19// On ARM, aq corresponds to a dmb ishst, aq+rl corresponds to a dmb ish. On
20// Intel, aq corresponds to an lfence, rl to an sfence, and aq+rl to an mfence
21// (or a lock prefix).
22//
23// Go's memory model requires that
24// - if a read happens after a write, the read must observe the write, and
25// that
26// - if a read happens concurrently with a write, the read may observe the
27// write.
28// aq is sufficient to guarantee this, so that's what we use here. (This jibes
29// with ARM, which uses dmb ishst.)
30
31#include "textflag.h"
32
33// func Cas(ptr *uint64, old, new uint64) bool
34// Atomically:
35// if(*val == old){
36// *val = new;
37// return 1;
38// } else {
39// return 0;
40// }
41TEXT ·Cas(SB), NOSPLIT, $0-17
42 MOV ptr+0(FP), A0
43 MOVW old+8(FP), A1
44 MOVW new+12(FP), A2
45cas_again:
46 LRW (A0), A3
47 BNE A3, A1, cas_fail
48 SCW A2, (A0), A4
49 BNE A4, ZERO, cas_again
50 MOV $1, A0
51 MOVB A0, ret+16(FP)
52 RET
53cas_fail:
54 MOV $0, A0
55 MOV A0, ret+16(FP)
56 RET
57
58// func Cas64(ptr *uint64, old, new uint64) bool
59TEXT ·Cas64(SB), NOSPLIT, $0-25
60 MOV ptr+0(FP), A0
61 MOV old+8(FP), A1
62 MOV new+16(FP), A2
63cas_again:
64 LRD (A0), A3
65 BNE A3, A1, cas_fail
66 SCD A2, (A0), A4
67 BNE A4, ZERO, cas_again
68 MOV $1, A0
69 MOVB A0, ret+24(FP)
70 RET
71cas_fail:
72 MOVB ZERO, ret+24(FP)
73 RET
74
75// func Load(ptr *uint32) uint32
76TEXT ·Load(SB),NOSPLIT|NOFRAME,$0-12
77 MOV ptr+0(FP), A0
78 LRW (A0), A0
79 MOVW A0, ret+8(FP)
80 RET
81
82// func Load8(ptr *uint8) uint8
83TEXT ·Load8(SB),NOSPLIT|NOFRAME,$0-9
84 MOV ptr+0(FP), A0
85 FENCE
86 MOVBU (A0), A1
87 FENCE
88 MOVB A1, ret+8(FP)
89 RET
90
91// func Load64(ptr *uint64) uint64
92TEXT ·Load64(SB),NOSPLIT|NOFRAME,$0-16
93 MOV ptr+0(FP), A0
94 LRD (A0), A0
95 MOV A0, ret+8(FP)
96 RET
97
98// func Store(ptr *uint32, val uint32)
99TEXT ·Store(SB), NOSPLIT, $0-12
100 MOV ptr+0(FP), A0
101 MOVW val+8(FP), A1
102 AMOSWAPW A1, (A0), ZERO
103 RET
104
105// func Store8(ptr *uint8, val uint8)
106TEXT ·Store8(SB), NOSPLIT, $0-9
107 MOV ptr+0(FP), A0
108 MOVBU val+8(FP), A1
109 FENCE
110 MOVB A1, (A0)
111 FENCE
112 RET
113
114// func Store64(ptr *uint64, val uint64)
115TEXT ·Store64(SB), NOSPLIT, $0-16
116 MOV ptr+0(FP), A0
117 MOV val+8(FP), A1
118 AMOSWAPD A1, (A0), ZERO
119 RET
120
121TEXT ·Casp1(SB), NOSPLIT, $0-25
122 JMP ·Cas64(SB)
123
124TEXT ·Casint32(SB),NOSPLIT,$0-17
125 JMP ·Cas(SB)
126
127TEXT ·Casint64(SB),NOSPLIT,$0-25
128 JMP ·Cas64(SB)
129
130TEXT ·Casuintptr(SB),NOSPLIT,$0-25
131 JMP ·Cas64(SB)
132
133TEXT ·CasRel(SB), NOSPLIT, $0-17
134 JMP ·Cas(SB)
135
136TEXT ·Loaduintptr(SB),NOSPLIT,$0-16
137 JMP ·Load64(SB)
138
139TEXT ·Storeint32(SB),NOSPLIT,$0-12
140 JMP ·Store(SB)
141
142TEXT ·Storeint64(SB),NOSPLIT,$0-16
143 JMP ·Store64(SB)
144
145TEXT ·Storeuintptr(SB),NOSPLIT,$0-16
146 JMP ·Store64(SB)
147
148TEXT ·Loaduint(SB),NOSPLIT,$0-16
149 JMP ·Loaduintptr(SB)
150
151TEXT ·Loadint32(SB),NOSPLIT,$0-12
152 JMP ·Load(SB)
153
154TEXT ·Loadint64(SB),NOSPLIT,$0-16
155 JMP ·Load64(SB)
156
157TEXT ·Xaddint32(SB),NOSPLIT,$0-20
158 JMP ·Xadd(SB)
159
160TEXT ·Xaddint64(SB),NOSPLIT,$0-24
161 MOV ptr+0(FP), A0
162 MOV delta+8(FP), A1
163 AMOADDD A1, (A0), A0
164 ADD A0, A1, A0
165 MOVW A0, ret+16(FP)
166 RET
167
168TEXT ·LoadAcq(SB),NOSPLIT|NOFRAME,$0-12
169 JMP ·Load(SB)
170
171TEXT ·LoadAcq64(SB),NOSPLIT|NOFRAME,$0-16
172 JMP ·Load64(SB)
173
174TEXT ·LoadAcquintptr(SB),NOSPLIT|NOFRAME,$0-16
175 JMP ·Load64(SB)
176
177// func Loadp(ptr unsafe.Pointer) unsafe.Pointer
178TEXT ·Loadp(SB),NOSPLIT,$0-16
179 JMP ·Load64(SB)
180
181// func StorepNoWB(ptr unsafe.Pointer, val unsafe.Pointer)
182TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
183 JMP ·Store64(SB)
184
185TEXT ·StoreRel(SB), NOSPLIT, $0-12
186 JMP ·Store(SB)
187
188TEXT ·StoreRel64(SB), NOSPLIT, $0-16
189 JMP ·Store64(SB)
190
191TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
192 JMP ·Store64(SB)
193
194// func Xchg(ptr *uint32, new uint32) uint32
195TEXT ·Xchg(SB), NOSPLIT, $0-20
196 MOV ptr+0(FP), A0
197 MOVW new+8(FP), A1
198 AMOSWAPW A1, (A0), A1
199 MOVW A1, ret+16(FP)
200 RET
201
202// func Xchg64(ptr *uint64, new uint64) uint64
203TEXT ·Xchg64(SB), NOSPLIT, $0-24
204 MOV ptr+0(FP), A0
205 MOV new+8(FP), A1
206 AMOSWAPD A1, (A0), A1
207 MOV A1, ret+16(FP)
208 RET
209
210// Atomically:
211// *val += delta;
212// return *val;
213
214// func Xadd(ptr *uint32, delta int32) uint32
215TEXT ·Xadd(SB), NOSPLIT, $0-20
216 MOV ptr+0(FP), A0
217 MOVW delta+8(FP), A1
218 AMOADDW A1, (A0), A2
219 ADD A2,A1,A0
220 MOVW A0, ret+16(FP)
221 RET
222
223// func Xadd64(ptr *uint64, delta int64) uint64
224TEXT ·Xadd64(SB), NOSPLIT, $0-24
225 MOV ptr+0(FP), A0
226 MOV delta+8(FP), A1
227 AMOADDD A1, (A0), A2
228 ADD A2, A1, A0
229 MOV A0, ret+16(FP)
230 RET
231
232// func Xadduintptr(ptr *uintptr, delta uintptr) uintptr
233TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
234 JMP ·Xadd64(SB)
235
236// func Xchgint32(ptr *int32, new int32) int32
237TEXT ·Xchgint32(SB), NOSPLIT, $0-20
238 JMP ·Xchg(SB)
239
240// func Xchgint64(ptr *int64, new int64) int64
241TEXT ·Xchgint64(SB), NOSPLIT, $0-24
242 JMP ·Xchg64(SB)
243
244// func Xchguintptr(ptr *uintptr, new uintptr) uintptr
245TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
246 JMP ·Xchg64(SB)
247
248// func And8(ptr *uint8, val uint8)
249TEXT ·And8(SB), NOSPLIT, $0-9
250 MOV ptr+0(FP), A0
251 MOVBU val+8(FP), A1
252 AND $3, A0, A2
253 AND $-4, A0
254 SLL $3, A2
255 XOR $255, A1
256 SLL A2, A1
257 XOR $-1, A1
258 AMOANDW A1, (A0), ZERO
259 RET
260
261// func Or8(ptr *uint8, val uint8)
262TEXT ·Or8(SB), NOSPLIT, $0-9
263 MOV ptr+0(FP), A0
264 MOVBU val+8(FP), A1
265 AND $3, A0, A2
266 AND $-4, A0
267 SLL $3, A2
268 SLL A2, A1
269 AMOORW A1, (A0), ZERO
270 RET
271
272// func And(ptr *uint32, val uint32)
273TEXT ·And(SB), NOSPLIT, $0-12
274 MOV ptr+0(FP), A0
275 MOVW val+8(FP), A1
276 AMOANDW A1, (A0), ZERO
277 RET
278
279// func Or(ptr *uint32, val uint32)
280TEXT ·Or(SB), NOSPLIT, $0-12
281 MOV ptr+0(FP), A0
282 MOVW val+8(FP), A1
283 AMOORW A1, (A0), ZERO
284 RET
285
286// func Or32(ptr *uint32, val uint32) uint32
287TEXT ·Or32(SB), NOSPLIT, $0-20
288 MOV ptr+0(FP), A0
289 MOVW val+8(FP), A1
290 AMOORW A1, (A0), A2
291 MOVW A2, ret+16(FP)
292 RET
293
294// func And32(ptr *uint32, val uint32) uint32
295TEXT ·And32(SB), NOSPLIT, $0-20
296 MOV ptr+0(FP), A0
297 MOVW val+8(FP), A1
298 AMOANDW A1, (A0), A2
299 MOVW A2, ret+16(FP)
300 RET
301
302// func Or64(ptr *uint64, val uint64) uint64
303TEXT ·Or64(SB), NOSPLIT, $0-24
304 MOV ptr+0(FP), A0
305 MOV val+8(FP), A1
306 AMOORD A1, (A0), A2
307 MOV A2, ret+16(FP)
308 RET
309
310// func And64(ptr *uint64, val uint64) uint64
311TEXT ·And64(SB), NOSPLIT, $0-24
312 MOV ptr+0(FP), A0
313 MOV val+8(FP), A1
314 AMOANDD A1, (A0), A2
315 MOV A2, ret+16(FP)
316 RET
317
318// func Anduintptr(ptr *uintptr, val uintptr) uintptr
319TEXT ·Anduintptr(SB), NOSPLIT, $0-24
320 JMP ·And64(SB)
321
322// func Oruintptr(ptr *uintptr, val uintptr) uintptr
323TEXT ·Oruintptr(SB), NOSPLIT, $0-24
324 JMP ·Or64(SB)
View as plain text