1// Copyright 2014 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "go_asm.h"
6#include "textflag.h"
7
8TEXT ·Casint32(SB), NOSPLIT, $0-17
9 B ·Cas(SB)
10
11TEXT ·Casint64(SB), NOSPLIT, $0-25
12 B ·Cas64(SB)
13
14TEXT ·Casuintptr(SB), NOSPLIT, $0-25
15 B ·Cas64(SB)
16
17TEXT ·CasRel(SB), NOSPLIT, $0-17
18 B ·Cas(SB)
19
20TEXT ·Loadint32(SB), NOSPLIT, $0-12
21 B ·Load(SB)
22
23TEXT ·Loadint64(SB), NOSPLIT, $0-16
24 B ·Load64(SB)
25
26TEXT ·Loaduintptr(SB), NOSPLIT, $0-16
27 B ·Load64(SB)
28
29TEXT ·Loaduint(SB), NOSPLIT, $0-16
30 B ·Load64(SB)
31
32TEXT ·Storeint32(SB), NOSPLIT, $0-12
33 B ·Store(SB)
34
35TEXT ·Storeint64(SB), NOSPLIT, $0-16
36 B ·Store64(SB)
37
38TEXT ·Storeuintptr(SB), NOSPLIT, $0-16
39 B ·Store64(SB)
40
41TEXT ·Xaddint32(SB), NOSPLIT, $0-20
42 B ·Xadd(SB)
43
44TEXT ·Xaddint64(SB), NOSPLIT, $0-24
45 B ·Xadd64(SB)
46
47TEXT ·Xadduintptr(SB), NOSPLIT, $0-24
48 B ·Xadd64(SB)
49
50TEXT ·Casp1(SB), NOSPLIT, $0-25
51 B ·Cas64(SB)
52
53// uint32 ·Load(uint32 volatile* addr)
54TEXT ·Load(SB),NOSPLIT,$0-12
55 MOVD ptr+0(FP), R0
56 LDARW (R0), R0
57 MOVW R0, ret+8(FP)
58 RET
59
60// uint8 ·Load8(uint8 volatile* addr)
61TEXT ·Load8(SB),NOSPLIT,$0-9
62 MOVD ptr+0(FP), R0
63 LDARB (R0), R0
64 MOVB R0, ret+8(FP)
65 RET
66
67// uint64 ·Load64(uint64 volatile* addr)
68TEXT ·Load64(SB),NOSPLIT,$0-16
69 MOVD ptr+0(FP), R0
70 LDAR (R0), R0
71 MOVD R0, ret+8(FP)
72 RET
73
74// void *·Loadp(void *volatile *addr)
75TEXT ·Loadp(SB),NOSPLIT,$0-16
76 MOVD ptr+0(FP), R0
77 LDAR (R0), R0
78 MOVD R0, ret+8(FP)
79 RET
80
81// uint32 ·LoadAcq(uint32 volatile* addr)
82TEXT ·LoadAcq(SB),NOSPLIT,$0-12
83 B ·Load(SB)
84
85// uint64 ·LoadAcquintptr(uint64 volatile* addr)
86TEXT ·LoadAcq64(SB),NOSPLIT,$0-16
87 B ·Load64(SB)
88
89// uintptr ·LoadAcq64(uintptr volatile* addr)
90TEXT ·LoadAcquintptr(SB),NOSPLIT,$0-16
91 B ·Load64(SB)
92
93TEXT ·StorepNoWB(SB), NOSPLIT, $0-16
94 B ·Store64(SB)
95
96TEXT ·StoreRel(SB), NOSPLIT, $0-12
97 B ·Store(SB)
98
99TEXT ·StoreRel64(SB), NOSPLIT, $0-16
100 B ·Store64(SB)
101
102TEXT ·StoreReluintptr(SB), NOSPLIT, $0-16
103 B ·Store64(SB)
104
105TEXT ·Store(SB), NOSPLIT, $0-12
106 MOVD ptr+0(FP), R0
107 MOVW val+8(FP), R1
108 STLRW R1, (R0)
109 RET
110
111TEXT ·Store8(SB), NOSPLIT, $0-9
112 MOVD ptr+0(FP), R0
113 MOVB val+8(FP), R1
114 STLRB R1, (R0)
115 RET
116
117TEXT ·Store64(SB), NOSPLIT, $0-16
118 MOVD ptr+0(FP), R0
119 MOVD val+8(FP), R1
120 STLR R1, (R0)
121 RET
122
123// uint32 Xchg(ptr *uint32, new uint32)
124// Atomically:
125// old := *ptr;
126// *ptr = new;
127// return old;
128TEXT ·Xchg(SB), NOSPLIT, $0-20
129 MOVD ptr+0(FP), R0
130 MOVW new+8(FP), R1
131 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
132 CBZ R4, load_store_loop
133 SWPALW R1, (R0), R2
134 MOVW R2, ret+16(FP)
135 RET
136load_store_loop:
137 LDAXRW (R0), R2
138 STLXRW R1, (R0), R3
139 CBNZ R3, load_store_loop
140 MOVW R2, ret+16(FP)
141 RET
142
143// uint64 Xchg64(ptr *uint64, new uint64)
144// Atomically:
145// old := *ptr;
146// *ptr = new;
147// return old;
148TEXT ·Xchg64(SB), NOSPLIT, $0-24
149 MOVD ptr+0(FP), R0
150 MOVD new+8(FP), R1
151 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
152 CBZ R4, load_store_loop
153 SWPALD R1, (R0), R2
154 MOVD R2, ret+16(FP)
155 RET
156load_store_loop:
157 LDAXR (R0), R2
158 STLXR R1, (R0), R3
159 CBNZ R3, load_store_loop
160 MOVD R2, ret+16(FP)
161 RET
162
163// bool Cas(uint32 *ptr, uint32 old, uint32 new)
164// Atomically:
165// if(*val == old){
166// *val = new;
167// return 1;
168// } else
169// return 0;
170TEXT ·Cas(SB), NOSPLIT, $0-17
171 MOVD ptr+0(FP), R0
172 MOVW old+8(FP), R1
173 MOVW new+12(FP), R2
174 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
175 CBZ R4, load_store_loop
176 MOVD R1, R3
177 CASALW R3, (R0), R2
178 CMP R1, R3
179 CSET EQ, R0
180 MOVB R0, ret+16(FP)
181 RET
182load_store_loop:
183 LDAXRW (R0), R3
184 CMPW R1, R3
185 BNE ok
186 STLXRW R2, (R0), R3
187 CBNZ R3, load_store_loop
188ok:
189 CSET EQ, R0
190 MOVB R0, ret+16(FP)
191 RET
192
193// bool ·Cas64(uint64 *ptr, uint64 old, uint64 new)
194// Atomically:
195// if(*val == old){
196// *val = new;
197// return 1;
198// } else {
199// return 0;
200// }
201TEXT ·Cas64(SB), NOSPLIT, $0-25
202 MOVD ptr+0(FP), R0
203 MOVD old+8(FP), R1
204 MOVD new+16(FP), R2
205 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
206 CBZ R4, load_store_loop
207 MOVD R1, R3
208 CASALD R3, (R0), R2
209 CMP R1, R3
210 CSET EQ, R0
211 MOVB R0, ret+24(FP)
212 RET
213load_store_loop:
214 LDAXR (R0), R3
215 CMP R1, R3
216 BNE ok
217 STLXR R2, (R0), R3
218 CBNZ R3, load_store_loop
219ok:
220 CSET EQ, R0
221 MOVB R0, ret+24(FP)
222 RET
223
224// uint32 xadd(uint32 volatile *ptr, int32 delta)
225// Atomically:
226// *val += delta;
227// return *val;
228TEXT ·Xadd(SB), NOSPLIT, $0-20
229 MOVD ptr+0(FP), R0
230 MOVW delta+8(FP), R1
231 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
232 CBZ R4, load_store_loop
233 LDADDALW R1, (R0), R2
234 ADD R1, R2
235 MOVW R2, ret+16(FP)
236 RET
237load_store_loop:
238 LDAXRW (R0), R2
239 ADDW R2, R1, R2
240 STLXRW R2, (R0), R3
241 CBNZ R3, load_store_loop
242 MOVW R2, ret+16(FP)
243 RET
244
245// uint64 Xadd64(uint64 volatile *ptr, int64 delta)
246// Atomically:
247// *val += delta;
248// return *val;
249TEXT ·Xadd64(SB), NOSPLIT, $0-24
250 MOVD ptr+0(FP), R0
251 MOVD delta+8(FP), R1
252 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
253 CBZ R4, load_store_loop
254 LDADDALD R1, (R0), R2
255 ADD R1, R2
256 MOVD R2, ret+16(FP)
257 RET
258load_store_loop:
259 LDAXR (R0), R2
260 ADD R2, R1, R2
261 STLXR R2, (R0), R3
262 CBNZ R3, load_store_loop
263 MOVD R2, ret+16(FP)
264 RET
265
266TEXT ·Xchgint32(SB), NOSPLIT, $0-20
267 B ·Xchg(SB)
268
269TEXT ·Xchgint64(SB), NOSPLIT, $0-24
270 B ·Xchg64(SB)
271
272TEXT ·Xchguintptr(SB), NOSPLIT, $0-24
273 B ·Xchg64(SB)
274
275TEXT ·And8(SB), NOSPLIT, $0-9
276 MOVD ptr+0(FP), R0
277 MOVB val+8(FP), R1
278 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
279 CBZ R4, load_store_loop
280 MVN R1, R2
281 LDCLRALB R2, (R0), R3
282 RET
283load_store_loop:
284 LDAXRB (R0), R2
285 AND R1, R2
286 STLXRB R2, (R0), R3
287 CBNZ R3, load_store_loop
288 RET
289
290TEXT ·Or8(SB), NOSPLIT, $0-9
291 MOVD ptr+0(FP), R0
292 MOVB val+8(FP), R1
293 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
294 CBZ R4, load_store_loop
295 LDORALB R1, (R0), R2
296 RET
297load_store_loop:
298 LDAXRB (R0), R2
299 ORR R1, R2
300 STLXRB R2, (R0), R3
301 CBNZ R3, load_store_loop
302 RET
303
304// func And(addr *uint32, v uint32)
305TEXT ·And(SB), NOSPLIT, $0-12
306 MOVD ptr+0(FP), R0
307 MOVW val+8(FP), R1
308 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
309 CBZ R4, load_store_loop
310 MVN R1, R2
311 LDCLRALW R2, (R0), R3
312 RET
313load_store_loop:
314 LDAXRW (R0), R2
315 AND R1, R2
316 STLXRW R2, (R0), R3
317 CBNZ R3, load_store_loop
318 RET
319
320// func Or(addr *uint32, v uint32)
321TEXT ·Or(SB), NOSPLIT, $0-12
322 MOVD ptr+0(FP), R0
323 MOVW val+8(FP), R1
324 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
325 CBZ R4, load_store_loop
326 LDORALW R1, (R0), R2
327 RET
328load_store_loop:
329 LDAXRW (R0), R2
330 ORR R1, R2
331 STLXRW R2, (R0), R3
332 CBNZ R3, load_store_loop
333 RET
334
335// func Or32(addr *uint32, v uint32) old uint32
336TEXT ·Or32(SB), NOSPLIT, $0-20
337 MOVD ptr+0(FP), R0
338 MOVW val+8(FP), R1
339 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
340 CBZ R4, load_store_loop
341 LDORALW R1, (R0), R2
342 MOVD R2, ret+16(FP)
343 RET
344load_store_loop:
345 LDAXRW (R0), R2
346 ORR R1, R2, R3
347 STLXRW R3, (R0), R4
348 CBNZ R4, load_store_loop
349 MOVD R2, ret+16(FP)
350 RET
351
352// func And32(addr *uint32, v uint32) old uint32
353TEXT ·And32(SB), NOSPLIT, $0-20
354 MOVD ptr+0(FP), R0
355 MOVW val+8(FP), R1
356 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
357 CBZ R4, load_store_loop
358 MVN R1, R2
359 LDCLRALW R2, (R0), R3
360 MOVD R3, ret+16(FP)
361 RET
362load_store_loop:
363 LDAXRW (R0), R2
364 AND R1, R2, R3
365 STLXRW R3, (R0), R4
366 CBNZ R4, load_store_loop
367 MOVD R2, ret+16(FP)
368 RET
369
370// func Or64(addr *uint64, v uint64) old uint64
371TEXT ·Or64(SB), NOSPLIT, $0-24
372 MOVD ptr+0(FP), R0
373 MOVD val+8(FP), R1
374 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
375 CBZ R4, load_store_loop
376 LDORALD R1, (R0), R2
377 MOVD R2, ret+16(FP)
378 RET
379load_store_loop:
380 LDAXR (R0), R2
381 ORR R1, R2, R3
382 STLXR R3, (R0), R4
383 CBNZ R4, load_store_loop
384 MOVD R2, ret+16(FP)
385 RET
386
387// func And64(addr *uint64, v uint64) old uint64
388TEXT ·And64(SB), NOSPLIT, $0-24
389 MOVD ptr+0(FP), R0
390 MOVD val+8(FP), R1
391 MOVBU internal∕cpu·ARM64+const_offsetARM64HasATOMICS(SB), R4
392 CBZ R4, load_store_loop
393 MVN R1, R2
394 LDCLRALD R2, (R0), R3
395 MOVD R3, ret+16(FP)
396 RET
397load_store_loop:
398 LDAXR (R0), R2
399 AND R1, R2, R3
400 STLXR R3, (R0), R4
401 CBNZ R4, load_store_loop
402 MOVD R2, ret+16(FP)
403 RET
404
405// func Anduintptr(addr *uintptr, v uintptr) old uintptr
406TEXT ·Anduintptr(SB), NOSPLIT, $0-24
407 B ·And64(SB)
408
409// func Oruintptr(addr *uintptr, v uintptr) old uintptr
410TEXT ·Oruintptr(SB), NOSPLIT, $0-24
411 B ·Or64(SB)
View as plain text