1// Copyright 2015 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5#include "textflag.h"
6
7// Linux/ARM atomic operations.
8
9// Because there is so much variation in ARM devices,
10// the Linux kernel provides an appropriate compare-and-swap
11// implementation at address 0xffff0fc0. Caller sets:
12// R0 = old value
13// R1 = new value
14// R2 = addr
15// LR = return address
16// The function returns with CS true if the swap happened.
17// http://lxr.linux.no/linux+v2.6.37.2/arch/arm/kernel/entry-armv.S#L850
18//
19// https://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=b49c0f24cf6744a3f4fd09289fe7cade349dead5
20//
21TEXT cas<>(SB),NOSPLIT,$0
22 MOVW $0xffff0fc0, R15 // R15 is hardware PC.
23
24TEXT ·Cas(SB),NOSPLIT|NOFRAME,$0
25 MOVB runtime·goarm(SB), R11
26 CMP $7, R11
27 BLT 2(PC)
28 JMP ·armcas(SB)
29 JMP kernelcas<>(SB)
30
31TEXT kernelcas<>(SB),NOSPLIT,$0
32 MOVW ptr+0(FP), R2
33 // trigger potential paging fault here,
34 // because we don't know how to traceback through __kuser_cmpxchg
35 MOVW (R2), R0
36 MOVW old+4(FP), R0
37 MOVW new+8(FP), R1
38 BL cas<>(SB)
39 BCC ret0
40 MOVW $1, R0
41 MOVB R0, ret+12(FP)
42 RET
43ret0:
44 MOVW $0, R0
45 MOVB R0, ret+12(FP)
46 RET
47
48// As for cas, memory barriers are complicated on ARM, but the kernel
49// provides a user helper. ARMv5 does not support SMP and has no
50// memory barrier instruction at all. ARMv6 added SMP support and has
51// a memory barrier, but it requires writing to a coprocessor
52// register. ARMv7 introduced the DMB instruction, but it's expensive
53// even on single-core devices. The kernel helper takes care of all of
54// this for us.
55
56// Use kernel helper version of memory_barrier, when compiled with GOARM < 7.
57TEXT memory_barrier<>(SB),NOSPLIT|NOFRAME,$0
58 MOVW $0xffff0fa0, R15 // R15 is hardware PC.
59
60TEXT ·Load(SB),NOSPLIT,$0-8
61 MOVW addr+0(FP), R0
62 MOVW (R0), R1
63
64 MOVB runtime·goarm(SB), R11
65 CMP $7, R11
66 BGE native_barrier
67 BL memory_barrier<>(SB)
68 B end
69native_barrier:
70 DMB MB_ISH
71end:
72 MOVW R1, ret+4(FP)
73 RET
74
75TEXT ·Store(SB),NOSPLIT,$0-8
76 MOVW addr+0(FP), R1
77 MOVW v+4(FP), R2
78
79 MOVB runtime·goarm(SB), R8
80 CMP $7, R8
81 BGE native_barrier
82 BL memory_barrier<>(SB)
83 B store
84native_barrier:
85 DMB MB_ISH
86
87store:
88 MOVW R2, (R1)
89
90 CMP $7, R8
91 BGE native_barrier2
92 BL memory_barrier<>(SB)
93 RET
94native_barrier2:
95 DMB MB_ISH
96 RET
97
98TEXT ·Load8(SB),NOSPLIT,$0-5
99 MOVW addr+0(FP), R0
100 MOVB (R0), R1
101
102 MOVB runtime·goarm(SB), R11
103 CMP $7, R11
104 BGE native_barrier
105 BL memory_barrier<>(SB)
106 B end
107native_barrier:
108 DMB MB_ISH
109end:
110 MOVB R1, ret+4(FP)
111 RET
112
113TEXT ·Store8(SB),NOSPLIT,$0-5
114 MOVW addr+0(FP), R1
115 MOVB v+4(FP), R2
116
117 MOVB runtime·goarm(SB), R8
118 CMP $7, R8
119 BGE native_barrier
120 BL memory_barrier<>(SB)
121 B store
122native_barrier:
123 DMB MB_ISH
124
125store:
126 MOVB R2, (R1)
127
128 CMP $7, R8
129 BGE native_barrier2
130 BL memory_barrier<>(SB)
131 RET
132native_barrier2:
133 DMB MB_ISH
134 RET
View as plain text