P99
p99_atomic_arm.h
Go to the documentation of this file.
1 /* This may look like nonsense, but it really is -*- mode: C; coding: utf-8 -*- */
2 /* */
3 /* Except for parts copied from previous work and as explicitly stated below, */
4 /* the authors and copyright holders for this work are as follows: */
5 /* (C) copyright 2012, 2014 Jens Gustedt, INRIA, France */
6 /* (C) copyright 2012 William Morris */
7 /* */
8 /* This file is free software; it is part of the P99 project. */
9 /* */
10 /* Licensed under the Apache License, Version 2.0 (the "License"); */
11 /* you may not use this file except in compliance with the License. */
12 /* You may obtain a copy of the License at */
13 /* */
14 /* http://www.apache.org/licenses/LICENSE-2.0 */
15 /* */
16 /* Unless required by applicable law or agreed to in writing, software */
17 /* distributed under the License is distributed on an "AS IS" BASIS, */
18 /* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */
19 /* See the License for the specific language governing permissions and */
20 /* limitations under the License. */
21 /* */
22 #ifndef P99_ATOMIC_ARM_H
23 #define P99_ATOMIC_ARM_H 1
24 
25 #ifndef P99_ATOMIC_H
26 # warning "never include this file directly, use p99_atomic.h, instead"
27 #endif
28 
29 #if !defined(__thumb__) && !defined(__thumb2__)
30 /* When in arm mode we can't do addressing with offset, here, so use
31  direct addressing. */
32 p99_inline
33 uint8_t p00_arm_ldrexb(uint8_t volatile*p00_ptr) {
34  uint8_t p00_ret;
35  __asm__ volatile ("ldrexb %0,[%1]\t@ load exclusive\n"
36  : "=&r" (p00_ret)
37  : "r" (p00_ptr)
38  : "cc", "memory"
39  );
40  return p00_ret;
41 }
42 
43 p99_inline
44 _Bool p00_arm_strexb(uint8_t volatile*p00_ptr, uint8_t p00_val) {
45  uint32_t p00_ret;
46  __asm__ volatile ("strexb %0,%1,[%2]\t@ store exclusive\n"
47  : "=&r" (p00_ret)
48  : "r" (p00_val), "r" (p00_ptr)
49  : "cc", "memory"
50  );
51  return p00_ret;
52 }
53 p99_inline
54 uint16_t p00_arm_ldrexh(uint16_t volatile*p00_ptr) {
55  uint16_t p00_ret;
56  __asm__ volatile ("ldrexh %0,[%1]\t@ load exclusive\n"
57  : "=&r" (p00_ret)
58  : "r" (p00_ptr)
59  : "cc", "memory"
60  );
61  return p00_ret;
62 }
63 
64 p99_inline
65 _Bool p00_arm_strexh(uint16_t volatile*p00_ptr, uint16_t p00_val) {
66  uint32_t p00_ret;
67  __asm__ volatile ("strexh %0,%1,[%2]\t@ store exclusive\n"
68  : "=&r" (p00_ret)
69  : "r" (p00_val), "r" (p00_ptr)
70  : "cc", "memory"
71  );
72  return p00_ret;
73 }
74 p99_inline
75 uint32_t p00_arm_ldrex(uint32_t volatile*p00_ptr) {
76  uint32_t p00_ret;
77  __asm__ volatile ("ldrex %0,[%1]\t@ load exclusive\n"
78  : "=&r" (p00_ret)
79  : "r" (p00_ptr)
80  : "cc", "memory"
81  );
82  return p00_ret;
83 }
84 
85 p99_inline
86 _Bool p00_arm_strex(uint32_t volatile*p00_ptr, uint32_t p00_val) {
87  uint32_t p00_ret;
88  __asm__ volatile ("strex %0,%1,[%2]\t@ store exclusive\n"
89  : "=&r" (p00_ret)
90  : "r" (p00_val), "r" (p00_ptr)
91  : "cc", "memory"
92  );
93  return p00_ret;
94 }
95 # if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8)
96 p99_inline
97 uint64_t p00_arm_ldrexd(uint64_t volatile*p00_ptr) {
98  uint64_t p00_ret;
99  __asm__ volatile ("ldrexd %0, %H0, [%1]\t@ load exclusive\n"
100  : "=&r" (p00_ret)
101  : "r" (p00_ptr)
102  : "cc", "memory"
103  );
104  return p00_ret;
105 }
106 
107 p99_inline
108 _Bool p00_arm_strex(uint64_t volatile*p00_ptr, uint64_t p00_val) {
109  uint32_t p00_ret;
110  __asm__ volatile ("strex %0, %1, %H1, [%2]\t@ store exclusive\n"
111  : "=&r" (p00_ret)
112  : "r" (p00_val), "r" (p00_ptr)
113  : "cc", "memory"
114  );
115  return p00_ret;
116 }
117 # endif
118 #else
119 /* When in thumb mode we can do addressing with offset, here, so use
120  the "m" constraint to the assembler. */
121 p99_inline
122 uint8_t p00_arm_ldrexb(uint8_t volatile*p00_ptr) {
123  uint8_t p00_ret;
124  __asm__ volatile ("ldrexb %0,%1\t@ load exclusive\n"
125  : "=&r" (p00_ret)
126  : "m" (p00_ptr)
127  : "cc", "memory"
128  );
129  return p00_ret;
130 }
131 
132 p99_inline
133 _Bool p00_arm_strexb(uint8_t volatile*p00_ptr, uint8_t p00_val) {
134  uint32_t p00_ret;
135  __asm__ volatile ("strexb %0,%1,%2\t@ store exclusive\n"
136  : "=&r" (p00_ret)
137  : "r" (p00_val), "m" (p00_ptr)
138  : "cc", "memory"
139  );
140  return p00_ret;
141 }
142 p99_inline
143 uint16_t p00_arm_ldrexh(uint16_t volatile*p00_ptr) {
144  uint16_t p00_ret;
145  __asm__ volatile ("ldrexh %0,%1\t@ load exclusive\n"
146  : "=&r" (p00_ret)
147  : "m" (p00_ptr)
148  : "cc", "memory"
149  );
150  return p00_ret;
151 }
152 
153 p99_inline
154 _Bool p00_arm_strexh(uint16_t volatile*p00_ptr, uint16_t p00_val) {
155  uint32_t p00_ret;
156  __asm__ volatile ("strexh %0,%1,%2\t@ store exclusive\n"
157  : "=&r" (p00_ret)
158  : "r" (p00_val), "m" (p00_ptr)
159  : "cc", "memory"
160  );
161  return p00_ret;
162 }
163 p99_inline
164 uint32_t p00_arm_ldrex(uint32_t volatile*p00_ptr) {
165  uint32_t p00_ret;
166  __asm__ volatile ("ldrex %0,%1\t@ load exclusive\n"
167  : "=&r" (p00_ret)
168  : "m" (p00_ptr)
169  : "cc", "memory"
170  );
171  return p00_ret;
172 }
173 
174 p99_inline
175 _Bool p00_arm_strex(uint32_t volatile*p00_ptr, uint32_t p00_val) {
176  uint32_t p00_ret;
177  __asm__ volatile ("strex %0,%1,%2\t@ store exclusive\n"
178  : "=&r" (p00_ret)
179  : "r" (p00_val), "m" (p00_ptr)
180  : "cc", "memory"
181  );
182  return p00_ret;
183 }
184 # if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8)
185 p99_inline
186 uint64_t p00_arm_ldrexd(uint64_t volatile*p00_ptr) {
187  uint64_t p00_ret;
188  __asm__ volatile ("ldrexd %0, %H0, %1\t@ load exclusive\n"
189  : "=&r" (p00_ret)
190  : "m" (p00_ptr)
191  : "cc", "memory"
192  );
193  return p00_ret;
194 }
195 
196 p99_inline
197 _Bool p00_arm_strexd(uint64_t volatile*p00_ptr, uint64_t p00_val) {
198  uint32_t p00_ret;
199  __asm__ volatile ("strexd %0, %1, %H1, %2\t@ store exclusive\n"
200  : "=&r" (p00_ret)
201  : "r" (p00_val), "m" (p00_ptr)
202  : "cc", "memory"
203  );
204  return p00_ret;
205 }
206 # endif
207 #endif
208 
209 p99_inline
210 uint8_t p00_atomic_exchange_1(uint8_t volatile* p00_objp, uint8_t p00_des) {
211  for (;;) {
212  uint8_t p00_ret = p00_arm_ldrexb(object);
213  if (!p00_arm_strexb(object, p00_des)) return p00_ret;
214  }
215 }
216 
217 p99_inline
218 uint16_t p00_atomic_exchange_2(uint16_t volatile* p00_objp, uint16_t p00_des) {
219  for (;;) {
220  uint16_t p00_ret = p00_arm_ldrexh(object);
221  if (!p00_arm_strexh(object, p00_des)) return p00_ret;
222  }
223 }
224 
225 p99_inline
226 uint32_t p00_atomic_exchange_4(uint32_t volatile* p00_objp, uint32_t p00_des) {
227  for (;;) {
228  uint32_t p00_ret = p00_arm_ldrex(object);
229  if (!p00_arm_strex(object, p00_des)) return p00_ret;
230  }
231 }
232 
233 #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || defined(P00_DOXYGEN)
234 p99_inline
235 uint64_t p00_atomic_exchange_8(uint64_t volatile* p00_objp, uint64_t p00_des) {
236  for (;;) {
237  uint64_t p00_ret = p00_arm_ldrexd(object);
238  if (!p00_arm_strexd(object, p00_des)) return p00_ret;
239  }
240 }
241 #endif
242 
243 #if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) && !defined(P00_DOXYGEN)
244 
245 p99_inline
246 uint32_t p00_sync_lock_test_and_set(uint32_t volatile *p00_objp) {
247  return __sync_lock_test_and_set(p00_objp, 1);
248 }
249 
250 p99_inline
251 void p00_sync_lock_release(uint32_t volatile *p00_objp) {
252  __sync_lock_release(p00_objp);
253 }
254 
255 p99_inline
256 void p00_mfence(void) {
257  __sync_synchronize();
258 }
259 
260 
261 #else
262 
289 p99_inline
290 uint32_t p00_sync_lock_test_and_set_internal(uint32_t volatile *object, memory_order p00_ord) {
291  for (;;) {
292  uint32_t p00_ret = p00_arm_ldrex(object);
293  /* Even if the result has been a 1 in p00_ret, We must imperatively
294  put a strex after the ldex since otherwise we would block other
295  threads when they try to access this. On the other hand even if
296  the strex doesn't succeed but p00_ret is already set, we are also
297  done. */
298  if (!p00_arm_strex(object, 1) || p00_ret) return p00_ret;
299  }
300 }
301 
302 p99_inline
303 void p00_sync_lock_release_internal(uint32_t volatile *object, memory_order p00_ord) {
304  __sync_lock_release(object);
305 }
306 
307 p99_inline
308 void p00_mfence_internal(memory_order p00_ord) {
309  __asm__ __volatile__("dmb":::"memory");
310 }
311 
312 #define p00_mfence(...) \
313 P99_IF_EMPTY(__VA_ARGS__) \
314  (p00_mfence_internal(memory_order_seq_cst)) \
315  (p00_mfence_internal(__VA_ARGS__))
316 
317 #define p00_sync_lock_release(...) \
318  P99_IF_LT(P99_NARG(__VA_ARGS__), 2) \
319  (p00_sync_lock_release_internal(__VA_ARGS__)) \
320  (p00_sync_lock_release_internal(P99_ALLBUTLAST(__VA_ARGS__)))
321 #define p00_sync_lock_test_and_set(...) \
322  P99_IF_LT(P99_NARG(__VA_ARGS__), 2) \
323  (p00_sync_lock_test_and_set_internal(__VA_ARGS__)) \
324  (p00_sync_lock_test_and_set_internal(P99_ALLBUTLAST(__VA_ARGS__)))
325 
326 p99_inline
327 uint32_t __sync_val_compare_and_swap_4(uint32_t volatile *object, uint32_t p00_pre, uint32_t p00_des) {
328  uint32_t p00_ret = 0;
329  for (;;) {
330  p00_ret = p00_arm_ldrex(object);
331  if (p00_pre != p00_ret) {
332  /* wrong value, cancel */
333  p00_arm_strex(object, p00_ret);
334  break;
335  } else {
336  if (!p00_arm_strex(object, p00_des)) break;
337  /* somebody else touched it, continue */
338  }
339  }
340  return p00_ret;
341 }
342 
343 p99_inline
344 uint32_t __sync_fetch_and_add_4(uint32_t volatile *object, uint32_t p00_val) {
345  uint32_t p00_ret = 0;
346  for (;;) {
347  p00_ret = p00_arm_ldrex(object);
348  uint32_t p00_des = p00_ret + p00_val;
349  if (!p00_arm_strex(object, p00_des)) break;
350  /* somebody else touched it, continue */
351  }
352  return p00_ret;
353 }
354 
355 p99_inline
356 uint32_t __sync_fetch_and_sub_4(uint32_t volatile *object, uint32_t p00_val) {
357  uint32_t p00_ret = 0;
358  for (;;) {
359  p00_ret = p00_arm_ldrex(object);
360  uint32_t p00_des = p00_ret - p00_val;
361  if (!p00_arm_strex(object, p00_des)) break;
362  /* somebody else touched it, continue */
363  }
364  return p00_ret;
365 }
366 
367 p99_inline
368 uint32_t __sync_fetch_and_or_4(uint32_t volatile *object, uint32_t p00_val) {
369  uint32_t p00_ret = 0;
370  for (;;) {
371  p00_ret = p00_arm_ldrex(object);
372  uint32_t p00_des = p00_ret | p00_val;
373  if (!p00_arm_strex(object, p00_des)) break;
374  /* somebody else touched it, continue */
375  }
376  return p00_ret;
377 }
378 
379 p99_inline
380 uint32_t __sync_fetch_and_and_4(uint32_t volatile *object, uint32_t p00_val) {
381  uint32_t p00_ret = 0;
382  for (;;) {
383  p00_ret = p00_arm_ldrex(object);
384  uint32_t p00_des = p00_ret & p00_val;
385  if (!p00_arm_strex(object, p00_des)) break;
386  /* somebody else touched it, continue */
387  }
388  return p00_ret;
389 }
390 
391 p99_inline
392 uint32_t __sync_fetch_and_xor_4(uint32_t volatile *object, uint32_t p00_val) {
393  uint32_t p00_ret = 0;
394  for (;;) {
395  p00_ret = p00_arm_ldrex(object);
396  uint32_t p00_des = p00_ret ^ p00_val;
397  if (!p00_arm_strex(object, p00_des)) break;
398  /* somebody else touched it, continue */
399  }
400  return p00_ret;
401 }
402 
403 #undef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4
404 #define __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 1
405 
406 
411 # endif
412 
413 #endif
__sync_fetch_and_or_4
uint32_t __sync_fetch_and_or_4(uint32_t volatile *object, uint32_t p00_val)
Definition: p99_atomic_arm.h:368
__sync_fetch_and_add_4
uint32_t __sync_fetch_and_add_4(uint32_t volatile *object, uint32_t p00_val)
Definition: p99_atomic_arm.h:344
p99_inline
#define p99_inline
Try to force a function always to be inlined.
Definition: p99_compiler.h:496
__sync_fetch_and_xor_4
uint32_t __sync_fetch_and_xor_4(uint32_t volatile *object, uint32_t p00_val)
Definition: p99_atomic_arm.h:392
__sync_fetch_and_sub_4
uint32_t __sync_fetch_and_sub_4(uint32_t volatile *object, uint32_t p00_val)
Definition: p99_atomic_arm.h:356
__sync_val_compare_and_swap_4
uint32_t __sync_val_compare_and_swap_4(uint32_t volatile *object, uint32_t p00_pre, uint32_t p00_des)
Definition: p99_atomic_arm.h:327
__sync_fetch_and_and_4
uint32_t __sync_fetch_and_and_4(uint32_t volatile *object, uint32_t p00_val)
Definition: p99_atomic_arm.h:380
Generated by   doxygen 1.8.17