atomic_utils.h

Go to the documentation of this file.

/*
 * SPDX-FileCopyrightText: 2020 Otto-von-Guericke-Universität Magdeburg
 * SPDX-License-Identifier: LGPL-2.1-only
 */
 
#pragma once

 
#include <limits.h>
#include <stdint.h>
 
#include "irq.h"
#include "macros/utils.h"
#include "sched.h"
 
#include "atomic_utils_arch.h" /* IWYU pragma: export */
 
#ifdef __cplusplus
extern "C" {
#endif
 
/* NOLINTBEGIN(bugprone-macro-parentheses, readability-inconsistent-declaration-parameter-name)
 *
 * The macros ATOMIC_LOAD_IMPL() and friends do not surround the argument used
 * to pass the type with parenthesis. Suppressing the clang-tidy warning here,
 * as adding parenthesis around a type would be a synstax error.
 *
 * The macro ATOMIC_FETCH_OP_IMPL() uses `val` as argument value. But we want
 * the declaration may be more specific (e.g. summand instead of val).
 */
 
/* Declarations and documentation: */
 
#if !defined(HAS_ATOMIC_BIT) || defined(DOXYGEN)
typedef struct {
    volatile uint8_t *dest;     
    uint8_t mask;               
} atomic_bit_u8_t;

typedef struct {
    volatile uint16_t *dest;    
    uint16_t mask;              
} atomic_bit_u16_t;

typedef struct {
    volatile uint32_t *dest;    
    uint32_t mask;              
} atomic_bit_u32_t;

typedef struct {
    volatile uint64_t *dest;    
    uint64_t mask;              
} atomic_bit_u64_t;

#endif /* HAS_ATOMIC_BIT */

#if UINT_MAX == UINT16_MAX
typedef atomic_bit_u16_t atomic_bit_unsigned_t;
#elif UINT_MAX == UINT32_MAX
typedef atomic_bit_u32_t atomic_bit_unsigned_t;
#else
typedef atomic_bit_u64_t atomic_bit_unsigned_t;
#endif

static inline uint8_t atomic_load_u8(const volatile uint8_t *var);
static inline uint16_t atomic_load_u16(const volatile uint16_t *var);
static inline uint32_t atomic_load_u32(const volatile uint32_t *var);
static inline uint64_t atomic_load_u64(const volatile uint64_t *var);
static inline unsigned atomic_load_unsigned(const volatile unsigned *var)
{
    if (sizeof(uint64_t) == sizeof(unsigned)) {
        return atomic_load_u64((const volatile uint64_t *)(uintptr_t)var);
    }
 
    if (sizeof(uint32_t) == sizeof(unsigned)) {
        return atomic_load_u32((const volatile uint32_t *)(uintptr_t)var);
    }
 
    return atomic_load_u16((const volatile uint16_t *)(uintptr_t)var);
}

static inline uintptr_t atomic_load_uintptr(const volatile uintptr_t *var) {
    if (sizeof(uintptr_t) == 2) {
        return atomic_load_u16((const volatile uint16_t *)var);
    }
 
    if (sizeof(uintptr_t) == 4) {
        return atomic_load_u32((const volatile uint32_t *)(uintptr_t)var);
    }
 
    return atomic_load_u64((const volatile uint64_t *)(uintptr_t)var);
}

static inline void * atomic_load_ptr(void **ptr_addr) {
    return (void *)atomic_load_uintptr((const volatile uintptr_t *)ptr_addr);
}

static inline kernel_pid_t atomic_load_kernel_pid(const volatile kernel_pid_t *var)
{
    return (kernel_pid_t)atomic_load_u16((const volatile uint16_t *)var);
}


static inline void atomic_store_u8(volatile uint8_t *dest, uint8_t val);
static inline void atomic_store_u16(volatile uint16_t *dest, uint16_t val);
static inline void atomic_store_u32(volatile uint32_t *dest, uint32_t val);
static inline void atomic_store_u64(volatile uint64_t *dest, uint64_t val);
static inline void atomic_store_unsigned(volatile unsigned *dest, unsigned val)
{
    if (sizeof(uint64_t) == sizeof(unsigned)) {
        atomic_store_u64((volatile uint64_t *)(uintptr_t)dest, val);
    }
    else if (sizeof(uint32_t) == sizeof(unsigned)) {
        atomic_store_u32((volatile uint32_t *)(uintptr_t)dest, val);
    }
    else {
        atomic_store_u16((volatile uint16_t *)(uintptr_t)dest, val);
    }
}

static inline void atomic_store_uintptr(volatile uintptr_t *dest, uintptr_t val)
{
    if (sizeof(uintptr_t) == 2) {
        atomic_store_u16((volatile uint16_t *)dest, (uint16_t)val);
    }
    else if (sizeof(uintptr_t) == 4) {
        atomic_store_u32((volatile uint32_t *)(uintptr_t)dest, (uint32_t)val);
    }
    else {
        atomic_store_u64((volatile uint64_t *)(uintptr_t)dest, (uint64_t)val);
    }
}

static inline void atomic_store_ptr(void **dest, const void *val) {
    atomic_store_uintptr((volatile uintptr_t *)dest, (uintptr_t)val);
}

static inline void atomic_store_kernel_pid(volatile kernel_pid_t *dest,
                                           kernel_pid_t val)
{
    atomic_store_u16((volatile uint16_t *)dest, (uint16_t)val);
}


static inline uint8_t atomic_fetch_add_u8(volatile uint8_t *dest,
                                          uint8_t summand);
static inline uint16_t atomic_fetch_add_u16(volatile uint16_t *dest,
                                            uint16_t summand);
static inline uint32_t atomic_fetch_add_u32(volatile uint32_t *dest,
                                            uint32_t summand);
static inline uint64_t atomic_fetch_add_u64(volatile uint64_t *dest,
                                            uint64_t summand);
static inline unsigned atomic_fetch_add_unsigned(volatile unsigned *dest,
                                                 unsigned summand)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return atomic_fetch_add_u64((volatile uint64_t *)(uintptr_t)dest, summand);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return atomic_fetch_add_u32((volatile uint32_t *)(uintptr_t)dest, summand);
    }
 
    return atomic_fetch_add_u16((volatile uint16_t *)(uintptr_t)dest, summand);
}


static inline uint8_t atomic_fetch_sub_u8(volatile uint8_t *dest,
                                          uint8_t subtrahend);
static inline uint16_t atomic_fetch_sub_u16(volatile uint16_t *dest,
                                            uint16_t subtrahend);
static inline uint32_t atomic_fetch_sub_u32(volatile uint32_t *dest,
                                            uint32_t subtrahend);
static inline uint64_t atomic_fetch_sub_u64(volatile uint64_t *dest,
                                            uint64_t subtrahend);
static inline unsigned atomic_fetch_sub_unsigned(volatile unsigned *dest,
                                                 unsigned subtrahend)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return atomic_fetch_sub_u64((volatile uint64_t *)(uintptr_t)dest, subtrahend);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return atomic_fetch_sub_u32((volatile uint32_t *)(uintptr_t)dest, subtrahend);
    }
 
    return atomic_fetch_sub_u16((volatile uint16_t *)(uintptr_t)dest, subtrahend);
}


static inline uint8_t atomic_fetch_or_u8(volatile uint8_t *dest, uint8_t val);
static inline uint16_t atomic_fetch_or_u16(volatile uint16_t *dest,
                                           uint16_t val);
static inline uint32_t atomic_fetch_or_u32(volatile uint32_t *dest,
                                           uint32_t val);
static inline uint64_t atomic_fetch_or_u64(volatile uint64_t *dest,
                                           uint64_t val);
static inline unsigned atomic_fetch_or_unsigned(volatile unsigned *dest,
                                                unsigned val)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return atomic_fetch_or_u64((volatile uint64_t *)(uintptr_t)dest, val);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return atomic_fetch_or_u32((volatile uint32_t *)(uintptr_t)dest, val);
    }
 
    return atomic_fetch_or_u16((volatile uint16_t *)(uintptr_t)dest, val);
}


static inline uint8_t atomic_fetch_xor_u8(volatile uint8_t *dest, uint8_t val);
static inline uint16_t atomic_fetch_xor_u16(volatile uint16_t *dest,
                                            uint16_t val);
static inline uint32_t atomic_fetch_xor_u32(volatile uint32_t *dest,
                                            uint32_t val);
static inline uint64_t atomic_fetch_xor_u64(volatile uint64_t *dest,
                                            uint64_t val);
static inline unsigned atomic_fetch_xor_unsigned(volatile unsigned *dest,
                                                 unsigned val)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return atomic_fetch_xor_u64((volatile uint64_t *)(uintptr_t)dest, val);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return atomic_fetch_xor_u32((volatile uint32_t *)(uintptr_t)dest, val);
    }
 
    return atomic_fetch_xor_u16((volatile uint16_t *)(uintptr_t)dest, val);
}


static inline uint8_t atomic_fetch_and_u8(volatile uint8_t *dest, uint8_t val);
static inline uint16_t atomic_fetch_and_u16(volatile uint16_t *dest,
                                            uint16_t val);
static inline uint32_t atomic_fetch_and_u32(volatile uint32_t *dest,
                                            uint32_t val);
static inline uint64_t atomic_fetch_and_u64(volatile uint64_t *dest,
                                            uint64_t val);
static inline unsigned atomic_fetch_and_unsigned(volatile unsigned *dest,
                                                 unsigned val)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return atomic_fetch_and_u64((volatile uint64_t *)(uintptr_t)dest, val);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return atomic_fetch_and_u32((volatile uint32_t *)(uintptr_t)dest, val);
    }
 
    return atomic_fetch_and_u16((volatile uint16_t *)(uintptr_t)dest, val);
}


static inline atomic_bit_u8_t atomic_bit_u8(volatile uint8_t *dest,
                                            uint8_t bit);

static inline atomic_bit_u16_t atomic_bit_u16(volatile uint16_t *dest,
                                              uint8_t bit);

static inline atomic_bit_u32_t atomic_bit_u32(volatile uint32_t *dest,
                                              uint8_t bit);

static inline atomic_bit_u64_t atomic_bit_u64(volatile uint64_t *dest,
                                              uint8_t bit);

static inline atomic_bit_unsigned_t atomic_bit_unsigned(volatile unsigned *dest,
                                                        uint8_t bit)
{
    /* Some archs define uint32_t as unsigned long, uint16_t as short etc.,
     * we need to cast. */
#if UINT_MAX == UINT16_MAX
    return atomic_bit_u16((uint16_t volatile *)dest, bit);
#elif UINT_MAX == UINT32_MAX
    return atomic_bit_u32((uint32_t volatile *)dest, bit);
#else
    return atomic_bit_u64((uint64_t volatile *)dest, bit);
#endif
}


static inline void atomic_set_bit_u8(atomic_bit_u8_t bit);
static inline void atomic_set_bit_u16(atomic_bit_u16_t bit);
static inline void atomic_set_bit_u32(atomic_bit_u32_t bit);
static inline void atomic_set_bit_u64(atomic_bit_u64_t bit);
static inline void atomic_set_bit_unsigned(atomic_bit_unsigned_t bit)
{
#if UINT_MAX == UINT16_MAX
      atomic_set_bit_u16(bit);
#elif UINT_MAX == UINT32_MAX
      atomic_set_bit_u32(bit);
#else
      atomic_set_bit_u64(bit);
#endif
}


static inline void atomic_clear_bit_u8(atomic_bit_u8_t bit);
static inline void atomic_clear_bit_u16(atomic_bit_u16_t bit);
static inline void atomic_clear_bit_u32(atomic_bit_u32_t bit);
static inline void atomic_clear_bit_u64(atomic_bit_u64_t bit);
static inline void atomic_clear_bit_unsigned(atomic_bit_unsigned_t bit)
{
#if UINT_MAX == UINT16_MAX
      atomic_clear_bit_u16(bit);
#elif UINT_MAX == UINT32_MAX
      atomic_clear_bit_u32(bit);
#else
      atomic_clear_bit_u64(bit);
#endif
}


static inline uint8_t semi_atomic_fetch_add_u8(volatile uint8_t *dest,
                                               uint8_t summand);
static inline uint16_t semi_atomic_fetch_add_u16(volatile uint16_t *dest,
                                                 uint16_t summand);
static inline uint32_t semi_atomic_fetch_add_u32(volatile uint32_t *dest,
                                                 uint32_t summand);
static inline uint64_t semi_atomic_fetch_add_u64(volatile uint64_t *dest,
                                                 uint64_t summand);
static inline unsigned semi_atomic_fetch_add_unsigned(volatile unsigned *dest,
                                                      unsigned summand)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return semi_atomic_fetch_add_u64((volatile uint64_t *)(uintptr_t)dest, summand);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return semi_atomic_fetch_add_u32((volatile uint32_t *)(uintptr_t)dest, summand);
    }
 
    return semi_atomic_fetch_add_u16((volatile uint16_t *)(uintptr_t)dest, summand);
}


static inline uint8_t semi_atomic_fetch_sub_u8(volatile uint8_t *dest,
                                               uint8_t subtrahend);
static inline uint16_t semi_atomic_fetch_sub_u16(volatile uint16_t *dest,
                                                 uint16_t subtrahend);
static inline uint32_t semi_atomic_fetch_sub_u32(volatile uint32_t *dest,
                                                 uint32_t subtrahend);
static inline uint64_t semi_atomic_fetch_sub_u64(volatile uint64_t *dest,
                                                 uint64_t subtrahend);
static inline unsigned semi_atomic_fetch_sub_unsigned(volatile unsigned *dest,
                                                      unsigned subtrahend)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return semi_atomic_fetch_sub_u64((volatile uint64_t *)(uintptr_t)dest, subtrahend);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return semi_atomic_fetch_sub_u32((volatile uint32_t *)(uintptr_t)dest, subtrahend);
    }
 
    return semi_atomic_fetch_sub_u16((volatile uint16_t *)(uintptr_t)dest, subtrahend);
}


static inline uint8_t semi_atomic_fetch_or_u8(volatile uint8_t *dest, uint8_t val);
static inline uint16_t semi_atomic_fetch_or_u16(volatile uint16_t *dest,
                                                uint16_t val);
static inline uint32_t semi_atomic_fetch_or_u32(volatile uint32_t *dest,
                                                uint32_t val);
static inline uint64_t semi_atomic_fetch_or_u64(volatile uint64_t *dest,
                                                uint64_t val);
static inline unsigned semi_atomic_fetch_or_unsigned(volatile unsigned *dest,
                                                     unsigned val)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return semi_atomic_fetch_or_u64((volatile uint64_t *)(uintptr_t)dest, val);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return semi_atomic_fetch_or_u32((volatile uint32_t *)(uintptr_t)dest, val);
    }
 
    return semi_atomic_fetch_or_u16((volatile uint16_t *)(uintptr_t)dest, val);
}


static inline uint8_t semi_atomic_fetch_xor_u8(volatile uint8_t *dest,
                                               uint8_t val);
static inline uint16_t semi_atomic_fetch_xor_u16(volatile uint16_t *dest,
                                                 uint16_t val);
static inline uint32_t semi_atomic_fetch_xor_u32(volatile uint32_t *dest,
                                                 uint32_t val);
static inline uint64_t semi_atomic_fetch_xor_u64(volatile uint64_t *dest,
                                                 uint64_t val);
static inline unsigned semi_atomic_fetch_xor_unsigned(volatile unsigned *dest,
                                                      unsigned val)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return semi_atomic_fetch_xor_u64((volatile uint64_t *)(uintptr_t)dest, val);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return semi_atomic_fetch_xor_u32((volatile uint32_t *)(uintptr_t)dest, val);
    }
 
    return semi_atomic_fetch_xor_u16((volatile uint16_t *)(uintptr_t)dest, val);
}


static inline uint8_t semi_atomic_fetch_and_u8(volatile uint8_t *dest,
                                               uint8_t val);
static inline uint16_t semi_atomic_fetch_and_u16(volatile uint16_t *dest,
                                                 uint16_t val);
static inline uint32_t semi_atomic_fetch_and_u32(volatile uint32_t *dest,
                                                 uint32_t val);
static inline uint64_t semi_atomic_fetch_and_u64(volatile uint64_t *dest,
                                                 uint64_t val);
static inline unsigned semi_atomic_fetch_and_unsigned(volatile unsigned *dest,
                                                      unsigned val)
{
    if (sizeof(unsigned) == sizeof(uint64_t)) {
        return semi_atomic_fetch_and_u64((volatile uint64_t *)(uintptr_t)dest, val);
    }
 
    if (sizeof(unsigned) == sizeof(uint32_t)) {
        return semi_atomic_fetch_and_u32((volatile uint32_t *)(uintptr_t)dest, val);
    }
 
    return semi_atomic_fetch_and_u16((volatile uint16_t *)(uintptr_t)dest, val);
}

 
/* Fallback implementations of atomic utility functions: */

#define ATOMIC_LOAD_IMPL(name, type)                                           \
    static inline type CONCAT(atomic_load_, name)(const volatile type *var)    \
    {                                                                          \
        unsigned state = irq_disable();                                        \
        type result = *var;                                                    \
        irq_restore(state);                                                    \
        return result;                                                         \
    }
 
#ifndef HAS_ATOMIC_LOAD_U8
ATOMIC_LOAD_IMPL(u8, uint8_t)
#endif
#ifndef HAS_ATOMIC_LOAD_U16
ATOMIC_LOAD_IMPL(u16, uint16_t)
#endif
#ifndef HAS_ATOMIC_LOAD_U32
ATOMIC_LOAD_IMPL(u32, uint32_t)
#endif
#ifndef HAS_ATOMIC_LOAD_U64
ATOMIC_LOAD_IMPL(u64, uint64_t)
#endif

#define ATOMIC_STORE_IMPL(name, type)                                          \
    static inline void CONCAT(atomic_store_, name)                             \
            (volatile type *dest, type val)                                    \
    {                                                                          \
        unsigned state = irq_disable();                                        \
        *dest = val;                                                           \
        irq_restore(state);                                                    \
    }
 
#ifndef HAS_ATOMIC_STORE_U8
ATOMIC_STORE_IMPL(u8, uint8_t)
#endif
#ifndef HAS_ATOMIC_STORE_U16
ATOMIC_STORE_IMPL(u16, uint16_t)
#endif
#ifndef HAS_ATOMIC_STORE_U32
ATOMIC_STORE_IMPL(u32, uint32_t)
#endif
#ifndef HAS_ATOMIC_STORE_U64
ATOMIC_STORE_IMPL(u64, uint64_t)
#endif

#define ATOMIC_FETCH_OP_IMPL(opname, op, name, type)                           \
    static inline type CONCAT4(atomic_fetch_, opname, _, name)                 \
            (volatile type *dest, type val)                                    \
    {                                                                          \
        unsigned state = irq_disable();                                        \
        const type result = *dest;                                             \
        *dest = result op val;                                                 \
        irq_restore(state);                                                    \
        return result;                                                         \
    }
 
#ifndef HAS_ATOMIC_FETCH_ADD_U8
ATOMIC_FETCH_OP_IMPL(add, +, u8, uint8_t)
#endif
#ifndef HAS_ATOMIC_FETCH_ADD_U16
ATOMIC_FETCH_OP_IMPL(add, +, u16, uint16_t)
#endif
#ifndef HAS_ATOMIC_FETCH_ADD_U32
ATOMIC_FETCH_OP_IMPL(add, +, u32, uint32_t)
#endif
#ifndef HAS_ATOMIC_FETCH_ADD_U64
ATOMIC_FETCH_OP_IMPL(add, +, u64, uint64_t)
#endif
 
#ifndef HAS_ATOMIC_FETCH_SUB_U8
ATOMIC_FETCH_OP_IMPL(sub, -, u8, uint8_t)
#endif
#ifndef HAS_ATOMIC_FETCH_SUB_U16
ATOMIC_FETCH_OP_IMPL(sub, -, u16, uint16_t)
#endif
#ifndef HAS_ATOMIC_FETCH_SUB_U32
ATOMIC_FETCH_OP_IMPL(sub, -, u32, uint32_t)
#endif
#ifndef HAS_ATOMIC_FETCH_SUB_U64
ATOMIC_FETCH_OP_IMPL(sub, -, u64, uint64_t)
#endif
 
#ifndef HAS_ATOMIC_FETCH_OR_U8
ATOMIC_FETCH_OP_IMPL(or, |, u8, uint8_t)
#endif
#ifndef HAS_ATOMIC_FETCH_OR_U16
ATOMIC_FETCH_OP_IMPL(or, |, u16, uint16_t)
#endif
#ifndef HAS_ATOMIC_FETCH_OR_U32
ATOMIC_FETCH_OP_IMPL(or, |, u32, uint32_t)
#endif
#ifndef HAS_ATOMIC_FETCH_OR_U64
ATOMIC_FETCH_OP_IMPL(or, |, u64, uint64_t)
#endif
 
#ifndef HAS_ATOMIC_FETCH_XOR_U8
ATOMIC_FETCH_OP_IMPL(xor, ^, u8, uint8_t)
#endif
#ifndef HAS_ATOMIC_FETCH_XOR_U16
ATOMIC_FETCH_OP_IMPL(xor, ^, u16, uint16_t)
#endif
#ifndef HAS_ATOMIC_FETCH_XOR_U32
ATOMIC_FETCH_OP_IMPL(xor, ^, u32, uint32_t)
#endif
#ifndef HAS_ATOMIC_FETCH_XOR_U64
ATOMIC_FETCH_OP_IMPL(xor, ^, u64, uint64_t)
#endif
 
#ifndef HAS_ATOMIC_FETCH_AND_U8
ATOMIC_FETCH_OP_IMPL(and, &, u8, uint8_t)
#endif
#ifndef HAS_ATOMIC_FETCH_AND_U16
ATOMIC_FETCH_OP_IMPL(and, &, u16, uint16_t)
#endif
#ifndef HAS_ATOMIC_FETCH_AND_U32
ATOMIC_FETCH_OP_IMPL(and, &, u32, uint32_t)
#endif
#ifndef HAS_ATOMIC_FETCH_AND_U64
ATOMIC_FETCH_OP_IMPL(and, &, u64, uint64_t)
#endif
 
#ifndef HAS_ATOMIC_BIT
static inline atomic_bit_u8_t atomic_bit_u8(volatile uint8_t *dest,
                                            uint8_t bit)
{
    atomic_bit_u8_t result = { .dest = dest, .mask = (uint8_t)(1U << bit) };
    return result;
}
static inline atomic_bit_u16_t atomic_bit_u16(volatile uint16_t *dest,
                                              uint8_t bit)
{
    atomic_bit_u16_t result = { .dest = dest, .mask = (uint16_t)(1U << bit) };
    return result;
}
static inline atomic_bit_u32_t atomic_bit_u32(volatile uint32_t *dest,
                                              uint8_t bit)
{
    atomic_bit_u32_t result = { .dest = dest, .mask = (uint32_t)(1UL << bit) };
    return result;
}
static inline atomic_bit_u64_t atomic_bit_u64(volatile uint64_t *dest,
                                              uint8_t bit)
{
    atomic_bit_u64_t result = { .dest = dest, .mask = (uint64_t)(1ULL << bit) };
    return result;
}
static inline void atomic_set_bit_u8(atomic_bit_u8_t bit)
{
    atomic_fetch_or_u8(bit.dest, bit.mask);
}
static inline void atomic_set_bit_u16(atomic_bit_u16_t bit)
{
    atomic_fetch_or_u16(bit.dest, bit.mask);
}
static inline void atomic_set_bit_u32(atomic_bit_u32_t bit)
{
    atomic_fetch_or_u32(bit.dest, bit.mask);
}
static inline void atomic_set_bit_u64(atomic_bit_u64_t bit)
{
    atomic_fetch_or_u64(bit.dest, bit.mask);
}
static inline void atomic_clear_bit_u8(atomic_bit_u8_t bit)
{
    atomic_fetch_and_u8(bit.dest, ~bit.mask);
}
static inline void atomic_clear_bit_u16(atomic_bit_u16_t bit)
{
    atomic_fetch_and_u16(bit.dest, ~bit.mask);
}
static inline void atomic_clear_bit_u32(atomic_bit_u32_t bit)
{
    atomic_fetch_and_u32(bit.dest, ~bit.mask);
}
static inline void atomic_clear_bit_u64(atomic_bit_u64_t bit)
{
    atomic_fetch_and_u64(bit.dest, ~bit.mask);
}
#endif
 
/* Provide semi_atomic_*() functions on top.
 *
 * - If atomic_<FOO>() is provided: Use this for semi_atomic_<FOO>() as well
 * - Else:
 *      - If matching `atomic_store_u<BITS>()` is provided: Only make final
 *        store atomic, as we can avoid touching the IRQ state register that
 *        way
 *      - Else: We need to disable and re-enable IRQs anyway, we just use the
 *        fallback implementation of `atomic_<FOO>()` for `semi_atomic<FOO>()`
 *        as well
 */
 
/* FETCH_ADD */
#if defined(HAS_ATOMIC_FETCH_ADD_U8) || !defined(HAS_ATOMIC_STORE_U8)
static inline uint8_t semi_atomic_fetch_add_u8(volatile uint8_t *dest,
                                               uint8_t val)
{
    return atomic_fetch_add_u8(dest, val);
}
#else
static inline uint8_t semi_atomic_fetch_add_u8(volatile uint8_t *dest,
                                               uint8_t val)
{
    uint8_t result = atomic_load_u8(dest);
    atomic_store_u8(dest, result + val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_ADD_U8 || !HAS_ATOMIC_STORE_U8 */
 
#if defined(HAS_ATOMIC_FETCH_ADD_U16) || !defined(HAS_ATOMIC_STORE_U16)
static inline uint16_t semi_atomic_fetch_add_u16(volatile uint16_t *dest,
                                                 uint16_t val)
{
    return atomic_fetch_add_u16(dest, val);
}
#else
static inline uint16_t semi_atomic_fetch_add_u16(volatile uint16_t *dest,
                                                 uint16_t val)
{
    uint16_t result = atomic_load_u16(dest);
    atomic_store_u16(dest, result + val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_ADD_U16 || !HAS_ATOMIC_STORE_U16 */
 
#if defined(HAS_ATOMIC_FETCH_ADD_U32) || !defined(HAS_ATOMIC_STORE_U32)
static inline uint32_t semi_atomic_fetch_add_u32(volatile uint32_t *dest,
                                                 uint32_t val)
{
    return atomic_fetch_add_u32(dest, val);
}
#else
static inline uint32_t semi_atomic_fetch_add_u32(volatile uint32_t *dest,
                                                 uint32_t val)
{
    uint32_t result = atomic_load_u32(dest);
    atomic_store_u32(dest, result + val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_ADD_U32 || !HAS_ATOMIC_STORE_U32 */
 
#if defined(HAS_ATOMIC_FETCH_ADD_U64) || !defined(HAS_ATOMIC_STORE_U64)
static inline uint64_t semi_atomic_fetch_add_u64(volatile uint64_t *dest,
                                                 uint64_t val)
{
    return atomic_fetch_add_u64(dest, val);
}
#else
static inline uint64_t semi_atomic_fetch_add_u64(volatile uint64_t *dest,
                                                 uint64_t val)
{
    atomic_store_u64(dest, *dest + val);
}
#endif /* HAS_ATOMIC_FETCH_ADD_U32 || !HAS_ATOMIC_STORE_U32 */
 
/* FETCH_SUB */
#if defined(HAS_ATOMIC_FETCH_SUB_U8) || !defined(HAS_ATOMIC_STORE_U8)
static inline uint8_t semi_atomic_fetch_sub_u8(volatile uint8_t *dest,
                                               uint8_t val)
{
    return atomic_fetch_sub_u8(dest, val);
}
#else
static inline uint8_t semi_atomic_fetch_sub_u8(volatile uint8_t *dest,
                                               uint8_t val)
{
    uint8_t result = atomic_load_u8(dest);
    atomic_store_u8(dest, result - val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_SUB_U8 || !HAS_ATOMIC_STORE_U8 */
 
#if defined(HAS_ATOMIC_FETCH_SUB_U16) || !defined(HAS_ATOMIC_STORE_U16)
static inline uint16_t semi_atomic_fetch_sub_u16(volatile uint16_t *dest,
                                                 uint16_t val)
{
    return atomic_fetch_sub_u16(dest, val);
}
#else
static inline uint16_t semi_atomic_fetch_sub_u16(volatile uint16_t *dest,
                                                 uint16_t val)
{
    uint16_t result = atomic_load_u16(dest);
    atomic_store_u16(dest, result - val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_SUB_U16 || !HAS_ATOMIC_STORE_U16 */
 
#if defined(HAS_ATOMIC_FETCH_SUB_U32) || !defined(HAS_ATOMIC_STORE_U32)
static inline uint32_t semi_atomic_fetch_sub_u32(volatile uint32_t *dest,
                                                 uint32_t val)
{
    return atomic_fetch_sub_u32(dest, val);
}
#else
static inline uint32_t semi_atomic_fetch_sub_u32(volatile uint32_t *dest,
                                                 uint32_t val)
{
    uint32_t result = atomic_load_u32(dest);
    atomic_store_u32(dest, result - val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_SUB_U32 || !HAS_ATOMIC_STORE_U64 */
 
#if defined(HAS_ATOMIC_FETCH_SUB_U64) || !defined(HAS_ATOMIC_STORE_U64)
static inline uint64_t semi_atomic_fetch_sub_u64(volatile uint64_t *dest,
                                                 uint64_t val)
{
    return atomic_fetch_sub_u64(dest, val);
}
#else
static inline uint64_t semi_atomic_fetch_sub_u64(volatile uint64_t *dest,
                                                 uint64_t val)
{
    uint64_t result = atomic_load_u64(dest);
    atomic_store_u64(dest, result - val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_SUB_U64 || !HAS_ATOMIC_STORE_U64 */
 
/* FETCH_OR */
#if defined(HAS_ATOMIC_FETCH_OR_U8) || !defined(HAS_ATOMIC_STORE_U8)
static inline uint8_t semi_atomic_fetch_or_u8(volatile uint8_t *dest,
                                              uint8_t val)
{
    return atomic_fetch_or_u8(dest, val);
}
#else
static inline uint8_t semi_atomic_fetch_or_u8(volatile uint8_t *dest,
                                              uint8_t val)
{
    uint8_t result = atomic_load_u8(dest);
    atomic_store_u8(dest, result | val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_OR_U8 || !HAS_ATOMIC_STORE_U8 */
 
#if defined(HAS_ATOMIC_FETCH_OR_U16) || !defined(HAS_ATOMIC_STORE_U16)
static inline uint16_t semi_atomic_fetch_or_u16(volatile uint16_t *dest,
                                                uint16_t val)
{
    return atomic_fetch_or_u16(dest, val);
}
#else
static inline uint16_t semi_atomic_fetch_or_u16(volatile uint16_t *dest,
                                                uint16_t val)
{
    uint16_t result = atomic_load_u16(dest);
    atomic_store_u16(dest, result | val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_OR_U16 || !HAS_ATOMIC_STORE_U16 */
 
#if defined(HAS_ATOMIC_FETCH_OR_U32) || !defined(HAS_ATOMIC_STORE_U32)
static inline uint32_t semi_atomic_fetch_or_u32(volatile uint32_t *dest,
                                                uint32_t val)
{
    return atomic_fetch_or_u32(dest, val);
}
#else
static inline uint32_t semi_atomic_fetch_or_u32(volatile uint32_t *dest,
                                                uint32_t val)
{
    uint32_t result = atomic_load_u32(dest);
    atomic_store_u32(dest, result | val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_OR_U32 || !HAS_ATOMIC_STORE_U32 */
 
#if defined(HAS_ATOMIC_FETCH_OR_U64) || !defined(HAS_ATOMIC_STORE_U64)
static inline uint64_t semi_atomic_fetch_or_u64(volatile uint64_t *dest,
                                                uint64_t val)
{
    return atomic_fetch_or_u64(dest, val);
}
#else
static inline uint64_t semi_atomic_fetch_or_u64(volatile uint64_t *dest,
                                                uint64_t val)
{
    uint64_t result = atomic_load_u64(dest);
    atomic_store_u64(dest, result | val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_OR_U64 || !HAS_ATOMIC_STORE_U64 */
 
/* FETCH_XOR */
#if defined(HAS_ATOMIC_FETCH_XOR_U8) || !defined(HAS_ATOMIC_STORE_U8)
static inline uint8_t semi_atomic_fetch_xor_u8(volatile uint8_t *dest,
                                               uint8_t val)
{
    return atomic_fetch_xor_u8(dest, val);
}
#else
static inline uint8_t semi_atomic_fetch_xor_u8(volatile uint8_t *dest,
                                               uint8_t val)
{
    uint8_t result = atomic_load_u8(dest);
    atomic_store_u8(dest, result ^ val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_XOR_U8 || !HAS_ATOMIC_STORE_U8 */
 
#if defined(HAS_ATOMIC_FETCH_XOR_U16) || !defined(HAS_ATOMIC_STORE_U16)
static inline uint16_t semi_atomic_fetch_xor_u16(volatile uint16_t *dest,
                                                 uint16_t val)
{
    return atomic_fetch_xor_u16(dest, val);
}
#else
static inline uint16_t semi_atomic_fetch_xor_u16(volatile uint16_t *dest,
                                                 uint16_t val)
{
    uint16_t result = atomic_load_u16(dest);
    atomic_store_u16(dest, result ^ val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_XOR_U16 || !HAS_ATOMIC_STORE_U16 */
 
#if defined(HAS_ATOMIC_FETCH_XOR_U32) || !defined(HAS_ATOMIC_STORE_U32)
static inline uint32_t semi_atomic_fetch_xor_u32(volatile uint32_t *dest,
                                                 uint32_t val)
{
    return atomic_fetch_xor_u32(dest, val);
}
#else
static inline uint32_t semi_atomic_fetch_xor_u32(volatile uint32_t *dest,
                                                 uint32_t val)
{
    uint32_t result = atomic_load_u32(dest);
    atomic_store_u32(dest, result ^ val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_XOR_U32 || !HAS_ATOMIC_STORE_U32 */
 
#if defined(HAS_ATOMIC_FETCH_XOR_U64) || !defined(HAS_ATOMIC_STORE_U64)
static inline uint64_t semi_atomic_fetch_xor_u64(volatile uint64_t *dest,
                                                 uint64_t val)
{
    return atomic_fetch_xor_u64(dest, val);
}
#else
static inline uint64_t semi_atomic_fetch_xor_u64(volatile uint64_t *dest,
                                                 uint64_t val)
{
    uint64_t result = atomic_load_u64(dest);
    atomic_store_u64(dest, result ^ val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_XOR_U64 || !HAS_ATOMIC_STORE_U64 */
 
/* FETCH_AND */
#if defined(HAS_ATOMIC_FETCH_AND_U8) || !defined(HAS_ATOMIC_STORE_U8)
static inline uint8_t semi_atomic_fetch_and_u8(volatile uint8_t *dest,
                                               uint8_t val)
{
    return atomic_fetch_and_u8(dest, val);
}
#else
static inline uint8_t semi_atomic_fetch_and_u8(volatile uint8_t *dest,
                                               uint8_t val)
{
    uint8_t result = atomic_load_u8(dest);
    atomic_store_u8(dest, result & val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_AND_U8 || !HAS_ATOMIC_STORE_U8 */
 
#if defined(HAS_ATOMIC_FETCH_AND_U16) || !defined(HAS_ATOMIC_STORE_U16)
static inline uint16_t semi_atomic_fetch_and_u16(volatile uint16_t *dest,
                                                 uint16_t val)
{
    return atomic_fetch_and_u16(dest, val);
}
#else
static inline uint16_t semi_atomic_fetch_and_u16(volatile uint16_t *dest,
                                                 uint16_t val)
{
    uint16_t result = atomic_load_u16(dest);
    atomic_store_u16(dest, result & val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_AND_U16 || !HAS_ATOMIC_STORE_U16 */
 
#if defined(HAS_ATOMIC_FETCH_AND_U32) || !defined(HAS_ATOMIC_STORE_U32)
static inline uint32_t semi_atomic_fetch_and_u32(volatile uint32_t *dest,
                                                 uint32_t val)
{
    return atomic_fetch_and_u32(dest, val);
}
#else
static inline uint32_t semi_atomic_fetch_and_u32(volatile uint32_t *dest,
                                                 uint32_t val)
{
    uint32_t result = atomic_load_u32(dest);
    atomic_store_u32(dest, result & val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_AND_U32 || !HAS_ATOMIC_STORE_U32 */
 
#if defined(HAS_ATOMIC_FETCH_AND_U64) || !defined(HAS_ATOMIC_STORE_U64)
static inline uint64_t semi_atomic_fetch_and_u64(volatile uint64_t *dest,
                                                 uint64_t val)
{
    return atomic_fetch_and_u64(dest, val);
}
#else
static inline uint64_t semi_atomic_fetch_and_u64(volatile uint64_t *dest,
                                                 uint64_t val)
{
    uint64_t result = atomic_load_u64(dest);
    atomic_store_u64(dest, result & val);
    return result;
}
#endif /* HAS_ATOMIC_FETCH_AND_U64 || !HAS_ATOMIC_STORE_U64 */
 
#ifdef __cplusplus
}
#endif
 
/* NOLINTEND(bugprone-macro-parentheses, readability-inconsistent-declaration-parameter-name) */