intrinsics.c (8111B)
1 /* See LICENSE for license details. */ 2 #include "compiler.h" 3 4 #if COMPILER_CLANG || COMPILER_GCC 5 #define force_inline inline __attribute__((always_inline)) 6 #elif COMPILER_MSVC 7 #define force_inline __forceinline 8 #endif 9 10 #if COMPILER_MSVC || (COMPILER_CLANG && OS_WINDOWS) 11 #pragma section(".rdata$", read) 12 #define read_only __declspec(allocate(".rdata$")) 13 #elif COMPILER_CLANG 14 #define read_only __attribute__((section(".rodata"))) 15 #elif COMPILER_GCC 16 /* TODO(rnp): how do we do this with gcc, putting it in rodata causes warnings and writing to 17 * it doesn't cause a fault */ 18 #define read_only 19 #endif 20 21 #if !defined(countof) 22 #define countof(a) (i64)(sizeof(a) / sizeof(*a)) 23 #endif 24 25 #if COMPILER_MSVC 26 #define alignas(n) __declspec(align(n)) 27 #define pack_struct(s) __pragma(pack(push, 1)) s __pragma(pack(pop)) 28 #define no_return __declspec(noreturn) 29 30 #define likely(x) (x) 31 #define unlikely(x) (x) 32 33 #define print_format(f, va) 34 35 #define assume(x) __assume(x) 36 #define debugbreak() __debugbreak() 37 #define unreachable() __assume(0) 38 39 #if ARCH_ARM64 40 #define cpu_yield() __yield() 41 #define store_fence() __dmb(0x0A) // 0x0A: ishst 42 #endif 43 44 #define atomic_add_u32(ptr, n) _InterlockedExchangeAdd((volatile u32 *)(ptr), (n)) 45 #define atomic_add_u64(ptr, n) _InterlockedExchangeAdd64((volatile u64 *)(ptr), (n)) 46 #define atomic_and_u32(ptr, n) _InterlockedAnd((volatile u32 *)(ptr), (n)) 47 #define atomic_and_u64(ptr, n) _InterlockedAnd64((volatile u64 *)(ptr), (n)) 48 #define atomic_cas_u32(ptr, cptr, n) (_InterlockedCompareExchange((volatile u32 *)(ptr), *(cptr), (n)) == *(cptr)) 49 #define atomic_cas_u64(ptr, cptr, n) (_InterlockedCompareExchange64((volatile u64 *)(ptr), *(cptr), (n)) == *(cptr)) 50 #define atomic_load_u32(ptr) *((volatile u32 *)(ptr)) 51 #define atomic_load_u64(ptr) *((volatile u64 *)(ptr)) 52 #define atomic_or_u32(ptr, n) _InterlockedOr((volatile u32 *)(ptr), (n)) 53 #define atomic_store_u32(ptr, n) *((volatile u32 *)(ptr)) = (u32)(n) 54 #define atomic_store_u64(ptr, n) *((volatile u64 *)(ptr)) = (u64)(n) 55 #define atomic_swap_u32(ptr, n) _InterlockedExchange((volatile u32 *)(ptr), n) 56 #define atomic_swap_u64(ptr, n) _InterlockedExchange64((volatile u64 *)(ptr), n) 57 58 #define atan2_f32(y, x) atan2f(y, x) 59 #define cos_f32(a) cosf(a) 60 #define sin_f32(a) sinf(a) 61 #define tan_f32(a) tanf(a) 62 #define ceil_f32(a) ceilf(a) 63 #define sqrt_f32(a) sqrtf(a) 64 65 #define exp_f64(a) exp(a) 66 #define sqrt_f64(a) sqrt(a) 67 68 #else 69 #define alignas(n) __attribute__((aligned(n))) 70 #define pack_struct(s) s __attribute__((packed)) 71 #define no_return __attribute__((noreturn)) 72 73 #define likely(x) (__builtin_expect(!!(x), 1)) 74 #define unlikely(x) (__builtin_expect(!!(x), 0)) 75 76 #define print_format(f, va) __attribute__((format(printf, f, va))) 77 78 #if COMPILER_CLANG 79 #define assume(x) __builtin_assume(x) 80 #else 81 #if defined(__has_attribute) 82 #if __has_attribute(assume) 83 #define assume(x) __attribute__((assume(x))) 84 #endif 85 #endif 86 #endif 87 #if !defined(assume) 88 #define assume(x) if (!(x)) unreachable() 89 #endif 90 #define unreachable() __builtin_unreachable() 91 #if ARCH_ARM64 92 /* TODO? debuggers just loop here forever and need a manual PC increment (step over) */ 93 #define debugbreak() asm volatile ("brk 0xf000") 94 #define cpu_yield() asm volatile ("yield") 95 #define store_fence() asm volatile ("dmb ishst" ::: "memory") 96 #else 97 #define debugbreak() asm volatile ("int3; nop") 98 #endif 99 100 #define atomic_add_u64(ptr, n) __atomic_fetch_add(ptr, n, __ATOMIC_SEQ_CST) 101 #define atomic_and_u64(ptr, n) __atomic_and_fetch(ptr, n, __ATOMIC_SEQ_CST) 102 #define atomic_cas_u64(ptr, cptr, n) __atomic_compare_exchange_n(ptr, cptr, n, 0, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST) 103 #define atomic_load_u64(ptr) __atomic_load_n(ptr, __ATOMIC_SEQ_CST) 104 #define atomic_or_u32(ptr, n) __atomic_or_fetch(ptr, n, __ATOMIC_SEQ_CST) 105 #define atomic_store_u64(ptr, n) __atomic_store_n(ptr, n, __ATOMIC_SEQ_CST) 106 #define atomic_swap_u64(ptr, n) __atomic_exchange_n(ptr, n, __ATOMIC_SEQ_CST) 107 #define atomic_add_u32 atomic_add_u64 108 #define atomic_and_u32 atomic_and_u64 109 #define atomic_cas_u32 atomic_cas_u64 110 #define atomic_load_u32 atomic_load_u64 111 #define atomic_store_u32 atomic_store_u64 112 #define atomic_swap_u32 atomic_swap_u64 113 114 #define atan2_f32(y, x) __builtin_atan2f(y, x) 115 #define cos_f32(a) __builtin_cosf(a) 116 #define sin_f32(a) __builtin_sinf(a) 117 #define tan_f32(a) __builtin_tanf(a) 118 #define ceil_f32(a) __builtin_ceilf(a) 119 #define sqrt_f32(a) __builtin_sqrtf(a) 120 121 #define exp_f64(a) __builtin_exp(a) 122 #define sqrt_f64(a) __builtin_sqrt(a) 123 124 #define popcount_u64(a) (u64)__builtin_popcountll(a) 125 #endif 126 127 #if COMPILER_MSVC 128 129 function force_inline u64 130 clz_u64(u64 a) 131 { 132 u64 result = 64, index; 133 if (a) { 134 _BitScanReverse64(&index, a); 135 result = index; 136 } 137 return result; 138 } 139 140 function force_inline u64 141 ctz_u64(u64 a) 142 { 143 u64 result = 64, index; 144 if (a) { 145 _BitScanForward64(&index, a); 146 result = index; 147 } 148 return result; 149 } 150 151 #else /* !COMPILER_MSVC */ 152 153 function force_inline u64 154 clz_u64(u32 a) 155 { 156 u64 result = 64; 157 if (a) result = (u64)__builtin_clzll(a); 158 return result; 159 } 160 161 function force_inline u64 162 ctz_u64(u64 a) 163 { 164 u64 result = 64; 165 if (a) result = (u64)__builtin_ctzll(a); 166 return result; 167 } 168 169 #endif 170 171 #if ARCH_ARM64 172 /* NOTE(rnp): we are only doing a handful of f32x4 operations so we will just use NEON and do 173 * the macro renaming thing. If you are implementing a serious wide vector operation you should 174 * use SVE(2) instead. The semantics are different however and the code will be written for an 175 * arbitrary vector bit width. In that case you will also need x86_64 code for determining 176 * the supported vector width (ideally at runtime though that may not be possible). 177 */ 178 #include <arm_neon.h> 179 typedef float32x4_t f32x4; 180 typedef int32x4_t i32x4; 181 typedef uint32x4_t u32x4; 182 183 #define add_f32x4(a, b) vaddq_f32(a, b) 184 #define cvt_i32x4_f32x4(a) vcvtq_f32_s32(a) 185 #define cvt_f32x4_i32x4(a) vcvtq_s32_f32(a) 186 #define div_f32x4(a, b) vdivq_f32(a, b) 187 #define dup_f32x4(f) vdupq_n_f32(f) 188 #define floor_f32x4(a) vrndmq_f32(a) 189 #define load_f32x4(a) vld1q_f32(a) 190 #define load_i32x4(a) vld1q_s32(a) 191 #define max_f32x4(a, b) vmaxq_f32(a, b) 192 #define min_f32x4(a, b) vminq_f32(a, b) 193 #define mul_f32x4(a, b) vmulq_f32(a, b) 194 #define set_f32x4(a, b, c, d) vld1q_f32((f32 []){d, c, b, a}) 195 #define sqrt_f32x4(a) vsqrtq_f32(a) 196 #define store_f32x4(o, a) vst1q_f32(o, a) 197 #define store_i32x4(o, a) vst1q_s32(o, a) 198 #define sub_f32x4(a, b) vsubq_f32(a, b) 199 200 #elif ARCH_X64 201 #include <immintrin.h> 202 typedef __m128 f32x4; 203 typedef __m128i i32x4; 204 typedef __m128i u32x4; 205 206 #define add_f32x4(a, b) _mm_add_ps(a, b) 207 #define cvt_i32x4_f32x4(a) _mm_cvtepi32_ps(a) 208 #define cvt_f32x4_i32x4(a) _mm_cvtps_epi32(a) 209 #define div_f32x4(a, b) _mm_div_ps(a, b) 210 #define dup_f32x4(f) _mm_set1_ps(f) 211 #define floor_f32x4(a) _mm_floor_ps(a) 212 #define load_f32x4(a) _mm_loadu_ps(a) 213 #define load_i32x4(a) _mm_loadu_si128((i32x4 *)a) 214 #define max_f32x4(a, b) _mm_max_ps(a, b) 215 #define min_f32x4(a, b) _mm_min_ps(a, b) 216 #define mul_f32x4(a, b) _mm_mul_ps(a, b) 217 #define set_f32x4(a, b, c, d) _mm_set_ps(a, b, c, d) 218 #define sqrt_f32x4(a) _mm_sqrt_ps(a) 219 #define store_f32x4(o, a) _mm_storeu_ps(o, a) 220 #define store_i32x4(o, a) _mm_storeu_si128((i32x4 *)o, a) 221 #define sub_f32x4(a, b) _mm_sub_ps(a, b) 222 223 #define cpu_yield _mm_pause 224 #define store_fence _mm_sfence 225 226 #endif 227 228 function force_inline f32 229 inf32(void) 230 { 231 union {u32 u; f32 f;} result; 232 result.u = 0x7F800000u; 233 return result.f; 234 }