beamformer_shared_memory.c (9540B)
1 /* See LICENSE for license details. */ 2 #define BEAMFORMER_SHARED_MEMORY_VERSION (32UL) 3 4 typedef enum { 5 BeamformerWorkKind_Compute, 6 BeamformerWorkKind_ComputeIndirect, 7 BeamformerWorkKind_CreateFilter, 8 BeamformerWorkKind_ExportBuffer, 9 } BeamformerWorkKind; 10 11 typedef struct { 12 BeamformerFilterParameters parameters; 13 u8 filter_slot; 14 u8 parameter_block; 15 static_assert(BeamformerFilterSlots <= 255, "CreateFilterContext only supports 255 filter slots"); 16 static_assert(BeamformerMaxParameterBlocks <= 255, "CreateFilterContext only supports 255 parameter blocks"); 17 } BeamformerCreateFilterContext; 18 19 typedef enum { 20 BeamformerExportKind_BeamformedData, 21 BeamformerExportKind_Stats, 22 } BeamformerExportKind; 23 24 typedef struct { 25 BeamformerExportKind kind; 26 u32 size; 27 } BeamformerExportContext; 28 29 #define BEAMFORMER_SHARED_MEMORY_LOCKS \ 30 X(ScratchSpace) \ 31 X(UploadRF) \ 32 X(ExportSync) \ 33 X(DispatchCompute) 34 35 #define X(name) BeamformerSharedMemoryLockKind_##name, 36 typedef enum {BEAMFORMER_SHARED_MEMORY_LOCKS BeamformerSharedMemoryLockKind_Count} BeamformerSharedMemoryLockKind; 37 #undef X 38 39 typedef struct { 40 BeamformerViewPlaneTag view_plane; 41 u32 parameter_block; 42 } BeamformerComputeWorkContext; 43 44 /* NOTE: discriminated union based on type */ 45 typedef struct { 46 BeamformerWorkKind kind; 47 BeamformerSharedMemoryLockKind lock; 48 union { 49 void *generic; 50 BeamformerComputeWorkContext compute_context; 51 BeamformerCreateFilterContext create_filter_context; 52 BeamformerExportContext export_context; 53 BeamformerShaderKind reload_shader; 54 }; 55 } BeamformWork; 56 57 typedef struct { 58 union { 59 u64 queue; 60 struct {u32 widx, ridx;}; 61 }; 62 BeamformWork work_items[1 << 6]; 63 } BeamformWorkQueue; 64 65 #define BEAMFORMER_PARAMETER_BLOCK_REGION_LIST \ 66 X(ComputePipeline, pipeline) \ 67 X(ChannelMapping, channel_mapping) \ 68 X(FocalVectors, focal_vectors) \ 69 X(Parameters, parameters) \ 70 X(SparseElements, sparse_elements) \ 71 X(TransmitReceiveOrientations, transmit_receive_orientations) \ 72 73 #define BEAMFORMER_PARAMETER_BLOCK_REGION_FLAG_LIST \ 74 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST \ 75 X(NotifyUI) \ 76 77 typedef enum { 78 #define X(k, ...) BeamformerParameterBlockRegion_##k, 79 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 80 #undef X 81 BeamformerParameterBlockRegion_Count 82 } BeamformerParameterBlockRegions; 83 84 typedef enum { 85 #define X(k, ...) BeamformerParameterRegionFlag_##k, 86 BEAMFORMER_PARAMETER_BLOCK_REGION_FLAG_LIST 87 #undef X 88 BeamformerParameterRegionFlag_Count, 89 } BeamformerParameterRegionFlags; 90 91 typedef union { 92 u8 filter_slot; 93 } BeamformerShaderParameters; 94 95 typedef struct { 96 BeamformerShaderKind shaders[BeamformerMaxComputeShaderStages]; 97 BeamformerShaderParameters parameters[BeamformerMaxComputeShaderStages]; 98 u32 shader_count; 99 BeamformerDataKind data_kind; 100 } BeamformerComputePipeline; 101 102 typedef struct { 103 alignas(16) union { 104 BeamformerParameters parameters; 105 struct { 106 BeamformerParametersHead parameters_head; 107 BeamformerUIParameters parameters_ui; 108 BeamformerExtraParameters parameters_extra; 109 }; 110 }; 111 112 /* NOTE(rnp): signals to the beamformer that a subregion of a block has been updated */ 113 u32 region_update_flags; 114 static_assert(BeamformerParameterRegionFlag_Count <= 32, ""); 115 116 BeamformerComputePipeline pipeline; 117 118 alignas(16) i16 channel_mapping[BeamformerMaxChannelCount]; 119 alignas(16) i16 sparse_elements[BeamformerMaxChannelCount]; 120 alignas(16) u8 transmit_receive_orientations[BeamformerMaxChannelCount]; 121 /* NOTE(rnp): interleaved transmit angle, focal depth pairs */ 122 alignas(16) v2 focal_vectors[BeamformerMaxChannelCount]; 123 } BeamformerParameterBlock; 124 static_assert(sizeof(BeamformerParameterBlock) % alignof(BeamformerParameterBlock) == 0, 125 "sizeof(BeamformerParametersBlock) must be a multiple of its alignment"); 126 127 #define X(k, field) [BeamformerParameterBlockRegion_##k] = offsetof(BeamformerParameterBlock, field), 128 read_only global u16 BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_Count] = { 129 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 130 }; 131 #undef X 132 133 typedef struct { 134 u32 version; 135 136 /* NOTE(rnp): causes future library calls to fail. 137 * see note in beamformer_invalidate_shared_memory() */ 138 b32 invalid; 139 140 /* NOTE(rnp): not used for locking on w32 but we can use these to peek at the status of 141 * the lock without leaving userspace. */ 142 i32 locks[(u32)BeamformerSharedMemoryLockKind_Count + (u32)BeamformerMaxParameterBlocks]; 143 144 /* NOTE(rnp): total number of parameter block regions the client has requested. 145 * used to calculate offset to scratch space and to track number of allocated 146 * semaphores on w32. Defaults to 1 but can be changed at runtime */ 147 u32 reserved_parameter_blocks; 148 149 /* TODO(rnp): this is really sucky. we need a better way to communicate this */ 150 u64 rf_block_rf_size; 151 152 // NOTE(rnp): currently this cannot be directly user readable. its interpretation 153 // requires beamformer implementation details 154 u64 beamformed_frame_buffer_size; 155 156 struct { 157 u64 max_rf_data_size; 158 b8 cuda; 159 b8 hilbert; 160 } capabilities; 161 162 BeamformerLiveImagingParameters live_imaging_parameters; 163 BeamformerLiveImagingDirtyFlags live_imaging_dirty_flags; 164 165 BeamformWorkQueue external_work_queue; 166 } BeamformerSharedMemory; 167 168 function BeamformWork * 169 beamform_work_queue_pop(BeamformWorkQueue *q) 170 { 171 BeamformWork *result = 0; 172 173 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 174 u64 val = atomic_load_u64(&q->queue); 175 u64 mask = countof(q->work_items) - 1; 176 u64 widx = val & mask; 177 u64 ridx = val >> 32 & mask; 178 179 if (ridx != widx) 180 result = q->work_items + ridx; 181 182 return result; 183 } 184 185 function void 186 beamform_work_queue_pop_commit(BeamformWorkQueue *q) 187 { 188 atomic_add_u64(&q->queue, 0x100000000ULL); 189 } 190 191 function BeamformWork * 192 beamform_work_queue_push(BeamformWorkQueue *q) 193 { 194 BeamformWork *result = 0; 195 196 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 197 u64 val = atomic_load_u64(&q->queue); 198 u64 mask = countof(q->work_items) - 1; 199 u64 widx = val & mask; 200 u64 ridx = val >> 32 & mask; 201 u64 next = (widx + 1) & mask; 202 203 if (val & 0x80000000) 204 atomic_and_u64(&q->queue, ~0x80000000); 205 206 if (next != ridx) { 207 result = q->work_items + widx; 208 zero_struct(result); 209 } 210 211 return result; 212 } 213 214 function void 215 beamform_work_queue_push_commit(BeamformWorkQueue *q) 216 { 217 atomic_add_u64(&q->queue, 1); 218 } 219 220 #if OS_WINDOWS 221 // NOTE(rnp): junk needed on w32 to watch a value across processes while yielding 222 // control back to the kernel. There are user level CPU instructions that allow 223 // this so why w32 can't do it in kernel mode sounds like shitty design to me. 224 DEBUG_IMPORT OSW32Semaphore os_w32_shared_memory_semaphores[countof(((BeamformerSharedMemory *)0)->locks)]; 225 #endif 226 227 function b32 228 beamformer_shared_memory_take_lock(BeamformerSharedMemory *sm, i32 lock, u32 timeout_ms) 229 { 230 #if OS_WINDOWS 231 b32 result = os_w32_semaphore_wait(os_w32_shared_memory_semaphores[lock], timeout_ms); 232 if (result) atomic_store_u32(sm->locks + lock, 1); 233 #else 234 b32 result = take_lock(sm->locks + lock, timeout_ms); 235 #endif 236 return result; 237 } 238 239 function void 240 beamformer_shared_memory_release_lock(BeamformerSharedMemory *sm, i32 lock) 241 { 242 release_lock(sm->locks + lock); 243 #if OS_WINDOWS 244 os_w32_semaphore_release(os_w32_shared_memory_semaphores[lock], 1); 245 #endif 246 } 247 248 function BeamformerParameterBlock * 249 beamformer_parameter_block(BeamformerSharedMemory *sm, u32 block) 250 { 251 assert(sm->reserved_parameter_blocks > block); 252 BeamformerParameterBlock *result = (typeof(result))((u8 *)(sm + 1) + block * sizeof(*result)); 253 return result; 254 } 255 256 function b32 257 beamformer_parameter_block_dirty(BeamformerSharedMemory *sm, u32 block) 258 { 259 b32 result = beamformer_parameter_block(sm, block)->region_update_flags != 0; 260 return result; 261 } 262 263 function BeamformerParameterBlock * 264 beamformer_parameter_block_lock(BeamformerSharedMemory *sm, u32 block, i32 timeout_ms) 265 { 266 assert(block < BeamformerMaxParameterBlocks); 267 BeamformerParameterBlock *result = 0; 268 if (beamformer_shared_memory_take_lock(sm, BeamformerSharedMemoryLockKind_Count + block, (u32)timeout_ms)) 269 result = beamformer_parameter_block(sm, block); 270 return result; 271 } 272 273 function void 274 beamformer_parameter_block_unlock(BeamformerSharedMemory *sm, u32 block) 275 { 276 assert(block < BeamformerMaxParameterBlocks); 277 beamformer_shared_memory_release_lock(sm, BeamformerSharedMemoryLockKind_Count + block); 278 } 279 280 function Arena 281 beamformer_shared_memory_scratch_arena(BeamformerSharedMemory *sm, i64 shared_memory_size) 282 { 283 assert(sm->reserved_parameter_blocks > 0); 284 BeamformerParameterBlock *last = beamformer_parameter_block(sm, sm->reserved_parameter_blocks - 1); 285 Arena result = {.beg = (u8 *)(last + 1), .end = (u8 *)sm + shared_memory_size}; 286 result.beg = arena_aligned_start(result, KB(4)); 287 return result; 288 } 289 290 function void 291 mark_parameter_block_region_dirty(BeamformerSharedMemory *sm, u32 block, BeamformerParameterBlockRegions region) 292 { 293 BeamformerParameterBlock *pb = beamformer_parameter_block(sm, block); 294 atomic_or_u32(&pb->region_update_flags, 1u << region); 295 } 296 297 function void 298 post_sync_barrier(BeamformerSharedMemory *sm, BeamformerSharedMemoryLockKind lock) 299 { 300 /* NOTE(rnp): debug: here it is not a bug to release the lock if it 301 * isn't held but elswhere it is */ 302 DEBUG_DECL(if (sm->locks[lock])) { 303 beamformer_shared_memory_release_lock(sm, lock); 304 } 305 }