beamformer_shared_memory.c (9763B)
1 /* See LICENSE for license details. */ 2 #define BEAMFORMER_SHARED_MEMORY_VERSION (37UL) 3 4 typedef enum { 5 BeamformerWorkKind_Compute, 6 BeamformerWorkKind_WaitThenCompute, 7 BeamformerWorkKind_CreateFilter, 8 BeamformerWorkKind_ExportBuffer, 9 } BeamformerWorkKind; 10 11 typedef struct { 12 BeamformerFilterParameters parameters; 13 u8 filter_slot; 14 u8 parameter_block; 15 static_assert(BeamformerFilterSlots <= 255, "CreateFilterContext only supports 255 filter slots"); 16 static_assert(BeamformerMaxParameterBlocks <= 255, "CreateFilterContext only supports 255 parameter blocks"); 17 } BeamformerCreateFilterContext; 18 19 typedef enum { 20 BeamformerExportKind_BeamformedData, 21 BeamformerExportKind_Stats, 22 } BeamformerExportKind; 23 24 typedef struct { 25 BeamformerExportKind kind; 26 u32 count; /* Number of items to export */ 27 u64 size; /* Total expected size of the exported data */ 28 } BeamformerExportContext; 29 30 #define BEAMFORMER_SHARED_MEMORY_LOCKS \ 31 X(ScratchSpace) \ 32 X(UploadRF) \ 33 X(ExportSync) \ 34 X(DispatchCompute) 35 36 #define X(name) BeamformerSharedMemoryLockKind_##name, 37 typedef enum {BEAMFORMER_SHARED_MEMORY_LOCKS BeamformerSharedMemoryLockKind_Count} BeamformerSharedMemoryLockKind; 38 #undef X 39 40 typedef struct { 41 m4 voxel_transform; 42 u64 rf_offset; 43 f32 focus_depth; 44 f32 steering_angle; 45 u16 tx_rx_orientation; 46 u16 parameter_set; 47 BeamformerViewPlaneTag view_plane; 48 49 static_assert(BeamformerMaxParameterBlocks <= U16_MAX, ""); 50 } BeamformerComputeWorkContext; 51 52 /* NOTE: discriminated union based on type */ 53 typedef struct { 54 BeamformerWorkKind kind; 55 BeamformerSharedMemoryLockKind lock; 56 union { 57 BeamformerComputeWorkContext compute_context; 58 BeamformerCreateFilterContext create_filter_context; 59 BeamformerExportContext export_context; 60 BeamformerShaderKind reload_shader; 61 }; 62 } BeamformWork; 63 64 typedef struct { 65 union { 66 u64 queue; 67 struct {u32 widx, ridx;}; 68 }; 69 BeamformWork work_items[1 << 6]; 70 } BeamformWorkQueue; 71 72 #define BEAMFORMER_PARAMETER_BLOCK_REGION_LIST \ 73 X(ComputePipeline, pipeline) \ 74 X(ChannelMapping, channel_mapping) \ 75 X(FocalVectors, focal_vectors) \ 76 X(Parameters, parameters) \ 77 X(SparseElements, sparse_elements) \ 78 X(TransmitReceiveOrientations, transmit_receive_orientations) \ 79 80 typedef enum { 81 #define X(k, ...) BeamformerParameterBlockRegion_##k, 82 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 83 #undef X 84 BeamformerParameterBlockRegion_Count 85 } BeamformerParameterBlockRegions; 86 87 typedef enum { 88 BeamformerParameterDirtyFlag_Parameters, 89 BeamformerParameterDirtyFlag_NotifyUI, 90 BeamformerParameterDirtyFlag_Count, 91 } BeamformerParameterDirtyFlags; 92 93 typedef union { 94 u8 filter_slot; 95 } BeamformerShaderParameters; 96 97 typedef struct { 98 BeamformerShaderKind shaders[BeamformerMaxComputeShaderStages]; 99 BeamformerShaderParameters parameters[BeamformerMaxComputeShaderStages]; 100 u32 shader_count; 101 BeamformerDataKind data_kind; 102 } BeamformerComputePipeline; 103 104 typedef struct { 105 alignas(16) BeamformerParameters parameters; 106 107 /* NOTE(rnp): signals to the beamformer that a subregion of a block has been updated */ 108 u32 region_update_flags; 109 static_assert(BeamformerParameterDirtyFlag_Count <= 32, ""); 110 111 BeamformerComputePipeline pipeline; 112 113 alignas(16) i16 channel_mapping[BeamformerMaxChannelCount]; 114 alignas(16) i16 sparse_elements[BeamformerMaxChannelCount]; 115 alignas(16) u8 transmit_receive_orientations[BeamformerMaxChannelCount]; 116 /* NOTE(rnp): interleaved transmit angle, focal depth pairs */ 117 alignas(16) v2 focal_vectors[BeamformerMaxChannelCount]; 118 } BeamformerParameterBlock; 119 static_assert(sizeof(BeamformerParameterBlock) % alignof(BeamformerParameterBlock) == 0, 120 "sizeof(BeamformerParametersBlock) must be a multiple of its alignment"); 121 122 #define X(k, field) [BeamformerParameterBlockRegion_##k] = offsetof(BeamformerParameterBlock, field), 123 read_only u16 BeamformerParameterBlockRegionOffsets[BeamformerParameterBlockRegion_Count] = { 124 BEAMFORMER_PARAMETER_BLOCK_REGION_LIST 125 }; 126 #undef X 127 128 typedef struct { 129 u32 version; 130 131 /* NOTE(rnp): causes future library calls to fail. 132 * see note in beamformer_invalidate_shared_memory() */ 133 b32 invalid; 134 135 /* NOTE(rnp): not used for locking on w32 but we can use these to peek at the status of 136 * the lock without leaving userspace. */ 137 i32 locks[(u32)BeamformerSharedMemoryLockKind_Count + (u32)BeamformerMaxParameterBlocks]; 138 139 /* NOTE(rnp): total number of parameter block regions the client has requested. 140 * used to calculate offset to scratch space and to track number of allocated 141 * semaphores on w32. Defaults to 1 but can be changed at runtime */ 142 u32 reserved_parameter_blocks; 143 144 /* TODO(rnp): this is really sucky. we need a better way to communicate this */ 145 u64 rf_block_rf_size; 146 147 b64 new_rf_upload; 148 149 // NOTE(rnp): currently this cannot be directly user readable. its interpretation 150 // requires beamformer implementation details 151 u64 beamformed_frame_buffer_size; 152 153 struct { 154 u64 max_rf_data_size; 155 b8 cuda; 156 b8 hilbert; 157 } capabilities; 158 159 BeamformerLiveImagingParameters live_imaging_parameters; 160 BeamformerLiveImagingDirtyFlags live_imaging_dirty_flags; 161 162 BeamformWorkQueue external_work_queue; 163 } BeamformerSharedMemory; 164 165 function BeamformWork * 166 beamform_work_queue_pop(BeamformWorkQueue *q) 167 { 168 BeamformWork *result = 0; 169 170 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 171 u64 val = atomic_load_u64(&q->queue); 172 u64 mask = countof(q->work_items) - 1; 173 u64 widx = val & mask; 174 u64 ridx = val >> 32 & mask; 175 176 if (ridx != widx) 177 result = q->work_items + ridx; 178 179 return result; 180 } 181 182 function void 183 beamform_work_queue_pop_commit(BeamformWorkQueue *q) 184 { 185 atomic_add_u64(&q->queue, 0x100000000ULL); 186 } 187 188 function BeamformWork * 189 beamform_work_queue_push(BeamformWorkQueue *q) 190 { 191 BeamformWork *result = 0; 192 193 static_assert(IsPowerOfTwo(countof(q->work_items)), "queue capacity must be a power of 2"); 194 u64 val = atomic_load_u64(&q->queue); 195 u64 mask = countof(q->work_items) - 1; 196 u64 widx = val & mask; 197 u64 ridx = val >> 32 & mask; 198 u64 next = (widx + 1) & mask; 199 200 if (val & 0x80000000) 201 atomic_and_u64(&q->queue, ~0x80000000); 202 203 if (next != ridx) { 204 result = q->work_items + widx; 205 zero_struct(result); 206 } 207 208 return result; 209 } 210 211 function void 212 beamform_work_queue_push_commit(BeamformWorkQueue *q) 213 { 214 atomic_add_u64(&q->queue, 1); 215 } 216 217 #if OS_WINDOWS 218 // NOTE(rnp): junk needed on w32 to watch a value across processes while yielding 219 // control back to the kernel. There are user level CPU instructions that allow 220 // this so why w32 can't do it in kernel mode sounds like shitty design to me. 221 DEBUG_IMPORT OSW32Semaphore os_w32_shared_memory_semaphores[countof(((BeamformerSharedMemory *)0)->locks)]; 222 #endif 223 224 function b32 225 beamformer_shared_memory_take_lock(BeamformerSharedMemory *sm, i32 lock, u32 timeout_ms) 226 { 227 #if OS_WINDOWS 228 b32 result = os_w32_semaphore_wait(os_w32_shared_memory_semaphores[lock], timeout_ms); 229 if (result) atomic_store_u32(sm->locks + lock, 1); 230 #else 231 b32 result = take_lock(sm->locks + lock, timeout_ms); 232 #endif 233 return result; 234 } 235 236 function void 237 beamformer_shared_memory_release_lock(BeamformerSharedMemory *sm, i32 lock) 238 { 239 release_lock(sm->locks + lock); 240 #if OS_WINDOWS 241 os_w32_semaphore_release(os_w32_shared_memory_semaphores[lock], 1); 242 #endif 243 } 244 245 function BeamformerParameterBlock * 246 beamformer_parameter_block(BeamformerSharedMemory *sm, u32 block) 247 { 248 assert(sm->reserved_parameter_blocks > block); 249 BeamformerParameterBlock *result = (typeof(result))((u8 *)(sm + 1) + block * sizeof(*result)); 250 return result; 251 } 252 253 function b32 254 beamformer_parameter_block_dirty(BeamformerSharedMemory *sm, u32 block) 255 { 256 b32 result = beamformer_parameter_block(sm, block)->region_update_flags != 0; 257 return result; 258 } 259 260 function BeamformerParameterBlock * 261 beamformer_parameter_block_lock(BeamformerSharedMemory *sm, u32 block, i32 timeout_ms) 262 { 263 assert(block < BeamformerMaxParameterBlocks); 264 BeamformerParameterBlock *result = 0; 265 if (beamformer_shared_memory_take_lock(sm, BeamformerSharedMemoryLockKind_Count + block, (u32)timeout_ms)) 266 result = beamformer_parameter_block(sm, block); 267 return result; 268 } 269 270 function void 271 beamformer_parameter_block_unlock(BeamformerSharedMemory *sm, u32 block) 272 { 273 assert(block < BeamformerMaxParameterBlocks); 274 beamformer_shared_memory_release_lock(sm, BeamformerSharedMemoryLockKind_Count + block); 275 } 276 277 function Arena * 278 beamformer_shared_memory_scratch_arena(BeamformerSharedMemory *sm, i64 shared_memory_size) 279 { 280 assert(sm->reserved_parameter_blocks > 0); 281 BeamformerParameterBlock *last = beamformer_parameter_block(sm, sm->reserved_parameter_blocks - 1); 282 u64 size = (u64)((u8 *)sm + shared_memory_size - (u8 *)(last + 1)); 283 Arena *result = arena_create(.optional_backing_store = (last + 1), .reserve_size = size, .commit_size = size, .flags = ArenaFlag_NoChain); 284 arena_pre_align(result, KB(4)); 285 return result; 286 } 287 288 function void * 289 beamformer_shared_memory_data_pointer(BeamformerSharedMemory *sm, i64 shared_memory_size) 290 { 291 Arena *arena = beamformer_shared_memory_scratch_arena(sm, shared_memory_size); 292 void *result = (u8 *)arena + arena->position; 293 return result; 294 } 295 296 function void 297 beamformed_shared_memory_dirty_flag(BeamformerSharedMemory *sm, u32 block, BeamformerParameterDirtyFlags flag) 298 { 299 BeamformerParameterBlock *pb = beamformer_parameter_block(sm, block); 300 atomic_or_u32(&pb->region_update_flags, 1u << flag); 301 } 302 303 function void 304 post_sync_barrier(BeamformerSharedMemory *sm, BeamformerSharedMemoryLockKind lock) 305 { 306 /* NOTE(rnp): debug: here it is not a bug to release the lock if it 307 * isn't held but elswhere it is */ 308 DEBUG_DECL(if (sm->locks[lock])) { 309 beamformer_shared_memory_release_lock(sm, lock); 310 } 311 }