beamformer_core.c (63153B)
1 /* See LICENSE for license details. */ 2 /* TODO(rnp): 3 * [ ]: refactor: DecodeMode_None should use a different mapping and optional conversion shader 4 * for rf only mode with no filter and demod/filter should gain the OutputFloats flag for iq 5 * case and rf mode with filter; this can also be used instead of first pass uniform 6 * [ ]: refactor: replace UploadRF with just the scratch_rf_size variable, 7 * use below to spin wait in library 8 * [ ]: utilize umonitor/umwait (intel), monitorx/mwaitx (amd), and wfe/sev (aarch64) 9 * for power efficient low latency waiting 10 * [ ]: refactor: split decode into reshape and decode 11 * - the check for first pass reshaping is the last non constant check 12 * in the shader 13 * - this will also remove the need for the channel mapping in the decode shader 14 * [X]: refactor: ui: reload only shader which is affected by the interaction 15 * [ ]: BeamformWorkQueue -> BeamformerWorkQueue 16 * [ ]: need to keep track of gpu memory in some way 17 * - want to be able to store more than 16 2D frames but limit 3D frames 18 * - maybe keep track of how much gpu memory is committed for beamformed images 19 * and use that to determine when to loop back over existing textures 20 * - to do this maybe use a circular linked list instead of a flat array 21 * - then have a way of querying how many frames are available for a specific point count 22 * [ ]: bug: reinit cuda on hot-reload 23 */ 24 25 #include "compiler.h" 26 27 #if defined(BEAMFORMER_DEBUG) && !defined(BEAMFORMER_EXPORT) && OS_WINDOWS 28 #define BEAMFORMER_EXPORT __declspec(dllexport) 29 #endif 30 31 #include "beamformer_internal.h" 32 33 global f32 dt_for_frame; 34 35 #define DECODE_FIRST_PASS_UNIFORM_LOC 1 36 37 #define DAS_LOCAL_SIZE_X 16 38 #define DAS_LOCAL_SIZE_Y 1 39 #define DAS_LOCAL_SIZE_Z 16 40 41 #define DAS_VOXEL_OFFSET_UNIFORM_LOC 2 42 #define DAS_CYCLE_T_UNIFORM_LOC 3 43 #define DAS_FAST_CHANNEL_UNIFORM_LOC 4 44 45 #define MIN_MAX_MIPS_LEVEL_UNIFORM_LOC 1 46 #define SUM_PRESCALE_UNIFORM_LOC 1 47 48 #if !BEAMFORMER_RENDERDOC_HOOKS 49 #define start_renderdoc_capture(...) 50 #define end_renderdoc_capture(...) 51 #else 52 global renderdoc_start_frame_capture_fn *start_frame_capture; 53 global renderdoc_end_frame_capture_fn *end_frame_capture; 54 #define start_renderdoc_capture(gl) if (start_frame_capture) start_frame_capture(gl, 0) 55 #define end_renderdoc_capture(gl) if (end_frame_capture) end_frame_capture(gl, 0) 56 #endif 57 58 typedef struct { 59 BeamformerFrame *frames; 60 u32 capacity; 61 u32 offset; 62 u32 cursor; 63 u32 needed_frames; 64 } ComputeFrameIterator; 65 66 function void 67 beamformer_compute_plan_release(BeamformerComputeContext *cc, u32 block) 68 { 69 assert(block < countof(cc->compute_plans)); 70 BeamformerComputePlan *cp = cc->compute_plans[block]; 71 if (cp) { 72 glDeleteBuffers(countof(cp->ubos), cp->ubos); 73 glDeleteTextures(countof(cp->textures), cp->textures); 74 for (u32 i = 0; i < countof(cp->filters); i++) 75 glDeleteBuffers(1, &cp->filters[i].ssbo); 76 cc->compute_plans[block] = 0; 77 SLLPushFreelist(cp, cc->compute_plan_freelist); 78 } 79 } 80 81 function BeamformerComputePlan * 82 beamformer_compute_plan_for_block(BeamformerComputeContext *cc, u32 block, Arena *arena) 83 { 84 assert(block < countof(cc->compute_plans)); 85 BeamformerComputePlan *result = cc->compute_plans[block]; 86 87 assert(result || arena); 88 89 if (!result) { 90 result = SLLPopFreelist(cc->compute_plan_freelist); 91 if (!result) result = push_struct_no_zero(arena, BeamformerComputePlan); 92 zero_struct(result); 93 cc->compute_plans[block] = result; 94 95 glCreateBuffers(countof(result->ubos), result->ubos); 96 97 Stream label = arena_stream(*arena); 98 #define X(k, t, ...) \ 99 glNamedBufferStorage(result->ubos[BeamformerComputeUBOKind_##k], sizeof(t), \ 100 0, GL_DYNAMIC_STORAGE_BIT); \ 101 stream_append_s8(&label, s8(#t "[")); \ 102 stream_append_u64(&label, block); \ 103 stream_append_byte(&label, ']'); \ 104 glObjectLabel(GL_BUFFER, result->ubos[BeamformerComputeUBOKind_##k], \ 105 label.widx, (c8 *)label.data); \ 106 label.widx = 0; 107 BEAMFORMER_COMPUTE_UBO_LIST 108 #undef X 109 110 #define X(_k, t, ...) t, 111 GLenum gl_kind[] = {BEAMFORMER_COMPUTE_TEXTURE_LIST_FULL}; 112 #undef X 113 read_only local_persist s8 tex_prefix[] = { 114 #define X(k, ...) s8_comp(#k "["), 115 BEAMFORMER_COMPUTE_TEXTURE_LIST_FULL 116 #undef X 117 }; 118 glCreateTextures(GL_TEXTURE_1D, BeamformerComputeTextureKind_Count - 1, result->textures); 119 for (u32 i = 0; i < BeamformerComputeTextureKind_Count - 1; i++) { 120 /* TODO(rnp): this could be predicated on channel count for this compute plan */ 121 glTextureStorage1D(result->textures[i], 1, gl_kind[i], BeamformerMaxChannelCount); 122 stream_append_s8(&label, tex_prefix[i]); 123 stream_append_u64(&label, block); 124 stream_append_byte(&label, ']'); 125 glObjectLabel(GL_TEXTURE, result->textures[i], label.widx, (c8 *)label.data); 126 label.widx = 0; 127 } 128 } 129 return result; 130 } 131 132 function void 133 beamformer_filter_update(BeamformerFilter *f, BeamformerFilterParameters fp, u32 block, u32 slot, Arena arena) 134 { 135 Stream sb = arena_stream(arena); 136 stream_append_s8s(&sb, 137 beamformer_filter_kind_strings[fp.kind % countof(beamformer_filter_kind_strings)], 138 s8("Filter[")); 139 stream_append_u64(&sb, block); 140 stream_append_s8(&sb, s8("][")); 141 stream_append_u64(&sb, slot); 142 stream_append_byte(&sb, ']'); 143 s8 label = arena_stream_commit(&arena, &sb); 144 145 void *filter = 0; 146 switch (fp.kind) { 147 case BeamformerFilterKind_Kaiser:{ 148 /* TODO(rnp): this should also support complex */ 149 /* TODO(rnp): implement this as an IFIR filter instead to reduce computation */ 150 filter = kaiser_low_pass_filter(&arena, fp.kaiser.cutoff_frequency, fp.sampling_frequency, 151 fp.kaiser.beta, (i32)fp.kaiser.length); 152 f->length = (i32)fp.kaiser.length; 153 f->time_delay = (f32)f->length / 2.0f / fp.sampling_frequency; 154 }break; 155 case BeamformerFilterKind_MatchedChirp:{ 156 typeof(fp.matched_chirp) *mc = &fp.matched_chirp; 157 f32 fs = fp.sampling_frequency; 158 f->length = (i32)(mc->duration * fs); 159 if (fp.complex) { 160 filter = baseband_chirp(&arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1, 0.5f); 161 f->time_delay = complex_filter_first_moment(filter, f->length, fs); 162 } else { 163 filter = rf_chirp(&arena, mc->min_frequency, mc->max_frequency, fs, f->length, 1); 164 f->time_delay = real_filter_first_moment(filter, f->length, fs); 165 } 166 }break; 167 InvalidDefaultCase; 168 } 169 170 f->parameters = fp; 171 172 glDeleteBuffers(1, &f->ssbo); 173 glCreateBuffers(1, &f->ssbo); 174 glNamedBufferStorage(f->ssbo, f->length * (i32)sizeof(f32) * (fp.complex? 2 : 1), filter, 0); 175 glObjectLabel(GL_BUFFER, f->ssbo, (i32)label.len, (c8 *)label.data); 176 } 177 178 function ComputeFrameIterator 179 compute_frame_iterator(BeamformerCtx *ctx, u32 start_index, u32 needed_frames) 180 { 181 start_index = start_index % ARRAY_COUNT(ctx->beamform_frames); 182 183 ComputeFrameIterator result; 184 result.frames = ctx->beamform_frames; 185 result.offset = start_index; 186 result.capacity = ARRAY_COUNT(ctx->beamform_frames); 187 result.cursor = 0; 188 result.needed_frames = needed_frames; 189 return result; 190 } 191 192 function BeamformerFrame * 193 frame_next(ComputeFrameIterator *bfi) 194 { 195 BeamformerFrame *result = 0; 196 if (bfi->cursor != bfi->needed_frames) { 197 u32 index = (bfi->offset + bfi->cursor++) % bfi->capacity; 198 result = bfi->frames + index; 199 } 200 return result; 201 } 202 203 function b32 204 beamformer_frame_compatible(BeamformerFrame *f, iv3 dim, GLenum gl_kind) 205 { 206 b32 result = gl_kind == f->gl_kind && iv3_equal(dim, f->dim); 207 return result; 208 } 209 210 function iv3 211 make_valid_output_points(i32 points[3]) 212 { 213 iv3 result; 214 result.E[0] = CLAMP(points[0], 1, gl_parameters.max_3d_texture_dim); 215 result.E[1] = CLAMP(points[1], 1, gl_parameters.max_3d_texture_dim); 216 result.E[2] = CLAMP(points[2], 1, gl_parameters.max_3d_texture_dim); 217 return result; 218 } 219 220 function void 221 alloc_beamform_frame(BeamformerFrame *out, iv3 out_dim, GLenum gl_kind, s8 name, Arena arena) 222 { 223 out->dim = make_valid_output_points(out_dim.E); 224 225 /* NOTE: allocate storage for beamformed output data; 226 * this is shared between compute and fragment shaders */ 227 u32 max_dim = (u32)MAX(out->dim.x, MAX(out->dim.y, out->dim.z)); 228 out->mips = (i32)ctz_u32(round_up_power_of_2(max_dim)) + 1; 229 230 out->gl_kind = gl_kind; 231 232 Stream label = arena_stream(arena); 233 stream_append_s8(&label, name); 234 stream_append_byte(&label, '['); 235 stream_append_hex_u64(&label, out->id); 236 stream_append_byte(&label, ']'); 237 238 glDeleteTextures(1, &out->texture); 239 glCreateTextures(GL_TEXTURE_3D, 1, &out->texture); 240 glTextureStorage3D(out->texture, out->mips, gl_kind, out->dim.x, out->dim.y, out->dim.z); 241 242 glTextureParameteri(out->texture, GL_TEXTURE_MIN_FILTER, GL_NEAREST); 243 glTextureParameteri(out->texture, GL_TEXTURE_MAG_FILTER, GL_NEAREST); 244 245 LABEL_GL_OBJECT(GL_TEXTURE, out->texture, stream_to_s8(&label)); 246 } 247 248 function void 249 update_hadamard_texture(BeamformerComputePlan *cp, i32 order, Arena arena) 250 { 251 f32 *hadamard = make_hadamard_transpose(&arena, order); 252 if (hadamard) { 253 cp->hadamard_order = order; 254 u32 *texture = cp->textures + BeamformerComputeTextureKind_Hadamard; 255 glDeleteTextures(1, texture); 256 glCreateTextures(GL_TEXTURE_2D, 1, texture); 257 glTextureStorage2D(*texture, 1, GL_R32F, order, order); 258 glTextureSubImage2D(*texture, 0, 0, 0, order, order, GL_RED, GL_FLOAT, hadamard); 259 260 Stream label = arena_stream(arena); 261 stream_append_s8(&label, s8("Hadamard")); 262 stream_append_i64(&label, order); 263 LABEL_GL_OBJECT(GL_TEXTURE, *texture, stream_to_s8(&label)); 264 } 265 } 266 267 function void 268 alloc_shader_storage(BeamformerCtx *ctx, u32 decoded_data_size, Arena arena) 269 { 270 BeamformerComputeContext *cc = &ctx->compute_context; 271 glDeleteBuffers(countof(cc->ping_pong_ssbos), cc->ping_pong_ssbos); 272 glCreateBuffers(countof(cc->ping_pong_ssbos), cc->ping_pong_ssbos); 273 274 cc->ping_pong_ssbo_size = decoded_data_size; 275 276 Stream label = arena_stream(arena); 277 stream_append_s8(&label, s8("PingPongSSBO[")); 278 i32 s_widx = label.widx; 279 for (i32 i = 0; i < countof(cc->ping_pong_ssbos); i++) { 280 glNamedBufferStorage(cc->ping_pong_ssbos[i], (iz)decoded_data_size, 0, 0); 281 stream_append_i64(&label, i); 282 stream_append_byte(&label, ']'); 283 LABEL_GL_OBJECT(GL_BUFFER, cc->ping_pong_ssbos[i], stream_to_s8(&label)); 284 stream_reset(&label, s_widx); 285 } 286 287 /* TODO(rnp): (25.08.04) cuda lib is heavily broken atm. First there are multiple RF 288 * buffers and cuda decode shouldn't assume that the data is coming from the rf_buffer 289 * ssbo. Second each parameter block may need a different hadamard matrix so ideally 290 * decode should just take the texture as a parameter. Third, none of these dimensions 291 * need to be pre-known by the library unless its allocating GPU memory which it shouldn't 292 * need to do. For now grab out of parameter block 0 but it is not correct */ 293 BeamformerParameterBlock *pb = beamformer_parameter_block(ctx->shared_memory, 0); 294 /* NOTE(rnp): these are stubs when CUDA isn't supported */ 295 cuda_register_buffers(cc->ping_pong_ssbos, countof(cc->ping_pong_ssbos), cc->rf_buffer.ssbo); 296 u32 decoded_data_dimension[3] = {pb->parameters.sample_count, pb->parameters.channel_count, pb->parameters.acquisition_count}; 297 cuda_init(pb->parameters.raw_data_dimensions.E, decoded_data_dimension); 298 } 299 300 function void 301 push_compute_timing_info(ComputeTimingTable *t, ComputeTimingInfo info) 302 { 303 u32 index = atomic_add_u32(&t->write_index, 1) % countof(t->buffer); 304 t->buffer[index] = info; 305 } 306 307 function b32 308 fill_frame_compute_work(BeamformerCtx *ctx, BeamformWork *work, BeamformerViewPlaneTag plane, 309 u32 parameter_block, b32 indirect) 310 { 311 b32 result = work != 0; 312 if (result) { 313 u32 frame_id = atomic_add_u32(&ctx->next_render_frame_index, 1); 314 u32 frame_index = frame_id % countof(ctx->beamform_frames); 315 work->kind = indirect? BeamformerWorkKind_ComputeIndirect : BeamformerWorkKind_Compute; 316 work->lock = BeamformerSharedMemoryLockKind_DispatchCompute; 317 work->compute_context.parameter_block = parameter_block; 318 work->compute_context.frame = ctx->beamform_frames + frame_index; 319 work->compute_context.frame->ready_to_present = 0; 320 work->compute_context.frame->view_plane_tag = plane; 321 work->compute_context.frame->id = frame_id; 322 } 323 return result; 324 } 325 326 function void 327 do_sum_shader(BeamformerComputeContext *cc, u32 *in_textures, u32 in_texture_count, 328 u32 out_texture, iv3 out_data_dim) 329 { 330 /* NOTE: zero output before summing */ 331 glClearTexImage(out_texture, 0, GL_RED, GL_FLOAT, 0); 332 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT); 333 334 glBindImageTexture(0, out_texture, 0, GL_TRUE, 0, GL_READ_WRITE, GL_RG32F); 335 for (u32 i = 0; i < in_texture_count; i++) { 336 glBindImageTexture(1, in_textures[i], 0, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 337 glDispatchCompute(ORONE((u32)out_data_dim.x / 32u), 338 ORONE((u32)out_data_dim.y), 339 ORONE((u32)out_data_dim.z / 32u)); 340 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 341 } 342 } 343 344 struct compute_cursor { 345 iv3 cursor; 346 uv3 dispatch; 347 iv3 target; 348 u32 points_per_dispatch; 349 u32 completed_points; 350 u32 total_points; 351 }; 352 353 function struct compute_cursor 354 start_compute_cursor(iv3 dim, u32 max_points) 355 { 356 struct compute_cursor result = {0}; 357 u32 invocations_per_dispatch = DAS_LOCAL_SIZE_X * DAS_LOCAL_SIZE_Y * DAS_LOCAL_SIZE_Z; 358 359 result.dispatch.y = MIN(max_points / invocations_per_dispatch, (u32)ceil_f32((f32)dim.y / DAS_LOCAL_SIZE_Y)); 360 361 u32 remaining = max_points / result.dispatch.y; 362 result.dispatch.x = MIN(remaining / invocations_per_dispatch, (u32)ceil_f32((f32)dim.x / DAS_LOCAL_SIZE_X)); 363 result.dispatch.z = MIN(remaining / (invocations_per_dispatch * result.dispatch.x), 364 (u32)ceil_f32((f32)dim.z / DAS_LOCAL_SIZE_Z)); 365 366 result.target.x = MAX(dim.x / (i32)result.dispatch.x / DAS_LOCAL_SIZE_X, 1); 367 result.target.y = MAX(dim.y / (i32)result.dispatch.y / DAS_LOCAL_SIZE_Y, 1); 368 result.target.z = MAX(dim.z / (i32)result.dispatch.z / DAS_LOCAL_SIZE_Z, 1); 369 370 result.points_per_dispatch = 1; 371 result.points_per_dispatch *= result.dispatch.x * DAS_LOCAL_SIZE_X; 372 result.points_per_dispatch *= result.dispatch.y * DAS_LOCAL_SIZE_Y; 373 result.points_per_dispatch *= result.dispatch.z * DAS_LOCAL_SIZE_Z; 374 375 result.total_points = (u32)(dim.x * dim.y * dim.z); 376 377 return result; 378 } 379 380 function iv3 381 step_compute_cursor(struct compute_cursor *cursor) 382 { 383 cursor->cursor.x += 1; 384 if (cursor->cursor.x >= cursor->target.x) { 385 cursor->cursor.x = 0; 386 cursor->cursor.y += 1; 387 if (cursor->cursor.y >= cursor->target.y) { 388 cursor->cursor.y = 0; 389 cursor->cursor.z += 1; 390 } 391 } 392 393 cursor->completed_points += cursor->points_per_dispatch; 394 395 iv3 result = cursor->cursor; 396 result.x *= (i32)cursor->dispatch.x * DAS_LOCAL_SIZE_X; 397 result.y *= (i32)cursor->dispatch.y * DAS_LOCAL_SIZE_Y; 398 result.z *= (i32)cursor->dispatch.z * DAS_LOCAL_SIZE_Z; 399 400 return result; 401 } 402 403 function b32 404 compute_cursor_finished(struct compute_cursor *cursor) 405 { 406 b32 result = cursor->completed_points >= cursor->total_points; 407 return result; 408 } 409 410 function m4 411 das_voxel_transform_matrix(BeamformerParameters *bp) 412 { 413 v3 extent = v3_abs(v3_sub(bp->output_max_coordinate, bp->output_min_coordinate)); 414 v3 points = v3_from_iv3(make_valid_output_points(bp->output_points.E)); 415 416 m4 T1 = m4_translation(v3_scale(v3_sub(points, (v3){{1.0f, 1.0f, 1.0f}}), -0.5f)); 417 m4 T2 = m4_translation(v3_add(bp->output_min_coordinate, v3_scale(extent, 0.5f))); 418 m4 S = m4_scale(v3_div(extent, points)); 419 420 m4 R; 421 switch (bp->acquisition_kind) { 422 case BeamformerAcquisitionKind_FORCES: 423 case BeamformerAcquisitionKind_UFORCES: 424 case BeamformerAcquisitionKind_Flash: 425 { 426 R = m4_identity(); 427 S.c[1].E[1] = 0; 428 T2.c[3].E[1] = 0; 429 }break; 430 case BeamformerAcquisitionKind_HERO_PA: 431 case BeamformerAcquisitionKind_HERCULES: 432 case BeamformerAcquisitionKind_UHERCULES: 433 case BeamformerAcquisitionKind_RCA_TPW: 434 case BeamformerAcquisitionKind_RCA_VLS: 435 { 436 R = m4_rotation_about_z(bp->beamform_plane ? 0.0f : 0.25f); 437 if (!(points.x > 1 && points.y > 1 && points.z > 1)) 438 T2.c[3].E[1] = bp->off_axis_pos; 439 }break; 440 default:{ R = m4_identity(); }break; 441 } 442 m4 result = m4_mul(R, m4_mul(T2, m4_mul(S, T1))); 443 return result; 444 } 445 446 function void 447 plan_compute_pipeline(BeamformerComputePlan *cp, BeamformerParameterBlock *pb) 448 { 449 b32 run_cuda_hilbert = 0; 450 b32 demodulate = 0; 451 452 for (u32 i = 0; i < pb->pipeline.shader_count; i++) { 453 switch (pb->pipeline.shaders[i]) { 454 case BeamformerShaderKind_CudaHilbert:{ run_cuda_hilbert = 1; }break; 455 case BeamformerShaderKind_Demodulate:{ demodulate = 1; }break; 456 default:{}break; 457 } 458 } 459 460 if (demodulate) run_cuda_hilbert = 0; 461 462 cp->iq_pipeline = demodulate || run_cuda_hilbert; 463 464 f32 sampling_frequency = pb->parameters.sampling_frequency; 465 u32 decimation_rate = MAX(pb->parameters.decimation_rate, 1); 466 u32 sample_count = pb->parameters.sample_count; 467 if (demodulate) { 468 sample_count /= (2 * decimation_rate); 469 sampling_frequency /= 2 * (f32)decimation_rate; 470 } 471 472 cp->rf_size = sample_count * pb->parameters.channel_count * pb->parameters.acquisition_count; 473 if (cp->iq_pipeline) cp->rf_size *= 8; 474 else cp->rf_size *= 4; 475 476 u32 das_sample_stride = 1; 477 u32 das_transmit_stride = sample_count; 478 u32 das_channel_stride = sample_count * pb->parameters.acquisition_count; 479 480 f32 time_offset = pb->parameters.time_offset; 481 482 BeamformerDataKind data_kind = pb->pipeline.data_kind; 483 cp->pipeline.shader_count = 0; 484 for (u32 i = 0; i < pb->pipeline.shader_count; i++) { 485 BeamformerShaderParameters *sp = pb->pipeline.parameters + i; 486 u32 slot = cp->pipeline.shader_count; 487 u32 shader = pb->pipeline.shaders[i]; 488 b32 commit = 0; 489 490 BeamformerShaderDescriptor *ld = cp->shader_descriptors + slot - 1; 491 BeamformerShaderDescriptor *sd = cp->shader_descriptors + slot; 492 zero_struct(sd); 493 494 switch (shader) { 495 case BeamformerShaderKind_CudaHilbert:{ commit = run_cuda_hilbert; }break; 496 case BeamformerShaderKind_Decode:{ 497 /* TODO(rnp): rework decode first and demodulate after */ 498 b32 first = slot == 0; 499 500 sd->bake.data_kind = data_kind; 501 if (!first) { 502 if (data_kind == BeamformerDataKind_Int16) { 503 sd->bake.data_kind = BeamformerDataKind_Int16Complex; 504 } else { 505 sd->bake.data_kind = BeamformerDataKind_Float32Complex; 506 } 507 } 508 509 BeamformerShaderKind *last_shader = cp->pipeline.shaders + slot - 1; 510 assert(first || ((*last_shader == BeamformerShaderKind_Demodulate || 511 *last_shader == BeamformerShaderKind_Filter))); 512 513 BeamformerShaderDecodeBakeParameters *db = &sd->bake.Decode; 514 db->decode_mode = pb->parameters.decode_mode; 515 db->transmit_count = pb->parameters.acquisition_count; 516 517 u32 channel_stride = pb->parameters.acquisition_count * pb->parameters.sample_count; 518 db->input_sample_stride = first? 1 : ld->bake.Filter.output_sample_stride; 519 db->input_channel_stride = first? channel_stride : ld->bake.Filter.output_channel_stride; 520 db->input_transmit_stride = first? pb->parameters.sample_count : 1; 521 522 db->output_sample_stride = das_sample_stride; 523 db->output_channel_stride = das_channel_stride; 524 db->output_transmit_stride = das_transmit_stride; 525 if (first) { 526 db->output_channel_stride *= decimation_rate; 527 db->output_transmit_stride *= decimation_rate; 528 } 529 530 if (run_cuda_hilbert) sd->bake.flags |= BeamformerShaderDecodeFlags_DilateOutput; 531 532 if (db->decode_mode == BeamformerDecodeMode_None) { 533 sd->layout = (uv3){{64, 1, 1}}; 534 535 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 536 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 537 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z); 538 } else if (db->transmit_count > 40) { 539 sd->bake.flags |= BeamformerShaderDecodeFlags_UseSharedMemory; 540 db->to_process = 2; 541 542 if (db->transmit_count == 48) 543 db->to_process = db->transmit_count / 16; 544 545 b32 use_16z = db->transmit_count == 48 || db->transmit_count == 80 || 546 db->transmit_count == 96 || db->transmit_count == 160; 547 sd->layout = (uv3){{4, 1, use_16z? 16 : 32}}; 548 549 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 550 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 551 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z / (f32)db->to_process); 552 } else { 553 db->to_process = 1; 554 555 /* NOTE(rnp): register caching. using more threads will cause the compiler to do 556 * contortions to avoid spilling registers. using less gives higher performance */ 557 /* TODO(rnp): may need to be adjusted to 16 on NVIDIA */ 558 sd->layout = (uv3){{32, 1, 1}}; 559 560 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 561 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 562 sd->dispatch.z = 1; 563 } 564 565 if (first) sd->dispatch.x *= decimation_rate; 566 567 /* NOTE(rnp): decode 2 samples per dispatch when data is i16 */ 568 if (first && data_kind == BeamformerDataKind_Int16) 569 sd->dispatch.x = (u32)ceil_f32((f32)sd->dispatch.x / 2); 570 571 commit = first || db->decode_mode != BeamformerDecodeMode_None; 572 }break; 573 case BeamformerShaderKind_Demodulate: 574 case BeamformerShaderKind_Filter: 575 { 576 b32 first = slot == 0; 577 b32 demod = shader == BeamformerShaderKind_Demodulate; 578 BeamformerFilter *f = cp->filters + sp->filter_slot; 579 580 time_offset += f->time_delay; 581 582 BeamformerShaderFilterBakeParameters *fb = &sd->bake.Filter; 583 fb->filter_length = (u32)f->length; 584 if (demod) sd->bake.flags |= BeamformerShaderFilterFlags_Demodulate; 585 if (f->parameters.complex) sd->bake.flags |= BeamformerShaderFilterFlags_ComplexFilter; 586 587 sd->bake.data_kind = data_kind; 588 if (!first) sd->bake.data_kind = BeamformerDataKind_Float32; 589 590 /* NOTE(rnp): when we are demodulating we pretend that the sampler was alternating 591 * between sampling the I portion and the Q portion of an IQ signal. Therefore there 592 * is an implicit decimation factor of 2 which must always be included. All code here 593 * assumes that the signal was sampled in such a way that supports this operation. 594 * To recover IQ[n] from the sampled data (RF[n]) we do the following: 595 * I[n] = RF[n] 596 * Q[n] = RF[n + 1] 597 * IQ[n] = I[n] - j*Q[n] 598 */ 599 if (demod) { 600 fb->demodulation_frequency = pb->parameters.demodulation_frequency; 601 fb->sampling_frequency = pb->parameters.sampling_frequency / 2; 602 fb->decimation_rate = decimation_rate; 603 fb->sample_count = pb->parameters.sample_count; 604 605 fb->output_channel_stride = das_channel_stride; 606 fb->output_sample_stride = das_sample_stride; 607 fb->output_transmit_stride = das_transmit_stride; 608 609 if (first) { 610 fb->input_channel_stride = pb->parameters.sample_count * pb->parameters.acquisition_count / 2; 611 fb->input_sample_stride = 1; 612 fb->input_transmit_stride = pb->parameters.sample_count / 2; 613 614 if (pb->parameters.decode_mode == BeamformerDecodeMode_None) { 615 sd->bake.flags |= BeamformerShaderFilterFlags_OutputFloats; 616 } else { 617 /* NOTE(rnp): output optimized layout for decoding */ 618 fb->output_channel_stride = das_channel_stride; 619 fb->output_sample_stride = pb->parameters.acquisition_count; 620 fb->output_transmit_stride = 1; 621 } 622 } else { 623 assert(cp->pipeline.shaders[slot - 1] == BeamformerShaderKind_Decode); 624 fb->input_channel_stride = ld->bake.Decode.output_channel_stride; 625 fb->input_sample_stride = ld->bake.Decode.output_sample_stride; 626 fb->input_transmit_stride = ld->bake.Decode.output_transmit_stride; 627 } 628 } else { 629 fb->decimation_rate = 1; 630 fb->output_channel_stride = sample_count * pb->parameters.acquisition_count; 631 fb->output_sample_stride = 1; 632 fb->output_transmit_stride = sample_count; 633 fb->input_channel_stride = sample_count * pb->parameters.acquisition_count; 634 fb->input_sample_stride = 1; 635 fb->input_transmit_stride = sample_count; 636 fb->sample_count = sample_count; 637 } 638 639 /* TODO(rnp): filter may need a different dispatch layout */ 640 sd->layout = (uv3){{128, 1, 1}}; 641 sd->dispatch.x = (u32)ceil_f32((f32)sample_count / (f32)sd->layout.x); 642 sd->dispatch.y = (u32)ceil_f32((f32)pb->parameters.channel_count / (f32)sd->layout.y); 643 sd->dispatch.z = (u32)ceil_f32((f32)pb->parameters.acquisition_count / (f32)sd->layout.z); 644 645 commit = 1; 646 }break; 647 case BeamformerShaderKind_DAS:{ 648 sd->bake.data_kind = BeamformerDataKind_Float32; 649 if (cp->iq_pipeline) 650 sd->bake.data_kind = BeamformerDataKind_Float32Complex; 651 652 BeamformerShaderDASBakeParameters *db = &sd->bake.DAS; 653 BeamformerDASUBO *du = &cp->das_ubo_data; 654 du->voxel_transform = das_voxel_transform_matrix(&pb->parameters); 655 du->xdc_element_pitch = pb->parameters.xdc_element_pitch; 656 db->sampling_frequency = sampling_frequency; 657 db->demodulation_frequency = pb->parameters.demodulation_frequency; 658 db->speed_of_sound = pb->parameters.speed_of_sound; 659 db->time_offset = time_offset; 660 db->f_number = pb->parameters.f_number; 661 db->acquisition_kind = pb->parameters.acquisition_kind; 662 db->sample_count = sample_count; 663 db->channel_count = pb->parameters.channel_count; 664 db->acquisition_count = pb->parameters.acquisition_count; 665 db->interpolation_mode = pb->parameters.interpolation_mode; 666 db->transmit_angle = pb->parameters.focal_vector.E[0]; 667 db->focus_depth = pb->parameters.focal_vector.E[1]; 668 db->transmit_receive_orientation = pb->parameters.transmit_receive_orientation; 669 670 // NOTE(rnp): old gcc will miscompile an assignment 671 mem_copy(du->xdc_transform.E, pb->parameters.xdc_transform.E, sizeof(du->xdc_transform)); 672 673 if (pb->parameters.single_focus) sd->bake.flags |= BeamformerShaderDASFlags_SingleFocus; 674 if (pb->parameters.single_orientation) sd->bake.flags |= BeamformerShaderDASFlags_SingleOrientation; 675 if (pb->parameters.coherency_weighting) sd->bake.flags |= BeamformerShaderDASFlags_CoherencyWeighting; 676 else sd->bake.flags |= BeamformerShaderDASFlags_Fast; 677 678 u32 id = pb->parameters.acquisition_kind; 679 if (id == BeamformerAcquisitionKind_UFORCES || id == BeamformerAcquisitionKind_UHERCULES) 680 sd->bake.flags |= BeamformerShaderDASFlags_Sparse; 681 682 sd->layout.x = DAS_LOCAL_SIZE_X; 683 sd->layout.y = DAS_LOCAL_SIZE_Y; 684 sd->layout.z = DAS_LOCAL_SIZE_Z; 685 686 commit = 1; 687 }break; 688 default:{ commit = 1; }break; 689 } 690 691 if (commit) { 692 u32 index = cp->pipeline.shader_count++; 693 cp->pipeline.shaders[index] = shader; 694 cp->pipeline.parameters[index] = *sp; 695 } 696 } 697 cp->pipeline.data_kind = data_kind; 698 } 699 700 function void 701 stream_push_shader_header(Stream *s, BeamformerShaderKind shader_kind, s8 header) 702 { 703 stream_append_s8s(s, s8("#version 460 core\n\n"), header); 704 705 switch (shader_kind) { 706 case BeamformerShaderKind_DAS:{ 707 stream_append_s8(s, s8("" 708 "layout(location = " str(DAS_VOXEL_OFFSET_UNIFORM_LOC) ") uniform ivec3 u_voxel_offset;\n" 709 "layout(location = " str(DAS_CYCLE_T_UNIFORM_LOC) ") uniform uint u_cycle_t;\n" 710 "layout(location = " str(DAS_FAST_CHANNEL_UNIFORM_LOC) ") uniform int u_channel;\n\n" 711 )); 712 }break; 713 case BeamformerShaderKind_Decode:{ 714 stream_append_s8s(s, s8("" 715 "layout(location = " str(DECODE_FIRST_PASS_UNIFORM_LOC) ") uniform bool u_first_pass;\n\n" 716 )); 717 }break; 718 case BeamformerShaderKind_MinMax:{ 719 stream_append_s8(s, s8("layout(location = " str(MIN_MAX_MIPS_LEVEL_UNIFORM_LOC) 720 ") uniform int u_mip_map;\n\n")); 721 }break; 722 case BeamformerShaderKind_Sum:{ 723 stream_append_s8(s, s8("layout(location = " str(SUM_PRESCALE_UNIFORM_LOC) 724 ") uniform float u_sum_prescale = 1.0;\n\n")); 725 }break; 726 default:{}break; 727 } 728 } 729 730 function void 731 load_compute_shader(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 shader_slot, Arena arena) 732 { 733 read_only local_persist s8 compute_headers[BeamformerShaderKind_ComputeCount] = { 734 /* X(name, type, gltype) */ 735 #define X(name, t, gltype) "\t" #gltype " " #name ";\n" 736 [BeamformerShaderKind_DAS] = s8_comp("layout(std140, binding = 0) uniform parameters {\n" 737 BEAMFORMER_DAS_UBO_PARAM_LIST 738 "};\n\n" 739 ), 740 #undef X 741 }; 742 743 BeamformerShaderKind shader = cp->pipeline.shaders[shader_slot]; 744 745 u32 program = 0; 746 i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[shader]; 747 if (reloadable_index != -1) { 748 BeamformerShaderKind base_shader = beamformer_reloadable_shader_kinds[reloadable_index]; 749 s8 path; 750 if (!BakeShaders) 751 path = push_s8_from_parts(&arena, os_path_separator(), s8("shaders"), 752 beamformer_reloadable_shader_files[reloadable_index]); 753 754 Stream shader_stream = arena_stream(arena); 755 stream_push_shader_header(&shader_stream, base_shader, compute_headers[base_shader]); 756 757 i32 header_vector_length = beamformer_shader_header_vector_lengths[reloadable_index]; 758 i32 *header_vector = beamformer_shader_header_vectors[reloadable_index]; 759 for (i32 index = 0; index < header_vector_length; index++) 760 stream_append_s8(&shader_stream, beamformer_shader_global_header_strings[header_vector[index]]); 761 762 BeamformerShaderDescriptor *sd = cp->shader_descriptors + shader_slot; 763 764 if (sd->layout.x != 0) { 765 stream_append_s8(&shader_stream, s8("layout(local_size_x = ")); 766 stream_append_u64(&shader_stream, sd->layout.x); 767 stream_append_s8(&shader_stream, s8(", local_size_y = ")); 768 stream_append_u64(&shader_stream, sd->layout.y); 769 stream_append_s8(&shader_stream, s8(", local_size_z = ")); 770 stream_append_u64(&shader_stream, sd->layout.z); 771 stream_append_s8(&shader_stream, s8(") in;\n\n")); 772 } 773 774 u32 *parameters = (u32 *)&sd->bake; 775 s8 *names = beamformer_shader_bake_parameter_names[reloadable_index]; 776 u32 float_bits = beamformer_shader_bake_parameter_float_bits[reloadable_index]; 777 i32 count = beamformer_shader_bake_parameter_counts[reloadable_index]; 778 779 for (i32 index = 0; index < count; index++) { 780 stream_append_s8s(&shader_stream, s8("#define "), names[index], 781 (float_bits & (1 << index))? s8(" uintBitsToFloat") : s8(" "), s8("(0x")); 782 stream_append_hex_u64(&shader_stream, parameters[index]); 783 stream_append_s8(&shader_stream, s8(")\n")); 784 } 785 786 stream_append_s8(&shader_stream, s8("#define DataKind (0x")); 787 stream_append_hex_u64(&shader_stream, sd->bake.data_kind); 788 stream_append_s8(&shader_stream, s8(")\n\n")); 789 790 s8 *flag_names = beamformer_shader_flag_strings[reloadable_index]; 791 u32 flag_count = beamformer_shader_flag_strings_count[reloadable_index]; 792 u32 flags = sd->bake.flags; 793 for (u32 bit = 0; bit < flag_count; bit++) { 794 stream_append_s8s(&shader_stream, s8("#define "), flag_names[bit], 795 (flags & (1 << bit))? s8(" 1") : s8(" 0"), s8("\n")); 796 } 797 798 stream_append_s8(&shader_stream, s8("\n#line 1\n")); 799 800 s8 shader_text; 801 if (BakeShaders) { 802 stream_append_s8(&shader_stream, beamformer_shader_data[reloadable_index]); 803 shader_text = arena_stream_commit(&arena, &shader_stream); 804 } else { 805 shader_text = arena_stream_commit(&arena, &shader_stream); 806 i64 length = os_read_entire_file((c8 *)path.data, arena.beg, arena_capacity(&arena, u8)); 807 shader_text.len += length; 808 arena_commit(&arena, length); 809 } 810 811 /* TODO(rnp): instance name */ 812 s8 shader_name = beamformer_shader_names[shader]; 813 program = load_shader(arena, &shader_text, (u32 []){GL_COMPUTE_SHADER}, 1, shader_name); 814 } 815 816 glDeleteProgram(cp->programs[shader_slot]); 817 cp->programs[shader_slot] = program; 818 } 819 820 function void 821 beamformer_commit_parameter_block(BeamformerCtx *ctx, BeamformerComputePlan *cp, u32 block, Arena arena) 822 { 823 BeamformerParameterBlock *pb = beamformer_parameter_block_lock(ctx->shared_memory, block, -1); 824 for EachBit(pb->dirty_regions, region) { 825 switch (region) { 826 case BeamformerParameterBlockRegion_ComputePipeline: 827 case BeamformerParameterBlockRegion_Parameters: 828 { 829 plan_compute_pipeline(cp, pb); 830 831 /* NOTE(rnp): these are both handled by plan_compute_pipeline() */ 832 u32 mask = 1 << BeamformerParameterBlockRegion_ComputePipeline | 833 1 << BeamformerParameterBlockRegion_Parameters; 834 pb->dirty_regions &= ~mask; 835 836 for (u32 shader_slot = 0; shader_slot < cp->pipeline.shader_count; shader_slot++) { 837 u128 hash = u128_hash_from_data(cp->shader_descriptors + shader_slot, sizeof(BeamformerShaderDescriptor)); 838 if (!u128_equal(hash, cp->shader_hashes[shader_slot])) 839 cp->dirty_programs |= 1 << shader_slot; 840 cp->shader_hashes[shader_slot] = hash; 841 } 842 843 #define X(k, t, v) glNamedBufferSubData(cp->ubos[BeamformerComputeUBOKind_##k], \ 844 0, sizeof(t), &cp->v ## _ubo_data); 845 BEAMFORMER_COMPUTE_UBO_LIST 846 #undef X 847 848 cp->acquisition_count = pb->parameters.acquisition_count; 849 cp->acquisition_kind = pb->parameters.acquisition_kind; 850 851 u32 decoded_data_size = cp->rf_size; 852 if (ctx->compute_context.ping_pong_ssbo_size < decoded_data_size) 853 alloc_shader_storage(ctx, decoded_data_size, arena); 854 855 if (cp->hadamard_order != (i32)cp->acquisition_count) 856 update_hadamard_texture(cp, (i32)cp->acquisition_count, arena); 857 858 cp->min_coordinate = pb->parameters.output_min_coordinate; 859 cp->max_coordinate = pb->parameters.output_max_coordinate; 860 861 cp->output_points = make_valid_output_points(pb->parameters.output_points.E); 862 cp->average_frames = pb->parameters.output_points.E[3]; 863 864 GLenum gl_kind = cp->iq_pipeline ? GL_RG32F : GL_R32F; 865 if (cp->average_frames > 1 && !beamformer_frame_compatible(ctx->averaged_frames + 0, cp->output_points, gl_kind)) { 866 alloc_beamform_frame(ctx->averaged_frames + 0, cp->output_points, gl_kind, s8("Averaged Frame"), arena); 867 alloc_beamform_frame(ctx->averaged_frames + 1, cp->output_points, gl_kind, s8("Averaged Frame"), arena); 868 } 869 }break; 870 case BeamformerParameterBlockRegion_ChannelMapping:{ 871 cuda_set_channel_mapping(pb->channel_mapping); 872 }break; 873 case BeamformerParameterBlockRegion_FocalVectors: 874 case BeamformerParameterBlockRegion_SparseElements: 875 case BeamformerParameterBlockRegion_TransmitReceiveOrientations: 876 { 877 BeamformerComputeTextureKind texture_kind = 0; 878 u32 pixel_type = 0, texture_format = 0; 879 switch (region) { 880 #define X(kind, _gl, tf, pt, ...) \ 881 case BeamformerParameterBlockRegion_## kind:{ \ 882 texture_kind = BeamformerComputeTextureKind_## kind; \ 883 texture_format = tf; \ 884 pixel_type = pt; \ 885 }break; 886 BEAMFORMER_COMPUTE_TEXTURE_LIST 887 #undef X 888 InvalidDefaultCase; 889 } 890 glTextureSubImage1D(cp->textures[texture_kind], 0, 0, BeamformerMaxChannelCount, 891 texture_format, pixel_type, 892 (u8 *)pb + BeamformerParameterBlockRegionOffsets[region]); 893 }break; 894 } 895 } 896 beamformer_parameter_block_unlock(ctx->shared_memory, block); 897 } 898 899 function void 900 do_compute_shader(BeamformerCtx *ctx, BeamformerComputePlan *cp, BeamformerFrame *frame, 901 BeamformerShaderKind shader, u32 shader_slot, BeamformerShaderParameters *sp, Arena arena) 902 { 903 BeamformerComputeContext *cc = &ctx->compute_context; 904 905 u32 program = cp->programs[shader_slot]; 906 glUseProgram(program); 907 908 u32 output_ssbo_idx = !cc->last_output_ssbo_index; 909 u32 input_ssbo_idx = cc->last_output_ssbo_index; 910 911 uv3 dispatch = cp->shader_descriptors[shader_slot].dispatch; 912 switch (shader) { 913 case BeamformerShaderKind_Decode:{ 914 glBindImageTexture(0, cp->textures[BeamformerComputeTextureKind_Hadamard], 0, 0, 0, GL_READ_ONLY, GL_R32F); 915 916 BeamformerDecodeMode mode = cp->shader_descriptors[shader_slot].bake.Decode.decode_mode; 917 if (shader_slot == 0) { 918 if (mode != BeamformerDecodeMode_None) { 919 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, cc->ping_pong_ssbos[input_ssbo_idx]); 920 glProgramUniform1ui(program, DECODE_FIRST_PASS_UNIFORM_LOC, 1); 921 922 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 923 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 924 } 925 } 926 927 if (mode != BeamformerDecodeMode_None) 928 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx]); 929 930 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, cc->ping_pong_ssbos[output_ssbo_idx]); 931 932 glProgramUniform1ui(program, DECODE_FIRST_PASS_UNIFORM_LOC, 0); 933 934 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 935 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 936 937 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 938 }break; 939 case BeamformerShaderKind_CudaDecode:{ 940 cuda_decode(0, output_ssbo_idx, 0); 941 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 942 }break; 943 case BeamformerShaderKind_CudaHilbert:{ 944 cuda_hilbert(input_ssbo_idx, output_ssbo_idx); 945 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 946 }break; 947 case BeamformerShaderKind_Filter: 948 case BeamformerShaderKind_Demodulate: 949 { 950 if (shader_slot != 0) 951 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx]); 952 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 2, cc->ping_pong_ssbos[output_ssbo_idx]); 953 glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 3, cp->filters[sp->filter_slot].ssbo); 954 955 glDispatchCompute(dispatch.x, dispatch.y, dispatch.z); 956 glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT); 957 958 cc->last_output_ssbo_index = !cc->last_output_ssbo_index; 959 }break; 960 case BeamformerShaderKind_MinMax:{ 961 for (i32 i = 1; i < frame->mips; i++) { 962 glBindImageTexture(0, frame->texture, i - 1, GL_TRUE, 0, GL_READ_ONLY, GL_RG32F); 963 glBindImageTexture(1, frame->texture, i - 0, GL_TRUE, 0, GL_WRITE_ONLY, GL_RG32F); 964 glProgramUniform1i(program, MIN_MAX_MIPS_LEVEL_UNIFORM_LOC, i); 965 966 u32 width = (u32)frame->dim.x >> i; 967 u32 height = (u32)frame->dim.y >> i; 968 u32 depth = (u32)frame->dim.z >> i; 969 glDispatchCompute(ORONE(width / 32), ORONE(height), ORONE(depth / 32)); 970 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 971 } 972 }break; 973 case BeamformerShaderKind_DAS:{ 974 local_persist u32 das_cycle_t = 0; 975 976 BeamformerShaderBakeParameters *bp = &cp->shader_descriptors[shader_slot].bake; 977 b32 fast = (bp->flags & BeamformerShaderDASFlags_Fast) != 0; 978 b32 sparse = (bp->flags & BeamformerShaderDASFlags_Sparse) != 0; 979 980 if (fast) { 981 glClearTexImage(frame->texture, 0, GL_RED, GL_FLOAT, 0); 982 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT); 983 glBindImageTexture(0, frame->texture, 0, GL_TRUE, 0, GL_READ_WRITE, cp->iq_pipeline ? GL_RG32F : GL_R32F); 984 } else { 985 glBindImageTexture(0, frame->texture, 0, GL_TRUE, 0, GL_WRITE_ONLY, cp->iq_pipeline ? GL_RG32F : GL_R32F); 986 } 987 988 u32 sparse_texture = cp->textures[BeamformerComputeTextureKind_SparseElements]; 989 if (!sparse) sparse_texture = 0; 990 991 glBindBufferBase(GL_UNIFORM_BUFFER, 0, cp->ubos[BeamformerComputeUBOKind_DAS]); 992 glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, cc->ping_pong_ssbos[input_ssbo_idx], 0, cp->rf_size); 993 glBindImageTexture(1, sparse_texture, 0, 0, 0, GL_READ_ONLY, GL_R16I); 994 glBindImageTexture(2, cp->textures[BeamformerComputeTextureKind_FocalVectors], 0, 0, 0, GL_READ_ONLY, GL_RG32F); 995 glBindImageTexture(3, cp->textures[BeamformerComputeTextureKind_TransmitReceiveOrientations], 0, 0, 0, GL_READ_ONLY, GL_R8I); 996 997 glProgramUniform1ui(program, DAS_CYCLE_T_UNIFORM_LOC, das_cycle_t++); 998 999 if (fast) { 1000 i32 loop_end; 1001 if (bp->DAS.acquisition_kind == BeamformerAcquisitionKind_RCA_VLS || 1002 bp->DAS.acquisition_kind == BeamformerAcquisitionKind_RCA_TPW) 1003 { 1004 /* NOTE(rnp): to avoid repeatedly sampling the whole focal vectors 1005 * texture we loop over transmits for VLS/TPW */ 1006 loop_end = (i32)bp->DAS.acquisition_count; 1007 } else { 1008 loop_end = (i32)bp->DAS.channel_count; 1009 } 1010 f32 percent_per_step = 1.0f / (f32)loop_end; 1011 cc->processing_progress = -percent_per_step; 1012 for (i32 index = 0; index < loop_end; index++) { 1013 cc->processing_progress += percent_per_step; 1014 /* IMPORTANT(rnp): prevents OS from coalescing and killing our shader */ 1015 glFinish(); 1016 glProgramUniform1i(program, DAS_FAST_CHANNEL_UNIFORM_LOC, index); 1017 glDispatchCompute((u32)ceil_f32((f32)frame->dim.x / DAS_LOCAL_SIZE_X), 1018 (u32)ceil_f32((f32)frame->dim.y / DAS_LOCAL_SIZE_Y), 1019 (u32)ceil_f32((f32)frame->dim.z / DAS_LOCAL_SIZE_Z)); 1020 glMemoryBarrier(GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 1021 } 1022 } else { 1023 #if 1 1024 /* TODO(rnp): compute max_points_per_dispatch based on something like a 1025 * transmit_count * channel_count product */ 1026 u32 max_points_per_dispatch = KB(64); 1027 struct compute_cursor cursor = start_compute_cursor(frame->dim, max_points_per_dispatch); 1028 f32 percent_per_step = (f32)cursor.points_per_dispatch / (f32)cursor.total_points; 1029 cc->processing_progress = -percent_per_step; 1030 for (iv3 offset = {0}; 1031 !compute_cursor_finished(&cursor); 1032 offset = step_compute_cursor(&cursor)) 1033 { 1034 cc->processing_progress += percent_per_step; 1035 /* IMPORTANT(rnp): prevents OS from coalescing and killing our shader */ 1036 glFinish(); 1037 glProgramUniform3iv(program, DAS_VOXEL_OFFSET_UNIFORM_LOC, 1, offset.E); 1038 glDispatchCompute(cursor.dispatch.x, cursor.dispatch.y, cursor.dispatch.z); 1039 } 1040 #else 1041 /* NOTE(rnp): use this for testing tiling code. The performance of the above path 1042 * should be the same as this path if everything is working correctly */ 1043 iv3 compute_dim_offset = {0}; 1044 glProgramUniform3iv(program, DAS_VOXEL_OFFSET_UNIFORM_LOC, 1, compute_dim_offset.E); 1045 glDispatchCompute((u32)ceil_f32((f32)dim.x / DAS_LOCAL_SIZE_X), 1046 (u32)ceil_f32((f32)dim.y / DAS_LOCAL_SIZE_Y), 1047 (u32)ceil_f32((f32)dim.z / DAS_LOCAL_SIZE_Z)); 1048 #endif 1049 } 1050 glMemoryBarrier(GL_TEXTURE_UPDATE_BARRIER_BIT|GL_SHADER_IMAGE_ACCESS_BARRIER_BIT); 1051 }break; 1052 case BeamformerShaderKind_Sum:{ 1053 u32 aframe_index = ctx->averaged_frame_index % ARRAY_COUNT(ctx->averaged_frames); 1054 BeamformerFrame *aframe = ctx->averaged_frames + aframe_index; 1055 aframe->id = ctx->averaged_frame_index; 1056 atomic_store_u32(&aframe->ready_to_present, 0); 1057 /* TODO(rnp): hack we need a better way of specifying which frames to sum; 1058 * this is fine for rolling averaging but what if we want to do something else */ 1059 assert(frame >= ctx->beamform_frames); 1060 assert(frame < ctx->beamform_frames + countof(ctx->beamform_frames)); 1061 u32 base_index = (u32)(frame - ctx->beamform_frames); 1062 u32 to_average = (u32)cp->average_frames; 1063 u32 frame_count = 0; 1064 u32 *in_textures = push_array(&arena, u32, BeamformerMaxSavedFrames); 1065 ComputeFrameIterator cfi = compute_frame_iterator(ctx, 1 + base_index - to_average, to_average); 1066 for (BeamformerFrame *it = frame_next(&cfi); it; it = frame_next(&cfi)) 1067 in_textures[frame_count++] = it->texture; 1068 1069 assert(to_average == frame_count); 1070 1071 glProgramUniform1f(program, SUM_PRESCALE_UNIFORM_LOC, 1 / (f32)frame_count); 1072 do_sum_shader(cc, in_textures, frame_count, aframe->texture, aframe->dim); 1073 aframe->min_coordinate = frame->min_coordinate; 1074 aframe->max_coordinate = frame->max_coordinate; 1075 aframe->compound_count = frame->compound_count; 1076 aframe->acquisition_kind = frame->acquisition_kind; 1077 }break; 1078 InvalidDefaultCase; 1079 } 1080 } 1081 1082 function s8 1083 shader_text_with_header(s8 header, s8 filepath, b32 has_file, BeamformerShaderKind shader_kind, Arena *arena) 1084 { 1085 Stream sb = arena_stream(*arena); 1086 stream_push_shader_header(&sb, shader_kind, header); 1087 stream_append_s8(&sb, s8("\n#line 1\n")); 1088 1089 s8 result; 1090 if (BakeShaders) { 1091 /* TODO(rnp): better handling of shaders with no backing file */ 1092 if (has_file) { 1093 i32 reloadable_index = beamformer_shader_reloadable_index_by_shader[shader_kind]; 1094 stream_append_s8(&sb, beamformer_shader_data[reloadable_index]); 1095 } 1096 result = arena_stream_commit(arena, &sb); 1097 } else { 1098 result = arena_stream_commit(arena, &sb); 1099 if (has_file) { 1100 i64 length = os_read_entire_file((c8 *)filepath.data, arena->beg, arena_capacity(arena, u8)); 1101 result.len += length; 1102 arena_commit(arena, length); 1103 } 1104 } 1105 1106 return result; 1107 } 1108 1109 /* NOTE(rnp): currently this function is only handling rendering shaders. 1110 * look at load_compute_shader for compute shaders */ 1111 function void 1112 beamformer_reload_shader(BeamformerCtx *ctx, BeamformerShaderReloadContext *src, Arena arena, s8 shader_name) 1113 { 1114 BeamformerShaderKind kind = beamformer_reloadable_shader_kinds[src->reloadable_info_index]; 1115 assert(kind == BeamformerShaderKind_Render3D); 1116 1117 s8 path = push_s8_from_parts(&arena, os_path_separator(), s8("shaders"), 1118 beamformer_reloadable_shader_files[src->reloadable_info_index]); 1119 1120 i32 shader_count = 1; 1121 BeamformerShaderReloadContext *link = src->link; 1122 while (link != src) { shader_count++; link = link->link; } 1123 1124 s8 *shader_texts = push_array(&arena, s8, shader_count); 1125 u32 *shader_types = push_array(&arena, u32, shader_count); 1126 1127 i32 index = 0; 1128 do { 1129 b32 has_file = link->reloadable_info_index >= 0; 1130 shader_texts[index] = shader_text_with_header(link->header, path, has_file, kind, &arena); 1131 shader_types[index] = link->gl_type; 1132 index++; 1133 link = link->link; 1134 } while (link != src); 1135 1136 u32 *shader = &ctx->frame_view_render_context.shader; 1137 glDeleteProgram(*shader); 1138 *shader = load_shader(arena, shader_texts, shader_types, shader_count, shader_name); 1139 ctx->frame_view_render_context.updated = 1; 1140 } 1141 1142 function void 1143 complete_queue(BeamformerCtx *ctx, BeamformWorkQueue *q, Arena *arena, iptr gl_context) 1144 { 1145 BeamformerComputeContext * cs = &ctx->compute_context; 1146 BeamformerSharedMemory * sm = ctx->shared_memory; 1147 1148 BeamformWork *work = beamform_work_queue_pop(q); 1149 while (work) { 1150 b32 can_commit = 1; 1151 switch (work->kind) { 1152 case BeamformerWorkKind_ExportBuffer:{ 1153 /* TODO(rnp): better way of handling DispatchCompute barrier */ 1154 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_DispatchCompute); 1155 beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)work->lock, (u32)-1); 1156 BeamformerExportContext *ec = &work->export_context; 1157 switch (ec->kind) { 1158 case BeamformerExportKind_BeamformedData:{ 1159 BeamformerFrame *frame = ctx->latest_frame; 1160 if (frame) { 1161 assert(frame->ready_to_present); 1162 u32 texture = frame->texture; 1163 iv3 dim = frame->dim; 1164 u32 out_size = (u32)dim.x * (u32)dim.y * (u32)dim.z * 2 * sizeof(f32); 1165 if (out_size <= ec->size) { 1166 glGetTextureImage(texture, 0, GL_RG, GL_FLOAT, (i32)out_size, 1167 beamformer_shared_memory_scratch_arena(sm).beg); 1168 } 1169 } 1170 }break; 1171 case BeamformerExportKind_Stats:{ 1172 ComputeTimingTable *table = ctx->compute_timing_table; 1173 /* NOTE(rnp): do a little spin to let this finish updating */ 1174 spin_wait(table->write_index != atomic_load_u32(&table->read_index)); 1175 ComputeShaderStats *stats = ctx->compute_shader_stats; 1176 if (sizeof(stats->table) <= ec->size) 1177 mem_copy(beamformer_shared_memory_scratch_arena(sm).beg, &stats->table, sizeof(stats->table)); 1178 }break; 1179 InvalidDefaultCase; 1180 } 1181 beamformer_shared_memory_release_lock(ctx->shared_memory, work->lock); 1182 post_sync_barrier(ctx->shared_memory, BeamformerSharedMemoryLockKind_ExportSync); 1183 }break; 1184 case BeamformerWorkKind_CreateFilter:{ 1185 /* TODO(rnp): this should probably get deleted and moved to lazy loading */ 1186 BeamformerCreateFilterContext *fctx = &work->create_filter_context; 1187 u32 block = fctx->parameter_block; 1188 u32 slot = fctx->filter_slot; 1189 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, block, arena); 1190 beamformer_filter_update(cp->filters + slot, fctx->parameters, block, slot, *arena); 1191 }break; 1192 case BeamformerWorkKind_ComputeIndirect:{ 1193 fill_frame_compute_work(ctx, work, work->compute_indirect_context.view_plane, 1194 work->compute_indirect_context.parameter_block, 1); 1195 } /* FALLTHROUGH */ 1196 case BeamformerWorkKind_Compute:{ 1197 DEBUG_DECL(glClearNamedBufferData(cs->ping_pong_ssbos[0], GL_RG32F, GL_RG, GL_FLOAT, 0);) 1198 DEBUG_DECL(glClearNamedBufferData(cs->ping_pong_ssbos[1], GL_RG32F, GL_RG, GL_FLOAT, 0);) 1199 DEBUG_DECL(glMemoryBarrier(GL_SHADER_STORAGE_BARRIER_BIT);) 1200 1201 push_compute_timing_info(ctx->compute_timing_table, 1202 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameBegin}); 1203 1204 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(cs, work->compute_context.parameter_block, arena); 1205 if (beamformer_parameter_block_dirty(sm, work->compute_context.parameter_block)) { 1206 u32 block = work->compute_context.parameter_block; 1207 beamformer_commit_parameter_block(ctx, cp, block, *arena); 1208 atomic_store_u32(&ctx->ui_dirty_parameter_blocks, (u32)(ctx->beamform_work_queue != q) << block); 1209 } 1210 1211 post_sync_barrier(ctx->shared_memory, work->lock); 1212 1213 u32 dirty_programs = atomic_swap_u32(&cp->dirty_programs, 0); 1214 static_assert(ISPOWEROF2(BeamformerMaxComputeShaderStages), 1215 "max compute shader stages must be power of 2"); 1216 assert((dirty_programs & ~((u32)BeamformerMaxComputeShaderStages - 1)) == 0); 1217 for EachBit(dirty_programs, slot) 1218 load_compute_shader(ctx, cp, (u32)slot, *arena); 1219 1220 atomic_store_u32(&cs->processing_compute, 1); 1221 start_renderdoc_capture(gl_context); 1222 1223 BeamformerFrame *frame = work->compute_context.frame; 1224 1225 GLenum gl_kind = cp->iq_pipeline ? GL_RG32F : GL_R32F; 1226 if (!beamformer_frame_compatible(frame, cp->output_points, gl_kind)) 1227 alloc_beamform_frame(frame, cp->output_points, gl_kind, s8("Beamformed_Data"), *arena); 1228 1229 frame->min_coordinate = cp->min_coordinate; 1230 frame->max_coordinate = cp->max_coordinate; 1231 frame->acquisition_kind = cp->acquisition_kind; 1232 frame->compound_count = cp->acquisition_count; 1233 1234 BeamformerComputeContext *cc = &ctx->compute_context; 1235 BeamformerComputePipeline *pipeline = &cp->pipeline; 1236 /* NOTE(rnp): first stage requires access to raw data buffer directly so we break 1237 * it out into a separate step. This way data can get released as soon as possible */ 1238 if (pipeline->shader_count > 0) { 1239 BeamformerRFBuffer *rf = &cs->rf_buffer; 1240 u32 slot = rf->compute_index % countof(rf->compute_syncs); 1241 1242 if (work->kind == BeamformerWorkKind_ComputeIndirect) { 1243 /* NOTE(rnp): compute indirect is used when uploading data. if compute thread 1244 * preempts upload it must wait for the fence to exist. then it must tell the 1245 * GPU to wait for upload to complete before it can start compute */ 1246 spin_wait(!atomic_load_u64(rf->upload_syncs + slot)); 1247 1248 glWaitSync(rf->upload_syncs[slot], 0, GL_TIMEOUT_IGNORED); 1249 glDeleteSync(rf->upload_syncs[slot]); 1250 rf->compute_index++; 1251 } else { 1252 slot = (rf->compute_index - 1) % countof(rf->compute_syncs); 1253 } 1254 1255 glBindBufferRange(GL_SHADER_STORAGE_BUFFER, 1, rf->ssbo, slot * rf->active_rf_size, rf->active_rf_size); 1256 1257 glBeginQuery(GL_TIME_ELAPSED, cc->shader_timer_ids[0]); 1258 do_compute_shader(ctx, cp, frame, pipeline->shaders[0], 0, pipeline->parameters + 0, *arena); 1259 glEndQuery(GL_TIME_ELAPSED); 1260 1261 if (work->kind == BeamformerWorkKind_ComputeIndirect) { 1262 atomic_store_u64(rf->compute_syncs + slot, glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0)); 1263 atomic_store_u64(rf->upload_syncs + slot, 0); 1264 } 1265 } 1266 1267 b32 did_sum_shader = 0; 1268 for (u32 i = 1; i < pipeline->shader_count; i++) { 1269 did_sum_shader |= pipeline->shaders[i] == BeamformerShaderKind_Sum; 1270 glBeginQuery(GL_TIME_ELAPSED, cc->shader_timer_ids[i]); 1271 do_compute_shader(ctx, cp, frame, pipeline->shaders[i], i, pipeline->parameters + i, *arena); 1272 glEndQuery(GL_TIME_ELAPSED); 1273 } 1274 1275 /* NOTE(rnp): the first of these blocks until work completes */ 1276 for (u32 i = 0; i < pipeline->shader_count; i++) { 1277 ComputeTimingInfo info = {0}; 1278 info.kind = ComputeTimingInfoKind_Shader; 1279 info.shader = pipeline->shaders[i]; 1280 glGetQueryObjectui64v(cc->shader_timer_ids[i], GL_QUERY_RESULT, &info.timer_count); 1281 push_compute_timing_info(ctx->compute_timing_table, info); 1282 } 1283 cs->processing_progress = 1; 1284 1285 frame->ready_to_present = 1; 1286 if (did_sum_shader) { 1287 u32 aframe_index = ((ctx->averaged_frame_index++) % countof(ctx->averaged_frames)); 1288 ctx->averaged_frames[aframe_index].view_plane_tag = frame->view_plane_tag; 1289 ctx->averaged_frames[aframe_index].ready_to_present = 1; 1290 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)(ctx->averaged_frames + aframe_index)); 1291 } else { 1292 atomic_store_u64((u64 *)&ctx->latest_frame, (u64)frame); 1293 } 1294 cs->processing_compute = 0; 1295 1296 push_compute_timing_info(ctx->compute_timing_table, 1297 (ComputeTimingInfo){.kind = ComputeTimingInfoKind_ComputeFrameEnd}); 1298 1299 end_renderdoc_capture(gl_context); 1300 }break; 1301 InvalidDefaultCase; 1302 } 1303 1304 if (can_commit) { 1305 beamform_work_queue_pop_commit(q); 1306 work = beamform_work_queue_pop(q); 1307 } 1308 } 1309 } 1310 1311 function void 1312 coalesce_timing_table(ComputeTimingTable *t, ComputeShaderStats *stats) 1313 { 1314 /* TODO(rnp): we do not currently do anything to handle the potential for a half written 1315 * info item. this could result in garbage entries but they shouldn't really matter */ 1316 1317 u32 target = atomic_load_u32(&t->write_index); 1318 u32 stats_index = (stats->latest_frame_index + 1) % countof(stats->table.times); 1319 1320 static_assert(BeamformerShaderKind_Count + 1 <= 32, "timing coalescence bitfield test"); 1321 u32 seen_info_test = 0; 1322 1323 while (t->read_index != target) { 1324 ComputeTimingInfo info = t->buffer[t->read_index % countof(t->buffer)]; 1325 switch (info.kind) { 1326 case ComputeTimingInfoKind_ComputeFrameBegin:{ 1327 assert(t->compute_frame_active == 0); 1328 t->compute_frame_active = 1; 1329 /* NOTE(rnp): allow multiple instances of same shader to accumulate */ 1330 mem_clear(stats->table.times[stats_index], 0, sizeof(stats->table.times[stats_index])); 1331 }break; 1332 case ComputeTimingInfoKind_ComputeFrameEnd:{ 1333 assert(t->compute_frame_active == 1); 1334 t->compute_frame_active = 0; 1335 stats->latest_frame_index = stats_index; 1336 stats_index = (stats_index + 1) % countof(stats->table.times); 1337 }break; 1338 case ComputeTimingInfoKind_Shader:{ 1339 stats->table.times[stats_index][info.shader] += (f32)info.timer_count / 1.0e9f; 1340 seen_info_test |= (1u << info.shader); 1341 }break; 1342 case ComputeTimingInfoKind_RF_Data:{ 1343 stats->latest_rf_index = (stats->latest_rf_index + 1) % countof(stats->table.rf_time_deltas); 1344 f32 delta = (f32)(info.timer_count - stats->last_rf_timer_count) / 1.0e9f; 1345 stats->table.rf_time_deltas[stats->latest_rf_index] = delta; 1346 stats->last_rf_timer_count = info.timer_count; 1347 seen_info_test |= (1 << BeamformerShaderKind_Count); 1348 }break; 1349 } 1350 /* NOTE(rnp): do this at the end so that stats table is always in a consistent state */ 1351 atomic_add_u32(&t->read_index, 1); 1352 } 1353 1354 if (seen_info_test) { 1355 for EachEnumValue(BeamformerShaderKind, shader) { 1356 if (seen_info_test & (1 << shader)) { 1357 f32 sum = 0; 1358 for EachElement(stats->table.times, i) 1359 sum += stats->table.times[i][shader]; 1360 stats->average_times[shader] = sum / countof(stats->table.times); 1361 } 1362 } 1363 1364 if (seen_info_test & (1 << BeamformerShaderKind_Count)) { 1365 f32 sum = 0; 1366 for EachElement(stats->table.rf_time_deltas, i) 1367 sum += stats->table.rf_time_deltas[i]; 1368 stats->rf_time_delta_average = sum / countof(stats->table.rf_time_deltas); 1369 } 1370 } 1371 } 1372 1373 DEBUG_EXPORT BEAMFORMER_COMPLETE_COMPUTE_FN(beamformer_complete_compute) 1374 { 1375 BeamformerCtx *ctx = (BeamformerCtx *)user_context; 1376 BeamformerSharedMemory *sm = ctx->shared_memory; 1377 complete_queue(ctx, &sm->external_work_queue, arena, gl_context); 1378 complete_queue(ctx, ctx->beamform_work_queue, arena, gl_context); 1379 } 1380 1381 function void 1382 beamformer_rf_buffer_allocate(BeamformerRFBuffer *rf, u32 rf_size, b32 nvidia) 1383 { 1384 assert((rf_size % 64) == 0); 1385 if (!nvidia) glUnmapNamedBuffer(rf->ssbo); 1386 glDeleteBuffers(1, &rf->ssbo); 1387 glCreateBuffers(1, &rf->ssbo); 1388 1389 u32 buffer_flags = GL_DYNAMIC_STORAGE_BIT; 1390 if (!nvidia) buffer_flags |= GL_MAP_PERSISTENT_BIT|GL_MAP_WRITE_BIT; 1391 1392 glNamedBufferStorage(rf->ssbo, countof(rf->compute_syncs) * rf_size, 0, buffer_flags); 1393 1394 if (!nvidia) { 1395 u32 access = GL_MAP_PERSISTENT_BIT|GL_MAP_WRITE_BIT|GL_MAP_FLUSH_EXPLICIT_BIT|GL_MAP_UNSYNCHRONIZED_BIT; 1396 rf->buffer = glMapNamedBufferRange(rf->ssbo, 0, (GLsizei)(countof(rf->compute_syncs) * rf_size), access); 1397 } 1398 1399 LABEL_GL_OBJECT(GL_BUFFER, rf->ssbo, s8("Raw_RF_SSBO")); 1400 rf->size = rf_size; 1401 } 1402 1403 DEBUG_EXPORT BEAMFORMER_RF_UPLOAD_FN(beamformer_rf_upload) 1404 { 1405 BeamformerSharedMemory *sm = ctx->shared_memory; 1406 BeamformerSharedMemoryLockKind scratch_lock = BeamformerSharedMemoryLockKind_ScratchSpace; 1407 BeamformerSharedMemoryLockKind upload_lock = BeamformerSharedMemoryLockKind_UploadRF; 1408 1409 u64 rf_block_rf_size; 1410 if (atomic_load_u32(sm->locks + upload_lock) && 1411 (rf_block_rf_size = atomic_swap_u64(&sm->rf_block_rf_size, 0))) 1412 { 1413 beamformer_shared_memory_take_lock(ctx->shared_memory, (i32)scratch_lock, (u32)-1); 1414 1415 BeamformerRFBuffer *rf = ctx->rf_buffer; 1416 BeamformerParameterBlock *b = beamformer_parameter_block(sm, (u32)(rf_block_rf_size >> 32ULL)); 1417 BeamformerParameters *bp = &b->parameters; 1418 BeamformerDataKind data_kind = b->pipeline.data_kind; 1419 1420 b32 nvidia = gl_parameters.vendor_id == GLVendor_NVIDIA; 1421 1422 rf->active_rf_size = (u32)round_up_to(rf_block_rf_size & 0xFFFFFFFFULL, 64); 1423 if unlikely(rf->size < rf->active_rf_size) 1424 beamformer_rf_buffer_allocate(rf, rf->active_rf_size, nvidia); 1425 1426 u32 slot = rf->insertion_index++ % countof(rf->compute_syncs); 1427 1428 /* NOTE(rnp): if the rest of the code is functioning then the first 1429 * time the compute thread processes an upload it must have gone 1430 * through this path. therefore it is safe to spin until it gets processed */ 1431 spin_wait(atomic_load_u64(rf->upload_syncs + slot)); 1432 1433 if (atomic_load_u64(rf->compute_syncs + slot)) { 1434 GLenum sync_result = glClientWaitSync(rf->compute_syncs[slot], 0, 1000000000); 1435 if (sync_result == GL_TIMEOUT_EXPIRED || sync_result == GL_WAIT_FAILED) { 1436 // TODO(rnp): what do? 1437 } 1438 glDeleteSync(rf->compute_syncs[slot]); 1439 } 1440 1441 u32 size = bp->channel_count * bp->acquisition_count * bp->sample_count * beamformer_data_kind_byte_size[data_kind]; 1442 u8 *data = beamformer_shared_memory_scratch_arena(sm).beg; 1443 1444 if (nvidia) glNamedBufferSubData(rf->ssbo, slot * rf->active_rf_size, (i32)size, data); 1445 else memory_copy_non_temporal(rf->buffer + slot * rf->active_rf_size, data, size); 1446 store_fence(); 1447 1448 beamformer_shared_memory_release_lock(ctx->shared_memory, (i32)scratch_lock); 1449 post_sync_barrier(ctx->shared_memory, upload_lock); 1450 1451 if (!nvidia) 1452 glFlushMappedNamedBufferRange(rf->ssbo, slot * rf->active_rf_size, (i32)rf->active_rf_size); 1453 1454 atomic_store_u64(rf->upload_syncs + slot, glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0)); 1455 atomic_store_u64(rf->compute_syncs + slot, 0); 1456 1457 os_wake_all_waiters(ctx->compute_worker_sync); 1458 1459 ComputeTimingInfo info = {.kind = ComputeTimingInfoKind_RF_Data}; 1460 glGetQueryObjectui64v(rf->data_timestamp_query, GL_QUERY_RESULT, &info.timer_count); 1461 glQueryCounter(rf->data_timestamp_query, GL_TIMESTAMP); 1462 push_compute_timing_info(ctx->compute_timing_table, info); 1463 } 1464 } 1465 1466 function void 1467 beamformer_queue_compute(BeamformerCtx *ctx, BeamformerFrame *frame, u32 parameter_block) 1468 { 1469 BeamformerSharedMemory *sm = ctx->shared_memory; 1470 BeamformerSharedMemoryLockKind dispatch_lock = BeamformerSharedMemoryLockKind_DispatchCompute; 1471 if (!sm->live_imaging_parameters.active && beamformer_shared_memory_take_lock(sm, (i32)dispatch_lock, 0)) 1472 { 1473 BeamformWork *work = beamform_work_queue_push(ctx->beamform_work_queue); 1474 BeamformerViewPlaneTag tag = frame ? frame->view_plane_tag : 0; 1475 if (fill_frame_compute_work(ctx, work, tag, parameter_block, 0)) 1476 beamform_work_queue_push_commit(ctx->beamform_work_queue); 1477 } 1478 os_wake_all_waiters(&ctx->compute_worker.sync_variable); 1479 } 1480 1481 #include "ui.c" 1482 1483 function void 1484 beamformer_process_input_events(BeamformerCtx *ctx, BeamformerInput *input, 1485 BeamformerInputEvent *events, u32 event_count) 1486 { 1487 for (u32 index = 0; index < event_count; index++) { 1488 BeamformerInputEvent *event = events + index; 1489 switch (event->kind) { 1490 1491 case BeamformerInputEventKind_ExecutableReload:{ 1492 ui_init(ctx, ctx->ui_backing_store); 1493 1494 #if BEAMFORMER_RENDERDOC_HOOKS 1495 start_frame_capture = input->renderdoc_start_frame_capture; 1496 end_frame_capture = input->renderdoc_end_frame_capture; 1497 #endif 1498 }break; 1499 1500 case BeamformerInputEventKind_FileEvent:{ 1501 BeamformerFileReloadContext *frc = event->file_watch_user_context; 1502 switch (frc->kind) { 1503 case BeamformerFileReloadKind_Shader:{ 1504 BeamformerShaderReloadContext *src = frc->shader_reload_context; 1505 BeamformerShaderKind kind = beamformer_reloadable_shader_kinds[src->reloadable_info_index]; 1506 beamformer_reload_shader(ctx, src, ctx->arena, beamformer_shader_names[kind]); 1507 }break; 1508 case BeamformerFileReloadKind_ComputeShader:{ 1509 BeamformerSharedMemory *sm = ctx->shared_memory; 1510 u32 reserved_blocks = sm->reserved_parameter_blocks; 1511 1512 for (u32 block = 0; block < reserved_blocks; block++) { 1513 BeamformerComputePlan *cp = beamformer_compute_plan_for_block(&ctx->compute_context, block, 0); 1514 for (u32 slot = 0; slot < cp->pipeline.shader_count; slot++) { 1515 i32 shader_index = beamformer_shader_reloadable_index_by_shader[cp->pipeline.shaders[slot]]; 1516 if (beamformer_reloadable_shader_kinds[shader_index] == frc->compute_shader_kind) 1517 atomic_or_u32(&cp->dirty_programs, 1 << slot); 1518 } 1519 } 1520 1521 if (ctx->latest_frame) 1522 beamformer_queue_compute(ctx, ctx->latest_frame, ctx->latest_frame->parameter_block); 1523 }break; 1524 InvalidDefaultCase; 1525 } 1526 }break; 1527 1528 InvalidDefaultCase; 1529 } 1530 } 1531 } 1532 1533 BEAMFORMER_EXPORT void 1534 beamformer_frame_step(BeamformerInput *input) 1535 { 1536 BeamformerCtx *ctx = BeamformerContextMemory(input->memory); 1537 1538 u64 current_time = os_timer_count(); 1539 dt_for_frame = (f64)(current_time - ctx->frame_timestamp) / os_system_info()->timer_frequency; 1540 ctx->frame_timestamp = current_time; 1541 1542 if (IsWindowResized()) { 1543 ctx->window_size.h = GetScreenHeight(); 1544 ctx->window_size.w = GetScreenWidth(); 1545 } 1546 1547 coalesce_timing_table(ctx->compute_timing_table, ctx->compute_shader_stats); 1548 1549 beamformer_process_input_events(ctx, input, input->event_queue, input->event_count); 1550 1551 BeamformerSharedMemory *sm = ctx->shared_memory; 1552 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_UploadRF)) 1553 os_wake_all_waiters(&ctx->upload_worker.sync_variable); 1554 if (atomic_load_u32(sm->locks + BeamformerSharedMemoryLockKind_DispatchCompute)) 1555 os_wake_all_waiters(&ctx->compute_worker.sync_variable); 1556 1557 BeamformerFrame *frame = ctx->latest_frame; 1558 BeamformerViewPlaneTag tag = frame? frame->view_plane_tag : 0; 1559 draw_ui(ctx, input, frame, tag); 1560 1561 ctx->frame_view_render_context.updated = 0; 1562 }