// HIP stream memory operations for the CPU MoE flag handshake. // // HIP WAIT addresses must come from hipMallocSignalMemory, one 64-bit word per allocation. // // ROCm 10 records hipStreamWrite/WaitValue64 into the graph under stream capture, one node per call. // ROCm 7.14 runs them while the stream is being captured but records nothing, so there we add explicit batch-memop nodes and splice them into the stream's dependency set. // Which of the two applies is measured once at runtime (records_native_memops), not read from the version number. // The probe then does a real capture + instantiate + replay through the chosen path. #include #include #include #include #include #include #include #include #include #include "gpu_sync.h" namespace freetoken::cpu_moe { namespace { void hip_check(hipError_t rc, const char* what) { if (rc != hipSuccess) throw std::runtime_error(std::string("CPU MoE HIP flag sync: ") + what + " failed: " + hipGetErrorString(rc)); } // Capability failures must not poison eager fallback or hide a caller's earlier error. struct ProbeErrorScope { ProbeErrorScope() { hip_check(hipPeekAtLastError(), "pending error before probe"); } ~ProbeErrorScope() { (void)hipGetLastError(); } }; hipError_t alloc_signal(uint64_t** out) { void* storage = nullptr; const hipError_t rc = hipExtMallocWithFlags(&storage, sizeof(uint64_t), hipMallocSignalMemory); if (rc == hipSuccess) { *out = static_cast(storage); __atomic_store_n(*out, 0ULL, __ATOMIC_RELEASE); } return rc; } void free_signal(uint64_t* signal) { if (signal != nullptr) (void)hipFree(signal); } volatile int64_t* as_flag(uint64_t* signal) { return reinterpret_cast(signal); } bool capturing(hipStream_t stream) { hipStreamCaptureStatus status = hipStreamCaptureStatusNone; hip_check(hipStreamIsCapturing(stream, &status), "capture status query"); return status == hipStreamCaptureStatusActive; } // Whether this runtime records a native hipStreamWriteValue64 issued under capture as a graph node. bool records_native_memops() { static const bool native = [] { ProbeErrorScope errors; uint64_t* word = nullptr; hipStream_t stream = nullptr; hipGraph_t graph = nullptr; hipError_t rc = alloc_signal(&word); if (rc == hipSuccess) rc = hipStreamCreateWithFlags(&stream, hipStreamNonBlocking); if (rc == hipSuccess) rc = hipStreamBeginCapture(stream, hipStreamCaptureModeRelaxed); const bool capture_started = rc == hipSuccess; if (rc == hipSuccess) rc = hipStreamWriteValue64(stream, word, 1, hipStreamWriteValueDefault); if (capture_started) { const hipError_t end_rc = hipStreamEndCapture(stream, &graph); if (rc == hipSuccess) rc = end_rc; } size_t nodes = 0; if (rc == hipSuccess) rc = hipGraphGetNodes(graph, nullptr, &nodes); if (graph != nullptr) (void)hipGraphDestroy(graph); if (stream != nullptr) { (void)hipStreamSynchronize(stream); (void)hipStreamDestroy(stream); } free_signal(word); return rc == hipSuccess && nodes == 1; }(); return native; } // Appends one batch-memop node to the capture of `stream`; the node keeps a pointer to `ops`. hipError_t add_capture_memop_node(hipStream_t stream, hipStreamBatchMemOpParams* ops, unsigned int count) { hipStreamCaptureStatus status = hipStreamCaptureStatusNone; hipGraph_t graph = nullptr; const hipGraphNode_t* dependencies = nullptr; size_t dependency_count = 0; hipError_t rc = hipStreamGetCaptureInfo_v2(stream, &status, nullptr, &graph, &dependencies, &dependency_count); if (rc != hipSuccess) return rc; if (status != hipStreamCaptureStatusActive || graph == nullptr) return hipErrorIllegalState; hipCtx_t context = nullptr; #if defined(__GNUC__) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" #endif rc = hipCtxGetCurrent(&context); #if defined(__GNUC__) #pragma GCC diagnostic pop #endif if (rc != hipSuccess) return rc; hipBatchMemOpNodeParams params{}; params.ctx = context; params.count = count; params.paramArray = ops; hipGraphNode_t node = nullptr; rc = hipGraphAddBatchMemOpNode(&node, graph, dependencies, dependency_count, ¶ms); if (rc != hipSuccess) return rc; rc = hipStreamUpdateCaptureDependencies(stream, &node, 1, hipStreamSetCaptureDependencies); // EndCapture rejects a node that was added but never joined to the stream. if (rc != hipSuccess && node != nullptr) (void)hipGraphDestroyNode(node); return rc; } hipStreamBatchMemOpParams write_op(uint64_t* address, uint64_t value) { hipStreamBatchMemOpParams op{}; op.writeValue.operation = hipStreamMemOpWriteValue64; op.writeValue.address = reinterpret_cast(address); op.writeValue.value64 = value; return op; } hipStreamBatchMemOpParams wait_op(uint64_t* address) { hipStreamBatchMemOpParams op{}; op.waitValue.operation = hipStreamMemOpWaitValue64; op.waitValue.address = reinterpret_cast(address); op.waitValue.value64 = 1; op.waitValue.flags = hipStreamWaitValueGte; return op; } bool needs_explicit_nodes(hipStream_t stream) { return capturing(stream) && !records_native_memops(); } // node_op is null for a native call, else the parameter slot of an explicit node, which the node keeps a pointer to. // The ops of one batch node run in parallel, so each write gets its own node and consecutive writes stay in order. hipError_t write_value(hipStream_t stream, uint64_t* address, uint64_t value, hipStreamBatchMemOpParams* node_op) { if (node_op == nullptr) return hipStreamWriteValue64(stream, address, value, hipStreamWriteValueDefault); *node_op = write_op(address, value); return add_capture_memop_node(stream, node_op, 1); } hipError_t wait_done(hipStream_t stream, uint64_t* done, hipStreamBatchMemOpParams* node_op) { if (node_op == nullptr) return hipStreamWaitValue64(stream, done, 1, hipStreamWaitValueGte, UINT64_MAX); *node_op = wait_op(done); return add_capture_memop_node(stream, node_op, 1); } } // namespace struct GpuSync::Impl { uint64_t* ready = nullptr; // signal memory uint64_t* done = nullptr; std::map publication_events; // Explicit nodes keep these addresses, so one parameter pair per distinct task, never moved (std::map nodes are stable). std::map> submit_ops; // done = 0, ready = task hipStreamBatchMemOpParams wait_op{}; }; // Locked fetch can use the signal words with a kernel wait when memops are unavailable. GpuSync::GpuSync(bool require_memops) : impl_(std::make_unique()) { hipError_t rc = alloc_signal(&impl_->ready); if (rc == hipSuccess) rc = alloc_signal(&impl_->done); if (rc != hipSuccess) { free_signal(impl_->ready); free_signal(impl_->done); hip_check(rc, "signal allocation"); } // Measure now: the measurement captures and allocates, which a graph capture in progress would reject. if (require_memops) (void)records_native_memops(); } GpuSync::~GpuSync() { for (const auto& [task, event] : impl_->publication_events) if (event != nullptr) (void)hipEventDestroy(event); free_signal(impl_->ready); free_signal(impl_->done); } bool GpuSync::probe(uintptr_t /*stream*/) { ProbeErrorScope errors; int device = 0; int can_wait = 0; if (hipGetDevice(&device) != hipSuccess || hipDeviceGetAttribute(&can_wait, hipDeviceAttributeCanUseStreamWaitValue, device) != hipSuccess || can_wait == 0) return false; const bool explicit_nodes = !records_native_memops(); uint64_t* ready = nullptr; uint64_t* done = nullptr; hipStream_t stream = nullptr; hipGraph_t graph = nullptr; hipGraphExec_t graph_exec = nullptr; std::array node_ops{}; hipError_t rc = alloc_signal(&ready); if (rc == hipSuccess) rc = alloc_signal(&done); if (rc == hipSuccess) rc = hipStreamCreateWithFlags(&stream, hipStreamNonBlocking); // A runtime that runs the WAIT during capture instead of recording it must not block here. if (rc == hipSuccess) __atomic_store_n(done, 1ULL, __ATOMIC_RELEASE); if (rc == hipSuccess) rc = hipStreamBeginCapture(stream, hipStreamCaptureModeRelaxed); const bool capture_started = rc == hipSuccess; if (rc == hipSuccess) rc = write_value(stream, ready, 1, explicit_nodes ? &node_ops[0] : nullptr); if (rc == hipSuccess) rc = wait_done(stream, done, explicit_nodes ? &node_ops[1] : nullptr); // End the capture even on failure so its stream and graph are not left active during cleanup. if (capture_started) { const hipError_t end_rc = hipStreamEndCapture(stream, &graph); if (rc == hipSuccess) rc = end_rc; } if (rc == hipSuccess) rc = hipStreamSynchronize(stream); size_t nodes = 0; if (rc == hipSuccess) rc = hipGraphGetNodes(graph, nullptr, &nodes); if (rc == hipSuccess && nodes != 2) rc = hipErrorNotSupported; if (rc == hipSuccess) rc = hipGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0); bool served = false; if (rc == hipSuccess) { __atomic_store_n(ready, 0ULL, __ATOMIC_RELEASE); __atomic_store_n(done, 0ULL, __ATOMIC_RELEASE); std::thread responder([&] { const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); while (__atomic_load_n(ready, __ATOMIC_ACQUIRE) == 0 && std::chrono::steady_clock::now() < deadline) std::this_thread::yield(); served = __atomic_load_n(ready, __ATOMIC_ACQUIRE) == 1; // The probe graph never resets done, so this also releases a launch that arrives after the deadline. __atomic_store_n(done, 1ULL, __ATOMIC_RELEASE); }); rc = hipGraphLaunch(graph_exec, stream); if (rc == hipSuccess) rc = hipStreamSynchronize(stream); responder.join(); } const bool ok = rc == hipSuccess && served; if (graph_exec != nullptr) (void)hipGraphExecDestroy(graph_exec); if (graph != nullptr) (void)hipGraphDestroy(graph); if (stream != nullptr) (void)hipStreamDestroy(stream); free_signal(ready); free_signal(done); return ok; } void GpuSync::gpu_submit(uintptr_t stream, uint64_t task) { auto s = reinterpret_cast(stream); hipStreamBatchMemOpParams* ops = needs_explicit_nodes(s) ? impl_->submit_ops[task].data() : nullptr; // Keep each task's event alive across graph capture, replay and recapture. auto& publication = impl_->publication_events[task]; if (publication == nullptr) hip_check(hipEventCreateWithFlags(&publication, hipEventDisableTiming | hipEventReleaseToSystem), "create input publication event"); // External recording retains the D2H publication point in every graph replay. hip_check(hipEventRecordWithFlags(publication, s, hipEventRecordExternal), "publish input copies"); // Reset done before raising ready, so the coordinator's completion for this step can never be wiped. hip_check(write_value(s, impl_->done, 0, ops), "write done"); hip_check(write_value(s, impl_->ready, task, ops ? ops + 1 : nullptr), "write ready"); } void GpuSync::gpu_wait(uintptr_t stream) { auto s = reinterpret_cast(stream); hip_check(wait_done(s, impl_->done, needs_explicit_nodes(s) ? &impl_->wait_op : nullptr), "wait done"); } bool GpuSync::host_func_capture_safe() { return false; } uint64_t GpuSync::cpu_poll() { const int64_t task = flag_load_acquire(as_flag(impl_->ready)); if (task == 0) return 0; flag_store_release(as_flag(impl_->ready), 0); return static_cast(task); } void GpuSync::cpu_done() { flag_store_release(as_flag(impl_->done), 1); } bool GpuSync::pending() const { return flag_load_acquire(as_flag(impl_->ready)) != 0 && flag_load_acquire(as_flag(impl_->done)) == 0; } uintptr_t GpuSync::ready_word() const { return reinterpret_cast(impl_->ready); } uintptr_t GpuSync::done_word() const { return reinterpret_cast(impl_->done); } } // namespace freetoken::cpu_moe