Files
ZihaoMuandLZ-QWQ 9cacd0896c fix(moe): publish rocm cpu expert inputs before notification (#764)
* fix(moe): prevent stale cpu expert inputs in rocm graphs

* fix(moe): publish rocm cpu inputs with graph events

* fix(moe): qualify rocm kernel io with ordered copy fallback

* fix(moe): enable locked hybrid staging on rocm

---------

Co-authored-by: LZ-QWQ <lz1830837889@163.com>
2026-11-02 21:29:42 -08:00

287 lines
12 KiB
C++

// HIP stream memory operations for the CPU MoE flag handshake.
//
// HIP WAIT addresses must come from hipMallocSignalMemory, one 64-bit word per allocation.
//
// ROCm 10 records hipStreamWrite/WaitValue64 into the graph under stream capture, one node per call.
// ROCm 7.14 runs them while the stream is being captured but records nothing, so there we add explicit batch-memop nodes and splice them into the stream's dependency set.
// Which of the two applies is measured once at runtime (records_native_memops), not read from the version number.
// The probe then does a real capture + instantiate + replay through the chosen path.
#include <array>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <map>
#include <stdexcept>
#include <string>
#include <thread>
#include <freetoken/hip_compat.h>
#include "gpu_sync.h"
namespace freetoken::cpu_moe {
namespace {
void hip_check(hipError_t rc, const char* what) {
if (rc != hipSuccess)
throw std::runtime_error(std::string("CPU MoE HIP flag sync: ") + what + " failed: " + hipGetErrorString(rc));
}
// Capability failures must not poison eager fallback or hide a caller's earlier error.
struct ProbeErrorScope {
ProbeErrorScope() { hip_check(hipPeekAtLastError(), "pending error before probe"); }
~ProbeErrorScope() { (void)hipGetLastError(); }
};
hipError_t alloc_signal(uint64_t** out) {
void* storage = nullptr;
const hipError_t rc = hipExtMallocWithFlags(&storage, sizeof(uint64_t), hipMallocSignalMemory);
if (rc == hipSuccess) {
*out = static_cast<uint64_t*>(storage);
__atomic_store_n(*out, 0ULL, __ATOMIC_RELEASE);
}
return rc;
}
void free_signal(uint64_t* signal) {
if (signal != nullptr) (void)hipFree(signal);
}
volatile int64_t* as_flag(uint64_t* signal) { return reinterpret_cast<volatile int64_t*>(signal); }
bool capturing(hipStream_t stream) {
hipStreamCaptureStatus status = hipStreamCaptureStatusNone;
hip_check(hipStreamIsCapturing(stream, &status), "capture status query");
return status == hipStreamCaptureStatusActive;
}
// Whether this runtime records a native hipStreamWriteValue64 issued under capture as a graph node.
bool records_native_memops() {
static const bool native = [] {
ProbeErrorScope errors;
uint64_t* word = nullptr;
hipStream_t stream = nullptr;
hipGraph_t graph = nullptr;
hipError_t rc = alloc_signal(&word);
if (rc == hipSuccess) rc = hipStreamCreateWithFlags(&stream, hipStreamNonBlocking);
if (rc == hipSuccess) rc = hipStreamBeginCapture(stream, hipStreamCaptureModeRelaxed);
const bool capture_started = rc == hipSuccess;
if (rc == hipSuccess) rc = hipStreamWriteValue64(stream, word, 1, hipStreamWriteValueDefault);
if (capture_started) {
const hipError_t end_rc = hipStreamEndCapture(stream, &graph);
if (rc == hipSuccess) rc = end_rc;
}
size_t nodes = 0;
if (rc == hipSuccess) rc = hipGraphGetNodes(graph, nullptr, &nodes);
if (graph != nullptr) (void)hipGraphDestroy(graph);
if (stream != nullptr) {
(void)hipStreamSynchronize(stream);
(void)hipStreamDestroy(stream);
}
free_signal(word);
return rc == hipSuccess && nodes == 1;
}();
return native;
}
// Appends one batch-memop node to the capture of `stream`; the node keeps a pointer to `ops`.
hipError_t add_capture_memop_node(hipStream_t stream, hipStreamBatchMemOpParams* ops, unsigned int count) {
hipStreamCaptureStatus status = hipStreamCaptureStatusNone;
hipGraph_t graph = nullptr;
const hipGraphNode_t* dependencies = nullptr;
size_t dependency_count = 0;
hipError_t rc = hipStreamGetCaptureInfo_v2(stream, &status, nullptr, &graph, &dependencies, &dependency_count);
if (rc != hipSuccess) return rc;
if (status != hipStreamCaptureStatusActive || graph == nullptr) return hipErrorIllegalState;
hipCtx_t context = nullptr;
#if defined(__GNUC__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
#endif
rc = hipCtxGetCurrent(&context);
#if defined(__GNUC__)
#pragma GCC diagnostic pop
#endif
if (rc != hipSuccess) return rc;
hipBatchMemOpNodeParams params{};
params.ctx = context;
params.count = count;
params.paramArray = ops;
hipGraphNode_t node = nullptr;
rc = hipGraphAddBatchMemOpNode(&node, graph, dependencies, dependency_count, &params);
if (rc != hipSuccess) return rc;
rc = hipStreamUpdateCaptureDependencies(stream, &node, 1, hipStreamSetCaptureDependencies);
// EndCapture rejects a node that was added but never joined to the stream.
if (rc != hipSuccess && node != nullptr) (void)hipGraphDestroyNode(node);
return rc;
}
hipStreamBatchMemOpParams write_op(uint64_t* address, uint64_t value) {
hipStreamBatchMemOpParams op{};
op.writeValue.operation = hipStreamMemOpWriteValue64;
op.writeValue.address = reinterpret_cast<hipDeviceptr_t>(address);
op.writeValue.value64 = value;
return op;
}
hipStreamBatchMemOpParams wait_op(uint64_t* address) {
hipStreamBatchMemOpParams op{};
op.waitValue.operation = hipStreamMemOpWaitValue64;
op.waitValue.address = reinterpret_cast<hipDeviceptr_t>(address);
op.waitValue.value64 = 1;
op.waitValue.flags = hipStreamWaitValueGte;
return op;
}
bool needs_explicit_nodes(hipStream_t stream) { return capturing(stream) && !records_native_memops(); }
// node_op is null for a native call, else the parameter slot of an explicit node, which the node keeps a pointer to.
// The ops of one batch node run in parallel, so each write gets its own node and consecutive writes stay in order.
hipError_t write_value(hipStream_t stream, uint64_t* address, uint64_t value, hipStreamBatchMemOpParams* node_op) {
if (node_op == nullptr) return hipStreamWriteValue64(stream, address, value, hipStreamWriteValueDefault);
*node_op = write_op(address, value);
return add_capture_memop_node(stream, node_op, 1);
}
hipError_t wait_done(hipStream_t stream, uint64_t* done, hipStreamBatchMemOpParams* node_op) {
if (node_op == nullptr) return hipStreamWaitValue64(stream, done, 1, hipStreamWaitValueGte, UINT64_MAX);
*node_op = wait_op(done);
return add_capture_memop_node(stream, node_op, 1);
}
} // namespace
struct GpuSync::Impl {
uint64_t* ready = nullptr; // signal memory
uint64_t* done = nullptr;
std::map<uint64_t, hipEvent_t> publication_events;
// Explicit nodes keep these addresses, so one parameter pair per distinct task, never moved (std::map nodes are stable).
std::map<uint64_t, std::array<hipStreamBatchMemOpParams, 2>> submit_ops; // done = 0, ready = task
hipStreamBatchMemOpParams wait_op{};
};
// Locked fetch can use the signal words with a kernel wait when memops are unavailable.
GpuSync::GpuSync(bool require_memops) : impl_(std::make_unique<Impl>()) {
hipError_t rc = alloc_signal(&impl_->ready);
if (rc == hipSuccess) rc = alloc_signal(&impl_->done);
if (rc != hipSuccess) {
free_signal(impl_->ready);
free_signal(impl_->done);
hip_check(rc, "signal allocation");
}
// Measure now: the measurement captures and allocates, which a graph capture in progress would reject.
if (require_memops) (void)records_native_memops();
}
GpuSync::~GpuSync() {
for (const auto& [task, event] : impl_->publication_events)
if (event != nullptr) (void)hipEventDestroy(event);
free_signal(impl_->ready);
free_signal(impl_->done);
}
bool GpuSync::probe(uintptr_t /*stream*/) {
ProbeErrorScope errors;
int device = 0;
int can_wait = 0;
if (hipGetDevice(&device) != hipSuccess ||
hipDeviceGetAttribute(&can_wait, hipDeviceAttributeCanUseStreamWaitValue, device) != hipSuccess || can_wait == 0)
return false;
const bool explicit_nodes = !records_native_memops();
uint64_t* ready = nullptr;
uint64_t* done = nullptr;
hipStream_t stream = nullptr;
hipGraph_t graph = nullptr;
hipGraphExec_t graph_exec = nullptr;
std::array<hipStreamBatchMemOpParams, 2> node_ops{};
hipError_t rc = alloc_signal(&ready);
if (rc == hipSuccess) rc = alloc_signal(&done);
if (rc == hipSuccess) rc = hipStreamCreateWithFlags(&stream, hipStreamNonBlocking);
// A runtime that runs the WAIT during capture instead of recording it must not block here.
if (rc == hipSuccess) __atomic_store_n(done, 1ULL, __ATOMIC_RELEASE);
if (rc == hipSuccess) rc = hipStreamBeginCapture(stream, hipStreamCaptureModeRelaxed);
const bool capture_started = rc == hipSuccess;
if (rc == hipSuccess) rc = write_value(stream, ready, 1, explicit_nodes ? &node_ops[0] : nullptr);
if (rc == hipSuccess) rc = wait_done(stream, done, explicit_nodes ? &node_ops[1] : nullptr);
// End the capture even on failure so its stream and graph are not left active during cleanup.
if (capture_started) {
const hipError_t end_rc = hipStreamEndCapture(stream, &graph);
if (rc == hipSuccess) rc = end_rc;
}
if (rc == hipSuccess) rc = hipStreamSynchronize(stream);
size_t nodes = 0;
if (rc == hipSuccess) rc = hipGraphGetNodes(graph, nullptr, &nodes);
if (rc == hipSuccess && nodes != 2) rc = hipErrorNotSupported;
if (rc == hipSuccess) rc = hipGraphInstantiate(&graph_exec, graph, nullptr, nullptr, 0);
bool served = false;
if (rc == hipSuccess) {
__atomic_store_n(ready, 0ULL, __ATOMIC_RELEASE);
__atomic_store_n(done, 0ULL, __ATOMIC_RELEASE);
std::thread responder([&] {
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
while (__atomic_load_n(ready, __ATOMIC_ACQUIRE) == 0 && std::chrono::steady_clock::now() < deadline)
std::this_thread::yield();
served = __atomic_load_n(ready, __ATOMIC_ACQUIRE) == 1;
// The probe graph never resets done, so this also releases a launch that arrives after the deadline.
__atomic_store_n(done, 1ULL, __ATOMIC_RELEASE);
});
rc = hipGraphLaunch(graph_exec, stream);
if (rc == hipSuccess) rc = hipStreamSynchronize(stream);
responder.join();
}
const bool ok = rc == hipSuccess && served;
if (graph_exec != nullptr) (void)hipGraphExecDestroy(graph_exec);
if (graph != nullptr) (void)hipGraphDestroy(graph);
if (stream != nullptr) (void)hipStreamDestroy(stream);
free_signal(ready);
free_signal(done);
return ok;
}
void GpuSync::gpu_submit(uintptr_t stream, uint64_t task) {
auto s = reinterpret_cast<hipStream_t>(stream);
hipStreamBatchMemOpParams* ops = needs_explicit_nodes(s) ? impl_->submit_ops[task].data() : nullptr;
// Keep each task's event alive across graph capture, replay and recapture.
auto& publication = impl_->publication_events[task];
if (publication == nullptr)
hip_check(hipEventCreateWithFlags(&publication, hipEventDisableTiming | hipEventReleaseToSystem),
"create input publication event");
// External recording retains the D2H publication point in every graph replay.
hip_check(hipEventRecordWithFlags(publication, s, hipEventRecordExternal), "publish input copies");
// Reset done before raising ready, so the coordinator's completion for this step can never be wiped.
hip_check(write_value(s, impl_->done, 0, ops), "write done");
hip_check(write_value(s, impl_->ready, task, ops ? ops + 1 : nullptr), "write ready");
}
void GpuSync::gpu_wait(uintptr_t stream) {
auto s = reinterpret_cast<hipStream_t>(stream);
hip_check(wait_done(s, impl_->done, needs_explicit_nodes(s) ? &impl_->wait_op : nullptr), "wait done");
}
bool GpuSync::host_func_capture_safe() { return false; }
uint64_t GpuSync::cpu_poll() {
const int64_t task = flag_load_acquire(as_flag(impl_->ready));
if (task == 0) return 0;
flag_store_release(as_flag(impl_->ready), 0);
return static_cast<uint64_t>(task);
}
void GpuSync::cpu_done() { flag_store_release(as_flag(impl_->done), 1); }
bool GpuSync::pending() const {
return flag_load_acquire(as_flag(impl_->ready)) != 0 && flag_load_acquire(as_flag(impl_->done)) == 0;
}
uintptr_t GpuSync::ready_word() const { return reinterpret_cast<uintptr_t>(impl_->ready); }
uintptr_t GpuSync::done_word() const { return reinterpret_cast<uintptr_t>(impl_->done); }
} // namespace freetoken::cpu_moe