mirror of
https://github.com/Comfy-Org/ComfyUI.git
synced 2026-11-23 01:53:32 +08:00
* Run optimize_for_gemv from pin_modules
Move the resident-weight relayout out of the cast miss path to a hit in
pin_modules, once per residency after the block's prefetch has fully
faulted and resolved. A weight is then never rewritten in place while
its transfer is in flight, and every forward in a step sees one layout.
The resident bytes already carry any baked lowvram patch and a fully
faulted block has no runtime weight functions, so the patch guards go.
* Combine compatible linears at first residency
Models declare comfy.ops.CombinedLinear groups over consecutive VBAR
linears; the dynamic loader loads a group's members back to back so
their allocations are consecutive. When a prefetched block fully hits,
pin_modules concatenates each group's resident weights in place with
cat_gathered, which saves the members' metadata, memmoves the codes
down over it and rewrites the metadata after them in the same layout
interpret_gathered_like would give one weight of the summed rows. The
members' weights are gone after that; the group then takes its turn in
the same optimize_for_gemv pass as a lone module, under its own
comfy_gemv_optimized flag that the model sets. The model executes the
group as one GEMM through CombinedLinear.forward or linear_input_act
and owns the split path when the group declines.
Combining moves bytes across member allocations, so a combined group
hits or misses as a whole: cast_modules_with_vbar faults all of a
group's members when it reaches the first one, and if any of them
missed, the group is dropped and every member reloads its canonical
weight in that same pass. A member's early fault is its pin for the
step, so no pin is taken twice and nothing is refaulted.
* Combine Qwen text-encoder projections with CombinedLinear
Qwen3 attention q/k/v and MLP gate/up, Qwen3.5 GatedDeltaNet qkv/z and GatedAttention q/k/v now run one combined GEMM when the resident weights share a quantized layout, falling back to the split projections otherwise.
* requirements: roll forward comfy-kitchen
* Make CombinedLinear follow its members' full precision setting
Text encoders load with full_precision_mm and CLIP.generate lifts it only
for the generation. The combined weight was built inside that window and
then kept running the quantized kernel for the following encode, which
emits the compute dtype into the encoder's float32 attention mask
("invalid dtype for bias" in SDPA) and silently skips full precision.
Read the flag live from the members and dequantize like they do.
* Resolve, combine and relayout a prefetched block on its first residency
pin_modules only resolved, combined and relaid out a block on a hit, so a
cold step ran the canonical layout eagerly and the first MMA decode of
the relaid weight was captured into the CUDA graph on the next hit.
Gate that work on every module having a VBAR slot and no runtime weight
function instead, and keep the hit requirement only for graph capture.
A miss is now resolved at pin time, so the eager warm pass runs the same
kernels the graph captures.
* Fault a combined group's members once when the group is dropped
cast_modules_with_vbar faults every member of a combined group when it
reaches the first one. If a member missed, the group was dropped in that
same iteration, so the later members saw an uncombined group and faulted
again. VBAR pins are counted and cleanup unpins once, so every partial
miss of a combined group leaked a pin and left those pages unevictable.
* Decline combining members the combined path cannot serve
A member with a per-call weight function or forced cast weights is transformed
on the split path every forward; the combined weight would skip that. A layout
without a linear_input_act kernel returns NotImplemented from the combined
forward, and the callers' split fallback then finds the members' weights gone.
* tests: cat_gathered, CombinedLinear residency combining and Qwen 3.5 decode paths
347 lines
14 KiB
Python
347 lines
14 KiB
Python
import logging
|
|
import threading
|
|
import warnings
|
|
import weakref
|
|
|
|
import comfy_kitchen as ck
|
|
import torch
|
|
|
|
import comfy_aimdo.malloc_graph
|
|
import comfy_aimdo.model_vbar
|
|
import comfy_aimdo.torch
|
|
from comfy.cli_args import args
|
|
import comfy.memory_management
|
|
import comfy.model_management
|
|
import comfy.ops
|
|
from comfy.quant_ops import QuantizedTensor
|
|
|
|
PREFETCH_QUEUES = []
|
|
GRAPH_WARMED_MODULES = weakref.WeakSet()
|
|
GRAPH_CAPTURE_STREAMS = {}
|
|
MALLOC_GRAPHS = {}
|
|
MALLOC_GRAPH_BREAKS = 0
|
|
MALLOC_GRAPH_ROGUES = 0
|
|
MALLOC_GRAPH_USED = False
|
|
|
|
def _malloc_graph_break():
|
|
global MALLOC_GRAPH_BREAKS
|
|
MALLOC_GRAPH_BREAKS += 1
|
|
logging.debug("Comfy model compiler graph break")
|
|
|
|
def malloc_graph_enabled(device):
|
|
return not args.disable_comfy_compiler and comfy.memory_management.aimdo_enabled and comfy.model_management.is_device_cuda(device)
|
|
|
|
class _PauseMallocGraph:
|
|
def __init__(self, sync=False):
|
|
self.sync = sync
|
|
|
|
def __enter__(self):
|
|
graph = MALLOC_GRAPHS.get(threading.get_ident())
|
|
if graph is not None and graph._comfy_active:
|
|
graph.pause(sync=self.sync)
|
|
|
|
def __exit__(self, *args):
|
|
graph = MALLOC_GRAPHS.get(threading.get_ident())
|
|
if graph is not None and graph._comfy_active:
|
|
graph.resume(sync=self.sync)
|
|
|
|
def pause_malloc_graph(sync=False):
|
|
return _PauseMallocGraph(sync)
|
|
|
|
class _MallocGraphScope:
|
|
def __init__(self, device):
|
|
self.device = device
|
|
|
|
def __enter__(self):
|
|
malloc_graph_begin(self.device)
|
|
|
|
def __exit__(self, exc_type, *args):
|
|
if exc_type is None:
|
|
malloc_graph_end()
|
|
else:
|
|
cleanup_malloc_graph()
|
|
|
|
def malloc_graph_scope(device):
|
|
return _MallocGraphScope(device)
|
|
|
|
def malloc_graph_recording():
|
|
return threading.get_ident() in MALLOC_GRAPHS
|
|
|
|
def malloc_graph_begin(device):
|
|
global MALLOC_GRAPH_USED
|
|
if not malloc_graph_enabled(device):
|
|
return
|
|
thread_id = threading.get_ident()
|
|
graph = MALLOC_GRAPHS.get(thread_id)
|
|
if graph is None:
|
|
graph = comfy_aimdo.malloc_graph.record(
|
|
comfy.model_management.current_stream(device), args.assert_graph_breaks
|
|
)
|
|
graph._comfy_cuda_graph_modules = weakref.WeakSet()
|
|
MALLOC_GRAPHS[thread_id] = graph
|
|
else:
|
|
graph.push()
|
|
if hasattr(ck, "set_allocation_context"):
|
|
ck.set_allocation_context(pause_malloc_graph())
|
|
graph._comfy_active = True
|
|
MALLOC_GRAPH_USED = True
|
|
|
|
def malloc_graph_end():
|
|
thread_id = threading.get_ident()
|
|
graph = MALLOC_GRAPHS.get(thread_id)
|
|
if graph is not None and graph._comfy_active:
|
|
if graph.pop():
|
|
_malloc_graph_break()
|
|
graph._comfy_active = False
|
|
|
|
def cleanup_malloc_graph():
|
|
global MALLOC_GRAPH_ROGUES
|
|
|
|
graph = MALLOC_GRAPHS.pop(threading.get_ident(), None)
|
|
if graph is not None:
|
|
if graph._comfy_active:
|
|
graph.abort()
|
|
graph._comfy_active = False
|
|
else:
|
|
comfy.model_management.synchronize()
|
|
for module in graph._comfy_cuda_graph_modules:
|
|
_drop_graph(module)
|
|
MALLOC_GRAPH_ROGUES += graph.rogue_count
|
|
del graph
|
|
|
|
def pin_modules(comfy_modules, device, dtype=None):
|
|
registerable_size = 0
|
|
for s in comfy_modules:
|
|
registerable_size += comfy.memory_management.vram_aligned_size([s.weight, s.bias])
|
|
for param_key in ("weight", "bias"):
|
|
lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
|
|
if lowvram_fn is not None:
|
|
registerable_size += lowvram_fn.memory_required()
|
|
|
|
offload_stream, fully_faulted = comfy.ops.cast_modules_with_vbar(comfy_modules, None, device, None, True, return_faulted=True)
|
|
if not (comfy_modules and comfy_modules[0]._pin_state["fast_disk"]):
|
|
comfy.model_management.ensure_pin_registerable(registerable_size)
|
|
comfy.model_management.sync_stream(device, offload_stream)
|
|
if fully_faulted and dtype is not None:
|
|
for comfy_module in comfy_modules:
|
|
comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)
|
|
# A miss whose modules all have VBAR slots and no runtime weight functions is resolved
|
|
# here rather than in the forward, so the block is combined and relaid out on its first
|
|
# residency and the eager warm pass runs the kernels the graph captures on the next hit.
|
|
elif dtype is not None and all(s._prefetch["signature"] is not None and not getattr(s, param_key + "_function", []) for s in comfy_modules for param_key in ("weight", "bias")):
|
|
for comfy_module in comfy_modules:
|
|
comfy.ops.resolve_cast_module_with_vbar(comfy_module, dtype, device, dtype, None, False, return_weights=False)
|
|
else:
|
|
return offload_stream, fully_faulted
|
|
relayout = []
|
|
for s in comfy_modules:
|
|
if s._v_relayout:
|
|
continue
|
|
s._v_relayout = True
|
|
group = s._combined_linear
|
|
if group is not None and s is group.linears[0]:
|
|
group.prepare(device)
|
|
relayout.append(group)
|
|
relayout.append(s)
|
|
for s in relayout:
|
|
# FIXME: Make this non_gemv_specific.
|
|
if isinstance(s, (comfy.ops.MixedPrecisionOp, comfy.ops.CombinedLinear)) and s.comfy_gemv_optimized and isinstance(s._v_weight, QuantizedTensor) and not s._full_precision_mm:
|
|
s._v_weight = s._v_weight.optimize_for_gemv(comfy_aimdo.torch.aimdo_to_tensor(s._v, device))
|
|
return offload_stream, fully_faulted
|
|
|
|
def cleanup_prefetched_modules(module, comfy_modules):
|
|
# Always drop the _prefetch marker, even if unpinning fails: a stale marker makes cast_bias_weight
|
|
# skip the fresh cast and read the old transfer state. Clean up every module, then raise the first error.
|
|
first_error = None
|
|
for s in comfy_modules:
|
|
prefetch = getattr(s, "_prefetch", None)
|
|
if prefetch is None:
|
|
continue
|
|
try:
|
|
for param_key in ("weight", "bias"):
|
|
lowvram_fn = getattr(s, param_key + "_lowvram_function", None)
|
|
if lowvram_fn is not None:
|
|
lowvram_fn.clear_prepared()
|
|
if prefetch["signature"] is not None:
|
|
comfy_aimdo.model_vbar.vbar_unpin(s._v)
|
|
except Exception as e:
|
|
if first_error is None:
|
|
first_error = e
|
|
finally:
|
|
delattr(s, "_prefetch")
|
|
if getattr(module, "_v_block_faulted", False):
|
|
try:
|
|
comfy_aimdo.model_vbar.vbar_unpin(module._v_block)
|
|
except Exception as e:
|
|
if first_error is None:
|
|
first_error = e
|
|
finally:
|
|
del module._v_block_faulted
|
|
if first_error is not None:
|
|
raise first_error
|
|
|
|
def _drop_graph(module):
|
|
graph = getattr(module, "_comfy_graph", None)
|
|
if graph is None:
|
|
return
|
|
# reset() through the bound method surfaces the allocator's benign
|
|
# "uncaptured free of a captured allocation" as catchable Python warnings;
|
|
# a plain del frees from the C++ dealloc path and spams stderr instead
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter("ignore")
|
|
graph["graph"].reset()
|
|
del module._comfy_graph
|
|
|
|
def cleanup_prefetch_queues():
|
|
global PREFETCH_QUEUES
|
|
global MALLOC_GRAPH_BREAKS
|
|
global MALLOC_GRAPH_ROGUES
|
|
global MALLOC_GRAPH_USED
|
|
|
|
queues = PREFETCH_QUEUES
|
|
# Reset before walking so a failing entry can't leave stale queues behind for every later node.
|
|
PREFETCH_QUEUES = []
|
|
first_error = None
|
|
try:
|
|
cleanup_malloc_graph()
|
|
except Exception as e:
|
|
first_error = e
|
|
for queue in queues:
|
|
# Only the head can have been pinned, as (offload_stream, (module, comfy_modules)). Later
|
|
# entries, and a head that was not reached yet, still hold the raw module or module group.
|
|
head = queue[0] if queue else None
|
|
if not isinstance(head, tuple) or len(head) != 2 or not isinstance(head[1], tuple):
|
|
continue
|
|
prefetched_module, comfy_modules = head[1]
|
|
try:
|
|
cleanup_prefetched_modules(prefetched_module, comfy_modules)
|
|
except Exception as e:
|
|
# Keep unpinning the other queues; report the first failure once they are done.
|
|
if first_error is None:
|
|
first_error = e
|
|
GRAPH_WARMED_MODULES.clear()
|
|
if MALLOC_GRAPH_USED:
|
|
logging.info("Comfy model compiler graph breaks: %d, rogues: %d", MALLOC_GRAPH_BREAKS, MALLOC_GRAPH_ROGUES)
|
|
MALLOC_GRAPH_BREAKS = 0
|
|
MALLOC_GRAPH_ROGUES = 0
|
|
MALLOC_GRAPH_USED = False
|
|
if first_error is not None:
|
|
raise first_error
|
|
|
|
def prefetch_queue_pop(queue, device, module, dtype=None, core=None, enable_graph=False, generator=None, malloc_scope=None):
|
|
malloc_graph = MALLOC_GRAPHS.get(threading.get_ident())
|
|
if malloc_graph is not None and not malloc_graph._comfy_active:
|
|
malloc_graph = None
|
|
enable_graph = enable_graph and malloc_graph is not None and not args.disable_cuda_graphs and comfy.model_management.is_device_cuda(device) and getattr(module, "_v_block", None) is not None
|
|
if queue is None:
|
|
if malloc_graph is not None and malloc_scope is not None:
|
|
if malloc_graph.iterate(malloc_scope if module is not None else None):
|
|
_malloc_graph_break()
|
|
if core is not None:
|
|
core()
|
|
return
|
|
|
|
capture_stream = None
|
|
if enable_graph:
|
|
capture_stream = GRAPH_CAPTURE_STREAMS.get(device)
|
|
if capture_stream is None:
|
|
capture_stream = torch.cuda.Stream(device=device)
|
|
# Keep PyTorch's persistent BLAS workspaces outside the allocation graph.
|
|
malloc_graph.pause()
|
|
with torch.cuda.stream(capture_stream):
|
|
torch.cuda.current_blas_handle()
|
|
one = torch.empty((2, 2), device=device)
|
|
torch.addmm(one[0], one, one)
|
|
malloc_graph.resume()
|
|
GRAPH_CAPTURE_STREAMS[device] = capture_stream
|
|
|
|
signature = None
|
|
graph_hit = False
|
|
graph = getattr(module, "_comfy_graph", None) if enable_graph else None
|
|
if graph is not None:
|
|
signature = comfy_aimdo.model_vbar.vbar_fault(module._v_block)
|
|
if signature is not None:
|
|
module._v_block_faulted = True
|
|
graph_hit = comfy_aimdo.model_vbar.vbar_signature_compare(signature, graph["signature"])
|
|
|
|
if malloc_graph is not None and malloc_scope is not None:
|
|
if malloc_graph.iterate(malloc_scope if module is not None and not graph_hit else None):
|
|
_malloc_graph_break()
|
|
|
|
consumed = queue.pop(0)
|
|
if consumed is not None:
|
|
offload_stream, prefetch_state = consumed
|
|
if offload_stream is not None:
|
|
offload_stream.wait_stream(comfy.model_management.current_stream(device))
|
|
prefetched_module, comfy_modules = prefetch_state
|
|
if comfy_modules is not None:
|
|
cleanup_prefetched_modules(prefetched_module, comfy_modules)
|
|
|
|
if graph_hit:
|
|
queue[0] = (None, (module, []))
|
|
graph["graph"].replay()
|
|
return
|
|
|
|
fully_faulted = False
|
|
prefetch = queue[0]
|
|
if prefetch is not None:
|
|
comfy_modules = []
|
|
prefetch_modules = prefetch if isinstance(prefetch, (list, tuple)) else (prefetch,)
|
|
for root in prefetch_modules:
|
|
for s in root.modules():
|
|
if hasattr(s, "_v"):
|
|
comfy_modules.append(s)
|
|
|
|
# Keep partially pinned groups reachable if pinning raises.
|
|
queue[0] = (None, (module, comfy_modules))
|
|
offload_stream, fully_faulted = pin_modules(comfy_modules, device, dtype)
|
|
queue[0] = (offload_stream, (module, comfy_modules))
|
|
|
|
if core is not None:
|
|
if enable_graph and fully_faulted and module in GRAPH_WARMED_MODULES:
|
|
if signature is None:
|
|
signature = comfy_aimdo.model_vbar.vbar_fault(module._v_block)
|
|
if signature is not None:
|
|
module._v_block_faulted = True
|
|
if signature is not None:
|
|
_drop_graph(module)
|
|
malloc_graph.pause()
|
|
graph = torch.cuda.CUDAGraph()
|
|
if generator is not None:
|
|
graph.register_generator_state(generator)
|
|
malloc_graph.resume()
|
|
# Capture-time VBAR eviction is safe after prior work completes.
|
|
comfy.model_management.synchronize()
|
|
capture_stream.wait_stream(comfy.model_management.current_stream(device))
|
|
malloc_graph.pause(sync=True)
|
|
with malloc_graph.use_stream(capture_stream):
|
|
with torch.cuda.graph(graph, stream=capture_stream, capture_error_mode="thread_local"):
|
|
malloc_graph.resume()
|
|
core()
|
|
malloc_graph.pause()
|
|
malloc_graph.resume(sync=True)
|
|
comfy.model_management.current_stream(device).wait_stream(capture_stream)
|
|
graph.replay()
|
|
module._comfy_graph = {"graph": graph, "signature": signature}
|
|
malloc_graph._comfy_cuda_graph_modules.add(module)
|
|
return
|
|
if capture_stream is None:
|
|
core()
|
|
else:
|
|
capture_stream.wait_stream(comfy.model_management.current_stream(device))
|
|
with torch.cuda.stream(capture_stream), malloc_graph.use_stream(capture_stream):
|
|
core()
|
|
comfy.model_management.current_stream(device).wait_stream(capture_stream)
|
|
GRAPH_WARMED_MODULES.add(module)
|
|
|
|
def make_prefetch_queue(queue, device, transformer_options):
|
|
if (not transformer_options.get("prefetch_dynamic_vbars", False)
|
|
or comfy.model_management.NUM_STREAMS == 0
|
|
or comfy.model_management.is_device_cpu(device)
|
|
or not comfy.model_management.device_supports_non_blocking(device)):
|
|
return None
|
|
|
|
queue = [None] + queue + [None]
|
|
PREFETCH_QUEUES.append(queue)
|
|
return queue
|