Vllm.ForwardContext (VLLM v0.3.0)

ForwardContext(no_compile_layers: dict[str, typing.Any], attn_metadata: dict[str, vllm.v1.attention.backend.AttentionMetadata] | list[dict[str, vllm.v1.attention.backend.AttentionMetadata]], virtual_engine: int, dp_metadata: vllm.forward_context.DPMetadata | None = None, cudagraph_runtime_mode: vllm.config.compilation.CUDAGraphMode = <CUDAGraphMode.NONE: 0>, batch_descriptor: vllm.forward_context.BatchDescriptor | None = None, ubatch_slices: list[vllm.v1.worker.ubatch_utils.UBatchSlice] | None = None, additional_kwargs: dict[str, typing.Any] = <factory>)

Summary

Types

t()

Functions

batch_descriptor(ref)

cudagraph_runtime_mode(ref)

dp_metadata(ref)

new(no_compile_layers, attn_metadata, virtual_engine, args, opts \\ [])

Initialize self. See help(type(self)) for accurate signature.

ubatch_slices(ref)

Types

t()

@opaque t()

Functions

batch_descriptor(ref)

@spec batch_descriptor(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

cudagraph_runtime_mode(ref)

@spec cudagraph_runtime_mode(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

dp_metadata(ref)

@spec dp_metadata(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

new(no_compile_layers, attn_metadata, virtual_engine, args, opts \\ [])

@spec new(%{optional(String.t()) => term()}, term(), integer(), [term()], keyword()) ::
  {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Initialize self. See help(type(self)) for accurate signature.

Parameters

no_compile_layers (%{optional(String.t()) => term()})
attn_metadata (term())
virtual_engine (integer())
dp_metadata (term() default: None)
cudagraph_runtime_mode (term() default: <CUDAGraphMode.NONE: 0>)
batch_descriptor (term() default: None)
ubatch_slices (term() default: None)
additional_kwargs (%{optional(String.t()) => term()} default: <factory>)

ubatch_slices(ref)

@spec ubatch_slices(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}