Vllm.ModelExecutor.PackedvLLMParameter (VLLM v0.3.0)

Copy Markdown View Source

Parameter for model weights which are packed on disk.

Example: GPTQ Marlin weights are int4 or int8, packed into int32. Extends the ModelWeightParameter to take in the packed factor, the packed dimension, and optionally, marlin tile size for marlin kernels. Adjusts the shard_size and shard_offset for fused linear layers model weight loading by accounting for packing and optionally, marlin tile size.

Summary

Types

t()

Functions

adjust_shard_indexes_for_packing(ref, shard_size, shard_offset, opts \\ [])

Python method PackedvLLMParameter.adjust_shard_indexes_for_packing.

bitblas_tile_size(ref)
data(ref)
device(ref)
dtype(ref)
grad(ref)
grad_fn(ref)
h(ref)
imag(ref)
input_dim(ref)
is_cpu(ref)
is_cuda(ref)
is_ipu(ref)
is_leaf(ref)
is_maia(ref)
is_meta(ref)
is_mkldnn(ref)
is_mps(ref)
is_mtia(ref)
is_nested(ref)
is_quantized(ref)
is_sparse(ref)
is_sparse_csr(ref)
is_vulkan(ref)
is_xla(ref)
is_xpu(ref)
itemsize(ref)
layout(ref)
m_h(ref)
m_t(ref)
marlin_tile_size(ref)
name(ref)
names(ref)
nbytes(ref)
ndim(ref)
new(packed_factor, packed_dim, args, opts \\ [])

Initialize the BasevLLMParameter

output_dim(ref)
output_nr(ref)
packed_dim(ref)
packed_factor(ref)
real(ref)
requires_grad(ref)
retains_grad(ref)
shape(ref)
t(ref)
volatile(ref)
weight_loader(ref)

Types

t()

@opaque t()

Functions

adjust_shard_indexes_for_packing(ref, shard_size, shard_offset, opts \\ [])

@spec adjust_shard_indexes_for_packing(SnakeBridge.Ref.t(), term(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method PackedvLLMParameter.adjust_shard_indexes_for_packing.

Parameters

  • shard_size (term())
  • shard_offset (term())

Returns

  • term()

bitblas_tile_size(ref)

@spec bitblas_tile_size(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

data(ref)

@spec data(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

device(ref)

@spec device(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

dtype(ref)

@spec dtype(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

grad(ref)

@spec grad(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

grad_fn(ref)

@spec grad_fn(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

h(ref)

@spec h(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

imag(ref)

@spec imag(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

input_dim(ref)

@spec input_dim(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_cpu(ref)

@spec is_cpu(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_cuda(ref)

@spec is_cuda(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_ipu(ref)

@spec is_ipu(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_leaf(ref)

@spec is_leaf(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_maia(ref)

@spec is_maia(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_meta(ref)

@spec is_meta(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_mkldnn(ref)

@spec is_mkldnn(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_mps(ref)

@spec is_mps(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_mtia(ref)

@spec is_mtia(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_nested(ref)

@spec is_nested(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_quantized(ref)

@spec is_quantized(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_sparse(ref)

@spec is_sparse(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_sparse_csr(ref)

@spec is_sparse_csr(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_vulkan(ref)

@spec is_vulkan(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_xla(ref)

@spec is_xla(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_xpu(ref)

@spec is_xpu(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

itemsize(ref)

@spec itemsize(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

layout(ref)

@spec layout(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

m_h(ref)

@spec m_h(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

m_t(ref)

@spec m_t(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

marlin_tile_size(ref)

@spec marlin_tile_size(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

name(ref)

@spec name(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

names(ref)

@spec names(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

nbytes(ref)

@spec nbytes(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

ndim(ref)

@spec ndim(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

new(packed_factor, packed_dim, args, opts \\ [])

@spec new(term(), integer(), [term()], keyword()) ::
  {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Initialize the BasevLLMParameter

Parameters

  • data - torch tensor with the parameter data
  • weight_loader - weight loader callable

Returns

a torch.nn.parameter

output_dim(ref)

@spec output_dim(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

output_nr(ref)

@spec output_nr(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

packed_dim(ref)

@spec packed_dim(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

packed_factor(ref)

@spec packed_factor(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

real(ref)

@spec real(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

requires_grad(ref)

@spec requires_grad(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

retains_grad(ref)

@spec retains_grad(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

shape(ref)

@spec shape(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

t(ref)

@spec t(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

volatile(ref)

@spec volatile(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

weight_loader(ref)

@spec weight_loader(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}