Vllm.ModelExecutor.BasevLLMParameter (VLLM v0.3.0)

Copy Markdown View Source

Base parameter for vLLM linear layers. Extends the torch.nn.parameter

by taking in a linear weight loader. Will copy the loaded weight into the parameter when the provided weight loader is called.

Summary

Types

t()

Functions

_assert_and_load(ref, loaded_weight, opts \\ [])

Python method BasevLLMParameter._assert_and_load.

_is_1d_and_scalar(ref, loaded_weight, opts \\ [])

Python method BasevLLMParameter._is_1d_and_scalar.

_shard_id_as_int(ref, shard_id, opts \\ [])

Python method BasevLLMParameter._shard_id_as_int.

data(ref)
device(ref)
dtype(ref)
grad(ref)
grad_fn(ref)
h(ref)
imag(ref)
is_cpu(ref)
is_cuda(ref)
is_ipu(ref)
is_leaf(ref)
is_maia(ref)
is_meta(ref)
is_mkldnn(ref)
is_mps(ref)
is_mtia(ref)
is_nested(ref)
is_quantized(ref)
is_sparse(ref)
is_sparse_csr(ref)
is_vulkan(ref)
is_xla(ref)
is_xpu(ref)
itemsize(ref)
layout(ref)
load_column_parallel_weight(ref, loaded_weight, opts \\ [])

Python method BasevLLMParameter.load_column_parallel_weight.

load_merged_column_weight(ref, loaded_weight, opts \\ [])

Python method BasevLLMParameter.load_merged_column_weight.

load_qkv_weight(ref, loaded_weight, opts \\ [])

Python method BasevLLMParameter.load_qkv_weight.

load_row_parallel_weight(ref, loaded_weight, opts \\ [])

Python method BasevLLMParameter.load_row_parallel_weight.

m_h(ref)
m_t(ref)
name(ref)
names(ref)
nbytes(ref)
ndim(ref)
new(data, weight_loader, opts \\ [])

Initialize the BasevLLMParameter

output_nr(ref)
real(ref)
requires_grad(ref)
retains_grad(ref)
shape(ref)
t(ref)
volatile(ref)
weight_loader(ref)

Types

t()

@opaque t()

Functions

_assert_and_load(ref, loaded_weight, opts \\ [])

@spec _assert_and_load(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method BasevLLMParameter._assert_and_load.

Parameters

  • loaded_weight (term())

Returns

  • term()

_is_1d_and_scalar(ref, loaded_weight, opts \\ [])

@spec _is_1d_and_scalar(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method BasevLLMParameter._is_1d_and_scalar.

Parameters

  • loaded_weight (term())

Returns

  • term()

_shard_id_as_int(ref, shard_id, opts \\ [])

@spec _shard_id_as_int(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method BasevLLMParameter._shard_id_as_int.

Parameters

  • shard_id (term())

Returns

  • integer()

data(ref)

@spec data(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

device(ref)

@spec device(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

dtype(ref)

@spec dtype(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

grad(ref)

@spec grad(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

grad_fn(ref)

@spec grad_fn(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

h(ref)

@spec h(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

imag(ref)

@spec imag(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_cpu(ref)

@spec is_cpu(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_cuda(ref)

@spec is_cuda(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_ipu(ref)

@spec is_ipu(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_leaf(ref)

@spec is_leaf(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_maia(ref)

@spec is_maia(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_meta(ref)

@spec is_meta(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_mkldnn(ref)

@spec is_mkldnn(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_mps(ref)

@spec is_mps(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_mtia(ref)

@spec is_mtia(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_nested(ref)

@spec is_nested(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_quantized(ref)

@spec is_quantized(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_sparse(ref)

@spec is_sparse(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_sparse_csr(ref)

@spec is_sparse_csr(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_vulkan(ref)

@spec is_vulkan(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_xla(ref)

@spec is_xla(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_xpu(ref)

@spec is_xpu(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

itemsize(ref)

@spec itemsize(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

layout(ref)

@spec layout(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

load_column_parallel_weight(ref, loaded_weight, opts \\ [])

@spec load_column_parallel_weight(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method BasevLLMParameter.load_column_parallel_weight.

Parameters

  • loaded_weight (term())

Returns

  • term()

load_merged_column_weight(ref, loaded_weight, opts \\ [])

@spec load_merged_column_weight(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method BasevLLMParameter.load_merged_column_weight.

Parameters

  • loaded_weight (term())
  • kwargs (term())

Returns

  • term()

load_qkv_weight(ref, loaded_weight, opts \\ [])

@spec load_qkv_weight(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method BasevLLMParameter.load_qkv_weight.

Parameters

  • loaded_weight (term())
  • kwargs (term())

Returns

  • term()

load_row_parallel_weight(ref, loaded_weight, opts \\ [])

@spec load_row_parallel_weight(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method BasevLLMParameter.load_row_parallel_weight.

Parameters

  • loaded_weight (term())

Returns

  • term()

m_h(ref)

@spec m_h(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

m_t(ref)

@spec m_t(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

name(ref)

@spec name(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

names(ref)

@spec names(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

nbytes(ref)

@spec nbytes(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

ndim(ref)

@spec ndim(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

new(data, weight_loader, opts \\ [])

@spec new(term(), term(), keyword()) ::
  {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Initialize the BasevLLMParameter

Parameters

  • data - torch tensor with the parameter data
  • weight_loader - weight loader callable

Returns

a torch.nn.parameter

output_nr(ref)

@spec output_nr(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

real(ref)

@spec real(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

requires_grad(ref)

@spec requires_grad(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

retains_grad(ref)

@spec retains_grad(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

shape(ref)

@spec shape(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

t(ref)

@spec t(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

volatile(ref)

@spec volatile(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

weight_loader(ref)

@spec weight_loader(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}