Vllm.PoolingParams (VLLM v0.3.0)

Copy Markdown View Source

API parameters for pooling models.

Attributes:

truncate_prompt_tokens: Controls prompt truncation.
    Set to -1 to use the model's default truncation size.
    Set to k to keep only the last k tokens (left truncation).
    Set to None to disable truncation.
dimensions: Reduce the dimensions of embeddings
    if model support matryoshka representation.
normalize: Deprecated, please use use_activation instead.
softmax: Deprecated, please use use_activation instead.
activation: Deprecated, please use use_activation instead.
use_activation: Whether to apply activation function to
    the classification outputs.

Summary

Types

t()

Functions

_merge_default_parameters(ref, args, opts \\ [])

vLLM: a high-throughput and memory-efficient inference engine for LLMs

_set_default_parameters(ref, model_config, opts \\ [])

vLLM: a high-throughput and memory-efficient inference engine for LLMs

_verify_step_pooling(ref, pooler_config, valid_parameters, opts \\ [])

vLLM: a high-throughput and memory-efficient inference engine for LLMs

_verify_valid_parameters(ref, opts \\ [])

vLLM: a high-throughput and memory-efficient inference engine for LLMs

activation(ref)
all_parameters(ref)
clone(ref, opts \\ [])

Returns a deep copy of the PoolingParams instance.

dimensions(ref)
extra_kwargs(ref)
new(args, opts \\ [])

Initialize self. See help(type(self)) for accurate signature.

normalize(ref)
output_kind(ref)
requires_token_ids(ref)
returned_token_ids(ref)
skip_reading_prefix_cache(ref)
softmax(ref)
step_tag_id(ref)
task(ref)
truncate_prompt_tokens(ref)
use_activation(ref)
valid_parameters(ref)
verify(ref, task, args, opts \\ [])

vLLM: a high-throughput and memory-efficient inference engine for LLMs

Types

t()

@opaque t()

Functions

_merge_default_parameters(ref, args, opts \\ [])

@spec _merge_default_parameters(SnakeBridge.Ref.t(), [term()], keyword()) ::
  {:ok, nil} | {:error, Snakepit.Error.t()}

vLLM: a high-throughput and memory-efficient inference engine for LLMs

Parameters

  • model_config (term() | nil default: None)

Returns

  • nil

_set_default_parameters(ref, model_config, opts \\ [])

@spec _set_default_parameters(SnakeBridge.Ref.t(), term() | nil, keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

vLLM: a high-throughput and memory-efficient inference engine for LLMs

Parameters

  • model_config (term() | nil)

Returns

  • term()

_verify_step_pooling(ref, pooler_config, valid_parameters, opts \\ [])

@spec _verify_step_pooling(SnakeBridge.Ref.t(), term(), [String.t()], keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

vLLM: a high-throughput and memory-efficient inference engine for LLMs

Parameters

  • pooler_config (term())
  • valid_parameters (list(String.t()))

Returns

  • term()

_verify_valid_parameters(ref, opts \\ [])

@spec _verify_valid_parameters(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

vLLM: a high-throughput and memory-efficient inference engine for LLMs

Returns

  • term()

activation(ref)

@spec activation(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

all_parameters(ref)

@spec all_parameters(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

clone(ref, opts \\ [])

@spec clone(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, Vllm.PoolingParamsClass.t()} | {:error, Snakepit.Error.t()}

Returns a deep copy of the PoolingParams instance.

Returns

  • Vllm.PoolingParamsClass.t()

dimensions(ref)

@spec dimensions(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

extra_kwargs(ref)

@spec extra_kwargs(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

new(args, opts \\ [])

@spec new(
  [term()],
  keyword()
) :: {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Initialize self. See help(type(self)) for accurate signature.

Parameters

  • args (term())
  • kwargs (term())

normalize(ref)

@spec normalize(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

output_kind(ref)

@spec output_kind(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

requires_token_ids(ref)

@spec requires_token_ids(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

returned_token_ids(ref)

@spec returned_token_ids(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

skip_reading_prefix_cache(ref)

@spec skip_reading_prefix_cache(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

softmax(ref)

@spec softmax(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

step_tag_id(ref)

@spec step_tag_id(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

task(ref)

@spec task(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

truncate_prompt_tokens(ref)

@spec truncate_prompt_tokens(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

use_activation(ref)

@spec use_activation(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

valid_parameters(ref)

@spec valid_parameters(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

verify(ref, task, args, opts \\ [])

@spec verify(SnakeBridge.Ref.t(), term(), [term()], keyword()) ::
  {:ok, nil} | {:error, Snakepit.Error.t()}

vLLM: a high-throughput and memory-efficient inference engine for LLMs

Parameters

  • task (term())

  • model_config (term() | nil default: None)

Returns

  • nil