Vllm.Config.SchedulerConfig (VLLM v0.3.0)

Copy Markdown View Source

Scheduler configuration.

Summary

Types

t()

Functions

_skip_none_validation(ref, value, handler, opts \\ [])

Skip validation if the value is None when initialisation is delayed.

async_scheduling(ref)
compute_hash(ref, opts \\ [])

WARNING: Whenever a new field is added to this config,

default_factory(ref, opts \\ [])

Factory method to create SchedulerConfig with default values for InitVars.

default_max_num_batched_tokens(ref)
default_max_num_seqs(ref)
disable_chunked_mm_input(ref)
disable_hybrid_kv_cache_manager(ref)
enable_chunked_prefill(ref)
encoder_cache_size(ref)
get_scheduler_cls(ref, opts \\ [])

Python method SchedulerConfig.get_scheduler_cls.

is_multimodal_model(ref)
long_prefill_token_threshold(ref)
max_long_partial_prefills(ref)
max_num_batched_tokens(ref)
max_num_encoder_input_tokens(ref)
max_num_partial_prefills(ref)
max_num_seqs(ref)
new(dataclass_self__, args, kwargs, opts \\ [])

Constructs SchedulerConfig.

policy(ref)
runner_type(ref)
scheduler_cls(ref)
stream_interval(ref)
verify_max_model_len(ref, max_model_len, opts \\ [])

Python method SchedulerConfig.verify_max_model_len.

Types

t()

@opaque t()

Functions

_skip_none_validation(ref, value, handler, opts \\ [])

@spec _skip_none_validation(SnakeBridge.Ref.t(), term(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Skip validation if the value is None when initialisation is delayed.

Parameters

  • value (term())
  • handler (term())

Returns

  • term()

async_scheduling(ref)

@spec async_scheduling(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

compute_hash(ref, opts \\ [])

@spec compute_hash(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, String.t()} | {:error, Snakepit.Error.t()}

WARNING: Whenever a new field is added to this config,

ensure that it is included in the factors list if it affects the computation graph.

Provide a hash that uniquely identifies all the configs that affect the structure of the computation graph from input ids/embeddings to the final hidden states, excluding anything before input ids/embeddings and after the final hidden states.

Returns

  • String.t()

default_factory(ref, opts \\ [])

@spec default_factory(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Factory method to create SchedulerConfig with default values for InitVars.

Parameters

  • kwargs (term())

Returns

  • term()

default_max_num_batched_tokens(ref)

@spec default_max_num_batched_tokens(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

default_max_num_seqs(ref)

@spec default_max_num_seqs(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

disable_chunked_mm_input(ref)

@spec disable_chunked_mm_input(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

disable_hybrid_kv_cache_manager(ref)

@spec disable_hybrid_kv_cache_manager(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

enable_chunked_prefill(ref)

@spec enable_chunked_prefill(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

encoder_cache_size(ref)

@spec encoder_cache_size(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

get_scheduler_cls(ref, opts \\ [])

@spec get_scheduler_cls(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Python method SchedulerConfig.get_scheduler_cls.

Returns

  • term()

is_multimodal_model(ref)

@spec is_multimodal_model(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

long_prefill_token_threshold(ref)

@spec long_prefill_token_threshold(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

max_long_partial_prefills(ref)

@spec max_long_partial_prefills(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

max_num_batched_tokens(ref)

@spec max_num_batched_tokens(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

max_num_encoder_input_tokens(ref)

@spec max_num_encoder_input_tokens(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

max_num_partial_prefills(ref)

@spec max_num_partial_prefills(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

max_num_seqs(ref)

@spec max_num_seqs(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

new(dataclass_self__, args, kwargs, opts \\ [])

@spec new(term(), term(), term(), keyword()) ::
  {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Constructs SchedulerConfig.

Parameters

  • dataclass_self__ (term())
  • args (term())
  • kwargs (term())

policy(ref)

@spec policy(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

runner_type(ref)

@spec runner_type(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

scheduler_cls(ref)

@spec scheduler_cls(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

stream_interval(ref)

@spec stream_interval(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

verify_max_model_len(ref, max_model_len, opts \\ [])

@spec verify_max_model_len(SnakeBridge.Ref.t(), integer(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method SchedulerConfig.verify_max_model_len.

Parameters

  • max_model_len (integer())

Returns

  • term()