Vllm.Config.ModelConfig (VLLM v0.3.0)

@spec _apply_dict_overrides(
  SnakeBridge.Ref.t(),
  term(),
  %{optional(String.t()) => term()},
  keyword()
) :: {:ok, nil} | {:error, Snakepit.Error.t()}

Apply dict overrides, handling both nested configs and dict values.

Parameters

config (term())
overrides (%{optional(String.t()) => term()})

Returns

nil

_get_convert_type(ref, architectures, runner_type, convert, opts \\ [])

@spec _get_convert_type(SnakeBridge.Ref.t(), [String.t()], term(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig._get_convert_type.

Parameters

architectures (list(String.t()))
runner_type (term())
convert (term())

Returns

term()

_get_default_convert_type(ref, architectures, runner_type, opts \\ [])

@spec _get_default_convert_type(SnakeBridge.Ref.t(), [String.t()], term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig._get_default_convert_type.

Parameters

architectures (list(String.t()))
runner_type (term())

Returns

term()

_get_default_runner_type(ref, architectures, opts \\ [])

@spec _get_default_runner_type(SnakeBridge.Ref.t(), [String.t()], keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig._get_default_runner_type.

Parameters

architectures (list(String.t()))

Returns

term()

_get_encoder_config(ref, opts \\ [])

@spec _get_encoder_config(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig._get_encoder_config.

Returns

term()

_get_runner_type(ref, architectures, runner, opts \\ [])

@spec _get_runner_type(SnakeBridge.Ref.t(), [String.t()], term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig._get_runner_type.

Parameters

architectures (list(String.t()))
runner (term())

Returns

term()

_get_transformers_backend_cls(ref, opts \\ [])

@spec _get_transformers_backend_cls(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, String.t()} | {:error, Snakepit.Error.t()}

Determine which Transformers modeling backend class will be used if

model_impl is set to transformers or auto.

Returns

String.t()

_lowercase_tokenizer_mode(ref, tokenizer_mode, opts \\ [])

@spec _lowercase_tokenizer_mode(SnakeBridge.Ref.t(), String.t(), keyword()) ::
  {:ok, String.t()} | {:error, Snakepit.Error.t()}

Python method ModelConfig._lowercase_tokenizer_mode.

Parameters

tokenizer_mode (String.t())

Returns

String.t()

_skip_none_validation(ref, value, handler, opts \\ [])

@spec _skip_none_validation(SnakeBridge.Ref.t(), term(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Skip validation if the value is None when initialisation is delayed.

Parameters

value (term())
handler (term())

Returns

term()

_try_verify_and_update_model_config(ref, opts \\ [])

@spec _try_verify_and_update_model_config(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig._try_verify_and_update_model_config.

Returns

term()

_update_nested(ref, target, updates, opts \\ [])

@spec _update_nested(
  SnakeBridge.Ref.t(),
  term(),
  %{optional(String.t()) => term()},
  keyword()
) ::
  {:ok, nil} | {:error, Snakepit.Error.t()}

Recursively updates a config or dict with nested updates.

Parameters

target (term())
updates (%{optional(String.t()) => term()})

Returns

nil

_verify_bnb_config(ref, opts \\ [])

@spec _verify_bnb_config(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, nil} | {:error, Snakepit.Error.t()}

The current version of bitsandbytes (0.46.1) with 8-bit models does not

yet support CUDA graph.

TODO Remove this when bitsandbytes supports.

Returns

nil

_verify_cuda_graph(ref, opts \\ [])

@spec _verify_cuda_graph(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, nil} | {:error, Snakepit.Error.t()}

Python method ModelConfig._verify_cuda_graph.

Returns

nil

_verify_quantization(ref, opts \\ [])

@spec _verify_quantization(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, nil} | {:error, Snakepit.Error.t()}

Python method ModelConfig._verify_quantization.

Returns

nil

_verify_with_expert_parallelism(ref, opts \\ [])

@spec _verify_with_expert_parallelism(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, nil} | {:error, Snakepit.Error.t()}

Python method ModelConfig._verify_with_expert_parallelism.

Returns

nil

allow_deprecated_quantization(ref)

@spec allow_deprecated_quantization(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

allowed_local_media_path(ref)

@spec allowed_local_media_path(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

allowed_media_domains(ref)

@spec allowed_media_domains(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

architecture(ref)

@spec architecture(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

architectures(ref)

@spec architectures(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

attn_type(ref)

@spec attn_type(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

code_revision(ref)

@spec code_revision(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

compute_hash(ref, opts \\ [])

@spec compute_hash(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, String.t()} | {:error, Snakepit.Error.t()}

WARNING: Whenever a new field is added to this config,

ensure that it is included in the factors list if it affects the computation graph.

Provide a hash that uniquely identifies all the configs that affect the structure of the computation graph from input ids/embeddings to the final hidden states, excluding anything before input ids/embeddings and after the final hidden states.

Returns

String.t()

config_format(ref)

@spec config_format(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

convert(ref)

@spec convert(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

disable_cascade_attn(ref)

@spec disable_cascade_attn(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

disable_sliding_window(ref)

@spec disable_sliding_window(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

dtype(ref)

@spec dtype(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

embedding_size(ref)

@spec embedding_size(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

enable_mm_embeds(ref)

@spec enable_mm_embeds(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

enable_prompt_embeds(ref)

@spec enable_prompt_embeds(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

enable_return_routed_experts(ref)

@spec enable_return_routed_experts(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

enable_sleep_mode(ref)

@spec enable_sleep_mode(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

enforce_eager(ref)

@spec enforce_eager(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

generation_config(ref)

@spec generation_config(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

get_and_verify_max_len(ref, max_model_len, opts \\ [])

@spec get_and_verify_max_len(SnakeBridge.Ref.t(), integer(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_and_verify_max_len.

Parameters

max_model_len (integer())

Returns

term()

get_diff_sampling_param(ref, opts \\ [])

@spec get_diff_sampling_param(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, %{optional(String.t()) => term()}} | {:error, Snakepit.Error.t()}

This method returns a dictionary containing the non-default sampling

parameters with override_generation_config applied.

The default sampling parameters are:

vLLM's neutral defaults if self.generation_config="vllm"
the model's defaults if self.generation_config="auto"
as defined in generation_config.json if self.generation_config="path/to/generation_config/dir"

Returns

%{optional(String.t()) => term()}

get_head_size(ref, opts \\ [])

@spec get_head_size(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_head_size.

Returns

integer()

get_hidden_size(ref, opts \\ [])

@spec get_hidden_size(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_hidden_size.

Returns

integer()

get_inputs_embeds_size(ref, opts \\ [])

@spec get_inputs_embeds_size(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_inputs_embeds_size.

Returns

integer()

get_layers_start_end_indices(ref, parallel_config, opts \\ [])

@spec get_layers_start_end_indices(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, {integer(), integer()}} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_layers_start_end_indices.

Parameters

parallel_config (term())

Returns

{integer(), integer()}

get_mamba_chunk_size(ref, opts \\ [])

@spec get_mamba_chunk_size(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Returns the mamba chunk size if it exists

Returns

term()

get_model_arch_config(ref, opts \\ [])

@spec get_model_arch_config(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_model_arch_config.

Returns

term()

get_multimodal_config(ref, opts \\ [])

@spec get_multimodal_config(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Get the multimodal configuration of the model.

Raises

ArgumentError - If the model is not multimodal.

Returns

term()

get_num_attention_heads(ref, parallel_config, opts \\ [])

@spec get_num_attention_heads(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_num_attention_heads.

Parameters

parallel_config (term())

Returns

integer()

get_num_experts(ref, opts \\ [])

@spec get_num_experts(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_num_experts.

Returns

integer()

get_num_kv_heads(ref, parallel_config, opts \\ [])

@spec get_num_kv_heads(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, integer()} | {:error, Snakepit.Error.t()}

Returns the number of KV heads per GPU.

Parameters

parallel_config (term())

Returns

integer()

get_num_layers(ref, parallel_config, opts \\ [])

@spec get_num_layers(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_num_layers.

Parameters

parallel_config (term())

Returns

integer()

get_num_layers_by_block_type(ref, parallel_config, args, opts \\ [])

@spec get_num_layers_by_block_type(SnakeBridge.Ref.t(), term(), [term()], keyword()) ::
  {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_num_layers_by_block_type.

Parameters

parallel_config (term())
block_type (term() default: 'attention')

Returns

integer()

get_sliding_window(ref, opts \\ [])

@spec get_sliding_window(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Get the sliding window size from the HF text config if present.

Returns

term()

get_total_num_hidden_layers(ref, opts \\ [])

@spec get_total_num_hidden_layers(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_total_num_hidden_layers.

Returns

integer()

get_total_num_kv_heads(ref, opts \\ [])

@spec get_total_num_kv_heads(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, integer()} | {:error, Snakepit.Error.t()}

Returns the total number of KV heads.

Returns

integer()

get_vocab_size(ref, opts \\ [])

@spec get_vocab_size(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.get_vocab_size.

Returns

integer()

has_inner_state(ref)

@spec has_inner_state(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

has_noops(ref)

@spec has_noops(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

head_dtype(ref)

@spec head_dtype(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

hf_config_path(ref)

@spec hf_config_path(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

hf_token(ref)

@spec hf_token(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

interleave_mm_strings(ref)

@spec interleave_mm_strings(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

io_processor_plugin(ref)

@spec io_processor_plugin(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_attention_free(ref)

@spec is_attention_free(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_chunked_prefill_supported(ref)

@spec is_chunked_prefill_supported(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_cross_encoder(ref)

@spec is_cross_encoder(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_deepseek_mla(ref)

@spec is_deepseek_mla(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_encoder_decoder(ref)

@spec is_encoder_decoder(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_hybrid(ref)

@spec is_hybrid(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_matryoshka(ref)

@spec is_matryoshka(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_mm_prefix_lm(ref)

@spec is_mm_prefix_lm(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_moe(ref)

@spec is_moe(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

is_multimodal_model(ref)

@spec is_multimodal_model(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_multimodal_raw_input_only_model(ref)

@spec is_multimodal_raw_input_only_model(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_pp_supported(ref)

@spec is_pp_supported(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_prefix_caching_supported(ref)

@spec is_prefix_caching_supported(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_quantized(ref)

@spec is_quantized(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

limit_mm_per_prompt(ref)

@spec limit_mm_per_prompt(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

logits_processor_pattern(ref)

@spec logits_processor_pattern(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

logits_processors(ref)

@spec logits_processors(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

logprobs_mode(ref)

@spec logprobs_mode(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

matryoshka_dimensions(ref)

@spec matryoshka_dimensions(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

max_logprobs(ref)

@spec max_logprobs(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

max_model_len(ref)

@spec max_model_len(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

maybe_pull_model_tokenizer_for_runai(ref, model, tokenizer, opts \\ [])

@spec maybe_pull_model_tokenizer_for_runai(
  SnakeBridge.Ref.t(),
  String.t(),
  String.t(),
  keyword()
) :: {:ok, nil} | {:error, Snakepit.Error.t()}

Pull model/tokenizer from Object Storage to temporary

directory when needed.

Parameters

model - Model name or path
tokenizer - Tokenizer name or path

Returns

nil

media_io_kwargs(ref)

@spec media_io_kwargs(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

mm_encoder_attn_backend(ref)

@spec mm_encoder_attn_backend(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

mm_encoder_tp_mode(ref)

@spec mm_encoder_tp_mode(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

mm_processor_cache_gb(ref)

@spec mm_processor_cache_gb(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

mm_processor_cache_type(ref)

@spec mm_processor_cache_type(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

mm_processor_kwargs(ref)

@spec mm_processor_kwargs(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

mm_shm_cache_max_object_size_mb(ref)

@spec mm_shm_cache_max_object_size_mb(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

model(ref)

@spec model(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

model_impl(ref)

@spec model_impl(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

model_weights(ref)

@spec model_weights(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

multimodal_config(ref)

@spec multimodal_config(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

new(dataclass_self__, args, kwargs, opts \\ [])

@spec new(term(), term(), term(), keyword()) ::
  {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Constructs ModelConfig.

Parameters

dataclass_self__ (term())
args (term())
kwargs (term())

override_attention_dtype(ref)

@spec override_attention_dtype(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

pooler_config(ref)

@spec pooler_config(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

quantization(ref)

@spec quantization(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

registry(ref)

@spec registry(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

requires_raw_input_tokens(ref)

@spec requires_raw_input_tokens(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

revision(ref)

@spec revision(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

runner(ref)

@spec runner(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

seed(ref)

@spec seed(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

served_model_name(ref)

@spec served_model_name(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

skip_mm_profiling(ref)

@spec skip_mm_profiling(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

skip_tokenizer_init(ref)

@spec skip_tokenizer_init(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

spec_target_max_model_len(ref)

@spec spec_target_max_model_len(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

supports_mamba_prefix_caching(ref)

@spec supports_mamba_prefix_caching(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

tokenizer(ref)

@spec tokenizer(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

tokenizer_mode(ref)

@spec tokenizer_mode(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

tokenizer_revision(ref)

@spec tokenizer_revision(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

trust_remote_code(ref)

@spec trust_remote_code(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

try_get_generation_config(ref, opts \\ [])

@spec try_get_generation_config(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, %{optional(String.t()) => term()}} | {:error, Snakepit.Error.t()}

This method attempts to retrieve the non-default values of the

generation config for this model.

The generation config can contain information about special tokens, as well as sampling parameters. Which is why this method exists separately to get_diff_sampling_param.

Returns

%{optional(String.t()) => term()}

use_mla(ref)

@spec use_mla(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

use_sep_token(ref)

@spec use_sep_token(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

uses_alibi(ref)

@spec uses_alibi(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

uses_mrope(ref)

@spec uses_mrope(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

uses_xdrope_dim(ref)

@spec uses_xdrope_dim(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

using_transformers_backend(ref, opts \\ [])

@spec using_transformers_backend(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, boolean()} | {:error, Snakepit.Error.t()}

Check if the model is using the Transformers modeling backend class.

Returns

boolean()

validate_model_config_after(ref, opts \\ [])

@spec validate_model_config_after(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, term()} | {:error, Snakepit.Error.t()}

Called after post_init

Returns

term()

validate_quantization_before(ref, value, opts \\ [])

@spec validate_quantization_before(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method ModelConfig.validate_quantization_before.

Parameters

value (term())

Returns

term()

verify_dual_chunk_attention_config(ref, load_config, opts \\ [])

@spec verify_dual_chunk_attention_config(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, nil} | {:error, Snakepit.Error.t()}

Python method ModelConfig.verify_dual_chunk_attention_config.

Parameters

load_config (term())

Returns

nil

verify_with_parallel_config(ref, parallel_config, opts \\ [])

@spec verify_with_parallel_config(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, nil} | {:error, Snakepit.Error.t()}

Python method ModelConfig.verify_with_parallel_config.

Parameters

parallel_config (term())

Returns

nil

video_pruning_rate(ref)

@spec video_pruning_rate(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}