@spec _create_processing_ctx(SnakeBridge.Ref.t(), term(), [term()], keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method MultiModalRegistry._create_processing_ctx.

Parameters

• model_config (term())
• observability_config (term() default: None)
• tokenizer (term() default: None)

Returns

• term()

@spec _create_processing_info(SnakeBridge.Ref.t(), term(), [term()], keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method MultiModalRegistry._create_processing_info.

Parameters

• model_config (term())
• observability_config (term() default: None)
• tokenizer (term() keyword-only default: None)

Returns

• term()

@spec _extract_mm_options(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Extract multimodal dummy options from model config.

Returns None if no configurable options are found, otherwise returns a mapping of modality names to their dummy options.

Parameters

• model_config (term())

Returns

• term()

@spec _get_model_cls(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method MultiModalRegistry._get_model_cls.

Parameters

• model_config (term())

Returns

• term()

@spec create_processor(SnakeBridge.Ref.t(), term(), [term()], keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Create a multi-modal processor for a specific model and tokenizer.

Parameters

• model_config (term())
• observability_config (term() default: None)
• tokenizer (term() keyword-only default: None)
• cache (term() keyword-only default: None)

Returns

• term()

@spec get_decoder_dummy_data(
  SnakeBridge.Ref.t(),
  term(),
  integer(),
  [term()],
  keyword()
) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Create dummy data for profiling the memory usage of a model.

The model is identified by model_config.

Parameters

• model_config (term())
• seq_len (integer())
• mm_counts (term() default: None)
• cache (term() keyword-only default: None)
• observability_config (term() keyword-only default: None)

Returns

• term()

@spec get_encdec_max_encoder_len(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, integer()} | {:error, Snakepit.Error.t()}

Get the maximum length of the encoder input for encoder-decoder models.

Parameters

• model_config (term())

Returns

• integer()

@spec get_encoder_dummy_data(
  SnakeBridge.Ref.t(),
  term(),
  integer(),
  [term()],
  keyword()
) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Create dummy data for profiling the memory usage of a model.

The model is identified by model_config.

Parameters

• model_config (term())
• seq_len (integer())
• mm_counts (term() default: None)
• cache (term() keyword-only default: None)
• observability_config (term() keyword-only default: None)

Returns

• term()

@spec get_max_tokens_per_item_by_modality(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Get the maximum number of tokens per data item from each modality based

on underlying model configuration.

Parameters

• model_config (term())
• cache (term() keyword-only default: None)
• profiler_limits (term() keyword-only default: None)
• observability_config (term() keyword-only default: None)

Returns

• term()

@spec get_mm_limits_per_prompt(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Get the maximum number of multi-modal input instances for each modality

that are allowed per prompt for a model class.

Parameters

• model_config (term())
• cache (term() keyword-only default: None)
• observability_config (term() keyword-only default: None)

Returns

• term()

@spec new(
  [term()],
  keyword()
) :: {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Initialize self. See help(type(self)) for accurate signature.

Parameters

• args (term())
• kwargs (term())

@spec register_processor(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Register a multi-modal processor to a model class. The processor

is constructed lazily, hence a factory method should be passed.

When the model receives multi-modal data, the provided function is invoked to transform the data into a dictionary of model inputs.

Parameters

• processor (term())
• info (term() keyword-only, required)
• dummy_inputs (term() keyword-only, required)

Returns

• term()

@spec supports_multimodal_inputs(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, boolean()} | {:error, Snakepit.Error.t()}

Checks if the model supports multimodal inputs.

Returns True if the model is multimodal with any non-zero supported modalities, otherwise returns False, effectively running in text-only mode.

Parameters

• model_config (term())

Returns

• boolean()