Vllm.Tokenizers.TokenizerLike (VLLM v0.3.0)

Copy Markdown View Source

Wrapper for Python class TokenizerLike.

Summary

Types

t()

Functions

all_special_ids(ref)
all_special_tokens(ref)
apply_chat_template(ref, messages, args, opts \\ [])

Python method TokenizerLike.apply_chat_template.

bos_token_id(ref)
convert_ids_to_tokens(ref, ids, args, opts \\ [])

Python method TokenizerLike.convert_ids_to_tokens.

convert_tokens_to_ids(ref, tokens, opts \\ [])

Python method TokenizerLike.convert_tokens_to_ids.

convert_tokens_to_string(ref, tokens, opts \\ [])

Python method TokenizerLike.convert_tokens_to_string.

decode(ref, ids, args, opts \\ [])

Python method TokenizerLike.decode.

encode(ref, text, args, opts \\ [])

Python method TokenizerLike.encode.

eos_token_id(ref)
from_pretrained(ref, path_or_repo_id, args, opts \\ [])

Python method TokenizerLike.from_pretrained.

get_added_vocab(ref, opts \\ [])

Python method TokenizerLike.get_added_vocab.

get_vocab(ref, opts \\ [])

Python method TokenizerLike.get_vocab.

is_fast(ref)
max_token_id(ref)
new(args, opts \\ [])

Constructs TokenizerLike.

num_special_tokens_to_add(ref, opts \\ [])

Python method TokenizerLike.num_special_tokens_to_add.

pad_token_id(ref)
truncation_side(ref)
vocab_size(ref)

Types

t()

@opaque t()

Functions

all_special_ids(ref)

@spec all_special_ids(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

all_special_tokens(ref)

@spec all_special_tokens(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

apply_chat_template(ref, messages, args, opts \\ [])

@spec apply_chat_template(SnakeBridge.Ref.t(), [term()], [term()], keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.apply_chat_template.

Parameters

  • messages (list(term()))
  • tools (term() default: None)
  • kwargs (term())

Returns

  • term()

bos_token_id(ref)

@spec bos_token_id(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

convert_ids_to_tokens(ref, ids, args, opts \\ [])

@spec convert_ids_to_tokens(SnakeBridge.Ref.t(), [integer()], [term()], keyword()) ::
  {:ok, [String.t()]} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.convert_ids_to_tokens.

Parameters

  • ids (list(integer()))
  • skip_special_tokens (boolean() default: False)

Returns

  • list(String.t())

convert_tokens_to_ids(ref, tokens, opts \\ [])

@spec convert_tokens_to_ids(SnakeBridge.Ref.t(), term(), keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.convert_tokens_to_ids.

Parameters

  • tokens (term())

Returns

  • term()

convert_tokens_to_string(ref, tokens, opts \\ [])

@spec convert_tokens_to_string(SnakeBridge.Ref.t(), [String.t()], keyword()) ::
  {:ok, String.t()} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.convert_tokens_to_string.

Parameters

  • tokens (list(String.t()))

Returns

  • String.t()

decode(ref, ids, args, opts \\ [])

@spec decode(SnakeBridge.Ref.t(), term(), [term()], keyword()) ::
  {:ok, String.t()} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.decode.

Parameters

  • ids (term())
  • skip_special_tokens (boolean() default: False)

Returns

  • String.t()

encode(ref, text, args, opts \\ [])

@spec encode(SnakeBridge.Ref.t(), String.t(), [term()], keyword()) ::
  {:ok, [integer()]} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.encode.

Parameters

  • text (String.t())
  • truncation (term() default: None)
  • max_length (term() default: None)
  • add_special_tokens (boolean() default: True)

Returns

  • list(integer())

eos_token_id(ref)

@spec eos_token_id(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

from_pretrained(ref, path_or_repo_id, args, opts \\ [])

@spec from_pretrained(SnakeBridge.Ref.t(), term(), [term()], keyword()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.from_pretrained.

Parameters

  • path_or_repo_id (term())
  • args (term())
  • trust_remote_code (boolean() keyword-only default: False)
  • revision (term() keyword-only default: None)
  • download_dir (term() keyword-only default: None)
  • kwargs (term())

Returns

  • term()

get_added_vocab(ref, opts \\ [])

@spec get_added_vocab(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, %{optional(String.t()) => integer()}} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.get_added_vocab.

Returns

  • %{optional(String.t()) => integer()}

get_vocab(ref, opts \\ [])

@spec get_vocab(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, %{optional(String.t()) => integer()}} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.get_vocab.

Returns

  • %{optional(String.t()) => integer()}

is_fast(ref)

@spec is_fast(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

max_token_id(ref)

@spec max_token_id(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

new(args, opts \\ [])

@spec new(
  [term()],
  keyword()
) :: {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Constructs TokenizerLike.

Parameters

  • args (term())
  • kwargs (term())

num_special_tokens_to_add(ref, opts \\ [])

@spec num_special_tokens_to_add(
  SnakeBridge.Ref.t(),
  keyword()
) :: {:ok, integer()} | {:error, Snakepit.Error.t()}

Python method TokenizerLike.num_special_tokens_to_add.

Returns

  • integer()

pad_token_id(ref)

@spec pad_token_id(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

truncation_side(ref)

@spec truncation_side(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

vocab_size(ref)

@spec vocab_size(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}