Vllm.Outputs.RequestStateStats (VLLM v0.3.0)

Copy Markdown View Source

Stats that need to be tracked across delta updates.

Summary

Types

t()

Functions

arrival_time(ref)
first_token_latency(ref)
first_token_ts(ref)
is_corrupted(ref)
last_token_ts(ref)
new(args, opts \\ [])

Initialize self. See help(type(self)) for accurate signature.

num_generation_tokens(ref)
queued_ts(ref)
scheduled_ts(ref)

Types

t()

@opaque t()

Functions

arrival_time(ref)

@spec arrival_time(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

first_token_latency(ref)

@spec first_token_latency(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

first_token_ts(ref)

@spec first_token_ts(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

is_corrupted(ref)

@spec is_corrupted(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

last_token_ts(ref)

@spec last_token_ts(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

new(args, opts \\ [])

@spec new(
  [term()],
  keyword()
) :: {:ok, SnakeBridge.Ref.t()} | {:error, Snakepit.Error.t()}

Initialize self. See help(type(self)) for accurate signature.

Parameters

  • num_generation_tokens (integer() default: 0)
  • arrival_time (float() default: 0.0)
  • queued_ts (float() default: 0.0)
  • scheduled_ts (float() default: 0.0)
  • first_token_ts (float() default: 0.0)
  • last_token_ts (float() default: 0.0)
  • first_token_latency (float() default: 0.0)
  • is_corrupted (boolean() default: False)

num_generation_tokens(ref)

@spec num_generation_tokens(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}

queued_ts(ref)

@spec queued_ts(SnakeBridge.Ref.t()) :: {:ok, term()} | {:error, Snakepit.Error.t()}

scheduled_ts(ref)

@spec scheduled_ts(SnakeBridge.Ref.t()) ::
  {:ok, term()} | {:error, Snakepit.Error.t()}