Rag.Pipeline (rag v0.3.4)

Pipeline definition and execution for composable RAG workflows.

Pipelines are sequences of steps that transform data through retrieval, reranking, context building, and generation.

Example

pipeline =
  Pipeline.new(:my_pipeline)
  |> Pipeline.add_step(
    name: :embed_query,
    module: MyApp.Steps,
    function: :embed_query,
    args: []
  )
  |> Pipeline.add_step(
    name: :retrieve,
    module: MyApp.Steps,
    function: :retrieve,
    args: [],
    inputs: [:embed_query]
  )

{:ok, result, context} = Pipeline.execute(pipeline, "What is RAG?")

Summary

Types

step()

t()

Functions

add_step(pipeline, step)

Adds a step to the pipeline.

execute(pipeline, input, opts \\ [])

Executes the pipeline with the given input.

new(name, opts \\ [])

Creates a new pipeline with the given name and options.

Types

step()

@type step() :: Rag.Pipeline.Step.t()

t()

@type t() :: %Rag.Pipeline{
  config: map(),
  description: String.t() | nil,
  metadata: map(),
  name: atom(),
  steps: [Rag.Pipeline.Step.t()]
}

Functions

add_step(pipeline, step)

@spec add_step(pipeline :: t(), step :: step() | keyword()) :: t()

Adds a step to the pipeline.

The step can be provided as a Step struct or as a keyword list that will be converted to a Step struct.

Examples

iex> pipeline = Pipeline.new(:test)
iex> step = %Pipeline.Step{name: :step1, module: MyModule, function: :my_func}
iex> Pipeline.add_step(pipeline, step)
%Pipeline{steps: [%Pipeline.Step{name: :step1}]}

iex> pipeline = Pipeline.new(:test)
iex> Pipeline.add_step(pipeline, name: :step1, module: MyModule, function: :my_func)
%Pipeline{steps: [%Pipeline.Step{name: :step1}]}

execute(pipeline, input, opts \\ [])

@spec execute(pipeline :: t(), input :: any(), opts :: keyword()) ::
  {:ok, result :: any(), context :: Rag.Pipeline.Context.t()} | {:error, term()}

Executes the pipeline with the given input.

Returns {:ok, result, context} on success or {:error, reason} on failure.

Options

:timeout - Overall timeout for the pipeline execution
:telemetry_metadata - Additional metadata to include in telemetry events

Examples

iex> pipeline = Pipeline.new(:test) |> Pipeline.add_step(...)
iex> Pipeline.execute(pipeline, "input data")
{:ok, result, %Context{}}

new(name, opts \\ [])

@spec new(name :: atom(), opts :: keyword()) :: t()

Creates a new pipeline with the given name and options.

Options

:description - A description of the pipeline
:config - Configuration map for the pipeline
:metadata - Additional metadata for the pipeline

Examples

iex> Pipeline.new(:my_pipeline)
%Pipeline{name: :my_pipeline, steps: [], config: %{}, metadata: %{}}

iex> Pipeline.new(:my_pipeline, description: "A test pipeline")
%Pipeline{name: :my_pipeline, description: "A test pipeline", steps: [], config: %{}, metadata: %{}}