webls/robots

Types

The policy for a specific robot

pub type Robot {
  Robot(
    user_agent: String,
    allowed_routes: List(String),
    disallowed_routes: List(String),
  )
}

</>

The configuration for a robots.txt file

pub type RobotsConfig {
  RobotsConfig(sitemap_url: String, robots: List(Robot))
}

</>

pub fn config(sitemap_url: String) -> RobotsConfig

Creates a robots config with a sitemap url

</>

pub fn robot(user_agent: String) -> Robot

Creates a robot policy

</>

pub fn to_string(config: RobotsConfig) -> String

</>

pub fn with_config_robot(
  config: RobotsConfig,
  robot: Robot,
) -> RobotsConfig

Adds a robot to the robots config

</>

pub fn with_config_robots(
  config: RobotsConfig,
  robots: List(Robot),
) -> RobotsConfig

Adds a list of robots to the robots config

</>

pub fn with_robot_allowed_route(
  robot: Robot,
  route: String,
) -> Robot

Adds a allowed route to the robot policy

</>

pub fn with_robot_allowed_routes(
  robot: Robot,
  routes: List(String),
) -> Robot

Adds a list of allowed routes to the robot policy

</>

pub fn with_robot_disallowed_route(
  robot: Robot,
  route: String,
) -> Robot

Adds a disallowed route to the robot policy

</>

pub fn with_robot_disallowed_routes(
  robot: Robot,
  routes: List(String),
) -> Robot

Adds a list of disallowed routes to the robot policy