Crawly.Settings (Crawly v0.17.2) View Source

Define Crawly setting types

Link to this section Summary

Link to this section Types

Specs

middleware() ::
  Crawly.Middlewares.DomainFilter
  | Crawly.Middlewares.UniqueRequest
  | Crawly.Middlewares.RobotsTxt
  | Crawly.Middlewares.AutoCookiesManager
  | {Crawly.Middlewares.UserAgent, [{:user_agents, [binary()]}]}

Specs

numeric_setting() :: pos_integer() | :disabled

Specs

pipeline() ::
  Crawly.Pipelines.JSONEncoder
  | {Crawly.Pipelines.DuplicatesFilter, [{:item_id, atom()}]}
  | {Crawly.Pipelines.Validate, [{:fields, [atom()]}]}
  | {Crawly.Pipelines.CSVEncoder, [{:fields, [atom()]}]}
  | {Crawly.Pipelines.WriteToFile, folder: binary(), extension: binary()}

Specs

retry() :: [
  retry_codes: [pos_integer()],
  max_retries: pos_integer(),
  ignored_middlewares: [module()]
]

Specs

t() :: [
  closespider_itemcount: numeric_setting(),
  closespider_timeout: pos_integer(),
  concurrent_requests_per_domain: pos_integer(),
  fetcher: Crawly.Fetchers.Fetcher.t(),
  retry: retry(),
  middlewares: [middleware()],
  pipelines: [pipeline()],
  on_spider_closed_callback: function()
]