middleware() ::
  Crawly.Middlewares.DomainFilter
  | Crawly.Middlewares.UniqueRequest
  | Crawly.Middlewares.RobotsTxt
  | Crawly.Middlewares.AutoCookiesManager
  | {Crawly.Middlewares.UserAgent, [{:user_agents, [binary()]}]}

numeric_setting() :: pos_integer() | :disabled

pipeline() ::
  Crawly.Pipelines.JSONEncoder
  | {Crawly.Pipelines.DuplicatesFilter, [{:item_id, atom()}]}
  | {Crawly.Pipelines.Validate, [{:fields, [atom()]}]}
  | {Crawly.Pipelines.CSVEncoder, [{:fields, [atom()]}]}
  | {Crawly.Pipelines.WriteToFile, folder: binary(), extension: binary()}

retry() :: [
  retry_codes: [pos_integer()],
  max_retries: pos_integer(),
  ignored_middlewares: [module()]
]

t() :: [
  closespider_itemcount: numeric_setting(),
  closespider_timeout: pos_integer(),
  concurrent_requests_per_domain: pos_integer(),
  fetcher: Crawly.Fetchers.Fetcher.t(),
  retry: retry(),
  middlewares: [middleware()],
  pipelines: [pipeline()],
  on_spider_closed_callback: function()
]