close_block_token() :: {:close_block, unused :: integer()}

comment_token() :: {:comment, {comment_type(), String.t()}}

comment_type() :: :c | :c_multiline

dquote_string_token() :: {:dquote_string, String.t()}

equal_token() :: {:=, unused :: integer()}

fold_token() :: {:fold, unused :: integer()}

newline_token() :: {:nl, unused :: integer()}

open_block_token() :: {:open_block, unused :: integer()}

raw_string_token() :: {:raw_string, String.t()}

semicolon_token() :: {:sc, unused :: integer()}

slashdash_token() :: {:slashdash, unused :: integer()}

space_token() :: {:space, {String.t(), len :: non_neg_integer()}}

term_token() :: {:term, String.t()}

token() ::
  open_block_token()
  | close_block_token()
  | slashdash_token()
  | comment_token()
  | dquote_string_token()
  | raw_string_token()
  | space_token()
  | newline_token()
  | equal_token()
  | semicolon_token()
  | fold_token()
  | term_token()

tokens() :: [token()]