caffeine_lang/frontend/tokenizer
Values
pub fn tokenize(
source: String,
) -> Result(
List(token.PositionedToken),
tokenizer_error.TokenizerError,
)Tokenizes source text into a list of positioned tokens.
pub fn tokenize(
source: String,
) -> Result(
List(token.PositionedToken),
tokenizer_error.TokenizerError,
)Tokenizes source text into a list of positioned tokens.