lexer

pub struct LexResult {
  tokens : Array[(
@moonbitlang/lexer/tokens.Token
Token, 
@moonbitlang/lexer/basic.Position
Position, 
@moonbitlang/lexer/basic.Position
Position)]
  errors : Array[(
@moonbitlang/lexer/basic.Position
Position, 
@moonbitlang/lexer/basic.Position
Position, LexicalError)]
  docstrings : Array[
@moonbitlang/core/list.List
List[(
@moonbitlang/lexer/basic.Location
Location, 
@moonbitlang/lexer/tokens.Comment
Comment)]]
}

pub enum LexicalError {
  IllegalCharacter(Char)
  UnterminatedString
  UnterminatedStringInVariableInterploation
  InterpInvalidMultilineString
  InterpInvalidComment
  InterpInvalidAttribute
  InterpMissingExpression
  InvalidDotInt(String)
  InvalidByteLiteral(String)
  Reserved_keyword(String)
}

fn tokens_from_string(name? : String, start_pos? : 
@moonbitlang/lexer/basic.Position
Position, is_interpolation? : Bool, comment~ : Bool, str : String) -> LexResult

fn tokens_from_string_with_utf16_location(name? : String, start_pos? : 
@moonbitlang/lexer/basic.Position
Position, is_interpolation? : Bool, comment~ : Bool, str : String) -> LexResult