mooncakes.io

🥮 mooncakes.io

/

/

src

/

README

trkbt10/micado_mbt/src/tokenizer does not have a README file

#
Edition

type Edition = Int

#
Mode

type Mode = Int

#
Tokenizer

pub struct Tokenizer {
  edition : Int
  mode : Int
  use_lexmatch_scanner : Bool
}

#
Tokenizer::new

fn Tokenizer::new() -> Tokenizer

#
Tokenizer::set_edition

fn Tokenizer::set_edition(self : Tokenizer, edition : Int) -> Tokenizer

#
Tokenizer::set_mode

fn Tokenizer::set_mode(self : Tokenizer, mode : Int) -> Tokenizer

#
Tokenizer::set_use_lexmatch_scanner

fn Tokenizer::set_use_lexmatch_scanner(self : Tokenizer, enabled : Bool) -> Tokenizer

#
Tokenizer::token_count

fn Tokenizer::token_count(self : Tokenizer, input : String) -> Int

#
Tokenizer::tokenize

fn Tokenizer::tokenize(self : Tokenizer, input : String) -> Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]

#
Tokenizer::tokenize_nbest

fn Tokenizer::tokenize_nbest(self : Tokenizer, input : String, n : Int) -> Array[Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]]

#
Tokenizer::tokenize_nbest_with_options

fn Tokenizer::tokenize_nbest_with_options(self : Tokenizer, input : String, n : Int, options : @trkbt10/micado_mbt/src/types.ParseOptions
ParseOptions) -> Array[@trkbt10/micado_mbt/src/types.NBestResult
NBestResult]

#
Tokenizer::tokenize_utf8

fn Tokenizer::tokenize_utf8(self : Tokenizer, input : BytesView) -> Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]

#
Tokenizer::tokenize_utf8_nbest

fn Tokenizer::tokenize_utf8_nbest(self : Tokenizer, input : BytesView, n : Int) -> Array[Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]]

#
Tokenizer::tokenize_utf8_nbest_with_options

fn Tokenizer::tokenize_utf8_nbest_with_options(self : Tokenizer, input : BytesView, n : Int, options : @trkbt10/micado_mbt/src/types.ParseOptions
ParseOptions) -> Array[@trkbt10/micado_mbt/src/types.NBestResult
NBestResult]

#
Tokenizer::tokenize_utf8_with_options

fn Tokenizer::tokenize_utf8_with_options(self : Tokenizer, input : BytesView, options : @trkbt10/micado_mbt/src/types.ParseOptions
ParseOptions) -> Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]

#
Tokenizer::tokenize_with_options

fn Tokenizer::tokenize_with_options(self : Tokenizer, input : String, options : @trkbt10/micado_mbt/src/types.ParseOptions
ParseOptions) -> Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]

#
EDITION_FULL

let EDITION_FULL : Int

#
EDITION_MINI

let EDITION_MINI : Int

#
EDITION_NANO

let EDITION_NANO : Int

#
EDITION_STANDARD

let EDITION_STANDARD : Int

#
MODE_NORMAL

let MODE_NORMAL : Int

#
MODE_SEARCH

let MODE_SEARCH : Int

#
token_count

fn token_count(input : String) -> Int

#
tokenize

fn tokenize(input : String) -> Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]

#
tokenize_nbest

fn tokenize_nbest(input : String, n : Int) -> Array[Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]]

#
tokenize_nbest_with_options

fn tokenize_nbest_with_options(input : String, n : Int, options : @trkbt10/micado_mbt/src/types.ParseOptions
ParseOptions) -> Array[@trkbt10/micado_mbt/src/types.NBestResult
NBestResult]

#
tokenize_utf8

fn tokenize_utf8(input : BytesView) -> Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]

#
tokenize_utf8_nbest

fn tokenize_utf8_nbest(input : BytesView, n : Int) -> Array[Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]]

#
tokenize_utf8_nbest_with_options

fn tokenize_utf8_nbest_with_options(input : BytesView, n : Int, options : @trkbt10/micado_mbt/src/types.ParseOptions
ParseOptions) -> Array[@trkbt10/micado_mbt/src/types.NBestResult
NBestResult]

#
tokenize_utf8_with_options

fn tokenize_utf8_with_options(input : BytesView, options : @trkbt10/micado_mbt/src/types.ParseOptions
ParseOptions) -> Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]

#
tokenize_with_options

fn tokenize_with_options(input : String, options : @trkbt10/micado_mbt/src/types.ParseOptions
ParseOptions) -> Array[@trkbt10/micado_mbt/src/types.Morpheme
Morpheme]

Source Files

tokenizer.mbt

Powered by MoonBit

Site source Report issue Packages Build queue Skills Statistics

© 2026 mooncakes.io