🥮 mooncakes.io
README

Lucius646/MoonSearch/analysis does not have a README file

#
TokenFilter

source
pub(open) trait TokenFilter {
  fn transform(Self, &TokenStream) -> &TokenStream
}

Wraps one TokenStream with a lazy token transformation.

#
TokenStream

source
pub(open) trait TokenStream {
  fn advance(Self) -> Bool
  fn token(Self) -> Token?
}

Stateful stream of analyzed tokens.

Tokenizers and token filters compose through this interface. As in Tantivy, token() exposes the token produced by the latest successful advance() call.

#
Tokenizer

source
pub(open) trait Tokenizer {
  fn token_stream(Self, String) -> &TokenStream
}

Splits source text into a TokenStream before filters are applied.

#
AnalysisError

source
pub(all) suberror AnalysisError {
  InvalidNgram(String)
  InvalidTokenGraph(String)
  TokenGraphTooComplex(Int)
  InvalidTokenizerName(String)
  UnknownTokenizer(String)
  FieldNotIndexed(
FieldId)
  EmptyQuery
} derive(Eq, 
Debug)

Errors raised while configuring or resolving text analysis.

#
EnglishStemmerFilter

source
pub struct EnglishStemmerFilter {
}

Porter2 English stemming filter compatible with Tantivy's en_stem era.

The filter expects lowercase English terms, so the built-in en_stem pipeline places LowerCaseFilter before it. Non-ASCII and mixed-script terms pass through unchanged.

#
EnglishStemmerFilter::new

source

#
LowerCaseFilter

source
pub struct LowerCaseFilter {
}

Unicode lowercase normalization.

#
LowerCaseFilter::new

source

#
NgramTokenizer

source
pub struct NgramTokenizer {
  min_gram : Int
  max_gram : Int
  prefix_only : Bool
}

Generic character N-gram tokenizer. It performs no linguistic segmentation.

Every token has position zero, matching Tantivy's NgramTokenizer semantics. UTF-8 byte offsets still identify the source span of each emitted gram.

#
NgramTokenizer::analyze

source
fn NgramTokenizer::analyze(self : NgramTokenizer, text : String) -> Array[Token]

#
NgramTokenizer::new

source
fn NgramTokenizer::new(min_gram : Int, max_gram : Int, prefix_only : Bool) -> NgramTokenizer raise AnalysisError

#
RawTokenizer

source
pub struct RawTokenizer {
}

Emits the complete non-empty input as one token.

#
RawTokenizer::analyze

source
fn RawTokenizer::analyze(self : RawTokenizer, text : String) -> Array[Token]

#
RawTokenizer::new

source

#
RemoveLongFilter

source
pub struct RemoveLongFilter {
  limit : Int
}

Removes tokens whose UTF-8 byte length exceeds the configured limit.

#
RemoveLongFilter::new

source
fn RemoveLongFilter::new(limit : Int) -> RemoveLongFilter

#
SimpleTokenizer

source
pub struct SimpleTokenizer {
}

Splits on Unicode whitespace, ASCII punctuation, and common CJK punctuation.

#
SimpleTokenizer::analyze

source
fn SimpleTokenizer::analyze(self : SimpleTokenizer, text : String) -> Array[Token]

#
SimpleTokenizer::new

source

#
StopWordFilter

source
pub struct StopWordFilter {
  stop_words : 
HashSet[String]
}

Removes exact token texts contained in an immutable stop-word snapshot.

Place this filter after LowerCaseFilter when stop words are normalized to lowercase. Removed tokens retain their position gaps in the surrounding stream.

#
StopWordFilter::new

source
fn StopWordFilter::new(stop_words : Array[String]) -> StopWordFilter

#
TextAnalyzer

source
pub struct TextAnalyzer {
  tokenizer : &Tokenizer
  filters : Array[&TokenFilter]
}

Composes one Tokenizer with an ordered filter chain.

#
TextAnalyzer::add_filter

source
fn TextAnalyzer::add_filter(self : TextAnalyzer, filter : &TokenFilter) -> Unit

#
TextAnalyzer::analyze

source
fn TextAnalyzer::analyze(self : TextAnalyzer, text : String) -> Array[Token]

Eager convenience API built on top of the stream consumed by indexing and query parsing.

#
TextAnalyzer::new

source
fn TextAnalyzer::new(tokenizer : &Tokenizer) -> TextAnalyzer

#
Token

source
pub(all) struct Token {
  text : String
  position : Int
  position_length : Int
  start_offset : Int
  end_offset : Int
} derive(Eq, 
Debug)

One analyzed token.

Offsets are UTF-8 byte offsets. position is a zero-based graph node and position_length is the number of original positions spanned by the token.

#
TokenGraph

source
pub struct TokenGraph {
  tokens : ReadOnlyArray[Token]
  start_position : Int
  end_position : Int
}

Immutable snapshot of a token stream interpreted as a directed acyclic graph. Every token is an edge from position to position + position_length.

#
TokenGraph::end_position

source
fn TokenGraph::end_position(self : TokenGraph) -> Int

#
TokenGraph::finite_strings

source
fn TokenGraph::finite_strings(self : TokenGraph) -> Array[TokenGraphPath] raise AnalysisError

Enumerates at most 256 finite strings, protecting query construction from accidental exponential graph expansion.

#
TokenGraph::finite_strings_with_limit

source
fn TokenGraph::finite_strings_with_limit(self : TokenGraph, max_paths : Int) -> Array[TokenGraphPath] raise AnalysisError

Enumerates all finite strings with an explicit expansion limit.

#
TokenGraph::from_tokens

source
fn TokenGraph::from_tokens(tokens : Array[Token]) -> TokenGraph raise AnalysisError

Validates and snapshots analyzed tokens as a TokenGraph.

Input order must be non-decreasing by position. Position lengths must be positive, and UTF-8 byte offsets must describe non-negative source spans.

#
TokenGraph::start_position

source
fn TokenGraph::start_position(self : TokenGraph) -> Int

#
TokenGraph::tokens

source
fn TokenGraph::tokens(self : TokenGraph) -> ReadOnlyArray[Token]

#
TokenGraphPath

source
pub struct TokenGraphPath {
  tokens : ReadOnlyArray[Token]
}

One finite linear path through a TokenGraph.

Tokens are normalized to path-local positions. A graph edge that spans multiple input positions becomes one token position in the finite string, while genuine gaps between graph nodes remain gaps.

#
TokenGraphPath::length

source
fn TokenGraphPath::length(self : TokenGraphPath) -> Int

#
TokenGraphPath::tokens

source
fn TokenGraphPath::tokens(self : TokenGraphPath) -> ReadOnlyArray[Token]

#
TokenizerManager

source
pub struct TokenizerManager {
  names : Array[String]
  tokenizers : Array[&Tokenizer]
}

Name-based registry for field indexing and query-time analysis.

#
TokenizerManager::analyze

source
fn TokenizerManager::analyze(self : TokenizerManager, name : String, text : String) -> Array[Token] raise AnalysisError

Eager convenience API for query construction, diagnostics, and tests.

#
TokenizerManager::contains

source
fn TokenizerManager::contains(self : TokenizerManager, name : String) -> Bool

#
TokenizerManager::new

source

#
TokenizerManager::register

source
fn TokenizerManager::register(self : TokenizerManager, name : String, tokenizer : &Tokenizer) -> Unit raise AnalysisError

#
TokenizerManager::register_tokenizer

source
fn TokenizerManager::register_tokenizer(self : TokenizerManager, name : String, tokenizer : &Tokenizer) -> Unit raise AnalysisError

Compatibility spelling for callers that explicitly register a Tokenizer.

#
TokenizerManager::snapshot

source

#
TokenizerManager::token_stream

source
fn TokenizerManager::token_stream(self : TokenizerManager, name : String, text : String) -> &TokenStream raise AnalysisError

Resolves a registered analysis pipeline as a lazy TokenStream.

#
TokenizerManager::with_defaults

source
fn TokenizerManager::with_defaults() -> TokenizerManager

Creates the built-in pipelines. NgramTokenizer remains opt-in because its parameters and use case must be chosen by the application.

#
WhitespaceAnalyzer

source
pub struct WhitespaceAnalyzer {
}

Analyzer that separates tokens at Unicode whitespace and performs no normalization. Retained as a compatibility wrapper around WhitespaceTokenizer.

#
WhitespaceAnalyzer::analyze

source
fn WhitespaceAnalyzer::analyze(self : WhitespaceAnalyzer, text : String) -> Array[Token]

Eager convenience API for callers that need a token collection.

#
WhitespaceAnalyzer::new

source

#
WhitespaceTokenizer

source
pub struct WhitespaceTokenizer {
}

Splits only on Unicode whitespace and performs no normalization.

#
WhitespaceTokenizer::analyze

source
fn WhitespaceTokenizer::analyze(self : WhitespaceTokenizer, text : String) -> Array[Token]

#
WhitespaceTokenizer::new

source

#
english_stem_analyzer

source
fn english_stem_analyzer() -> TextAnalyzer

Creates the same high-level pipeline as Tantivy's en_stem preset.

Source Files

Powered by MoonBit

Site sourceReport issuePackagesBuild queueSkillsStatistics

© 2026 mooncakes.io