Deterministic phonetic encoding, explainable name matching, indexing, and deduplication for MoonBit
moon add zbhzs1/moonbit-phoneticimport {
"zbhzs1/moonbit-phonetic" @phonetic,
}let config = @phonetic.balanced_match_config()
match @phonetic.match_names("John Smith", "Smith Jon", config) {
Ok(evidence) => println(evidence.matched)
Err(_) => println("invalid input or configuration")
}let index = match @phonetic.NameIndex::new(config, [@phonetic.Soundex]) {
Ok(value) => value
Err(_) => abort("invalid index configuration")
}
ignore(index.insert("person-1", "Robert"))
ignore(index.insert("person-2", "Rupert"))
let result = index.query("Robert", 10, 0.0)moon run examples/basic
moon run examples/matching
moon run examples/deduplicate
moon run examples/linkagemoon clean
moon fmt --check
moon info
moon check
moon build
moon test
moon run examples/basic
moon run examples/matching
moon run examples/deduplicate
moon run examples/linkage
moon publish --dry-runpub(all) struct AcceptedLink {
left_id : String
right_id : String
evidence : MatchEvidence
} derive(Eq, Debug)pub(all) struct DedupEdge {
left_id : String
right_id : String
evidence : MatchEvidence
} derive(Eq, Debug)pub(all) enum DedupError {
InvalidUnionFindSize(Int)
UnionFindIndexOutOfBounds(Int, Int)
DuplicateInputId(String)
DedupIndexFailed(IndexError)
DedupMatchFailed(MatchError)
} derive(Eq, Debug)pub(all) struct DedupGroup {
representative : DedupInput
members : Array[DedupInput]
} derive(Eq, Debug)pub(all) struct DedupReport {
groups : Array[DedupGroup]
singletons : Array[DedupInput]
accepted_edges : Array[DedupEdge]
summary : DedupSummary
} derive(Eq, Debug)pub(all) struct IndexCandidateRecord {
record : NameRecord
normalized_name : String
shared_bucket_keys : Array[String]
} derive(Eq, Debug)pub(all) struct IndexCandidateSet {
query : String
normalized_query : String
query_bucket_keys : Array[String]
records : Array[IndexCandidateRecord]
} derive(Eq, Debug)pub(all) enum IndexError {
InvalidIndexMatchConfig(MatchError)
NoBlockingAlgorithms
DuplicateBlockingAlgorithm(Algorithm)
EmptyRecordId
DuplicateRecord(String)
MissingRecord(String)
IndexNormalizationFailed(NormalizationError)
NoBlockingKeys(String)
InvalidQueryLimit(Int)
InvalidMinimumScore(Double)
IndexMatchFailed(MatchError)
UnsupportedSnapshotVersion(Int)
DuplicateSnapshotRecord(String)
InconsistentSnapshotRecord(String)
} derive(Eq, Debug)pub(all) struct IndexMatch {
record : NameRecord
normalized_name : String
evidence : MatchEvidence
} derive(Eq, Debug)pub(all) struct IndexQueryResult {
query : String
normalized_query : String
matches : Array[IndexMatch]
summary : IndexQuerySummary
} derive(Eq, Debug)pub(all) struct IndexSnapshot {
version : Int
config : MatchConfig
blocking_algorithms : Array[Algorithm]
records : Array[IndexSnapshotRecord]
} derive(Eq, Debug)pub(all) struct IndexSnapshotRecord {
record : NameRecord
normalized : String
bucket_keys : Array[String]
} derive(Eq, Debug)pub(all) struct LinkCandidate {
left_id : String
right_id : String
score : Double
accepted : Bool
evidence : MatchEvidence
} derive(Eq, Debug)pub(all) struct LinkageConfig {
match_config : MatchConfig
blocking_algorithms : Array[Algorithm]
max_candidates_per_left : Int
minimum_score : Double
retain_rejected : Bool
} derive(Eq, Debug)pub(all) enum LinkageError {
InvalidLinkageMatchConfig(MatchError)
NoLinkageBlockingAlgorithms
DuplicateLinkageBlockingAlgorithm(Algorithm)
InvalidLinkageCandidateLimit(Int)
InvalidLinkageMinimumScore(Double)
EmptyLinkageRecordId(LinkageSide)
DuplicateLinkageRecordId(LinkageSide, String)
LinkageIndexFailed(LinkageSide, String, IndexError)
LinkageMatchFailed(String, String, MatchError)
} derive(Eq, Debug)pub(all) struct LinkageReport {
links : Array[AcceptedLink]
unmatched_left_ids : Array[String]
unmatched_right_ids : Array[String]
rejected_candidates : Array[LinkCandidate]
stats : LinkageStats
notices : Array[String]
} derive(Eq, Debug)pub(all) struct MatchConfig {
normalization : NormalizationConfig
encoders : Array[EncoderWeight]
metric : SimilarityMetric
string_weight : Double
threshold : Double
token_policy : TokenPolicy
unmatched_token_penalty : Double
} derive(Eq, Debug)fn MatchConfig::with_normalization(self : MatchConfig, normalization : NormalizationConfig) -> MatchConfigfn MatchConfig::with_token_settings(self : MatchConfig, token_policy : TokenPolicy, unmatched_token_penalty : Double) -> MatchConfigpub(all) enum MatchError {
InvalidThreshold(Double)
InvalidEncoderWeight(Algorithm, Double)
InvalidStringWeight(Double)
NoPositiveComponents
DuplicateEncoder(Algorithm)
InvalidUnmatchedTokenPenalty(Double)
InvalidNormalizationConfig(NormalizationError)
InvalidSimilarityMetric(SimilarityError)
NormalizationFailed(NormalizationError)
SimilarityFailed(SimilarityError)
} derive(Eq, Debug)pub(all) struct MatchEvidence {
left_normalized : String
right_normalized : String
left_keys : Array[EncodedKeys]
right_keys : Array[EncodedKeys]
components : Array[ComponentScore]
score : Double
threshold : Double
matched : Bool
notices : Array[String]
} derive(Eq, Debug)pub struct NameIndex {
config : MatchConfig
blocking_algorithms : Array[Algorithm]
records : Map[String, NameRecord]
normalized : Map[String, String]
record_bucket_keys : Map[String, Array[String]]
buckets : Map[String, Array[String]]
}fn NameIndex::candidate_records(self : NameIndex, name : String) -> Result[IndexCandidateSet, IndexError]fn NameIndex::new(config : MatchConfig, blocking_algorithms : Array[Algorithm]) -> Result[NameIndex, IndexError]fn NameIndex::query(self : NameIndex, name : String, limit : Int, minimum_score : Double) -> Result[IndexQueryResult, IndexError]pub(all) struct NormalizationConfig {
case_policy : CasePolicy
punctuation : CharacterPolicy
whitespace : CharacterPolicy
hyphen : CharacterPolicy
apostrophe : CharacterPolicy
digits : CharacterPolicy
fold_latin_diacritics : Bool
max_input_length : Int
} derive(Eq, Debug)fn NormalizationConfig::with_case_policy(self : NormalizationConfig, case_policy : CasePolicy) -> NormalizationConfigfn NormalizationConfig::with_character_policies(self : NormalizationConfig, punctuation : CharacterPolicy, whitespace : CharacterPolicy, hyphen : CharacterPolicy, apostrophe : CharacterPolicy, digits : CharacterPolicy) -> NormalizationConfigfn NormalizationConfig::with_max_input_length(self : NormalizationConfig, max_input_length : Int) -> NormalizationConfigpub(all) struct NormalizationResult {
original : String
normalized : String
tokens : Array[String]
notices : Array[NormalizationNotice]
truncated : Bool
} derive(Eq, Debug)fn caverphone1(input : String) -> Stringfn caverphone2(input : String) -> Stringfn deduplicate_names(inputs : Array[DedupInput], config : MatchConfig, blocking_algorithms : Array[Algorithm], include_singletons : Bool) -> Result[DedupReport, DedupError]fn encode_keys(input : String, algorithm : Algorithm, normalization : NormalizationConfig) -> Result[EncodedKeys, NormalizationError]fn jaro_similarity(left : String, right : String) -> Doublefn jaro_winkler_similarity(left : String, right : String, scaling : Double) -> Result[Double, SimilarityError]fn levenshtein_distance(left : String, right : String) -> Intfn levenshtein_similarity(left : String, right : String) -> Doublefn link_name_datasets(left : Array[LinkRecord], right : Array[LinkRecord], config : LinkageConfig) -> Result[LinkageReport, LinkageError]fn match_names(left : String, right : String, config : MatchConfig) -> Result[MatchEvidence, MatchError]fn match_rating_codex(input : String) -> Stringfn match_rating_compare(left : String, right : String) -> Boolfn metaphone(input : String) -> Stringfn normalize_name(input : String, config : NormalizationConfig) -> Result[NormalizationResult, NormalizationError]fn nysiis(input : String) -> Stringfn refined_soundex(input : String) -> Stringfn soundex(input : String) -> Stringfn string_similarity(left : String, right : String, metric : SimilarityMetric) -> Result[Double, SimilarityError]Deterministic phonetic encoding, explainable name matching, indexing, and deduplication for MoonBit