A small robots.txt parser and access decision library for MoonBit.
moon add cauchyQ/moonbit-robots@0.1.0moon check
moon test
moon run cmd/mainlet file = #|User-agent: *
#|Disallow: /private
#|Allow: /private/public
#|
debug_inspect(allowed(file, "MoonBot", "/private/a"), content="false")
debug_inspect(allowed(file, "MoonBot", "/private/public/a"), content="true")println(explain(file, "MoonBot", ["/", "/private/a", "/private/public/a"]))debug_inspect(
filter_urls(file, "MoonBot", [
"https://example.test/",
"https://example.test/private/a",
]),
content="[\"https://example.test/\"]",
)let xml = #|<urlset>
#| <url><loc>https://example.test/a</loc></url>
#|</urlset>
debug_inspect(sitemap_urls(xml), content="[\"https://example.test/a\"]")println(compliance_report(file))
println(decision_matrix(file, ["MoonBot", "OtherBot"], ["/", "/private/a"]))pub struct Scenario {
name : String
robots : String
agent : String
expectations : Array[PathExpectation]
} derive(Eq, Debug)fn compare_agents(text : String, left_agent : String, right_agent : String, paths : Array[String]) -> Stringfn compare_versions(old_text : String, new_text : String, agent : String, paths : Array[String]) -> Stringfn next_allowed_tick(text : String, agent : String, previous_tick : Int) -> Intfn profile(name : String, agent : String, seed_urls : Array[String], max_urls : Int) -> CrawlProfilefn protects_sensitive_defaults(text : String, agent : String) -> Boolfn recipe_private_public(private_prefix : String, public_prefix : String) -> Stringfn scenario(name : String, robots : String, agent : String, expectations : Array[PathExpectation]) -> Scenariofn sitemap_coverage_report(xml : String, robots_text : String, agent : String) -> StringA small robots.txt parser and access decision library for MoonBit.