moonbit-robots

A small robots.txt parser and access decision library for MoonBit.

robots
robotstxt
crawler
parser
moonbit
moon add cauchyQ/moonbit-robots@0.1.0
Download zip
Author
Version
0.1.0
License
MIT
Last updated
13 days ago
Downloads
4
README

#moonbit-robots

moonbit-robots is a small robots.txt parser and access decision library written in MoonBit. It helps crawler-like tools decide whether a user agent may visit a path, while also exposing metadata such as Sitemap, Host, and Crawl-delay.

#Features

  • parse User-agent, Allow, Disallow, Crawl-delay, Sitemap, and Host
  • choose the most specific user-agent group
  • apply longest-match rule precedence
  • support * wildcards and $ end anchors in path rules
  • ignore comments and blank lines
  • validate malformed lines and invalid crawl-delay values
  • render compact summaries and per-path explanations
  • extract paths from full URLs and filter URL lists
  • lint duplicate rules, unknown directives, and misplaced groups
  • render a parsed robots model back to robots.txt text
  • normalize URL paths, dot segments, extensions, and percent escapes
  • extract simple sitemap URL entries and sitemap index locations
  • build crawl frontiers and compare policy changes
  • generate audit, inventory, statistics, and compliance reports

#Install

After publishing:

moon add cauchyQ/moonbit-robots@0.1.0

Local development:

moon check moon test moon run cmd/main

#Example

let file = #|User-agent: *
#|Disallow: /private
#|Allow: /private/public
#|

debug_inspect(allowed(file, "MoonBot", "/private/a"), content="false")
debug_inspect(allowed(file, "MoonBot", "/private/public/a"), content="true")

For a readable report:

println(explain(file, "MoonBot", ["/", "/private/a", "/private/public/a"]))

URL filtering:

debug_inspect(
filter_urls(file, "MoonBot", [
"https://example.test/",
"https://example.test/private/a",
]),
content="[\"https://example.test/\"]",
)

Sitemap extraction:

let xml = #|<urlset>
#| <url><loc>https://example.test/a</loc></url>
#|</urlset>
debug_inspect(sitemap_urls(xml), content="[\"https://example.test/a\"]")

Policy audit:

println(compliance_report(file))
println(decision_matrix(file, ["MoonBot", "OtherBot"], ["/", "/private/a"]))

#Scope

This library does not fetch web pages and does not implement a crawler. It only parses robots.txt text and answers access-rule questions. That keeps the package dependency-free and easy to embed in other MoonBit tools.

#License

MIT

#
AccessRecord

pub struct AccessRecord {
verb : String
url : String
status : Int
agent : String
} derive(Eq,
Debug
)

#
Coverage

pub struct Coverage {
total : Int
allowed : Int
blocked : Int
unmatched : Int
} derive(Eq,
Debug
)

#
CrawlPolicy

pub struct CrawlPolicy {
agent : String
crawl_delay : Int?
sitemaps : Array[String]
hosts : Array[String]
group_agents : Array[String]
} derive(Eq,
Debug
)

#
CrawlProfile

pub struct CrawlProfile {
name : String
agent : String
seed_urls : Array[String]
max_urls : Int
} derive(Eq,
Debug
)

#
Decision

pub struct Decision {
allowed : Bool
agent : String
path : String
matched_kind : String
matched_pattern : String
} derive(Eq,
Debug
)

#
Directive

pub struct Directive {
line : Int
key : String
value : String
raw : String
} derive(Eq,
Debug
)

#
FetchSlot

pub struct FetchSlot {
url : String
earliest_tick : Int
} derive(Eq,
Debug
)

#
FrontierItem

pub struct FrontierItem {
url : String
path : String
allowed : Bool
reason : String
} derive(Eq,
Debug
)

#
Group

pub struct Group {
agents : Array[String]
rules : Array[Rule]
crawl_delay : Int?
} derive(Eq,
Debug
)

#
PathExpectation

pub struct PathExpectation {
path : String
allowed : Bool
} derive(Eq,
Debug
)

#
Robots

pub struct Robots {
groups : Array[Group]
sitemaps : Array[String]
hosts : Array[String]
} derive(Eq,
Debug
)

#
RobotsStats

pub struct RobotsStats {
groups : Int
agents : Int
allows : Int
disallows : Int
crawl_delays : Int
sitemaps : Int
hosts : Int
} derive(Eq,
Debug
)

#
Rule

pub struct Rule {
kind : String
pattern : String
} derive(Eq,
Debug
)

#
Scenario

pub struct Scenario {
name : String
robots : String
agent : String
expectations : Array[PathExpectation]
} derive(Eq,
Debug
)

#
SitemapEntry

pub struct SitemapEntry {
loc : String
lastmod : String
changefreq : String
priority : String
} derive(Eq,
Debug
)

#
access_record

fn access_record(verb : String, url : String, status : Int, agent : String) -> AccessRecord

#
access_summary

fn access_summary(records : Array[AccessRecord]) -> String

#
agent_family

fn agent_family(agent : String) -> String

#
agent_matches_token

fn agent_matches_token(agent : String, token : String) -> Bool

#
agent_report

fn agent_report(agent : String, tokens : Array[String]) -> String

#
agent_score_against

fn agent_score_against(agent : String, token : String) -> Int

#
agent_tokens

fn agent_tokens(agent : String) -> Array[String]

#
allow_all

fn allow_all(agent : String) -> Robots

#
allow_public_private

fn allow_public_private(private_prefix : String, public_prefix : String) -> Robots

#
allow_rule

fn allow_rule(pattern : String) -> Rule

#
allowed

fn allowed(text : String, agent : String, path : String) -> Bool

#
allowed_paths

fn allowed_paths(text : String, agent : String, paths : Array[String]) -> Array[String]

#
append_host

fn append_host(robots : Robots, host : String) -> Robots

#
append_sitemap

fn append_sitemap(robots : Robots, sitemap : String) -> Robots

#
ascii_space

fn ascii_space(ch : Char) -> Bool

#
attach_path

fn attach_path(base_url : String, path : String) -> String

#
audit

fn audit(text : String) -> Array[String]

#
audit_report

fn audit_report(text : String) -> String

#
auto_coverage_report

fn auto_coverage_report(text : String, agent : String) -> String

#
best_agent_token

fn best_agent_token(agent : String, tokens : Array[String]) -> String

#
best_group

fn best_group(robots : Robots, agent : String) -> Group?

#
block_all

fn block_all(agent : String) -> Robots

#
block_paths

fn block_paths(agent : String, paths : Array[String]) -> Robots

#
blocked_report

fn blocked_report(text : String, agent : String, urls : Array[String]) -> String

#
bool_word

fn bool_word(value : Bool) -> String

#
built_in_scenarios

fn built_in_scenarios() -> Array[Scenario]

#
built_in_scenarios_report

fn built_in_scenarios_report() -> String

#
can_fetch_url

fn can_fetch_url(text : String, agent : String, url : String) -> Bool

#
canonical_match_path

fn canonical_match_path(path : String) -> String

#
classify_path

fn classify_path(path : String) -> String

#
clean_path

fn clean_path(path : String) -> String

#
common_path_prefix

fn common_path_prefix(paths : Array[String]) -> String

#
common_two_path_prefix

fn common_two_path_prefix(left : String, right : String) -> String

#
compact

fn compact(text : String) -> String

#
compare_agents

fn compare_agents(text : String, left_agent : String, right_agent : String, paths : Array[String]) -> String

#
compare_versions

fn compare_versions(old_text : String, new_text : String, agent : String, paths : Array[String]) -> String

#
compliance_report

fn compliance_report(text : String) -> String

#
contains_rule

fn contains_rule(rules : Array[Rule], target : Rule) -> Bool

#
contains_string

fn contains_string(values : Array[String], value : String) -> Bool

#
contains_string_case_insensitive

fn contains_string_case_insensitive(values : Array[String], value : String) -> Bool

#
count_status

fn count_status(records : Array[AccessRecord], status : Int) -> Int

#
coverage

fn coverage(text : String, agent : String, paths : Array[String]) -> Coverage

#
coverage_line

fn coverage_line(c : Coverage) -> String

#
coverage_report

fn coverage_report(text : String, agent : String, paths : Array[String]) -> String

#
crawl_delay_or

fn crawl_delay_or(text : String, agent : String, fallback : Int) -> Int

#
csv_escape

fn csv_escape(text : String) -> String

#
csv_row

fn csv_row(values : Array[String]) -> String

#
decide

fn decide(robots : Robots, agent : String, path : String) -> Decision

#
decide_group

fn decide_group(group : Group, agent : String, path : String) -> Decision

#
decide_url

fn decide_url(robots : Robots, agent : String, url : String) -> Decision

#
decision_line

fn decision_line(decision : Decision) -> String

#
decision_matrix

fn decision_matrix(text : String, agents : Array[String], paths : Array[String]) -> String

#
decisions_csv

fn decisions_csv(text : String, agent : String, paths : Array[String]) -> String

#
decode_percent_path

fn decode_percent_path(path : String) -> String

#
dedupe_urls

fn dedupe_urls(urls : Array[String]) -> Array[String]

#
delay_text

fn delay_text(delay : Int?) -> String

#
diagnostics_report

fn diagnostics_report(text : String, agent : String, paths : Array[String]) -> String

#
directive_count

fn directive_count(text : String, key : String) -> Int

#
directive_report

fn directive_report(text : String) -> String

#
directive_suggestion_report

fn directive_suggestion_report(text : String) -> String

#
directive_values

fn directive_values(text : String, key : String) -> Array[String]

#
directives

fn directives(text : String) -> Array[Directive]

#
directives_for_key

fn directives_for_key(text : String, key : String) -> Array[Directive]

#
disallow_rule

fn disallow_rule(pattern : String) -> Rule

#
disallowed_paths

fn disallowed_paths(text : String, agent : String, paths : Array[String]) -> Array[String]

#
due_urls

fn due_urls(now : Int, slots : Array[FetchSlot]) -> Array[String]

#
edit_distance

fn edit_distance(left : String, right : String) -> Int

#
empty_robots

fn empty_robots() -> Robots

#
encode_path_spaces

fn encode_path_spaces(path : String) -> String

#
expect_path

fn expect_path(path : String, allowed : Bool) -> PathExpectation

#
explain

fn explain(text : String, agent : String, paths : Array[String]) -> String

#
explain_urls

fn explain_urls(text : String, agent : String, urls : Array[String]) -> String

#
fenced

fn fenced(text : String) -> String

#
filter_urls

fn filter_urls(text : String, agent : String, urls : Array[String]) -> Array[String]

#
first_directive_value

fn first_directive_value(text : String, key : String) -> String?

#
frontier

fn frontier(text : String, agent : String, urls : Array[String]) -> Array[FrontierItem]

#
frontier_allowed

fn frontier_allowed(items : Array[FrontierItem]) -> Array[String]

#
frontier_blocked

fn frontier_blocked(items : Array[FrontierItem]) -> Array[String]

#
frontier_csv

fn frontier_csv(items : Array[FrontierItem]) -> String

#
frontier_report

fn frontier_report(items : Array[FrontierItem]) -> String

#
group_add_allow

fn group_add_allow(group : Group, pattern : String) -> Group

#
group_add_disallow

fn group_add_disallow(group : Group, pattern : String) -> Group

#
group_agent_score

fn group_agent_score(group : Group, agent : String) -> Int

#
group_for_agents

fn group_for_agents(agents : Array[String]) -> Group

#
group_matches_agent

fn group_matches_agent(group : Group, agent : String) -> Bool

#
group_rule_count

fn group_rule_count(group : Group, kind : String) -> Int

#
group_set_delay

fn group_set_delay(group : Group, delay : Int) -> Group

#
group_summary

fn group_summary(group : Group) -> String

#
group_urls_by_host

fn group_urls_by_host(urls : Array[String]) -> Array[String]

#
group_with

fn group_with(agents : Array[String], rules : Array[Rule]) -> Group

#
has_global_block

fn has_global_block(robots : Robots) -> Bool

#
has_percent_escape

fn has_percent_escape(text : String) -> Bool

#
health_report

fn health_report(text : String) -> String

#
hex_value

fn hex_value(ch : Char) -> Int?

#
host_matches_robots

fn host_matches_robots(robots : Robots, url : String) -> Bool

#
invalid_percent_offsets

fn invalid_percent_offsets(text : String) -> Array[Int]

#
inventory_report

fn inventory_report(text : String) -> String

#
invert_group

fn invert_group(group : Group) -> Group

#
is_allow

fn is_allow(rule : Rule) -> Bool

#
is_clean

fn is_clean(text : String) -> Bool

#
is_disallow

fn is_disallow(rule : Rule) -> Bool

#
is_due

fn is_due(now : Int, slot : FetchSlot) -> Bool

#
join_ints

fn join_ints(values : Array[Int], sep : String) -> String

#
join_lines

fn join_lines(lines : Array[String]) -> String

#
join_with

fn join_with(values : Array[String], sep : String) -> String

#
known_directive

fn known_directive(key : String) -> Bool

#
largest_group

fn largest_group(robots : Robots) -> Group?

#
largest_group_summary

fn largest_group_summary(text : String) -> String

#
lint

fn lint(text : String) -> Array[String]

#
lint_report

fn lint_report(text : String) -> String

#
lower_ascii

fn lower_ascii(text : String) -> String

#
markdown_decision_table

fn markdown_decision_table(robots : Robots, agent : String, paths : Array[String]) -> String

#
markdown_full

fn markdown_full(text : String, agent : String, paths : Array[String]) -> String

#
markdown_hosts

fn markdown_hosts(robots : Robots) -> String

#
markdown_inventory

fn markdown_inventory(text : String) -> String

#
markdown_report

fn markdown_report(text : String, agent : String, paths : Array[String]) -> String

#
markdown_sitemaps

fn markdown_sitemaps(robots : Robots) -> String

#
merge_groups_by_agent

fn merge_groups_by_agent(robots : Robots) -> Robots

#
min_int

fn min_int(a : Int, b : Int) -> Int

#
missing_common_blocks

fn missing_common_blocks(text : String, agent : String) -> Array[String]

#
new_group

fn new_group(agent : String) -> Group

#
next_allowed_tick

fn next_allowed_tick(text : String, agent : String, previous_tick : Int) -> Int

#
next_batch

fn next_batch(text : String, agent : String, urls : Array[String], limit : Int) -> Array[String]

#
normalize_agent

fn normalize_agent(agent : String) -> String

#
normalize_path

fn normalize_path(path : String) -> String

#
normalize_robots

fn normalize_robots(robots : Robots) -> Robots

#
origin

fn origin(url : String) -> String?

#
origin_or

fn origin_or(url : String, fallback : String) -> String

#
parse

fn parse(text : String) -> Robots

#
parse_errors

fn parse_errors(text : String) -> Array[String]

#
parse_non_negative_int

fn parse_non_negative_int(text : String) -> Int?

#
parse_report

fn parse_report(text : String) -> String

#
path_basename

fn path_basename(path : String) -> String

#
path_depth

fn path_depth(path : String) -> Int

#
path_extension

fn path_extension(path : String) -> String

#
path_has_extension

fn path_has_extension(path : String, ext : String) -> Bool

#
path_parent

fn path_parent(path : String) -> String

#
path_score

fn path_score(path : String) -> Int

#
pattern_matches

fn pattern_matches(pattern : String, path : String) -> Bool

#
pending_urls

fn pending_urls(now : Int, slots : Array[FetchSlot]) -> Array[String]

#
percent_report

fn percent_report(text : String) -> String

#
policy_for

fn policy_for(text : String, agent : String) -> CrawlPolicy

#
policy_from

fn policy_from(robots : Robots, agent : String) -> CrawlPolicy

#
policy_summary

fn policy_summary(policy : CrawlPolicy) -> String

#
profile

fn profile(name : String, agent : String, seed_urls : Array[String], max_urls : Int) -> CrawlProfile

#
profile_markdown

fn profile_markdown(text : String, p : CrawlProfile) -> String

#
profile_plan

fn profile_plan(text : String, p : CrawlProfile) -> Array[String]

#
profile_report

fn profile_report(text : String, p : CrawlProfile) -> String

#
protects_sensitive_defaults

fn protects_sensitive_defaults(text : String, agent : String) -> Bool

#
rank_agents

fn rank_agents(agent : String, tokens : Array[String]) -> Array[String]

#
recipe_allow_all

fn recipe_allow_all(agent : String) -> String

#
recipe_api_docs

fn recipe_api_docs(sitemap : String) -> String

#
recipe_block_all

fn recipe_block_all(agent : String) -> String

#
recipe_blog

fn recipe_blog(sitemap : String) -> String

#
recipe_catalog

fn recipe_catalog() -> String

#
recipe_for_kind

fn recipe_for_kind(kind : String, sitemap : String) -> String

#
recipe_names

fn recipe_names() -> Array[String]

#
recipe_private_public

fn recipe_private_public(private_prefix : String, public_prefix : String) -> String

#
recipe_static_site

fn recipe_static_site(sitemap : String) -> String

#
records_for_agent

fn records_for_agent(records : Array[AccessRecord], agent : String) -> Array[AccessRecord]

#
remove_duplicate_paths

fn remove_duplicate_paths(paths : Array[String]) -> Array[String]

#
remove_duplicate_rules

fn remove_duplicate_rules(group : Group) -> Group

#
remove_rules_with_prefix

fn remove_rules_with_prefix(group : Group, prefix : String) -> Group

#
render

fn render(robots : Robots) -> String

#
render_group

fn render_group(group : Group) -> String

#
render_rule

fn render_rule(rule : Rule) -> String

#
replay_access

fn replay_access(text : String, records : Array[AccessRecord]) -> Array[String]

#
replay_report

fn replay_report(text : String, records : Array[AccessRecord]) -> String

#
risky_patterns

fn risky_patterns(robots : Robots) -> Array[String]

#
robots_digest

fn robots_digest(text : String) -> Array[String]

#
robots_from_group

fn robots_from_group(group : Group) -> Robots

#
robots_source_report

fn robots_source_report(urls : Array[String]) -> String

#
robots_url

fn robots_url(site_url : String) -> String

#
robots_urls_for

fn robots_urls_for(urls : Array[String]) -> Array[String]

#
robots_with_metadata

fn robots_with_metadata(groups : Array[Group], sitemaps : Array[String], hosts : Array[String]) -> Robots

#
rule_inventory

fn rule_inventory(robots : Robots) -> Array[String]

#
rule_matches

fn rule_matches(rule : Rule, path : String) -> Bool

#
rule_suggestion

fn rule_suggestion(text : String, agent : String, path : String) -> String

#
rule_weight

fn rule_weight(rule : Rule) -> Int

#
rules_csv

fn rules_csv(text : String) -> String

#
rules_for_kind

fn rules_for_kind(group : Group, kind : String) -> Array[String]

#
run_scenario

fn run_scenario(case : Scenario) -> Array[String]

#
run_scenarios

fn run_scenarios(cases : Array[Scenario]) -> Array[String]

#
same_host

fn same_host(url : String, host : String) -> Bool

#
same_origin_url

fn same_origin_url(left : String, right : String) -> Bool

#
sample_child_path

fn sample_child_path(pattern : String) -> String

#
sample_paths_from_rules

fn sample_paths_from_rules(robots : Robots) -> Array[String]

#
scenario

fn scenario(name : String, robots : String, agent : String, expectations : Array[PathExpectation]) -> Scenario

#
scenario_markdown

fn scenario_markdown(case : Scenario) -> String

#
scenario_matrix

fn scenario_matrix(case : Scenario) -> String

#
scenario_report

fn scenario_report(case : Scenario) -> String

#
scenarios_report

fn scenarios_report(cases : Array[Scenario]) -> String

#
schedule_fetches

fn schedule_fetches(text : String, agent : String, urls : Array[String], start_tick : Int) -> Array[FetchSlot]

#
schedule_report

fn schedule_report(slots : Array[FetchSlot]) -> String

#
security_patch

fn security_patch(agent : String) -> String

#
security_score

fn security_score(text : String, agent : String) -> Int

#
sensitive_path_patterns

fn sensitive_path_patterns() -> Array[String]

#
sensitive_paths_allowed

fn sensitive_paths_allowed(text : String, agent : String) -> Array[String]

#
sensitive_report

fn sensitive_report(text : String, agent : String) -> String

#
shadowed_rules

fn shadowed_rules(group : Group) -> Array[String]

#
sitemap_by_extension

fn sitemap_by_extension(xml : String, ext : String) -> Array[String]

#
sitemap_coverage_report

fn sitemap_coverage_report(xml : String, robots_text : String, agent : String) -> String

#
sitemap_csv

fn sitemap_csv(xml : String) -> String

#
sitemap_entries

fn sitemap_entries(xml : String) -> Array[SitemapEntry]

#
sitemap_index_locations

fn sitemap_index_locations(xml : String) -> Array[String]

#
sitemap_inventory

fn sitemap_inventory(xml : String) -> String

#
sitemap_report

fn sitemap_report(xml : String) -> String

#
sitemap_same_host

fn sitemap_same_host(xml : String, host : String) -> Array[String]

#
sitemap_urls

fn sitemap_urls(xml : String) -> Array[String]

#
sitemap_urls_allowed

fn sitemap_urls_allowed(xml : String, robots_text : String, agent : String) -> Array[String]

#
sitemap_urls_blocked

fn sitemap_urls_blocked(xml : String, robots_text : String, agent : String) -> Array[String]

#
sort_paths_by_depth

fn sort_paths_by_depth(paths : Array[String]) -> Array[String]

#
split_char

fn split_char(text : String, sep : Char) -> Array[String]

#
split_lines

fn split_lines(text : String) -> Array[String]

#
split_once

fn split_once(text : String, sep : String) -> Array[String]

#
starts_with_any

fn starts_with_any(text : String, prefixes : Array[String]) -> Bool

#
stats

fn stats(text : String) -> RobotsStats

#
stats_line

fn stats_line(s : RobotsStats) -> String

#
stats_of

fn stats_of(robots : Robots) -> RobotsStats

#
stats_table

fn stats_table(text : String) -> String

#
string_chars

fn string_chars(text : String) -> Array[Char]

#
strip_fragment

fn strip_fragment(text : String) -> String

#
strip_inline_comment

fn strip_inline_comment(line : String) -> String

#
strip_scheme_and_authority

fn strip_scheme_and_authority(text : String) -> String

#
suggest_directive

fn suggest_directive(key : String) -> String?

#
suggested_security_group

fn suggested_security_group(agent : String) -> Group

#
summary

fn summary(robots : Robots) -> String

#
text_replace

fn text_replace(text : String, from : String, to : String) -> String

#
transform_report

fn transform_report(text : String) -> String

#
trim_ascii

fn trim_ascii(text : String) -> String

#
unique_lower

fn unique_lower(values : Array[String]) -> Array[String]

#
unknown_directives

fn unknown_directives(text : String) -> Array[Directive]

#
upper_ascii

fn upper_ascii(text : String) -> String

#
url_host

fn url_host(url : String) -> String?

#
url_path

fn url_path(url_or_path : String) -> String

#
url_path_without_query

fn url_path_without_query(url_or_path : String) -> String

#
url_paths

fn url_paths(urls : Array[String]) -> Array[String]

#
url_query

fn url_query(url_or_path : String) -> String?

#
url_scheme_or

fn url_scheme_or(url : String, fallback : String) -> String

#
validation_report

fn validation_report(text : String) -> String

#
wildcard_prefix_match

fn wildcard_prefix_match(pattern : String, path : String) -> Bool

#
wildcard_suffix_ok

fn wildcard_suffix_ok(pattern : String, path : String) -> Bool

#
xml_blocks

fn xml_blocks(xml : String, tag : String) -> Array[String]

#
xml_tag_text

fn xml_tag_text(xml : String, tag : String) -> String

#
xml_unescape

fn xml_unescape(text : String) -> String