"""A robots.txt parser following RFC 9309.

Only the parts the study needs, but the parts it needs are done properly:

* Fields are case-insensitive; product tokens are matched case-insensitively.
* A group is one or more consecutive user-agent lines followed by that group's
  rules. A user-agent line that follows a rule line starts a new group.
* Several groups may name the same product token; RFC 9309 section 2.2.1 says
  their rules are merged.
* A crawler uses the group whose product token is the longest match for its own
  name, falling back to the "*" group only if no group names it.
* Path matching supports "*" and "$"; the most specific (longest) matching rule
  wins, and Allow wins ties (section 2.2.2).
"""
import re

FIELD = re.compile(r"^([A-Za-z0-9_\-]+)\s*:\s*(.*)$")


def parse(body):
    """Return (groups, sitemaps, other_fields).

    groups is a list of (set_of_lowercase_tokens, [(rule_type, value)]).
    """
    groups = []
    cur_tokens, cur_rules, last_was_ua = None, None, False
    sitemaps, other = [], []
    for raw in body.splitlines():
        line = raw.split("#", 1)[0].strip().lstrip("")
        if not line:
            continue
        m = FIELD.match(line)
        if not m:
            other.append(line[:80])
            continue
        field, value = m.group(1).strip().lower(), m.group(2).strip()
        if field == "sitemap":
            sitemaps.append(value)
            last_was_ua = False
        elif field == "user-agent":
            if not last_was_ua:
                if cur_tokens:
                    groups.append((cur_tokens, cur_rules))
                cur_tokens, cur_rules = set(), []
            if value:
                cur_tokens.add(value.lower())
            last_was_ua = True
        elif field in ("allow", "disallow"):
            if cur_tokens is None:
                cur_tokens, cur_rules = set(), []
            cur_rules.append((field, value))
            last_was_ua = False
        else:
            other.append(field)
            last_was_ua = False
    if cur_tokens:
        groups.append((cur_tokens, cur_rules))
    return groups, sitemaps, other


def _to_regex(pattern):
    out = ["^"]
    for i, ch in enumerate(pattern):
        if ch == "*":
            out.append(".*")
        elif ch == "$" and i == len(pattern) - 1:
            out.append("$")
        else:
            out.append(re.escape(ch))
    try:
        return re.compile("".join(out))
    except re.error:
        return None


def _matches(rule_value, path):
    if rule_value == "":
        return False
    if "*" not in rule_value and "$" not in rule_value:
        return path.startswith(rule_value)
    rx = _to_regex(rule_value)
    return bool(rx and rx.match(path))


def allowed(rules, path="/"):
    """RFC 9309 section 2.2.2 longest-match evaluation. True if path is allowed."""
    best_len, best_type = -1, None
    for rtype, value in rules:
        if _matches(value, path):
            n = len(value)
            if n > best_len or (n == best_len and rtype == "allow"):
                best_len, best_type = n, rtype
    if best_type is None:
        return True
    return best_type == "allow"


def group_for(groups, token):
    """Rules that apply to `token`: merged rules of every group naming it.

    Returns (rules, matched) where matched is the token string used, or None
    when the file never names this crawler (the caller decides whether the "*"
    group then applies).
    """
    tok = token.lower()
    hits = [rules for tokens, rules in groups if tok in tokens]
    if hits:
        return [r for rules in hits for r in rules], tok
    return None, None


def star_rules(groups):
    hits = [rules for tokens, rules in groups if "*" in tokens]
    return [r for rules in hits for r in rules] if hits else None


def state(groups, token):
    """Per-crawler state.

    'block'   - the file names this crawler and disallows the site root
    'partial' - the file names it and disallows something, but not the root
    'allow'   - the file names it and disallows nothing
    'absent'  - the file never names it (the '*' group may still apply, which we
                record separately; an AI-blocking intent is never expressed this way)
    """
    rules, _ = group_for(groups, token)
    if rules is None:
        return "absent"
    if not allowed(rules, "/"):
        return "block"
    if any(t == "disallow" and v for t, v in rules):
        return "partial"
    return "allow"


def star_state(groups):
    rules = star_rules(groups)
    if rules is None:
        return "absent"
    if not allowed(rules, "/"):
        return "block"
    if any(t == "disallow" and v for t, v in rules):
        return "partial"
    return "allow"