#!/usr/bin/env python3
"""Re-fetch robots.txt for every site named in the report and save the bytes.

Fetches go through the same rotating proxy as the main run (PROXY_URL in the
environment, never written to any output). Each file in evidence/ carries the
fetch timestamp, final URL, HTTP status, the user-agent used, the byte count and
the SHA-256 of the body, followed by the body. The state printed for each site is
recomputed from the file that was just saved, so nothing is asserted about a
named site without a saved file behind it.
"""
import hashlib, json, os, subprocess, sys, tempfile, time
import robots_rfc9309 as R

PROXY = os.environ.get("PROXY_URL")
ENV = dict(os.environ)
if PROXY:
    ENV.update(https_proxy=PROXY, http_proxy=PROXY, ALL_PROXY=PROXY)

UA = ("FentnerResearchBot/1.0 (+https://fentner.com/research; robots.txt "
      "measurement study; contact: research@fentner.com)")
UA_B = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
        "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36")
OUT = "evidence"
DOMAINS = json.load(open(sys.argv[1]))
VENDORS = json.load(open("vendors.json"))
TOKENS = [t["token"] for v in VENDORS.values() for t in v["tokens"]]


def get(url, ua, attempts=3):
    last = (-1, url, "", "")
    for _ in range(attempts):
        fd, path = tempfile.mkstemp(prefix="ev_")
        os.close(fd)
        cmd = ["curl", "-s", "-L", "-k", "--compressed", "--connect-timeout", "12",
               "--max-time", "40", "--max-redirs", "5", "-A", ua,
               "-H", "Accept: */*", "-H", "Accept-Language: en-GB,en;q=0.9",
               "-o", path, "-w", "%{http_code}\t%{url_effective}\t%{content_type}", url]
        try:
            p = subprocess.run(cmd, capture_output=True, timeout=70, env=ENV)
            meta = p.stdout.decode("utf-8", "replace").split("\t")
            code = int(meta[0]) if meta and meta[0].isdigit() else -1
            body = open(path, "rb").read().decode("utf-8", "replace")
            last = (code, meta[1] if len(meta) > 1 else url,
                    meta[2] if len(meta) > 2 else "", body)
        except Exception:
            pass
        finally:
            try: os.unlink(path)
            except OSError: pass
        if last[0] == 200:
            return last + (ua,)
    return last + (ua,)


os.makedirs(OUT, exist_ok=True)
results = {}
for dom in DOMAINS:
    url = f"https://{dom}/robots.txt"
    st, final, ctype, body, used = get(url, UA)
    if st != 200:
        st, final, ctype, body, used = get(url, UA_B)
    stamp = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
    digest = hashlib.sha256(body.encode()).hexdigest()
    header = (f"# fetched {stamp}\n# url {final}\n# http {st}\n"
              f"# content-type {ctype}\n# user-agent {used}\n"
              f"# via rotating proxy: {'yes' if PROXY else 'no'}\n"
              f"# sha256 {digest}\n# bytes {len(body.encode())}\n\n")
    open(os.path.join(OUT, dom + ".txt"), "w").write(header + body)
    groups, _sm, _o = R.parse(body)
    row = {"http": st, "url": final, "fetched": stamp, "bytes": len(body.encode()),
           "sha256": digest, "star": R.star_state(groups), "tokens": {}}
    for tok in TOKENS:
        s = R.state(groups, tok.lower())
        if s != "absent":
            row["tokens"][tok] = s
    results[dom] = row
    print(f"{dom:22s} http={st:4d} bytes={row['bytes']:7d} star={row['star']:8s} " +
          ", ".join(f"{k}={v}" for k, v in row["tokens"].items()), flush=True)
json.dump(results, open("evidence_summary.json", "w"), indent=1)