"""Fetch /robots.txt for the Tranco top N with two user-agents, via curl.

curl is used rather than urllib because it enforces a hard wall-clock limit per
request; a server that trickles bytes cannot stall the run. Each domain is
fetched twice: once with an honest research user-agent naming the study, once
with a browser user-agent, so that a site refusing the research agent can be
identified instead of being recorded as having no robots.txt.
"""
import json, gzip, os, subprocess, sys, tempfile, hashlib, time
from concurrent.futures import ThreadPoolExecutor

UA_RESEARCH = ("FentnerResearchBot/1.0 (+https://fentner.com/research; "
               "robots.txt measurement study; contact: research@fentner.com)")
UA_BROWSER = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
              "(KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36")
MAXB = 300000


def get(url, ua):
    fd, path = tempfile.mkstemp(prefix="rb_")
    os.close(fd)
    cmd = ["curl", "-s", "-L", "-k", "--compressed",
           "--connect-timeout", "8", "--max-time", "20", "--max-redirs", "5",
           "--max-filesize", str(MAXB * 4),
           "-A", ua, "-H", "Accept: */*", "-H", "Accept-Language: en-GB,en;q=0.9",
           "-o", path, "-w", "%{http_code}\t%{url_effective}\t%{content_type}", url]
    try:
        p = subprocess.run(cmd, capture_output=True, timeout=40)
        meta = p.stdout.decode("utf-8", "replace").split("\t")
        code = int(meta[0]) if meta and meta[0].isdigit() else -1
        final = meta[1] if len(meta) > 1 else url
        ctype = meta[2][:120] if len(meta) > 2 else ""
        with open(path, "rb") as f:
            body = f.read(MAXB).decode("utf-8", "replace")
        if p.returncode != 0 and code == 0:
            code = -1
    except Exception:
        code, final, ctype, body = -1, url, "", ""
    finally:
        try: os.unlink(path)
        except OSError: pass
    return {"status": code, "url": final, "ctype": ctype, "body": body}


def sha(s):
    return hashlib.sha256(s.encode("utf-8", "replace")).hexdigest()[:16]


def job(t):
    rank, d = t
    u = f"https://{d}/robots.txt"
    r = get(u, UA_RESEARCH)
    if r["status"] in (-1, 0):
        u2 = f"https://www.{d}/robots.txt"
        r2 = get(u2, UA_RESEARCH)
        if r2["status"] not in (-1, 0):
            u, r = u2, r2
    b = get(u, UA_BROWSER)
    out = {"rank": rank, "domain": d, "url": u,
           "r_status": r["status"], "r_url": r["url"], "r_ctype": r["ctype"],
           "b_status": b["status"], "b_url": b["url"], "b_ctype": b["ctype"],
           "r_sha": sha(r["body"]), "b_sha": sha(b["body"]),
           "r_len": len(r["body"]), "b_len": len(b["body"]),
           "r_body": r["body"]}
    out["b_body"] = "" if out["r_sha"] == out["b_sha"] else b["body"]
    return out


lo, hi, listfile, outfile = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3], sys.argv[4]
skip = set(json.load(open(sys.argv[5]))) if len(sys.argv) > 5 else set()
doms = []
for line in open(listfile):
    rk, d = line.strip().split(",", 1); rk = int(rk)
    if lo <= rk <= hi and d not in skip:
        doms.append((rk, d))
    if rk > hi:
        break
print("to fetch:", len(doms), flush=True)
t0 = time.time()
with gzip.open(outfile, "wt") as f, ThreadPoolExecutor(max_workers=20) as ex:
    futs = [ex.submit(job, t) for t in doms]
    for i, fu in enumerate(futs):
        f.write(json.dumps(fu.result()) + "\n")
        if i % 500 == 0:
            f.flush()
            print(i, len(doms), f"{time.time()-t0:.0f}s", flush=True)
print("done", time.time() - t0, flush=True)