#!/usr/bin/env python3
"""Report 5-gram overlap between one report and every other report in the repo."""
import re, sys, glob, os, collections

TARGET = sys.argv[1]
DIRS = sys.argv[2:] or [os.path.dirname(TARGET)]


def words(path):
    t = open(path).read()
    t = re.sub(r"^---.*?^---", "", t, flags=re.S | re.M)          # front matter
    t = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", t)                 # link targets
    t = re.sub(r"[^A-Za-z0-9' ]+", " ", t).lower()
    return t.split()


def grams(ws, n=5):
    return {" ".join(ws[i:i + n]) for i in range(len(ws) - n + 1)}


tw = words(TARGET)
tg = grams(tw)
hits = collections.defaultdict(set)
for d in DIRS:
    for p in sorted(glob.glob(os.path.join(d, "*.md"))):
        if os.path.abspath(p) == os.path.abspath(TARGET):
            continue
        common = tg & grams(words(p))
        for c in common:
            hits[c].add(os.path.basename(p))
print(f"{TARGET}: {len(tw)} words, {len(tg)} distinct 5-grams")
print(f"overlapping 5-grams with other reports: {len(hits)}")
for g, fs in sorted(hits.items()):
    print(f"  {g!r}  <- {sorted(fs)}")