"""Summarize a local HackerTube export. Does not download anything.

Usage:
  python3 hackertube-quickstart.py /path/to/unzipped/hackertube-data

The zip is https://hackertube.org/download/hackertube-data.zip (see /api/export).
This script was run on 2026-11-03 against the 2026-10-31 export.

Credit HackerTube (https://hackertube.org) and the open-index/hacker-news dataset
(https://huggingface.co/datasets/open-index/hacker-news). License: ODC-By 1.0.
https://opendatacommons.org/licenses/by/1-0/
"""

import json
import re
import struct
import sys
from collections import Counter, defaultdict
from pathlib import Path

REC = {"rec", "strong_rec"}


def load_jsonl(path: Path) -> list[dict]:
    rows = []
    with path.open(encoding="utf-8") as handle:
        for line in handle:
            line = line.strip()
            if line:
                rows.append(json.loads(line))
    return rows


def normalize_title(title: str) -> str:
    return re.sub(r"[^a-z0-9]+", " ", (title or "").lower()).strip()


def read_npy_header(path: Path) -> tuple[str, tuple[int, ...], int]:
    blob = path.read_bytes()
    if blob[:6] != b"\x93NUMPY":
        raise SystemExit("embeddings.npy is not a NumPy file")
    header_len = struct.unpack("<H", blob[8:10])[0]
    header = blob[10 : 10 + header_len].decode("latin1").strip()
    descr = header.split("'descr': ")[1].split(",")[0].strip().strip("'")
    shape_text = header.split("'shape': ")[1].split("}")[0].strip().rstrip(",")
    shape = tuple(int(part) for part in shape_text.strip("() ").split(",") if part.strip())
    return descr, shape, len(blob)


def main() -> None:
    if len(sys.argv) != 2:
        raise SystemExit("usage: python3 hackertube-quickstart.py /path/to/hackertube-data")
    root = Path(sys.argv[1])
    videos = load_jsonl(root / "videos.jsonl")
    mentions = load_jsonl(root / "mentions.jsonl")
    related = load_jsonl(root / "related.jsonl")
    taxonomy = json.loads((root / "taxonomy.json").read_text(encoding="utf-8"))
    embedding_ids = json.loads((root / "embedding_ids.json").read_text(encoding="utf-8"))
    descr, shape, npy_bytes = read_npy_header(root / "embeddings.npy")

    by_vid: dict[str, list[dict]] = defaultdict(list)
    for mention in mentions:
        by_vid[mention["vid"]].append(mention)

    mismatches = 0
    strong_field_differs = 0
    stances: Counter[str] = Counter()
    content_types: Counter[str] = Counter()
    shelves: Counter[str] = Counter()
    levels: Counter[str] = Counter()
    category_videos: Counter[str] = Counter()
    area_videos: Counter[str] = Counter()
    categories_per_video: Counter[int] = Counter()
    year_mentions: Counter[str] = Counter()
    year_recs: Counter[str] = Counter()
    year_videos: dict[str, set[str]] = defaultdict(set)
    text_lengths = []
    title_groups: Counter[str] = Counter()

    for mention in mentions:
        stances[mention.get("stance")] += 1
        year = (mention.get("month") or "")[:4]
        year_mentions[year] += 1
        text_lengths.append(len(mention.get("text") or ""))
        if mention.get("stance") in REC:
            year_recs[year] += 1
            year_videos[year].add(mention["vid"])

    for video in videos:
        kept = by_vid[video["vid"]]
        rec_months = set()
        strong_months = set()
        recommender_ids = set()
        stance_counts: Counter[str] = Counter()
        for mention in kept:
            stance = mention.get("stance")
            stance_counts[stance] += 1
            if stance in REC:
                rec_months.add(mention.get("month"))
                recommender_ids.add(mention.get("author"))
            if stance == "strong_rec":
                strong_months.add(mention.get("month"))
        hn = video["hn"]
        expected_score = len(rec_months) + 0.5 * len(strong_months)
        if (
            hn.get("mentions") != len(kept)
            or hn.get("rec_months") != len(rec_months)
            or hn.get("recommenders") != len(recommender_ids)
            or hn.get("strong_recs") != stance_counts["strong_rec"]
            or dict(hn.get("stances") or {}) != dict(stance_counts)
            or abs(float(video.get("score")) - expected_score) > 1e-9
        ):
            mismatches += 1
        if hn.get("strong_recs") != len(strong_months):
            strong_field_differs += 1
        content_types[video.get("content_type")] += 1
        shelves[video.get("shelf_life")] += 1
        levels[video.get("level")] += 1
        title_groups[normalize_title(video.get("title") or "")] += 1
        seen_categories = set()
        seen_areas = set()
        for category in video.get("categories") or []:
            seen_categories.add(category["id"])
            seen_areas.add(category["area"])
        categories_per_video[len(seen_categories)] += 1
        for category_id in seen_categories:
            category_videos[category_id] += 1
        for area_id in seen_areas:
            area_videos[area_id] += 1

    areas = taxonomy["areas"]
    categories = [category for area in areas for category in area["categories"]]
    category_names = {category["id"]: category["name"] for category in categories}
    area_names = {area["id"]: area["name"] for area in areas}
    video_ids = {video["vid"] for video in videos}
    rec_authors = {
        mention["author"] for mention in mentions if mention.get("stance") in REC
    }
    duplicate_groups = [count for count in title_groups.values() if count > 1]
    text_lengths.sort()
    zero_recommenders = sum(1 for video in videos if video["hn"]["recommenders"] == 0)

    print(f"videos {len(videos)}")
    print(f"mentions {len(mentions)}")
    print(f"related {len(related)}")
    print(f"areas {len(areas)}")
    print(f"categories {len(categories)}")
    print(f"taxonomy_version {taxonomy.get('version')}")
    print(f"embedding_ids {len(embedding_ids)}")
    print(f"embedding_id_set_matches_videos {set(embedding_ids) == video_ids}")
    print(f"npy_descr {descr}")
    print(f"npy_shape {shape[0]} {shape[1]}")
    print(f"npy_bytes {npy_bytes}")
    print(f"score_mismatches {mismatches}")
    print(f"strong_recs_field_differs_from_strong_months {strong_field_differs}")
    print(f"recommenders_eq_0 {zero_recommenders}")
    print(f"sum_hn_recommenders {sum(video['hn']['recommenders'] for video in videos)}")
    print(f"distinct_mention_authors {len({mention['author'] for mention in mentions})}")
    print(f"distinct_recommender_authors {len(rec_authors)}")
    print(f"mentions_gt_20 {sum(1 for video in videos if video['hn']['mentions'] > 20)}")
    print(f"text_len_min {text_lengths[0]}")
    print(f"text_len_median {text_lengths[len(text_lengths) // 2]}")
    print(f"text_len_max {text_lengths[-1]}")
    print(f"text_len_eq_1500 {sum(1 for length in text_lengths if length == 1500)}")
    print(f"duplicate_title_groups {len(duplicate_groups)}")
    print(f"duplicate_title_extra_rows {sum(count - 1 for count in duplicate_groups)}")
    print("stances")
    for name, count in stances.most_common():
        print(f"  {name} {count}")
    print("content_type")
    for name, count in content_types.most_common():
        print(f"  {name} {count}")
    print("level")
    for name, count in levels.most_common():
        print(f"  {name} {count}")
    print("shelf_life")
    for name, count in shelves.most_common():
        print(f"  {name} {count}")
    print("categories_per_video")
    for size, count in sorted(categories_per_video.items()):
        print(f"  {size} {count}")
    print(f"area_memberships {sum(area_videos.values())}")
    print("top_categories")
    for category_id, count in category_videos.most_common(8):
        print(f"  {count} {category_names[category_id]}")
    print("areas")
    for area_id, count in area_videos.most_common():
        print(f"  {count} {area_names[area_id]}")
    print("mentions_by_year")
    for year in sorted(year_mentions):
        print(
            f"  {year} mentions {year_mentions[year]} "
            f"recommendations {year_recs[year]} videos {len(year_videos[year])}"
        )
    print("top_videos")
    for video in videos[:5]:
        hn = video["hn"]
        print(
            f"  {video['score']} {hn['rec_months']} {hn['strong_recs']} "
            f"{hn['recommenders']} {video['channel']} | {video['title']}"
        )
    if mismatches:
        raise SystemExit(f"{mismatches} videos did not match the published score formula")


if __name__ == "__main__":
    main()
