import urllib.request
import urllib.error
import re
import time
import json
import os
import subprocess
from datetime import datetime

base_url = "https://solved.ac"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5'
}

GROUPS_JSON_PATH = os.path.join(os.path.dirname(__file__), "crawled_groups.json")
CONTESTS_JSON_PATH = os.path.join(os.path.dirname(__file__), "crawled_contests.json")
GROUPS_MD_PATH = os.path.join(os.path.dirname(__file__), "solved_ac_groups.md")

def fetch_group_html(group_id):
    url = f"{base_url}/en/problems/archive/group/{group_id}"
    req = urllib.request.Request(url, headers=headers)
    while True:
        try:
            with urllib.request.urlopen(req, timeout=15) as response:
                return response.read().decode('utf-8')
        except urllib.error.HTTPError as e:
            if e.code == 404:
                print(f"Group {group_id}: HTTP 404 Not Found")
                return None
            now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            print(f"\n[HTTP {e.code} Error] Throttled or server error on Group {group_id} at {now_str}.")
            print("Backing off and sleeping for 15 minutes (900 seconds) before retrying...")
            time.sleep(15 * 60)
            print(f"Resuming Group {group_id} retry at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}...")
            continue
        except Exception as e:
            now_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
            print(f"\n[Connection Error: {e}] on Group {group_id} at {now_str}.")
            print("Backing off and sleeping for 15 minutes (900 seconds) before retrying...")
            time.sleep(15 * 60)
            print(f"Resuming Group {group_id} retry at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}...")
            continue

def generate_markdown_report(groups_data, contests_mapped):
    """
    Renders solved_ac_groups.md directly from the cached groups hierarchy.
    """
    lines = [
        "# Solved.ac ICPC Contest Groups & Archive Pages Index",
        "",
        "This report documents the complete, fully-expanded hierarchical tree of all **ICPC contest groups and individual contest archive pages** indexed on the live **solved.ac** platform.",
        "",
        f"**Total Discovered Group Archives:** {len(groups_data)}",
        f"**Total Discovered Contest Archives:** {len(contests_mapped)}",
        "",
        "---",
        "",
        "## 🏆 Contest Archives Tree",
        ""
    ]

    def render_subtree(group_id, level=0):
        gid_str = str(group_id)
        if gid_str not in groups_data:
            return []
        
        g = groups_data[gid_str]
        indent = "    " * level
        sub_lines = []
        
        # Group bullet
        g_name = g.get("name") or f"Group {group_id}"
        sub_lines.append(f"{indent}*   **[{g_name} (Group {group_id})](https://solved.ac/problems/archive/group/{group_id})**")
        
        # Child groups
        for cg in g.get("child_groups", []):
            sub_lines.extend(render_subtree(cg["id"], level + 1))
            
        # Child contests
        for cc in g.get("child_contests", []):
            c_indent = "    " * (level + 1)
            c_name = cc.get("name") or f"Contest {cc['id']}"
            sub_lines.append(f"{c_indent}*   [{c_name}](https://solved.ac/problems/archive/contest/{cc['id']})")
            
        return sub_lines

    # Render ICPC tree (root Group 1)
    if "1" in groups_data:
        lines.extend(render_subtree(1, 0))

    # Render Group 90 tree (NAIPC / UCIPC)
    if "90" in groups_data:
        lines.append("")
        lines.extend(render_subtree(90, 0))

    with open(GROUPS_MD_PATH, "w", encoding="utf-8") as f:
        f.write("\n".join(lines) + "\n")
    print(f"Generated {GROUPS_MD_PATH}")

def crawl_all_groups_and_contests():
    groups_data = {}
    contests_mapped = {}
    
    # Load previous cache if available
    if os.path.exists(GROUPS_JSON_PATH):
        try:
            with open(GROUPS_JSON_PATH, "r", encoding="utf-8") as f:
                groups_data = json.load(f)
            print(f"Loaded {len(groups_data)} groups from {GROUPS_JSON_PATH}")
        except Exception:
            pass

    if os.path.exists(CONTESTS_JSON_PATH):
        try:
            with open(CONTESTS_JSON_PATH, "r", encoding="utf-8") as f:
                contests_mapped = json.load(f)
            print(f"Loaded {len(contests_mapped)} contests from {CONTESTS_JSON_PATH}")
        except Exception:
            pass

    visited_groups = set(groups_data.keys())
    queue = []
    
    # Check if roots need crawling
    for root_id, root_path in [("1", "ICPC"), ("90", "North American Invitational Programming Contest")]:
        if root_id not in visited_groups:
            queue.append((root_id, root_path))
        else:
            # Enqueue unvisited child groups
            for cg in groups_data[root_id].get("child_groups", []):
                cid_str = str(cg["id"])
                if cid_str not in visited_groups:
                    queue.append((cid_str, f"{root_path} > {cg.get('name', cid_str)}"))

    # Also enqueue any child groups in groups_data that haven't been visited yet
    for gid, ginfo in list(groups_data.items()):
        for cg in ginfo.get("child_groups", []):
            cid_str = str(cg["id"])
            if cid_str not in visited_groups and cid_str not in [q[0] for q in queue]:
                queue.append((cid_str, f"{ginfo.get('path', '')} > {cg.get('name', cid_str)}"))

    if queue:
        # Start persistent Node parser for group pages
        node_proc = subprocess.Popen(
            ["node", "-e", """
const readline = require("readline");
const { parseGroupTSR } = require("./parse_contest.js");
const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: false });

rl.on("line", (line) => {
  if (!line.trim()) return;
  try {
    const req = JSON.parse(line);
    const parsed = parseGroupTSR(req.html);
    console.log(JSON.stringify({ id: req.id, data: parsed }));
  } catch (err) {
    console.log(JSON.stringify({ id: null, error: err.message }));
  }
});
"""],
            stdin=subprocess.PIPE,
            stdout=subprocess.PIPE,
            text=True,
            bufsize=1
        )

        print(f"Starting group crawl for {len(queue)} queued groups...")

        while queue:
            group_id, group_path = queue.pop(0)
            if group_id in visited_groups:
                continue
                
            visited_groups.add(group_id)
            print(f"[{len(visited_groups)} visited, {len(queue)} queued] Crawling Group {group_id} ({group_path})...")
            
            html = fetch_group_html(group_id)
            if not html:
                continue
                
            payload = json.dumps({"id": group_id, "html": html}) + "\n"
            node_proc.stdin.write(payload)
            node_proc.stdin.flush()
            resp_line = node_proc.stdout.readline()
            parsed = json.loads(resp_line)
            data = parsed.get("data", {})
            
            if data.get("error"):
                print(f"  Parse error on Group {group_id}: {data['error']}")
                continue
                
            cg = data.get("contestGroup", {})
            group_name = cg.get("contestGroupName", group_path)
            
            child_groups = data.get("childGroups", [])
            child_contests = data.get("childContests", [])
            
            groups_data[group_id] = {
                "group_id": int(group_id),
                "name": group_name,
                "fullName": cg.get("contestGroupFullName", ""),
                "parent_id": cg.get("parentContestGroupId"),
                "path": group_path,
                "child_groups": child_groups,
                "child_contests": child_contests
            }
            
            # Queue child groups without any filter
            for child in child_groups:
                cid = str(child["id"])
                cname = child.get("name", f"Group {cid}")
                if cid not in visited_groups:
                    queue.append((cid, f"{group_path} > {cname}"))
                    
            # Map child contests
            for contest in child_contests:
                contest_id = str(contest["id"])
                contest_name = contest.get("name", f"Contest {contest_id}")
                if contest_id not in contests_mapped:
                    contests_mapped[contest_id] = {
                        "name": contest_name,
                        "fullName": contest.get("fullName", contest_name),
                        "year": contest.get("year"),
                        "problemCount": contest.get("problemCount", 0),
                        "path": group_path
                    }
                else:
                    if group_path not in contests_mapped[contest_id]["path"]:
                        contests_mapped[contest_id]["path"] += f" | {group_path}"
                        
            # Save checkpoints
            if len(visited_groups) % 10 == 0:
                with open(GROUPS_JSON_PATH, "w", encoding="utf-8") as out_g:
                    json.dump(groups_data, out_g, indent=2, ensure_ascii=False)
                with open(CONTESTS_JSON_PATH, "w", encoding="utf-8") as out_c:
                    json.dump(contests_mapped, out_c, indent=2, ensure_ascii=False)
                    
            time.sleep(0.4)

        node_proc.terminate()

    # Save final JSONs
    with open(GROUPS_JSON_PATH, "w", encoding="utf-8") as out_g:
        json.dump(groups_data, out_g, indent=2, ensure_ascii=False)
    with open(CONTESTS_JSON_PATH, "w", encoding="utf-8") as out_c:
        json.dump(contests_mapped, out_c, indent=2, ensure_ascii=False)
    print(f"Saved {len(groups_data)} groups to {GROUPS_JSON_PATH}")
    print(f"Saved {len(contests_mapped)} contests to {CONTESTS_JSON_PATH}")

    # Render Markdown report
    generate_markdown_report(groups_data, contests_mapped)

if __name__ == "__main__":
    crawl_all_groups_and_contests()