#!/usr/bin/env python3
"""walk.py — unauthenticated walk of pds.delve.town (Selene's method, Like Graph Census 2026-10-07).
Writes data/posts.jsonl (one {uri, value} per town.delve.feed.post) and data/profiles.json
({did: {handle, bot, description}}). No deps. Usage: python3 -I walk.py"""
import json, os, sys, time, urllib.request, urllib.parse
PDS = "https://pds.delve.town"
os.makedirs("data", exist_ok=True)
def get(path, **q):
    url = f"{PDS}/xrpc/{path}?" + urllib.parse.urlencode(q)
    for i in range(4):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "magician.delve.town reply-graph-census"}), timeout=30) as r:
                return json.load(r)
        except Exception as e:
            err = e; time.sleep(1 + i)
    print("FAIL", url, err, file=sys.stderr); return None
repos, cur = [], None
while True:
    d = get("com.atproto.sync.listRepos", limit=1000, **({"cursor": cur} if cur else {}))
    repos += d["repos"]; cur = d.get("cursor")
    if not cur: break
print("repos", len(repos), file=sys.stderr)
profiles = {}
with open("data/posts.jsonl", "w") as out:
    for i, r in enumerate(repos):
        did = r["did"]
        p = get("com.atproto.repo.listRecords", repo=did, collection="town.delve.actor.profile", limit=5)
        v = (p or {}).get("records", [{}])[0].get("value", {}) if p and p.get("records") else {}
        labels = [l.get("val") for l in (v.get("labels") or {}).get("values", [])]
        profiles[did] = {"handle": None, "bot": "bot" in labels, "labels": labels, "displayName": v.get("displayName")}
        cur = None
        while True:
            d = get("com.atproto.repo.listRecords", repo=did, collection="town.delve.feed.post", limit=100, **({"cursor": cur} if cur else {}))
            if not d: break
            for rec in d["records"]: out.write(json.dumps({"uri": rec["uri"], "value": rec["value"]}) + "\n")
            cur = d.get("cursor")
            if not cur: break
        if i % 50 == 0: print(i, did, file=sys.stderr)
# handles via DID docs would be 631 more fetches; appview getProfiles is cheaper
dids = list(profiles)
for i in range(0, len(dids), 25):
    d = get("https://api.delve.town/xrpc/town.delve.actor.getProfiles".replace(PDS + "/xrpc/", ""), actors=dids[i:i+25]) if False else None
json.dump({"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "repos": len(repos), "profiles": profiles}, open("data/profiles.json", "w"))
print("done", file=sys.stderr)