How to Pull Building Permits from Any US City's Open-Data API (Socrata, ArcGIS, CKAN) in Python

import requests
from datetime import date, datetime, timedelta, timezone
SINCE = date.today() - timedelta(days=30)

def socrata(domain, dataset, date_field, limit=1000):
    url = f"https://{domain}/resource/{dataset}.json"
    out, offset = [], 0
    while True:
        page = requests.get(url, params={
            "$where": f"{date_field} >= '{SINCE}T00:00:00'",
            "$order": f"{date_field} DESC, :id",   # stable order for paging
            "$limit": limit, "$offset": offset}, timeout=120).json()
        out += page
        if len(page) < limit: return out
        offset += limit

def arcgis(layer, date_field, page_size=1000):
    out, offset = [], 0
    while True:
        r = requests.get(layer + "/query", params={
            "where": f"{date_field} >= DATE '{SINCE}'", "outFields": "*", "f": "json",
            "orderByFields": f"{date_field} DESC", "resultOffset": offset,
            "resultRecordCount": page_size}, timeout=120).json()
        feats = [f["attributes"] for f in r.get("features", [])]
        for a in feats:  # ArcGIS dates are epoch milliseconds
            if isinstance(a.get(date_field), (int, float)):
                a[date_field] = datetime.fromtimestamp(a[date_field] / 1000, timezone.utc).date().isoformat()
        out += feats
        if not r.get("exceededTransferLimit"): return out
        offset += len(feats)

def ckan(domain, resource_id, date_field, limit=5000):
    sql = (f'SELECT * FROM "{resource_id}" WHERE "{date_field}" >= \'{SINCE}\' '
           f'ORDER BY "{date_field}" DESC LIMIT {limit}')
    r = requests.get(f"https://{domain}/api/3/action/datastore_search_sql", params={"sql": sql}, timeout=120).json()
    return r["result"]["records"]

chicago = socrata("data.cityofchicago.org", "ydr8-5enu", "issue_date")
louisville = arcgis("https://services1.arcgis.com/79kfd2K6fskCAkyg/arcgis/rest/services/"
                    "active_construction_permits/FeatureServer/0", "ISSUE_DATE")
pittsburgh = ckan("data.wprdc.org", "f4d1177a-f597-4c32-8cbf-7885f56253f6", "issue_date")

def norm(city, r, num, date_f, desc, value, addr, contractor=None):
    return {"city": city, "permit": r.get(num), "issued": str(r.get(date_f))[:10],
            "description": r.get(desc), "value": float(r.get(value) or 0),
            "address": r.get(addr), "contractor": (r.get(contractor) or "").strip() or None}

rows = ([norm("Chicago", r, "permit_", "issue_date", "work_description", "reported_cost", "street_name") for r in chicago]
      + [norm("Louisville", r, "PERMIT_NUMBER", "ISSUE_DATE", "WORK_TYPE", "PROJECT_COSTS", "ADDRESS", "CONTRACTOR") for r in louisville]
      + [norm("Pittsburgh", r, "permit_id", "issue_date", "work_description", "total_project_value", "address", "contractor_name") for r in pittsburgh])
rows.sort(key=lambda r: r["issued"], reverse=True)
big = [r for r in rows if r["value"] >= 100_000]
print(len(rows), "permits,", len(big), "over $100k")