Keep a local copy current

Store per-record SHA-256 values, fetch only changed or new records, and report removals.

Store per-record SHA-256 values, fetch only changed or new records, and report removals.

What you will produce: A deterministic local snapshot and checkpoint, without assuming a change feed or webhook exists.

1. Download the first snapshot and store every hash

On the first run, download and verify every directory record, then store a checkpoint map from district slug to SHA-256.

curl

mkdir -p de-records
base=https://www.districtfacts.com
curl -sS "$base/data/v1/de/districts.json" -o de-directory.json
jq -r '.[] | [.slug, .api_url, .sha256] | @tsv' de-directory.json | while IFS=
  

\t' read -r slug path expected; do curl -sS "$base$path" -o "de-records/$slug.json"; [ "$(shasum -a 256 "de-records/$slug.json" | awk '{print $1}')" = "$expected" ] || exit 1; done
jq 'map({key: .slug, value: .sha256}) | from_entries' de-directory.json > de-checkpoint.json
echo "synced $(jq 'length' de-checkpoint.json) records"

JavaScript

const fs = await import("node:fs/promises");
const nodeCrypto = await import("node:crypto");
const base = "https://www.districtfacts.com";
const directory = await fetch(base + "/data/v1/de/districts.json").then(response => response.json());
await fs.mkdir("de-records", { recursive: true });
for (const record of directory) {
  const body = Buffer.from(await fetch(base + record.api_url).then(response => response.arrayBuffer()));
  if (nodeCrypto.createHash("sha256").update(body).digest("hex") !== record.sha256) throw new Error(`hash mismatch for ${record.slug}`);
  await fs.writeFile(`de-records/${record.slug}.json`, body);
}
const checkpoint = Object.fromEntries(directory.map(record => [record.slug, record.sha256]));
await fs.writeFile("de-checkpoint.json", JSON.stringify(checkpoint, null, 2));
console.log({ synced: Object.keys(checkpoint).length });

Python

import hashlib, json, os
from urllib.request import urlopen
base = "https://www.districtfacts.com"
with urlopen(base + "/data/v1/de/districts.json") as response: directory = json.load(response)
os.makedirs("de-records", exist_ok=True)
for record in directory:
    with urlopen(base + record["api_url"]) as response: body = response.read()
    if hashlib.sha256(body).hexdigest() != record["sha256"]: raise ValueError(f"hash mismatch for {record['slug']}")
    with open(f"de-records/{record['slug']}.json", "wb") as output: output.write(body)
checkpoint = {record["slug"]: record["sha256"] for record in directory}
with open("de-checkpoint.json", "w", encoding="utf-8") as output: json.dump(checkpoint, output, indent=2)
print({"synced": len(checkpoint)})

Sample output

Committed-release example
{ "synced": 43 }

2. Fetch only changed or new records

Compare the next directory with the saved map. Verify each response hash before replacing the local file.

curl

mkdir -p de-records
base=https://www.districtfacts.com
curl -sS "$base/data/v1/de/districts.json" -o de-directory.next.json
jq -r --slurpfile old de-checkpoint.json '.[] | select($old[0][.slug] != .sha256) | [.slug, .api_url, .sha256] | @tsv' de-directory.next.json | while IFS=
  

\t' read -r slug path expected; do curl -sS "$base$path" -o "de-records/$slug.json"; [ "$(shasum -a 256 "de-records/$slug.json" | awk '{print $1}')" = "$expected" ] || exit 1; echo "updated $slug"; done

JavaScript

const previous = JSON.parse(await fs.readFile("de-checkpoint.json", "utf8"));
const nextDirectory = await fetch(base + "/data/v1/de/districts.json").then(response => response.json());
await fs.mkdir("de-records", { recursive: true });
for (const record of nextDirectory) {
  if (previous[record.slug] === record.sha256) continue;
  const body = Buffer.from(await fetch(base + record.api_url).then(response => response.arrayBuffer()));
  if (nodeCrypto.createHash("sha256").update(body).digest("hex") !== record.sha256) throw new Error(`hash mismatch for ${record.slug}`);
  await fs.writeFile(`de-records/${record.slug}.json`, body);
  console.log("updated", record.slug);
}

Python

import hashlib, os
with open("de-checkpoint.json", encoding="utf-8") as source: previous = json.load(source)
with urlopen("https://www.districtfacts.com/data/v1/de/districts.json") as response: next_directory = json.load(response)
os.makedirs("de-records", exist_ok=True)
for record in next_directory:
    if previous.get(record["slug"]) == record["sha256"]: continue
    with urlopen("https://www.districtfacts.com" + record["api_url"]) as response: body = response.read()
    if hashlib.sha256(body).hexdigest() != record["sha256"]: raise ValueError(f"hash mismatch for {record['slug']}")
    with open(f"de-records/{record['slug']}.json", "wb") as output: output.write(body)
    print("updated", record["slug"])

Sample output

Sample output when one record changes
updated capital-school-district

3. Report removals and save the checkpoint

Report a slug absent from the new directory for review. Do not silently delete its local file.

curl

jq -r --slurpfile next de-directory.next.json 'keys[] as $slug | select(any($next[0][]; .slug == $slug) | not) | "removed \($slug)"' de-checkpoint.json
jq 'map({key: .slug, value: .sha256}) | from_entries' de-directory.next.json > de-checkpoint.json

JavaScript

const nextCheckpoint = Object.fromEntries(nextDirectory.map(record => [record.slug, record.sha256]));
for (const slug of Object.keys(previous)) if (!(slug in nextCheckpoint)) console.log("removed", slug);
await fs.writeFile("de-checkpoint.json", JSON.stringify(nextCheckpoint, null, 2));

Python

next_checkpoint = {record["slug"]: record["sha256"] for record in next_directory}
for slug in previous:
    if slug not in next_checkpoint: print("removed", slug)
with open("de-checkpoint.json", "w", encoding="utf-8") as output: json.dump(next_checkpoint, output, indent=2)

Sample output

Sample output
removed example-district

4. Use ETag on the keyed host, and wait out a 429

Send the keyed host’s weak ETag in `If-None-Match`. A 304 consumes the minute bucket but not day, month, or bulk counters. The keyed host answers 429 when a limit is reached: wait the number of seconds in `Retry-After` (one second if the header is absent), retry, and stop after a fixed number of attempts. The open host used in the steps above applies no quota and sends no rate-limit headers.

curl

url=https://api.districtfacts.com/data/v1/de/districts.json
for attempt in 1 2 3 4 5; do
  headers=$(mktemp)
  code=$(curl -sS -D "$headers" -o de-directory.keyed.json -w '%{http_code}' -H "Authorization: Bearer $DISTRICTFACTS_API_KEY" -H 'If-None-Match: W/"<sha256>"' "$url")
  [ "$code" != 429 ] && break
  [ "$attempt" = 5 ] && break
  wait=$(awk 'tolower($1)=="retry-after:" {gsub("\r", "", $2); print $2}' "$headers")
  sleep "${wait:-1}"
done
case "$code" in
  304) echo unchanged ;;
  200) grep -i '^etag:' "$headers" ;;
  *) echo "HTTP $code" >&2; exit 1 ;;
esac

JavaScript

const url = "https://api.districtfacts.com/data/v1/de/districts.json";
const headers = { Authorization: `Bearer ${process.env.DISTRICTFACTS_API_KEY}`, "If-None-Match": 'W/"<sha256>"' };
let response;
for (let attempt = 0; attempt < 5; attempt++) {
  response = await fetch(url, { headers });
  if (response.status !== 429 || attempt === 4) break;
  const seconds = Number(response.headers.get("retry-after") ?? 1);
  await new Promise(resolve => setTimeout(resolve, seconds * 1000));
}
if (response.status === 304) console.log("unchanged");
else if (!response.ok) throw new Error(`HTTP ${response.status}`);
else console.log({ etag: response.headers.get("etag"), directory: await response.json() });

Python

import json, os, time
from urllib.error import HTTPError
from urllib.request import Request, urlopen
request = Request("https://api.districtfacts.com/data/v1/de/districts.json", headers={"Authorization": f"Bearer {os.environ['DISTRICTFACTS_API_KEY']}", "If-None-Match": 'W/"<sha256>"'})
for attempt in range(5):
    try:
        with urlopen(request) as response: print({"etag": response.headers["ETag"], "directory": json.load(response)})
        break
    except HTTPError as error:
        if error.code == 304:
            print("unchanged")
            break
        if error.code != 429 or attempt == 4: raise
        time.sleep(int(error.headers.get("Retry-After", "1")))

Sample output

HTTP/1.1 304 Not Modified
ETag: W/"<sha256>"

There is no change feed or webhook. A changed hash can reflect a correction, definition change, format release, or new period.

Developer documentation