Download a whole state
Check the advertised byte size, stream the NDJSON file, and verify its line count against the directory.
Check the advertised byte size, stream the NDJSON file, and verify its line count against the directory.
What you will produce: A validated newline-delimited file with one district record per directory row, in slug order.
1. Read `bulk_bytes` before downloading
Use the index to find the bulk URL and advertised size before allocating storage or starting a transfer. The current release sizes are shown below.
curl
curl -sS https://www.districtfacts.com/data/v1/index.json | jq '.states[] | select(.state == "de") | {bulk_url, bulk_bytes}'JavaScript
const index = await fetch("https://www.districtfacts.com/data/v1/index.json").then(response => response.json());
const state = index.states.find(entry => entry.state === "de");
console.log({ bulk_url: state.bulk_url, bulk_bytes: state.bulk_bytes });Python
import json
from urllib.request import urlopen
with urlopen("https://www.districtfacts.com/data/v1/index.json") as response: index = json.load(response)
state = next(entry for entry in index["states"] if entry["state"] == "de")
print({"bulk_url": state["bulk_url"], "bulk_bytes": state["bulk_bytes"]})Sample output
Committed-release example
{ "bulk_url": "/data/v1/de/bulk.ndjson", "bulk_bytes": 20364509 }2. Download and stream the open file
The open host returns NDJSON directly. Process one complete district object per line.
curl
curl --fail --silent --show-error https://www.districtfacts.com/data/v1/de/bulk.ndjson -o de-districts.ndjson
head -n 1 de-districts.ndjson | jq '{slug, name}'JavaScript
const { Readable } = await import("node:stream");
const { createInterface } = await import("node:readline");
const response = await fetch("https://www.districtfacts.com/data/v1/de/bulk.ndjson");
if (!response.ok || !response.body) throw new Error(`HTTP ${response.status}`);
const lines = createInterface({ input: Readable.fromWeb(response.body) });
let count = 0;
for await (const line of lines) { JSON.parse(line); count += 1; }
console.log({ count });Python
import json
from urllib.request import urlopen
count = 0
with urlopen("https://www.districtfacts.com/data/v1/de/bulk.ndjson", timeout=60) as response:
for line in response:
json.loads(line)
count += 1
print({"count": count})Sample output
Committed-release example
{ "count": 43 }3. Ask for compression and resume large downloads
Send `Accept-Encoding: gzip` and the host compresses the NDJSON, making the transfer much smaller on the wire; the decompressed bytes and line count are unchanged. The host also accepts byte `Range` requests, so an interrupted uncompressed download can resume where it stopped. Use one or the other: a resume offset counts bytes of the uncompressed file, so do not resume a compressed transfer; restart it instead.
curl
# Compressed full download (curl decompresses on arrival)
curl --fail --compressed -o de-districts.ndjson https://www.districtfacts.com/data/v1/de/bulk.ndjson
# Or: uncompressed download that resumes after an interruption
curl --fail -C - -o de-districts.ndjson https://www.districtfacts.com/data/v1/de/bulk.ndjsonJavaScript
const { Readable } = await import("node:stream");
const { createInterface } = await import("node:readline");
// Node fetch decompresses gzip automatically.
const response = await fetch("https://www.districtfacts.com/data/v1/de/bulk.ndjson");
if (!response.ok || !response.body) throw new Error(`HTTP ${response.status}`);
const lines = createInterface({ input: Readable.fromWeb(response.body) });
let count = 0;
for await (const line of lines) { JSON.parse(line); count += 1; }
console.log({ count });Python
import gzip
import json
from urllib.request import Request, urlopen
request = Request("https://www.districtfacts.com/data/v1/de/bulk.ndjson", headers={"Accept-Encoding": "gzip"})
with urlopen(request) as response:
stream = gzip.GzipFile(fileobj=response) if response.headers.get("Content-Encoding") == "gzip" else response
records = [json.loads(line) for line in stream]
print({"count": len(records)})Sample output
The same NDJSON lines as the plain download; verify the count as in the next step4. Check the line count against the directory
The bulk file is newline-terminated and must have exactly one line per directory row.
curl
expected=$(curl -sS https://www.districtfacts.com/data/v1/de/districts.json | jq 'length')
actual=$(wc -l < de-districts.ndjson | tr -d ' ')
[ "$actual" = "$expected" ] || exit 1
echo "verified $actual records"JavaScript
const directory = await fetch("https://www.districtfacts.com/data/v1/de/districts.json").then(response => response.json());
if (count !== directory.length) throw new Error(`count mismatch: ${count} != ${directory.length}`);
console.log(`verified ${count} records`);Python
with urlopen("https://www.districtfacts.com/data/v1/de/districts.json") as response: directory = json.load(response)
if count != len(directory): raise ValueError(f"count mismatch: {count} != {len(directory)}")
print(f"verified {count} records")Sample output
verified 43 records5. Follow the keyed redirect without forwarding the key
A keyed bulk GET returns 302 with a `Location` signed for ten minutes. Request that signed URL separately, without the API Authorization header.
curl
headers=$(mktemp)
curl -sS -D "$headers" -o /dev/null -H "Authorization: Bearer $DISTRICTFACTS_API_KEY" https://api.districtfacts.com/data/v1/de/bulk.ndjson
location=$(awk 'tolower($1)=="location:" {$1=""; sub(/^ /, ""); sub(/\r$/, ""); print}' "$headers")
curl --fail --silent --show-error "$location" -o de-districts.ndjsonJavaScript
const first = await fetch("https://api.districtfacts.com/data/v1/de/bulk.ndjson", { headers: { Authorization: `Bearer ${process.env.DISTRICTFACTS_API_KEY}` }, redirect: "manual" });
if (first.status !== 302) throw new Error(`HTTP ${first.status}`);
const location = first.headers.get("location");
if (!location) throw new Error("Missing Location");
const download = await fetch(location); // No Authorization header.
console.log({ bytes: (await download.arrayBuffer()).byteLength });Python
import os
from urllib.error import HTTPError
from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
class NoRedirect(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl): return None
request = Request("https://api.districtfacts.com/data/v1/de/bulk.ndjson", headers={"Authorization": f"Bearer {os.environ['DISTRICTFACTS_API_KEY']}"})
try: build_opener(NoRedirect).open(request)
except HTTPError as response:
if response.code != 302: raise
location = response.headers["Location"]
with urlopen(location, timeout=60) as download: body = download.read() # No Authorization header.
print({"bytes": len(body)})Sample output
Response sequence
302 Location: <signed URL valid for ten minutes>
200 <NDJSON bytes>