Download a whole state

Check the advertised byte size, stream the NDJSON file, and verify its line count against the directory.

Check the advertised byte size, stream the NDJSON file, and verify its line count against the directory.

What you will produce: A validated newline-delimited file with one district record per directory row, in slug order.

1. Read `bulk_bytes` before downloading

Use the index to find the bulk URL and advertised size before allocating storage or starting a transfer. The current release sizes are shown below.

curl

curl -sS https://www.districtfacts.com/data/v1/index.json | jq '.states[] | select(.state == "de") | {bulk_url, bulk_bytes}'

JavaScript

const index = await fetch("https://www.districtfacts.com/data/v1/index.json").then(response => response.json());
const state = index.states.find(entry => entry.state === "de");
console.log({ bulk_url: state.bulk_url, bulk_bytes: state.bulk_bytes });

Python

import json
from urllib.request import urlopen
with urlopen("https://www.districtfacts.com/data/v1/index.json") as response: index = json.load(response)
state = next(entry for entry in index["states"] if entry["state"] == "de")
print({"bulk_url": state["bulk_url"], "bulk_bytes": state["bulk_bytes"]})

Sample output

Committed-release example
{ "bulk_url": "/data/v1/de/bulk.ndjson", "bulk_bytes": 20364509 }

2. Download and stream the open file

The open host returns NDJSON directly. Process one complete district object per line.

curl

curl --fail --silent --show-error https://www.districtfacts.com/data/v1/de/bulk.ndjson -o de-districts.ndjson
head -n 1 de-districts.ndjson | jq '{slug, name}'

JavaScript

const { Readable } = await import("node:stream");
const { createInterface } = await import("node:readline");
const response = await fetch("https://www.districtfacts.com/data/v1/de/bulk.ndjson");
if (!response.ok || !response.body) throw new Error(`HTTP ${response.status}`);
const lines = createInterface({ input: Readable.fromWeb(response.body) });
let count = 0;
for await (const line of lines) { JSON.parse(line); count += 1; }
console.log({ count });

Python

import json
from urllib.request import urlopen
count = 0
with urlopen("https://www.districtfacts.com/data/v1/de/bulk.ndjson", timeout=60) as response:
    for line in response:
        json.loads(line)
        count += 1
print({"count": count})

Sample output

Committed-release example
{ "count": 43 }

3. Ask for compression and resume large downloads

Send `Accept-Encoding: gzip` and the host compresses the NDJSON, making the transfer much smaller on the wire; the decompressed bytes and line count are unchanged. The host also accepts byte `Range` requests, so an interrupted uncompressed download can resume where it stopped. Use one or the other: a resume offset counts bytes of the uncompressed file, so do not resume a compressed transfer; restart it instead.

curl

# Compressed full download (curl decompresses on arrival)
curl --fail --compressed -o de-districts.ndjson https://www.districtfacts.com/data/v1/de/bulk.ndjson

# Or: uncompressed download that resumes after an interruption
curl --fail -C - -o de-districts.ndjson https://www.districtfacts.com/data/v1/de/bulk.ndjson

JavaScript

const { Readable } = await import("node:stream");
const { createInterface } = await import("node:readline");
// Node fetch decompresses gzip automatically.
const response = await fetch("https://www.districtfacts.com/data/v1/de/bulk.ndjson");
if (!response.ok || !response.body) throw new Error(`HTTP ${response.status}`);
const lines = createInterface({ input: Readable.fromWeb(response.body) });
let count = 0;
for await (const line of lines) { JSON.parse(line); count += 1; }
console.log({ count });

Python

import gzip
import json
from urllib.request import Request, urlopen
request = Request("https://www.districtfacts.com/data/v1/de/bulk.ndjson", headers={"Accept-Encoding": "gzip"})
with urlopen(request) as response:
    stream = gzip.GzipFile(fileobj=response) if response.headers.get("Content-Encoding") == "gzip" else response
    records = [json.loads(line) for line in stream]
print({"count": len(records)})

Sample output

The same NDJSON lines as the plain download; verify the count as in the next step

4. Check the line count against the directory

The bulk file is newline-terminated and must have exactly one line per directory row.

curl

expected=$(curl -sS https://www.districtfacts.com/data/v1/de/districts.json | jq 'length')
actual=$(wc -l < de-districts.ndjson | tr -d ' ')
[ "$actual" = "$expected" ] || exit 1
echo "verified $actual records"

JavaScript

const directory = await fetch("https://www.districtfacts.com/data/v1/de/districts.json").then(response => response.json());
if (count !== directory.length) throw new Error(`count mismatch: ${count} != ${directory.length}`);
console.log(`verified ${count} records`);

Python

with urlopen("https://www.districtfacts.com/data/v1/de/districts.json") as response: directory = json.load(response)
if count != len(directory): raise ValueError(f"count mismatch: {count} != {len(directory)}")
print(f"verified {count} records")

Sample output

verified 43 records

5. Follow the keyed redirect without forwarding the key

A keyed bulk GET returns 302 with a `Location` signed for ten minutes. Request that signed URL separately, without the API Authorization header.

curl

headers=$(mktemp)
curl -sS -D "$headers" -o /dev/null -H "Authorization: Bearer $DISTRICTFACTS_API_KEY" https://api.districtfacts.com/data/v1/de/bulk.ndjson
location=$(awk 'tolower($1)=="location:" {$1=""; sub(/^ /, ""); sub(/\r$/, ""); print}' "$headers")
curl --fail --silent --show-error "$location" -o de-districts.ndjson

JavaScript

const first = await fetch("https://api.districtfacts.com/data/v1/de/bulk.ndjson", { headers: { Authorization: `Bearer ${process.env.DISTRICTFACTS_API_KEY}` }, redirect: "manual" });
if (first.status !== 302) throw new Error(`HTTP ${first.status}`);
const location = first.headers.get("location");
if (!location) throw new Error("Missing Location");
const download = await fetch(location); // No Authorization header.
console.log({ bytes: (await download.arrayBuffer()).byteLength });

Python

import os
from urllib.error import HTTPError
from urllib.request import HTTPRedirectHandler, Request, build_opener, urlopen
class NoRedirect(HTTPRedirectHandler):
    def redirect_request(self, req, fp, code, msg, headers, newurl): return None
request = Request("https://api.districtfacts.com/data/v1/de/bulk.ndjson", headers={"Authorization": f"Bearer {os.environ['DISTRICTFACTS_API_KEY']}"})
try: build_opener(NoRedirect).open(request)
except HTTPError as response:
    if response.code != 302: raise
    location = response.headers["Location"]
with urlopen(location, timeout=60) as download: body = download.read()  # No Authorization header.
print({"bytes": len(body)})

Sample output

Response sequence
302 Location: <signed URL valid for ten minutes>
200 <NDJSON bytes>

Developer documentation