The Web: HTTP and APIs 🌐

What actually happens

Your program opens a connection, sends a small block of text describing what it wants, and gets a small block of text back with the answer attached. That is HTTP. It is genuinely this simple.

GET /repos/python/cpython HTTP/1.1
Host: api.github.com
Accept: application/json
User-Agent: python-school-example

--- and the reply ---

HTTP/1.1 200 OK
Content-Type: application/json
X-RateLimit-Remaining: 59

{"name": "cpython", "stargazers_count": 63000, ...}

requests, the one everyone uses

import requests

response = requests.get("https://api.github.com/repos/python/cpython", timeout=10)

print(response.status_code)
print(response.headers["content-type"])

data = response.json()
print(data["full_name"], "has", data["stargazers_count"], "stars")

No run button on this lesson's network examples: the school's in-browser Python has no network access, deliberately. Install requests in a virtual environment (Lesson 26) and run these on your own machine.

Doing it properly

import requests


def fetch_repo(owner: str, name: str) -> dict:
    """Fetch one repository, raising a clear error on failure."""
    url = f"https://api.github.com/repos/{owner}/{name}"
    response = requests.get(
        url,
        headers={
            "Accept": "application/vnd.github+json",
            "User-Agent": "python-school-example",
        },
        timeout=10,
    )
    response.raise_for_status()      # turns 4xx and 5xx into an exception
    return response.json()


try:
    repo = fetch_repo("python", "cpython")
    print(repo["description"])
except requests.HTTPError as err:
    print(f"HTTP {err.response.status_code}: {err.response.reason}")
except requests.Timeout:
    print("the server took too long")
except requests.RequestException as err:
    print(f"network problem: {err}")

Query parameters, headers and POST

import requests

# query string, built and escaped for you
response = requests.get(
    "https://api.github.com/search/repositories",
    params={"q": "language:python stars:>10000", "sort": "stars", "per_page": 5},
    timeout=10,
)

for repo in response.json()["items"]:
    print(f"{repo['stargazers_count']:>7,}  {repo['full_name']}")

# sending JSON
created = requests.post(
    "https://httpbin.org/post",
    json={"name": "Guybrush", "role": "captain"},
    timeout=10,
)
print(created.json()["json"])

# a form, and a file
requests.post("https://httpbin.org/post", data={"field": "value"}, timeout=10)

Use params= rather than gluing a query string together yourself: it escapes spaces and symbols correctly. json= sets the content type and encodes the body; data= sends a form instead.

Authentication, and keeping the key out of your code

import os
import requests

API_KEY = os.environ.get("WEATHER_API_KEY")
if not API_KEY:
    raise SystemExit("Set WEATHER_API_KEY first. Never hard-code it.")

response = requests.get(
    "https://api.example.com/v1/forecast",
    headers={"Authorization": f"Bearer {API_KEY}"},
    timeout=10,
)

# .env  (and add .env to .gitignore, today, before you forget)
WEATHER_API_KEY=abc123
GITHUB_TOKEN=ghp_xxxxxxxx

from dotenv import load_dotenv      # pip install python-dotenv
import os

load_dotenv()
key = os.environ["WEATHER_API_KEY"]

Sessions: faster, and less repetition

import requests

with requests.Session() as session:
    session.headers.update({
        "User-Agent": "python-school-example",
        "Accept": "application/json",
    })

    for name in ["cpython", "peps"]:
        response = session.get(f"https://api.github.com/repos/python/{name}", timeout=10)
        if response.ok:
            print(name, response.json()["stargazers_count"])

A session reuses the underlying TCP connection, which makes repeated calls to the same host noticeably faster, and lets you set headers and cookies once. Use one whenever you make more than a single request.

Being a good citizen

import time
import requests


def fetch_all(urls, session, delay=0.5, max_retries=3):
    """Fetch every URL politely: rate limited, with retries and backoff."""
    results = []

    for url in urls:
        for attempt in range(1, max_retries + 1):
            response = session.get(url, timeout=10)

            if response.status_code == 429:
                wait = int(response.headers.get("Retry-After", 2 ** attempt))
                print(f"rate limited, waiting {wait}s")
                time.sleep(wait)
                continue

            if response.status_code >= 500:
                wait = 2 ** attempt
                print(f"server error, retrying in {wait}s")
                time.sleep(wait)
                continue

            response.raise_for_status()
            results.append(response.json())
            break
        else:
            print(f"giving up on {url}")

        time.sleep(delay)

    return results

• Honour 429 and Retry-After. Ignoring them gets your key banned.
• Back off exponentially on server errors: 1s, 2s, 4s. Hammering a struggling server is how a small outage becomes a large one.
• Never retry a POST blindly. You may create the same order twice.
• Identify yourself in the User-Agent. Some APIs require it, and it lets an operator contact you instead of blocking you.

Without any packages at all

import json
import urllib.request

request = urllib.request.Request(
    "https://api.github.com/repos/python/cpython",
    headers={"User-Agent": "python-school-example"},
)

with urllib.request.urlopen(request, timeout=10) as response:
    data = json.loads(response.read().decode("utf-8"))

print(data["name"])

urllib is in the standard library and needs nothing installed, which matters on a locked-down machine or in a tiny container. It is clumsier than requests for anything complicated, which is exactly why requests is the most downloaded package on PyPI.

import requests


def get_pokemon(name: str) -> dict | None:
    """Fetch one pokemon, or None if there is no such thing."""
    response = requests.get(f"https://pokeapi.co/api/v2/pokemon/{name.lower()}", timeout=10)
    if response.status_code == 404:
        return None
    response.raise_for_status()
    return response.json()


for name in ["pikachu", "guybrush"]:
    data = get_pokemon(name)
    if data is None:
        print(f"{name}: no such pokemon")
        continue
    types = ", ".join(t["type"]["name"] for t in data["types"])
    print(f"{data['name']}: {data['height']}dm, {data['weight']}hg, type {types}")

import json
from pathlib import Path

import requests

CACHE = Path("api-cache")


def fetch_cached(name: str, max_age_calls: int = 1) -> dict:
    """Fetch a pokemon, using a local cache file when one exists."""
    CACHE.mkdir(exist_ok=True)
    cache_file = CACHE / f"{name.lower()}.json"

    if cache_file.exists():
        print(f"  (cache hit: {name})")
        return json.loads(cache_file.read_text(encoding="utf-8"))

    response = requests.get(f"https://pokeapi.co/api/v2/pokemon/{name.lower()}", timeout=10)
    response.raise_for_status()
    data = response.json()

    cache_file.write_text(json.dumps(data), encoding="utf-8")
    print(f"  (fetched and cached: {name})")
    return data