import os, pathlib, re, shutil, subprocess, tempfile

# A tiny program that prints the arguments it receives, each in brackets so spaces are visible.
SHOWARGS = '#!/bin/sh\nprintf "%d argument(s):" "$#"\nfor a in "$@"; do printf " [%s]" "$a"; done\necho\n'

def bash(line, cwd, env):
    """Run one line with bash, after a fixed prelude, and return what it printed."""
    prelude = 'f="my notes.txt"; empty=""\n'
    r = subprocess.run([shutil.which("bash"), "--norc", "--noprofile", "-c", prelude + line],
                       cwd=cwd, env=env, capture_output=True, text=True)
    if r.returncode:
        why = re.search(r"([^:\n]+: command not found)", r.stderr)
        return f"exit status {r.returncode}" + (f': "{why.group(1).strip()}"' if why else "")
    return r.stdout.rstrip("\n")

with tempfile.TemporaryDirectory() as tmp:
    root = pathlib.Path(tmp)
    (root / "bin").mkdir(); (root / "demo").mkdir()
    tool = root / "bin" / "showargs"; tool.write_text(SHOWARGS); tool.chmod(0o755)
    for name in ("a.txt", "b c.txt", "notes.md"): (root / "demo" / name).write_text("")
    env = {"PATH": f"{root / 'bin'}:{os.environ['PATH']}", "HOME": "/home/learner", "LC_ALL": "C"}

    groups = {
        "1. The shell splits a line into words at unquoted spaces; quotes keep spaces in": [
            "showargs hello   world", 'showargs "hello   world"',
            "showargs it\\'s \"say \\\"hi\\\"\" 'a\\nb'"],
        "2. $name is replaced, then split again unless it's in double quotes": [
            "showargs $f", 'showargs "$f"', "showargs $empty", 'showargs "$empty"',
            "showargs 'single $HOME' \"double $HOME\""],
        "3. Globs are matched against file names; a glob that matches nothing is left alone": [
            "showargs *.txt", "showargs *.csv", 'showargs "*.txt"'],
        "4. Other expansions: tilde, braces, arithmetic, command substitution": [
            'showargs ~/notes "~/notes"', "showargs file{1..3}.log {red,green}-pen",
            "showargs $((2 + 3 * 4))",
            "showargs $(printf 'one   two')", 'showargs "$(printf \'one   two\')"'],
        "5. The order matters: a substitution's result is split and then globbed": [
            "showargs $(printf '*.txt')"],
        "6. Spaces around = make it a command, not an assignment": [
            "x = 5", 'x=5 y="a b"; showargs $x "$y"'],
    }
    for title, lines in groups.items():
        print(title)
        for line in lines:
            print(f"   $ {line}\n     {bash(line, root / 'demo', env)}")

    print("7. Without a shell there is no expansion: the program gets the list exactly as given")
    r = subprocess.run([str(tool), "*.txt", "my notes.txt", "$HOME"], cwd=root / "demo",
                       env=env, capture_output=True, text=True)
    print(f"   subprocess.run(['showargs', '*.txt', 'my notes.txt', '$HOME'])\n     {r.stdout.strip()}")

1. The shell splits a line into words at unquoted spaces; quotes keep spaces in
   $ showargs hello   world
     2 argument(s): [hello] [world]
   $ showargs "hello   world"
     1 argument(s): [hello   world]
   $ showargs it\'s "say \"hi\"" 'a\nb'
     3 argument(s): [it's] [say "hi"] [a\nb]
2. $name is replaced, then split again unless it's in double quotes
   $ showargs $f
     2 argument(s): [my] [notes.txt]
   $ showargs "$f"
     1 argument(s): [my notes.txt]
   $ showargs $empty
     0 argument(s):
   $ showargs "$empty"
     1 argument(s): []
   $ showargs 'single $HOME' "double $HOME"
     2 argument(s): [single $HOME] [double /home/learner]
3. Globs are matched against file names; a glob that matches nothing is left alone
   $ showargs *.txt
     2 argument(s): [a.txt] [b c.txt]
   $ showargs *.csv
     1 argument(s): [*.csv]
   $ showargs "*.txt"
     1 argument(s): [*.txt]
4. Other expansions: tilde, braces, arithmetic, command substitution
   $ showargs ~/notes "~/notes"
     2 argument(s): [/home/learner/notes] [~/notes]
   $ showargs file{1..3}.log {red,green}-pen
     5 argument(s): [file1.log] [file2.log] [file3.log] [red-pen] [green-pen]
   $ showargs $((2 + 3 * 4))
     1 argument(s): [14]
   $ showargs $(printf 'one   two')
     2 argument(s): [one] [two]
   $ showargs "$(printf 'one   two')"
     1 argument(s): [one   two]
5. The order matters: a substitution's result is split and then globbed
   $ showargs $(printf '*.txt')
     2 argument(s): [a.txt] [b c.txt]
6. Spaces around = make it a command, not an assignment
   $ x = 5
     exit status 127: "x: command not found"
   $ x=5 y="a b"; showargs $x "$y"
     2 argument(s): [5] [a b]
7. Without a shell there is no expansion: the program gets the list exactly as given
   subprocess.run(['showargs', '*.txt', 'my notes.txt', '$HOME'])
     3 argument(s): [*.txt] [my notes.txt] [$HOME]

import os, pathlib, shutil, subprocess, tempfile

# A tiny program that writes one line to each output stream and exits with the status it's given.
TALK = '#!/bin/sh\necho "to stdout"\necho "to stderr" >&2\nexit "${1:-0}"\n'

def show(script, cwd, env):
    """Run a script with bash and report where each piece of text ended up."""
    for f in cwd.glob("*.txt"): f.unlink()
    r = subprocess.run([shutil.which("bash"), "--norc", "--noprofile", "-c", script],
                       cwd=cwd, env=env, capture_output=True, text=True)
    first, *rest = script.splitlines()
    print(f"   $ {first}" + "".join(f"\n   > {line}" for line in rest))
    places = [("the terminal's stdout", r.stdout), ("the terminal's stderr", r.stderr)]
    places += [(f"file {f.name}", f.read_text()) for f in sorted(cwd.glob("*.txt"))]
    for place, text in places:
        if text: print(f"     {place + ':':24}" + " | ".join(text.splitlines()))
    if r.returncode: print(f"     {'the script exited with:':24}status {r.returncode}")

with tempfile.TemporaryDirectory() as tmp:
    root = pathlib.Path(tmp); (root / "bin").mkdir(); (root / "work").mkdir()
    tool = root / "bin" / "talk"; tool.write_text(TALK); tool.chmod(0o755)
    env = {"PATH": f"{root / 'bin'}:{os.environ['PATH']}", "HOME": "/home/learner", "LC_ALL": "C"}
    work = root / "work"

    print("1. Every program starts with three streams: 0 stdin, 1 stdout, 2 stderr")
    for s in ["talk", "talk > out.txt", "talk 2> /dev/null"]: show(s, work, env)

    print("2. Redirections are applied left to right, and 2>&1 means 'make 2 point where 1 points now'")
    for s in ["talk > out.txt 2>&1", "talk 2>&1 > out.txt"]: show(s, work, env)

    print("3. A pipe connects stdout only; stderr goes around it unless you redirect it")
    for s in ["talk | tr a-z A-Z", "talk 2>&1 | tr a-z A-Z", "talk 2>&1 | tee all.txt | grep -c std"]:
        show(s, work, env)

    print("4. Every command exits with a status: 0 means success, anything else is a failure")
    for s in ['talk 3 2> /dev/null; echo "status $?"',
              "talk 3 > /dev/null 2>&1 && echo worked || echo failed",
              'if grep -q needle /dev/null; then echo found; else echo "not found, status $?"; fi']:
        show(s, work, env)

    print("5. A pipeline's status is its last command's, unless pipefail is set")
    for s in ['false | true; echo "status $?"', 'set -o pipefail; false | true; echo "status $?"',
              'false | true | talk 2 > /dev/null 2>&1; echo "statuses ${PIPESTATUS[*]}"',
              'yes | head -n 2; echo "statuses ${PIPESTATUS[*]}"']:
        show(s, work, env)

    print("6. set -e stops at the first failure, except inside a condition, however deep")
    for s in ['set -e; false; echo "never printed"',
              'set -e\ncheck() { false; echo "check carried on after false"; }\n'
              'check || echo "check failed"']:
        show(s, work, env)

    print("7. A here-document feeds lines to stdin; quoting the marker turns expansion off")
    for s in ["name=Ada\ncat <<EOF\nHello, $name\nEOF", "name=Ada\ncat <<'EOF'\nHello, $name\nEOF"]:
        show(s, work, env)

1. Every program starts with three streams: 0 stdin, 1 stdout, 2 stderr
   $ talk
     the terminal's stdout:  to stdout
     the terminal's stderr:  to stderr
   $ talk > out.txt
     the terminal's stderr:  to stderr
     file out.txt:           to stdout
   $ talk 2> /dev/null
     the terminal's stdout:  to stdout
2. Redirections are applied left to right, and 2>&1 means 'make 2 point where 1 points now'
   $ talk > out.txt 2>&1
     file out.txt:           to stdout | to stderr
   $ talk 2>&1 > out.txt
     the terminal's stdout:  to stderr
     file out.txt:           to stdout
3. A pipe connects stdout only; stderr goes around it unless you redirect it
   $ talk | tr a-z A-Z
     the terminal's stdout:  TO STDOUT
     the terminal's stderr:  to stderr
   $ talk 2>&1 | tr a-z A-Z
     the terminal's stdout:  TO STDOUT | TO STDERR
   $ talk 2>&1 | tee all.txt | grep -c std
     the terminal's stdout:  2
     file all.txt:           to stdout | to stderr
4. Every command exits with a status: 0 means success, anything else is a failure
   $ talk 3 2> /dev/null; echo "status $?"
     the terminal's stdout:  to stdout | status 3
   $ talk 3 > /dev/null 2>&1 && echo worked || echo failed
     the terminal's stdout:  failed
   $ if grep -q needle /dev/null; then echo found; else echo "not found, status $?"; fi
     the terminal's stdout:  not found, status 1
5. A pipeline's status is its last command's, unless pipefail is set
   $ false | true; echo "status $?"
     the terminal's stdout:  status 0
   $ set -o pipefail; false | true; echo "status $?"
     the terminal's stdout:  status 1
   $ false | true | talk 2 > /dev/null 2>&1; echo "statuses ${PIPESTATUS[*]}"
     the terminal's stdout:  statuses 1 0 2
   $ yes | head -n 2; echo "statuses ${PIPESTATUS[*]}"
     the terminal's stdout:  y | y | statuses 141 0
6. set -e stops at the first failure, except inside a condition, however deep
   $ set -e; false; echo "never printed"
     the script exited with: status 1
   $ set -e
   > check() { false; echo "check carried on after false"; }
   > check || echo "check failed"
     the terminal's stdout:  check carried on after false
7. A here-document feeds lines to stdin; quoting the marker turns expansion off
   $ name=Ada
   > cat <<EOF
   > Hello, $name
   > EOF
     the terminal's stdout:  Hello, Ada
   $ name=Ada
   > cat <<'EOF'
   > Hello, $name
   > EOF
     the terminal's stdout:  Hello, $name

import os, pathlib, random, shutil, subprocess, tempfile, time

def make_log(path):
    """A web server's access log: client, time, request, status, bytes sent, milliseconds taken."""
    rng = random.Random(3)
    clients = ["203.0.113.7"] * 9 + ["198.51.100.23"] * 6 + ["192.0.2.41"] * 4 + ["203.0.113.90"] * 3 + \
              ["198.51.100.5", "192.0.2.200"]
    pages = [("GET", "/", 200), ("GET", "/api/orders", 200), ("POST", "/api/orders", 201),
             ("GET", "/api/users", 200), ("GET", "/static/app.js", 200), ("POST", "/login", 302),
             ("GET", "/old-page", 404), ("GET", "/api/orders", 500), ("GET", "/api/report", 503)]
    lines = []
    for i in range(48):
        method, page, status = rng.choice(pages)
        ms = rng.randint(900, 2400) if status >= 500 else rng.randint(8, 640)
        stamp = time.strftime("%d/%b/%Y:%H:%M:%S", time.gmtime(1773306000 + 37 * i))
        lines.append(f'{rng.choice(clients)} - - [{stamp} +0000] "{method} {page} HTTP/1.1" '
                     f'{status} {rng.randint(120, 9000)} {ms}ms')
    path.write_text("\n".join(lines) + "\n")

def make_tree(root):
    """A small project folder, with a few files last changed long ago."""
    old = time.time() - 40 * 86400
    for name, text, aged in [("src/app.py", "# TODO: handle empty carts\n", False),
                             ("src/util.py", "x = 1\n", True), ("docs/guide.md", "", True),
                             ("tests/test_app.py", "# TODO: test refunds\n", False),
                             ("build/out.tmp", "", False), ("old notes.tmp", "", True),
                             ("node_modules/pkg/index.js", "", False), ("src/menu.js", "", False)]:
        p = root / name; p.parent.mkdir(parents=True, exist_ok=True); p.write_text(text)
        if aged: os.utime(p, (old, old))

def q(cmd, cwd, env):
    r = subprocess.run([shutil.which("bash"), "--norc", "--noprofile", "-c", cmd], cwd=cwd, env=env,
                       capture_output=True, text=True)
    first, *rest = cmd.splitlines()
    print(f"   $ {first}" + "".join(f"\n   > {line.strip()}" for line in rest))
    for line in r.stdout.splitlines(): print(f"     {line}")

with tempfile.TemporaryDirectory() as tmp:
    root = pathlib.Path(tmp); make_log(root / "access.log"); make_tree(root / "project")
    env = {"PATH": os.environ["PATH"], "HOME": "/home/learner", "LC_ALL": "C"}
    print("The log has lines like this one:")
    q("head -n 1 access.log", root, env)

    print("grep picks lines that match a regular expression")
    q("""grep -cE '" 5[0-9]{2} ' access.log""", root, env)
    q("""grep -E '" 5[0-9]{2} ' access.log | cut -d ' ' -f 1,7,9,11 | head -n 3""", root, env)
    q("""grep -oE '/api/[a-z]+' access.log | sort | uniq -c | awk '{print $1, $2}'""", root, env)

    print("awk splits each line into fields, $1 to $NF, and runs a little program on them")
    q("""awk '{n[$1]++} END {for (ip in n) print n[ip], ip}' access.log | sort -k1,1nr -k2,2 | head -n 3""",
      root, env)
    q("""awk '$NF + 0 > 1400 {print $7, $9, $NF}' access.log""", root, env)
    q("""awk '{c = substr($9, 1, 1) "xx"; t[c] += $NF; n[c]++}
     END {for (c in n) printf "%s %2d requests, average %4.0f ms\\n", c, n[c], t[c] / n[c]}' access.log |
     sort""",
      root, env)

    print("sed edits a stream: here it hides the last two parts of each address, and finds missing pages")
    q("""sed -E 's/^([0-9]+\\.[0-9]+)\\.[0-9]+\\.[0-9]+/\\1.x.x/' access.log |
     cut -d ' ' -f 1,7 | head -n 3""",
      root, env)
    q("""sed -nE 's/.*"GET ([^ ]+) [^"]*" 404 .*/\\1/p' access.log | sort -u""", root, env)

    print("find walks a folder tree and selects files by name, type, age and more")
    q("find . -name '*.py' | sort", root / "project", env)
    q("find . -name node_modules -prune -o -name '*.js' -print", root / "project", env)
    q("find . -type f -mtime +30 | sort", root / "project", env)
    q("grep -rn --include='*.py' TODO . | sort", root / "project", env)

    print("Names with spaces break naive pipelines; -print0 and xargs -0 separate names with a zero byte")
    q("find . -name '*.tmp' | xargs rm 2> /dev/null; find . -name '*.tmp' | sort", root / "project", env)
    q("find . -name '*.tmp' -print0 | xargs -0 rm; find . -name '*.tmp' | wc -l | tr -d ' '",
      root / "project", env)

The log has lines like this one:
   $ head -n 1 access.log
     192.0.2.41 - - [12/Mar/2026:09:00:00 +0000] "GET /api/users HTTP/1.1" 200 2256 614ms
grep picks lines that match a regular expression
   $ grep -cE '" 5[0-9]{2} ' access.log
     5
   $ grep -E '" 5[0-9]{2} ' access.log | cut -d ' ' -f 1,7,9,11 | head -n 3
     192.0.2.200 /api/report 503 1698ms
     203.0.113.7 /api/report 503 1383ms
     198.51.100.23 /api/report 503 1455ms
   $ grep -oE '/api/[a-z]+' access.log | sort | uniq -c | awk '{print $1, $2}'
     11 /api/orders
     3 /api/report
     4 /api/users
awk splits each line into fields, $1 to $NF, and runs a little program on them
   $ awk '{n[$1]++} END {for (ip in n) print n[ip], ip}' access.log | sort -k1,1nr -k2,2 | head -n 3
     19 203.0.113.7
     13 198.51.100.23
     8 192.0.2.41
   $ awk '$NF + 0 > 1400 {print $7, $9, $NF}' access.log
     /api/report 503 1698ms
     /api/report 503 1455ms
     /api/orders 500 1454ms
   $ awk '{c = substr($9, 1, 1) "xx"; t[c] += $NF; n[c]++}
   > END {for (c in n) printf "%s %2d requests, average %4.0f ms\n", c, n[c], t[c] / n[c]}' access.log |
   > sort
     2xx 25 requests, average  344 ms
     3xx  6 requests, average  500 ms
     4xx 12 requests, average  435 ms
     5xx  5 requests, average 1387 ms
sed edits a stream: here it hides the last two parts of each address, and finds missing pages
   $ sed -E 's/^([0-9]+\.[0-9]+)\.[0-9]+\.[0-9]+/\1.x.x/' access.log |
   > cut -d ' ' -f 1,7 | head -n 3
     192.0.x.x /api/users
     192.0.x.x /login
     203.0.x.x /
   $ sed -nE 's/.*"GET ([^ ]+) [^"]*" 404 .*/\1/p' access.log | sort -u
     /old-page
find walks a folder tree and selects files by name, type, age and more
   $ find . -name '*.py' | sort
     ./src/app.py
     ./src/util.py
     ./tests/test_app.py
   $ find . -name node_modules -prune -o -name '*.js' -print
     ./src/menu.js
   $ find . -type f -mtime +30 | sort
     ./docs/guide.md
     ./old notes.tmp
     ./src/util.py
   $ grep -rn --include='*.py' TODO . | sort
     ./src/app.py:1:# TODO: handle empty carts
     ./tests/test_app.py:1:# TODO: test refunds
Names with spaces break naive pipelines; -print0 and xargs -0 separate names with a zero byte
   $ find . -name '*.tmp' | xargs rm 2> /dev/null; find . -name '*.tmp' | sort
     ./old notes.tmp
   $ find . -name '*.tmp' -print0 | xargs -0 rm; find . -name '*.tmp' | wc -l | tr -d ' '
     0

import os, pathlib, shutil, subprocess, tempfile

TIDY = r'''#!/usr/bin/env bash
# tidy: sort the files at the top of a folder into subfolders by type. Safe to run again.
set -euo pipefail

usage() { echo "usage: tidy [-n] FOLDER    -n: only show what would move"; }

dry_run=false
while getopts "nh" opt; do
  case "$opt" in
    n) dry_run=true ;;
    h) usage; exit 0 ;;
    *) usage >&2; exit 2 ;;
  esac
done
shift $((OPTIND - 1))
[ "$#" -eq 1 ] || { usage >&2; exit 2; }
folder=${1%/}
[ -d "$folder" ] || { echo "tidy: no such folder: $folder" >&2; exit 1; }

# mkdir is atomic, so only one run can hold the lock; the trap removes it however the script ends
lock="$folder/.tidy.lock"
mkdir "$lock" 2> /dev/null || { echo "tidy: already running on $folder" >&2; exit 1; }
trap 'rmdir "$lock"' EXIT

category() {
  local ext
  ext=$(printf '%s' "${1##*.}" | tr '[:upper:]' '[:lower:]')
  case "$ext" in
    jpg|jpeg|png|gif|heic) echo images ;;
    pdf|doc|docx|txt|md|csv) echo documents ;;
    zip|tar|gz|dmg) echo archives ;;
    *) echo other ;;
  esac
}

free_name() {   # $2, or "$2 (1)", "$2 (2)"... whichever doesn't exist yet in folder $1
  local dir=$1 name=$2 base ext n=1 candidate=$2
  if [[ $name == ?*.* ]]; then base=${name%.*} ext=.${name##*.}; else base=$name ext=; fi
  while [ -e "$dir/$candidate" ]; do candidate="$base ($n)$ext"; n=$((n + 1)); done
  printf '%s\n' "$candidate"
}

count=0
for path in "$folder"/*; do
  [ -f "$path" ] || continue          # skip folders, and the unexpanded * of an empty folder
  name=${path##*/}
  dest=$(category "$name")
  target=$(free_name "$folder/$dest" "$name")
  if $dry_run; then
    echo "would move: $name -> $dest/$target"
  else
    mkdir -p "$folder/$dest"
    mv -n "$path" "$folder/$dest/$target"
    echo "moved: $name -> $dest/$target"
  fi
  count=$((count + 1))
done
[ "$count" -gt 0 ] || echo "nothing to tidy in $folder"
'''

def run(cmd, cwd, env):
    r = subprocess.run([shutil.which("bash"), "--norc", "--noprofile", "-c", cmd], cwd=cwd, env=env,
                       capture_output=True, text=True)
    print(f"$ {cmd}")
    for line in r.stdout.splitlines(): print(f"   {line}")
    for line in r.stderr.splitlines(): print(f"   (stderr) {line}")
    if r.returncode: print(f"   (exit status {r.returncode})")

def tree(folder):
    for p in sorted(folder.rglob("*")):
        if p.is_file(): print(f"   {p.relative_to(folder.parent)}")

with tempfile.TemporaryDirectory() as tmp:
    root = pathlib.Path(tmp); (root / "bin").mkdir()
    script = root / "bin" / "tidy"; script.write_text(TIDY); script.chmod(0o755)
    downloads = root / "Downloads"
    for name in ["Holiday 2025.JPG", "invoice-0423.pdf", "report.pdf", "notes.txt", "photo.png",
                 "backup.tar.gz", "Makefile", "documents/report.pdf", "projects/plan.md"]:
        p = downloads / name; p.parent.mkdir(parents=True, exist_ok=True); p.write_text(name)
    env = {"PATH": f"{root / 'bin'}:{os.environ['PATH']}", "HOME": "/home/learner", "LC_ALL": "C"}

    run("tidy", root, env)
    run("tidy -n Downloads", root, env)
    run("tidy Downloads/", root, env)
    print("After one run:"); tree(downloads)
    run("tidy Downloads", root, env)
    run("tidy Nowhere", root, env)
    run('mkdir Downloads/.tidy.lock; tidy Downloads || echo "refused, status $?"; '
        'rmdir Downloads/.tidy.lock', root, env)
    kept = (downloads / "documents/report.pdf").read_text() == "documents/report.pdf"
    print(f"no lock left behind: {not (downloads / '.tidy.lock').exists()}; "
          f"the report already in documents/ is untouched: {kept}")

$ tidy
   (stderr) usage: tidy [-n] FOLDER    -n: only show what would move
   (exit status 2)
$ tidy -n Downloads
   would move: Holiday 2025.JPG -> images/Holiday 2025.JPG
   would move: Makefile -> other/Makefile
   would move: backup.tar.gz -> archives/backup.tar.gz
   would move: invoice-0423.pdf -> documents/invoice-0423.pdf
   would move: notes.txt -> documents/notes.txt
   would move: photo.png -> images/photo.png
   would move: report.pdf -> documents/report (1).pdf
$ tidy Downloads/
   moved: Holiday 2025.JPG -> images/Holiday 2025.JPG
   moved: Makefile -> other/Makefile
   moved: backup.tar.gz -> archives/backup.tar.gz
   moved: invoice-0423.pdf -> documents/invoice-0423.pdf
   moved: notes.txt -> documents/notes.txt
   moved: photo.png -> images/photo.png
   moved: report.pdf -> documents/report (1).pdf
After one run:
   Downloads/archives/backup.tar.gz
   Downloads/documents/invoice-0423.pdf
   Downloads/documents/notes.txt
   Downloads/documents/report (1).pdf
   Downloads/documents/report.pdf
   Downloads/images/Holiday 2025.JPG
   Downloads/images/photo.png
   Downloads/other/Makefile
   Downloads/projects/plan.md
$ tidy Downloads
   nothing to tidy in Downloads
$ tidy Nowhere
   (stderr) tidy: no such folder: Nowhere
   (exit status 1)
$ mkdir Downloads/.tidy.lock; tidy Downloads || echo "refused, status $?"; rmdir Downloads/.tidy.lock
   refused, status 1
   (stderr) tidy: already running on Downloads
no lock left behind: True; the report already in documents/ is untouched: True

import os, pathlib, posixpath, shutil, subprocess, tempfile

def sh(cmd, cwd):
    """Run a shell command; the rest of the example inspects what it did with Python's os module."""
    r = subprocess.run([shutil.which("bash"), "--norc", "--noprofile", "-c", cmd], cwd=cwd,
                       env={"PATH": os.environ["PATH"], "LC_ALL": "C"}, capture_output=True, text=True)
    print(f"$ {cmd}" + (f"   (exit status {r.returncode})" if r.returncode else ""))
    for line in r.stdout.splitlines(): print(f"   {line}")

def describe(d, *names):
    """For each name: what kind of entry it is, which inode it names, and how many names that inode has."""
    inodes = {}
    for name in names:
        p = d / name
        if p.is_symlink():
            target = os.readlink(p)
            state = "works" if p.exists() else "is broken: nothing at that path"
            print(f"   {name:12} symbolic link to {target!r}, which {state}")
        elif p.exists():
            st = p.stat(); label = inodes.setdefault(st.st_ino, chr(ord("A") + len(inodes)))
            print(f"   {name:12} file, inode {label}, {st.st_nlink} name(s), "
                  f"contains {p.read_text().strip()!r}")
        else:
            print(f"   {name:12} does not exist")

print("1. A path is directions through the tree; . is here, .. is one level up")
for cwd, path in [("/home/learner/project", "notes.txt"), ("/home/learner/project", "../data/./in.csv"),
                  ("/home/learner/project", "/etc/hosts"), ("/", "home/learner/../../tmp")]:
    print(f"   from {cwd:22} {path:24} is {posixpath.normpath(posixpath.join(cwd, path))}")

with tempfile.TemporaryDirectory() as tmp:
    d = pathlib.Path(tmp)
    print("2. A hard link is a second name for the same inode; a symbolic link is a file holding a path")
    sh("echo 'version 1' > notes.txt; ln notes.txt hard.txt; ln -s notes.txt soft.txt", d)
    describe(d, "notes.txt", "hard.txt", "soft.txt")
    sh("echo 'version 2' > hard.txt", d)
    describe(d, "notes.txt", "soft.txt")

    print("3. Renaming changes a name, not the file; symbolic links follow names, so they can break")
    sh("mv notes.txt renamed.txt", d)
    describe(d, "renamed.txt", "hard.txt", "soft.txt")
    sh("rm renamed.txt", d)
    describe(d, "hard.txt", "soft.txt")

    print("4. A relative link is resolved from the folder the link is in, not from where you made it")
    (d / "dotfiles").mkdir(); (d / "home").mkdir(); (d / "dotfiles" / "zshrc").write_text("# my settings\n")
    sh("ln -s dotfiles/zshrc home/.zshrc; ln -s ../dotfiles/zshrc home/.zshrc-fixed", d)
    describe(d / "home", ".zshrc", ".zshrc-fixed")

    print("5. Deleting removes a name; the data is freed only when no name and no open file refers to it")
    log = open(d / "big.log", "w"); log.write("x" * 1_000_000); log.flush()
    sh("rm big.log; ls big.log 2> /dev/null || echo 'no such name any more'", d)
    st = os.fstat(log.fileno())
    print(f"   but the program that opened it still has it: {st.st_size:,} bytes, {st.st_nlink} names")
    log.close()

1. A path is directions through the tree; . is here, .. is one level up
   from /home/learner/project  notes.txt                is /home/learner/project/notes.txt
   from /home/learner/project  ../data/./in.csv         is /home/learner/data/in.csv
   from /home/learner/project  /etc/hosts               is /etc/hosts
   from /                      home/learner/../../tmp   is /tmp
2. A hard link is a second name for the same inode; a symbolic link is a file holding a path
$ echo 'version 1' > notes.txt; ln notes.txt hard.txt; ln -s notes.txt soft.txt
   notes.txt    file, inode A, 2 name(s), contains 'version 1'
   hard.txt     file, inode A, 2 name(s), contains 'version 1'
   soft.txt     symbolic link to 'notes.txt', which works
$ echo 'version 2' > hard.txt
   notes.txt    file, inode A, 2 name(s), contains 'version 2'
   soft.txt     symbolic link to 'notes.txt', which works
3. Renaming changes a name, not the file; symbolic links follow names, so they can break
$ mv notes.txt renamed.txt
   renamed.txt  file, inode A, 2 name(s), contains 'version 2'
   hard.txt     file, inode A, 2 name(s), contains 'version 2'
   soft.txt     symbolic link to 'notes.txt', which is broken: nothing at that path
$ rm renamed.txt
   hard.txt     file, inode A, 1 name(s), contains 'version 2'
   soft.txt     symbolic link to 'notes.txt', which is broken: nothing at that path
4. A relative link is resolved from the folder the link is in, not from where you made it
$ ln -s dotfiles/zshrc home/.zshrc; ln -s ../dotfiles/zshrc home/.zshrc-fixed
   .zshrc       symbolic link to 'dotfiles/zshrc', which is broken: nothing at that path
   .zshrc-fixed symbolic link to '../dotfiles/zshrc', which works
5. Deleting removes a name; the data is freed only when no name and no open file refers to it
$ rm big.log; ls big.log 2> /dev/null || echo 'no such name any more'
   no such name any more
   but the program that opened it still has it: 1,000,000 bytes, 0 names

import os, pathlib, shutil, stat, subprocess, tempfile

if os.geteuid() == 0:
    raise SystemExit("Run this as an ordinary user: root may do almost anything, whatever the bits say.")

def sh(cmd, cwd):
    r = subprocess.run([shutil.which("bash"), "--norc", "--noprofile", "-c", cmd], cwd=cwd,
                       env={"PATH": os.environ["PATH"], "LC_ALL": "C"}, capture_output=True, text=True)
    return r.returncode, r.stdout.strip()

def mode(p): return stat.filemode(os.lstat(p).st_mode)

def attempt(what, cmd, cwd):
    status, out = sh(cmd, cwd)
    if status: result = f"refused (status {status})"
    else: result = "allowed" + (f", printed {out!r}" if out else "")
    print(f"   {what:42} {result}")

print("1. Nine bits: read, write, execute, for the owner, the group, and everyone else")
for bits in (0o644, 0o755, 0o600, 0o750, 0o400):
    who = [stat.filemode(stat.S_IFREG | bits)[i:i + 3] for i in (1, 4, 7)]
    print(f"   {bits:o}  {stat.filemode(stat.S_IFREG | bits)}   "
          f"owner {who[0]}  group {who[1]}  others {who[2]}")

with tempfile.TemporaryDirectory() as tmp:
    d = pathlib.Path(tmp)
    print("2. chmod sets bits by number or by name; umask removes bits from everything you create")
    (d / "plan.txt").write_text("plan\n"); os.chmod(d / "plan.txt", 0o644)
    for change in ("u+x", "go-r", "a+r", "640"):
        sh(f"chmod {change} plan.txt", d)
        print(f"   chmod {change:5} plan.txt   now {mode(d / 'plan.txt')}")
    sh("umask 022; touch a.txt; mkdir a-dir; umask 077; touch b.txt; mkdir b-dir", d)
    for name in ("a.txt", "a-dir", "b.txt", "b-dir"):
        print(f"   created with umask {'022' if name[0] == 'a' else '077'}: {name:6} {mode(d / name)}")

    print("3. On a file, r lets you read it, w lets you change it, x lets you run it")
    (d / "secret.txt").write_text("42\n"); os.chmod(d / "secret.txt", 0o200)
    attempt("cat secret.txt  (mode --w-------)", "cat secret.txt", d)
    (d / "fixed.txt").write_text("v1\n"); os.chmod(d / "fixed.txt", 0o444)
    attempt("append to fixed.txt  (mode r--r--r--)", "echo v2 >> fixed.txt", d)
    (d / "hello.sh").write_text("echo hello from the script\n"); os.chmod(d / "hello.sh", 0o644)
    attempt("./hello.sh  (mode rw-r--r--)", "./hello.sh", d)
    attempt("bash hello.sh  (bash only reads it)", "bash hello.sh", d)

    print("4. On a folder, r lets you list names, w lets you add and remove names, x lets you go in")
    box = d / "box"; box.mkdir(); (box / "note.txt").write_text("inside\n")
    os.chmod(box, 0o600)
    attempt("echo box/*  (mode rw-------: no x)", "echo box/*", d)
    attempt("cat box/note.txt", "cat box/note.txt", d)
    os.chmod(box, 0o300)
    attempt("ls box  (mode -wx------: no r)", "ls box", d)
    attempt("cat box/note.txt, knowing the name", "cat box/note.txt", d)
    os.chmod(box, 0o500)
    attempt("rm box/note.txt  (mode r-x------: no w)", "rm -f box/note.txt", d)
    os.chmod(box, 0o700)
    attempt("rm fixed.txt  (r--r--r--, folder writable)", "rm -f fixed.txt", d)
    os.chmod(d / "secret.txt", 0o600)

1. Nine bits: read, write, execute, for the owner, the group, and everyone else
   644  -rw-r--r--   owner rw-  group r--  others r--
   755  -rwxr-xr-x   owner rwx  group r-x  others r-x
   600  -rw-------   owner rw-  group ---  others ---
   750  -rwxr-x---   owner rwx  group r-x  others ---
   400  -r--------   owner r--  group ---  others ---
2. chmod sets bits by number or by name; umask removes bits from everything you create
   chmod u+x   plan.txt   now -rwxr--r--
   chmod go-r  plan.txt   now -rwx------
   chmod a+r   plan.txt   now -rwxr--r--
   chmod 640   plan.txt   now -rw-r-----
   created with umask 022: a.txt  -rw-r--r--
   created with umask 022: a-dir  drwxr-xr-x
   created with umask 077: b.txt  -rw-------
   created with umask 077: b-dir  drwx------
3. On a file, r lets you read it, w lets you change it, x lets you run it
   cat secret.txt  (mode --w-------)          refused (status 1)
   append to fixed.txt  (mode r--r--r--)      refused (status 1)
   ./hello.sh  (mode rw-r--r--)               refused (status 126)
   bash hello.sh  (bash only reads it)        allowed, printed 'hello from the script'
4. On a folder, r lets you list names, w lets you add and remove names, x lets you go in
   echo box/*  (mode rw-------: no x)         allowed, printed 'box/note.txt'
   cat box/note.txt                           refused (status 1)
   ls box  (mode -wx------: no r)             refused (status 1)
   cat box/note.txt, knowing the name         allowed, printed 'inside'
   rm box/note.txt  (mode r-x------: no w)    refused (status 1)
   rm fixed.txt  (r--r--r--, folder writable) allowed

import os, pathlib, shutil, subprocess, tempfile

def sh(cmd, cwd, env, args=()):
    r = subprocess.run([shutil.which("bash"), "--norc", "--noprofile", *args, "-c", cmd], cwd=cwd, env=env,
                       capture_output=True, text=True, stdin=subprocess.DEVNULL)
    return r.stdout.replace(str(cwd), "$DEMO").rstrip("\n"), r.returncode

def show(cmd, cwd, env, args=()):
    out, status = sh(cmd, cwd, env, args)
    first, *rest = cmd.splitlines()
    print(f"   $ {first}" + "".join(f"\n   > {line}" for line in rest))
    for line in out.splitlines(): print(f"     {line}")
    if status: print(f"     (exit status {status}" + (": command not found)" if status == 127 else ")"))

with tempfile.TemporaryDirectory() as tmp:
    demo = pathlib.Path(tmp).resolve()
    for folder, text in (("bin1", "greet from bin1"), ("bin2", "greet from bin2")):
        (demo / folder).mkdir(); tool = demo / folder / "greet"
        tool.write_text(f"#!/bin/sh\necho '{text}'\n"); tool.chmod(0o755)
    (demo / "sub").mkdir()
    (demo / "setup.sh").write_text('export GREETING=hello\ncd sub\n')
    base = {"PATH": os.environ["PATH"], "HOME": "/home/learner", "LC_ALL": "C"}
    print("($DEMO stands for the example's temporary folder)")

    print("1. PATH is a list of folders; the shell runs the first matching program it finds")
    show('PATH="$DEMO/bin1:$DEMO/bin2:$PATH"; greet', demo, dict(base, DEMO=str(demo)))
    show('PATH="$DEMO/bin2:$DEMO/bin1:$PATH"; greet; type -a greet', demo, dict(base, DEMO=str(demo)))
    show("greet", demo, base)
    show("type cd export greet", demo, dict(base, PATH=f"{demo}/bin1:{base['PATH']}"))

    print("2. A child process gets a copy of the exported variables; changes never flow back up")
    show('x=1; bash -c \'echo "child sees x=[$x]"\'', demo, base)
    show('export x=1; bash -c \'echo "child sees x=[$x]"\'', demo, base)
    show('x=2 bash -c \'echo "child sees x=[$x]"\'; echo "the shell still has x=[$x]"', demo, base)
    show("export x=1; bash -c 'x=99; cd /'; echo \"x=$x, still in $PWD\"", demo, base)

    print("3. Running a script starts a child; sourcing it runs its lines in this shell")
    show('bash setup.sh; echo "GREETING=[$GREETING], in $PWD"', demo, base)
    show('. ./setup.sh; echo "GREETING=[$GREETING], in $PWD"', demo, base)

    print("4. Which startup files bash reads depends on how it was started")
    home = demo / "home"; home.mkdir()
    for name in (".bash_profile", ".bashrc", ".profile"):
        (home / name).write_text(f"echo '   (bash read ~/{name})'\n")
    env = dict(base, HOME=str(home))
    for label, args in (("a script: neither login nor interactive", []),
                        ("a login shell (bash -l)", ["-l"]), ("an interactive shell (bash -i)", ["-i"]),
                        ("an interactive login shell, like a new terminal on macOS", ["-l", "-i"])):
        out = subprocess.run([shutil.which("bash"), *args, "-c", "true"], env=env, cwd=demo,
                             capture_output=True, text=True, stdin=subprocess.DEVNULL).stdout
        print(f"   {label}:\n{out.rstrip() or '   (no startup files)'}")

($DEMO stands for the example's temporary folder)
1. PATH is a list of folders; the shell runs the first matching program it finds
   $ PATH="$DEMO/bin1:$DEMO/bin2:$PATH"; greet
     greet from bin1
   $ PATH="$DEMO/bin2:$DEMO/bin1:$PATH"; greet; type -a greet
     greet from bin2
     greet is $DEMO/bin2/greet
     greet is $DEMO/bin1/greet
   $ greet
     (exit status 127: command not found)
   $ type cd export greet
     cd is a shell builtin
     export is a shell builtin
     greet is $DEMO/bin1/greet
2. A child process gets a copy of the exported variables; changes never flow back up
   $ x=1; bash -c 'echo "child sees x=[$x]"'
     child sees x=[]
   $ export x=1; bash -c 'echo "child sees x=[$x]"'
     child sees x=[1]
   $ x=2 bash -c 'echo "child sees x=[$x]"'; echo "the shell still has x=[$x]"
     child sees x=[2]
     the shell still has x=[]
   $ export x=1; bash -c 'x=99; cd /'; echo "x=$x, still in $PWD"
     x=1, still in $DEMO
3. Running a script starts a child; sourcing it runs its lines in this shell
   $ bash setup.sh; echo "GREETING=[$GREETING], in $PWD"
     GREETING=[], in $DEMO
   $ . ./setup.sh; echo "GREETING=[$GREETING], in $PWD"
     GREETING=[hello], in $DEMO/sub
4. Which startup files bash reads depends on how it was started
   a script: neither login nor interactive:
   (no startup files)
   a login shell (bash -l):
   (bash read ~/.bash_profile)
   an interactive shell (bash -i):
   (bash read ~/.bashrc)
   an interactive login shell, like a new terminal on macOS:
   (bash read ~/.bash_profile)

import os, pathlib, shutil, subprocess, tempfile

INSTALL = r'''#!/usr/bin/env bash
# install.sh: link each file in this repository to where its program looks for it. Safe to rerun.
set -euo pipefail
repo=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)
backup="$HOME/.dotfiles-backup"

link() {   # link() FILE-IN-REPO PATH-UNDER-HOME
  local src="$repo/$1" dst="$HOME/$2"
  if [ -L "$dst" ] && [ "$(readlink "$dst")" = "$src" ]; then
    echo "ok          ~/$2"; return
  fi
  mkdir -p "$(dirname "$dst")"
  if [ -e "$dst" ] || [ -L "$dst" ]; then      # something else is there: keep it, never delete it
    mkdir -p "$backup"
    mv "$dst" "$backup/${2//\//_}"
    echo "backed up   ~/$2 to ~/.dotfiles-backup/${2//\//_}"
  fi
  ln -s "$src" "$dst"
  echo "linked      ~/$2 -> $1"
}

link zshrc .zshrc
link gitconfig .gitconfig
link gitignore_global .config/git/ignore
[ -e "$HOME/.gitconfig.local" ] || echo "note: put this machine's email in ~/.gitconfig.local"
'''
FILES = {
    "install.sh": INSTALL,
    "zshrc": 'export EDITOR=vim\nalias gs="git status --short"\n',
    "gitconfig": "[user]\n\tname = Ada Lovelace\n[include]\n\tpath = ~/.gitconfig.local\n"
                 "[init]\n\tdefaultBranch = main\n",
    "gitignore_global": ".DS_Store\n*.swp\n",
    ".gitignore": "*.local\n",
}

def sh(cmd, cwd, env):
    r = subprocess.run([shutil.which("bash"), "--norc", "--noprofile", "-c", cmd], cwd=cwd, env=env,
                       capture_output=True, text=True)
    print(f"$ {cmd}")
    for line in (r.stdout + r.stderr).replace(env["HOME"], "~").splitlines(): print(f"   {line}")

with tempfile.TemporaryDirectory() as tmp:
    root = pathlib.Path(tmp).resolve()
    repo, home = root / "dotfiles", root / "home"; repo.mkdir(); home.mkdir()
    for name, text in FILES.items(): (repo / name).write_text(text)
    (repo / "install.sh").chmod(0o755)
    (home / ".zshrc").write_text("# the old settings this laptop came with\n")
    env = {"PATH": os.environ["PATH"], "HOME": str(home), "LC_ALL": "C", "GIT_CONFIG_NOSYSTEM": "1"}

    print("1. The first run links everything, keeping a backup of what was already there")
    sh("dotfiles/install.sh", root, env)
    print("2. A second run changes nothing: running it again is always safe")
    sh("dotfiles/install.sh", root, env)
    for name in (".zshrc", ".gitconfig", ".config/git/ignore"):
        p = home / name
        print(f"   ~/{name:18} -> {pathlib.Path(os.readlink(p)).relative_to(root)}")
    print(f"   the old ~/.zshrc is kept: {(home / '.dotfiles-backup/.zshrc').read_text().strip()!r}")

    print("3. Git reads the linked config, and the include adds settings that never go in the repository")
    (home / ".gitconfig.local").write_text("[user]\n\temail = ada@example.org\n")
    sh("git config --show-origin --get user.name; git config --show-origin --get user.email", root, env)

    print("4. The global ignore file applies in every repository on this machine")
    sh("git init -q project && cd project && touch .DS_Store notes.md && git status --short", root, env)

    print("5. The repository's own .gitignore keeps per-machine files out of it")
    sh("cd dotfiles && git init -q && touch secrets.local && git add -A && git status --short", root, env)

1. The first run links everything, keeping a backup of what was already there
$ dotfiles/install.sh
   backed up   ~/.zshrc to ~/.dotfiles-backup/.zshrc
   linked      ~/.zshrc -> zshrc
   linked      ~/.gitconfig -> gitconfig
   linked      ~/.config/git/ignore -> gitignore_global
   note: put this machine's email in ~/.gitconfig.local
2. A second run changes nothing: running it again is always safe
$ dotfiles/install.sh
   ok          ~/.zshrc
   ok          ~/.gitconfig
   ok          ~/.config/git/ignore
   note: put this machine's email in ~/.gitconfig.local
   ~/.zshrc             -> dotfiles/zshrc
   ~/.gitconfig         -> dotfiles/gitconfig
   ~/.config/git/ignore -> dotfiles/gitignore_global
   the old ~/.zshrc is kept: '# the old settings this laptop came with'
3. Git reads the linked config, and the include adds settings that never go in the repository
$ git config --show-origin --get user.name; git config --show-origin --get user.email
   file:~/.gitconfig	Ada Lovelace
   file:~/.gitconfig.local	ada@example.org
4. The global ignore file applies in every repository on this machine
$ git init -q project && cd project && touch .DS_Store notes.md && git status --short
   ?? notes.md
5. The repository's own .gitignore keeps per-machine files out of it
$ cd dotfiles && git init -q && touch secrets.local && git add -A && git status --short
   A  .gitignore
   A  gitconfig
   A  gitignore_global
   A  install.sh
   A  zshrc

import hashlib, os, pathlib, subprocess, tempfile, zlib

# Fixed names and times, and no personal git settings, so every hash below is the same on any machine.
ENV = {"PATH": os.environ["PATH"], "HOME": "/home/learner", "LC_ALL": "C",
       "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull,
       "GIT_AUTHOR_NAME": "Ada Lovelace", "GIT_AUTHOR_EMAIL": "ada@example.org",
       "GIT_COMMITTER_NAME": "Ada Lovelace", "GIT_COMMITTER_EMAIL": "ada@example.org"}
clock = [1772355600]                                   # 2026-03-01 09:00 UTC, advanced a minute per commit

def git(*args, cwd, data=None):
    env = dict(ENV, GIT_AUTHOR_DATE=f"@{clock[0]} +0000", GIT_COMMITTER_DATE=f"@{clock[0]} +0000")
    r = subprocess.run(["git", *args], cwd=cwd, env=env, input=data, capture_output=True, check=True)
    return r.stdout.decode().strip()

def commit(message, cwd):
    clock[0] += 60
    git("commit", "-q", "-m", message, cwd=cwd)
    return git("rev-parse", "HEAD", cwd=cwd)

def object_id(kind, body):
    """Every git object is named by the SHA-1 of a header ("blob 6", say), a zero byte, and the body."""
    return hashlib.sha1(f"{kind} {len(body)}".encode() + b"\0" + body).hexdigest()

def tree_id(folder):
    """A tree lists its entries, sorted by name: mode, name, a zero byte, and the entry's id as 20 bytes."""
    entries = []
    for p in folder.iterdir():
        if p.name == ".git": continue
        if p.is_dir(): entries.append((p.name + "/", b"40000", p.name, tree_id(p)))
        else: entries.append((p.name, b"100644", p.name, object_id("blob", p.read_bytes())))
    body = b"".join(mode + b" " + name.encode() + b"\0" + bytes.fromhex(oid)
                    for _, mode, name, oid in sorted(entries))
    return object_id("tree", body)

with tempfile.TemporaryDirectory() as tmp:
    repo = pathlib.Path(tmp)
    git("init", "-q", "-b", "main", cwd=repo)

    print("1. A blob is a file's contents, named by its hash")
    mine = object_id("blob", b"hello\n")
    print(f"   by hand:              {mine}")
    print(f"   git hash-object:      {git('hash-object', '-w', '--stdin', cwd=repo, data=b'hello\n')}")
    stored = repo / ".git" / "objects" / mine[:2] / mine[2:]
    print(f"   stored in .git/objects/{mine[:2]}/{mine[2:8]}..., compressed; inflated it is "
          f"{zlib.decompress(stored.read_bytes())!r}")

    print("2. A tree is a folder listing; a commit names a tree, its parents, an author and a message")
    (repo / "src").mkdir()
    (repo / "README.md").write_text("# Shop\n"); (repo / "src" / "app.py").write_text("print('shop')\n")
    git("add", ".", cwd=repo)
    print(f"   tree by hand:         {tree_id(repo)}")
    print(f"   git write-tree:       {git('write-tree', cwd=repo)}")
    first = commit("Start the shop", repo)
    text = (f"tree {tree_id(repo)}\nauthor Ada Lovelace <ada@example.org> {clock[0]} +0000\n"
            f"committer Ada Lovelace <ada@example.org> {clock[0]} +0000\n\nStart the shop\n")
    print(f"   commit by hand:       {object_id('commit', text.encode())}")
    print(f"   git commit:           {first}")
    print("   git cat-file -p HEAD:")
    for line in git("cat-file", "-p", "HEAD", cwd=repo).splitlines(): print(f"      {line}".rstrip())
    print("   git cat-file -p HEAD^{tree}:")
    for line in git("cat-file", "-p", "HEAD^{tree}", cwd=repo).splitlines(): print(f"      {line}")

    print("3. A new commit is a new snapshot; whatever didn't change is shared, not copied")
    (repo / "README.md").write_text("# Shop\nOpen 9 to 5.\n"); git("add", ".", cwd=repo)
    second = commit("Add opening hours", repo)
    for name, rev in (("first", first), ("second", second)):
        print(f"   {name:6} commit {rev[:10]}  tree {git('rev-parse', rev + '^{tree}', cwd=repo)[:10]}"
              f"  src/ {git('rev-parse', rev + ':src', cwd=repo)[:10]}"
              f"  README.md {git('rev-parse', rev + ':README.md', cwd=repo)[:10]}")
    print(f"   the second commit's parent is the first: {git('rev-parse', 'HEAD^', cwd=repo) == first}")

    print("4. Identical contents are stored once, whatever the file is called")
    count = lambda: git("count-objects", "-v", cwd=repo).splitlines()[0].split()[1]
    before = count()
    (repo / "src" / "copy.py").write_text("print('shop')\n"); git("add", ".", cwd=repo)
    print(f"   objects stored before adding src/copy.py, a copy of app.py: {before}; after: {count()}")
    print(f"   both names point to blob {git('rev-parse', ':src/copy.py', cwd=repo)[:10]}"
          f" = {git('rev-parse', ':src/app.py', cwd=repo)[:10]}")
    kinds = git("cat-file", "--batch-all-objects", "--batch-check=%(objecttype)", cwd=repo).split()
    tally = ", ".join(f"{kinds.count(k)} {k}s" for k in ("blob", "tree", "commit"))
    print(f"   objects in the repository: {tally}")

1. A blob is a file's contents, named by its hash
   by hand:              ce013625030ba8dba906f756967f9e9ca394464a
   git hash-object:      ce013625030ba8dba906f756967f9e9ca394464a
   stored in .git/objects/ce/013625..., compressed; inflated it is b'blob 6\x00hello\n'
2. A tree is a folder listing; a commit names a tree, its parents, an author and a message
   tree by hand:         c93e4e95d9da0e7e336125c34b0ed66cfa186cd0
   git write-tree:       c93e4e95d9da0e7e336125c34b0ed66cfa186cd0
   commit by hand:       883e680149aecf0110fda9335267363c06bca680
   git commit:           883e680149aecf0110fda9335267363c06bca680
   git cat-file -p HEAD:
      tree c93e4e95d9da0e7e336125c34b0ed66cfa186cd0
      author Ada Lovelace <ada@example.org> 1772355660 +0000
      committer Ada Lovelace <ada@example.org> 1772355660 +0000

      Start the shop
   git cat-file -p HEAD^{tree}:
      100644 blob a00621bb3a9b990ee018f8d04c2d3140800d6ca6	README.md
      040000 tree 5234dadb279379214f5a218b331d5db0a198db4f	src
3. A new commit is a new snapshot; whatever didn't change is shared, not copied
   first  commit 883e680149  tree c93e4e95d9  src/ 5234dadb27  README.md a00621bb3a
   second commit 70188257a5  tree 080446fce0  src/ 5234dadb27  README.md 9921ceefb6
   the second commit's parent is the first: True
4. Identical contents are stored once, whatever the file is called
   objects stored before adding src/copy.py, a copy of app.py: 9; after: 9
   both names point to blob 9cf595fed1 = 9cf595fed1
   objects in the repository: 4 blobs, 3 trees, 2 commits

import os, pathlib, subprocess, tempfile

ENV = {"PATH": os.environ["PATH"], "HOME": "/home/learner", "LC_ALL": "C",
       "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull,
       "GIT_AUTHOR_NAME": "Ada Lovelace", "GIT_AUTHOR_EMAIL": "ada@example.org",
       "GIT_COMMITTER_NAME": "Ada Lovelace", "GIT_COMMITTER_EMAIL": "ada@example.org"}
clock = [1772355600]

def git(*args, cwd):
    env = dict(ENV, GIT_AUTHOR_DATE=f"@{clock[0]} +0000", GIT_COMMITTER_DATE=f"@{clock[0]} +0000")
    return subprocess.run(["git", *args], cwd=cwd, env=env, capture_output=True, text=True,
                          check=True).stdout.rstrip()

def commit(message, cwd):
    clock[0] += 60
    with (cwd / "log.txt").open("a") as f: f.write(message + "\n")
    git("add", "log.txt", cwd=cwd); git("commit", "-q", "-m", message, cwd=cwd)
    return git("rev-parse", "--short", "HEAD", cwd=cwd)

def files(repo, *names):
    for name in names:
        text = (repo / ".git" / name).read_text().strip()
        if not text.startswith("ref:"): text = text[:12] + "... (40 hex digits)"
        print(f"   .git/{name:18} contains: {text}")

def graph(repo):
    for line in git("log", "--all", "--graph", "--format=%h %s%d", cwd=repo).splitlines():
        print(f"      {line}".rstrip())

with tempfile.TemporaryDirectory() as tmp:
    repo = pathlib.Path(tmp); git("init", "-q", "-b", "main", cwd=repo)
    a = commit("A: first", repo); b = commit("B: second", repo)

    print("1. A branch is a file holding a commit's id; HEAD is a file naming the current branch")
    files(repo, "HEAD", "refs/heads/main")
    print("2. Making a branch writes one small file; nothing is copied")
    git("branch", "feature", cwd=repo); files(repo, "refs/heads/feature")
    print("3. A commit moves the current branch, and only that one")
    git("switch", "-q", "feature", cwd=repo); c = commit("C: on feature", repo)
    files(repo, "HEAD", "refs/heads/feature", "refs/heads/main"); graph(repo)

    print("4. Detached HEAD: HEAD holds a commit id instead of a branch name")
    git("switch", "-q", "--detach", b, cwd=repo); files(repo, "HEAD")
    d = commit("D: an experiment", repo)
    git("switch", "-q", "main", cwd=repo)
    print(f"   back on main, D ({d}) is on no branch, so log --all no longer shows it:"); graph(repo)

    print("5. The reflog records every move of HEAD, so D can still be found and given a name")
    for line in git("reflog", "--format=%h %gs", cwd=repo).splitlines()[:4]: print(f"      {line}")
    git("branch", "experiment", d, cwd=repo); graph(repo)

    print("6. reset moves the current branch to another commit; the reflog can move it back")
    git("reset", "-q", "--hard", a, cwd=repo)
    print(f"   after reset --hard {a}, main is at {git('rev-parse', '--short', 'main', cwd=repo)}; "
          f"main's reflog: {', '.join(git('reflog', 'main', '--format=%h', cwd=repo).split())}")
    git("reset", "-q", "--hard", "main@{1}", cwd=repo)
    lines = len((repo / "log.txt").read_text().splitlines())
    print(f"   after reset --hard main@{{1}}, main is at {git('rev-parse', '--short', 'main', cwd=repo)} "
          f"again, and log.txt has {lines} lines")

1. A branch is a file holding a commit's id; HEAD is a file naming the current branch
   .git/HEAD               contains: ref: refs/heads/main
   .git/refs/heads/main    contains: c7ebc43f34bf... (40 hex digits)
2. Making a branch writes one small file; nothing is copied
   .git/refs/heads/feature contains: c7ebc43f34bf... (40 hex digits)
3. A commit moves the current branch, and only that one
   .git/HEAD               contains: ref: refs/heads/feature
   .git/refs/heads/feature contains: cb1a81f51ed7... (40 hex digits)
   .git/refs/heads/main    contains: c7ebc43f34bf... (40 hex digits)
      * cb1a81f C: on feature (HEAD -> feature)
      * c7ebc43 B: second (main)
      * 87b02ea A: first
4. Detached HEAD: HEAD holds a commit id instead of a branch name
   .git/HEAD               contains: c7ebc43f34bf... (40 hex digits)
   back on main, D (a02cb58) is on no branch, so log --all no longer shows it:
      * cb1a81f C: on feature (feature)
      * c7ebc43 B: second (HEAD -> main)
      * 87b02ea A: first
5. The reflog records every move of HEAD, so D can still be found and given a name
      c7ebc43 checkout: moving from a02cb58cf95097efe98514c7c1793137bfdbe6c7 to main
      a02cb58 commit: D: an experiment
      c7ebc43 checkout: moving from feature to c7ebc43
      cb1a81f commit: C: on feature
      * a02cb58 D: an experiment (experiment)
      | * cb1a81f C: on feature (feature)
      |/
      * c7ebc43 B: second (HEAD -> main)
      * 87b02ea A: first
6. reset moves the current branch to another commit; the reflog can move it back
   after reset --hard 87b02ea, main is at 87b02ea; main's reflog: 87b02ea, c7ebc43, 87b02ea
   after reset --hard main@{1}, main is at c7ebc43 again, and log.txt has 2 lines

import os, pathlib, subprocess, tempfile

ENV = {"PATH": os.environ["PATH"], "HOME": "/home/learner", "LC_ALL": "C",
       "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull,
       "GIT_AUTHOR_NAME": "Ada Lovelace", "GIT_AUTHOR_EMAIL": "ada@example.org",
       "GIT_COMMITTER_NAME": "Ada Lovelace", "GIT_COMMITTER_EMAIL": "ada@example.org"}
clock = [1772355600]

def git(*args, cwd, check=True):
    clock[0] += 60                                     # every command happens a minute after the last
    env = dict(ENV, GIT_AUTHOR_DATE=f"@{clock[0]} +0000", GIT_COMMITTER_DATE=f"@{clock[0]} +0000")
    r = subprocess.run(["git", *args], cwd=cwd, env=env, capture_output=True, text=True, check=check)
    return r.stdout.rstrip() if check else r

def edit(repo, branch, line, text, message):
    """Switch to a branch, replace one line of menu.txt, and commit."""
    git("switch", "-q", branch, cwd=repo)
    lines = (repo / "menu.txt").read_text().splitlines(); lines[line - 1] = text
    (repo / "menu.txt").write_text("\n".join(lines) + "\n")
    git("commit", "-q", "-am", message, cwd=repo)

def version(repo, rev): return git("show", f"{rev}:menu.txt", cwd=repo).splitlines()

def graph(repo):
    for line in git("log", "--graph", "--format=%h %s%d", "main", cwd=repo).splitlines():
        print(f"      {line}".rstrip())

with tempfile.TemporaryDirectory() as tmp:
    repo = pathlib.Path(tmp); git("init", "-q", "-b", "main", cwd=repo)
    (repo / "menu.txt").write_text("Soup\nBread\nSalad\nPasta\nCake\nTea\nWater\n")
    git("add", ".", cwd=repo); git("commit", "-q", "-m", "Base menu", cwd=repo)
    for b in ("lunch", "dinner", "drinks"): git("branch", b, cwd=repo)
    edit(repo, "lunch", 2, "Sourdough", "Lunch: sourdough")
    edit(repo, "main", 5, "Lemon cake", "Main: lemon cake")

    print("1. A merge compares each side with the merge base, the last commit both sides share")
    base = git("merge-base", "main", "lunch", cwd=repo)
    print(f"   merge base of main and lunch: {base[:7]} {git('log', '-1', '--format=%s', base, cwd=repo)}")
    b, ours, theirs = version(repo, base), version(repo, "main"), version(repo, "lunch")
    git("merge", "-q", "--no-edit", "lunch", cwd=repo)
    result = (repo / "menu.txt").read_text().splitlines()
    print(f"   {'line':5}{'base':12}{'main (ours)':14}{'lunch (theirs)':16}{'rule':24}merged")
    for i, (x, o, t, m) in enumerate(zip(b, ours, theirs, result), 1):
        rule = "nobody changed it" if o == t == x else "only theirs changed" if o == x else \
               "only ours changed" if t == x else "both, the same way" if o == t else "both: conflict"
        print(f"   {i:<5}{x:12}{o:14}{t:16}{rule:24}{m}")

    print("2. A conflict: both sides changed the same line in different ways")
    edit(repo, "dinner", 2, "Garlic bread", "Dinner: garlic bread")
    git("switch", "-q", "main", cwd=repo)
    r = git("merge", "--no-edit", "dinner", cwd=repo, check=False)
    print(f"   git merge dinner exits with status {r.returncode}; unmerged: "
          f"{git('diff', '--name-only', '--diff-filter=U', cwd=repo)}")
    for line in (repo / "menu.txt").read_text().splitlines()[:7]: print(f"      {line}")
    resolved = ["Soup", "Sourdough or garlic bread", "Salad", "Pasta", "Lemon cake", "Tea", "Water"]
    (repo / "menu.txt").write_text("\n".join(resolved) + "\n")
    git("add", "menu.txt", cwd=repo); git("commit", "-q", "--no-edit", cwd=repo)
    print("   resolved by hand, added, committed:"); graph(repo)

    print("3. When one side has no new commits, merging just moves the branch forward")
    git("switch", "-q", "-c", "tea", cwd=repo); edit(repo, "tea", 6, "Mint tea", "Tea: mint")
    git("switch", "-q", "main", cwd=repo); git("merge", "-q", "--ff-only", "tea", cwd=repo)
    same = git("rev-parse", "main", cwd=repo) == git("rev-parse", "tea", cwd=repo)
    print(f"   main and tea are now the same commit: {same}")

    print("4. A rebase replays commits on a new base: same changes, new commits, new ids")
    git("switch", "-q", "drinks", cwd=repo)
    for drink in ("Lemonade", "Coffee"):
        with (repo / "menu.txt").open("a") as f: f.write(drink + "\n")
        git("commit", "-q", "-am", f"Drinks: {drink.lower()}", cwd=repo)
    before = git("rev-list", "--reverse", "main..drinks", cwd=repo).split()
    git("rebase", "-q", "main", cwd=repo)
    after = git("rev-list", "--reverse", "main..drinks", cwd=repo).split()
    def patch(rev):                                    # the lines a commit removes and adds
        diff = git("show", "--format=", "-U0", rev, cwd=repo).splitlines()
        return [l for l in diff if l[:1] in "+-" and not l.startswith(("+++", "---"))]
    for old, new in zip(before, after):
        print(f"   {git('log', '-1', '--format=%s', new, cwd=repo):18} {old[:7]} -> {new[:7]}   "
              f"same change: {patch(old) == patch(new)}")
    git("switch", "-q", "main", cwd=repo); git("merge", "-q", "--ff-only", "drinks", cwd=repo)
    print("   then main fast-forwards to it, and history stays a straight line from the merge:")
    graph(repo)

1. A merge compares each side with the merge base, the last commit both sides share
   merge base of main and lunch: 4e0be3c Base menu
   line base        main (ours)   lunch (theirs)  rule                    merged
   1    Soup        Soup          Soup            nobody changed it       Soup
   2    Bread       Bread         Sourdough       only theirs changed     Sourdough
   3    Salad       Salad         Salad           nobody changed it       Salad
   4    Pasta       Pasta         Pasta           nobody changed it       Pasta
   5    Cake        Lemon cake    Cake            only ours changed       Lemon cake
   6    Tea         Tea           Tea             nobody changed it       Tea
   7    Water       Water         Water           nobody changed it       Water
2. A conflict: both sides changed the same line in different ways
   git merge dinner exits with status 1; unmerged: menu.txt
      Soup
      <<<<<<< HEAD
      Sourdough
      =======
      Garlic bread
      >>>>>>> dinner
      Salad
   resolved by hand, added, committed:
      *   23d796f Merge branch 'dinner' (HEAD -> main)
      |\
      | * 86a5223 Dinner: garlic bread (dinner)
      * |   c3ca78a Merge branch 'lunch'
      |\ \
      | * | a96b1aa Lunch: sourdough (lunch)
      | |/
      * / 3288f75 Main: lemon cake
      |/
      * 4e0be3c Base menu (drinks)
3. When one side has no new commits, merging just moves the branch forward
   main and tea are now the same commit: True
4. A rebase replays commits on a new base: same changes, new commits, new ids
   Drinks: lemonade   e18dd94 -> 6248552   same change: True
   Drinks: coffee     c5b9e4a -> f8fe437   same change: True
   then main fast-forwards to it, and history stays a straight line from the merge:
      * f8fe437 Drinks: coffee (HEAD -> main, drinks)
      * 6248552 Drinks: lemonade
      * 1cd8cf6 Tea: mint (tea)
      *   23d796f Merge branch 'dinner'
      |\
      | * 86a5223 Dinner: garlic bread (dinner)
      * |   c3ca78a Merge branch 'lunch'
      |\ \
      | * | a96b1aa Lunch: sourdough (lunch)
      | |/
      * / 3288f75 Main: lemon cake
      |/
      * 4e0be3c Base menu

import os, pathlib, re, subprocess, sys, tempfile

ENV = {"PATH": os.environ["PATH"], "HOME": "/home/learner", "LC_ALL": "C",
       "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull,
       "GIT_AUTHOR_NAME": "Ada Lovelace", "GIT_AUTHOR_EMAIL": "ada@example.org",
       "GIT_COMMITTER_NAME": "Ada Lovelace", "GIT_COMMITTER_EMAIL": "ada@example.org"}
clock = [1772355600]

def git(*args, cwd):
    clock[0] += 60
    env = dict(ENV, GIT_AUTHOR_DATE=f"@{clock[0]} +0000", GIT_COMMITTER_DATE=f"@{clock[0]} +0000")
    return subprocess.run(["git", *args], cwd=cwd, env=env, capture_output=True, text=True,
                          check=True).stdout.rstrip()

GOOD = '''import re, unicodedata

def slug(title):
    """Make a web address from a title: "Creme Brulee, 2 Ways!" -> "creme-brulee-2-ways"."""
    plain = unicodedata.normalize("NFKD", title).encode("ascii", "ignore").decode()
    return re.sub(r"[^a-z0-9]+", "-", plain.lower()).strip("-")
'''
BAD = GOOD.replace('    plain = unicodedata.normalize("NFKD", title).encode("ascii", "ignore").decode()\n'
                   '    return re.sub(r"[^a-z0-9]+", "-", plain.lower()).strip("-")\n',
                   '    return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")\n')
CHECK = ('from recipes import slug\n'
         'raise SystemExit(slug("Crème Brûlée, 2 Ways!") != "creme-brulee-2-ways")\n')

with tempfile.TemporaryDirectory() as tmp:
    root = pathlib.Path(tmp); repo = root / "site"; repo.mkdir()
    git("init", "-q", "-b", "main", cwd=repo)
    (repo / "recipes.py").write_text(GOOD); (repo / "list.txt").write_text("")
    git("add", ".", cwd=repo); git("commit", "-q", "-m", "Add slug() for recipe addresses", cwd=repo)
    number = {git("rev-parse", "HEAD", cwd=repo): 1}
    for n in range(2, 41):                             # 39 more commits; number 27 breaks slug()
        if n == 27:
            (repo / "recipes.py").write_text(BAD); message = "Simplify slug()"
        else:
            with (repo / "list.txt").open("a") as f: f.write(f"recipe {n}\n")
            message = f"Add recipe {n}"
        git("commit", "-q", "-am", message, cwd=repo)
        number[git("rev-parse", "HEAD", cwd=repo)] = n
    (root / "check.py").write_text(CHECK)
    (root / "check.sh").write_text(f'#!/bin/sh\nPYTHONPATH=. exec "{sys.executable}" ../check.py\n')
    (root / "check.sh").chmod(0o755)
    run = lambda: subprocess.run(["../check.sh"], cwd=repo, env=ENV, capture_output=True).returncode
    first = git("rev-list", "--max-parents=0", "HEAD", cwd=repo)
    print(f"40 commits. The test (../check.sh) exits with status {run()} at HEAD: something broke.")

    print("1. git bisect run: a binary search over history, with the test deciding good or bad")
    git("bisect", "start", "HEAD", first, cwd=repo)
    git("bisect", "run", "../check.sh", cwd=repo)
    bad = git("rev-parse", "refs/bisect/bad", cwd=repo)
    steps = [l.split()[2:4] for l in git("bisect", "log", cwd=repo).splitlines()
             if l.startswith(("git bisect good ", "git bisect bad "))]
    tried = [f"{number[h]} ({verdict})" for verdict, h in steps]
    print(f"   it tested commits {', '.join(tried[:-1])} and {tried[-1]}")
    print(f"   {len(steps)} tests for 39 suspects; the first bad commit is number {number[bad]}: "
          f"{git('log', '-1', '--format=%h %s', bad, cwd=repo)}")
    git("bisect", "reset", cwd=repo)

    print("2. git log -S: the commits that added or removed a piece of text")
    for line in git("log", "-S", "normalize", "--format=%h %s", cwd=repo).splitlines(): print(f"   {line}")

    print("3. git blame: which commit last changed each line")
    blame = git("blame", "--line-porcelain", "-L", "3,5", "recipes.py", cwd=repo).splitlines()
    commits = [l.split()[0] for l in blame if re.match(r"[0-9a-f]{40} \d+ \d+", l)]
    summaries = [l[len("summary "):] for l in blame if l.startswith("summary ")]
    code = [l[1:] for l in blame if l.startswith("\t")]
    for h, summary, line in zip(commits, summaries, code):
        print(f"   {h[:7]} {summary[:31]:31} | {line}")

    print("4. git revert: a new commit that undoes an old one, without rewriting history")
    git("revert", "--no-edit", bad, cwd=repo)
    print(f"   {git('log', '-1', '--format=%h %s', cwd=repo)}; the test now exits with status {run()}")

40 commits. The test (../check.sh) exits with status 1 at HEAD: something broke.
1. git bisect run: a binary search over history, with the test deciding good or bad
   it tested commits 20 (good), 30 (bad), 25 (good), 27 (bad) and 26 (good)
   5 tests for 39 suspects; the first bad commit is number 27: 8ec864a Simplify slug()
2. git log -S: the commits that added or removed a piece of text
   8ec864a Simplify slug()
   dae9473 Add slug() for recipe addresses
3. git blame: which commit last changed each line
   dae9473 Add slug() for recipe addresses | def slug(title):
   dae9473 Add slug() for recipe addresses |     """Make a web address from a title: "Creme Brulee, 2 Ways!" -> "creme-brulee-2-ways"."""
   8ec864a Simplify slug()                 |     return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
4. git revert: a new commit that undoes an old one, without rewriting history
   e53f276 Revert "Simplify slug()"; the test now exits with status 0

import os, pathlib, subprocess, tempfile

# Bare repositories in a temporary folder stand in for GitHub: "upstream" is the project, "fork" is
# the contributor's copy of it. Grace maintains the project; Ada contributes.
BASE = {"PATH": os.environ["PATH"], "HOME": "/home/learner", "LC_ALL": "C",
        "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": os.devnull}
PEOPLE = {"grace": ("Grace Hopper", "grace@example.org"), "ada": ("Ada Lovelace", "ada@example.org")}
clock = [1772355600]

def git(who, *args, cwd, check=True):
    clock[0] += 60
    name, email = PEOPLE[who]
    env = dict(BASE, GIT_AUTHOR_NAME=name, GIT_AUTHOR_EMAIL=email, GIT_COMMITTER_NAME=name,
               GIT_COMMITTER_EMAIL=email, GIT_AUTHOR_DATE=f"@{clock[0]} +0000",
               GIT_COMMITTER_DATE=f"@{clock[0]} +0000")
    r = subprocess.run(["git", *args], cwd=cwd, env=env, capture_output=True, text=True)
    if check and r.returncode: raise SystemExit(r.stderr)
    return r.stdout.rstrip() if check else r

def change(who, repo, name, text, message):
    (repo / name).write_text(text)
    git(who, "add", name, cwd=repo); git(who, "commit", "-q", "-m", message, cwd=repo)

def graph(who, repo, *revs):
    hide = "--decorate-refs-exclude=refs/remotes/*/HEAD"    # newer gits add these; older ones don't
    for line in git(who, "log", "--graph", "--format=%h %an: %s%d", hide, *revs, cwd=repo).splitlines():
        print(f"      {line}".rstrip())

with tempfile.TemporaryDirectory() as tmp:
    gh = pathlib.Path(tmp).resolve()
    print("1. Grace creates the project; Ada forks it on GitHub and clones her fork")
    git("grace", "init", "-q", "--bare", "-b", "main", "upstream.git", cwd=gh)
    git("grace", "clone", "-q", "upstream.git", "grace", cwd=gh)
    grace, ada = gh / "grace", gh / "ada"
    git("grace", "switch", "-q", "-c", "main", cwd=grace)
    change("grace", grace, "README.md", "# Recipes\nA small recipe site.\n", "Start the project")
    git("grace", "push", "-q", "origin", "main", cwd=grace)
    git("ada", "clone", "-q", "--bare", "upstream.git", "fork.git", cwd=gh)      # what the Fork button does
    git("ada", "clone", "-q", "fork.git", "ada", cwd=gh)
    git("ada", "remote", "add", "upstream", str(gh / "upstream.git"), cwd=ada)
    for line in git("ada", "remote", "-v", cwd=ada).splitlines():
        name, rest = line.split("\t"); url, kind = rest.rsplit(" ", 1)      # paths may contain spaces
        print(f"   Ada's remote {name:8} is {pathlib.Path(url).name} {kind}")

    print("2. Ada works on a branch and pushes it to her fork; that's what a pull request is opened from")
    git("ada", "switch", "-q", "-c", "fix-typo", cwd=ada)
    change("ada", ada, "README.md", "# Recipes\nA small site for recipes.\n", "Fix the README's wording")
    git("ada", "push", "-q", "-u", "origin", "fix-typo", cwd=ada)
    print(f"   {git('ada', 'status', '-sb', cwd=ada).splitlines()[0]}")

    print("3. Meanwhile Grace pushes to the project; a push that would lose her commit is refused")
    change("grace", grace, "CONTRIBUTING.md", "Open a pull request.\n", "Add contributing notes")
    git("grace", "push", "-q", "origin", "main", cwd=grace)
    git("ada", "switch", "-q", "main", cwd=ada)
    change("ada", ada, "notes.txt", "todo\n", "A commit made on main by mistake")
    r = git("ada", "push", "--porcelain", "upstream", "main", cwd=ada, check=False)
    print(f"   git push upstream main: status {r.returncode}, "
          f"{[l for l in r.stdout.splitlines() if l.startswith('!')][0].split(chr(9))[-1]}")
    git("ada", "reset", "-q", "--hard", "origin/main", cwd=ada)

    print("4. Ada brings her branch up to date: fetch upstream, rebase onto it, force-push with a lease")
    git("ada", "fetch", "-q", "upstream", cwd=ada)
    git("ada", "switch", "-q", "fix-typo", cwd=ada); git("ada", "rebase", "-q", "upstream/main", cwd=ada)
    git("ada", "push", "-q", "--force-with-lease", "origin", "fix-typo", cwd=ada)
    graph("ada", ada, "fix-typo")

    print("5. Grace reviews and merges the pull request: a merge commit on the project's main")
    git("grace", "fetch", "-q", str(gh / "fork.git"), "fix-typo", cwd=grace)
    git("grace", "merge", "-q", "--no-ff", "-m", "Merge pull request #1 from ada/fix-typo", "FETCH_HEAD",
        cwd=grace)
    git("grace", "push", "-q", "origin", "main", cwd=grace)
    graph("grace", grace, "main")

    print("6. Ada syncs: update her main from upstream, push it to her fork, delete the merged branch")
    git("ada", "switch", "-q", "main", cwd=ada)
    git("ada", "pull", "-q", "--ff-only", "upstream", "main", cwd=ada)
    git("ada", "push", "-q", "origin", "main", ":fix-typo", cwd=ada)     # ":name" deletes a remote branch
    git("ada", "branch", "-q", "-D", "fix-typo", cwd=ada)
    same = {git("ada", "rev-parse", r, cwd=gh / repo) for repo, r in
            (("upstream.git", "main"), ("fork.git", "main"), ("ada", "main"), ("grace", "main"))}
    refs = git("ada", "for-each-ref", "--format=%(refname)", "refs/heads", "refs/remotes", cwd=ada).split()
    branches = [r.split("/", 2)[2] for r in refs if not r.endswith("/HEAD")]
    print(f"   all four copies of main are the same commit: {len(same) == 1}")
    print(f"   Ada's branches and remote-tracking branches: {', '.join(branches)}")

1. Grace creates the project; Ada forks it on GitHub and clones her fork
   Ada's remote origin   is fork.git (fetch)
   Ada's remote origin   is fork.git (push)
   Ada's remote upstream is upstream.git (fetch)
   Ada's remote upstream is upstream.git (push)
2. Ada works on a branch and pushes it to her fork; that's what a pull request is opened from
   ## fix-typo...origin/fix-typo
3. Meanwhile Grace pushes to the project; a push that would lose her commit is refused
   git push upstream main: status 1, [rejected] (fetch first)
4. Ada brings her branch up to date: fetch upstream, rebase onto it, force-push with a lease
      * caddb90 Ada Lovelace: Fix the README's wording (HEAD -> fix-typo, origin/fix-typo)
      * 5202fa0 Grace Hopper: Add contributing notes (upstream/main)
      * 35e30f9 Grace Hopper: Start the project (origin/main, main)
5. Grace reviews and merges the pull request: a merge commit on the project's main
      *   b997fe3 Grace Hopper: Merge pull request #1 from ada/fix-typo (HEAD -> main, origin/main)
      |\
      | * caddb90 Ada Lovelace: Fix the README's wording
      |/
      * 5202fa0 Grace Hopper: Add contributing notes
      * 35e30f9 Grace Hopper: Start the project
6. Ada syncs: update her main from upstream, push it to her fork, delete the merged branch
   all four copies of main are the same commit: True
   Ada's branches and remote-tracking branches: main, origin/main, upstream/main

import ast, re

# 1. "Go to file": find the files whose paths contain the query's letters in order, best first.
FILES = ["README.md", "pyproject.toml", "src/shop/__init__.py", "src/shop/models/order.py",
         "src/shop/models/data_loader.py", "src/shop/views/menu.py", "src/shop/views/order_detail.py",
         "src/shop/utils/money.py", "tests/test_models.py", "tests/test_order_detail.py", "docs/deploy.md"]

def fuzzy(query, path):
    """The best score for finding the query's letters in order in the path, and where they are.
    A letter scores more at the start of a word, right after the previous match, or in the file name;
    letters skipped between matches cost a little."""
    q, p, n = query.lower(), path.lower(), len(path)
    name_start = path.rfind("/") + 1
    def value(j): return 1 + 8 * (j == 0 or p[j - 1] in "/_-.") + 2 * (j >= name_start)
    best = {}                                          # best[(i, j)]: score with q[i] matched at p[j]
    for i, ch in enumerate(q):
        for j in range(n):
            if p[j] != ch: continue
            if i == 0: best[(i, j)] = (value(j) - 0.1 * j, [j])
            else:
                options = [(s + value(j) + (8 if k == j - 1 else -0.2 * (j - k - 1)), at + [j])
                           for (pi, k), (s, at) in best.items() if pi == i - 1 and k < j]
                if options: best[(i, j)] = max(options)
    finals = [v for (i, _), v in best.items() if i == len(q) - 1]
    return max(finals) if finals else None

def marked(path, at): return "".join(f"[{c}]" if i in at else c for i, c in enumerate(path))

for query in ("mdl", "tmod", "odet", "ord"):
    ranked = sorted(((fuzzy(query, f), f) for f in FILES if fuzzy(query, f)), key=lambda t: -t[0][0])
    print(f"go to file '{query}':")
    for (score, at), f in ranked[:3]: print(f"   {score:5.1f}  {marked(f, at)}")

# 2. Search in files: every line that matches, with its file and line number, like rg or grep -rn
PROJECT = {"src/shop/models/order.py": "class Order:\n    total = 0\n",
           "src/shop/views/menu.py": "def show(menu):\n    # TODO: prices\n    return menu\n",
           "src/shop/utils/money.py": "def cents(x):\n    return round(x * 100)  # TODO: negative?\n"}
print("search for TODO in every file:")
for path, text in sorted(PROJECT.items()):
    for n, line in enumerate(text.splitlines(), 1):
        if re.search(r"TODO", line): print(f"   {path}:{n}: {line.strip()}")

# 3. Rename: text replacement versus a rename that knows what each name refers to
SOURCE = '''def total(items):
    return sum(items)

class Receipt:
    total = 0                              # a different thing with the same name

def report(carts):
    totals = [total(cart) for cart in carts]
    print("total:", total(totals))
'''

def rename_function(source, old, new):
    """Rename the module-level function `old`: its definition, and each use that refers to it."""
    tree = ast.parse(source); spots = []
    def visit(node, shadowed):
        if isinstance(node, ast.FunctionDef) and node.name == old and not shadowed:
            line = source.splitlines()[node.lineno - 1]
            spots.append((node.lineno, line.index(old, node.col_offset)))
        if isinstance(node, ast.Name) and node.id == old and not shadowed:
            spots.append((node.lineno, node.col_offset))
        if isinstance(node, (ast.ClassDef, ast.FunctionDef)):      # a name assigned inside is a new thing
            assigned = {t.id for n in ast.walk(node) for t in getattr(n, "targets", [])
                        if isinstance(t, ast.Name)}
            args = {a.arg for a in node.args.args} if isinstance(node, ast.FunctionDef) else set()
            shadowed = shadowed or old in assigned | args
            for child in node.body: visit(child, shadowed)
        else:
            for child in ast.iter_child_nodes(node): visit(child, shadowed)
    for child in tree.body: visit(child, False)
    lines = source.splitlines(keepends=True)
    for row, col in sorted(spots, reverse=True):
        lines[row - 1] = lines[row - 1][:col] + new + lines[row - 1][col + len(old):]
    return "".join(lines)

def changed(before, after):
    return [f"   line {n}: {b.strip()}  ->  {a.strip()}"
            for n, (b, a) in enumerate(zip(before.splitlines(), after.splitlines()), 1) if a != b]

for label, after in (("replace every 'total'", SOURCE.replace("total", "grand_total")),
                     ("replace whole words only, \\btotal\\b", re.sub(r"\btotal\b", "grand_total", SOURCE)),
                     ("rename the function, as an editor's Rename Symbol does",
                      rename_function(SOURCE, "total", "grand_total"))):
    print(f"{label}:"); print("\n".join(changed(SOURCE, after)))
    compile(after, "after.py", "exec")

go to file 'mdl':
    28.1  src/shop/[m]odels/[d]ata_[l]oader.py
    15.5  tests/test_[m]o[d]e[l]s.py
     9.7  src/shop/[m]o[d]e[l]s/order.py
go to file 'tmod':
    42.6  tests/[t]est_[m][o][d]els.py
go to file 'odet':
    41.9  tests/test_[o]rder_[d][e][t]ail.py
    41.5  src/shop/views/[o]rder_[d][e][t]ail.py
    20.0  src/shop/m[o][d][e]ls/da[t]a_loader.py
go to file 'ord':
    31.9  tests/test_[o][r][d]er_detail.py
    31.5  src/shop/views/[o][r][d]er_detail.py
    31.4  src/shop/models/[o][r][d]er.py
search for TODO in every file:
   src/shop/utils/money.py:2: return round(x * 100)  # TODO: negative?
   src/shop/views/menu.py:2: # TODO: prices
replace every 'total':
   line 1: def total(items):  ->  def grand_total(items):
   line 5: total = 0                              # a different thing with the same name  ->  grand_total = 0                              # a different thing with the same name
   line 8: totals = [total(cart) for cart in carts]  ->  grand_totals = [grand_total(cart) for cart in carts]
   line 9: print("total:", total(totals))  ->  print("grand_total:", grand_total(grand_totals))
replace whole words only, \btotal\b:
   line 1: def total(items):  ->  def grand_total(items):
   line 5: total = 0                              # a different thing with the same name  ->  grand_total = 0                              # a different thing with the same name
   line 8: totals = [total(cart) for cart in carts]  ->  totals = [grand_total(cart) for cart in carts]
   line 9: print("total:", total(totals))  ->  print("grand_total:", grand_total(totals))
rename the function, as an editor's Rename Symbol does:
   line 1: def total(items):  ->  def grand_total(items):
   line 8: totals = [total(cart) for cart in carts]  ->  totals = [grand_total(cart) for cart in carts]
   line 9: print("total:", total(totals))  ->  print("total:", grand_total(totals))

import random, traceback

def average_prices(text):
    """The average price in each category of a menu, from lines like '[drinks]' and 'tea 2.50'."""
    result, category, prices = {}, None, []
    for line in text.splitlines():
        line = line.strip()
        if not line or line.startswith("#"): continue
        if line.startswith("["):
            if category is not None:
                result[category] = average(prices)
            category, prices = line.strip("[]"), []
        else:
            name, price = line.rsplit(" ", 1)
            prices.append(float(price))
    if category is not None and prices:
        result[category] = average(prices)
    return result

def average(numbers): return round(sum(numbers) / len(numbers), 2)

def make_menu():
    """A menu of about a hundred lines; one category, part-way down, has nothing in it yet."""
    rng = random.Random(15); lines = ["# The Corner Cafe menu"]
    for category in ["breakfast", "soups", "salads", "mains", "specials", "sides", "cakes",
                     "ice cream", "tea", "coffee", "juice", "wine"]:
        lines += ["", f"[{category}]"]
        if category == "specials": lines.append("# coming soon"); continue
        lines += [f"{category} item {i} {rng.randint(150, 1800) / 100:.2f}" for i in range(1, 8)]
    return lines + ["# prices include service"]

menu = make_menu()
print(f"1. The program fails on the real menu, {len(menu)} lines long")
try:
    average_prices("\n".join(menu))
except Exception as error:
    frames = traceback.extract_tb(error.__traceback__)
    print(f"   the last line of the traceback says what went wrong: {type(error).__name__}: {error}")
    print("   the lines above it say where; from the bottom up, the innermost call first:")
    for f in reversed(frames):
        name = f.name if f.name.startswith("<") else f.name + "()"
        print(f"      line {f.lineno:2} in {name:16} {f.line}")

def fails(lines):
    try: average_prices("\n".join(lines)); return False
    except ZeroDivisionError: return True

def shrink(lines, fails):
    """Delta debugging (Zeller, 2002): keep dropping chunks of the input while it still fails."""
    n, tests, sizes = 2, 0, [len(lines)]
    while len(lines) >= 2:
        size = -(-len(lines) // n)
        chunks = [lines[i:i + size] for i in range(0, len(lines), size)]
        for i in range(len(chunks)):
            rest = [line for j, c in enumerate(chunks) if j != i for line in c]
            tests += 1
            if fails(rest):
                lines, n = rest, max(n - 1, 2); sizes.append(len(lines)); break
        else:
            if n >= len(lines): break
            n = min(2 * n, len(lines))
    return lines, tests, sizes

print("2. Shrinking the input automatically, keeping only what's needed to fail")
small, tests, sizes = shrink(menu, fails)
print(f"   {tests} test runs; the failing input went from {' -> '.join(map(str, sizes))} lines:")
for line in small: print(f"      {line}")
print(f"   with the first line alone it works: {not fails(small[:1])}; "
      f"with the second alone: {not fails(small[1:])}")

print("3. The bug report this makes possible")
print(f"""   average_prices() raises ZeroDivisionError when a category has no items and another follows.
   To reproduce:  average_prices("{small[0]}\\n{small[1]}")
   Expected:      {{}} or {{'{small[0].strip('[]')}': None}}, not a crash
   Actually:      ZeroDivisionError: division by zero, in average(), called from line 11
   Tried:         the same with an empty category at the end of the menu works""")

1. The program fails on the real menu, 104 lines long
   the last line of the traceback says what went wrong: ZeroDivisionError: division by zero
   the lines above it say where; from the bottom up, the innermost call first:
      line 20 in average()        def average(numbers): return round(sum(numbers) / len(numbers), 2)
      line 11 in average_prices() result[category] = average(prices)
      line 35 in <module>         average_prices("\n".join(menu))
2. Shrinking the input automatically, keeping only what's needed to fail
   30 test runs; the failing input went from 104 -> 52 -> 26 -> 19 -> 14 -> 10 -> 8 -> 6 -> 4 -> 3 -> 2 lines:
      [specials]
      [sides]
   with the first line alone it works: True; with the second alone: True
3. The bug report this makes possible
   average_prices() raises ZeroDivisionError when a category has no items and another follows.
   To reproduce:  average_prices("[specials]\n[sides]")
   Expected:      {} or {'specials': None}, not a crash
   Actually:      ZeroDivisionError: division by zero, in average(), called from line 11
   Tried:         the same with an empty category at the end of the menu works

import random
from dataclasses import dataclass

# A small model of memory, of the kind spaced-repetition programs are built on. Its constants are
# chosen to show the shape of the effect; the experiments described in the chapter are the evidence.
#   - Each fact has a stability S: the number of days until your chance of recalling it falls to 90%.
#     t days after you last saw it, the chance is 1 / (1 + t / (9 * S)). A new fact has S = 1.
#   - Recalling it makes S larger, by more the nearer you were to forgetting: S * (1 + 15 * (1 - chance)).
#     A review of something you can recall easily adds almost nothing.
#   - Failing to recall it sets S back to 1: you look up the answer and start again.
def chance(days, stability): return 1 / (1 + days / (9 * stability))

@dataclass
class Fact:
    S: float = 1.0
    seen: float = 0.0
    reviews: int = 0
    box: int = 1

    def review(self, day, rng):
        p = chance(day - self.seen, self.S)
        recalled = rng.random() < p
        self.S = self.S * (1 + 15 * (1 - p)) if recalled else 1.0
        self.seen, self.reviews = day, self.reviews + 1
        return recalled

def cram(fact, rng):                                  # four more passes through the cards, that evening
    for k in range(1, 5): fact.review(k / 48, rng)

def fixed(fact, rng):                                 # the same five days for every fact
    for day in (1, 3, 7, 14, 28): fact.review(day, rng)

def leitner(fact, rng):
    """Leitner boxes: a fact in box k comes back 2**(k-1) days later; recalling it moves it up a box,
    forgetting it sends it back to box 1. Hard facts are seen often, easy ones rarely."""
    day = 1
    while day <= 30:
        fact.box = min(fact.box + 1, 5) if fact.review(day, rng) else 1
        day += 2 ** (fact.box - 1)

def at_ninety(fact, rng):
    """What modern schedulers do: bring each fact back just as its predicted recall falls to 90%."""
    day = fact.seen + fact.S
    while day <= 30:
        fact.review(day, rng); day = fact.seen + fact.S

print("200 facts learned on day 0, then reviewed only during the first 30 days")
print(f"   {'plan':38}{'reviews per fact':>17}{'recall, day 60':>16}{'recall, day 90':>16}")
for label, plan in (("no review", None), ("cram: 4 more passes on day 0", cram),
                    ("fixed: days 1, 3, 7, 14 and 28", fixed), ("Leitner boxes", leitner),
                    ("each fact when recall reaches 90%", at_ninety)):
    rng = random.Random(16); facts = [Fact() for _ in range(200)]
    for f in facts:
        if plan: plan(f, rng)
    reviews = sum(f.reviews for f in facts) / len(facts)
    later = [sum(chance(day - f.seen, f.S) for f in facts) / len(facts) for day in (60, 90)]
    print(f"   {label:38}{reviews:>17.1f}{later[0]:>16.0%}{later[1]:>16.0%}")

print("one fact under the 90% rule: the gaps between reviews grow as it becomes stable")
f, rng = Fact(), random.Random(3); days = []
while f.seen + f.S <= 365:
    f.review(f.seen + f.S, rng); days.append(round(f.seen))
print("   reviewed on days " + ", ".join(map(str, days)) + f"; by then it would last {f.S:.0f} days")

200 facts learned on day 0, then reviewed only during the first 30 days
   plan                                   reviews per fact  recall, day 60  recall, day 90
   no review                                           0.0             13%              9%
   cram: 4 more passes on day 0                        4.0             15%             10%
   fixed: days 1, 3, 7, 14 and 28                      5.0             81%             74%
   Leitner boxes                                       4.9             83%             74%
   each fact when recall reaches 90%                   4.8             86%             78%
one fact under the 90% rule: the gaps between reviews grow as it becomes stable
   reviewed on days 1, 4, 10, 25, 64, 162; by then it would last 244 days

import ast, math
from fractions import Fraction
from decimal import Decimal

# 1. An expression is evaluated one operation at a time. This stepper shows each step.
def steps(source):
    tree = ast.parse(source, mode="eval")
    def reduce(node):                                     # evaluate the first operation that's ready
        if isinstance(node, ast.BinOp):
            if isinstance(node.left, ast.Constant) and isinstance(node.right, ast.Constant):
                return ast.Constant(eval(ast.unparse(node))), True
            node.left, done = reduce(node.left)
            if not done: node.right, done = reduce(node.right)
            return node, done
        if isinstance(node, ast.UnaryOp) and isinstance(node.operand, ast.Constant):
            return ast.Constant(eval(ast.unparse(node))), True
        return node, False
    out = [ast.unparse(tree)]
    while True:
        tree.body, done = reduce(tree.body)
        if not done: return out
        out.append(ast.unparse(tree))
for line in steps("2 + 3 * (7 - 4) ** 2 // 5"):
    print("   " + line)

# 2. Every value has a type, and the type decides what operations mean
for v in (42, 4.2, "42", True, None, [4, 2], (4, 2), {4: 2}):
    print(f"{v!r:>8} has type {type(v).__name__}")
print(f"\n'42' * 2 = {'42' * 2!r}, but 42 * 2 = {42 * 2}; '42' + 2 raises: ", end="")
try: "42" + 2
except TypeError as e: print(e)

# 3. Integers never overflow; floats are binary fractions and round
print(f"\n2 ** 100 = {2 ** 100:,}")
print(f"0.1 + 0.2 = {0.1 + 0.2!r}; equal to 0.3? {0.1 + 0.2 == 0.3}; "
      f"math.isclose: {math.isclose(0.1 + 0.2, 0.3)}")
print(f"0.1 is really {Decimal(0.1)}")
print(f"exact alternatives: Fraction(1, 10) * 3 = {Fraction(1, 10) * 3}, "
      f"Decimal('0.1') * 3 = {Decimal('0.1') * 3}")
print(f"1e308 * 10 = {1e308 * 10}; float('nan') == float('nan'): {float('nan') == float('nan')}")

# 4. Three kinds of division, and rounding that surprises people
for a, b in ((7, 2), (-7, 2)):
    print(f"{a:>2} / {b} = {a / b:<5} {a:>2} // {b} = {a // b:<3} {a:>2} % {b} = {a % b}")
print(f"int(-3.7) = {int(-3.7)} (towards zero); math.floor(-3.7) = {math.floor(-3.7)}; "
      f"round(2.5) = {round(2.5)}, round(3.5) = {round(3.5)} (to even)")

# 5. Conversions and truthiness
print(f"\nint('42') = {int('42')}; bool('False') = {bool('False')}; bool('') = {bool('')}")
for text in ("4.2", "forty-two"):
    try: int(text)
    except ValueError as e: print(f"int({text!r}) raises ValueError: {e}")
falsy = [v for v in (0, 0.0, "", [], {}, None, "0", [0], " ") if not v]
print(f"falsy: {falsy}; everything else, including '0', [0] and ' ', is truthy")
print(f"and/or return an operand: 0 or 'default' = {0 or 'default'!r}; "
      f"'x' and 'y' = {'x' and 'y'!r}; [] and print('never') = {[] and print('never')!r}")

   2 + 3 * (7 - 4) ** 2 // 5
   2 + 3 * 3 ** 2 // 5
   2 + 3 * 9 // 5
   2 + 27 // 5
   2 + 5
   7
      42 has type int
     4.2 has type float
    '42' has type str
    True has type bool
    None has type NoneType
  [4, 2] has type list
  (4, 2) has type tuple
  {4: 2} has type dict

'42' * 2 = '4242', but 42 * 2 = 84; '42' + 2 raises: can only concatenate str (not "int") to str

2 ** 100 = 1,267,650,600,228,229,401,496,703,205,376
0.1 + 0.2 = 0.30000000000000004; equal to 0.3? False; math.isclose: True
0.1 is really 0.1000000000000000055511151231257827021181583404541015625
exact alternatives: Fraction(1, 10) * 3 = 3/10, Decimal('0.1') * 3 = 0.3
1e308 * 10 = inf; float('nan') == float('nan'): False
 7 / 2 = 3.5    7 // 2 = 3    7 % 2 = 1
-7 / 2 = -3.5  -7 // 2 = -4  -7 % 2 = 1
int(-3.7) = -3 (towards zero); math.floor(-3.7) = -4; round(2.5) = 2, round(3.5) = 4 (to even)

int('42') = 42; bool('False') = True; bool('') = False
int('4.2') raises ValueError: invalid literal for int() with base 10: '4.2'
int('forty-two') raises ValueError: invalid literal for int() with base 10: 'forty-two'
falsy: [0, 0.0, '', [], {}, None]; everything else, including '0', [0] and ' ', is truthy
and/or return an operand: 0 or 'default' = 'default'; 'x' and 'y' = 'y'; [] and print('never') = []

import builtins

# 1. Assignment binds a name to an object. Two names can be bound to one object.
scores = [70, 85]
backup = scores                      # not a copy: a second name for the same list
backup.append(90)                    # mutate the object through one name ...
print(f"scores = {scores}, backup is scores: {backup is scores}")    # ... and both names see it
backup = [0]                         # rebinding a name leaves the object alone
print(f"after rebinding backup: scores = {scores}, backup = {backup}")

count = 5
before = count
count += 1                           # ints can't change, so += makes a new int and rebinds count
print(f"count = {count}, before = {before}")
total = scores
total += [100]                       # but on a list, += changes the list in place
print(f"after total += [100], scores = {scores}")

a, b = 1, 2
a, b = b, a                          # the right side is built first, then unpacked: a swap
print(f"swapped: a = {a}, b = {b}")

# 2. Where Python looks for a name: Local, Enclosing function, Global, then Built-in
level = "global"
def outer():
    level = "enclosing"
    def inner():
        found = "built-in" if len is builtins.len else "shadowed"
        return f"inner sees {level!r}; len is the {found} one"
    return inner()
print("\n" + outer())

list = ["a", "b"]                    # shadowing a built-in with a global name
try: list("xyz")
except TypeError as e: print(f"after list = [...], list('xyz') fails: {e}")
del list                             # the global goes; the built-in is visible again
print(f"after del list: list('xyz') = {list('xyz')}")

# 3. Assigning anywhere in a function makes the name local to the whole function
visits = 0
def record_visit():
    visits += 1                      # reads the local visits, which has no value yet
try: record_visit()
except UnboundLocalError as e: print(f"\nrecord_visit(): UnboundLocalError: {e}")
def record_visit_fixed():
    global visits                    # say so explicitly, and the name means the global
    visits += 1
record_visit_fixed(); record_visit_fixed()
print(f"with global: visits = {visits}")

def make_counter():
    n = 0
    def step():
        nonlocal n                   # the same rule one level out: n belongs to make_counter
        n += 1
        return n
    return step
step = make_counter()
print(f"a counter with nonlocal: {[step() for _ in range(3)]}")

# 4. Blocks don't make scopes: a loop variable outlives its loop; a comprehension's doesn't
for i in range(3):
    pass
print(f"\nafter the for loop, i = {i}")
squares = [j * j for j in range(3)]
print(f"after the comprehension, j defined: {'j' in globals()}")
if (n := len(squares)) > 2:          # := assigns inside an expression
    print(f"the walrus bound n = {n} inside an if condition")

scores = [70, 85, 90], backup is scores: True
after rebinding backup: scores = [70, 85, 90], backup = [0]
count = 6, before = 5
after total += [100], scores = [70, 85, 90, 100]
swapped: a = 2, b = 1

inner sees 'enclosing'; len is the built-in one
after list = [...], list('xyz') fails: 'list' object is not callable
after del list: list('xyz') = ['x', 'y', 'z']

record_visit(): UnboundLocalError: cannot access local variable 'visits' where it is not associated with a value
with global: visits = 2
a counter with nonlocal: [1, 2, 3]

after the for loop, i = 2
after the comprehension, j defined: False
the walrus bound n = 3 inside an if condition

# 1. Insertion sort, checking its loop invariant on every pass: items[:i] is always sorted
def insertion_sort(items, show=False):
    items = list(items)
    for i in range(1, len(items)):
        assert items[:i] == sorted(items[:i]), "invariant broken"      # true before each pass
        current, j = items[i], i - 1
        while j >= 0 and items[j] > current:     # shift bigger items right to make a gap
            items[j + 1] = items[j]
            j -= 1
        items[j + 1] = current
        if show: print(f"   after pass {i}: sorted {items[:i + 1]} | still to do {items[i + 1:]}")
    return items
insertion_sort([5, 2, 9, 1, 7], show=True)

# 2. Binary search. Invariant: if target is in xs, it's in xs[lo:hi]. Variant: hi - lo shrinks.
def search(xs, target):
    lo, hi, steps = 0, len(xs), 0
    while lo < hi:
        steps += 1
        mid = (lo + hi) // 2
        if xs[mid] < target: lo = mid + 1        # target can't be at mid or left of it
        elif xs[mid] > target: hi = mid          # target can't be at mid or right of it
        else: return mid, steps
    return None, steps
def search_bug1(xs, target):                     # an off-by-one: the last element is never checked
    lo, hi = 0, len(xs) - 1
    while lo < hi:
        mid = (lo + hi) // 2
        if xs[mid] < target: lo = mid + 1
        elif xs[mid] > target: hi = mid
        else: return mid
    return None
def search_bug2(xs, target, limit=100):          # lo = mid instead of mid + 1: it can stop shrinking
    lo, hi, n = 0, len(xs), 0
    while lo < hi:
        n += 1
        if n > limit: return "stuck"
        mid = (lo + hi) // 2
        if xs[mid] < target: lo = mid
        elif xs[mid] > target: hi = mid
        else: return mid
    return None

def check(fn):
    """Every sorted list of distinct values from 0, 2, 4, ... of length 0 to 6, every target."""
    cases = failures = 0; first = None
    for n in range(7):
        xs = list(range(0, 2 * n, 2))
        for target in range(-1, 2 * n + 1):
            got = fn(xs, target)
            got = got[0] if isinstance(got, tuple) else got
            want = xs.index(target) if target in xs else None
            cases += 1
            if got != want:
                failures += 1; first = first or (xs, target, want, got)
    return cases, failures, first
print()
for name, fn in (("correct", search), ("bug 1: hi = len - 1", search_bug1),
                 ("bug 2: lo = mid", search_bug2)):
    cases, failures, first = check(fn)
    detail = (f"; first: search({first[0]}, {first[1]}) gave {first[3]!r}, wanted {first[2]!r}"
              if first else "")
    print(f"{name:20} {cases} cases, {failures} wrong{detail}")
big = list(range(1_000_000))
worst = max(search(big, t)[1] for t in (0, 499_999, 999_999, -5))
print(f"\nsearching a million items takes at most {worst} steps")

# 3. for ... else: the else runs only if the loop wasn't left by break
def first_negative(nums):
    for i, x in enumerate(nums):
        if x < 0:
            break
    else:
        return "none"
    return f"index {i}"
print(f"first negative in [3, 1, -4, 1]: {first_negative([3, 1, -4, 1])}; "
      f"in [3, 1]: {first_negative([3, 1])}")

   after pass 1: sorted [2, 5] | still to do [9, 1, 7]
   after pass 2: sorted [2, 5, 9] | still to do [1, 7]
   after pass 3: sorted [1, 2, 5, 9] | still to do [7]
   after pass 4: sorted [1, 2, 5, 7, 9] | still to do []

correct              56 cases, 0 wrong
bug 1: hi = len - 1  56 cases, 6 wrong; first: search([0], 0) gave None, wanted 0
bug 2: lo = mid      56 cases, 27 wrong; first: search([0], 1) gave 'stuck', wanted None

searching a million items takes at most 20 steps
first negative in [3, 1, -4, 1]: index 2; in [3, 1]: none

import string

TEXT = """It was the best of times, it was the worst of times, it was the age of wisdom,
it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity."""

# 1. A problem broken into small functions, each with one job, input in and result out
def words(text):
    """Lower-case words, with punctuation stripped."""
    return [w.strip(string.punctuation) for w in text.lower().split()]
def count(items):
    counts = {}
    for item in items:
        counts[item] = counts.get(item, 0) + 1
    return counts
def top(counts, n, ignore=()):
    kept = [(word, c) for word, c in counts.items() if word not in ignore]
    return sorted(kept, key=lambda pair: (-pair[1], pair[0]))[:n]      # most common, then a-z
def report(pairs):
    return "\n".join(f"   {word:<8} {'#' * c} {c}" for word, c in pairs)
print(report(top(count(words(TEXT)), 5, ignore={"the", "of", "it", "was"})))

# 2. Pure functions return a result; impure ones change something. Know which you're calling.
nums = [3, 1, 2]
result = sorted(nums)                    # pure: returns a new list, leaves nums alone
print(f"\nsorted(nums) = {result}, nums still {nums}")
result = nums.sort()                     # impure: sorts nums in place, and returns None
print(f"nums.sort() returned {result}; nums is now {nums}")

# 3. A default value is evaluated once, when the def runs. A mutable default is shared.
def add_tag(tag, tags=[]):
    tags.append(tag)
    return tags
print(f"\nadd_tag('a') = {add_tag('a')}, then add_tag('b') = {add_tag('b')}  (the same list!)")
def add_tag_fixed(tag, tags=None):
    tags = [] if tags is None else tags
    tags.append(tag)
    return tags
print(f"fixed: {add_tag_fixed('a')}, then {add_tag_fixed('b')}")

# 4. The ways to pass arguments
def describe(name, /, age, *hobbies, city="nowhere", **extra):
    return f"{name}, {age}, hobbies {hobbies}, city {city}, extra {extra}"
print("\n" + describe("Ada", 36, "maths", "poetry", city="London", title="Countess"))
for bad in ("describe(name='Ada', age=36)", "describe('Ada')"):
    try: eval(bad)
    except TypeError as e: print(f"{bad} -> TypeError: {e}")

# 5. Functions are values: store them, pass them, return them
def make_scaler(factor):
    def scale(x): return x * factor
    return scale
double, triple = make_scaler(2), make_scaler(3)
ops = {"double": double, "triple": triple, "negate": lambda x: -x}
print(f"\n{[(name, op(7)) for name, op in ops.items()]}")
print(f"sorted by length, then a-z: {sorted(['pear', 'fig', 'banana', 'kiwi'], key=lambda w: (len(w), w))}")

   age      ## 2
   epoch    ## 2
   times    ## 2
   belief   # 1
   best     # 1

sorted(nums) = [1, 2, 3], nums still [3, 1, 2]
nums.sort() returned None; nums is now [1, 2, 3]

add_tag('a') = ['a'], then add_tag('b') = ['a', 'b']  (the same list!)
fixed: ['a'], then ['b']

Ada, 36, hobbies ('maths', 'poetry'), city London, extra {'title': 'Countess'}
describe(name='Ada', age=36) -> TypeError: describe() missing 1 required positional argument: 'name'
describe('Ada') -> TypeError: describe() missing 1 required positional argument: 'age'

[('double', 14), ('triple', 21), ('negate', -7)]
sorted by length, then a-z: ['fig', 'kiwi', 'pear', 'banana']

import sys
from functools import cache

# 1. Each call gets its own frame on the call stack. Watch the stack grow, then unwind.
def factorial(n, depth=0):
    pad = "   " * depth
    print(f"{pad}factorial({n}) called")
    if n <= 1:                                    # the base case: no further call
        print(f"{pad}factorial({n}) returns 1")
        return 1
    result = n * factorial(n - 1, depth + 1)      # the recursive case: a smaller problem
    print(f"{pad}factorial({n}) returns {n} * {result // n} = {result}")
    return result
factorial(4)

# 2. Recursive data invites recursive code: a list can contain lists
def total(item):
    if isinstance(item, list):
        return sum(total(x) for x in item)
    return item
def depth(item):
    if not isinstance(item, list): return 0
    return 1 + max((depth(x) for x in item), default=0)
nested = [1, [2, [3, 4], []], [[[5]]], 6]
print(f"\nnested = {nested}: total {total(nested)}, nesting depth {depth(nested)}")

# 3. The same subproblem solved again and again, unless you remember the answers
calls = 0
def fib(n):
    global calls; calls += 1
    return n if n < 2 else fib(n - 1) + fib(n - 2)
@cache
def fib_memo(n):
    global calls; calls += 1
    return n if n < 2 else fib_memo(n - 1) + fib_memo(n - 2)
for f in (fib, fib_memo):
    calls = 0
    print(f"{f.__name__}(25) = {f(25):,} after {calls:,} calls")

# 4. Every frame costs stack space, so Python stops runaway recursion
def sum_to(n):
    return 0 if n == 0 else n + sum_to(n - 1)
print(f"\nPython's recursion limit is {sys.getrecursionlimit()} frames")
for n in (500, 5000):
    try: print(f"sum_to({n}) = {sum_to(n):,}")
    except RecursionError: print(f"sum_to({n}): RecursionError, the stack would be {n:,} frames deep")

# 5. Any recursion can be turned into a loop with a stack you manage yourself
def flatten_recursive(item):
    if not isinstance(item, list): return [item]
    return [x for sub in item for x in flatten_recursive(sub)]
def flatten_iterative(item):
    out, stack = [], [item]
    while stack:
        top = stack.pop()
        if isinstance(top, list): stack.extend(reversed(top))   # reversed keeps left-to-right order
        else: out.append(top)
    return out
deep = 0
for _ in range(10_000): deep = [deep]                         # 10,000 lists, one inside the next
print(f"flatten(nested): recursive {flatten_recursive(nested)}, iterative {flatten_iterative(nested)}")
try: flatten_recursive(deep)
except RecursionError: print("10,000 levels deep: the recursive version overflows; "
                             f"the iterative one returns {flatten_iterative(deep)}")

# 6. Some problems are naturally recursive: the towers of Hanoi
def hanoi(n, source="A", spare="B", target="C"):
    if n == 0: return []
    return hanoi(n - 1, source, target, spare) + [(source, target)] + hanoi(n - 1, spare, source, target)
moves = hanoi(3)
print(f"\nHanoi with 3 discs: {len(moves)} moves: {' '.join(a + '>' + b for a, b in moves)}")
print("moves for 1 to 10 discs:", [len(hanoi(n)) for n in range(1, 11)])

factorial(4) called
   factorial(3) called
      factorial(2) called
         factorial(1) called
         factorial(1) returns 1
      factorial(2) returns 2 * 1 = 2
   factorial(3) returns 3 * 2 = 6
factorial(4) returns 4 * 6 = 24

nested = [1, [2, [3, 4], []], [[[5]]], 6]: total 21, nesting depth 4
fib(25) = 75,025 after 242,785 calls
fib_memo(25) = 75,025 after 26 calls

Python's recursion limit is 1000 frames
sum_to(500) = 125,250
sum_to(5000): RecursionError, the stack would be 5,000 frames deep
flatten(nested): recursive [1, 2, 3, 4, 5, 6], iterative [1, 2, 3, 4, 5, 6]
10,000 levels deep: the recursive version overflows; the iterative one returns [0]

Hanoi with 3 discs: 7 moves: A>C A>B C>B A>C B>A B>C A>C
moves for 1 to 10 discs: [1, 3, 7, 15, 31, 63, 127, 255, 511, 1023]

# 1. Lists are ordered and changeable; tuples are ordered and fixed
planets = ["Mercury", "Venus", "Earth", "Mars", "Jupiter"]
print(f"planets[0] = {planets[0]!r}, planets[-1] = {planets[-1]!r}, planets[1:3] = {planets[1:3]}")
print(f"planets[::2] = {planets[::2]}, planets[::-1][:2] = {planets[::-1][:2]}")
first, *middle, last = planets                    # unpacking, with the rest gathered up
print(f"first = {first!r}, middle = {middle}, last = {last!r}")
point = (3, 4)
try: point[0] = 5
except TypeError as e: print(f"point[0] = 5 -> TypeError: {e}")
distance = {(0, 0): "origin", point: "five away"}  # tuples can be dict keys; lists can't
try: distance[[1, 2]] = "nope"
except TypeError as e: print(f"a list as a key -> TypeError: {e}")

# 2. Why append is cheap and insert(0) is not: a list is an array that grows by reallocating
print()
class DynamicArray:
    """Counts the element copies a list does as it grows, for a given growth policy."""
    def __init__(self, grow):
        self.grow, self.capacity, self.size, self.copies, self.shifts = grow, 1, 0, 0, 0
    def append(self):
        if self.size == self.capacity:                 # full: allocate a bigger block, copy all
            self.copies += self.size
            self.capacity = self.grow(self.capacity)
        self.size += 1
    def insert_front(self):
        self.append()
        self.shifts += self.size - 1                   # every existing item moves up one place
for n in (1_000, 100_000):
    by_one, doubling = DynamicArray(lambda c: c + 1), DynamicArray(lambda c: 2 * c)
    for _ in range(n): by_one.append(); doubling.append()
    print(f"{n:>7,} appends: growing by 1 copies {by_one.copies:>13,} items; "
          f"doubling copies {doubling.copies:>7,}, {doubling.copies / n:.2f} per append")
front = DynamicArray(lambda c: 2 * c)
for _ in range(10_000): front.insert_front()
print(f" 10,000 inserts at the front shift {front.shifts:,} items, {front.shifts // 10_000:,} per insert")

# 3. 'in' on a list checks items one by one
comparisons = 0
class Counted:
    def __init__(self, v): self.v = v
    def __eq__(self, other):
        global comparisons; comparisons += 1
        return self.v == other.v
items = [Counted(i) for i in range(10_000)]
Counted(9_999) in items
print(f"\nfinding the last of 10,000 items with 'in': {comparisons:,} comparisons")

# 4. Sorting is stable: items that compare equal keep their original order
people = [("Ana", 31), ("Ben", 25), ("Cy", 31), ("Di", 25)]
by_age = sorted(people, key=lambda p: p[1])
print(f"\nsorted by age: {by_age}  (Ben before Di, Ana before Cy, as in the input)")
print(f"oldest first, then by name: {sorted(people, key=lambda p: (-p[1], p[0]))}")

# 5. A classic bug: changing a list while looping over it
nums = [2, 4, 5, 6, 8]
for x in nums:
    if x % 2 == 0: nums.remove(x)                  # the loop's position skips the next item
print(f"\nremoving evens while iterating leaves {nums}")
nums = [2, 4, 5, 6, 8]
nums = [x for x in nums if x % 2 != 0]             # build a new list instead
print(f"building a new list instead leaves {nums}")

planets[0] = 'Mercury', planets[-1] = 'Jupiter', planets[1:3] = ['Venus', 'Earth']
planets[::2] = ['Mercury', 'Earth', 'Jupiter'], planets[::-1][:2] = ['Jupiter', 'Mars']
first = 'Mercury', middle = ['Venus', 'Earth', 'Mars'], last = 'Jupiter'
point[0] = 5 -> TypeError: 'tuple' object does not support item assignment
a list as a key -> TypeError: cannot use 'list' as a dict key (unhashable type: 'list')

  1,000 appends: growing by 1 copies       499,500 items; doubling copies   1,023, 1.02 per append
100,000 appends: growing by 1 copies 4,999,950,000 items; doubling copies 131,071, 1.31 per append
 10,000 inserts at the front shift 49,995,000 items, 4,999 per insert

finding the last of 10,000 items with 'in': 10,000 comparisons

sorted by age: [('Ben', 25), ('Di', 25), ('Ana', 31), ('Cy', 31)]  (Ben before Di, Ana before Cy, as in the input)
oldest first, then by name: [('Ana', 31), ('Cy', 31), ('Ben', 25), ('Di', 25)]

removing evens while iterating leaves [4, 5, 8]
building a new list instead leaves [5]

from collections import Counter, defaultdict

# 1. A dict maps keys to values; a set holds unique keys. Both remember insertion order (dicts).
stock = {"apples": 12, "pears": 0}
stock["figs"] = 7
print(f"stock = {stock}; stock.get('kiwis', 0) = {stock.get('kiwis', 0)}")
try: stock["kiwis"]
except KeyError as e: print(f"stock['kiwis'] -> KeyError: {e}")
in_stock = {name for name, n in stock.items() if n > 0}
print(f"in stock: {sorted(in_stock)}; 'pears' in stock: {'pears' in stock} (the key exists, with 0)")

words = "the cat and the hat and the bat".split()
print(f"\nCounter: {Counter(words).most_common(2)}")
by_letter = defaultdict(list)
for w in words: by_letter[w[0]].append(w)
print(f"grouped by first letter: {dict(by_letter)}")
print(f"unique, in first-seen order: {list(dict.fromkeys(words))}")
a, b = set("listen"), set("silent")
print(f"set('listen') == set('silent'): {a == b}; "
      f"{{1, 2, 3}} & {{2, 3, 4}} = {({1, 2, 3} & {2, 3, 4})}, | = {({1, 2, 3} | {2, 3, 4})}, "
      f"- = {({1, 2, 3} - {2, 3, 4})}")

# 2. How a dict finds a key without searching: hash it to a slot, then probe a few slots
def weak_hash(s):                            # Python randomises str hashes per run; these are fixed
    h = 0
    for ch in s: h = (h * 31 + ord(ch)) % 2**32
    return h
def fnv_hash(s):                             # FNV-1a: mixes every character into every bit
    h = 2166136261
    for ch in s: h = ((h ^ ord(ch)) * 16777619) % 2**32
    return h
class HashTable:
    def __init__(self, hash_fn, size=8):
        self.hash, self.slots, self.count, self.probes = hash_fn, [None] * size, 0, 0
    def _find(self, key):
        i = self.hash(key) % len(self.slots)
        while True:
            self.probes += 1
            if self.slots[i] is None or self.slots[i][0] == key: return i
            i = (i + 1) % len(self.slots)    # occupied by another key: try the next slot
    def put(self, key, value):
        if (self.count + 1) * 3 > len(self.slots) * 2:      # keep it under two-thirds full
            old = [s for s in self.slots if s]
            self.slots, self.count = [None] * (len(self.slots) * 4), 0
            for k, v in old: self.put(k, v)
        i = self._find(key)
        if self.slots[i] is None: self.count += 1
        self.slots[i] = (key, value)
    def get(self, key):
        slot = self.slots[self._find(key)]
        return slot[1] if slot else None
names = [f"user{i}" for i in range(10_000)]
print()
for hash_fn in (weak_hash, fnv_hash):
    table = HashTable(hash_fn)
    for i, name in enumerate(names): table.put(name, i)
    table.probes = 0
    assert all(table.get(name) == i for i, name in enumerate(names))
    print(f"10,000 keys in {len(table.slots):,} slots with {hash_fn.__name__}: "
          f"{table.probes / len(names):.2f} probes per lookup")
print(f"the same lookups in a list would average {sum(range(1, 10_001)) / 10_000:,.1f} comparisons")

# 3. Keys must be hashable, and equal keys must hash equally
class Point:
    def __init__(self, x, y): self.x, self.y = x, y
    def __eq__(self, other): return (self.x, self.y) == (other.x, other.y)
try: {Point(1, 2): "a"}
except TypeError as e: print(f"\nPoint with __eq__ but no __hash__ -> TypeError: {e}")
class BadPoint(Point):
    def __hash__(self): return id(self)      # breaks the rule: equal points, different hashes
places = {BadPoint(1, 2): "home"}
print(f"BadPoint(1, 2) in places: {BadPoint(1, 2) in places}  (it's equal, but hashed elsewhere)")
class GoodPoint(Point):
    def __hash__(self): return hash((self.x, self.y))
places = {GoodPoint(1, 2): "home"}
print(f"GoodPoint(1, 2) in places: {GoodPoint(1, 2) in places}")

stock = {'apples': 12, 'pears': 0, 'figs': 7}; stock.get('kiwis', 0) = 0
stock['kiwis'] -> KeyError: 'kiwis'
in stock: ['apples', 'figs']; 'pears' in stock: True (the key exists, with 0)

Counter: [('the', 3), ('and', 2)]
grouped by first letter: {'t': ['the', 'the', 'the'], 'c': ['cat'], 'a': ['and', 'and'], 'h': ['hat'], 'b': ['bat']}
unique, in first-seen order: ['the', 'cat', 'and', 'hat', 'bat']
set('listen') == set('silent'): True; {1, 2, 3} & {2, 3, 4} = {2, 3}, | = {1, 2, 3, 4}, - = {1}

10,000 keys in 32,768 slots with weak_hash: 8.12 probes per lookup
10,000 keys in 32,768 slots with fnv_hash: 1.18 probes per lookup
the same lookups in a list would average 5,000.5 comparisons

Point with __eq__ but no __hash__ -> TypeError: cannot use 'Point' as a dict key (unhashable type: 'Point')
BadPoint(1, 2) in places: False  (it's equal, but hashed elsewhere)
GoodPoint(1, 2) in places: True