From c9c90920cc35d0dffa227935dbd449dc3ec33d7c Mon Sep 17 00:00:00 2001 From: adamsjdavid Date: Sat, 12 Sep 2026 18:29:13 -0400 Subject: [PATCH] v1.1.0: headerless mode for audits printed without row/column headings Sniffs the format on page 1: printed row numbers -> gridded mode (unchanged); UUID-keyed rows -> new headerless mode, which anchors rows on per-record UUIDs and takes the column map from the page-1 header labels. Handles fused cells (ID+Name+Org+networks), mid-value line wraps, two-timestamp Time Frame ranges, wrapped header labels, and per-export schema drift. Validated against three monthly JC PD audits: 120/120 cross-engine spot checks, 0 anomalies, field-identical to the standalone converter it replaces. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012bQ28wRBiQYcSR9FzV9U7c --- README.md | 135 +++++++++++++++------------ VERIFICATION.md | 35 +++++++ pinion/__init__.py | 11 ++- pinion/cli.py | 61 +++++++++++- pinion/headerless.py | 215 +++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 6 files changed, 394 insertions(+), 65 deletions(-) create mode 100644 pinion/headerless.py diff --git a/README.md b/README.md index a8f75bb..5b6df62 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # pinion-pdf +*Those who would sacrifice essential Balls to preserve a little Ligma deserve Deez Nuts.* — Dave Adams + Someone took a spreadsheet, hit "Print", and handed out the PDF. pinion turns that PDF back into a CSV — and then proves the conversion is right. @@ -12,7 +14,7 @@ pip install pymupdf python3 -m pinion input.pdf output.csv ``` -Or install it properly and get a `pinion` command: +Or install it and get a `pinion` command: ``` pip install . @@ -24,59 +26,79 @@ the first 50 batches first. ## Why you can't just use a normal PDF tool -The obvious tools (pdftotext, pdfplumber, Camelot, Tabula) fail on printed -spreadsheets, for three separate reasons. +The obvious tools (pdftotext, pdfplumber, Camelot, Tabula) have some common +trappings. -**Trap 1 — text spills over the column next door.** Some columns are too -narrow for what's in them, so the text is drawn straight over the top of -the neighbouring cell. You, being a human with a brain, can see the issue; -a position-based sort cannot. pdfplumber genuinely returns this for one -cell: +### Trap 1 — Text Spillover +Some columns can be too narrow for what's in them, and text spills into neighbors. +You, being a human with a brain, can see the issue - OCR or position-based sort cannot. -``` -'l,o 0o5k:u5p6 :-0 M0 oAbMil eUTC' -``` +Something like `pdfplumber` will interleave spillover text, giving you unrecoverable nonsense. -That is `lookup - Mobile` and `05:56:00 AM UTC` interleaved letter by -letter — unrecoverable once mixed. The fix: place text by where it -**starts**, never by where it sits. A cell's text always begins at its own -column's left edge, even when the tail spills past the far edge. +Our approach measure start points and then uses the PDF's embedded information about write +order to trace the text. -**Trap 2 — blacked-out cells are invisible to text extraction.** A -redacted cell has a black rectangle drawn over it. Rectangles are graphics, -not text, so every text extractor reports the cell as simply empty — and -you can no longer tell "nobody filled this in" from "somebody removed -this". pinion finds the rectangles itself and emits a `reason_redacted` -TRUE/FALSE column plus the bar's width in points (a rough proxy for how -much text was removed). It also checks whether any text survives *under* -each bar — if a redaction is cosmetic rather than destructive, it says so -loudly. +### Trap 2 - Redactions Get Lost +Blacked out cells (which are graphics) can be invisible to text extractors. -**Trap 3 — counting lines gives the wrong total.** Every page carries an -"A B C D" strip labelling the columns, which looks exactly like a row of -data to a line-counting script. Count lines naively and you invent one -phantom row per page. +This might conflate 'redacted' with 'empty', but we generally want to preserve this context. + +Our approach searches for redaction boxes, tracks their size, and adds some extra csv fields +in case this information is relevant later. It also picks up on bad redactions +(text survival under the boxes). + + +### Trap 3 - Line Counts are Janky + +Every page in the typical export format carries an "A B C D ..." strip labeling columns. + +To a eegular old parser, that's a row...but it's obviously not. + +We avoid phantom rows by stripping those tracking rows from the output. + + +## Two print formats + +pinion sniffs page 1 and picks the right strategy on its own: + +**Gridded** — Excel printed with row/column headings on. Printed row +numbers and the letter strip anchor everything; this is the mode described +below. + +**Headerless** — the same audits printed via "Microsoft Print to PDF" with +headings off. No gridlines, no row numbers, adjacent cells fused into +single text runs, cells wrapping mid-value, and a schema that drifts +between exports (one month has a License Plate column, the next doesn't). +Here pinion anchors rows on the UUID that starts every record and takes +the column map from the file's own page-1 header labels. Columns absent +from a given export are reported loudly and left empty — never guessed. + +If a file is neither, pinion says so and stops. ## How the pages are laid out -A wide spreadsheet prints in vertical slices: page 1 holds columns A–D for -rows 1–32, page 2 holds E–F for those same rows, and so on until the -letters start over at A and the next 32 rows begin. One set of those pages -is a "batch"; pinion works out the batch size itself by watching for the -letter strip to restart. +A wide spreadsheet prints in vertical slices. In the original data used to build this tool, +page 1 held columns A-D for 32 rows, page 2 held E-F, etc. -Rebuilding one spreadsheet row therefore means stitching together pieces -from several pages — which is why the printed row numbers in the left -margin matter so much. pinion anchors every row to Excel's own numbering -and verifies the chain runs 1..N with no gaps or repeats. +For nomenclature, one set of those pages make a 'batch'. The pattern then repeats. -## Checking its own working +Rebuilding a row naturally requires stiching together several pages. + +We track our calculated rows against the reported row numbers to make sure we haven't +lost the plot as we process things. Any misses or phantoms -> explosion. + +## How We Verify Accuracy + +Every run contains internal consistency checks to make sure things haven't gone off the rails. + +We track and log any row number deviation, unplaceable text, unplaceable redaction bars, +and any time we made an assumption regarding a row number, etc because of a redaction bar overlap. + +If a layout assumption fails during processing, we stop and yell about it. -pinion never hands you plausible-looking output without evidence. Every run -ends with: ``` -CHECKING OUR OWN WORKING +OUTPUT VALIDATION -------------------------------------------------------------- rows written ............... 1,048,575 rows expected .............. 1,048,575 ok @@ -90,7 +112,7 @@ CHECKING OUR OWN WORKING blackout bars found ........ 2,714 rows affected .............. 1,549 which columns .............. {'G': 2702, 'H': 3, 'I': 1, '?': 8} - text readable under bars ... none (deleted, not merely covered) + text readable under bars ... none ``` If a layout assumption fails, pinion stops with a specific complaint @@ -98,25 +120,22 @@ instead of guessing. Wrong-but-plausible output is the one failure mode this tool must never have. See [VERIFICATION.md](VERIFICATION.md) for how the reference conversion was validated. -## A warning about truncated data +## Data Truncation Warning -Excel tops out at 1,048,576 rows; anything past that was thrown away -before the PDF existed. pinion warns when a sheet lands exactly on that -ceiling — data that stops precisely at the maximum was almost certainly -cut off there, so every total from such a file is a floor, not a full -count. We can warn about it, but we can't recover data that isn't in the -PDF. +Excel tops out at 1,048,576 rows. -## Layout of the code +If you get a result with 1,048,575 rows + a header row, you must either immediately purchase +a lottery ticket or consider the possibility of data truncation. + +We issue a warning, but the truncated data is an issue in the source PDF. + +## General Code Layout ``` -pinion/page.py — reading what the PDF drew on a page -pinion/measure.py — measuring the layout (one function per fact) -pinion/convert.py — rebuilding the rows -pinion/cli.py — run it, then check our own working +pinion/page.py — Read the PDF page layout +pinion/measure.py — Measure the gridded layout +pinion/convert.py — Rebuilt rows using our measurements +pinion/headerless.py — The no-headings variant (UUID + header-label anchors) +pinion/cli.py — Format sniffing, execution harness + inline sanity checks ``` ---- - -*Those who would sacrifice essential Ligma to preserve a little Balls -deserve Deez Nuts.* — Dave Adams diff --git a/VERIFICATION.md b/VERIFICATION.md index 0c9af23..effad8d 100644 --- a/VERIFICATION.md +++ b/VERIFICATION.md @@ -61,3 +61,38 @@ diffing against known-good output or by a check line going red: absent: the export was truncated at the ceiling, and roughly 17% of the original records are estimated to be missing. All totals from the file are floors. + +# Headerless mode (v1.1.0) + +Validated against three monthly Johnson City TN PD audit PDFs printed via +"Microsoft Print to PDF" without headings (48/74/69 pages; 2,032 / 2,437 / +2,607 rows). + +Checks passed on all three: + +1. **Cross-engine spot check.** 40 random rows per file re-verified + against pdftotext: 120/120 after making the probes wrap-tolerant (a + cell that wraps mid-value in the PDF never appears as one contiguous + string in any text extractor's output — probe fragments, not lines). +2. **Range sanity.** Every two-timestamp Time Frame parses and runs + start <= end (7,074 ranges, 0 disordered). +3. **Anomaly counters at zero**: no bad UUIDs, no unplaced runs, no + strays in the Name/Org windows. + +Bugs the checks caught during development: + +- **Wrapped header labels** ('Total Networks ' / 'Searched' on stacked + runs) made a whole column invisible to name-matching; its values landed + in the Org Name window and were silently overwritten. Fixed by merging + stacked header runs; guarded by the strays counter. +- **A false-alarm verification failure**: 43 spot-checks "failed" because + the probe assumed cells appear contiguously in pdftotext output. Glyph + level inspection proved the CSV was right and the check was naive. + Kept here as a reminder: verify the verifier before trusting a red + result. + +Schema drift is real: of the three sibling exports, one has all 13 +columns, two lack License Plate entirely, and Total Networks Searched +moves between its own column and being fused into the ID run. Headerless +mode takes each file's schema from its own header and reports what's +absent. diff --git a/pinion/__init__.py b/pinion/__init__.py index 9ae007e..ced9ec4 100644 --- a/pinion/__init__.py +++ b/pinion/__init__.py @@ -5,10 +5,11 @@ measures the PDF before it reads it -- page size, column widths, column and row counts are all derived from the file -- rebuilds every row, detects blacked-out cells, and then checks its own working. - MAP: page.py - reading what the PDF drew on a page - measure.py - measuring the layout - convert.py - rebuilding the rows - cli.py - run it, then check our own working + MAP: page.py - reading what the PDF drew on a page + measure.py - measuring the gridded layout + convert.py - rebuilding gridded rows + headerless.py - the no-headings variant (UUID + header anchors) + cli.py - sniff the format, run it, check our own working """ -__version__ = "1.0.0" +__version__ = "1.1.0" diff --git a/pinion/cli.py b/pinion/cli.py index fed2db2..328f54d 100644 --- a/pinion/cli.py +++ b/pinion/cli.py @@ -7,8 +7,10 @@ try: except ImportError: sys.exit("Missing dependency. Run: pip install pymupdf") +from .page import all_runs, run_start, run_text from .measure import measure from .convert import convert +from .headerless import looks_headerless, convert_headerless def report(label, value, ok=None, bad=""): @@ -24,9 +26,30 @@ def main(): ap.add_argument("pdf") ap.add_argument("out_csv") ap.add_argument("--limit", type=int, default=0, - help="only convert the first N batches, for a trial run") + help="only convert the first N batches (gridded) or " + "pages (headerless), for a trial run") args = ap.parse_args() + # Which of the two known print formats is this? A gridded Excel print + # puts its row numbers leftmost on the page; the headerless audit + # variant has no headings at all but keys every row with a UUID. + doc = pymupdf.open(args.pdf) + runs0 = all_runs(doc[0]) + if not runs0: + sys.exit("Page 1 has no text at all. Is this a scan?") + leftmost = min(runs0, key=lambda r: run_start(r)[0]) + if run_text(leftmost).strip().isdigit(): + doc.close() + run_gridded(args) + elif looks_headerless(doc[0]): + run_headerless(doc, args) + else: + sys.exit("Unrecognised layout: no printed row numbers (gridded " + "Excel print) and no UUID-keyed rows (headerless audit " + "print). pinion doesn't know this format yet.") + + +def run_gridded(args): M = measure(args.pdf) n_batches = min(args.limit, M["n_batches"]) if args.limit else M["n_batches"] @@ -76,3 +99,39 @@ def main(): print("\n WARNING: exactly 1,048,576 rows. That's an Excel magic number") print(" The source was almost certainly truncated.") print(" Go tell the source they owe you more data.") + + +def run_headerless(doc, args): + n_pages = min(args.limit, doc.page_count) if args.limit else doc.page_count + print("HEADERLESS AUDIT PRINT (no row/column headings; anchoring on") + print("the page-1 header labels and per-row UUIDs)") + print("-" * 62) + with open(args.out_csv, "w", newline="", encoding="utf-8") as fh: + r = convert_headerless(doc, csv.writer(fh), n_pages) + doc.close() + s = r["stats"] + + print(f"\nWritten to {args.out_csv}\n") + print("-" * 62) + report("rows written", f"{r['n_rows']:,}") + report("columns in this export", f"{len(r['names'])}/13", + not r["missing"], + f"absent from the PDF itself: {r['missing']}") + report("org name (derived)", repr(r["org"])) + report("time-frame ranges", + f"{r['tf_ok']:,} valid pairs, {r['tf_single']} single", + r["tf_bad"] == 0, f"{r['tf_bad']} MALFORMED") + report("rows with a bad uuid", s["rows_bad_uuid"], + s["rows_bad_uuid"] == 0, "ROWS DROPPED") + report("unplaced runs", s["runs_no_column"] + s["runs_below_last_anchor"], + s["runs_no_column"] + s["runs_below_last_anchor"] == 0, + "SOME TEXT LOST") + report("strays in name/org windows", s["strays_in_name_org_windows"], + s["strays_in_name_org_windows"] == 0, "COLUMN MAPPED WRONG") + report("fused SearchTime split", s["searchtime_splits"]) + report("blackout bars found", len(r["bars"])) + if r["missing"]: + print(f"\n NOTE: this export has no {', '.join(r['missing'])} " + "column(s).") + print(" That data was never in the PDF; the CSV column is left " + "empty.") diff --git a/pinion/headerless.py b/pinion/headerless.py new file mode 100644 index 0000000..264e442 --- /dev/null +++ b/pinion/headerless.py @@ -0,0 +1,215 @@ +"""The headerless variant: audit exports printed WITHOUT row/column headings. + +Some Flock audit PDFs come out of "Microsoft Print to PDF" with Excel's +headings switched off. Everything the gridded mode anchors on is missing, +and this format brings its own traps on top: + + - No gridlines and no printed row numbers, so rows are anchored on the + one structural fact left: every row starts with a UUID in the ID + column. Column edges come from the page-1 header labels' x-positions. + - Adjacent cells FUSE into one text run when columns are tight: ID + + Name + Org Name (+ sometimes the networks count) arrive as a single + string and must be split apart again. Search Time occasionally fuses + with Search Type; the fixed 27-character timestamp gives the split + point. + - Long header labels wrap onto stacked runs ('Total Networks ' over + 'Searched'), so header runs sharing an x are merged before matching + names. Miss this and a whole column silently lands in the wrong + window -- it happened; the strays counter now guards it. + - Cells wrap mid-value ('05/05/2026, 04:12:32 ' + 'AM UTC'), and the + wrap point moves between exports. Fragments are rejoined by position. + - Time Frame is TWO stacked timestamps -- the full start/end range -- + joined here as 'start - end'. + - The schema itself drifts between exports (License Plate present one + month, gone the next), so columns are taken from each file's own + header, and missing ones are reported, not guessed. +""" + +import re +import sys +import collections +import statistics +from bisect import bisect_left + +from .page import all_runs, run_text, solid_regions + +CANON = ["ID", "Name", "Org Name", "Total Networks Searched", "Time Frame", + "License Plate", "Reason", "Case #", "Filters", "Search Time", + "Search Type", "Text Prompt", "Moderation"] +UUID = re.compile(r'^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}') +TS = re.compile(r'^\d{2}/\d{2}/\d{4}, \d{2}:\d{2}:\d{2} [AP]M UTC$') +TS_PREFIX = re.compile(r'^(\d{2}/\d{2}/\d{4}, \d{2}:\d{2}:\d{2} [AP]M UTC)(.+)$') + + +def looks_headerless(page0): + """Does page 1 carry UUID-keyed rows? That's this variant's signature.""" + return any(UUID.match(run_text(r)) for r in all_runs(page0)) + + +def parse_header(page0_runs, header_cut): + """Column names and x-positions from the page-1 header band. + + A long label wraps onto stacked runs at the same x, so runs sharing an + x are merged (in y-order) before searching for the canonical names. + Char positions are kept so names inside a fused label still yield + their own x. + """ + hdr = [r for r in page0_runs if r["chars"][0][2][1] < header_cut] + by_x = collections.defaultdict(list) + for r in hdr: + x0 = r["chars"][0][2][0] + key = next((k for k in by_x if abs(k - x0) < 1.2), x0) + by_x[key].append(r) + + col_x = {} + for x0, rs in by_x.items(): + rs.sort(key=lambda r: r["chars"][0][2][1]) + text = " ".join(run_text(r).strip() for r in rs) + cmap, pos = {}, 0 + for r in rs: + raw = run_text(r) + lead = len(raw) - len(raw.lstrip()) + for i in range(len(raw.strip())): + cmap[pos + i] = r["chars"][lead + i] + pos += len(raw.strip()) + 1 + for name in CANON: + i = text.find(name) + if i >= 0 and name not in col_x and i in cmap: + col_x[name] = cmap[i][2][0] + + ordered = sorted((x, n) for n, x in col_x.items()) + names = [n for _, n in ordered] + edges = [x for x, _ in ordered] + [1e9] + missing = [n for n in CANON if n not in col_x] + return names, edges, missing + + +def convert_headerless(doc, out, n_pages): + """Rebuild every row and write it to `out` (a csv.writer). Returns a + stats dict for the checks at the end.""" + p1 = all_runs(doc[0]) + anchors1 = sorted(r["chars"][0][2][1] for r in p1 if UUID.match(run_text(r))) + header_cut = anchors1[0] - 6.0 + names, edges, missing = parse_header(p1, header_cut) + + def col_at(x): + i = bisect_left(edges, x + 2.5) - 1 + return names[i] if 0 <= i < len(names) else None + + stats = collections.Counter() + stats["columns_missing"] = 0 # ensure keys exist even when zero + rows, gaps, bars = [], [], [] + for pno in range(n_pages): + runs = all_runs(doc[pno]) + anchors = sorted((r["chars"][0][2][1], r) for r in runs + if UUID.match(run_text(r))) + if not anchors: + stats["pages_without_rows"] += 1 + continue + ys = [a[0] for a in anchors] + gaps += [b - a for a, b in zip(ys, ys[1:])] + page_rows = [collections.defaultdict(list) for _ in anchors] + + # place every text run in its row (nearest anchor at or below it) + # and its column (by where the run STARTS) + for r in runs: + x, y = r["chars"][0][2] + if pno == 0 and y < header_cut: + continue + i = bisect_left(ys, y - 1.0) + if i == len(ys): + i -= 1 + if y > ys[-1] + 2.0: + stats["runs_below_last_anchor"] += 1 + c = col_at(x) + if c is None: + stats["runs_no_column"] += 1 + continue + page_rows[i][c].append((y, x, run_text(r))) + + # blackout bars, same shape test as the gridded mode + pitch = statistics.median(gaps) if gaps else 12.0 + for box in solid_regions(doc[pno]): + if box.width <= 1.2 or box.height <= pitch * 0.5: + continue + yc = (box.y0 + box.y1) / 2 + i = min(bisect_left(ys, yc - 1.0), len(ys) - 1) + bars.append((pno, i, round(box.width, 1))) + page_rows[i]["_bar"] = max(page_rows[i].get("_bar") or 0, + round(box.width, 1)) + rows.extend(page_rows) + + # --- assemble cells from fragments --- + def join(parts): + parts = [p[2].strip() for p in sorted(parts)] + parts = [p for p in parts if p] + if len(parts) == 2 and all(TS.match(p) for p in parts): + return " - ".join(parts) # Time Frame start/end range + return re.sub(r'\s+', ' ', " ".join(parts)).strip() + + out_rows, middles = [], [] + for row in rows: + bar = row.pop("_bar", None) + # Name/Org come only from splitting the fused ID run; anything + # assigned to them directly means a column landed in the wrong + # window and would be silently clobbered below. + if row.get("Name") or row.get("Org Name"): + stats["strays_in_name_org_windows"] += 1 + cell = {c: join(v) for c, v in row.items()} + + fused = cell.get("ID", "") + m = UUID.match(fused) + if not m: + stats["rows_bad_uuid"] += 1 + continue + rest = fused[36:] + digits = re.search(r'(\d+)$', rest) + if digits and not cell.get("Total Networks Searched"): + cell["Total Networks Searched"] = digits.group(1) + rest = rest[:digits.start()] + cell["ID"] = m.group(0) + cell["_rest"] = rest + middles.append(rest) + + st = cell.get("Search Time", "") + pm = TS_PREFIX.match(st) + if pm: + cell["Search Time"] = pm.group(1) + cell["Search Type"] = (pm.group(2) + " " + + cell.get("Search Type", "")).strip() + stats["searchtime_splits"] += 1 + cell["_bar"] = bar + out_rows.append(cell) + + # Name / Org split: the org is the longest common suffix of every + # fused middle -- constant per file, while names vary + org = middles[0] if middles else "" + for s in middles[1:]: + while org and not s.endswith(org): + org = org[1:] + if len(org) < 4: + sys.exit(f"Could not derive a constant Org Name (got {org!r}); " + "refusing to split names by guesswork.") + for cell in out_rows: + cell["Name"] = cell.pop("_rest")[:-len(org)].strip() + cell["Org Name"] = org + + out.writerow(CANON + ["reason_redacted", "redaction_bar_pt"]) + for cell in out_rows: + bar = cell.pop("_bar") + out.writerow([cell.get(c, "") for c in CANON] + + ["TRUE" if bar else "FALSE", bar or ""]) + + tf_ok = tf_single = tf_bad = 0 + for cell in out_rows: + tf = cell.get("Time Frame", "") + if " - " in tf: + a, b = tf.split(" - ", 1) + tf_ok += bool(TS.match(a) and TS.match(b)) + tf_bad += not (TS.match(a) and TS.match(b)) + elif tf: + tf_single += 1 + stats["columns_missing"] = len(missing) + return {"n_rows": len(out_rows), "org": org, "names": names, + "missing": missing, "tf_ok": tf_ok, "tf_single": tf_single, + "tf_bad": tf_bad, "bars": bars, "stats": stats} diff --git a/pyproject.toml b/pyproject.toml index 0e6cd54..2a8ed7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pinion-pdf" -version = "1.0.0" +version = "1.1.0" description = "Turn a spreadsheet that was printed to PDF back into a CSV, and prove the conversion is right." readme = "README.md" requires-python = ">=3.9"