commit 74b06be742fb22b86b446c13ca1c28743affbb87 Author: adamsjdavid Date: Mon Sep 7 00:34:28 2026 -0400 pinion-pdf 1.0.0: rebuild a CSV from a spreadsheet printed to PDF Derives page layout from the file itself (batch size, column edges, row grid, heading font), places text by start-x to survive cell overflow, detects blackout redactions including filled-path bars and single-character bars, and self-verifies via Excel's printed row-number chain. Output byte-identical to four independently written implementations across a 1,048,576-row reference file. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_012bQ28wRBiQYcSR9FzV9U7c diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..457490f --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.pyc +*.egg-info/ +dist/ +build/ +.venv/ +*.csv diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..fbdd49c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 David Adams + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..a8f75bb --- /dev/null +++ b/README.md @@ -0,0 +1,122 @@ +# pinion-pdf + +Someone took a spreadsheet, hit "Print", and handed out the PDF. pinion +turns that PDF back into a CSV — and then proves the conversion is right. + +It measures the PDF before it reads it, so page size, column widths, and +column/row counts are all derived from the file itself. Nothing about any +particular document is hardcoded. + +``` +pip install pymupdf +python3 -m pinion input.pdf output.csv +``` + +Or install it properly and get a `pinion` command: + +``` +pip install . +pinion input.pdf output.csv +``` + +A 131,072-page file takes about 5 minutes. Add `--limit 50` to trial-run +the first 50 batches first. + +## Why you can't just use a normal PDF tool + +The obvious tools (pdftotext, pdfplumber, Camelot, Tabula) fail on printed +spreadsheets, for three separate reasons. + +**Trap 1 — text spills over the column next door.** Some columns are too +narrow for what's in them, so the text is drawn straight over the top of +the neighbouring cell. You, being a human with a brain, can see the issue; +a position-based sort cannot. pdfplumber genuinely returns this for one +cell: + +``` +'l,o 0o5k:u5p6 :-0 M0 oAbMil eUTC' +``` + +That is `lookup - Mobile` and `05:56:00 AM UTC` interleaved letter by +letter — unrecoverable once mixed. The fix: place text by where it +**starts**, never by where it sits. A cell's text always begins at its own +column's left edge, even when the tail spills past the far edge. + +**Trap 2 — blacked-out cells are invisible to text extraction.** A +redacted cell has a black rectangle drawn over it. Rectangles are graphics, +not text, so every text extractor reports the cell as simply empty — and +you can no longer tell "nobody filled this in" from "somebody removed +this". pinion finds the rectangles itself and emits a `reason_redacted` +TRUE/FALSE column plus the bar's width in points (a rough proxy for how +much text was removed). It also checks whether any text survives *under* +each bar — if a redaction is cosmetic rather than destructive, it says so +loudly. + +**Trap 3 — counting lines gives the wrong total.** Every page carries an +"A B C D" strip labelling the columns, which looks exactly like a row of +data to a line-counting script. Count lines naively and you invent one +phantom row per page. + +## How the pages are laid out + +A wide spreadsheet prints in vertical slices: page 1 holds columns A–D for +rows 1–32, page 2 holds E–F for those same rows, and so on until the +letters start over at A and the next 32 rows begin. One set of those pages +is a "batch"; pinion works out the batch size itself by watching for the +letter strip to restart. + +Rebuilding one spreadsheet row therefore means stitching together pieces +from several pages — which is why the printed row numbers in the left +margin matter so much. pinion anchors every row to Excel's own numbering +and verifies the chain runs 1..N with no gaps or repeats. + +## Checking its own working + +pinion never hands you plausible-looking output without evidence. Every run +ends with: + +``` +CHECKING OUR OWN WORKING +-------------------------------------------------------------- + rows written ............... 1,048,575 + rows expected .............. 1,048,575 ok + row numbers 1..N unbroken .. yes ok + text we could not place .... 0 by column, 0 by row ok + bars we could not place .... 0 ok + printed numbers disagreed .. 0 ok + row numbers hidden by bars . 8 (placed by spacing instead) + cells drawn in pieces ...... 191 (rejoined; nothing lost) + + blackout bars found ........ 2,714 + rows affected .............. 1,549 + which columns .............. {'G': 2702, 'H': 3, 'I': 1, '?': 8} + text readable under bars ... none (deleted, not merely covered) +``` + +If a layout assumption fails, pinion stops with a specific complaint +instead of guessing. Wrong-but-plausible output is the one failure mode +this tool must never have. See [VERIFICATION.md](VERIFICATION.md) for how +the reference conversion was validated. + +## A warning about truncated data + +Excel tops out at 1,048,576 rows; anything past that was thrown away +before the PDF existed. pinion warns when a sheet lands exactly on that +ceiling — data that stops precisely at the maximum was almost certainly +cut off there, so every total from such a file is a floor, not a full +count. We can warn about it, but we can't recover data that isn't in the +PDF. + +## Layout of the code + +``` +pinion/page.py — reading what the PDF drew on a page +pinion/measure.py — measuring the layout (one function per fact) +pinion/convert.py — rebuilding the rows +pinion/cli.py — run it, then check our own working +``` + +--- + +*Those who would sacrifice essential Ligma to preserve a little Balls +deserve Deez Nuts.* — Dave Adams diff --git a/VERIFICATION.md b/VERIFICATION.md new file mode 100644 index 0000000..0c9af23 --- /dev/null +++ b/VERIFICATION.md @@ -0,0 +1,63 @@ +# How the reference conversion was validated + +pinion was developed against a real file: a 131,072-page, 360 MB PDF of a +law-enforcement network-audit spreadsheet (1,048,576 rows × 14 columns, +printed from Excel via PDF-XChange on Windows). Every design decision below +was forced by something that actually went wrong on that file. + +## Independent checks the output passed + +1. **Row-number chain.** Excel prints its own row numbers in the left + margin. The reconstruction reproduces 1..1,048,576 with no gaps and no + duplicates. Any method that drops or invents rows fails this + immediately — a competing hand count of the same file reported + 1,081,343 rows, which is exactly one phantom row per page (Trap 3) and + also exceeds Excel's hard row limit, so it can be rejected on + arithmetic alone. + +2. **Cross-engine spot check.** 40 randomly sampled rows were re-extracted + with pdftotext (Poppler — a completely independent PDF engine) and + matched on ID, agency, and printed row number: 40/40. + +3. **Implementation-independent reproduction.** Four separately written + implementations (a hardcoded-geometry minimal script, a heavily + commented teaching version, a parallel self-verifying version, and this + package) produce **byte-identical** CSVs across all 1,048,575 data + rows, verified with `cmp`. + +4. **Internal consistency.** Every value in the ID column is a + well-formed UUID (1,048,575 of 1,048,575); every row has exactly the + same field count; zero text runs failed to map to a row or column. + +## Bugs the checks caught during development + +These are the reasons the self-checks exist. Each was caught only by +diffing against known-good output or by a check line going red: + +- **Single-character redactions are ~3.4pt wide** — narrower than some + grid marks. A size-based bar filter dropped 5 of them. Fix: filter by + shape (taller than half a row, thicker than a hairline), not by area. +- **Some blackout bars are filled four-sided paths, not rectangles.** + Visually identical, different PDF instructions. Inspecting individual + line segments missed 4 bars, including one 121pt wide. Fix: measure + whole filled regions, never segments. +- **Two redaction bars cover the printed row number itself**, making the + gutter unreadable on those pages. Anchoring rows solely on printed + numbers silently dropped both rows. Fix: anchor on geometry, use the + printed numbers as a cross-check (`printed numbers disagreed` must be 0). +- **A one-character join bug** (`" ".join` vs `"".join`) corrupted the 191 + cells whose text the PDF drew in multiple pieces. Caught by byte-diff. + +## Known artifacts in the source data (not conversion errors) + +- 19 rows contain an unmapped `ff` ligature (the subset font has no + Unicode mapping for it): `Sex O�enses`, `Tra�ic`. Exact-match filters on + those strings will miss these rows. +- 141 rows contain double-encoded UTF-8 from the source system + (`100’s`, `–`). Present in the PDF itself. +- The sheet ends at exactly 1,048,576 rows — Excel's ceiling — with the + final row fully populated. Comparison of per-day record counts against + the file's claimed date range shows several end-of-range days thinned or + absent: the export was truncated at the ceiling, and roughly 17% of the + original records are estimated to be missing. All totals from the file + are floors. diff --git a/pinion/__init__.py b/pinion/__init__.py new file mode 100644 index 0000000..9ae007e --- /dev/null +++ b/pinion/__init__.py @@ -0,0 +1,14 @@ +"""pinion-pdf: turn a spreadsheet that was printed to PDF back into a CSV. + +Someone took a spreadsheet, hit "Print", and handed out the PDF. pinion +measures the PDF before it reads it -- page size, column widths, column and +row counts are all derived from the file -- rebuilds every row, detects +blacked-out cells, and then checks its own working. + + MAP: page.py - reading what the PDF drew on a page + measure.py - measuring the layout + convert.py - rebuilding the rows + cli.py - run it, then check our own working +""" + +__version__ = "1.0.0" diff --git a/pinion/__main__.py b/pinion/__main__.py new file mode 100644 index 0000000..4e28416 --- /dev/null +++ b/pinion/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +main() diff --git a/pinion/cli.py b/pinion/cli.py new file mode 100644 index 0000000..fed2db2 --- /dev/null +++ b/pinion/cli.py @@ -0,0 +1,78 @@ +import sys +import csv +import argparse + +try: + import pymupdf # noqa: F401 (probed here for a friendly error) +except ImportError: + sys.exit("Missing dependency. Run: pip install pymupdf") + +from .measure import measure +from .convert import convert + + +def report(label, value, ok=None, bad=""): + """One line of the results table. ok=None means purely informational.""" + tail = "" if ok is None else (" ok" if ok else f" <-- {bad}") + print(f" {label + ' ':.<28} {value}{tail}") + + +def main(): + ap = argparse.ArgumentParser( + prog="pinion", + description="Rebuild a CSV from a spreadsheet that was printed to PDF.") + ap.add_argument("pdf") + ap.add_argument("out_csv") + ap.add_argument("--limit", type=int, default=0, + help="only convert the first N batches, for a trial run") + args = ap.parse_args() + + M = measure(args.pdf) + n_batches = min(args.limit, M["n_batches"]) if args.limit else M["n_batches"] + + print("CONVERTING (takes a few minutes)") + print("-" * 62) + doc = pymupdf.open(args.pdf) + with open(args.out_csv, "w", newline="", encoding="utf-8") as fh: + stats = convert(doc, M, csv.writer(fh), n_batches) + doc.close() + + expected = n_batches * M["rows_per_page"] - 1 + misplaced = stats["text_no_column"] + stats["text_no_row"] + tally = {} + for _row, letter, _width in stats["blackouts"]: + tally[letter] = tally.get(letter, 0) + 1 + + print(f"\nWritten to {args.out_csv}\n") + print("-" * 62) + report("rows written", f"{stats['n_rows']:,}") + report("rows expected", f"{expected:,}", stats["n_rows"] == expected, "MISMATCH") + report("row numbers 1..N unbroken", "yes" if stats["chain_ok"] else "NO", + stats["chain_ok"], "GAPS OR DUPLICATES") + report("unplaced text", + f"{stats['text_no_column']:,} by column, {stats['text_no_row']:,} by row", + misplaced == 0, "SOME TEXT LOST") + report("unplaced bars", f"{stats['bars_no_row']:,}", + stats["bars_no_row"] == 0, "REDACTIONS LOST") + report("disagreed numbers", f"{stats['gutter_disagreed']:,}", + stats["gutter_disagreed"] == 0, "LAYOUT WRONG") + report("row numbers hidden by bars", + f"{stats['gutter_unreadable']:,} (placed by spacing instead)") + report("cells drawn in pieces", + f"{stats['pieces']:,} (rejoined; nothing lost)") + print() + report("blackout bars found", f"{len(stats['blackouts']):,}") + report("rows affected", f"{len(set(b[0] for b in stats['blackouts'])):,}") + report("which columns", tally or "none") + if stats["readable"]: + report("TEXT READABLE UNDER BARS", f"{len(stats['readable']):,}", + False, "REDACTION IS COSMETIC, HAHA! TEXT RECOVERABLE") + for row, letter, text in stats["readable"][:10]: + print(f" row {row:,} column {letter}: {text!r}") + else: + report("text readable under bars", "none (deleted)") + + if n_batches * M["rows_per_page"] == 1048576: + print("\n WARNING: exactly 1,048,576 rows. That's an Excel magic number") + print(" The source was almost certainly truncated.") + print(" Go tell the source they owe you more data.") diff --git a/pinion/convert.py b/pinion/convert.py new file mode 100644 index 0000000..4e69335 --- /dev/null +++ b/pinion/convert.py @@ -0,0 +1,119 @@ +from .page import all_runs, run_start, run_text, row_numbers, solid_regions + + +def which_column(edges, x): + """Which column does text starting at x belong to? + Checked against text start with a slight fudge factor.""" + for i in range(len(edges) - 1): + if edges[i] - 0.6 <= x < edges[i + 1] - 0.6: + return i + return None + + +def convert(doc, M, out, n_batches): + """Rebuild every row and write it to `out`.""" + rpp, pitch = M["rows_per_page"], M["pitch"] + stats = {"text_no_column": 0, "text_no_row": 0, "bars_no_row": 0, + "pieces": 0, "gutter_disagreed": 0, "gutter_unreadable": 0, + "n_rows": 0, "chain_ok": True, + "blackouts": [], "readable": []} + next_expected = 1 + + for batch in range(n_batches): + if batch and batch % max(1, n_batches // 10) == 0: + print(f" {100 * batch // n_batches}% ...") + cells, redacted, known_rows = {}, {}, set() + + for slice_no in range(M["ppb"]): + page = doc[batch * M["ppb"] + slice_no] + edges, offset = M["bands"][slice_no], M["offsets"][slice_no] + runs = all_runs(page) + data = [r for r in runs if r["font"] != M["font"]] + if not data: + continue + + # STEP 1 - Read this page's printed row numbers. + printed = row_numbers(runs, M["font"], M["table_left"]) + stats["gutter_unreadable"] += rpp - len(printed) + + # STEP 2 - Vertical position -> row number. + # This makes sure redaction-covered row numbers don't mess us up. + # (Yes, it happens) + if printed: + anchor_y = min(printed) + anchor_n = printed[anchor_y] + else: + anchor_y = M["first_row_y"] + anchor_n = batch * rpp + 1 + + def row_at(y): + n = anchor_n + round((y - anchor_y) / pitch) + lo = anchor_n - (anchor_n - 1) % rpp + return n if lo <= n < lo + rpp else None + + # If we can read the number, might as well validate it. + for y, n in printed.items(): + if row_at(y) != n: + stats["gutter_disagreed"] += 1 + known_rows.add(n) + + # STEP 3 - Text -> cell. + for run in data: + x, y = run_start(run) + row = row_at(y) + if row is None: + stats["text_no_row"] += 1 + continue + col = which_column(edges, x) + if col is None: + stats["text_no_column"] += 1 + continue + slot = cells.setdefault(row, [[] for _ in range(M["n_cols"])]) + slot[offset + col].append((x, run_text(run))) + + # STEP 4 - Blackout bars (Trap 2). + for box in solid_regions(page): + if box.width <= M["bar_min_w"] or box.height <= M["bar_min_h"]: + continue + row = row_at((box.y0 + box.y1) / 2) + if row is None: + stats["bars_no_row"] += 1 + continue + col = which_column(edges, box.x0) + letters = M["strips"][slice_no] + label = letters[col] if col is not None else "?" + stats["blackouts"].append((row, label, round(box.width, 1))) + redacted[row] = max(redacted.get(row, 0), round(box.width, 1)) + + # Did any writing survive underneath the bar? + # If so, point and laugh. And also report it. + for run in data: + if row_at(run_start(run)[1]) != row: + continue + under = [chr(c[0]) for c in run["chars"] + if box.x0 - 0.5 <= c[2][0] <= box.x1 - 1.5] + if under: + stats["readable"].append((row, label, "".join(under))) + + # STEP 5 - Write it all out. + for row in sorted(set(cells) | known_rows): + slot = cells.get(row) or [[] for _ in range(M["n_cols"])] + line = [] + for cell in slot: + if len(cell) > 1: + stats["pieces"] += 1 + # sorting by x rejoins a cell drawn in several pieces + line.append("".join(t for _, t in sorted(cell)).strip()) + if row == 1: + # header row names our two extra redaction columns + out.writerow(line + ["reason_redacted", "redaction_bar_pt"]) + else: + bar = redacted.get(row) + out.writerow(line + ["TRUE" if bar else "FALSE", bar or ""]) + stats["n_rows"] += 1 + # Row numbers must count 1,2,3... with no gaps or repeats. Duh. + if row != next_expected: + stats["chain_ok"] = False + next_expected = row + 1 + + return stats diff --git a/pinion/measure.py b/pinion/measure.py new file mode 100644 index 0000000..66c7cc9 --- /dev/null +++ b/pinion/measure.py @@ -0,0 +1,155 @@ +import sys +import statistics + +import pymupdf + +from .page import all_runs, run_start, run_text, row_numbers, vertical_rules + + +def find_heading_font(page0): + """Excel draws its labelling (row numbers, letter strip) in a different + font from the data, and the row numbers sit leftmost on the page. So: + whatever font the leftmost text uses is the labelling font.""" + leftmost = min(all_runs(page0), key=lambda r: run_start(r)[0]) + if not run_text(leftmost).strip().isdigit(): + sys.exit("Expected a row number at the far left of page 1, found " + f"{run_text(leftmost)!r}. Was this printed from a spreadsheet " + "with row and column headings switched on?") + return leftmost["font"] + + +def find_table_left(page0, font): + """Where cell data begins. Left of this: margin. Right of this: table.""" + return min(run_start(r)[0] for r in all_runs(page0) if r["font"] != font) + + +def column_letters(page, font, table_left): + """The 'A B C D' strip above the table, read left to right.""" + got = [] + for run in all_runs(page): + if run["font"] != font: + continue + for ch in run["chars"]: + if ch[2][0] >= table_left and chr(ch[0]).isalpha(): + got.append((ch[2][0], chr(ch[0]))) + return "".join(c for _, c in sorted(got)) + + +def find_batch_shape(doc, font, table_left): + """How many pages hold one batch of rows, and which columns each shows. + We follow the letter strip page by page until it starts over at 'A'.""" + strips = [column_letters(doc[p], font, table_left) + for p in range(min(40, doc.page_count))] + if not strips[0].startswith("A"): + sys.exit(f"Page 1 starts at column {strips[0][:1]!r}, not 'A'. This " + "does not look like the first page of a printed spreadsheet.") + ppb = next((i for i in range(1, len(strips)) + if strips[i].startswith("A")), None) + if ppb is None: + sys.exit("Could not find where the column letters start over at 'A'.") + if doc.page_count % ppb: + sys.exit(f"The file has {doc.page_count} pages, which does not divide " + f"evenly into batches of {ppb}. The layout is not uniform " + "and this tool cannot safely rebuild it.") + return strips[:ppb], ppb, doc.page_count // ppb + + +def find_row_grid(page0_runs, font, table_left): + """Rows per page, row height, and where the first row sits -- read off + the printed row numbers, which is exact. Assumes rows are evenly spaced + and the same height throughout the file.""" + gutter = row_numbers(page0_runs, font, table_left) + if len(gutter) < 2: + sys.exit("Could not read Excel's printed row numbers from the left " + "margin of page 1. They are required to place rows exactly.") + ys = sorted(gutter) + if gutter[ys[0]] != 1: + sys.exit(f"Page 1 begins at spreadsheet row {gutter[ys[0]]}, not " + "row 1. This may not be the start of the document.") + pitch = statistics.median(b - a for a, b in zip(ys, ys[1:])) + return len(gutter), pitch, ys[0] + + +def find_column_edges(doc, font, strips, ppb, n_batches): + """Column boundaries for each page type. Excel only draws a boundary + where a cell has a border, so no single page shows all of them -- we + sample many pages of each type and combine. Edges closer than 2pt are + the same line drawn twice; anything left of the first cell is margin, + except the last such line, which is the table's own left edge.""" + step = max(1, n_batches // 80) + bands = [] + for slice_no in range(ppb): + found, starts = set(), [] + for b in range(0, n_batches, step): + page = doc[b * ppb + slice_no] + found |= vertical_rules(page) + starts += [run_start(r)[0] for r in all_runs(page) + if r["font"] != font] + if not starts: + sys.exit(f"No cell text found anywhere on page type {slice_no + 1}.") + leftmost_cell = min(starts) + edges = [] + for x in sorted(found): + if x > leftmost_cell: + if edges and x - edges[-1] < 2.0: + continue + edges.append(x) + else: + edges = [x] + wanted = len(strips[slice_no]) + 1 + if len(edges) != wanted: + sys.exit(f"On page type {slice_no + 1} the column letters say " + f"there are {len(strips[slice_no])} columns " + f"({strips[slice_no]}), which needs {wanted} edges, but " + f"{len(edges)} were found.") + bands.append(edges) + return bands + + +def measure(path): + """Establish every fact the converter needs, narrating as we go.""" + doc = pymupdf.open(path) + page0_runs = all_runs(doc[0]) + + print("MEASURING THE PAGE") + print("-" * 62) + font = find_heading_font(doc[0]) + print(f" heading font ......... {font}") + + table_left = find_table_left(doc[0], font) + print(f" table starts at x .... {table_left:.1f}pt " + f"(left of this is the row-number margin)") + + strips, ppb, n_batches = find_batch_shape(doc, font, table_left) + print(f" pages per batch ...... {ppb} column strips {strips}") + print(f" batches in the file .. {n_batches:,}") + + rows_per_page, pitch, first_row_y = find_row_grid(page0_runs, font, table_left) + print(f" rows per page ........ {rows_per_page}") + print(f" row height ........... {pitch:.2f}pt " + f"(first row sits at y={first_row_y:.1f})") + + bands = find_column_edges(doc, font, strips, ppb, n_batches) + for i, edges in enumerate(bands): + print(f" page type {i + 1} ({strips[i]:<8}) {len(edges) - 1} columns, " + f"edges {' '.join(f'{e:.0f}' for e in edges)}") + + # Telling a blackout bar from a grid line: separate by SHAPE, not size. + # A single-character blackout is only ~3pt wide, but it is always as + # tall as the writing it hides; a rule is long but paper-thin. + bar_min_w, bar_min_h = 1.2, pitch * 0.5 + print(f" blackout bars ........ solid shapes over {bar_min_w}pt wide " + f"and {bar_min_h:.1f}pt tall") + print(f" spreadsheet shape .... {n_batches * rows_per_page:,} rows " + f"x {sum(len(s) for s in strips)} columns\n") + + doc.close() + return { + "font": font, "table_left": table_left, "strips": strips, + "ppb": ppb, "n_batches": n_batches, "bands": bands, + "rows_per_page": rows_per_page, "pitch": pitch, + "first_row_y": first_row_y, "bar_min_w": bar_min_w, + "bar_min_h": bar_min_h, + "n_cols": sum(len(s) for s in strips), + "offsets": [sum(len(s) for s in strips[:i]) for i in range(ppb)], + } diff --git a/pinion/page.py b/pinion/page.py new file mode 100644 index 0000000..93b5235 --- /dev/null +++ b/pinion/page.py @@ -0,0 +1,65 @@ +"""Reading what the PDF actually drew on a page. + +Small views onto the raw page. Everything downstream is built from these. +The key one is all_runs(): it returns each piece of text AS THE PDF DREW IT, +one run per spreadsheet cell, which is what dodges Trap 1 -- overlapping +cells stay separate because the PDF drew them separately. +""" + + +def all_runs(page): + """Every piece of text on the page, one run per spreadsheet cell.""" + return [r for r in page.get_texttrace() if r["chars"]] + + +def run_start(run): + """Where a piece of text begins, as (across, down).""" + return run["chars"][0][2] + + +def run_text(run): + """The text of a run. Word Wrap would really screw this up; don't let + the PDF boomers know.""" + return "".join(chr(c[0]) for c in run["chars"]) + + +def solid_regions(page): + """Every solid filled region, taken as a whole. A blackout bar can be a + plain rectangle OR a filled four-sided path -- visually identical, + different in the PDF -- so we measure whole regions, never segments.""" + return [d["rect"] for d in page.get_drawings() if d["type"] == "f"] + + +def vertical_rules(page): + """x-positions of thin vertical lines: <1.2pt wide, >8pt tall, drawn + either as skinny rectangles or as literal line segments.""" + xs = set() + for d in page.get_drawings(): + for item in d["items"]: + if item[0] == "re" and item[1].width < 1.2 and item[1].height > 8: + xs.add(round(item[1].x0 + item[1].width / 2, 1)) + elif item[0] == "l": + a, b = item[1], item[2] + if abs(a.x - b.x) < 1.2 and abs(a.y - b.y) > 8: + xs.add(round((a.x + b.x) / 2, 1)) + return xs + + +def row_numbers(runs, heading_font, table_left): + """Excel's printed row numbers -> {vertical position: row number}. + Digits of one number are drawn separately, so group them by line and + read left to right.""" + lines = {} + for run in runs: + if run["font"] != heading_font: + continue + for ch in run["chars"]: + x, y = ch[2] + if x < table_left: + lines.setdefault(round(y, 1), []).append((x, chr(ch[0]))) + out = {} + for y, chars in lines.items(): + text = "".join(c for _, c in sorted(chars)) + if text.isdigit(): + out[y] = int(text) + return out diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0e6cd54 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "pinion-pdf" +version = "1.0.0" +description = "Turn a spreadsheet that was printed to PDF back into a CSV, and prove the conversion is right." +readme = "README.md" +requires-python = ">=3.9" +license = { text = "MIT" } +authors = [{ name = "David Adams" }] +dependencies = ["pymupdf"] + +[project.scripts] +pinion = "pinion.cli:main" + +[tool.setuptools] +packages = ["pinion"]