pinion-pdf 1.0.0: rebuild a CSV from a spreadsheet printed to PDF

Derives page layout from the file itself (batch size, column edges, row
grid, labelling font), places text by start-x to survive cell overflow,
detects blackout redactions including filled-path bars and
single-character bars, and self-verifies via Excel's printed row-number
chain. Output byte-identical to four independently written
implementations across a 1,048,576-row reference file.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012bQ28wRBiQYcSR9FzV9U7c

pinion/measure.py

@@ -0,0 +1,163 @@
+"""Measuring the layout before reading a single row.
+
+One function per fact. None of them print; measure() at the bottom runs
+them in order and narrates. If a fact can't be established we stop with a
+specific complaint instead of guessing -- wrong-but-plausible output is the
+one failure mode this tool must never have.
+"""
+
+import sys
+import statistics
+
+import pymupdf
+
+from .page import all_runs, run_start, run_text, row_numbers, vertical_rules
+
+
+def find_furniture_font(page0):
+    """Excel draws its labelling (row numbers, letter strip) in a different
+    font from the data, and the row numbers sit leftmost on the page. So:
+    whatever font the leftmost text uses is the labelling font."""
+    leftmost = min(all_runs(page0), key=lambda r: run_start(r)[0])
+    if not run_text(leftmost).strip().isdigit():
+        sys.exit("Expected a row number at the far left of page 1, found "
+                 f"{run_text(leftmost)!r}. Was this printed from a spreadsheet "
+                 "with row and column headings switched on?")
+    return leftmost["font"]
+
+
+def find_table_left(page0, font):
+    """Where cell data begins. Left of this: margin. Right of this: table."""
+    return min(run_start(r)[0] for r in all_runs(page0) if r["font"] != font)
+
+
+def column_letters(page, font, table_left):
+    """The 'A B C D' strip above the table, read left to right."""
+    got = []
+    for run in all_runs(page):
+        if run["font"] != font:
+            continue
+        for ch in run["chars"]:
+            if ch[2][0] >= table_left and chr(ch[0]).isalpha():
+                got.append((ch[2][0], chr(ch[0])))
+    return "".join(c for _, c in sorted(got))
+
+
+def find_batch_shape(doc, font, table_left):
+    """How many pages hold one batch of rows, and which columns each shows.
+    We follow the letter strip page by page until it starts over at 'A'."""
+    strips = [column_letters(doc[p], font, table_left)
+              for p in range(min(40, doc.page_count))]
+    if not strips[0].startswith("A"):
+        sys.exit(f"Page 1 starts at column {strips[0][:1]!r}, not 'A'. This "
+                 "does not look like the first page of a printed spreadsheet.")
+    ppb = next((i for i in range(1, len(strips))
+                if strips[i].startswith("A")), None)
+    if ppb is None:
+        sys.exit("Could not find where the column letters start over at 'A'.")
+    if doc.page_count % ppb:
+        sys.exit(f"The file has {doc.page_count} pages, which does not divide "
+                 f"evenly into batches of {ppb}. The layout is not uniform "
+                 "and this tool cannot safely rebuild it.")
+    return strips[:ppb], ppb, doc.page_count // ppb
+
+
+def find_row_grid(page0_runs, font, table_left):
+    """Rows per page, row height, and where the first row sits -- read off
+    the printed row numbers, which is exact. Assumes rows are evenly spaced
+    and the same height throughout the file."""
+    gutter = row_numbers(page0_runs, font, table_left)
+    if len(gutter) < 2:
+        sys.exit("Could not read Excel's printed row numbers from the left "
+                 "margin of page 1. They are required to place rows exactly.")
+    ys = sorted(gutter)
+    if gutter[ys[0]] != 1:
+        sys.exit(f"Page 1 begins at spreadsheet row {gutter[ys[0]]}, not "
+                 "row 1. This may not be the start of the document.")
+    pitch = statistics.median(b - a for a, b in zip(ys, ys[1:]))
+    return len(gutter), pitch, ys[0]
+
+
+def find_column_edges(doc, font, strips, ppb, n_batches):
+    """Column boundaries for each page type. Excel only draws a boundary
+    where a cell has a border, so no single page shows all of them -- we
+    sample many pages of each type and combine. Edges closer than 2pt are
+    the same line drawn twice; anything left of the first cell is margin,
+    except the last such line, which is the table's own left edge."""
+    step = max(1, n_batches // 80)
+    bands = []
+    for slice_no in range(ppb):
+        found, starts = set(), []
+        for b in range(0, n_batches, step):
+            page = doc[b * ppb + slice_no]
+            found |= vertical_rules(page)
+            starts += [run_start(r)[0] for r in all_runs(page)
+                       if r["font"] != font]
+        if not starts:
+            sys.exit(f"No cell text found anywhere on page type {slice_no + 1}.")
+        leftmost_cell = min(starts)
+        edges = []
+        for x in sorted(found):
+            if x > leftmost_cell:
+                if edges and x - edges[-1] < 2.0:
+                    continue
+                edges.append(x)
+            else:
+                edges = [x]
+        wanted = len(strips[slice_no]) + 1
+        if len(edges) != wanted:
+            sys.exit(f"On page type {slice_no + 1} the column letters say "
+                     f"there are {len(strips[slice_no])} columns "
+                     f"({strips[slice_no]}), which needs {wanted} edges, but "
+                     f"{len(edges)} were found.")
+        bands.append(edges)
+    return bands
+
+
+def measure(path):
+    """Establish every fact the converter needs, narrating as we go."""
+    doc = pymupdf.open(path)
+    page0_runs = all_runs(doc[0])
+
+    print("MEASURING THE PAGE")
+    print("-" * 62)
+    font = find_furniture_font(doc[0])
+    print(f"  furniture font ....... {font}")
+
+    table_left = find_table_left(doc[0], font)
+    print(f"  table starts at x .... {table_left:.1f}pt "
+          f"(left of this is the row-number margin)")
+
+    strips, ppb, n_batches = find_batch_shape(doc, font, table_left)
+    print(f"  pages per batch ...... {ppb}   column strips {strips}")
+    print(f"  batches in the file .. {n_batches:,}")
+
+    rows_per_page, pitch, first_row_y = find_row_grid(page0_runs, font, table_left)
+    print(f"  rows per page ........ {rows_per_page}")
+    print(f"  row height ........... {pitch:.2f}pt "
+          f"(first row sits at y={first_row_y:.1f})")
+
+    bands = find_column_edges(doc, font, strips, ppb, n_batches)
+    for i, edges in enumerate(bands):
+        print(f"  page type {i + 1} ({strips[i]:<8}) {len(edges) - 1} columns, "
+              f"edges {' '.join(f'{e:.0f}' for e in edges)}")
+
+    # Telling a blackout bar from a grid line: separate by SHAPE, not size.
+    # A single-character blackout is only ~3pt wide, but it is always as
+    # tall as the writing it hides; a rule is long but paper-thin.
+    bar_min_w, bar_min_h = 1.2, pitch * 0.5
+    print(f"  blackout bars ........ solid shapes over {bar_min_w}pt wide "
+          f"and {bar_min_h:.1f}pt tall")
+    print(f"  spreadsheet shape .... {n_batches * rows_per_page:,} rows "
+          f"x {sum(len(s) for s in strips)} columns\n")
+
+    doc.close()
+    return {
+        "font": font, "table_left": table_left, "strips": strips,
+        "ppb": ppb, "n_batches": n_batches, "bands": bands,
+        "rows_per_page": rows_per_page, "pitch": pitch,
+        "first_row_y": first_row_y, "bar_min_w": bar_min_w,
+        "bar_min_h": bar_min_h,
+        "n_cols": sum(len(s) for s in strips),
+        "offsets": [sum(len(s) for s in strips[:i]) for i in range(ppb)],
+    }