daveandClaude Fable 5
74b06be742
Derives page layout from the file itself (batch size, column edges, row grid, heading font), places text by start-x to survive cell overflow, detects blackout redactions including filled-path bars and single-character bars, and self-verifies via Excel's printed row-number chain. Output byte-identical to four independently written implementations across a 1,048,576-row reference file. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_012bQ28wRBiQYcSR9FzV9U7c
156 lines
6.6 KiB
Python
156 lines
6.6 KiB
Python
import sys
|
|
import statistics
|
|
|
|
import pymupdf
|
|
|
|
from .page import all_runs, run_start, run_text, row_numbers, vertical_rules
|
|
|
|
|
|
def find_heading_font(page0):
|
|
"""Excel draws its labelling (row numbers, letter strip) in a different
|
|
font from the data, and the row numbers sit leftmost on the page. So:
|
|
whatever font the leftmost text uses is the labelling font."""
|
|
leftmost = min(all_runs(page0), key=lambda r: run_start(r)[0])
|
|
if not run_text(leftmost).strip().isdigit():
|
|
sys.exit("Expected a row number at the far left of page 1, found "
|
|
f"{run_text(leftmost)!r}. Was this printed from a spreadsheet "
|
|
"with row and column headings switched on?")
|
|
return leftmost["font"]
|
|
|
|
|
|
def find_table_left(page0, font):
|
|
"""Where cell data begins. Left of this: margin. Right of this: table."""
|
|
return min(run_start(r)[0] for r in all_runs(page0) if r["font"] != font)
|
|
|
|
|
|
def column_letters(page, font, table_left):
|
|
"""The 'A B C D' strip above the table, read left to right."""
|
|
got = []
|
|
for run in all_runs(page):
|
|
if run["font"] != font:
|
|
continue
|
|
for ch in run["chars"]:
|
|
if ch[2][0] >= table_left and chr(ch[0]).isalpha():
|
|
got.append((ch[2][0], chr(ch[0])))
|
|
return "".join(c for _, c in sorted(got))
|
|
|
|
|
|
def find_batch_shape(doc, font, table_left):
|
|
"""How many pages hold one batch of rows, and which columns each shows.
|
|
We follow the letter strip page by page until it starts over at 'A'."""
|
|
strips = [column_letters(doc[p], font, table_left)
|
|
for p in range(min(40, doc.page_count))]
|
|
if not strips[0].startswith("A"):
|
|
sys.exit(f"Page 1 starts at column {strips[0][:1]!r}, not 'A'. This "
|
|
"does not look like the first page of a printed spreadsheet.")
|
|
ppb = next((i for i in range(1, len(strips))
|
|
if strips[i].startswith("A")), None)
|
|
if ppb is None:
|
|
sys.exit("Could not find where the column letters start over at 'A'.")
|
|
if doc.page_count % ppb:
|
|
sys.exit(f"The file has {doc.page_count} pages, which does not divide "
|
|
f"evenly into batches of {ppb}. The layout is not uniform "
|
|
"and this tool cannot safely rebuild it.")
|
|
return strips[:ppb], ppb, doc.page_count // ppb
|
|
|
|
|
|
def find_row_grid(page0_runs, font, table_left):
|
|
"""Rows per page, row height, and where the first row sits -- read off
|
|
the printed row numbers, which is exact. Assumes rows are evenly spaced
|
|
and the same height throughout the file."""
|
|
gutter = row_numbers(page0_runs, font, table_left)
|
|
if len(gutter) < 2:
|
|
sys.exit("Could not read Excel's printed row numbers from the left "
|
|
"margin of page 1. They are required to place rows exactly.")
|
|
ys = sorted(gutter)
|
|
if gutter[ys[0]] != 1:
|
|
sys.exit(f"Page 1 begins at spreadsheet row {gutter[ys[0]]}, not "
|
|
"row 1. This may not be the start of the document.")
|
|
pitch = statistics.median(b - a for a, b in zip(ys, ys[1:]))
|
|
return len(gutter), pitch, ys[0]
|
|
|
|
|
|
def find_column_edges(doc, font, strips, ppb, n_batches):
|
|
"""Column boundaries for each page type. Excel only draws a boundary
|
|
where a cell has a border, so no single page shows all of them -- we
|
|
sample many pages of each type and combine. Edges closer than 2pt are
|
|
the same line drawn twice; anything left of the first cell is margin,
|
|
except the last such line, which is the table's own left edge."""
|
|
step = max(1, n_batches // 80)
|
|
bands = []
|
|
for slice_no in range(ppb):
|
|
found, starts = set(), []
|
|
for b in range(0, n_batches, step):
|
|
page = doc[b * ppb + slice_no]
|
|
found |= vertical_rules(page)
|
|
starts += [run_start(r)[0] for r in all_runs(page)
|
|
if r["font"] != font]
|
|
if not starts:
|
|
sys.exit(f"No cell text found anywhere on page type {slice_no + 1}.")
|
|
leftmost_cell = min(starts)
|
|
edges = []
|
|
for x in sorted(found):
|
|
if x > leftmost_cell:
|
|
if edges and x - edges[-1] < 2.0:
|
|
continue
|
|
edges.append(x)
|
|
else:
|
|
edges = [x]
|
|
wanted = len(strips[slice_no]) + 1
|
|
if len(edges) != wanted:
|
|
sys.exit(f"On page type {slice_no + 1} the column letters say "
|
|
f"there are {len(strips[slice_no])} columns "
|
|
f"({strips[slice_no]}), which needs {wanted} edges, but "
|
|
f"{len(edges)} were found.")
|
|
bands.append(edges)
|
|
return bands
|
|
|
|
|
|
def measure(path):
|
|
"""Establish every fact the converter needs, narrating as we go."""
|
|
doc = pymupdf.open(path)
|
|
page0_runs = all_runs(doc[0])
|
|
|
|
print("MEASURING THE PAGE")
|
|
print("-" * 62)
|
|
font = find_heading_font(doc[0])
|
|
print(f" heading font ......... {font}")
|
|
|
|
table_left = find_table_left(doc[0], font)
|
|
print(f" table starts at x .... {table_left:.1f}pt "
|
|
f"(left of this is the row-number margin)")
|
|
|
|
strips, ppb, n_batches = find_batch_shape(doc, font, table_left)
|
|
print(f" pages per batch ...... {ppb} column strips {strips}")
|
|
print(f" batches in the file .. {n_batches:,}")
|
|
|
|
rows_per_page, pitch, first_row_y = find_row_grid(page0_runs, font, table_left)
|
|
print(f" rows per page ........ {rows_per_page}")
|
|
print(f" row height ........... {pitch:.2f}pt "
|
|
f"(first row sits at y={first_row_y:.1f})")
|
|
|
|
bands = find_column_edges(doc, font, strips, ppb, n_batches)
|
|
for i, edges in enumerate(bands):
|
|
print(f" page type {i + 1} ({strips[i]:<8}) {len(edges) - 1} columns, "
|
|
f"edges {' '.join(f'{e:.0f}' for e in edges)}")
|
|
|
|
# Telling a blackout bar from a grid line: separate by SHAPE, not size.
|
|
# A single-character blackout is only ~3pt wide, but it is always as
|
|
# tall as the writing it hides; a rule is long but paper-thin.
|
|
bar_min_w, bar_min_h = 1.2, pitch * 0.5
|
|
print(f" blackout bars ........ solid shapes over {bar_min_w}pt wide "
|
|
f"and {bar_min_h:.1f}pt tall")
|
|
print(f" spreadsheet shape .... {n_batches * rows_per_page:,} rows "
|
|
f"x {sum(len(s) for s in strips)} columns\n")
|
|
|
|
doc.close()
|
|
return {
|
|
"font": font, "table_left": table_left, "strips": strips,
|
|
"ppb": ppb, "n_batches": n_batches, "bands": bands,
|
|
"rows_per_page": rows_per_page, "pitch": pitch,
|
|
"first_row_y": first_row_y, "bar_min_w": bar_min_w,
|
|
"bar_min_h": bar_min_h,
|
|
"n_cols": sum(len(s) for s in strips),
|
|
"offsets": [sum(len(s) for s in strips[:i]) for i in range(ppb)],
|
|
}
|