From aa1f7f18b634c2a60507388a06025923d1755f9e Mon Sep 17 00:00:00 2001 From: Adhaar Sharma Date: Wed, 28 Oct 2026 14:45:41 -0400 Subject: [PATCH 1/4] Map what VBA macros touch; add `macro` command and an xlsm fixture - New `macros` module reads each VBA procedure's source: the sheets and ranges it writes and uses (literal addresses, variables, With blocks, sheets passed to other procedures, code names, string constants, named ranges, sheets it creates), the procedures it calls, and the event that runs it. Apps Script functions are listed with their triggers. - REPORT.md: one line per procedure under Scripts; modules with no code (the empty one Excel gives every sheet) are counted, not listed. - graph.json: `macro` nodes with `writes` / `uses` / `calls` edges, and `calls` edges from formula groups to the VBA functions they use. - `sheetgraph macro [name]` lists macros or prints a module's or procedure's code; `find` also searches macro code. - Read sheets' VBA code names from xlsx/xlsm (`Sheet.code_name`). - tests/fixtures/macros.xlsm, built by make_macros.py (which writes the VBA project from scratch), and tests/macros.rs. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 + README.md | 16 +- skills/sheetgraph/SKILL.md | 4 +- src/analyze.rs | 4 + src/graph.rs | 46 ++ src/lib.rs | 1 + src/macros.rs | 860 ++++++++++++++++++++++++++++++++++ src/main.rs | 14 +- src/model.rs | 15 + src/ooxml.rs | 5 +- src/query.rs | 69 +++ src/report.rs | 39 +- tests/fixtures/macros.xlsm | Bin 0 -> 7448 bytes tests/fixtures/make_macros.py | 316 +++++++++++++ tests/macros.rs | 190 ++++++++ 15 files changed, 1561 insertions(+), 28 deletions(-) create mode 100644 src/macros.rs create mode 100644 tests/fixtures/macros.xlsm create mode 100644 tests/fixtures/make_macros.py create mode 100644 tests/macros.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index afae34f..cd936e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,16 @@ All notable changes to this project are documented here. The format follows - `install.sh`, a one-line installer for macOS and Linux that downloads the latest release binary. - Agent skill (`skills/sheetgraph/SKILL.md`) and a Claude Code plugin marketplace (`.claude-plugin/`), so agents run sheetgraph on their own when asked about a spreadsheet. - README: before/after demo image (`assets/demo.png`) and the script that generates its sample workbook. +- Macros: each VBA procedure's trigger (workbook and sheet events, `Auto_Open`), the sheets and ranges it writes and + uses, the procedures it calls, and the formula cells that call it as a function, in REPORT.md and as `macro` nodes + with `writes` / `uses` / `calls` edges in graph.json. Apps Script functions are listed with their triggers. +- `sheetgraph macro [name]` lists macros or prints one module's or procedure's code; `find` also searches + macro code. +- Sheets' VBA code names are read from xlsx/xlsm (`Sheet.code_name`). + +### Changed + +- REPORT.md counts modules with no code (the empty module Excel gives every sheet) instead of listing each one. ## [0.1.0] diff --git a/README.md b/README.md index 4baf46a..4f7371a 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,8 @@ sheetgraph extract ... -o sheetgraph-out --creds service-accou # drill down (all read workbook.json.gz; no network) sheetgraph cells sheetgraph-out/ "'Revenue Input'!A1:F20" sheetgraph trace sheetgraph-out/ "Monthly!E10" --depth 3 # precedents + dependents -sheetgraph find sheetgraph-out/ "VAT" +sheetgraph find sheetgraph-out/ "VAT" # values, formulas, notes, macro code +sheetgraph macro sheetgraph-out/ [Module1 | RollUp] # macros and what they touch; code with a name sheetgraph sheet sheetgraph-out/ "Ledger" --full # untruncated section sheetgraph report sheetgraph-out/ [--full] # re-render without refetching ``` @@ -126,12 +127,14 @@ let graph = sheetgraph::graph::build(&wb, &a); // serde_json::Value sheetgraph-out/ INDEX.md every workbook, sizes, links between workbooks (IMPORTRANGE, Excel external links) /REPORT.md the map: read this first - /graph.json nodes (sheets, blocks, formula groups, charts, pivots, externals) + edges + /graph.json nodes (sheets, blocks, formula groups, charts, pivots, externals, macros) + edges /workbook.json.gz every cell (see Known gaps for very large files): value, formula, display text, style, validation, note, link /meta.json ``` -REPORT.md contains, per workbook: sheet list with used ranges and sheet→sheet data flow; external links (IMPORTRANGE / IMPORTDATA / … with targets resolved from cells, Excel links to other workbooks, refs to deleted sheets, Connected-Sheets data sources); named ranges; function census incl. **non-builtin functions** (Apps Script custom functions / Named Functions / VBA UDFs), dynamic refs (INDIRECT/OFFSET) and volatile functions; scripts (VBA modules from xlsm; Apps Script on request); developer metadata (left by add-ons/integrations). +REPORT.md contains, per workbook: sheet list with used ranges and sheet→sheet data flow; external links (IMPORTRANGE / IMPORTDATA / … with targets resolved from cells, Excel links to other workbooks, refs to deleted sheets, Connected-Sheets data sources); named ranges; function census incl. **non-builtin functions** (Apps Script custom functions / Named Functions / VBA UDFs), dynamic refs (INDIRECT/OFFSET) and volatile functions; scripts (VBA modules from xlsm; Apps Script on request), with each procedure's trigger (`Workbook_Open`, `Worksheet_Change`, `onEdit`), the sheets and ranges it writes and uses, the procedures it calls, and the formula cells that call it as a function; modules with no code (the empty one Excel gives every sheet) are counted, not listed; developer metadata (left by add-ons/integrations). + +graph.json edges: `reads` (formula group → the blocks and groups its references land on), `plots` (chart → source), `summarizes` (pivot → source), `imports` (formula → external workbook or URL), `writes` / `uses` (macro → what it changes or reads; a sheet the macro creates has `"to": null` and `missing_sheet`), `calls` (macro → macro, and formula group → the VBA function it calls). Per sheet: blocks (tables with column profiles + samples, or small blocks verbatim), formula copy-groups and families with flags (`[array/spills]`, `[external]`, `[dynamic ref]`, `[#REF!]`, error counts), data validation (dropdowns, checkboxes, range-sourced lists), conditional formats, visual formatting regions, merges, filters/filter views, protected ranges, hidden rows/cols, banding, Excel tables, charts and pivots (with source ranges), notes and comments, hyperlinks. @@ -146,6 +149,7 @@ Point the agent at `INDEX.md`, then the `REPORT.md` it needs, and let it run the - "same layout as X" means a sheet is a structural clone of an earlier one with different data; `sheetgraph sheet "" --full` shows it in full. - `⋯ rows A–B summarized` means a very large sheet's middle rows were summarized per column rather than kept: the column profiles still count every row, but sample rows and `cells` don't show those rows. - Flags: `[external]` another file, `[dynamic ref]` INDIRECT/OFFSET (graph edges incomplete), `[array/spills]`, `[#REF!]`, `[N errors]`. +- Under Scripts, `RollUp: writes Summary!A1:B1,B2:B5 · uses Data!E10,… · calls Totals` is one procedure. `,…` means it also reaches cells whose address is built at run time (`Cells(i, 2)`); `(not in file)` is a sheet the macro creates; `ActiveSheet` is whichever sheet is active when it runs. `sheetgraph macro RollUp` prints its code. ## Layout @@ -157,9 +161,10 @@ Point the agent at `INDEX.md`, then the `REPORT.md` it needs, and let it run the | `ooxml` | xlsx/xlsm package parts calamine skips: styles, validation, conditional formats, comments, hyperlinks, frozen/hidden rows and cols, filters, protection, tables, charts, pivots, external workbook links | | `formula` | reference lexer (quoted and non-English sheet names, open-ended `A2:A`, whole rows/cols, `'[Book.xlsx]Sheet'`), R1C1 keys, function census | | `analyze` | copy-groups, families, blocks, cross-sheet edges, externals | +| `macros` | VBA / Apps Script procedures: triggers, sheets and ranges each writes and uses, calls | | `report` | REPORT.md | | `graph` | graph.json | -| `query` | cells / trace / find | +| `query` | cells / trace / find / macro | | `store` | on-disk layout, INDEX.md | ## Known gaps @@ -168,6 +173,7 @@ Point the agent at `INDEX.md`, then the `REPORT.md` it needs, and let it run the - **Local files**: xlsx/xlsm get the same coverage as Google Sheets, plus Excel tables and links to other workbooks (`'[Budget.xlsx]Data'!B2`). Images, shapes, sparklines and slicers aren't read (the report says when a file has them). xls, xlsb and ods get values, formulas, names and merges only. - Number-format display text (e.g. `$1,234.50`) is only rendered for dates in local files; the format itself is on every cell. - Spill ranges of array formulas are flagged on the formula but the spilled cells look like plain values. +- **What a macro touches** is read from its code, not by running it: literal addresses, sheets held in variables, `With` blocks, sheets passed to other procedures, code names, string constants and named ranges are followed; an address built at run time counts as the whole sheet, and a sheet picked at run time (from a cell, a loop over all sheets) isn't known. Which button or shape runs a macro isn't read, and Apps Script code is listed by function without the ranges it touches. - INDIRECT/OFFSET targets are dynamic, so those edges are missing (flagged in the report). - **Very large local files**: up to 10 million cells are kept whole (about 0.2–0.4 GB of memory per million). Past that, a big sheet keeps its last 1,000 rows and its formulas, and summarizes the rows in between per column (counts, min and max, most common values; no sums), so memory stays near 2–3 GB however big the file is (a 160 MB xlsx with 44 million cells: 3 GB, 84 s). REPORT.md's column profiles still count every row and say which rows were summarized; `cells` only lists kept rows, and `find` reports text matches in summarized rows by column. The budget is `xlsx::CELL_BUDGET`; `xlsx::read_limited` takes another. - **Very large Google Sheets** are read whole, without that budget. Google allows up to 20 million cells, so a sheet near the limit can take several GB of memory. @@ -178,7 +184,7 @@ Point the agent at `INDEX.md`, then the `REPORT.md` it needs, and let it run the cargo test ``` -`tests/fixtures/features.xlsx` exercises every xlsx feature the reader handles. It is generated by `tests/fixtures/make_features.py` (openpyxl 3.1.5); rerun the script after changing it. +`tests/fixtures/features.xlsx` exercises every xlsx feature the reader handles. It is generated by `tests/fixtures/make_features.py` (openpyxl 3.1.5); rerun the script after changing it. `tests/fixtures/macros.xlsm` is the VBA fixture, generated the same way by `tests/fixtures/make_macros.py`, which also writes its VBA project from scratch. See [CONTRIBUTING.md](CONTRIBUTING.md) before opening an issue or pull request, and [SECURITY.md](SECURITY.md) to report a vulnerability. diff --git a/skills/sheetgraph/SKILL.md b/skills/sheetgraph/SKILL.md index 20367bd..fb978f3 100644 --- a/skills/sheetgraph/SKILL.md +++ b/skills/sheetgraph/SKILL.md @@ -42,13 +42,15 @@ Notation: - `▸ range table …` is a data block, one line per column: type and range, `enum` for small value sets, `fmt` for the number format, `ƒN` when computed. - `⋯ rows A–B summarized`: a very large sheet's middle rows are summarized per column, not kept. - Flags: `[external]` another file, `[dynamic ref]` INDIRECT/OFFSET (graph edges incomplete), `[array/spills]`, `[#REF!]`, `[N errors]`. +- Scripts: one line per macro with its trigger, what it `writes` and `uses`, and what it `calls`. `,…` means more cells whose address is built at run time; `(not in file)` is a sheet the macro creates. Values a macro writes are pasted values in the sheet, not formulas: check here before calling a number hardcoded. ## 4. Drill down for exact values (offline, fast) ```bash sheetgraph cells sheetgraph-out/ "'Sheet Name'!A1:F20" # exact values, formulas, formats sheetgraph trace sheetgraph-out/ "Summary!B4" --depth 3 # precedents and dependents -sheetgraph find sheetgraph-out/ "VAT" # search text, formulas and names +sheetgraph find sheetgraph-out/ "VAT" # search text, formulas, names and macro code +sheetgraph macro sheetgraph-out/ RollUp # a macro's code (no name: every macro and what it touches) sheetgraph sheet sheetgraph-out/ "Ledger" --full # one sheet's section, untruncated ``` diff --git a/src/analyze.rs b/src/analyze.rs index 6b59430..99c5373 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -22,6 +22,8 @@ pub struct Analysis { /// References to sheets that do not exist in this workbook. pub broken_refs: BTreeMap, pub named: HashMap, Rect)>, + /// Procedures in the workbook's macro / script modules. + pub macros: Vec, } #[derive(Debug, Clone, Default)] @@ -109,12 +111,14 @@ pub fn analyze(wb: &Workbook) -> Analysis { func_cells: BTreeMap::new(), broken_refs: BTreeMap::new(), named, + macros: Vec::new(), }; for (si, sh) in wb.sheets.iter().enumerate() { let sa = analyze_sheet(wb, si, sh, &mut a); a.sheets.push(sa); } find_clones(wb, &mut a); + a.macros = crate::macros::analyze(wb, &a); a } diff --git a/src/graph.rs b/src/graph.rs index 3a18aef..01e8b8c 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -2,6 +2,7 @@ use crate::a1::{quote_sheet, Rect}; use crate::analyze::Analysis; +use crate::macros::At; use crate::model::Workbook; use serde_json::{json, Value as J}; @@ -15,6 +16,9 @@ pub fn group_id(si: usize, gi: usize) -> String { pub fn block_id(si: usize, bi: usize) -> String { format!("s{}.b{}", si + 1, bi + 1) } +pub fn macro_id(wb: &Workbook, p: &crate::macros::Proc) -> String { + format!("macro:{}.{}", wb.scripts[p.script].name, p.name) +} pub fn build(wb: &Workbook, a: &Analysis) -> J { let mut nodes = Vec::new(); @@ -86,6 +90,48 @@ pub fn build(wb: &Workbook, a: &Analysis) -> J { for n in &wb.named_ranges { nodes.push(json!({"id": format!("name:{}", n.name), "type": "named_range", "range": n.range})); } + for p in &a.macros { + let id = macro_id(wb, p); + let sc = &wb.scripts[p.script]; + nodes.push(json!({ + "id": id, "type": "macro", "lang": sc.kind, "module": sc.name, "name": p.name, "kind": p.kind, + "lines": format!("{}-{}", p.lines.0, p.lines.1), + "trigger": p.trigger.as_ref().map(|t| t.event.clone()), + "trigger_sheet": p.trigger.as_ref().and_then(|t| t.sheet).map(|s| format!("s{}", s + 1)), + })); + for t in &p.touches { + let kind = if t.write { "writes" } else { "uses" }; + let range = crate::macros::place(wb, t); + match &t.at { + At::Sheet(s) => { + let ts = t.rect.map(|r| targets(a, *s, &r, None)).unwrap_or_default(); + if ts.is_empty() || ts.len() > MAX_TARGETS { + edges.push(json!({"from": id, "to": format!("s{}", s + 1), "kind": kind, "range": range})); + } else { + for to in ts { + edges.push(json!({"from": id, "to": to, "kind": kind, "range": range})); + } + } + } + At::Missing(name) => edges.push(json!({"from": id, "to": null, "kind": kind, "range": range, "missing_sheet": name})), + // Which sheet is active when it runs isn't known. + At::Active => {} + } + } + for &c in &p.calls { + edges.push(json!({"from": id, "to": macro_id(wb, &a.macros[c]), "kind": "calls"})); + } + } + // Worksheet formulas calling VBA functions (UDFs). + for (si, sa) in a.sheets.iter().enumerate() { + for (gi, g) in sa.groups.iter().enumerate() { + for f in &g.funcs { + if let Some(p) = a.macros.iter().find(|p| p.kind == "function" && p.name.eq_ignore_ascii_case(f)) { + edges.push(json!({"from": group_id(si, gi), "to": macro_id(wb, p), "kind": "calls"})); + } + } + } + } json!({ "schema": SCHEMA, "source": wb.source, diff --git a/src/lib.rs b/src/lib.rs index 1db0156..c778857 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ pub mod analyze; pub mod formula; pub mod graph; pub mod gsheets; +pub mod macros; pub mod model; pub mod ooxml; pub mod query; diff --git a/src/macros.rs b/src/macros.rs new file mode 100644 index 0000000..9345aec --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,860 @@ +//! Macro code: the procedures in each script module and, for VBA, which sheets +//! and ranges each one reads or changes, which procedures it calls, and which +//! event runs it. +//! +//! This reads the source without running it, so it follows what is written +//! out: `Worksheets("Data").Range("A1:B5")`, a sheet or range held in a +//! variable (`Set ws = …`), a `With` block, a sheet passed to another +//! procedure's parameter, a code name (`Sheet1.Range`), a named range. An +//! address built at run time (`Range("A" & i)`, `Cells(i, 2)`) counts as +//! somewhere on that sheet. + +use crate::a1::{col_index, parse_rect, quote_sheet, Rect}; +use crate::analyze::{sheet_index, Analysis}; +use crate::model::Workbook; +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +#[derive(Debug, Clone)] +pub struct Proc { + /// Index into `wb.scripts`. + pub script: usize, + pub name: String, + /// "sub" | "function" | "property" + pub kind: &'static str, + /// First and last line in the module, 1-based. + pub lines: (usize, usize), + pub trigger: Option, + /// Sorted by (sheet, range). + pub touches: Vec, + /// Indices into the procedure list, sorted. + pub calls: Vec, + /// Declared `As Worksheet`: a helper that hands back a sheet by name. + pub returns_sheet: bool, +} + +/// An event that runs a procedure without anyone calling it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Trigger { + /// "Open", "Change", "BeforeClose", … (Apps Script: "onEdit", "doGet", …) + pub event: String, + /// The sheet whose event it is; None for the workbook. + pub sheet: Option, +} + +/// The sheet a reference lands on. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum At { + Sheet(usize), + /// Named in the code but not in the file: one the macro creates, or one + /// since deleted or renamed. + Missing(String), + /// Whichever sheet is active when it runs. + Active, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Touch { + pub at: At, + /// None: somewhere on the sheet (an address built at run time). + pub rect: Option, + /// Assigns to it or changes it (`.Value = …`, `.ClearContents`, `.Delete`, …). + pub write: bool, +} + +/// Methods that change the range or sheet they're called on. +const MUTATORS: &[&str] = &[ + "add", "autofill", "autofit", "clear", "clearcomments", "clearcontents", "clearformats", "copypicture", "delete", "filldown", + "fillright", "insert", "merge", "pastespecial", "removeduplicates", "replace", "sort", "texttocolumns", "unmerge", +]; +/// Members that move away from the literal address, so only the sheet is known. +const RELATIVE: &[&str] = &["columns", "currentregion", "end", "entirecolumn", "entirerow", "offset", "resize", "rows", "specialcells", "usedrange"]; +/// Statements whose `=` compares rather than assigns. +const NO_ASSIGN: &[&str] = &[ + "call", "case", "debug", "do", "exit", "for", "goto", "loop", "msgbox", "next", "on", "resume", "return", "select", "until", "wend", "while", +]; +/// Apps Script simple triggers and web-app entry points. +const JS_TRIGGERS: &[&str] = &["onOpen", "onEdit", "onChange", "onSelectionChange", "onFormSubmit", "onInstall", "doGet", "doPost"]; + +pub fn analyze(wb: &Workbook, a: &Analysis) -> Vec { + let mut procs = Vec::new(); + let mut bodies: Vec>> = Vec::new(); + let mut params: Vec> = Vec::new(); + let code_names: HashMap = + wb.sheets.iter().enumerate().filter_map(|(i, s)| Some((s.code_name.as_ref()?.to_ascii_lowercase(), i))).collect(); + // String constants stand in for sheet names and addresses + // (`Const RAW As String = "Raw Data"`); a module's own come first. + let mut global_consts = HashMap::new(); + let mut module_consts = Vec::new(); + for sc in &wb.scripts { + let mut mine = HashMap::new(); + if sc.kind == "vba" { + for (_, st) in statements(&sc.source) { + if let Some((name, value, public)) = string_const(&st) { + if public { + global_consts.entry(name.clone()).or_insert_with(|| value.clone()); + } + mine.insert(name, value); + } + } + } + module_consts.push(mine); + } + for (si, sc) in wb.scripts.iter().enumerate() { + if sc.kind == "vba" { + let me = code_names.get(&sc.name.to_ascii_lowercase()).copied(); + let mut open: Option<(Proc, Vec, Vec>)> = None; + let mut last = 0; + for (line, mut st) in statements(&sc.source) { + last = line; + for t in &mut st { + if let Tok::Id(w) = t { + let lw = w.to_ascii_lowercase(); + if let Some(v) = module_consts[si].get(&lw).or_else(|| global_consts.get(&lw)) { + *t = Tok::Str(v.clone()); + } + } + } + if let Some((kind, name, ps, returns_sheet)) = proc_header(&st) { + let trigger = vba_trigger(&name, me); + let p = Proc { script: si, name, kind, lines: (line, line), trigger, touches: Vec::new(), calls: Vec::new(), returns_sheet }; + open = Some((p, ps, Vec::new())); + } else if id_is(st.first(), "end") && ["sub", "function", "property"].iter().any(|k| id_is(st.get(1), k)) { + if let Some((mut p, ps, body)) = open.take() { + p.lines.1 = line; + procs.push(p); + params.push(ps); + bodies.push(body); + } + } else if let Some((_, _, body)) = &mut open { + body.push(st); + } + } + // A module cut off before `End Sub` still has the procedure. + if let Some((mut p, ps, body)) = open { + p.lines.1 = last.max(p.lines.0); + procs.push(p); + params.push(ps); + bodies.push(body); + } + } else { + let total = sc.source.lines().count(); + let start = procs.len(); + for (n, line) in sc.source.lines().enumerate() { + if let Some(name) = js_function(line) { + let trigger = JS_TRIGGERS.contains(&name.as_str()).then(|| Trigger { event: name.clone(), sheet: None }); + procs.push(Proc { script: si, name, kind: "function", lines: (n + 1, total), trigger, touches: Vec::new(), calls: Vec::new(), returns_sheet: false }); + params.push(Vec::new()); + bodies.push(Vec::new()); + } + } + // Each function runs until the next one starts. + let starts: Vec = procs[start..].iter().skip(1).map(|p| p.lines.0).collect(); + for (p, next) in procs[start..].iter_mut().zip(starts) { + p.lines.1 = next - 1; + } + } + } + + let mut by_name: HashMap> = HashMap::new(); + for (i, p) in procs.iter().enumerate() { + by_name.entry(p.name.to_ascii_lowercase()).or_default().push(i); + } + // Sheets passed as arguments bind the callee's parameters, which can pass + // them on again: repeat until no binding grows. + let mut bindings: Vec> = vec![HashMap::new(); procs.len()]; + for _ in 0..10 { + let mut next = bindings.clone(); + for (pi, body) in bodies.iter().enumerate() { + if wb.scripts[procs[pi].script].kind != "vba" { + continue; + } + let me = code_names.get(&wb.scripts[procs[pi].script].name.to_ascii_lowercase()).copied(); + let mut sc = Scope { wb, a, code_names: &code_names, by_name: &by_name, procs: &procs, me, this: pi, vars: bindings[pi].clone(), with: Vec::new() }; + let mut out = Out::default(); + for st in body { + sc.statement(st, &mut out); + } + let mut touches: BTreeMap<(At, Option), bool> = BTreeMap::new(); + for t in out.touches { + *touches.entry((t.at, t.rect)).or_default() |= t.write; + } + procs[pi].touches = touches.into_iter().map(|((at, rect), write)| Touch { at, rect, write }).collect(); + procs[pi].calls = out.calls.iter().map(|(c, _)| *c).collect::>().into_iter().collect(); + for (c, args) in out.calls { + for (k, b) in args.into_iter().enumerate() { + if let (Some(b), Some(p)) = (b, params[c].get(k)) { + next[c].entry(p.clone()).or_default().merge(&b); + } + } + } + } + if next == bindings { + break; + } + bindings = next; + } + procs +} + +/// Formula cells that call this procedure (a VBA function used as a worksheet function). +pub fn formula_cells(a: &Analysis, p: &Proc) -> u64 { + if p.kind != "function" { + return 0; + } + a.custom_funcs.iter().filter(|(f, _)| f.eq_ignore_ascii_case(&p.name)).map(|(_, n)| n).sum() +} + +fn sheet_label(wb: &Workbook, at: &At) -> String { + match at { + At::Sheet(s) => quote_sheet(&wb.sheets[*s].name), + At::Missing(n) => quote_sheet(n), + At::Active => "ActiveSheet".into(), + } +} + +/// `Sheet!A1`, `Sheet`, `Out!A1 (not in file)`, `ActiveSheet!A1`. +pub fn place(wb: &Workbook, t: &Touch) -> String { + let mut s = sheet_label(wb, &t.at); + if let Some(r) = t.rect { + s += &format!("!{r}"); + } + if matches!(t.at, At::Missing(_)) { + s += " (not in file)"; + } + s +} + +/// One line on what a procedure does: its trigger, formula use, the ranges it +/// writes and uses, the procedures it calls. `max` caps each list. None when +/// there's nothing to say beyond its name. +pub fn describe(wb: &Workbook, a: &Analysis, p: &Proc, max: usize) -> Option { + let list = |v: Vec, sep: &str| { + let more = v.len().saturating_sub(max); + let mut s = v.into_iter().take(max).collect::>().join(sep); + if more > 0 { + s += &format!(" +{more} more"); + } + s + }; + let mut parts = Vec::new(); + if let Some(t) = &p.trigger { + match t.sheet { + Some(s) => parts.push(format!("runs on {} {}", quote_sheet(&wb.sheets[s].name), t.event)), + None if wb.scripts[p.script].kind == "vba" => parts.push(format!("runs on workbook {}", t.event)), + None => parts.push(format!("trigger {}", t.event)), + } + } + let n = formula_cells(a, p); + if n > 0 { + parts.push(format!("used in {n} formula cell{}", if n == 1 { "" } else { "s" })); + } + // Per sheet: `Sheet!A1,B2:C5` (ranges inside another listed one dropped), + // with `,…` when it also reaches cells whose address is built at run time. + for (write, verb) in [(true, "writes"), (false, "uses")] { + let mut by_sheet: BTreeMap<&At, (bool, Vec)> = BTreeMap::new(); + for t in p.touches.iter().filter(|t| t.write == write) { + let e = by_sheet.entry(&t.at).or_default(); + match t.rect { + Some(r) => e.1.push(r), + None => e.0 = true, + } + } + let inside = |r: &Rect, o: &Rect| o != r && o.r0 <= r.r0 && r.r1 <= o.r1 && o.c0 <= r.c0 && r.c1 <= o.c1; + let sheets: Vec = by_sheet + .into_iter() + .map(|(at, (whole, rects))| { + let rects: Vec = rects.iter().filter(|r| !rects.iter().any(|o| inside(r, o))).map(|r| r.a1()).collect(); + let mut s = sheet_label(wb, at); + if !rects.is_empty() { + let more = rects.len().saturating_sub(max); + s += &format!("!{}", rects.into_iter().take(max).collect::>().join(",")); + if more > 0 { + s += &format!(",+{more}"); + } + if whole { + s += ",…"; + } + } + if matches!(at, At::Missing(_)) { + s += " (not in file)"; + } + s + }) + .collect(); + if !sheets.is_empty() { + parts.push(format!("{verb} {}", list(sheets, "; "))); + } + } + if !p.calls.is_empty() { + parts.push(format!("calls {}", list(p.calls.iter().map(|&c| a.macros[c].name.clone()).collect(), ", "))); + } + (!parts.is_empty()).then(|| parts.join(" · ")) +} + +fn vba_trigger(name: &str, me: Option) -> Option { + let lower = name.to_ascii_lowercase(); + if let Some(ev) = lower.strip_prefix("workbook_") { + return Some(Trigger { event: name[name.len() - ev.len()..].to_string(), sheet: None }); + } + if let Some(ev) = lower.strip_prefix("worksheet_") + && me.is_some() + { + return Some(Trigger { event: name[name.len() - ev.len()..].to_string(), sheet: me }); + } + match lower.as_str() { + "auto_open" => Some(Trigger { event: "Open".into(), sheet: None }), + "auto_close" => Some(Trigger { event: "BeforeClose".into(), sheet: None }), + _ => None, + } +} + +fn js_function(line: &str) -> Option { + let t = line.trim_start(); + let rest = ["async function ", "function "].iter().find_map(|k| t.strip_prefix(k))?; + let name: String = rest.trim_start().chars().take_while(|c| c.is_alphanumeric() || *c == '_' || *c == '$').collect(); + (!name.is_empty()).then_some(name) +} + +// ---------------------------------------------------------------- tokens + +#[derive(Debug, Clone, PartialEq)] +enum Tok { + Id(String), + Str(String), + Num(f64), + Op(String), +} + +fn id_is(t: Option<&Tok>, w: &str) -> bool { + matches!(t, Some(Tok::Id(s)) if s.eq_ignore_ascii_case(w)) +} + +fn op_is(t: Option<&Tok>, o: &str) -> bool { + matches!(t, Some(Tok::Op(s)) if s == o) +} + +/// One line of VBA, up to its comment. +fn tokenize(line: &str) -> Vec { + let cs: Vec = line.chars().collect(); + let mut out = Vec::new(); + let mut i = 0; + while i < cs.len() { + let c = cs[i]; + if c.is_whitespace() { + i += 1; + } else if c == '\'' { + break; + } else if c == '"' { + let mut s = String::new(); + i += 1; + while i < cs.len() { + if cs[i] == '"' { + if cs.get(i + 1) == Some(&'"') { + s.push('"'); + i += 2; + continue; + } + i += 1; + break; + } + s.push(cs[i]); + i += 1; + } + out.push(Tok::Str(s)); + } else if c.is_ascii_digit() { + let st = i; + while i < cs.len() && (cs[i].is_ascii_digit() || cs[i] == '.') { + i += 1; + } + out.push(Tok::Num(cs[st..i].iter().collect::().parse().unwrap_or(f64::NAN))); + } else if c.is_alphabetic() || c == '_' { + let st = i; + while i < cs.len() && (cs[i].is_alphanumeric() || cs[i] == '_') { + i += 1; + } + let id: String = cs[st..i].iter().collect(); + if cs.get(i) == Some(&'$') { + i += 1; // Left$(…) + } + if id.eq_ignore_ascii_case("rem") && out.is_empty() { + break; + } + out.push(Tok::Id(id)); + } else { + let two: String = cs[i..(i + 2).min(cs.len())].iter().collect(); + if [":=", "<=", ">=", "<>"].contains(&two.as_str()) { + out.push(Tok::Op(two)); + i += 2; + } else { + out.push(Tok::Op(c.to_string())); + i += 1; + } + } + } + out +} + +/// A module's statements with the line each starts on: continuation lines +/// (` _`) joined, `:`-separated statements split. +fn statements(src: &str) -> Vec<(usize, Vec)> { + let mut out = Vec::new(); + let mut cur: Vec = Vec::new(); + let mut start = 0; + for (n, line) in src.lines().enumerate() { + if cur.is_empty() { + start = n + 1; + } + let mut toks = tokenize(line); + let cont = matches!(toks.last(), Some(Tok::Id(s)) if s == "_"); + if cont { + toks.pop(); + } + cur.extend(toks); + if !cont { + for st in std::mem::take(&mut cur).split(|t| op_is(Some(t), ":")) { + if !st.is_empty() { + out.push((start, st.to_vec())); + } + } + } + } + if !cur.is_empty() { + out.push((start, cur)); + } + out +} + +/// `[Public|Private] Const Name [As String] = "text"`: (lowercased name, text, public). +fn string_const(t: &[Tok]) -> Option<(String, String, bool)> { + let public = id_is(t.first(), "public") || id_is(t.first(), "global"); + let i = usize::from(public || id_is(t.first(), "private")); + if !id_is(t.get(i), "const") { + return None; + } + let Some(Tok::Id(name)) = t.get(i + 1) else { return None }; + let eq = t.iter().position(|x| op_is(Some(x), "="))?; + match &t[eq + 1..] { + [Tok::Str(v)] => Some((name.to_ascii_lowercase(), v.clone(), public)), + _ => None, + } +} + +/// `[Public|Private|Friend|Static] Sub|Function|Property Get|Let|Set Name(params)`. +fn proc_header(t: &[Tok]) -> Option<(&'static str, String, Vec, bool)> { + let mut i = 0; + while ["public", "private", "friend", "static"].iter().any(|k| id_is(t.get(i), k)) { + i += 1; + } + let kind = if id_is(t.get(i), "sub") { + "sub" + } else if id_is(t.get(i), "function") { + "function" + } else if id_is(t.get(i), "property") { + i += 1; + "property" + } else { + return None; + }; + let Some(Tok::Id(name)) = t.get(i + 1) else { return None }; + let mut params = Vec::new(); + let mut close = i + 1; + if op_is(t.get(i + 2), "(") { + close = close_paren(t, i + 2); + for arg in split_args(&t[i + 3..close]) { + let mut j = 0; + while ["optional", "byval", "byref", "paramarray"].iter().any(|k| id_is(arg.get(j), k)) { + j += 1; + } + if let Some(Tok::Id(p)) = arg.get(j) { + params.push(p.to_ascii_lowercase()); + } + } + } + let sheet = id_is(t.get(close + 1), "as") && id_is(t.get(close + 2), "worksheet"); + Some((kind, name.clone(), params, sheet)) +} + +/// Index of the `)` closing the `(` at `open`, or the end. +fn close_paren(t: &[Tok], open: usize) -> usize { + let mut depth = 0; + for (i, tok) in t.iter().enumerate().skip(open) { + if op_is(Some(tok), "(") { + depth += 1; + } else if op_is(Some(tok), ")") { + depth -= 1; + if depth == 0 { + return i; + } + } + } + t.len() +} + +fn split_args(t: &[Tok]) -> Vec<&[Tok]> { + if t.is_empty() { + return Vec::new(); + } + let mut out = Vec::new(); + let (mut depth, mut start) = (0i32, 0); + for (i, tok) in t.iter().enumerate() { + match tok { + Tok::Op(o) if o == "(" => depth += 1, + Tok::Op(o) if o == ")" => depth -= 1, + Tok::Op(o) if o == "," && depth == 0 => { + out.push(&t[start..i]); + start = i + 1; + } + _ => {} + } + } + out.push(&t[start..]); + out +} + +// ---------------------------------------------------------------- resolution + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct Place { + at: At, + rect: Option, +} + +/// What a variable, parameter or expression refers to. +#[derive(Debug, Clone, Default, PartialEq)] +struct Binding { + places: BTreeSet, + /// A range rather than a whole sheet, so using it bare reads or writes cells. + range: bool, +} + +impl Binding { + fn sheet(at: At) -> Binding { + Binding { places: [Place { at, rect: None }].into(), range: false } + } + fn merge(&mut self, o: &Binding) { + self.places.extend(o.places.iter().cloned()); + self.range |= o.range; + } +} + +struct Hit { + b: Binding, + write: bool, + /// Reaches cells or the sheet itself, not just names a sheet to pass on. + counts: bool, + end: usize, +} + +#[derive(Default)] +struct Out { + touches: Vec, + /// (callee, what each argument refers to) + calls: Vec<(usize, Vec>)>, +} + +struct Scope<'a> { + wb: &'a Workbook, + a: &'a Analysis, + code_names: &'a HashMap, + by_name: &'a HashMap>, + procs: &'a [Proc], + /// The sheet whose module this is: `Me`, and where a bare `Range` points. + me: Option, + this: usize, + vars: HashMap, + with: Vec, +} + +impl Scope<'_> { + fn statement(&mut self, t: &[Tok], out: &mut Out) { + let head = match t.first() { + Some(Tok::Id(w)) => w.to_ascii_lowercase(), + _ => String::new(), + }; + match head.as_str() { + // The target of `With` / `Set` is reached through what follows, + // so only calls in it are recorded here. + "with" => { + let b = self.whole(&t[1..]); + self.scan(&t[1..], None, out, b.is_none()); + self.with.push(b.unwrap_or_default()); + } + "end" => { + if id_is(t.get(1), "with") { + self.with.pop(); + } + } + "set" if matches!(t.get(1), Some(Tok::Id(_))) && op_is(t.get(2), "=") => { + let Some(Tok::Id(v)) = t.get(1) else { return }; + let b = self.whole(&t[3..]); + self.scan(&t[3..], None, out, b.is_none()); + match b { + Some(b) => self.vars.insert(v.to_ascii_lowercase(), b), + None => self.vars.remove(&v.to_ascii_lowercase()), + }; + } + "dim" | "redim" | "const" | "attribute" | "option" | "declare" | "public" | "private" | "global" => {} + "if" | "elseif" => { + let then = t.iter().position(|x| id_is(Some(x), "then")).unwrap_or(t.len()); + self.scan(&t[1..then], None, out, true); + if then + 1 < t.len() { + self.statement(&t[then + 1..], out); + } + } + "else" => self.statement(&t[1..], out), + h if NO_ASSIGN.contains(&h) => self.scan(t, None, out, true), + _ => { + // A single-line If's `… Else …` tail is its own statement. + if let Some(e) = t.iter().position(|x| id_is(Some(x), "else")) { + self.statement(&t[..e], out); + self.statement(&t[e..], out); + return; + } + // `ws.Name = "Out"`: ws now holds the sheet called Out. + if let [Tok::Id(v), Tok::Op(d), Tok::Id(n), Tok::Op(e), Tok::Str(name)] = t + && d == "." + && n.eq_ignore_ascii_case("name") + && e == "=" + { + let at = sheet_index(self.wb, name).map_or_else(|| At::Missing(name.clone()), At::Sheet); + self.vars.insert(v.to_ascii_lowercase(), Binding::sheet(at)); + } + let mut depth = 0; + let eq = t.iter().position(|x| { + match x { + Tok::Op(o) if o == "(" => depth += 1, + Tok::Op(o) if o == ")" => depth -= 1, + _ => {} + } + depth == 0 && op_is(Some(x), "=") + }); + self.scan(t, eq, out, true); + } + } + } + + /// Record the calls in `t` and, with `touches`, every reference; those + /// before `eq` are assigned to. + fn scan(&self, t: &[Tok], eq: Option, out: &mut Out, touches: bool) { + let mut i = 0; + while i < t.len() { + if let Tok::Id(w) = &t[i] + && !op_is(i.checked_sub(1).and_then(|p| t.get(p)), ".") + && let Some(callees) = self.callees(w) + { + let args = if op_is(t.get(i + 1), "(") { + split_args(&t[i + 2..close_paren(t, i + 1).min(t.len())]) + } else if i == 0 || id_is(t.get(i - 1), "call") { + split_args(&t[i + 1..]) + } else { + Vec::new() + }; + let bound: Vec> = args.iter().map(|a| self.whole(a)).collect(); + for c in callees { + out.calls.push((c, bound.clone())); + } + } + match self.resolve(t, i) { + Some(h) => { + if touches && h.counts { + let write = h.write || eq.is_some_and(|e| i < e); + out.touches.extend(h.b.places.iter().map(|p| Touch { at: p.at.clone(), rect: p.rect, write })); + } + i = h.end.max(i + 1); + } + None => i += 1, + } + } + } + + /// Procedures `w` may call: the one in this module, else any of that name. + fn callees(&self, w: &str) -> Option> { + let all = self.by_name.get(&w.to_ascii_lowercase())?; + let mine = self.procs[self.this].script; + let same: Vec = all.iter().copied().filter(|&c| self.procs[c].script == mine).collect(); + let v: Vec = if same.is_empty() { all.clone() } else { same }; + let v: Vec = v.into_iter().filter(|&c| c != self.this).collect(); + (!v.is_empty()).then_some(v) + } + + /// What `t` as a whole refers to, allowing a workbook prefix + /// (`ThisWorkbook.Worksheets("Data")`). + fn whole(&self, t: &[Tok]) -> Option { + for k in 0..t.len() { + if let Some(h) = self.resolve(t, k) { + return (h.end == t.len()).then_some(h.b); + } + if !matches!(t[k], Tok::Id(_)) && !op_is(Some(&t[k]), ".") { + return None; + } + } + None + } + + /// A reference starting at `t[i]`: its base (sheet expression, variable, + /// code name, `With` target, bare `Range`) and the member chain after it. + fn resolve(&self, t: &[Tok], i: usize) -> Option { + let prev_dot = i > 0 && op_is(t.get(i - 1), "."); + let (mut b, mut j, mut counts, mut write) = match &t[i] { + Tok::Op(o) if o == "." => { + let leading = i == 0 || matches!(&t[i - 1], Tok::Op(p) if p != ")"); + let top = self.with.last().filter(|b| leading && !b.places.is_empty())?; + (top.clone(), i, false, false) + } + Tok::Id(w) => { + let lw = w.to_ascii_lowercase(); + if lw == "worksheets" || lw == "sheets" { + let at = self.sheet_arg(t, i + 1, true, true)?; + (Binding::sheet(at), i + 4, true, false) + } else if prev_dot { + return None; + } else if lw == "range" || lw == "cells" { + let mut b = Binding::sheet(self.me_at()); + let mut write = false; + let j = self.member(&mut b, &mut write, t, i); + (b, j, true, write) + } else if lw == "me" { + (Binding::sheet(At::Sheet(self.me?)), i + 1, false, false) + } else if lw == "activesheet" { + (Binding::sheet(At::Active), i + 1, false, false) + } else if let Some(v) = self.vars.get(&lw) { + (v.clone(), i + 1, v.range, false) + } else if let Some(&s) = self.code_names.get(&lw) { + (Binding::sheet(At::Sheet(s)), i + 1, false, false) + } else { + // A helper given a sheet's name: SheetByName("Data") + let creates = self.callees(w).is_some_and(|c| c.iter().any(|&c| self.procs[c].returns_sheet)); + (Binding::sheet(self.sheet_arg(t, i + 1, false, creates)?), i + 4, true, false) + } + } + _ => return None, + }; + while op_is(t.get(j), ".") && matches!(t.get(j + 1), Some(Tok::Id(_))) { + counts = true; + j = self.member(&mut b, &mut write, t, j + 1); + } + Some(Hit { b, write, counts, end: j }) + } + + /// Apply the member at `t[m]` (with its arguments) to `b`; returns the index after it. + fn member(&self, b: &mut Binding, write: &mut bool, t: &[Tok], m: usize) -> usize { + let Some(Tok::Id(name)) = t.get(m) else { return m + 1 }; + let name = name.to_ascii_lowercase(); + let (args, next) = if op_is(t.get(m + 1), "(") { + let c = close_paren(t, m + 1); + (Some(&t[m + 2..c.min(t.len())]), c + 1) + } else { + (None, m + 1) + }; + if (name == "range" || name == "cells") && !b.range { + let (sheet, rect) = self.range_args(&name, args); + b.places = b.places.iter().map(|p| Place { at: sheet.map_or_else(|| p.at.clone(), At::Sheet), rect }).collect(); + b.range = true; + } else if RELATIVE.contains(&name.as_str()) || ((name == "range" || name == "cells") && args.is_some()) { + b.places = b.places.iter().map(|p| Place { at: p.at.clone(), rect: None }).collect(); + b.range = true; + } else if MUTATORS.contains(&name.as_str()) { + *write = true; + } + next + } + + fn me_at(&self) -> At { + self.me.map_or(At::Active, At::Sheet) + } + + /// `("Name")` naming a sheet of this workbook (or, when `sure` it names a + /// sheet, any name), or `(3)` with `sure`. + fn sheet_arg(&self, t: &[Tok], k: usize, numeric: bool, sure: bool) -> Option { + if !op_is(t.get(k), "(") || !op_is(t.get(k + 2), ")") { + return None; + } + match t.get(k + 1)? { + Tok::Str(s) => match sheet_index(self.wb, s) { + Some(i) => Some(At::Sheet(i)), + None => sure.then(|| At::Missing(s.clone())), + }, + Tok::Num(n) if numeric && *n >= 1.0 && (*n as usize) <= self.wb.sheets.len() => Some(At::Sheet(*n as usize - 1)), + _ => None, + } + } + + /// The sheet (from a named range) and address of `Range(…)` / `Cells(…)` + /// arguments, when they are written out. + fn range_args(&self, kind: &str, args: Option<&[Tok]>) -> (Option, Option) { + let Some(args) = args else { return (None, None) }; + let parts = split_args(args); + if kind == "cells" { + return (None, cells_rect(&parts)); + } + let (mut sheet, mut acc): (Option, Option) = (None, None); + for p in &parts { + let r = match p { + [Tok::Str(s)] => match parse_rect(s) { + Some(r) => r, + None => { + let Some(&(s, r)) = self.a.named.get(&s.to_ascii_uppercase()) else { return (None, None) }; + sheet = s.or(sheet); + r + } + }, + // Cells(1, 2), or .Cells(1, 2) inside a With + [Tok::Id(c), Tok::Op(o), rest @ ..] | [Tok::Op(_), Tok::Id(c), Tok::Op(o), rest @ ..] + if c.eq_ignore_ascii_case("cells") && o == "(" && !matches!(&p[0], Tok::Op(x) if x != ".") => + { + match cells_rect(&split_args(&rest[..rest.len().saturating_sub(1)])) { + Some(r) => r, + None => return (None, None), + } + } + _ => return (None, None), + }; + acc = Some(acc.map_or(r, |a| a.union(&r))); + } + (sheet, acc) + } +} + +/// `Cells(2, 3)` / `Cells(2, "C")` with literal arguments. +fn cells_rect(parts: &[&[Tok]]) -> Option { + let [[Tok::Num(r)], [col]] = parts else { return None }; + let c = match col { + Tok::Num(c) if *c >= 1.0 => *c as u32 - 1, + Tok::Str(s) => col_index(s)?, + _ => return None, + }; + (*r >= 1.0).then(|| Rect::cell(*r as u32 - 1, c)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn statements_join_continuations_and_drop_comments() { + let st = statements("x = \"it's \"\"quoted\"\"\" ' comment\nIf a Then _\n b = 1: c = 2\nRem gone"); + assert_eq!(st.len(), 3); + assert_eq!(st[0].1[2], Tok::Str("it's \"quoted\"".into())); + assert_eq!((st[1].0, st[1].1.len()), (2, 6)); + assert_eq!((st[2].0, st[2].1.len()), (2, 3)); + } + + #[test] + fn string_consts() { + let c = |src: &str| string_const(&statements(src)[0].1); + assert_eq!(c("Private Const RAW As String = \"Raw Data\""), Some(("raw".into(), "Raw Data".into(), false))); + assert_eq!(c("Public Const OUT = \"Out\""), Some(("out".into(), "Out".into(), true))); + assert_eq!(c("Const N As Long = 252"), None); + assert_eq!(c("Const A = \"x\" & \"y\""), None); + } + + #[test] + fn headers() { + let (kind, name, ps, _) = proc_header(&tokenize("Private Sub Totals(ByVal ws As Worksheet, Optional n As Long = 2)")).unwrap(); + assert_eq!((kind, name.as_str(), ps), ("sub", "Totals", vec!["ws".to_string(), "n".to_string()])); + assert!(proc_header(&tokenize("Private Function FreshSheet(nm As String) As Worksheet")).unwrap().3); + assert_eq!(proc_header(&tokenize("Public Property Get Rate() As Double")).unwrap().1, "Rate"); + assert!(proc_header(&tokenize("Private Declare PtrSafe Function GetTickCount Lib \"kernel32\" () As Long")).is_none()); + assert_eq!(js_function(" async function onEdit(e) {").as_deref(), Some("onEdit")); + } +} diff --git a/src/main.rs b/src/main.rs index 2baf460..a14a986 100644 --- a/src/main.rs +++ b/src/main.rs @@ -55,7 +55,14 @@ enum Cmd { #[arg(long)] down: bool, }, - /// Search values, formulas and notes. + /// List macro / script modules and procedures with the sheets and ranges each + /// writes or uses; with NAME (module, procedure or Module.Procedure), print its code. + #[command(visible_alias = "macros")] + Macro { + dir: PathBuf, + name: Option, + }, + /// Search values, formulas, notes and macro code. Find { dir: PathBuf, text: String, @@ -178,6 +185,11 @@ fn main() -> Result<()> { let a = analyze::analyze(&wb); emit(&query::find(&wb, &a, &text, limit))?; } + Cmd::Macro { dir, name } => { + let wb = store::load(&dir)?; + let a = analyze::analyze(&wb); + emit(&query::macros(&wb, &a, name.as_deref())?)?; + } Cmd::Rebuild { out, full } => { let opts = if full { report::Opts::full() } else { report::Opts::default() }; for e in std::fs::read_dir(&out)? { diff --git a/src/model.rs b/src/model.rs index 51ad275..1629f53 100644 --- a/src/model.rs +++ b/src/model.rs @@ -66,6 +66,10 @@ pub struct Sheet { pub frozen_cols: u32, #[serde(default, skip_serializing_if = "Option::is_none")] pub tab_color: Option, + /// The sheet's VBA code name (`Sheet1`), which macros can use in place of + /// `Worksheets("Data")` and which names the sheet's own code module. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_name: Option, /// Sparse, sorted by (row, col). pub cells: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -373,6 +377,17 @@ pub struct Script { pub source: String, } +impl Script { + /// Whether the module holds any code. Excel gives every sheet and the + /// workbook a module, and most stay `Attribute …` / `Option …` lines only. + pub fn has_code(&self) -> bool { + self.source.lines().map(str::trim).any(|l| { + let lower = l.to_ascii_lowercase(); + !(l.is_empty() || l.starts_with('\'') || lower.starts_with("attribute ") || lower.starts_with("option ") || lower.starts_with("rem ")) + }) + } +} + fn is_false(b: &bool) -> bool { !*b } diff --git a/src/ooxml.rs b/src/ooxml.rs index b5cc6e7..d1b950e 100644 --- a/src/ooxml.rs +++ b/src/ooxml.rs @@ -449,7 +449,10 @@ fn read_sheet(pkg: &mut Pkg, part: &str, kind: &str, sh: &mut Sheet, cx: &mut Sh for n in &sx.nodes { match n.name.as_str() { - "sheetPr" => sh.tab_color = n.child("tabColor").and_then(|c| cx.styles.colors.of(c)), + "sheetPr" => { + sh.tab_color = n.child("tabColor").and_then(|c| cx.styles.colors.of(c)); + sh.code_name = n.attr("codeName").map(str::to_string); + } "sheetViews" => { if let Some(p) = n.child("sheetView").and_then(|v| v.child("pane")).filter(|p| p.attr("state").is_some_and(|s| s.starts_with("frozen"))) { sh.frozen_rows = p.num("ySplit").unwrap_or(0); diff --git a/src/query.rs b/src/query.rs index 13e2137..4227523 100644 --- a/src/query.rs +++ b/src/query.rs @@ -283,6 +283,17 @@ pub fn find(wb: &Workbook, a: &Analysis, needle: &str, limit: usize) -> String { } } } + for (i, sc) in wb.scripts.iter().enumerate() { + for (ln, line) in sc.source.lines().enumerate() { + if line.to_lowercase().contains(&n) { + hits += 1; + if hits <= limit { + let p = a.macros.iter().find(|p| p.script == i && p.lines.0 <= ln + 1 && ln < p.lines.1); + let _ = writeln!(s, "{} {}:{}{}: {}", sc.kind, sc.name, ln + 1, p.map(|p| format!(" ({})", p.name)).unwrap_or_default(), trunc(line.trim(), 160)); + } + } + } + } if hits > limit { let _ = writeln!(s, "… {} hits total (raise --limit)", hits); } @@ -310,3 +321,61 @@ pub fn find(wb: &Workbook, a: &Analysis, needle: &str, limit: usize) -> String { } s } + +/// Without `name`, every module and procedure with what each touches. With +/// it, the code of a module, a procedure, or `Module.Procedure`, numbered. +pub fn macros(wb: &Workbook, a: &Analysis, name: Option<&str>) -> Result { + let mut s = String::new(); + let Some(name) = name else { + if wb.scripts.is_empty() { + return Ok("no macros or scripts in this workbook\n".into()); + } + let mut empty = Vec::new(); + for (i, sc) in wb.scripts.iter().enumerate() { + if !sc.has_code() { + empty.push(sc.name.as_str()); + continue; + } + let _ = writeln!(s, "{} {} ({} lines)", sc.kind, sc.name, sc.source.lines().count()); + for p in a.macros.iter().filter(|p| p.script == i) { + let d = crate::macros::describe(wb, a, p, usize::MAX).map(|d| format!(" · {d}")).unwrap_or_default(); + let _ = writeln!(s, " {} {} lines {}-{}{d}", p.kind, p.name, p.lines.0, p.lines.1); + } + } + if !empty.is_empty() { + let _ = writeln!(s, "no code: {}", empty.join(", ")); + } + return Ok(s); + }; + let numbered = |s: &mut String, src: &str, from: usize, to: usize| { + for (n, line) in src.lines().enumerate().skip(from - 1).take(to + 1 - from) { + let _ = writeln!(s, "{:>5} {}", n + 1, line); + } + }; + if let Some(sc) = wb.scripts.iter().find(|sc| sc.name.eq_ignore_ascii_case(name)) { + let _ = writeln!(s, "{} {} ({} lines)", sc.kind, sc.name, sc.source.lines().count()); + numbered(&mut s, &sc.source, 1, sc.source.lines().count().max(1)); + return Ok(s); + } + let (module, proc) = match name.split_once('.') { + Some((m, p)) => (Some(m), p), + None => (None, name), + }; + let found: Vec<_> = a + .macros + .iter() + .filter(|p| p.name.eq_ignore_ascii_case(proc) && module.is_none_or(|m| wb.scripts[p.script].name.eq_ignore_ascii_case(m))) + .collect(); + if found.is_empty() { + let mut names: Vec = wb.scripts.iter().filter(|sc| sc.has_code()).map(|sc| sc.name.clone()).collect(); + names.extend(a.macros.iter().map(|p| format!("{}.{}", wb.scripts[p.script].name, p.name))); + return Err(anyhow!("no module or procedure {name:?}; have: {}", trunc(&names.join(", "), 600))); + } + for p in found { + let sc = &wb.scripts[p.script]; + let d = crate::macros::describe(wb, a, p, usize::MAX).map(|d| format!("\n{d}")).unwrap_or_default(); + let _ = writeln!(s, "{} {}.{} lines {}-{}{d}", p.kind, sc.name, p.name, p.lines.0, p.lines.1); + numbered(&mut s, &sc.source, p.lines.0, p.lines.1); + } + Ok(s) +} diff --git a/src/report.rs b/src/report.rs index cb6bf22..66c7473 100644 --- a/src/report.rs +++ b/src/report.rs @@ -251,9 +251,25 @@ pub fn render(wb: &Workbook, a: &Analysis, o: &Opts) -> String { if !wb.scripts.is_empty() { let _ = writeln!(s, "\n## Scripts"); - for sc in &wb.scripts { - let procs = procedures(&sc.source); - let _ = writeln!(s, "{} {} ({} lines): {}", sc.kind, sc.name, sc.source.lines().count(), trunc(&procs.join(", "), 400)); + let mut empty = Vec::new(); + for (i, sc) in wb.scripts.iter().enumerate() { + if !sc.has_code() { + empty.push(sc.name.as_str()); + continue; + } + let procs: Vec<_> = a.macros.iter().filter(|p| p.script == i).collect(); + let names: Vec<&str> = procs.iter().map(|p| p.name.as_str()).collect(); + let _ = writeln!(s, "{} {} ({} lines): {}", sc.kind, sc.name, sc.source.lines().count(), trunc(&names.join(", "), 400)); + for p in procs { + if let Some(d) = crate::macros::describe(wb, a, p, 6) { + let _ = writeln!(s, " {}: {}", p.name, trunc(&d, 300)); + } + } + } + if !empty.is_empty() { + let n = empty.len(); + let names = if n > 8 { format!("{} …", empty[..8].join(", ")) } else { empty.join(", ") }; + let _ = writeln!(s, "{n} module{} with no code: {names}", if n == 1 { "" } else { "s" }); } } if !wb.developer_metadata.is_empty() { @@ -276,23 +292,6 @@ pub fn render(wb: &Workbook, a: &Analysis, o: &Opts) -> String { s } -fn procedures(src: &str) -> Vec { - let mut out = Vec::new(); - for line in src.lines() { - let t = line.trim_start(); - let lower = t.to_ascii_lowercase(); - for kw in ["function ", "sub ", "public sub ", "private sub ", "public function ", "private function ", "async function "] { - if lower.starts_with(kw) { - let name: String = t[kw.len()..].chars().take_while(|c| c.is_alphanumeric() || *c == '_').collect(); - if !name.is_empty() { - out.push(name); - } - } - } - } - out -} - // ---------------------------------------------------------------- sheet pub fn render_sheet(wb: &Workbook, a: &Analysis, si: usize, o: &Opts) -> String { diff --git a/tests/fixtures/macros.xlsm b/tests/fixtures/macros.xlsm new file mode 100644 index 0000000000000000000000000000000000000000..2f0d363d7be678ad67a45d0043d817fc3a3344d3 GIT binary patch literal 7448 zcmZ`;1yCJZlfF1Xg1ZI{?(Xh{;O=tqi+hmZ?!gHJcL)x_E!f2Z1lQmmAo$+Ax4U_d z-G8cPPSwg2Ujea{4RN@bA_FlfA9d$HW+@fNsd{~w zD5SU~y>WgF;<8g3)+FhqIa;QO1HK05$e4h2x6p$okuV3h$2B@%_w$H>etgW6M#!ik z2XD#sq2DHfWPKM+qFa0HP}#l&S)?CE7u55}ZnO_Z(EAzKhNNiejnD;&OO}*~Db9vZ zx;jH;dk*m$k{vS}*YGKr@A2P?wl{<~UA(m2`yACJji1ypv4!z<#ls`~VSu5cD&!(q z#Lq4)JD%9#I__pCC+@Kcm;DYbE`PGf@GBMR5c}NvI3@%R=V4gPI^OK#a?6{#nd7>< z`GonDuCOf_jp6T#FP}bbl^v9u4Gtzy1LA{@CnV&_wOUxHcR&#RdEE^;7-!c$HTQ#BgrM|WbE#X`FQ);|63 znp~BDI$Jv99H)G3dM*V~raaF0iOYF(#vd2dLrlF6vrURF_i{gx5)N8j>McBwurHy0 zRrO|2%1h{ecM9N8cp6&*Bh6%FS63X~7jvB!jb2=m$LGsPff#=#6evcyCszySEO!)K z6q=dVd#>(2T}+f<3*B^Pvld27%}KqWajdz4`?sRfgG#a~5Cz8|H#+1tcQjLRb_BVw zn1DdPok|I;gxtf50>9R&>fV$dNm}qyQYbQ~9n9<&(nx0?e0y>PE8^hgHAIxp96>?T zm?m|`yk*b9K8PwhOL;3iGMpEgx8+mCwj3B;ONdrLMherLZ$B3h!iK^-y3tg#H^VG# z#B39NlXrpLT-1!OKP?^cE-5?h3v9S2g_UVSICn@G zGjH!6LF^i8M<%71Mk)iSZ-9`S70E?}=yTN6M#WFEymVOHKtkskXV|xh&9#c%{USs~ z<}_j2VD;5&sm`qC`h&&}n6#k@yaI2B~?04m&?g=!q z!afV;b6XZ(Pc{Cdt{R%<`pI5yO?b9+!Po~)lb4^KXk4=^(Jn=zwE{Iy4_1HD{(En~ zkGdzXKn#Ke0{{?0R!=*Yzx~+5(b?A2(b4v|asKk@{={yF9#(Wo|Ne6X^H-I;L<}UehY)(shbVl4fy5Qm^r%a0+U}RvZR$Z2V(?;mOz#TAjrlHVu+<44f+L4Uyyn z8+b~kN;$J`O-4a}sZ#EF{Pp#bnE~#ll?Eb-%#^K4%21p; z3D&H=-M2(L(G=wvc8;7;^5gZBr!5}(!&dy&WJd;pW|@64PgB7{#j)TbEv^i;BMYWb1#P2ZjnYuSbZA=YL9J}$zW-V?03Z&$vk}fVzRV9 zcS$6tK@S}QxvRAcXg3B>on`AxL%+d~@Hq0!By&_!W;bXDNZrAR9-MXnn9{G`z7^@T%I(;E6J zhI|k;)EcNDpB$GwN!9ue>;EPH;x5+70+w&0v<3+@lp_*_bnClO#WM#3c+HRP^5KRVW<@9V4!UGPh7Z# zk{}0=;VJd-MI9`9pr~|?M7ZJn3&dgaSLnkRzh3!W2#t)dwjAEhIb9I_J0#QfvnbKw z0e~#pzsoozBwc|P_7;CX*?zm=k*=Q00x$Nvlj;dit1Elt0q(%YdMp=f_r?13;3H+S z5De`YJ1_HuVE;}TZuF5<)T2bD)=WWI#=X07DVd#JUayN|X4Bo!@Z@t(m|uUMG9OF>bH3ZB>_v-)d#ge;V^`}K>6hHcSl za)36-eRgyTCe@Jc)-9RvJ4xHJnEt8uF<%76<3K;`to0N@DyOt_c0Alm08ihfqKa>| zSYzZ`PvVq*{4AKRL4^=8V<_-aCg@fFQuV{-hqkW;9NyV}{NuLC{?_GC#Yb(K;G}z( z-L2f>q&X$=aZo$si?cgQJ(Dn62LUW+eqt+eU+~pcLLoIf_~ZuZKv~d=Of$8{01V#B ziFT%JTfWlvyLuzzKsi_1^_gNWS(-{JcZjKV3xl^o4wW5C1Ti}_%|?Y7y5I#{YF_BC z`hg0JH7viT4#c6=nDFaz#|$B@JZ-s!;9uy{gd15dKU^m5?hp#4C$BWxmp*;a#)0~P zzsgz0#n?aVnbLJunI%Yqg*ldt3@;Y?fr4K;lg9qwCHIv3)QWBLY(zy z%3@cVv0V+^2%o% zK5fAhiJ2o#+Z_HeT%Y?RqcaBGog3~jD2hN5Y47^Rzxo9E6LvDeIvnLZ-CuYL=CuBu_orZ)h?5TEa>7%jC6+WDYw zv^kl$xmaY;;80EHL10gz%GYC^hGkVgykb$?tw93T&^xc`uo0@qQ3gNiR1W2$U?}4v zW6GK(;}yy0Ul6Eu>@djnY4CPRaWrCnz;ECDG3cbM){N=ihz8Bx(#BWnZNX+$biT@j z@`6z>TnBSQ$>8YQPG(k>H--D{2aj&!^9RsZ8MJMI9iyKH5>@Z6mN#!5k_2QGv%LB- zpj@S(DN~5yaq-xJ6fyX+)!r;)cc$Tl4st}E*4xZuc7$Plc7lY&|_DytKWN`)+c!kPIoB(IYmfpzy zX-gSNaOv>`wWg;pcKs--6YAduc9rZ)4$oNk__XFBMGV z^f`GG7)jtFfFa{`OqZh{9V9OdPaYm5UPYMI(uTc+2M$On6O8gcJ*TLU6u+ACgq1eI zHzZ$x!(UANfhj1eX8EXTH-FgCQCGX!CE-cr%>V*446N41JXptvgUO^tlq%zPEU(fY zs~NKxeT6c6S;p{mbvmBYgEp(J&l~U!k$rnh21xf^)yKjCur) z?N?JyDLP7gho7f^dLoMZ{O&V~=4~fIug|5wntQwx4u+PO11+`@RMaM+b0t%pN8ooX zfIHTAi2~A54y02?3>HDZT+bO+{1><5w zT6H&4xDGU_*zt)1@0KXb=FLcV#`|t^IhOIr?e&B50~%L;k6a< zqYExB&y7jFy#6H+tlim3iH>ec;y0P&ATE4}cMAj6`K0pee$q+U+x1Jg?amH31zO{g zcFU+}&m%iv-_8kHp9ye3%v$i>Gq1;t&O$pcto?q%ZJIj+b*dtJ_BBY-o=bTV)wv6J zb@O`cxS-SIV`^hHd- zVFdi8yWR-h7rF60GN(c3$K;?o!J3K8*RC7T_#u8QT#PL(9!sm2&W_j4v@EzRMwbf` zGZQlX+M>Ccb-rjUxIU65bH1xwVec<^L7N?(pTdgQWX{%YNH6b@f>9-s61`wp33q7a zcE6H``v&u>*Nt*WVy%DmZ+Ik*Za70>NVE_rQ=2=mEM{? z6j93R#z!;EUuA=fM2L&ZMt8D+Oi8G`napSiyJ$?9F;L$=f)_cytxhuMH3+bBsROQs zli5x&8n~o`532EPG`(XY7`aiyoZ;noVGY=2>AtdZjB-N@1H~eIG2hOj(T2HbTid%d zYs0dZ4Lh%!kM~QbaB9FjnUC>5=GqPAN-{|HSYSTISV?AcNA^{_JW@FI@tOvEh43b7 zB>k$CjKxs1*uXY1q)^8CLA`NjQ7fKBGpf)>*V8U}&mW5&and=8ckZ3e+b~~ma^LYT z;9=>L$rFeUgdL$z2g13#_Nz^JD8LWYi9rlBd2JszE>=~y+D|HK8308<5M4)qbDhlh zkrEXQt$B{W?Se?vuOeq@{))lIir?T+)<^gSpQxPot@+D_Mx4)76k@o0x=0~zqLv*8 z1jN0d$g(%a@2k)#%u@lwnd&6_ip7;Wj+at`eRRS!!mkJh+1*LoNBIs6k@#iA)Qdpn zEW`nnzFl*(e9Tkq`pEe;MG<)%l^t&d;WEIBu}CP{@2V4aVp;F=#6OXnz1@W4l^Oo* zP#U92XRWJUA5$8NpWr5MClhfa=3wfCz?&L8-;VyiNExkq*xE*O#H>21Fk}7UhDC<+ zs{jM;yR^=_`B`t>T>pdj?#TnK-TX#Iw~F~?vE{?b{VmvPTp)N11SXA=CndL)X*y#a z#={>A7*=E@pGgE`X^sTwcq?VHrgGd}UvFR&)1GGaicTqc%jRwf?GvA4Y$lvw+T<-Y z1p~XtdPkwKNAgf+L5Onv8RK}jsISyV2F$#u`~CT0^SP<%D*2e03t4fZOY#^z3rPyq zP1cGEkCa*udLH4lc4hmP#k3k;IpVdNw0t+N?3}ChGd<<`cgdJHBdSz}a3lVZrWft+ zlJQRhAm;6)GK3wuPoXChZ9}4AtMoB}rgAixI8C$cPPQIY1wF6dA-aC&IXFPcu0|?b#a+jo_~z3pv)#?y5{97Q7iRgvTs+!0nX+>C z+>*wga`beUoScEL`Ap0*d2E~&-JKet`|PxJE@N>T*l};xL5yOcFAJOqh?a3mKor0@ zQ(2c&9zMwYYCp8X;D%R&cOj|&Zm@O9{OrZQO+S+c<2wQ|`zVBK!2e@<7odfOs|(9t z%U?ZT$gG}YCM!OKWeaLvn9y_)R)@8YQM1P@FOxQ?Uhacc?@D({JX!^ZCKForYOv-S zEym$c7UPMpE^gdl_hviKco{jq zP{y@OZ!{-Puq$(o-#PVogpkKxj`014uf;nThm$1c$;)3F8ww0UL_yc`Lz%$)`D#VI z3QR4}!9Mixr>rwVtrYx->aA-xl?yY~gyh`Su+9_rV3re{22Z8356spI9HV9=8(}kpn79loww*kLCb2pzs>NTBF8kZg2X-v^ zh=^TBU3)Fr7F@wMG!?wmuraV1i7Qq;`f6j9pBmLId30$54$6@~ikVqyUm%-#OypNB zbt^@+Nk=*w0eS?bMy|HiXjN` z*BE~kN1$|NjeCjU+8nH`4m6PWDU;9<;%iZ19>8*&FPA_LrFY|14ZGL}s>mSQoUPv& zRVrA&aic1yrNtZWH_wmtb~EpmEsT}k2O5%~?R&}{mCUpzAFB_tdpkRu2wu3Y_I=@R zG_Ft`q;wOkR)%-0D{zOXBkxgxMZnG1%9?)D>X>Mlty*k?*AC+37CrdDlTYw|MR(gj)fhC?uTLD<;GjzgrDL9LV+`|ig)LBx zj}A|#kg56OBsJ;2{t_5(1jg*}*92>D6mP_VJ%$IHSCw;u^ataQ1+VjdeGbFH$4Z(1 zI!8r6EEF_ z`t0SpH~uxgvWRVGQjM~~?kIZid`@cv54b3@6FuvuPt@XQ_5dS YK}7-Xcc%*(z##*AA+93wdx!z}FSKPLhX4Qo literal 0 HcmV?d00001 diff --git a/tests/fixtures/make_macros.py b/tests/fixtures/make_macros.py new file mode 100644 index 0000000..17d7239 --- /dev/null +++ b/tests/fixtures/make_macros.py @@ -0,0 +1,316 @@ +"""Build tests/fixtures/macros.xlsm: a small macro-enabled workbook for the VBA +reader. openpyxl writes the sheets; the VBA project (xl/vbaProject.bin) is +written here from scratch, since openpyxl can only copy an existing one. + +vbaProject.bin is a Compound File (MS-CFB) holding the VBA storage: the `dir` +stream lists the modules and each module stream holds its source, compressed +as MS-OVBA 2.4.1 describes. The project carries just what a reader needs to +list modules and read their source (no compiled code, no references). + + pip install openpyxl==3.1.5 + python tests/fixtures/make_macros.py +""" + +import struct +import zipfile +from io import BytesIO +from pathlib import Path + +from openpyxl import Workbook + +OUT = Path(__file__).with_name("macros.xlsm") + +# Module name → (procedural?, source). Sheet1 and Sheet2 are the document +# modules behind the Data and Summary sheets; Excel adds one per sheet, and +# they stay attribute-only unless someone writes event code in them. +DOC_ATTRS = """Attribute VB_Base = "0{00020820-0000-0000-C000-000000000046}" +Attribute VB_GlobalNameSpace = False +Attribute VB_Creatable = False +Attribute VB_PredeclaredId = True +Attribute VB_Exposed = True +Attribute VB_TemplateDerived = False +Attribute VB_Customizable = True +""" + +MODULES = { + "ThisWorkbook": (False, 'Attribute VB_Name = "ThisWorkbook"\n' + DOC_ATTRS + """Option Explicit + +Private Sub Workbook_Open() + RollUp +End Sub +"""), + "Sheet1": (False, 'Attribute VB_Name = "Sheet1"\n' + DOC_ATTRS), + "Sheet2": (False, 'Attribute VB_Name = "Sheet2"\n' + DOC_ATTRS + "Option Explicit\n"), + "Module1": (True, """Attribute VB_Name = "Module1" +Option Explicit + +' Roll the month's revenue up into Summary. +Public Sub RollUp() + Dim src As Worksheet + Set src = ThisWorkbook.Worksheets("Data") + With Worksheets("Summary") + .Range("A1:B1").Value = Array("Region", "Revenue") + .Range("B2:B5").ClearContents + End With + Totals src, Sheet2 + src.Range("A1:F1").Font.Bold = True +End Sub + +Private Sub Totals(ws As Worksheet, dest As Worksheet) + Dim i As Long + For i = 2 To 9 + dest.Cells(2, 2).Value = dest.Cells(2, 2).Value + _ + ws.Cells(i, 5).Value + Next i + MsgBox "Total: " & ws.Range("E10").Value ' "Summary" here is just text +End Sub + +Function Margin(price As Double, cost As Double) As Double + Margin = (price - cost) / price +End Function +"""), +} + + +def build() -> bytes: + wb = Workbook() + wb.code_name = "ThisWorkbook" + data = wb.active + data.title = "Data" + data.sheet_properties.codeName = "Sheet1" + data.append(["Date", "Region", "Units", "Price", "Revenue", "Margin"]) + for i in range(8): + r = i + 2 + data.append([f"2026-01-{i + 5:02}", ["North", "South", "East", "West"][i % 4], 10 + i, 4 + i, f"=C{r}*D{r}", f"=Margin(D{r},3)"]) + summary = wb.create_sheet("Summary") + summary.sheet_properties.codeName = "Sheet2" + summary["A1"], summary["B1"] = "Region", "Revenue" + out = BytesIO() + wb.save(out) + return out.getvalue() + + +# ---------------------------------------------------------------- MS-OVBA + + +def compress(data: bytes) -> bytes: + """MS-OVBA 2.4.1.3.1: a signature byte, then one chunk per 4096 bytes.""" + out = bytearray(b"\x01") + for start in range(0, len(data), 4096): + chunk = data[start:start + 4096] + body = compress_chunk(chunk) + if len(body) > 4096 and len(chunk) == 4096: + out += struct.pack(" bytes: + body = bytearray() + pos = 0 + while pos < len(chunk): + flag_at = len(body) + body.append(0) + for bit in range(8): + if pos >= len(chunk): + break + bit_count = max(4, (pos - 1).bit_length()) if pos > 1 else 4 + max_len = (0xFFFF >> bit_count) + 3 + best_len, best_off = 0, 0 + for cand in range(pos - 1, -1, -1): + n = 0 + while n < max_len and pos + n < len(chunk) and chunk[cand + n] == chunk[pos + n]: + n += 1 + if n > best_len: + best_len, best_off = n, pos - cand + if best_len >= 3: + body[flag_at] |= 1 << bit + body += struct.pack(" bytes: + return struct.pack(" bytes: + """MS-OVBA 2.3.4.2: project information, no references, then the modules.""" + d = record(0x0001, struct.pack(" bytes: + lines = ['ID="{00000000-0000-0000-0000-000000000000}"'] + for name, (procedural, _) in MODULES.items(): + lines.append(f"Module={name}" if procedural else f"Document={name}/&H00000000") + lines += ['Name="VBAProject"', 'HelpContextID="0"', 'VersionCompatible32="393222000"', ""] + lines += ["[Host Extender Info]", "&H00000001={3832D640-CF90-11CF-8E43-00A0C911005A};VBE;&H00000000", ""] + return "\r\n".join(lines).encode("cp1252") + + +def vba_project() -> bytes: + vba = {"dir": compress(dir_stream()), "_VBA_PROJECT": b"\xcc\x61\xff\xff\x00\x00\x00"} + for name, (_, src) in MODULES.items(): + vba[name] = compress(src.replace("\n", "\r\n").encode("cp1252")) + return cfb({"PROJECT": project_stream(), "VBA": vba}) + + +# ---------------------------------------------------------------- MS-CFB + +SECTOR, MINI, CUTOFF = 512, 64, 4096 +FREE, END, FATSECT, NOSTREAM = 0xFFFFFFFF, 0xFFFFFFFE, 0xFFFFFFFD, 0xFFFFFFFF + + +def cfb(tree: dict) -> bytes: + """A version-3 compound file. `tree` maps names to bytes (streams) or + dicts (storages). Streams under 4096 bytes live in the mini stream.""" + entries = [{"name": "Root Entry", "type": 5, "kids": []}] + + def add(parent, items): + for name, v in items.items(): + e = {"name": name, "type": 1 if isinstance(v, dict) else 2, "kids": [], "data": None if isinstance(v, dict) else v} + entries.append(e) + parent["kids"].append(len(entries) - 1) + if isinstance(v, dict): + add(e, v) + + add(entries[0], tree) + + mini = bytearray() + minifat = [] + big = [] # (entry, data) stored in regular sectors + for e in entries[1:]: + if e["type"] != 2: + continue + if len(e["data"]) >= CUTOFF: + big.append(e) + continue + n = max(1, -(-len(e["data"]) // MINI)) + e["start"] = len(minifat) + minifat += [len(minifat) + i + 1 for i in range(n - 1)] + [END] + mini += e["data"].ljust(n * MINI, b"\0") + + # Sector layout: FAT, directory, mini FAT, mini stream, big streams. + dir_secs = -(-len(entries) // 4) + minifat_secs = -(-len(minifat) * 4 // SECTOR) if minifat else 0 + mini_secs = -(-len(mini) // SECTOR) + big_secs = [-(-len(e["data"]) // SECTOR) for e in big] + rest = dir_secs + minifat_secs + mini_secs + sum(big_secs) + fat_secs = 1 + while fat_secs * (SECTOR // 4) < fat_secs + rest: + fat_secs += 1 + assert fat_secs <= 109, "needs DIFAT sectors" + + fat = [FATSECT] * fat_secs + body = bytearray() + + def chain(data: bytes, n: int) -> int: + start = len(fat) + fat.extend([len(fat) + i + 1 for i in range(n - 1)] + [END]) + body.extend(data.ljust(n * SECTOR, b"\0")) + return start + + dir_start = len(fat) + fat.extend([len(fat) + i + 1 for i in range(dir_secs - 1)] + [END]) + dir_at = len(body) + body.extend(b"\0" * dir_secs * SECTOR) + minifat += [FREE] * (minifat_secs * SECTOR // 4 - len(minifat)) + minifat_start = chain(b"".join(struct.pack("= hi: + return NOSTREAM + mid = (lo + hi) // 2 + entries[ids[mid]]["left"] = build(lo, mid) + entries[ids[mid]]["right"] = build(mid + 1, hi) + return ids[mid] + + return build(0, len(ids)) + + for e in entries: + e["child"] = tree_of(e["kids"]) if e["type"] != 2 else NOSTREAM + + raw = bytearray() + for e in entries: + name = e["name"].encode("utf-16-le") + b"\0\0" + raw += name.ljust(64, b"\0") + struct.pack(" bytes: + with zipfile.ZipFile(BytesIO(xlsx)) as z: + files = {n: z.read(n) for n in z.namelist()} + types = files["[Content_Types].xml"].decode() + types = types.replace( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml", + "application/vnd.ms-excel.sheet.macroEnabled.main+xml", + ).replace("", '') + rels = files["xl/_rels/workbook.xml.rels"].decode().replace( + "", + '', + ) + files.update({"[Content_Types].xml": types.encode(), "xl/_rels/workbook.xml.rels": rels.encode(), "xl/vbaProject.bin": vba_project()}) + out = BytesIO() + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as z: + for n in sorted(files, key=lambda n: (n != "[Content_Types].xml", n)): + z.writestr(n, files[n]) + return out.getvalue() + + +if __name__ == "__main__": + OUT.write_bytes(add_vba(build())) + print(f"wrote {OUT}") diff --git a/tests/macros.rs b/tests/macros.rs new file mode 100644 index 0000000..9fdadd4 --- /dev/null +++ b/tests/macros.rs @@ -0,0 +1,190 @@ +//! VBA macros: read from `tests/fixtures/macros.xlsm` (built by +//! `tests/fixtures/make_macros.py`), then what each procedure touches and +//! calls, in the report, graph.json and the `macro` / `find` commands. + +use sheetgraph::a1::parse_rect; +use sheetgraph::analyze::analyze; +use sheetgraph::macros::{describe, formula_cells, At, Proc, Touch}; +use sheetgraph::model::{NamedRange, Script, Sheet, Workbook}; +use std::path::PathBuf; +use std::process::Command; + +const FIXTURE: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/macros.xlsm"); + +fn load() -> Workbook { + sheetgraph::load(FIXTURE, None, false).expect("fixture loads") +} + +fn proc<'a>(procs: &'a [Proc], name: &str) -> &'a Proc { + procs.iter().find(|p| p.name == name).unwrap_or_else(|| panic!("no procedure {name}")) +} + +fn touch(sheet: usize, a1: Option<&str>, write: bool) -> Touch { + Touch { at: At::Sheet(sheet), rect: a1.map(|r| parse_rect(r).unwrap()), write } +} + +#[test] +fn modules_and_code_names() { + let wb = load(); + let names: Vec<(&str, bool)> = wb.scripts.iter().map(|s| (s.name.as_str(), s.has_code())).collect(); + assert_eq!(names, [("Module1", true), ("Sheet1", false), ("Sheet2", false), ("ThisWorkbook", true)]); + assert!(wb.scripts[0].source.contains("Public Sub RollUp()")); + let code: Vec<_> = wb.sheets.iter().map(|s| s.code_name.as_deref()).collect(); + assert_eq!(code, [Some("Sheet1"), Some("Sheet2")]); +} + +#[test] +fn what_each_procedure_touches() { + let wb = load(); + let a = analyze(&wb); + let (data, summary) = (0, 1); + + let roll = proc(&a.macros, "RollUp"); + assert_eq!((roll.kind, roll.lines), ("sub", (5, 14))); + // With Worksheets("Summary") … .Range("A1:B1").Value = …, .Range("B2:B5").ClearContents, + // and src (Set to Worksheets("Data")) … .Font.Bold = True. + assert_eq!( + roll.touches, + [touch(data, Some("A1:F1"), true), touch(summary, Some("A1:B1"), true), touch(summary, Some("B2:B5"), true)] + ); + assert_eq!(roll.calls, [a.macros.iter().position(|p| p.name == "Totals").unwrap()]); + + // Totals(ws, dest) is called with src and the code name Sheet2. The text + // "Summary" in a comment doesn't count; Cells(i, 5) is somewhere on Data. + let totals = proc(&a.macros, "Totals"); + assert_eq!(totals.touches, [touch(data, None, false), touch(data, Some("E10"), false), touch(summary, Some("B2"), true)]); + assert_eq!(describe(&wb, &a, totals, 6).unwrap(), "writes Summary!B2 · uses Data!E10,…"); + + let margin = proc(&a.macros, "Margin"); + assert_eq!((margin.kind, formula_cells(&a, margin)), ("function", 8)); + assert!(margin.touches.is_empty()); + + let open = proc(&a.macros, "Workbook_Open"); + let t = open.trigger.as_ref().expect("an event procedure"); + assert_eq!((t.event.as_str(), t.sheet), ("Open", None)); + assert_eq!(describe(&wb, &a, open, 6).unwrap(), "runs on workbook Open · calls RollUp"); +} + +#[test] +fn report_and_graph() { + let wb = load(); + let a = analyze(&wb); + let report = sheetgraph::report::render(&wb, &a, &sheetgraph::report::Opts::default()); + assert!(report.contains("vba Module1 (27 lines): RollUp, Totals, Margin\n"), "{report}"); + assert!(report.contains(" RollUp: writes Data!A1:F1; Summary!A1:B1,B2:B5 · calls Totals\n"), "{report}"); + assert!(report.contains(" Margin: used in 8 formula cells\n"), "{report}"); + assert!(report.contains("2 modules with no code: Sheet1, Sheet2\n"), "{report}"); + assert!(!report.contains("vba Sheet1"), "{report}"); + + let g = sheetgraph::graph::build(&wb, &a); + let node = g["nodes"].as_array().unwrap().iter().find(|n| n["id"] == "macro:ThisWorkbook.Workbook_Open").expect("macro node"); + assert_eq!((node["type"].as_str(), node["trigger"].as_str(), node["lines"].as_str()), (Some("macro"), Some("Open"), Some("11-13"))); + let edges: Vec<(String, String, String)> = g["edges"] + .as_array() + .unwrap() + .iter() + .filter(|e| e["kind"] != "reads") + .map(|e| (e["from"].as_str().unwrap().into(), e["to"].as_str().unwrap_or("-").into(), e["kind"].as_str().unwrap().into())) + .collect(); + let has = |from: &str, to: &str, kind: &str| edges.iter().any(|e| e == &(from.into(), to.into(), kind.into())); + assert!(has("macro:Module1.RollUp", "s2.b1", "writes"), "{edges:?}"); + assert!(has("macro:Module1.RollUp", "macro:Module1.Totals", "calls"), "{edges:?}"); + assert!(has("macro:ThisWorkbook.Workbook_Open", "macro:Module1.RollUp", "calls"), "{edges:?}"); + // Data!F2:F9 is =Margin(D2,3) + assert!(has("s1.f2", "macro:Module1.Margin", "calls"), "{edges:?}"); +} + +#[test] +fn macro_and_find_commands() { + let out: PathBuf = std::env::temp_dir().join(format!("sheetgraph-macros-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&out); + let run = |args: &[&str]| { + let o = Command::new(env!("CARGO_BIN_EXE_sheetgraph")).args(args).output().unwrap(); + assert!(o.status.success(), "{args:?}: {}", String::from_utf8_lossy(&o.stderr)); + String::from_utf8(o.stdout).unwrap() + }; + run(&["extract", FIXTURE, "--out", out.to_str().unwrap()]); + let dir = out.join("macros"); + let dir = dir.to_str().unwrap(); + + let list = run(&["macro", dir]); + assert!(list.contains(" sub Totals lines 16-23 · writes Summary!B2 · uses Data!E10,…\n"), "{list}"); + assert!(list.ends_with("no code: Sheet1, Sheet2\n"), "{list}"); + + let code = run(&["macro", dir, "module1.totals"]); + assert!(code.starts_with("sub Module1.Totals lines 16-23\n"), "{code}"); + assert!(code.contains(" 16 Private Sub Totals(ws As Worksheet, dest As Worksheet)\n"), "{code}"); + assert!(code.trim_end().ends_with("23 End Sub"), "{code}"); + assert_eq!(run(&["macro", dir, "ThisWorkbook"]).lines().count(), 14, "header and 13 lines"); + + let found = run(&["find", dir, "clearcontents"]); + assert_eq!(found, "vba Module1:10 (RollUp): .Range(\"B2:B5\").ClearContents\n"); + let _ = std::fs::remove_dir_all(&out); +} + +/// Patterns from real workbooks, on a hand-made workbook: string constants as +/// sheet names, a helper `As Worksheet` that creates a sheet, a sheet renamed +/// through its variable, `With` + `.Range(.Cells(…), .Cells(…))`, a single-line +/// `If … Then … Else …`, named ranges, and a sheet's own module. +#[test] +fn resolution_patterns() { + let sheet = |name: &str, code: &str| Sheet { name: name.into(), code_name: Some(code.into()), ..Default::default() }; + let vba = |name: &str, source: &str| Script { kind: "vba".into(), name: name.into(), source: source.into() }; + let wb = Workbook { + sheets: vec![sheet("Raw Data", "Sheet1"), sheet("Calc", "shCalc")], + named_ranges: vec![NamedRange { name: "TaxRate".into(), range: "Calc!$B$2".into() }], + scripts: vec![ + vba( + "Module1", + r#"Private Const RAW As String = "Raw Data" +Private Const OUT_SHEET As String = "Output" + +Sub Build() + Dim src As Worksheet, dst As Worksheet + Set src = ThisWorkbook.Sheets(RAW) + Set dst = FreshSheet(OUT_SHEET) + With shCalc + .Range(.Cells(2, 1), .Cells(9, "C")).Copy + If src.Range("A1").Value = "" Then .Range("D1").Value = 0 Else .Range("D1").Value = 1 + End With + dst.Range("A1").PasteSpecial + Range("TaxRate").Value = 0.2 +End Sub + +Function FreshSheet(nm As String) As Worksheet + Dim ws As Worksheet + Set ws = ThisWorkbook.Worksheets.Add + ws.Name = nm + Set FreshSheet = ws +End Function +"#, + ), + vba("Sheet1", "Private Sub Worksheet_Change(ByVal Target As Range)\n Range(\"A1\").Interior.Color = vbYellow\nEnd Sub\n"), + ], + ..Default::default() + }; + let a = analyze(&wb); + let (raw, calc) = (0, 1); + let r = |s: &str| Some(parse_rect(s).unwrap()); + + let build = proc(&a.macros, "Build"); + assert_eq!( + build.touches, + [ + Touch { at: At::Sheet(raw), rect: r("A1"), write: false }, + Touch { at: At::Sheet(calc), rect: r("D1"), write: true }, + Touch { at: At::Sheet(calc), rect: r("A2:C9"), write: false }, + Touch { at: At::Sheet(calc), rect: r("B2"), write: true }, + Touch { at: At::Missing("Output".into()), rect: r("A1"), write: true }, + ] + ); + assert_eq!( + describe(&wb, &a, build, 6).unwrap(), + "writes Calc!D1,B2; Output!A1 (not in file) · uses 'Raw Data'!A1; Calc!A2:C9 · calls FreshSheet" + ); + + // Unqualified Range in a sheet's own module is that sheet. + let change = proc(&a.macros, "Worksheet_Change"); + assert_eq!(change.trigger.as_ref().map(|t| (t.event.as_str(), t.sheet)), Some(("Change", Some(raw)))); + assert_eq!(change.touches, [Touch { at: At::Sheet(raw), rect: r("A1"), write: true }]); +} From 0ff3814c55ac2c32252932c0f7d8177c5db6548a Mon Sep 17 00:00:00 2001 From: Adhaar Sharma Date: Wed, 28 Oct 2026 19:26:17 -0400 Subject: [PATCH 2/4] Fix 3D and external-ref graph edges; read file settings, shapes and rich cells Graph fixes: - 3D references (Jan:Dec!B2) now link every sheet in the span. - References into another workbook no longer add a broken edge. File level: document and custom properties, calculation mode and iterative settings (Excel and Google), locked structure, data connections with passwords removed, and Power Query code as scripts. Tabs: pictures, shapes, text boxes and form buttons with their anchors; the graph links a button to the macro it runs and a control to its cell. Cells: numbers read as displayed via a number-format renderer, pictures in cells and data types (Excel), people/file chips and rich-text runs (Excel and Google), unlocked cells and hidden formulas. Also guards against a calamine panic on unusual VBA compression, and adds tests/audit.rs over a generated audit workbook as a regression suite. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 20 ++ README.md | 16 +- skills/sheetgraph/SKILL.md | 3 + src/analyze.rs | 27 +- src/formula.rs | 21 ++ src/graph.rs | 20 ++ src/gsheets.rs | 105 +++++- src/lib.rs | 1 + src/macros.rs | 14 + src/model.rs | 137 +++++++- src/numfmt.rs | 278 ++++++++++++++++ src/ooxml.rs | 619 ++++++++++++++++++++++++++++++++++- src/query.rs | 39 ++- src/report.rs | 188 ++++++++++- src/xlsx.rs | 21 +- tests/audit.rs | 168 ++++++++++ tests/fixtures/audit.xlsx | Bin 0 -> 21220 bytes tests/fixtures/make_audit.py | 351 ++++++++++++++++++++ 18 files changed, 1973 insertions(+), 55 deletions(-) create mode 100644 src/numfmt.rs create mode 100644 tests/audit.rs create mode 100644 tests/fixtures/audit.xlsx create mode 100644 tests/fixtures/make_audit.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cd936e8..0decd7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,26 @@ All notable changes to this project are documented here. The format follows - `sheetgraph macro [name]` lists macros or prints one module's or procedure's code; `find` also searches macro code. - Sheets' VBA code names are read from xlsx/xlsm (`Sheet.code_name`). +- File-level settings: document and custom properties, calculation mode and iterative calculation (Excel and + Google), and whether the sheet structure is locked, at the top of REPORT.md. +- Data connections (ODBC, OLE DB, web, text, Power Query) with passwords removed, and Power Query (M) code as + `power_query` scripts that `macro` and `find` read. +- Pictures, shapes, text boxes and form controls over the grid (`Sheet.shapes`), with their text, alt text, the macro + a button runs and a control's linked cell; graph.json gets `shape` nodes with `runs` and `writes` edges. +- Pictures placed in cells and Excel data types (Stocks, Geography) with their fields (`Cell.obj`); Google people and + file chips the same way. +- Text with mixed formatting inside a cell (`Cell.runs`), from Excel and Google (`textFormatRuns`). +- Locked/unlocked cells and hidden formulas (`Style.unlocked`, `Style.hidden_formula`), listed per sheet as the + inputs left editable under protection. +- Numbers in xlsx/xlsm show as their format displays them (`25%`, `$12.50`, `(1,234)`), not only dates. +- `tests/fixtures/audit.xlsx` and `tests/audit.rs`: one workbook with every item on the coverage checklist. + +### Fixed + +- A reference across sheets (`=SUM(Jan:Dec!B2)`) linked only the last sheet; it now links every sheet in the span. +- A reference to another workbook (`'[Other.xlsx]Data'!A1`) also produced a broken `reads` edge in graph.json. +- An xlsm whose VBA uses compression calamine doesn't expect no longer aborts the extraction; the macros are reported + missing and everything else is kept. ### Changed diff --git a/README.md b/README.md index 4f7371a..1d43fd6 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,11 @@ sheetgraph-out/ REPORT.md contains, per workbook: sheet list with used ranges and sheet→sheet data flow; external links (IMPORTRANGE / IMPORTDATA / … with targets resolved from cells, Excel links to other workbooks, refs to deleted sheets, Connected-Sheets data sources); named ranges; function census incl. **non-builtin functions** (Apps Script custom functions / Named Functions / VBA UDFs), dynamic refs (INDIRECT/OFFSET) and volatile functions; scripts (VBA modules from xlsm; Apps Script on request), with each procedure's trigger (`Workbook_Open`, `Worksheet_Change`, `onEdit`), the sheets and ranges it writes and uses, the procedures it calls, and the formula cells that call it as a function; modules with no code (the empty one Excel gives every sheet) are counted, not listed; developer metadata (left by add-ons/integrations). -graph.json edges: `reads` (formula group → the blocks and groups its references land on), `plots` (chart → source), `summarizes` (pivot → source), `imports` (formula → external workbook or URL), `writes` / `uses` (macro → what it changes or reads; a sheet the macro creates has `"to": null` and `missing_sheet`), `calls` (macro → macro, and formula group → the VBA function it calls). +REPORT.md also starts with what changes how the whole file behaves: document properties, calculation mode (manual calculation means stored values may be stale; iterative means circular references are intended), a locked sheet structure, and data connections and Power Query code. -Per sheet: blocks (tables with column profiles + samples, or small blocks verbatim), formula copy-groups and families with flags (`[array/spills]`, `[external]`, `[dynamic ref]`, `[#REF!]`, error counts), data validation (dropdowns, checkboxes, range-sourced lists), conditional formats, visual formatting regions, merges, filters/filter views, protected ranges, hidden rows/cols, banding, Excel tables, charts and pivots (with source ranges), notes and comments, hyperlinks. +graph.json edges: `reads` (formula group → the blocks and groups its references land on), `plots` (chart → source), `summarizes` (pivot → source), `imports` (formula → external workbook or URL), `writes` / `uses` (macro → what it changes or reads; a sheet the macro creates has `"to": null` and `missing_sheet`), `calls` (macro → macro, and formula group → the VBA function it calls), `runs` (button or shape → the macro it runs), and `writes` from a form control to its linked cell. + +Per sheet: blocks (tables with column profiles + samples, or small blocks verbatim), formula copy-groups and families with flags (`[array/spills]`, `[external]`, `[dynamic ref]`, `[#REF!]`, error counts), data validation (dropdowns, checkboxes, range-sourced lists), conditional formats, visual formatting regions, merges, filters/filter views, protected ranges, hidden rows/cols, banding, Excel tables, charts and pivots (with source ranges), pictures, shapes, text boxes and buttons over the grid (with their text and the macro they run), pictures in cells and data types (Excel Stocks/Geography, Google people and file chips), text with mixed formatting, unlocked input cells and hidden formulas, notes and comments, hyperlinks. ## Giving it to an AI agent @@ -158,7 +160,8 @@ Point the agent at `INDEX.md`, then the `REPORT.md` it needs, and let it run the | `model` | source-agnostic full-fidelity workbook (sparse cells, deduped styles/validations) | | `gsheets` | auth (service account JWT / ADC refresh / token), chunked parallel grid fetch with retry, conversion | | `xlsx` | calamine: values, formulas, defined names, merges, VBA; csv | -| `ooxml` | xlsx/xlsm package parts calamine skips: styles, validation, conditional formats, comments, hyperlinks, frozen/hidden rows and cols, filters, protection, tables, charts, pivots, external workbook links | +| `ooxml` | xlsx/xlsm package parts calamine skips: styles, validation, conditional formats, comments, hyperlinks, frozen/hidden rows and cols, filters, protection, tables, charts, pivots, external workbook links, rich text, pictures and data types in cells, shapes and form controls, document properties, calculation settings, connections, Power Query | +| `numfmt` | Excel number formats applied to numbers (`25%`, `$12.50`) | | `formula` | reference lexer (quoted and non-English sheet names, open-ended `A2:A`, whole rows/cols, `'[Book.xlsx]Sheet'`), R1C1 keys, function census | | `analyze` | copy-groups, families, blocks, cross-sheet edges, externals | | `macros` | VBA / Apps Script procedures: triggers, sheets and ranges each writes and uses, calls | @@ -170,10 +173,11 @@ Point the agent at `INDEX.md`, then the `REPORT.md` it needs, and let it run the ## Known gaps - **Apps Script / macros / add-ons on Google Sheets**: the Sheets API doesn't expose bound scripts, installed add-ons or triggers, and the Apps Script API doesn't work with service accounts. `--apps-script ` fetches a project with user OAuth (scope `script.projects.readonly`). Custom functions are still detected from formulas. -- **Local files**: xlsx/xlsm get the same coverage as Google Sheets, plus Excel tables and links to other workbooks (`'[Budget.xlsx]Data'!B2`). Images, shapes, sparklines and slicers aren't read (the report says when a file has them). xls, xlsb and ods get values, formulas, names and merges only. -- Number-format display text (e.g. `$1,234.50`) is only rendered for dates in local files; the format itself is on every cell. +- **Local files**: xlsx/xlsm get the same coverage as Google Sheets, plus Excel tables, links to other workbooks (`'[Budget.xlsx]Data'!B2`), shapes and pictures, connections and Power Query. Sparklines, slicers and ActiveX controls aren't read (the report says when a file has them). xls, xlsb and ods get values, formulas, names and merges only. +- **Pictures on Google Sheets**: the Sheets API doesn't return pictures placed over the grid or inserted into cells (an `=IMAGE()` formula is kept). +- Number-format display text covers the common formats (percent, currency, thousands, decimals, scientific, sections); fractions, conditional sections and formats that mix digits with literals (`000-00-0000`) show the plain value. The format itself is on every cell. - Spill ranges of array formulas are flagged on the formula but the spilled cells look like plain values. -- **What a macro touches** is read from its code, not by running it: literal addresses, sheets held in variables, `With` blocks, sheets passed to other procedures, code names, string constants and named ranges are followed; an address built at run time counts as the whole sheet, and a sheet picked at run time (from a cell, a loop over all sheets) isn't known. Which button or shape runs a macro isn't read, and Apps Script code is listed by function without the ranges it touches. +- **What a macro touches** is read from its code, not by running it: literal addresses, sheets held in variables, `With` blocks, sheets passed to other procedures, code names, string constants and named ranges are followed; an address built at run time counts as the whole sheet, and a sheet picked at run time (from a cell, a loop over all sheets) isn't known. Apps Script code is listed by function without the ranges it touches. - INDIRECT/OFFSET targets are dynamic, so those edges are missing (flagged in the report). - **Very large local files**: up to 10 million cells are kept whole (about 0.2–0.4 GB of memory per million). Past that, a big sheet keeps its last 1,000 rows and its formulas, and summarizes the rows in between per column (counts, min and max, most common values; no sums), so memory stays near 2–3 GB however big the file is (a 160 MB xlsx with 44 million cells: 3 GB, 84 s). REPORT.md's column profiles still count every row and say which rows were summarized; `cells` only lists kept rows, and `find` reports text matches in summarized rows by column. The budget is `xlsx::CELL_BUDGET`; `xlsx::read_limited` takes another. - **Very large Google Sheets** are read whole, without that budget. Google allows up to 20 million cells, so a sheet near the limit can take several GB of memory. diff --git a/skills/sheetgraph/SKILL.md b/skills/sheetgraph/SKILL.md index fb978f3..5bd3923 100644 --- a/skills/sheetgraph/SKILL.md +++ b/skills/sheetgraph/SKILL.md @@ -42,6 +42,9 @@ Notation: - `▸ range table …` is a data block, one line per column: type and range, `enum` for small value sets, `fmt` for the number format, `ƒN` when computed. - `⋯ rows A–B summarized`: a very large sheet's middle rows are summarized per column, not kept. - Flags: `[external]` another file, `[dynamic ref]` INDIRECT/OFFSET (graph edges incomplete), `[array/spills]`, `[#REF!]`, `[N errors]`. +- The lines under the title are file settings. `calculation: manual` means stored values may be stale; `iterative` means circular references are intended, not errors. +- `▣` lines are pictures in cells and data types (stocks, places, people chips); `[image: …]` in a cell is a picture, not text. `✎ mixed formatting` shows runs as `[text]{bold fg#FF0000}`. `unlocked cells` are the inputs a protected sheet leaves editable. +- Shape lines (`button "Run roll-up" at G9:H10 → runs RollUp`, `text box "…"`) are things drawn over the grid; text boxes often hold instructions. - Scripts: one line per macro with its trigger, what it `writes` and `uses`, and what it `calls`. `,…` means more cells whose address is built at run time; `(not in file)` is a sheet the macro creates. Values a macro writes are pasted values in the sheet, not formulas: check here before calling a number hardcoded. ## 4. Drill down for exact values (offline, fast) diff --git a/src/analyze.rs b/src/analyze.rs index 99c5373..f4f28e8 100644 --- a/src/analyze.rs +++ b/src/analyze.rs @@ -173,6 +173,18 @@ pub fn sheet_index(wb: &Workbook, name: &str) -> Option { }) } +/// Sheets a reference's sheet part covers: the one sheet, or every sheet from +/// first to last (in tab order) for a 3D ref like `Jan:Dec`. Empty if unknown. +pub fn sheet_span(wb: &Workbook, name: &str) -> Vec { + if let Some(i) = sheet_index(wb, name) { + return vec![i]; + } + match name.split_once(':').map(|(a, b)| (sheet_index(wb, a), sheet_index(wb, b))) { + Some((Some(x), Some(y))) => (x.min(y)..=x.max(y)).collect(), + _ => Vec::new(), + } +} + /// Index of the cell at (r, c) in a sheet's sorted cell list. pub fn find_cell(sh: &Sheet, r: u32, c: u32) -> Option<&Cell> { sh.cells.binary_search_by_key(&(r, c), |x| (x.r, x.c)).ok().map(|i| &sh.cells[i]) @@ -335,11 +347,18 @@ fn resolve_refs(wb: &Workbook, si: usize, lx: &Lexed, rect: &Rect, a: &Analysis) let mut out = Vec::new(); for t in &lx.refs { let (Some(a0), Some(a1)) = (t.rect_at(0, 0), t.rect_at(dr, dc)) else { continue }; - let (sheet, sheet_name) = match &t.sheet { - None => (Some(si), None), - Some(n) => (sheet_index(wb, n), Some(n.clone())), + let rect = a0.union(&a1); + let Some(n) = &t.sheet else { + out.push(GRef { sheet: Some(si), sheet_name: None, rect, via_name: None }); + continue; }; - out.push(GRef { sheet, sheet_name, rect: a0.union(&a1), via_name: None }); + let span = sheet_span(wb, n); + if span.is_empty() { + out.push(GRef { sheet: None, sheet_name: Some(n.clone()), rect, via_name: None }); + } + for s in span { + out.push(GRef { sheet: Some(s), sheet_name: Some(n.clone()), rect, via_name: None }); + } } for n in &lx.names { if let Some((sh, r)) = a.named.get(&n.to_ascii_uppercase()) { diff --git a/src/formula.rs b/src/formula.rs index 87e4fd5..b53e43d 100644 --- a/src/formula.rs +++ b/src/formula.rs @@ -127,6 +127,17 @@ pub fn lex(f: &str) -> Lexed { i = lex_ref_after_sheet(f, start, i + 1, Some(word.to_string()), &mut out); continue; } + // 3D ref across a run of sheets: `Jan:Dec!B2` (sheet "Jan:Dec"). + if i < b.len() && b[i] == b':' { + let mut k = i + 1; + while k < b.len() && is_ident(b[k]) { + k += 1; + } + if k > i + 1 && k < b.len() && b[k] == b'!' { + i = lex_ref_after_sheet(f, start, k + 1, Some(f[start..k].to_string()), &mut out); + continue; + } + } if i < b.len() && b[i] == b'[' { // Table1[Col] out.names.push(word.to_string()); @@ -435,6 +446,16 @@ mod tests { assert_eq!(args, vec!["\"abc\"", "\"S!A1:B2\""]); } + #[test] + fn three_d_refs() { + for f in ["=SUM(Jan:Dec!B2)", "=SUM('Jan:Dec'!B2)"] { + let lx = lex(f); + assert_eq!(lx.refs.len(), 1, "{f}"); + assert_eq!(lx.refs[0].sheet.as_deref(), Some("Jan:Dec"), "{f}"); + assert!(lx.names.is_empty(), "{f}"); + } + } + #[test] fn external_workbook_refs() { let lx = lex("='[Budget 2026.xlsx]Data'!B2+'[Rates.xlsx]'!Margin*C1"); diff --git a/src/graph.rs b/src/graph.rs index 01e8b8c..e68fa69 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -54,6 +54,10 @@ pub fn build(wb: &Workbook, a: &Analysis) -> J { _ => r.rect.a1(), }; let Some(t) = r.sheet else { + // `[Book.xlsx]Sheet` refs are `imports` edges to the external node. + if r.sheet_name.as_deref().is_some_and(|n| n.starts_with('[')) { + continue; + } edges.push(json!({"from": group_id(si, gi), "to": null, "kind": "reads", "range": range_txt, "broken": true})); continue; }; @@ -74,6 +78,22 @@ pub fn build(wb: &Workbook, a: &Analysis) -> J { link_a1(wb, a, &id, src, "plots", &mut edges); } } + for (xi, x) in sh.shapes.iter().enumerate().filter(|(_, x)| x.on_click.is_some() || x.linked_cell.is_some()) { + let id = format!("s{}.shape{}", si + 1, xi + 1); + nodes.push(json!({ + "id": id, "type": "shape", "sheet": format!("s{}", si + 1), "kind": x.kind, "text": x.text, + "range": x.anchor.map(|r| r.a1()), + })); + if let Some(m) = &x.on_click { + for p in a.macros.iter().filter(|p| crate::macros::clicks(m, wb, p)) { + edges.push(json!({"from": id, "to": macro_id(wb, p), "kind": "runs"})); + } + } + if let Some(cell) = &x.linked_cell { + let a1 = if cell.contains('!') { cell.clone() } else { format!("{}!{cell}", quote_sheet(&sh.name)) }; + link_a1(wb, a, &id, &a1, "writes", &mut edges); + } + } for p in &sh.pivots { let id = format!("s{}.pivot{}", si + 1, crate::a1::cell_name(p.anchor.0, p.anchor.1)); nodes.push(json!({"id": id, "type": "pivot", "sheet": format!("s{}", si + 1), "rows": p.rows, "cols": p.cols, "values": p.values})); diff --git a/src/gsheets.rs b/src/gsheets.rs index 3d4cff8..a7d4b52 100644 --- a/src/gsheets.rs +++ b/src/gsheets.rs @@ -5,7 +5,7 @@ use crate::a1::{col_name, quote_sheet, Rect, OPEN}; use crate::model::*; use anyhow::{anyhow, bail, Context, Result}; use serde::Deserialize; -use serde_json::Value as J; +use serde_json::{json, Value as J}; use std::collections::HashMap; use std::sync::{mpsc, Mutex}; use std::time::{Duration, Instant}; @@ -172,6 +172,8 @@ struct CellData { effective_value: Option, formatted_value: Option, user_entered_format: Option, + text_format_runs: Vec, + chip_runs: Vec, data_validation: Option, note: Option, hyperlink: Option, @@ -226,7 +228,7 @@ impl ExtValue { pub fn fetch(auth: &Auth, id_or_url: &str, verbose: bool) -> Result { let id = crate::formula::spreadsheet_id(id_or_url); - let meta_fields = "spreadsheetId,spreadsheetUrl,properties(title,locale,timeZone),namedRanges,dataSources,developerMetadata,\ + let meta_fields = "spreadsheetId,spreadsheetUrl,properties(title,locale,timeZone,autoRecalc,iterativeCalculationSettings),namedRanges,dataSources,developerMetadata,\ sheets(properties,merges,conditionalFormats,protectedRanges,basicFilter,filterViews,charts,bandedRanges,developerMetadata)"; let meta: J = serde_json::from_str(&auth.get(&format!("{SHEETS}/{id}"), &[("fields", meta_fields)])?)?; @@ -242,6 +244,7 @@ pub fn fetch(auth: &Auth, id_or_url: &str, verbose: bool) -> Result { }, ..Default::default() }; + wb.calc = calc(&meta["properties"]); // Drive metadata is best-effort: needs the drive scope and visibility. match auth.get( &format!("https://www.googleapis.com/drive/v3/files/{id}"), @@ -337,7 +340,8 @@ pub fn fetch(auth: &Auth, id_or_url: &str, verbose: bool) -> Result { } let total = jobs.len(); let fields = "sheets(data(startRow,startColumn,rowMetadata(hiddenByUser),columnMetadata(hiddenByUser),\ - rowData(values(userEnteredValue,effectiveValue,formattedValue,userEnteredFormat,dataValidation,note,hyperlink,pivotTable))))"; + rowData(values(userEnteredValue,effectiveValue,formattedValue,userEnteredFormat,dataValidation,note,hyperlink,pivotTable,\ + textFormatRuns,chipRuns))))"; let names: Vec<(String, u32)> = wb.sheets.iter().map(|s| (s.name.clone(), s.grid_cols)).collect(); let queue = Mutex::new(jobs.into_iter()); let (tx, rx) = mpsc::channel::>(); @@ -454,7 +458,13 @@ fn absorb( }; let s = cd.user_entered_format.as_ref().map(style_of).filter(|s| *s != Style::default()).map(|s| styles.intern(s)); let dv = cd.data_validation.as_ref().map(|d| dvs.intern(validation(d))); - let cell = Cell { r, c, v, f, t, s, dv, note: cd.note, link: cd.hyperlink }; + let text = cd.formatted_value.as_deref().or(match &v { + Value::Str(s) => Some(s.as_str()), + _ => None, + }); + let runs = text.map(|t| runs(t, &cd.text_format_runs)).unwrap_or_default(); + let obj = chip(text, &cd.chip_runs); + let cell = Cell { r, c, v, f, t, s, dv, note: cd.note, link: cd.hyperlink, runs, obj }; if cell.is_blank() && cell.s.is_none() && cell.dv.is_none() && cell.note.is_none() { continue; } @@ -545,7 +555,63 @@ fn style_of(f: &J) -> Style { font, size, wrap: f["wrapStrategy"].as_str() == Some("WRAP"), + ..Default::default() + } +} + +/// Recalculation settings, when they differ from Google's default (on change, no iteration). +fn calc(p: &J) -> Option { + let mode = match p["autoRecalc"].as_str() { + Some("MINUTE") => "on change and every minute", + Some("HOUR") => "on change and every hour", + _ => "on change", + }; + let it = &p["iterativeCalculationSettings"]; + let iterative = it.is_object(); + if !iterative && mode == "on change" { + return None; + } + Some(Calc { + mode: mode.into(), + iterative, + max_iterations: it["maxIterations"].as_u64().map(|n| n as u32), + max_change: it["convergenceThreshold"].as_f64(), + }) +} + +/// A cell's text split where its formatting changes (`textFormatRuns`). +/// Indexes count UTF-16 code units. +fn runs(text: &str, fmt_runs: &[J]) -> Vec { + let units: Vec = text.encode_utf16().collect(); + let mut out: Vec = Vec::new(); + for (i, r) in fmt_runs.iter().enumerate() { + let start = r["startIndex"].as_u64().unwrap_or(0) as usize; + let end = fmt_runs.get(i + 1).and_then(|n| n["startIndex"].as_u64()).map_or(units.len(), |e| e as usize); + let Some(slice) = units.get(start.min(units.len())..end.min(units.len())).filter(|s| !s.is_empty()) else { continue }; + let f = style_of(&json!({ "textFormat": r["format"] })); + out.push(Run { text: String::from_utf16_lossy(slice), f }); + } + if !runs_differ(&out) { + return Vec::new(); } + out +} + +/// Smart chips: people, Drive files and other links. +fn chip(text: Option<&str>, chips: &[J]) -> Option { + let mut objs = chips.iter().filter_map(|c| { + let c = &c["chip"]; + let (kind, target) = if let Some(email) = c["personProperties"]["email"].as_str() { + ("person", email) + } else { + let uri = c["richLinkProperties"]["uri"].as_str()?; + (if uri.contains("docs.google.com") || uri.contains("drive.google.com") { "file" } else { "link" }, uri) + }; + Some((kind, target.to_string())) + }); + let (kind, target) = objs.next()?; + let props = objs.map(|(k, t)| (k.to_string(), t)).collect(); + Some(CellObject { kind: kind.into(), text: text.map(Into::into), target: Some(target), props }) } fn cond_values(cond: &J) -> Vec { @@ -687,3 +753,34 @@ fn pivot(p: &J, anchor: (u32, u32), id_to_name: &HashMap) -> Pivot filters: arr(&p["filterSpecs"]).iter().map(col).collect(), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn text_runs_and_chips() { + // Shapes as the Sheets API returns them; indexes are UTF-16 units. + let fmt = json!([ + {"startIndex": 0, "format": {}}, + {"startIndex": 6, "format": {"bold": true, "foregroundColorStyle": {"rgbColor": {"red": 1}}}}, + ]); + let r = runs("Café: due", fmt.as_array().unwrap()); + assert_eq!(r.iter().map(|r| r.text.as_str()).collect::>(), ["Café: ", "due"]); + assert!(r[1].f.bold && r[1].f.fg.as_deref() == Some("#FF0000")); + let same = json!([{"startIndex": 0, "format": {"italic": true}}, {"startIndex": 2, "format": {"italic": true}}]); + assert!(runs("abcd", same.as_array().unwrap()).is_empty()); + + let chips = json!([ + {"startIndex": 0, "chip": {"personProperties": {"email": "ana@example.com", "displayFormat": "DEFAULT"}}}, + {"startIndex": 4, "chip": {"richLinkProperties": {"uri": "https://docs.google.com/document/d/abc", "mimeType": "application/vnd.google-apps.document"}}}, + ]); + let o = chip(Some("Ana and the plan"), chips.as_array().unwrap()).unwrap(); + assert_eq!((o.kind.as_str(), o.target.as_deref()), ("person", Some("ana@example.com"))); + assert_eq!(o.props, [("file".to_string(), "https://docs.google.com/document/d/abc".to_string())]); + + assert_eq!(calc(&json!({"autoRecalc": "ON_CHANGE"})), None); + let c = calc(&json!({"autoRecalc": "HOUR", "iterativeCalculationSettings": {"maxIterations": 50, "convergenceThreshold": 0.01}})).unwrap(); + assert_eq!((c.mode.as_str(), c.iterative, c.max_iterations, c.max_change), ("on change and every hour", true, Some(50), Some(0.01))); + } +} diff --git a/src/lib.rs b/src/lib.rs index c778857..4fa20f0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,7 @@ pub mod graph; pub mod gsheets; pub mod macros; pub mod model; +pub mod numfmt; pub mod ooxml; pub mod query; pub mod report; diff --git a/src/macros.rs b/src/macros.rs index 9345aec..6a25e98 100644 --- a/src/macros.rs +++ b/src/macros.rs @@ -227,6 +227,14 @@ pub fn place(wb: &Workbook, t: &Touch) -> String { /// One line on what a procedure does: its trigger, formula use, the ranges it /// writes and uses, the procedures it calls. `max` caps each list. None when /// there's nothing to say beyond its name. +/// Whether a shape's `on_click` (`RollUp` or `Module1.RollUp`) names this procedure. +pub fn clicks(on_click: &str, wb: &Workbook, p: &Proc) -> bool { + match on_click.rsplit_once('.') { + Some((m, n)) => n.eq_ignore_ascii_case(&p.name) && m.eq_ignore_ascii_case(&wb.scripts[p.script].name), + None => on_click.eq_ignore_ascii_case(&p.name), + } +} + pub fn describe(wb: &Workbook, a: &Analysis, p: &Proc, max: usize) -> Option { let list = |v: Vec, sep: &str| { let more = v.len().saturating_sub(max); @@ -244,6 +252,12 @@ pub fn describe(wb: &Workbook, a: &Analysis, p: &Proc, max: usize) -> Option parts.push(format!("trigger {}", t.event)), } } + for sh in &wb.sheets { + for x in sh.shapes.iter().filter(|x| x.on_click.as_deref().is_some_and(|m| clicks(m, wb, p))) { + let label = x.text.as_deref().or(Some(x.name.as_str()).filter(|n| !n.is_empty())).map(|t| format!(" \"{}\"", crate::report::trunc(t, 40))); + parts.push(format!("run by {}{} on {}", x.kind, label.unwrap_or_default(), quote_sheet(&sh.name))); + } + } let n = formula_cells(a, p); if n > 0 { parts.push(format!("used in {n} formula cell{}", if n == 1 { "" } else { "s" })); diff --git a/src/model.rs b/src/model.rs index 1629f53..a7576c4 100644 --- a/src/model.rs +++ b/src/model.rs @@ -24,6 +24,18 @@ pub struct Workbook { /// name them as `'[file]Sheet'!A1`. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub external_links: Vec, + /// Document properties: title, author, description, … then custom ones. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub properties: Vec<(String, String)>, + /// Calculation settings, when not the default (automatic, no iteration). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub calc: Option, + /// Sheets can't be added, removed, renamed or unhidden. + #[serde(default, skip_serializing_if = "is_false")] + pub structure_locked: bool, + /// Data connections: databases, web queries, text imports, Power Query. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub connections: Vec, /// Anything the backend could not extract, so consumers know what is missing. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub warnings: Vec, @@ -93,6 +105,9 @@ pub struct Sheet { /// Excel tables (ListObjects); formulas reach them as `Name[Column]`. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub tables: Vec, + /// Pictures, shapes, text boxes and form controls placed over the grid. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub shapes: Vec, /// Rows summarized instead of kept, when the workbook was too big to hold /// whole (see [`crate::xlsx::CELL_BUDGET`]). #[serde(default, skip_serializing_if = "Option::is_none")] @@ -186,21 +201,66 @@ pub struct Cell { pub note: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub link: Option, + /// Text whose formatting changes partway (one word bold, say), as runs. + /// Only kept when the runs differ from each other. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub runs: Vec, + /// A picture in the cell, or a data type (stock, place, person chip). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub obj: Option, } impl Cell { pub fn is_blank(&self) -> bool { - self.v.is_empty() && self.f.is_none() + self.v.is_empty() && self.f.is_none() && self.obj.is_none() } /// What a person would see in the cell. pub fn display(&self) -> String { - match &self.t { - Some(t) => t.clone(), - None => self.v.to_string(), + match (&self.t, &self.obj) { + (Some(t), _) => t.clone(), + (None, Some(o)) if self.v.is_empty() => match &o.text { + Some(t) => format!("[{}: {t}]", o.kind), + None => format!("[{}]", o.kind), + }, + _ => self.v.to_string(), } } } +/// A stretch of a cell's text with its own formatting (only the font +/// properties of [`Style`] are used). +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct Run { + pub text: String, + #[serde(default, skip_serializing_if = "Style::is_plain")] + pub f: Style, +} + +/// Whether runs differ in emphasis (bold, italic, underline, strike, colour). +/// Runs that only switch font or size are mostly font fallback for another +/// script, and aren't worth keeping. +pub fn runs_differ(runs: &[Run]) -> bool { + let key = |r: &Run| (r.f.bold, r.f.italic, r.f.underline, r.f.strike, r.f.fg.clone()); + runs.iter().any(|r| key(r) != key(&runs[0])) +} + +/// What a cell holds besides a plain value. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct CellObject { + /// "image" (a picture in the cell), "entity" (Excel linked data type such + /// as Stocks or Geography), "person", "file", "link" (Google smart chips). + pub kind: String, + /// Alt text of a picture, or the data type's display name. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text: Option, + /// Picture file inside the package, or a chip's URL / email. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub target: Option, + /// A data type's fields (`Price = 412.5`, `Population = 67750000`). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub props: Vec<(String, String)>, +} + #[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] #[serde(untagged)] pub enum Value { @@ -269,9 +329,19 @@ pub struct Style { pub size: Option, #[serde(default, skip_serializing_if = "is_false")] pub wrap: bool, + /// Left editable when the sheet is protected (cells are locked by default). + #[serde(default, skip_serializing_if = "is_false")] + pub unlocked: bool, + /// The formula is hidden from the formula bar when the sheet is protected. + #[serde(default, skip_serializing_if = "is_false")] + pub hidden_formula: bool, } impl Style { + pub fn is_plain(&self) -> bool { + *self == Style::default() + } + pub fn is_date(&self) -> bool { matches!(self.num_type.as_deref(), Some("DATE" | "DATE_TIME" | "TIME")) } @@ -369,9 +439,66 @@ pub struct ExternalLink { pub sheets: Vec, } +/// How the workbook recalculates. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)] +pub struct Calc { + /// "automatic", "manual", "automatic except tables" (Excel); "on change", + /// "on change and every minute", "on change and every hour" (Google). + pub mode: String, + /// Circular references are allowed and iterated. + #[serde(default, skip_serializing_if = "is_false")] + pub iterative: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_iterations: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_change: Option, +} + +/// A data connection the workbook refreshes from. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct Connection { + pub name: String, + /// "odbc", "oledb", "web", "text", "power query", … + pub kind: String, + /// Connection string or URL, with passwords removed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// SQL, table name or query run on the source. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub command: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "is_false")] + pub refresh_on_open: bool, +} + +/// Something drawn over the grid. +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct Shape { + /// "picture", "shape", "text box", "button", "checkbox", "dropdown", … + pub kind: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + pub name: String, + /// Cells it covers. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub anchor: Option, + /// Text shown on it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text: Option, + /// Alt text (descriptions of pictures). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub alt: Option, + /// Macro run when it's clicked. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub on_click: Option, + /// Cell a form control reads and writes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub linked_cell: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct Script { - /// "vba" | "apps_script" | "macro" + /// "vba" | "apps_script" | "power_query" pub kind: String, pub name: String, pub source: String, diff --git a/src/numfmt.rs b/src/numfmt.rs new file mode 100644 index 0000000..0a99e48 --- /dev/null +++ b/src/numfmt.rs @@ -0,0 +1,278 @@ +//! Excel number formats applied to numbers, so a cell reads as it shows on +//! screen (`25%`, `$12.50`, `(1,234)`) rather than as its stored value. +//! +//! Covers the common shapes: sections (`pos;neg;zero`), literals, `%`, +//! thousands separators and `,` scaling, fixed and optional decimals, +//! scientific notation, colour and currency tags. Anything else (dates, +//! fractions, conditions, digits mixed with literals) returns None, and the +//! caller keeps the plain value. + +use crate::model::fmt_num; + +/// `n` formatted with `code`, or None when the format isn't one this handles. +pub fn format(n: f64, code: &str) -> Option { + if !n.is_finite() { + return None; + } + let sections = split_sections(code); + if sections.iter().any(|s| s.contains("[>") || s.contains("[<") || s.contains("[=")) { + return None; + } + let (sec, signed) = match (sections.len(), n) { + (_, n) if n > 0.0 => (sections[0].as_str(), false), + (1, n) if n < 0.0 => (sections[0].as_str(), true), + (_, n) if n < 0.0 => (sections[1].as_str(), false), + (l, _) if l >= 3 => (sections[2].as_str(), false), + _ => (sections[0].as_str(), false), + }; + let body = render(n.abs(), sec)?; + Some(if signed && body.chars().any(|c| c.is_ascii_digit() && c != '0') { format!("-{body}") } else { body }) +} + +fn split_sections(code: &str) -> Vec { + let mut out = vec![String::new()]; + let mut chars = code.chars(); + while let Some(c) = chars.next() { + match c { + ';' => out.push(String::new()), + '"' => { + let last = out.last_mut().unwrap(); + last.push('"'); + for q in chars.by_ref() { + last.push(q); + if q == '"' { + break; + } + } + } + '\\' => { + let last = out.last_mut().unwrap(); + last.push('\\'); + last.extend(chars.next()); + } + _ => out.last_mut().unwrap().push(c), + } + } + out +} + +enum Piece { + Lit(String), + Num, +} + +/// One section: literals around a single run of digit placeholders. +fn render(v: f64, sec: &str) -> Option { + let mut pieces: Vec = Vec::new(); + let mut pat = String::new(); // digit placeholders, '.', ',', 'E+00' + let mut lit = String::new(); + let mut general = false; + let mut percent = 0; + let sec = replace_general(sec); + let mut chars = sec.chars().peekable(); + let flush = |lit: &mut String, pieces: &mut Vec| { + if !lit.is_empty() { + pieces.push(Piece::Lit(std::mem::take(lit))); + } + }; + while let Some(c) = chars.next() { + match c { + '"' => { + for q in chars.by_ref() { + if q == '"' { + break; + } + lit.push(q); + } + } + '\\' => lit.extend(chars.next()), + '_' => { + chars.next(); + lit.push(' '); + } + '*' => { + chars.next(); + } + '[' => { + let tag: String = chars.by_ref().take_while(|c| *c != ']').collect(); + // [$€-407] is a currency symbol; [$-409] only a locale; [Red] a colour. + if let Some(cur) = tag.strip_prefix('$') { + lit.push_str(cur.split('-').next().unwrap_or("")); + } else if tag.chars().next().is_some_and(|c| "hHmMsS".contains(c)) { + return None; // elapsed time + } + } + '0' | '#' | '?' | '.' => { + if pieces.iter().any(|p| matches!(p, Piece::Num)) { + return None; // digits broken up by literals (phone numbers, fractions) + } + flush(&mut lit, &mut pieces); + pat.push(c); + while let Some(&d) = chars.peek() { + if "0#?.,".contains(d) { + pat.push(d); + chars.next(); + } else if (d == 'E' || d == 'e') && !pat.contains('E') { + chars.next(); + let sign = chars.next().filter(|s| *s == '+' || *s == '-')?; + pat.push('E'); + pat.push(sign); + } else { + break; + } + } + pieces.push(Piece::Num); + } + '%' => { + percent += 1; + lit.push('%'); + } + GENERAL => { + flush(&mut lit, &mut pieces); + general = true; + pieces.push(Piece::Num); + } + '@' | '/' => return None, + c if c.is_ascii_alphabetic() && !matches!(c, 'E' | 'e') => { + // y m d h s and AM/PM belong to dates and times. + if "yYmMdDhHsSaA".contains(c) { + return None; + } + lit.push(c); + } + _ => lit.push(c), + } + } + flush(&mut lit, &mut pieces); + let v = v * 100f64.powi(percent); + let num = if general { + fmt_num(v) + } else if pat.is_empty() { + String::new() + } else { + number(v, &pat) + }; + // `_)` and `* ` only pad for alignment. + let out: String = pieces.into_iter().map(|p| match p { + Piece::Lit(s) => s, + Piece::Num => num.clone(), + }).collect(); + Some(out.trim().to_string()) +} + +/// Stands in for the word `General` (outside quotes) while a section is read. +const GENERAL: char = '\u{1}'; + +fn replace_general(sec: &str) -> String { + let mut out = String::new(); + let mut rest = sec; + let mut quoted = false; + while let Some(c) = rest.chars().next() { + if !quoted && rest.get(..7).is_some_and(|w| w.eq_ignore_ascii_case("general")) { + out.push(GENERAL); + rest = &rest[7..]; + continue; + } + quoted ^= c == '"'; + out.push(c); + rest = &rest[c.len_utf8()..]; + } + out +} + +/// `v` (not negative) through a placeholder pattern like `#,##0.00` or `0.0E+00`. +fn number(mut v: f64, pat: &str) -> String { + let (mant, exp) = match pat.find('E') { + Some(i) => (&pat[..i], Some(&pat[i + 1..])), + None => (pat, None), + }; + // Commas right after the last digit placeholder scale by 1000 each. + let trimmed = mant.trim_end_matches(','); + for _ in 0..mant.len() - trimmed.len() { + v /= 1000.0; + } + let (int_pat, dec_pat) = trimmed.split_once('.').unwrap_or((trimmed, "")); + let thousands = int_pat.contains(','); + let min_int = int_pat.chars().filter(|c| *c == '0').count(); + let (min_dec, max_dec) = (dec_pat.chars().filter(|c| *c == '0' || *c == '?').count(), dec_pat.chars().filter(|c| "0#?".contains(*c)).count()); + + let mut exponent = 0i32; + if exp.is_some() && v != 0.0 { + exponent = v.log10().floor() as i32; + v /= 10f64.powi(exponent); + // Rounding can carry the mantissa to 10.0. + if format!("{v:.max_dec$}").starts_with("10") { + v /= 10.0; + exponent += 1; + } + } + let fixed = format!("{v:.max_dec$}"); + let (int_s, dec_s) = fixed.split_once('.').unwrap_or((&fixed, "")); + let mut dec = dec_s.to_string(); + while dec.len() > min_dec && dec.ends_with('0') { + dec.pop(); + } + let int_s = int_s.trim_start_matches('0'); + let mut int = format!("{:0>min_int$}", int_s); + if thousands && int.len() > 3 { + let mut grouped = String::new(); + for (i, ch) in int.chars().enumerate() { + if i > 0 && (int.len() - i) % 3 == 0 { + grouped.push(','); + } + grouped.push(ch); + } + int = grouped; + } + let mut out = int; + if !dec.is_empty() || (dec_pat.is_empty() && trimmed.contains('.')) { + out.push('.'); + out.push_str(&dec); + } + if let Some(e) = exp { + let (sign, digits) = e.split_at(1); + let width = digits.chars().filter(|c| *c == '0').count(); + let s = if exponent < 0 { "-" } else if sign == "+" { "+" } else { "" }; + out.push_str(&format!("E{s}{:0>width$}", exponent.abs())); + } + out +} + +#[cfg(test)] +mod tests { + use super::format; + + #[test] + fn common_formats() { + let f = |n: f64, code: &str| format(n, code); + assert_eq!(f(0.25, "0%").as_deref(), Some("25%")); + assert_eq!(f(0.1234, "0.0%").as_deref(), Some("12.3%")); + assert_eq!(f(12.5, "$#,##0.00").as_deref(), Some("$12.50")); + assert_eq!(f(1234567.0, "#,##0").as_deref(), Some("1,234,567")); + assert_eq!(f(-1234.5, "$#,##0.00_);($#,##0.00)").as_deref(), Some("($1,234.50)")); + assert_eq!(f(1234.5, "$#,##0.00_);($#,##0.00)").as_deref(), Some("$1,234.50")); + assert_eq!(f(0.0, "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"??_);_(@_)").as_deref(), Some("-")); + assert_eq!(f(-1500.0, "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"??_);_(@_)").as_deref(), Some("(1,500)")); + assert_eq!(f(-5.0, "#,##0").as_deref(), Some("-5")); + assert_eq!(f(0.0, "#,##0;(#,##0);\"-\"").as_deref(), Some("-")); + assert_eq!(f(-7.0, "#,##0;[Red](#,##0)").as_deref(), Some("(7)")); + assert_eq!(f(0.00012, "0.00E+00").as_deref(), Some("1.20E-04")); + assert_eq!(f(123456.0, "0.0E+0").as_deref(), Some("1.2E+5")); + assert_eq!(f(1_500_000.0, "0.0,,\"M\"").as_deref(), Some("1.5M")); + assert_eq!(f(1234.5, "[$€-407] #,##0.00").as_deref(), Some("€ 1,234.50")); + assert_eq!(f(5.0, "0.00").as_deref(), Some("5.00")); + assert_eq!(f(0.5, "#.00").as_deref(), Some(".50")); + assert_eq!(f(1.23456, "0.0#").as_deref(), Some("1.23")); + assert_eq!(f(3.1, "0.0#").as_deref(), Some("3.1")); + assert_eq!(f(42.0, "General\" units\"").as_deref(), Some("42 units")); + assert_eq!(f(42.0, ";;;").as_deref(), Some("")); + assert_eq!(f(-0.001, "0.0").as_deref(), Some("0.0")); + } + + #[test] + fn declines_what_it_cannot_render() { + for code in ["yyyy-mm-dd", "h:mm", "# ?/?", "000-00-0000", "[>100]0;0.0", "@", "[h]:mm"] { + assert_eq!(format(1.0, code), None, "{code}"); + } + } +} diff --git a/src/ooxml.rs b/src/ooxml.rs index d1b950e..89a1aa0 100644 --- a/src/ooxml.rs +++ b/src/ooxml.rs @@ -7,8 +7,9 @@ //! failed extraction. Values and formulas still come from calamine. use crate::a1::{col_name, parse_pt, parse_rect, quote_sheet, Rect, OPEN}; -use crate::model::{ - Cell, Chart, CondFormat, ExternalLink, Filter, Interner, Pivot, Protected, Sheet, Style, Table, Validation, Workbook, +use crate::model::{fmt_num, Value, + Calc, Cell, CellObject, Chart, CondFormat, Connection, ExternalLink, Filter, Interner, Pivot, Protected, Run, Script, Shape, Sheet, Style, + Table, Validation, Workbook, }; use anyhow::{anyhow, Result}; use quick_xml::events::{BytesRef, BytesStart, Event}; @@ -35,13 +36,22 @@ pub fn enrich(path: &Path, wb: &mut Workbook) -> Result<()> { styles.xfs.iter().map(|s| if *s == Style::default() { None } else { Some(interner.intern(s.clone())) }).collect(); let mut dvs = Interner::::default(); let mut skipped: BTreeSet<&str> = BTreeSet::new(); + let rich = part_of("sharedStrings").map(|p| rich_strings(&mut pkg, &p, &styles)).unwrap_or_default(); + let objects = rich_values(&mut pkg); wb.external_links = external_links(&mut pkg, &root, &rels); + wb.calc = calc(&root); + wb.structure_locked = root.child("workbookProtection").is_some_and(|p| p.flag("lockStructure")); + wb.properties = properties(&mut pkg); + if let Some(p) = part_of("connections") { + wb.connections = connections(&mut pkg, &p); + } + wb.scripts.extend(power_query(&mut pkg)); for s in root.child("sheets").map(|s| s.children("sheet").collect::>()).unwrap_or_default() { let (Some(name), Some(rel)) = (s.attr("name"), s.attr("id").and_then(|id| rels.get(id))) else { continue }; let Some(si) = wb.sheets.iter().position(|x| x.name == name) else { continue }; - let mut cx = SheetCx { styles: &styles, xf_ids: &xf_ids, dvs: &mut dvs, skipped: &mut skipped }; + let mut cx = SheetCx { styles: &styles, xf_ids: &xf_ids, dvs: &mut dvs, skipped: &mut skipped, rich: &rich, objects: &objects }; if let Err(e) = read_sheet(&mut pkg, &rel.target, &rel.kind, &mut wb.sheets[si], &mut cx) { wb.warnings.push(format!("{name}: {e:#}")); } @@ -69,7 +79,7 @@ pub fn enrich(path: &Path, wb: &mut Workbook) -> Result<()> { } } - for (prefix, what) in [("xl/media/", "images"), ("xl/slicers/", "slicers"), ("xl/timelines/", "timelines")] { + for (prefix, what) in [("xl/slicers/", "slicers"), ("xl/timelines/", "timelines"), ("xl/activex/", "ActiveX controls")] { if pkg.names.keys().any(|n| n.starts_with(prefix)) { skipped.insert(what); } @@ -114,6 +124,13 @@ impl Pkg { Some(r) } + fn bytes(&mut self, part: &str) -> Option> { + let name = self.names.get(&part.to_ascii_lowercase())?.clone(); + let mut out = Vec::new(); + std::io::Read::read_to_end(&mut self.zip.by_name(&name).ok()?, &mut out).ok()?; + Some(out) + } + fn node(&mut self, part: &str) -> Option { parse(&mut self.reader(part)?).ok() } @@ -202,6 +219,14 @@ impl Node { walk(self, name, &mut out); out } + /// All text below, in document order. + fn all_text(&self) -> String { + let mut s = self.text.clone(); + for k in &self.kids { + s.push_str(&k.all_text()); + } + s + } /// Concatenated text of every `` below (rich-text runs). fn runs(&self) -> String { self.all("t").iter().map(|t| t.text.as_str()).collect() @@ -274,6 +299,10 @@ struct SheetCx<'a> { xf_ids: &'a [Option], dvs: &'a mut Interner, skipped: &'a mut BTreeSet<&'static str>, + /// Shared strings with mixed formatting, by index. + rich: &'a HashMap>, + /// Pictures and data types by cell `vm` (value metadata) index, from 0. + objects: &'a [Option], } /// One pass over a worksheet part: cell style ids and hidden rows are taken @@ -288,10 +317,18 @@ struct SheetXml { shared_fixes: Vec<(u32, u32, String)>, /// Summarized rows ([`crate::model::Elided`]): their cell styles are skipped. gap: Option<(u32, u32)>, + /// Read shared-string indexes (only when some shared strings are rich). + want_sst: bool, + /// Cells holding a shared string: (row, col, index). + sst_cells: Vec<(u32, u32, u32)>, + /// Inline rich strings (`…`). + inline: Vec<(u32, u32, Node)>, + /// Cells with value metadata (`vm`): pictures in cells, data types. + vm: Vec<(u32, u32, u32)>, } -fn read_sheet_xml(rd: &mut Reader, gap: Option<(u32, u32)>) -> Result { - let mut out = SheetXml { gap, ..Default::default() }; +fn read_sheet_xml(rd: &mut Reader, gap: Option<(u32, u32)>, want_sst: bool) -> Result { + let mut out = SheetXml { gap, want_sst, ..Default::default() }; let mut buf = Vec::new(); loop { buf.clear(); @@ -319,9 +356,23 @@ fn read_sheet_data(rd: &mut Reader, out: &mut SheetXml, buf: &mut // `r` is optional on rows and cells; position then follows the previous one. let (mut row, mut next_col, mut started) = (0u32, 0u32, false); let (mut cell, mut masters) = ((0u32, 0u32), HashMap::new()); + let mut shared_str = false; loop { buf.clear(); + let kept = |out: &SheetXml, r: u32| !out.gap.is_some_and(|(a, b)| (a..=b).contains(&r)); match rd.read_event_into(buf)? { + Event::Start(e) if e.local_name().as_ref() == "v" && shared_str && out.want_sst => { + let v = read_children(rd, "v".into(), Vec::new(), buf)?; + if let (Ok(i), true) = (v.text.trim().parse::(), kept(out, cell.0)) { + out.sst_cells.push((cell.0, cell.1, i)); + } + } + Event::Start(e) if e.local_name().as_ref() == "is" => { + let n = read_children(rd, "is".into(), Vec::new(), buf)?; + if n.child("r").is_some() && kept(out, cell.0) { + out.inline.push((cell.0, cell.1, n)); + } + } Event::Start(e) if e.local_name().as_ref() == "f" => { let f = read_children(rd, "f".into(), attrs_of(&e), buf)?; shared_formula(&f, cell, &mut masters, &mut out.shared_fixes); @@ -348,17 +399,23 @@ fn read_sheet_data(rd: &mut Reader, out: &mut SheetXml, buf: &mut } } "c" => { - let (mut pos, mut s) = (None, None); + let (mut pos, mut s, mut vm) = (None, None, None); + shared_str = false; for a in e.attributes().flatten() { match a.key.local_name().as_ref() { "r" => pos = parse_pt(&a.value).and_then(|p| Some((p.row?, p.col?))), "s" => s = a.value.parse::().ok(), + "t" => shared_str = a.value.as_ref() == "s", + "vm" => vm = a.value.parse::().ok(), _ => {} } } let (r, c) = pos.unwrap_or((row, next_col)); next_col = c + 1; cell = (r, c); + if let Some(vm) = vm.filter(|v| *v > 0) { + out.vm.push((r, c, vm)); + } if let Some(s) = s.filter(|s| *s != 0) && !out.gap.is_some_and(|(a, b)| (a..=b).contains(&r)) { @@ -409,7 +466,7 @@ fn push_span(spans: &mut Vec<(u32, u32)>, i: u32) { fn read_sheet(pkg: &mut Pkg, part: &str, kind: &str, sh: &mut Sheet, cx: &mut SheetCx) -> Result<()> { let sx = { let mut rd = pkg.reader(part).ok_or_else(|| anyhow!("missing part {part}"))?; - read_sheet_xml(&mut rd, sh.elided.as_ref().map(|e| e.rows))? + read_sheet_xml(&mut rd, sh.elided.as_ref().map(|e| e.rows), !cx.rich.is_empty())? }; let rels = pkg.rels(part); if kind == "chartsheet" { @@ -426,8 +483,43 @@ fn read_sheet(pkg: &mut Pkg, part: &str, kind: &str, sh: &mut Sheet, cx: &mut Sh && let Some(x) = cell_at(&mut sh.cells, &mut at, r, c) { x.s = Some(*id); + // Numbers as the format shows them (25%, $12.50); dates already are. + if let (Value::Num(n), None, Some(st)) = (&x.v, &x.t, cx.styles.xfs.get(xf as usize)) + && !st.is_date() + && let Some(t) = st.num_fmt.as_deref().and_then(|code| crate::numfmt::format(*n, code)) + && t != fmt_num(*n) + { + x.t = Some(t); + } + } + } + for &(r, c, i) in &sx.sst_cells { + if let Some(runs) = cx.rich.get(&i) + && let Some(x) = cell_at(&mut sh.cells, &mut at, r, c) + { + x.runs = runs.clone(); + } + } + for (r, c, n) in &sx.inline { + let runs = rich_runs(n, cx.styles); + if !runs.is_empty() + && let Some(x) = cell_at(&mut sh.cells, &mut at, *r, *c) + { + x.runs = runs; } } + for &(r, c, vm) in &sx.vm { + let Some(Some(o)) = cx.objects.get(vm as usize - 1) else { continue }; + let Some(x) = cell_at(&mut sh.cells, &mut at, r, c) else { continue }; + // The stored value of a picture or data type cell is #VALUE!. + if x.f.is_none() || matches!(x.v, Value::Err { .. }) { + x.v = match (&o.text, o.kind.as_str()) { + (Some(t), "entity") => Value::Str(t.clone()), + _ => Value::Empty, + }; + } + x.obj = Some(o.clone()); + } for (r, c, f) in sx.shared_fixes { if let Some(x) = cell_at(&mut sh.cells, &mut at, r, c) { x.f = Some(f); @@ -514,6 +606,13 @@ fn read_sheet(pkg: &mut Pkg, part: &str, kind: &str, sh: &mut Sheet, cx: &mut Sh if let Some(rel) = n.attr("id").and_then(|id| rels.get(id)) { let target = rel.target.clone(); sh.charts.extend(charts(pkg, &target)); + sh.shapes.extend(drawing_shapes(pkg, &target)); + } + } + "legacyDrawing" => { + if let Some(rel) = n.attr("id").and_then(|id| rels.get(id)) { + let target = rel.target.clone(); + sh.shapes.extend(form_controls(pkg, &target)); } } "extLst" => { @@ -783,6 +882,263 @@ fn table(n: &Node) -> Option
{ }) } +// ---------------------------------------------------------------- rich text & rich values + +/// Shared strings whose runs are formatted differently, by index. Streams the +/// part so a large plain string table costs nothing kept. +fn rich_strings(pkg: &mut Pkg, part: &str, st: &Styles) -> HashMap> { + match pkg.reader(part) { + Some(mut rd) => rich_string_table(&mut rd, st), + None => HashMap::new(), + } +} + +fn rich_string_table(rd: &mut Reader, st: &Styles) -> HashMap> { + let mut out = HashMap::new(); + let (mut buf, mut i) = (Vec::new(), 0u32); + loop { + buf.clear(); + match rd.read_event_into(&mut buf) { + Ok(Event::Start(e)) if e.local_name().as_ref() == "si" => { + let Ok(n) = read_children(rd, "si".into(), Vec::new(), &mut buf) else { break }; + if n.child("r").is_some() { + let runs = rich_runs(&n, st); + if !runs.is_empty() { + out.insert(i, runs); + } + } + i += 1; + } + Ok(Event::Empty(e)) if e.local_name().as_ref() == "si" => i += 1, + Ok(Event::Eof) | Err(_) => break, + _ => {} + } + } + out +} + +/// Runs of an `` / ``, merged where formatting repeats; empty when +/// every run looks the same. +fn rich_runs(n: &Node, st: &Styles) -> Vec { + let mut out: Vec = Vec::new(); + for k in &n.kids { + let (text, f) = match k.name.as_str() { + "t" => (k.text.clone(), Style::default()), + "r" => (k.children("t").map(|t| t.text.as_str()).collect(), k.child("rPr").map(|p| st.font_style(p)).unwrap_or_default()), + _ => continue, + }; + match out.last_mut() { + Some(last) if last.f == f => last.text.push_str(&text), + _ if text.is_empty() => {} + _ => out.push(Run { text, f }), + } + } + if !crate::model::runs_differ(&out) { + out.clear(); + } + out +} + +/// A rich value's structure type and its (key, key type, value) fields. +type RichFields<'a> = (&'a str, Vec<(&'a str, &'a str, &'a str)>); + +/// What each cell `vm` index points at: a picture placed in the cell or a +/// linked data type. Follows metadata.xml → rich value → its structure (and, +/// for pictures, richValueRel.xml → the image part). +fn rich_values(pkg: &mut Pkg) -> Vec> { + let Some(meta) = pkg.node("xl/metadata.xml") else { return Vec::new() }; + let types: Vec<&str> = meta.child("metadataTypes").map(|t| t.children("metadataType").map(|m| m.attr("name").unwrap_or("")).collect()).unwrap_or_default(); + let rv_index: Vec> = meta + .children("futureMetadata") + .find(|f| f.attr("name") == Some("XLRICHVALUE")) + .map(|f| f.children("bk").map(|bk| bk.all("rvb").first().and_then(|r| r.num("i"))).collect()) + .unwrap_or_default(); + let structs: Vec<(String, Vec<(String, String)>)> = pkg + .node("xl/richData/rdrichvaluestructure.xml") + .map(|n| { + n.children("s") + .map(|s| { + let keys = s.children("k").map(|k| (k.attr("n").unwrap_or("").to_string(), k.attr("t").unwrap_or("").to_string())).collect(); + (s.attr("t").unwrap_or("").to_string(), keys) + }) + .collect() + }) + .unwrap_or_default(); + let values: Vec<(usize, Vec)> = pkg + .node("xl/richData/rdrichvalue.xml") + .map(|n| n.children("rv").map(|rv| (rv.num("s").unwrap_or(0) as usize, rv.children("v").map(|v| v.text.clone()).collect())).collect()) + .unwrap_or_default(); + let rel_part = "xl/richData/richValueRel.xml"; + let rels = pkg.rels(rel_part); + let images: Vec> = pkg + .node(rel_part) + .map(|n| n.children("rel").map(|r| r.attr("id").and_then(|id| rels.get(id)).map(|x| x.target.clone())).collect()) + .unwrap_or_default(); + + // A rich value's fields by key name. + let fields = |i: usize| -> Option> { + let (s, vs) = values.get(i)?; + let (t, keys) = structs.get(*s)?; + Some((t.as_str(), keys.iter().zip(vs).map(|((k, kt), v)| (k.as_str(), kt.as_str(), v.as_str())).collect())) + }; + let object = |i: usize| -> Option { + let (t, kv) = fields(i)?; + let get = |name: &str| kv.iter().find(|(k, _, _)| *k == name).map(|(_, _, v)| v.to_string()); + if t.to_ascii_lowercase().contains("image") { + let target = get("_rvRel:LocalImageIdentifier").and_then(|r| r.parse::().ok()).and_then(|r| images.get(r).cloned().flatten()); + return Some(CellObject { kind: "image".into(), text: get("Text").filter(|t| !t.is_empty()), target, props: Vec::new() }); + } + let display = get("_DisplayString")?; + let mut props = Vec::new(); + for (k, kt, v) in &kv { + if k.starts_with('_') || k.starts_with('%') || props.len() >= 30 { + continue; + } + let v = match *kt { + "" | "s" | "b" | "d" => v.to_string(), + // Nested rich value: show its display name. + "r" => match v.parse::().ok().and_then(fields) { + Some((_, nested)) => match nested.iter().find(|(k, _, _)| *k == "_DisplayString") { + Some((_, _, d)) => d.to_string(), + None => continue, + }, + None => continue, + }, + _ => continue, + }; + props.push((k.to_string(), v)); + } + Some(CellObject { kind: "entity".into(), text: Some(display), target: None, props }) + }; + meta.child("valueMetadata") + .map(|vm| { + vm.children("bk") + .map(|bk| { + let rc = bk.child("rc")?; + let ty = types.get(rc.num("t")?.checked_sub(1)? as usize)?; + if *ty != "XLRICHVALUE" { + return None; + } + object((*rv_index.get(rc.num("v")? as usize)?)? as usize) + }) + .collect() + }) + .unwrap_or_default() +} + +// ---------------------------------------------------------------- shapes + +/// Pictures, shapes and text boxes on a drawing part (charts are read apart). +fn drawing_shapes(pkg: &mut Pkg, drawing: &str) -> Vec { + let Some(d) = pkg.node(drawing) else { return Vec::new() }; + let mut out = Vec::new(); + for a in d.kids.iter().filter(|k| k.name.ends_with("Anchor")) { + let rect = anchor_rect(a); + for k in &a.kids { + drawn(k, rect, &mut out); + } + } + out +} + +fn drawn(k: &Node, anchor: Option, out: &mut Vec) { + let props = |nv: &str| k.child(nv).and_then(|n| n.child("cNvPr")); + let mut shape = |kind: &str, p: Option<&Node>, text: Option| { + let alt = p.and_then(|p| p.attr("descr")).filter(|d| !d.is_empty()).map(String::from); + let on_click = k.attr("macro").and_then(macro_name); + if kind != "picture" && text.is_none() && on_click.is_none() && alt.is_none() { + return; // plain lines and boxes + } + out.push(Shape { + kind: kind.into(), + name: p.and_then(|p| p.attr("name")).unwrap_or("").into(), + anchor, + text, + alt, + on_click, + linked_cell: None, + }); + }; + match k.name.as_str() { + "pic" => shape("picture", props("nvPicPr"), None), + "sp" => { + let text_box = k.child("nvSpPr").and_then(|n| n.child("cNvSpPr")).is_some_and(|c| c.flag("txBox")); + let text = k.child("txBody").map(|b| b.children("p").map(|p| p.runs()).collect::>().join("\n").trim().to_string()).filter(|t| !t.is_empty()); + shape(if text_box { "text box" } else { "shape" }, props("nvSpPr"), text); + } + "grpSp" => { + for kid in &k.kids { + drawn(kid, anchor, out); + } + } + _ => {} + } +} + +/// Cells a drawing anchor covers. An edge at offset 0 sits on the cell's +/// border, so that row or column isn't covered. +fn anchor_rect(a: &Node) -> Option { + let pos = |n: &Node| -> Option<(u32, u32, bool)> { + let num = |k: &str| n.child(k)?.text.trim().parse::().ok(); + Some((num("row")?, num("col")?, num("rowOff").unwrap_or(0) == 0 && num("colOff").unwrap_or(0) == 0)) + }; + let (r0, c0, _) = pos(a.child("from")?)?; + let (mut r1, mut c1) = (r0, c0); + if let Some((r, c, on_edge)) = a.child("to").and_then(pos) { + let back = |v: u32, start: u32| if on_edge && v > start { v - 1 } else { v }; + (r1, c1) = (back(r, r0), back(c, c0)); + } + Some(Rect::new(r0, c0, r1, c1)) +} + +/// `[0]!RollUp`, `'Book.xlsm'!Module1.RollUp` → `RollUp`, `Module1.RollUp`. +fn macro_name(s: &str) -> Option { + let s = s.trim().trim_start_matches('='); + let name = s.rsplit('!').next().unwrap_or(s).trim_matches('\'').trim(); + (!name.is_empty()).then(|| name.to_string()) +} + +/// Form controls (buttons, checkboxes, dropdowns) from a VML drawing. +/// Comment boxes live there too and are skipped. +fn form_controls(pkg: &mut Pkg, part: &str) -> Vec { + let Some(n) = pkg.node(part) else { return Vec::new() }; + let mut out = Vec::new(); + for sh in n.all("shape") { + let Some(cd) = sh.child("ClientData") else { continue }; + let kind = match cd.attr("ObjectType").unwrap_or("") { + "Note" => continue, + "Button" => "button", + "Checkbox" => "checkbox", + "Drop" => "dropdown", + "List" => "list box", + "Radio" => "option button", + "Spin" => "spinner", + "Scroll" => "scroll bar", + "GBox" => "group box", + "Label" => "label", + "Edit" => "edit box", + other => other, + } + .to_ascii_lowercase(); + let text = sh.child("textbox").map(|t| t.all_text().trim().to_string()).filter(|t| !t.is_empty()); + let anchor = cd.child("Anchor").and_then(|a| { + let v: Vec = a.text.split(',').filter_map(|x| x.trim().parse().ok()).collect(); + (v.len() == 8).then(|| Rect::new(v[2], v[0], v[6], v[4])) + }); + let fmla = |k: &str| cd.child(k).map(|x| x.text.trim().to_string()).filter(|t| !t.is_empty()); + out.push(Shape { + kind, + name: sh.attr("alt").unwrap_or("").into(), + anchor, + text, + alt: None, + on_click: fmla("FmlaMacro").as_deref().and_then(macro_name), + linked_cell: fmla("FmlaLink"), + }); + } + out +} + // ---------------------------------------------------------------- charts & pivots /// Charts on a drawing part, in the order they're anchored. @@ -879,6 +1235,204 @@ fn pivot(pkg: &mut Pkg, part: &str) -> Option { }) } +// ---------------------------------------------------------------- workbook settings + +fn calc(root: &Node) -> Option { + let c = root.child("calcPr")?; + let mode = match c.attr("calcMode") { + Some("manual") => "manual", + Some("autoNoTable") => "automatic except tables", + _ => "automatic", + }; + let iterative = c.flag("iterate"); + if !iterative && mode == "automatic" { + return None; + } + Some(Calc { + mode: mode.into(), + iterative, + max_iterations: iterative.then(|| c.num("iterateCount").unwrap_or(100)), + max_change: iterative.then(|| c.attr("iterateDelta").and_then(|d| d.parse().ok()).unwrap_or(0.001)), + }) +} + +/// Core, extended and custom document properties, in that order. +fn properties(pkg: &mut Pkg) -> Vec<(String, String)> { + let rels = pkg.rels(""); + let part = |kind: &str| rels.values().find(|r| r.kind == kind).map(|r| r.target.clone()); + let mut out: Vec<(String, String)> = Vec::new(); + let mut add = |k: &str, v: &str| { + let v = v.trim(); + if !v.is_empty() { + out.push((k.to_string(), v.to_string())); + } + }; + if let Some(n) = part("core-properties").and_then(|p| pkg.node(&p)) { + for (tag, name) in [ + ("title", "title"), + ("subject", "subject"), + ("creator", "author"), + ("description", "description"), + ("keywords", "keywords"), + ("category", "category"), + ("contentStatus", "status"), + ("lastModifiedBy", "last modified by"), + ("created", "created"), + ("modified", "modified"), + ] { + if let Some(x) = n.child(tag) { + add(name, &x.text); + } + } + } + if let Some(n) = part("extended-properties").and_then(|p| pkg.node(&p)) { + for (tag, name) in [("Company", "company"), ("Manager", "manager")] { + if let Some(x) = n.child(tag) { + add(name, &x.text); + } + } + } + if let Some(n) = part("custom-properties").and_then(|p| pkg.node(&p)) { + for p in n.children("property") { + if let (Some(name), Some(v)) = (p.attr("name"), p.kids.first()) { + add(name, &v.text); + } + } + } + out +} + +fn connections(pkg: &mut Pkg, part: &str) -> Vec { + let Some(n) = pkg.node(part) else { return Vec::new() }; + let mut out = Vec::new(); + for c in n.children("connection") { + let mut x = Connection { + name: c.attr("name").unwrap_or("").into(), + description: c.attr("description").map(String::from), + refresh_on_open: c.flag("refreshOnLoad"), + kind: match c.attr("type") { + Some("1") => "odbc", + Some("2") => "dao", + Some("3") => "file", + Some("4") => "web", + Some("5") => "oledb", + Some("6") => "text", + Some("7") => "ado", + Some("8") => "dsp", + _ => "connection", + } + .into(), + ..Default::default() + }; + if let Some(db) = c.child("dbPr") { + let conn = db.attr("connection").unwrap_or(""); + // Power Query loads go through the Mashup provider; Location names the query. + if conn.contains("Microsoft.Mashup.OleDb") { + x.kind = "power query".into(); + x.source = conn_field(conn, "location"); + } else { + x.source = Some(scrub_secrets(conn)).filter(|s| !s.is_empty()); + } + x.command = db.attr("command").map(String::from); + } else if let Some(w) = c.child("webPr") { + x.source = w.attr("url").map(String::from); + } else if let Some(t) = c.child("textPr") { + x.source = t.attr("sourceFile").map(String::from); + } else if let Some(o) = c.child("olapPr") { + x.kind = "olap".into(); + x.source = o.attr("connection").map(scrub_secrets); + } + out.push(x); + } + out +} + +fn conn_field(conn: &str, key: &str) -> Option { + conn.split(';').find_map(|kv| kv.split_once('=').filter(|(k, _)| k.trim().eq_ignore_ascii_case(key)).map(|(_, v)| v.trim().to_string())) +} + +/// A connection string without its password. +fn scrub_secrets(conn: &str) -> String { + conn.split(';') + .filter(|kv| { + let k = kv.split('=').next().unwrap_or("").trim().to_ascii_lowercase(); + !matches!(k.as_str(), "pwd" | "password" | "jet oledb:database password") + }) + .collect::>() + .join(";") +} + +/// Power Query (M) code: base64 in a `customXml` part, holding a zip with +/// `Formulas/Section1.m`. One script per query. +fn power_query(pkg: &mut Pkg) -> Vec