diff --git a/README.md b/README.md index eb96529..4c91007 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,9 @@ Go-idiomatic rather than a gem port. optional **shaped-text** API (GSUB/GPOS via go-opentype) for Arabic/Indic/CJK. - **Images** — JPEG embedded directly (DCTDecode); PNG and any `image.Image` rasterised as XObjects (FlateDecode) with an `/SMask` for alpha. + **Pixel-identical bitmaps are shared**: each image is content-addressed by its + *uncompressed* samples, so a repeat is embedded — and compressed — once per + document and every placement, on any page, references the one XObject. - **Pages** — standard sizes (A3/A4/A5/Letter/Legal/Tabloid), portrait/landscape, custom sizes; `Pt`/`Mm`/`In` unit helpers. - **Widget bridge** — `Page.AddWidget` and `Page.AddWidgetVector` "print" a diff --git a/doc.go b/doc.go index 729aeb3..6786e9d 100644 --- a/doc.go +++ b/doc.go @@ -39,11 +39,29 @@ // charstring-subsetted FontFile3 / CIDFontType0 whose glyph numbering is // preserved, so an Identity /CIDToGIDMap suffices. // +// # Images +// +// DrawJPEG stores the original JPEG bytes as a DCTDecode stream, so nothing is +// re-encoded. DrawPNG and DrawImage walk any image.Image into 8-bit DeviceRGB +// samples compressed with FlateDecode, and an image that is not fully opaque +// gets its alpha channel as a DeviceGray /SMask. +// +// An image is content-addressed by its uncompressed samples together with the +// width, height, colour space, bits per component and the alpha samples — the +// raw JPEG bytes in the DCTDecode case. Pixel-identical bitmaps therefore share +// a single XObject across the whole document, whichever page paints them and +// whichever entry point embedded them; the samples are hashed before they are +// compressed, so a repeat never pays for compression twice. This matters for +// pages that reuse one icon many times: an HTML renderer feeding pdfkit 166 +// rasterised copies of a handful of icon SVGs now emits a handful of streams. +// // # Determinism // // With the zero Options the output contains no timestamps and a content-derived // /ID, so identical inputs produce byte-identical documents. Set Options.Now to -// stamp creation and modification dates. +// stamp creation and modification dates. The image cache is consulted but never +// iterated: object order and /Im numbering follow first-sighting order, so +// deduplication does not put map iteration on the output path. // // # Widget bridge // diff --git a/document.go b/document.go index eaeaf35..057a786 100644 --- a/document.go +++ b/document.go @@ -60,6 +60,12 @@ type Document struct { use map[*Font]*fontUse // per-document glyph usage, keyed by font images []*imageXObject // registration order; index drives the /Im name outline []outlineItem // document outline (bookmarks), in document order + + // imgCache maps an image's content address (see imageKey) to the XObject + // already registered for it, so a bitmap drawn many times is embedded + // once. It is consulted, never iterated: the emitted order and the /Im + // numbering come from the images slice, so output stays deterministic. + imgCache map[string]*imageXObject } // outlineItem is one entry in the document outline (the viewer's bookmark tree): @@ -85,9 +91,10 @@ func (d *Document) AddOutlineItem(title string, level, pageIndex int) { // New returns a new, empty Document configured by opts. func New(opts Options) *Document { return &Document{ - opts: opts, - fontIx: map[*Font]int{}, - use: map[*Font]*fontUse{}, + opts: opts, + fontIx: map[*Font]int{}, + use: map[*Font]*fontUse{}, + imgCache: map[string]*imageXObject{}, } } @@ -124,6 +131,21 @@ func (d *Document) registerImage(x *imageXObject) string { return name } +// imageFor returns the XObject registered for the content address key, calling +// build only the first time that content is seen. Every later draw of a +// pixel-identical bitmap — on this page or any other, since the cache belongs +// to the document — reuses the same object and the same /Im name, so it +// costs one stream in the file instead of one per placement. +func (d *Document) imageFor(key string, build func() *imageXObject) *imageXObject { + if x, ok := d.imgCache[key]; ok { + return x + } + x := build() + x.name = d.registerImage(x) + d.imgCache[key] = x + return x +} + // builder assembles the flat list of indirect objects and assigns their // numbers. Object number n lives at objs[n-1]. type builder struct { diff --git a/image.go b/image.go index 506e6da..de8c7b6 100644 --- a/image.go +++ b/image.go @@ -6,6 +6,7 @@ package pdfkit import ( "bytes" + "crypto/sha256" "fmt" "image" "image/png" @@ -25,9 +26,29 @@ type imageXObject struct { smask *imageXObject } +// imageKey content-addresses an image by the bytes a PDF consumer would see +// once the filter is undone: the *uncompressed* samples plus the geometry and +// colour parameters that give them meaning, and the alpha samples that would +// become a soft mask. The lengths go into the digest ahead of the payloads so +// no shift of bytes between the two slices can forge a match. +// +// Hashing the raw samples (rather than the filtered stream) is what lets a +// duplicate be recognised *before* it pays for compression. +func imageKey(filter, colorSpace string, w, h, bpc int, samples, alpha []byte) string { + sum := sha256.New() + fmt.Fprintf(sum, "%s|%s|%d|%d|%d|%d|%d|", filter, colorSpace, w, h, bpc, len(samples), len(alpha)) + sum.Write(samples) + sum.Write(alpha) + return string(sum.Sum(nil)) +} + // DrawImage embeds img and paints it into the rectangle r (in points). Any // alpha channel becomes a soft mask, so partially transparent images composite // correctly. Sample data is FlateDecode-compressed. +// +// Pixel-identical images share one XObject document-wide: the samples are +// hashed before they are compressed, so redrawing the same bitmap costs a hash +// and nothing else. func (p *Page) DrawImage(img image.Image, r Rect) { b := img.Bounds() w, h := b.Dx(), b.Dy() @@ -44,24 +65,31 @@ func (p *Page) DrawImage(img image.Image, r Rect) { } } } - x := &imageXObject{ - width: w, - height: h, - colorSpace: "DeviceRGB", - bpc: 8, - filter: "FlateDecode", - data: flateCompress(rgb), + if !hasAlpha { + alpha = nil // a fully opaque image gets no soft mask, and keys as such } - if hasAlpha { - x.smask = &imageXObject{ + key := imageKey("FlateDecode", "DeviceRGB", w, h, 8, rgb, alpha) + x := p.doc.imageFor(key, func() *imageXObject { + x := &imageXObject{ width: w, height: h, - colorSpace: "DeviceGray", + colorSpace: "DeviceRGB", bpc: 8, filter: "FlateDecode", - data: flateCompress(alpha), + data: flateCompress(rgb), } - } + if alpha != nil { + x.smask = &imageXObject{ + width: w, + height: h, + colorSpace: "DeviceGray", + bpc: 8, + filter: "FlateDecode", + data: flateCompress(alpha), + } + } + return x + }) p.placeImage(x, r) } @@ -95,21 +123,24 @@ func (p *Page) DrawJPEG(data []byte, r Rect) error { default: return fmt.Errorf("pdfkit: unsupported JPEG component count %d", comps) } - x := &imageXObject{ - width: w, - height: h, - colorSpace: cs, - bpc: 8, - filter: "DCTDecode", - data: data, - } + key := imageKey("DCTDecode", cs, w, h, 8, data, nil) + x := p.doc.imageFor(key, func() *imageXObject { + return &imageXObject{ + width: w, + height: h, + colorSpace: cs, + bpc: 8, + filter: "DCTDecode", + data: data, + } + }) p.placeImage(x, r) return nil } -// placeImage registers the XObject and emits the operators to paint it into r. +// placeImage emits the operators that paint an already-registered XObject +// into r, and records it as a resource this page uses. func (p *Page) placeImage(x *imageXObject, r Rect) { - x.name = p.doc.registerImage(x) p.usedImages[x] = true p.Save() p.Transform(r.Width, 0, 0, r.Height, r.X, r.Y) diff --git a/image_test.go b/image_test.go index a7d9e31..8cd2afe 100644 --- a/image_test.go +++ b/image_test.go @@ -7,6 +7,7 @@ package pdfkit import ( "bytes" "image" + "image/color" "image/jpeg" "strings" "testing" @@ -88,6 +89,189 @@ func TestJPEGInfoErrors(t *testing.T) { } } +// writeDoc serialises doc and fails the test if that errors. +func writeDoc(t *testing.T, doc *Document) []byte { + t.Helper() + var buf bytes.Buffer + if err := doc.Write(&buf); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// countImageObjects counts the image XObject streams actually emitted. +func countImageObjects(t *testing.T, doc *Document) int { + t.Helper() + return bytes.Count(writeDoc(t, doc), []byte("/Subtype /Image")) +} + +func TestDrawImageDedupIdentical(t *testing.T) { + doc := New(Options{}) + p := doc.AddPage(A4) + img := makeOpaqueImage() + p.DrawImage(img, Rect{Width: 10, Height: 10}) + p.DrawImage(makeOpaqueImage(), Rect{X: 20, Width: 10, Height: 10}) // same pixels, other value + + if len(doc.images) != 1 { + t.Fatalf("registered %d XObjects, want 1", len(doc.images)) + } + if n := countImageObjects(t, doc); n != 1 { + t.Errorf("emitted %d image objects, want 1", n) + } + // Both placements must still paint, and both through the shared name. + if got := strings.Count(content(p), "/Im0 Do"); got != 2 { + t.Errorf("/Im0 Do appears %d times, want 2", got) + } + if strings.Contains(content(p), "/Im1") { + t.Error("a second resource name leaked into the content stream") + } +} + +func TestDrawImageDistinctNotDeduped(t *testing.T) { + doc := New(Options{}) + p := doc.AddPage(A4) + p.DrawImage(makeOpaqueImage(), Rect{Width: 10, Height: 10}) + p.DrawImage(makeTestImage(), Rect{X: 20, Width: 10, Height: 10}) // different pixels *and* alpha + + if len(doc.images) != 2 { + t.Fatalf("registered %d XObjects, want 2", len(doc.images)) + } + // Two placements, two names; the second carries a soft mask, so three + // image streams reach the file. + if n := countImageObjects(t, doc); n != 3 { + t.Errorf("emitted %d image objects, want 3 (two images + one soft mask)", n) + } + c := content(p) + if !strings.Contains(c, "/Im0 Do") || !strings.Contains(c, "/Im1 Do") { + t.Errorf("both placements should paint distinct XObjects: %q", c) + } +} + +func TestDrawImageAlphaIsPartOfTheKey(t *testing.T) { + // Same RGB samples, different alpha: the two must not collapse into one. + opaque := image.NewNRGBA(image.Rect(0, 0, 2, 2)) + translucent := image.NewNRGBA(image.Rect(0, 0, 2, 2)) + for y := 0; y < 2; y++ { + for x := 0; x < 2; x++ { + opaque.Set(x, y, color.NRGBA{R: 200, G: 100, B: 50, A: 255}) + translucent.Set(x, y, color.NRGBA{R: 200, G: 100, B: 50, A: 128}) + } + } + doc := New(Options{}) + p := doc.AddPage(A4) + p.DrawImage(opaque, Rect{Width: 4, Height: 4}) + p.DrawImage(translucent, Rect{X: 10, Width: 4, Height: 4}) + + if len(doc.images) != 2 { + t.Fatalf("alpha ignored by the key: registered %d XObjects, want 2", len(doc.images)) + } + if doc.images[0].smask != nil { + t.Error("opaque image should have no soft mask") + } + if doc.images[1].smask == nil { + t.Error("translucent image should have a soft mask") + } +} + +func TestDrawPNGDedup(t *testing.T) { + doc := New(Options{}) + p := doc.AddPage(A4) + encoded := pngBytes(makeTestImage()) + for i := 0; i < 3; i++ { + if err := p.DrawPNG(encoded, Rect{X: float64(i * 10), Width: 5, Height: 5}); err != nil { + t.Fatal(err) + } + } + if len(doc.images) != 1 { + t.Fatalf("registered %d XObjects, want 1", len(doc.images)) + } + // One image plus its soft mask. + if n := countImageObjects(t, doc); n != 2 { + t.Errorf("emitted %d image objects, want 2 (image + soft mask)", n) + } + if got := strings.Count(content(p), "/Im0 Do"); got != 3 { + t.Errorf("/Im0 Do appears %d times, want 3", got) + } +} + +func TestDrawJPEGDedup(t *testing.T) { + doc := New(Options{}) + p := doc.AddPage(A4) + data := jpegBytes() + if err := p.DrawJPEG(data, Rect{Width: 8, Height: 8}); err != nil { + t.Fatal(err) + } + // A byte-identical copy, not the same slice, so identity cannot be doing + // the work. + if err := p.DrawJPEG(append([]byte(nil), data...), Rect{X: 20, Width: 8, Height: 8}); err != nil { + t.Fatal(err) + } + if err := p.DrawJPEG(craftJPEG(4), Rect{Width: 1, Height: 1}); err != nil { + t.Fatal(err) + } + if len(doc.images) != 2 { + t.Fatalf("registered %d XObjects, want 2", len(doc.images)) + } + if n := countImageObjects(t, doc); n != 2 { + t.Errorf("emitted %d image objects, want 2", n) + } +} + +func TestImageDedupAcrossPages(t *testing.T) { + doc := New(Options{}) + p1 := doc.AddPage(A4) + p2 := doc.AddPage(A4) + p1.DrawImage(makeOpaqueImage(), Rect{Width: 10, Height: 10}) + p2.DrawImage(makeOpaqueImage(), Rect{Width: 10, Height: 10}) + + if len(doc.images) != 1 { + t.Fatalf("registered %d XObjects, want 1 shared across pages", len(doc.images)) + } + if n := countImageObjects(t, doc); n != 1 { + t.Errorf("emitted %d image objects, want 1", n) + } + // Each page must still list it in its own /Resources /XObject dictionary. + for i, p := range []*Page{p1, p2} { + if !p.usedImages[doc.images[0]] { + t.Errorf("page %d does not claim the shared XObject as a resource", i) + } + if !strings.Contains(content(p), "/Im0 Do") { + t.Errorf("page %d does not paint the shared XObject", i) + } + } + if n := bytes.Count(writeDoc(t, doc), []byte("/XObject <<")); n != 2 { + t.Errorf("%d /XObject resource dictionaries, want one per page", n) + } +} + +func TestImageDedupStaysDeterministic(t *testing.T) { + build := func() *Document { + doc := New(Options{}) + p := doc.AddPage(A4) + // Interleave repeats and fresh content so a map-ordered registration + // would show up as a difference between the two writes. + p.DrawImage(makeOpaqueImage(), Rect{Width: 4, Height: 4}) + p.DrawImage(makeTestImage(), Rect{X: 10, Width: 4, Height: 4}) + p.DrawImage(makeOpaqueImage(), Rect{X: 20, Width: 4, Height: 4}) + if err := p.DrawJPEG(jpegBytes(), Rect{X: 30, Width: 4, Height: 4}); err != nil { + t.Fatal(err) + } + p.DrawImage(makeTestImage(), Rect{X: 40, Width: 4, Height: 4}) + return doc + } + doc := build() + first, second := writeDoc(t, doc), writeDoc(t, doc) + if !bytes.Equal(first, second) { + t.Error("two writes of one document differ") + } + if other := writeDoc(t, build()); !bytes.Equal(first, other) { + t.Error("two identically built documents differ") + } + if len(doc.images) != 3 { + t.Errorf("registered %d XObjects, want 3", len(doc.images)) + } +} + func TestShortSOF(t *testing.T) { // SOF marker with a declared length under 8 bytes. data := []byte{0xFF, 0xD8, 0xFF, 0xC0, 0x00, 0x06, 0x08, 0x00, 0x10, 0x00, 0x20}