diff --git a/check_exposed.go b/check_exposed.go index 2168bca..6d0ffb4 100644 --- a/check_exposed.go +++ b/check_exposed.go @@ -96,11 +96,14 @@ func dataType(p string) string { // commit, the files git tracks at the deployed one are (Repo.Tracked), and // what Composer installs (plugins, vendor) is only counted. notes say what // was only counted. Pure and deterministic: the probes ask about these -// paths, and the exposed check joins their answers back to them. -func Candidates(f *FileFacts, repo *Repo) (out []Candidate, notes []string) { +// paths, and the exposed check joins their answers back to them. site (nil +// when the collector didn't run) says which documents are media library +// attachments: a site without year/month folders keeps them in uploads' root. +func Candidates(f *FileFacts, repo *Repo, site *SiteFacts) (out []Candidate, notes []string) { if f == nil { return nil, nil } + attached := site.media() uploads := f.Content + "/uploads/" compare := repo != nil && repo.Dir != "" && !stale(f, repo) var tracked map[string]bool @@ -130,7 +133,7 @@ func Candidates(f *FileFacts, repo *Repo) (out []Candidate, notes []string) { t := dataType(p) rest := strings.TrimPrefix(p, uploads) doc := t == "document" || t == "archive" - isMedia := inUploads && doc && mediaRE.MatchString(rest) + isMedia := inUploads && doc && (mediaRE.MatchString(rest) || attached[path.Clean(rest)]) why := "" switch { case !inUploads && t == "document": @@ -200,7 +203,7 @@ func Candidates(f *FileFacts, repo *Repo) (out []Candidate, notes []string) { // are documents in an active plugin's own uploads folder (pluginUploads). func (c *checker) exposed() { f, w := c.in.Facts.Files, c.in.Facts.Web - cands, notes := Candidates(f, c.in.Repo) + cands, notes := Candidates(f, c.in.Repo, c.in.Facts.Site) for _, n := range notes { c.info("%s", n) } diff --git a/check_files_test.go b/check_files_test.go index c9bab7e..3768c5f 100644 --- a/check_files_test.go +++ b/check_files_test.go @@ -152,7 +152,7 @@ func TestExposed(t *testing.T) { {Path: "web/app/plugins/woocommerce/sample-data/x.csv", Size: 7, Mtime: 7}, // no install to compare {Path: "web/app/uploads/old.zip", Size: 8, Mtime: 8}, }} - cands, notes := Candidates(f, nil) + cands, notes := Candidates(f, nil, nil) var paths []string for _, c := range cands { paths = append(paths, c.Path) @@ -463,7 +463,7 @@ func TestCandidatesAtDeployedCommit(t *testing.T) { {Path: "web/app/export.csv"}, }} cands := func() (paths []string) { - c, _ := Candidates(f, repo) + c, _ := Candidates(f, repo, nil) for _, x := range c { paths = append(paths, x.Path) } diff --git a/collector.php b/collector.php index 9110e45..8c42089 100644 --- a/collector.php +++ b/collector.php @@ -671,6 +671,27 @@ function_exists('proc_nice') && @proc_nice(19); $wpda['linked_uploads'] = array_keys($wpda_linked); $wpda_timed('linked'); +// Documents that are media library attachments, as paths below uploads (a +// network site's with its sites//): uploaded on purpose, also where the +// site keeps its uploads in uploads' root rather than year/month folders. +$wpda_media = []; +empty($wpda_skip['content']) && $wpda_each(function (int $blog) use ($wpdb, $wpda_stream, $wpda_rows, &$wpda_media, &$wpda_errors) { + $prefix = is_multisite() && ! is_main_site($blog) ? "sites/{$blog}/" : ''; + $err = $wpda_stream( + "SELECT meta_id AS wpda_k, meta_value FROM {$wpdb->postmeta} WHERE meta_key = '_wp_attached_file' AND meta_value REGEXP '[.](pdf|docx?|xlsx?|pptx?|odt|ods|odp|rtf|csv|txt|zip|rar|7z)$'", + 'meta_id', + function ($r) use (&$wpda_media, $prefix) { + $wpda_media[$prefix . ltrim((string) $r->meta_value, '/')] = true; + }, + $wpda_rows + ); + if ($err !== null && $err !== 'truncated') { + $wpda_errors[] = 'media files: ' . $err; + } +}); +$wpda['media_files'] = array_keys($wpda_media); +$wpda_timed('media'); + // Files are the hasher's (hasher.php), which runs without WordPress. // Checksums against wordpress.org, for sites whose files can't be compared diff --git a/facts.go b/facts.go index ab30bb6..bbd48c2 100644 --- a/facts.go +++ b/facts.go @@ -110,12 +110,30 @@ type SiteFacts struct { Sensitive map[string]string `json:"sensitive"` // option (blog-tagged) → its value's hash // LinkedUploads are the files below uploads that published posts and their // fields link to (every blog's), as paths below uploads: linked on purpose. - LinkedUploads []string `json:"linked_uploads"` - Timings map[string]float64 `json:"timings"` // seconds per collector section - PeakMB float64 `json:"peak_mb"` // the PHP process's peak memory, WordPress included - Errors []string `json:"errors"` - - linkedSet map[string]bool + LinkedUploads []string `json:"linked_uploads"` + // MediaFiles are the documents below uploads that are media library + // attachments (every blog's _wp_attached_file, as paths below uploads): + // uploaded on purpose, wherever the site keeps its uploads. + MediaFiles []string `json:"media_files"` + Timings map[string]float64 `json:"timings"` // seconds per collector section + PeakMB float64 `json:"peak_mb"` // the PHP process's peak memory, WordPress included + Errors []string `json:"errors"` + + linkedSet, mediaSet map[string]bool +} + +// media is MediaFiles as a set; none for nil s. +func (s *SiteFacts) media() map[string]bool { + if s == nil { + return nil + } + if s.mediaSet == nil { + s.mediaSet = map[string]bool{} + for _, p := range s.MediaFiles { + s.mediaSet[path.Clean(p)] = true + } + } + return s.mediaSet } // linked is LinkedUploads as a set. diff --git a/owned_test.go b/owned_test.go index 68699e9..3b88b7b 100644 --- a/owned_test.go +++ b/owned_test.go @@ -87,6 +87,36 @@ func TestPluginOwnedUploadsPHP(t *testing.T) { } } +// A document that is a media library attachment is a media library upload +// wherever it sits (a site without year/month folders keeps them in uploads' +// root): counted, or for review when its name suggests people's data; one +// that isn't stays urgent. +func TestExposedMediaAttachments(t *testing.T) { + f := &FileFacts{Content: "web/app", Data: []DataFile{ + {Path: "web/app/uploads/Toimitussisältö-2018.pdf", Size: 1, Mtime: 1}, + {Path: "web/app/uploads/customers-2019.pdf", Size: 2, Mtime: 2}, + {Path: "web/app/uploads/sites/2/hinnasto.pdf", Size: 3, Mtime: 3}, + {Path: "web/app/uploads/stray.pdf", Size: 4, Mtime: 4}, + }} + served := map[string]Answer{} + for _, d := range f.Data { + served[d.Path] = Answer{Status: 200, Type: "application/pdf"} + } + s := site0() + s.MediaFiles = []string{"Toimitussisältö-2018.pdf", "customers-2019.pdf", "sites/2/hinnasto.pdf"} + r := Check(Input{Facts: Facts{At: now, Site: s, Files: f, Web: &WebFacts{Base: "https://site.example", Files: served}}}).Report + got := map[string]Severity{} + for _, fl := range kind(r, KindExposed) { + got[strings.SplitN(fl.Untrusted, " (", 2)[0]] = fl.Severity + } + if len(got) != 2 || got["web/app/uploads/stray.pdf"] != Urgent || got["web/app/uploads/customers-2019.pdf"] != Review { + t.Fatalf("exposed flags %v", got) + } + if !strings.Contains(noteTexts(r), "2 documents in the media library not checked") { + t.Errorf("notes %s", noteTexts(r)) + } +} + // A downloadable document a published page links to is counted; unlinked, or // linked with a name that suggests people's data, it's still flagged. func TestExposedLinkedDocuments(t *testing.T) { diff --git a/runner/runner.go b/runner/runner.go index f7abf60..7c03c63 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -303,7 +303,7 @@ func (r *run) gather(ctx context.Context, script string) audit.Facts { t := where t.Content = facts.Files.Content var o web.Options - cands, _ := audit.Candidates(facts.Files, r.repo) + cands, _ := audit.Candidates(facts.Files, r.repo, facts.Site) // Site is set before located closes for _, c := range cands { o.Files = append(o.Files, c.Path) }