diff --git a/.github/scripts/rows/fetch-symbols.ps1 b/.github/scripts/rows/fetch-symbols.ps1 index fdb4d14..118c4d7 100644 --- a/.github/scripts/rows/fetch-symbols.ps1 +++ b/.github/scripts/rows/fetch-symbols.ps1 @@ -18,7 +18,10 @@ $url, $packed = switch ($Browser) { if ($ResolveOnly) { $url; return } $target = if ($packed) { "$Out.gz" } else { $Out } curl.exe -fsSL --retry 3 -o $target $url -if ($LASTEXITCODE -ne 0) { throw "The symbol server has no PDB at $url yet" } +# curl exits with 22 when the server answers with an error, which for a symbol server means the PDB +# is not published yet. The caller may then make the row without it, so that case has its own type. +if ($LASTEXITCODE -eq 22) { throw [System.IO.FileNotFoundException]::new("The symbol server has no PDB at $url yet") } +if ($LASTEXITCODE -ne 0) { throw "Could not download ${url}: curl exited with $LASTEXITCODE" } if ($packed) { $in = [IO.File]::OpenRead($target) $gzip = [IO.Compression.GZipStream]::new($in, [IO.Compression.CompressionMode]::Decompress) diff --git a/.github/workflows/browser-rows.yml b/.github/workflows/browser-rows.yml index 505e7a7..d99737e 100644 --- a/.github/workflows/browser-rows.yml +++ b/.github/workflows/browser-rows.yml @@ -17,6 +17,12 @@ # A failure is never silent: a failed build opens or updates the issue named after it, with the step # that failed, its reason and the capture of the failing window. Each step records its reason in # `failure.txt` before it stops. +# +# A maker can ship a build before its PDB reaches the symbol server. Then the row is made from the +# fingerprints of the prior row's functions, each of which must match one place in the new module, +# and every offset, slot and size is read again from the new module as for a row made from symbols. +# Such a row is marked, so once the PDB appears the job makes the row again from it and compares the +# two. A difference opens an issue. name: Browser rows on: @@ -29,6 +35,15 @@ on: description: "Rows to take out of the published table, as browser@version separated by semicolons. Leave empty for a normal run." required: false default: "" + trial: + description: "Make rows with the code of the branch this run is started on, and keep them only in the run's files: nothing is proposed, published or reported." + type: boolean + required: false + default: false + only: + description: "With trial: the builds to make, as browser@version separated by semicolons, instead of those the feeds list." + required: false + default: "" permissions: contents: write @@ -107,7 +122,8 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: develop + # A trial makes rows with the code it is started on; every other run uses develop. + ref: ${{ inputs.trial && github.ref || 'develop' }} - name: Build the row tools working-directory: rivet @@ -126,6 +142,8 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} RIVET_TABLE_PUBLIC_KEY: ${{ vars.RIVET_TABLE_PUBLIC_KEY }} + TRIAL: ${{ inputs.trial }} + ONLY: ${{ inputs.only }} run: | $feeds = Join-Path $env:RUNNER_TEMP "feeds" New-Item -ItemType Directory -Force $feeds | Out-Null @@ -148,6 +166,20 @@ jobs: & "$tools/chrome-builds.exe" pending "$feeds/chrome-stable.json" "$feeds/chrome-beta.json" "$feeds/edge.json" "$feeds/brave.json" "$feeds/known.json" | Set-Content "$feeds/pending.json" if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Get-Content "$feeds/pending.json" + $only = @($env:ONLY -split ';' | ForEach-Object { $_.Trim() } | Where-Object { $_ }) + if ($only.Count) { + if ($env:TRIAL -ne 'true') { throw "Builds are named only for a trial run." } + # A named build need not be in the feeds any more: Edge's update service still serves it. + $listed = @(Get-Content -Raw "$feeds/pending.json" | ConvertFrom-Json) + $builds = @(foreach ($name in $only) { + $browser, $version = $name -split '@', 2 + $known = $listed | Where-Object { $_.browser -eq $browser -and $_.version -eq $version } | Select-Object -First 1 + if ($known) { $known } else { [pscustomobject]@{ browser = $browser; version = $version; channel = 'stable'; download = '' } } + }) + $json = $builds | ConvertTo-Json -Compress -AsArray + "builds=$json" >> $env:GITHUB_OUTPUT + exit 0 + } $builds = foreach ($build in @(Get-Content -Raw "$feeds/pending.json" | ConvertFrom-Json)) { # A proposal waits for its person, and a run must not spend its turn on it again. $branch = "feat/row-$($build.browser -replace ' ','-')-$($build.version)".ToLower() @@ -161,15 +193,48 @@ jobs: "::warning::$($build.browser) $($build.version) has no row and cannot be fetched yet: $_" } } + # A row made without its PDB is made again from the PDB once the symbol server has it, and + # the two are compared. Such a row may wait in develop's table or on a row branch. + $sources = @([pscustomobject]@{ source = 'develop'; file = 'rivet/runtime/windows/rows/table.json' }) + foreach ($line in @(git ls-remote --heads origin)) { + $branch = ($line -split "`t")[1] -replace '^refs/heads/', '' + if (-not ($branch -like 'feat/row-*' -or $branch -like 'feat/rows-*') -or $branch -like 'feat/row-pdb-*') { continue } + git fetch --quiet --depth 1 origin $branch + if ($LASTEXITCODE -ne 0) { continue } + $file = Join-Path $feeds ("table-" + ($branch -replace '[^A-Za-z0-9.]+', '-') + ".json") + git show "FETCH_HEAD:rivet/runtime/windows/rows/table.json" | Set-Content $file + if ($LASTEXITCODE -eq 0) { $sources += [pscustomobject]@{ source = $branch; file = $file } } + } + $open = @(gh issue list --state open --label browser-rows --limit 100 --json title | ConvertFrom-Json | ForEach-Object title) + $seen = @{} + $rechecks = foreach ($source in $sources) { + foreach ($row in @((Get-Content -Raw $source.file | ConvertFrom-Json).rows)) { + $mark = $row.made_without_pdb + if (-not $mark) { continue } + $browser = $row.frame.browser + $version = $row.frame.version + if ($seen["$browser $version"]) { continue } + $seen["$browser $version"] = $true + # A confirmation waits for review, and a difference waits on its issue. + $slug = "$($browser -replace ' ','-')-$version".ToLower() + git ls-remote --exit-code --heads origin "feat/row-pdb-$slug" | Out-Null + if ($LASTEXITCODE -eq 0) { continue } + if ($open -contains "Row differs from its PDB: $browser $version") { continue } + $url = ./.github/scripts/rows/fetch-symbols.ps1 -Browser $browser -Key $mark.pdb_key -Out unused -ResolveOnly + curl.exe -fsSL -r 0-0 -o NUL $url + if ($LASTEXITCODE -ne 0) { "$browser $version still has no PDB."; continue } + [pscustomobject]@{ browser = $browser; version = $version; channel = $mark.channel; download = $mark.download; recheck = $source.source } + } + } $global:LASTEXITCODE = 0 - $builds = @($builds | Where-Object { $_ -isnot [string] } | Select-Object -First ([int]$env:MOST_PER_RUN)) + $builds = @(@($builds) + @($rechecks) | Where-Object { $_ -and $_ -isnot [string] } | Select-Object -First ([int]$env:MOST_PER_RUN)) # Through the pipeline: -InputObject would wrap the whole list as one element of another list. $json = if ($builds.Count) { $builds | ConvertTo-Json -Compress -AsArray } else { "[]" } "builds=$json" >> $env:GITHUB_OUTPUT exit 0 - name: Tell a person that the watch failed - if: failure() + if: failure() && !inputs.trial env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -191,7 +256,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: develop + ref: ${{ inputs.trial && github.ref || 'develop' }} - uses: actions/download-artifact@v4 with: @@ -208,35 +273,114 @@ jobs: -Out (Join-Path $env:RUNNER_TEMP "browser") -Output $env:GITHUB_OUTPUT - name: Download the symbols of this build and of the row before it + id: symbols + env: + RECHECK: ${{ matrix.build.recheck }} run: | trap { Add-Content (Join-Path $env:RUNNER_TEMP failure.txt) "Download the symbols: $_"; break } $browser = "${{ matrix.build.browser }}" $version = "${{ matrix.build.version }}" - $prior = Join-Path $env:RUNNER_TEMP "prior.json" + $temp = $env:RUNNER_TEMP + $prior = Join-Path $temp "prior.json" $tables = @("--table", "rivet/runtime/windows/rows/table.json") - ./tools/rivet-rows.exe prior $browser $version @tables | Set-Content $prior + $before = $version + if ($env:RECHECK) { + # The row made without its PDB, from where it is kept, and the prior row it was made from. + $stored = Join-Path $temp "stored-table.json" + if ($env:RECHECK -eq 'develop') { + Copy-Item rivet/runtime/windows/rows/table.json $stored + } else { + git fetch --quiet --depth 1 origin $env:RECHECK + if ($LASTEXITCODE -ne 0) { throw "Could not fetch $env:RECHECK" } + git show "FETCH_HEAD:rivet/runtime/windows/rows/table.json" | Set-Content $stored + if ($LASTEXITCODE -ne 0) { throw "$env:RECHECK has no table" } + } + $row = @((Get-Content -Raw $stored | ConvertFrom-Json).rows) | + Where-Object { $_.frame.browser -eq $browser -and $_.frame.version -eq $version } | Select-Object -First 1 + if (-not $row.made_without_pdb) { throw "$env:RECHECK holds no row of $browser $version made without its PDB" } + $before = $row.made_without_pdb.prior_version + $tables += @("--table", $stored) + } + ./tools/rivet-rows.exe prior $browser $before @tables | Set-Content $prior if ($LASTEXITCODE -ne 0) { throw "rivet-rows found no prior row for $browser $version" } - $priorVersion = (Get-Content -Raw $prior | ConvertFrom-Json).frame.version - $symbols = (Get-Content -Raw rivet/runtime/windows/rows/table.json | ConvertFrom-Json).symbols - $priorKey = $symbols."$browser $priorVersion" + $priorRow = Get-Content -Raw $prior | ConvertFrom-Json + $priorVersion = $priorRow.frame.version + $priorKey = $null + foreach ($file in @($tables | Where-Object { $_ -ne '--table' })) { + if (-not $priorKey) { $priorKey = (Get-Content -Raw $file | ConvertFrom-Json).symbols."$browser $priorVersion" } + } if (-not $priorKey) { throw "The table holds no symbol key for the prior row $browser $priorVersion. Add it to rows/table.json with chrome-builds pdb-key." } $newKey = ./tools/chrome-builds.exe pdb-key "${{ steps.build.outputs.module }}" if ($LASTEXITCODE -ne 0) { throw "chrome-builds could not read the symbol key of the new module" } - ./.github/scripts/rows/fetch-symbols.ps1 -Browser $browser -Key $priorKey -Out (Join-Path $env:RUNNER_TEMP "prior.pdb") - ./.github/scripts/rows/fetch-symbols.ps1 -Browser $browser -Key ($newKey -split '/')[1] -Out (Join-Path $env:RUNNER_TEMP "new.pdb") + # The prior row's PDB names what the row is made from, so without it nothing is made. + ./.github/scripts/rows/fetch-symbols.ps1 -Browser $browser -Key $priorKey -Out (Join-Path $temp "prior.pdb") + try { + ./.github/scripts/rows/fetch-symbols.ps1 -Browser $browser -Key ($newKey -split '/')[1] -Out (Join-Path $temp "new.pdb") + "no_pdb=false" >> $env:GITHUB_OUTPUT + } catch [System.IO.FileNotFoundException] { + if ($env:RECHECK) { throw } + "::warning::$($_.Exception.Message). The row is made from the fingerprints of the $browser $priorVersion row." + "no_pdb=true" >> $env:GITHUB_OUTPUT + # The prior build gives the fingerprints when the prior row stored none, and in any case + # tells which prior addresses are function starts, so a match elsewhere does not count. + $found = Join-Path $temp "prior-build.txt" + $fetch = @{ Browser = $browser; Version = $priorVersion; Out = (Join-Path $temp "prior-browser"); Output = $found } + try { + ./.github/scripts/rows/fetch-browser.ps1 @fetch -Channel "${{ matrix.build.channel }}" + } catch { + if ("${{ matrix.build.channel }}" -eq 'stable') { throw "The symbol server has no PDB for $browser $version, and the prior build $priorVersion could not be fetched for its fingerprints: $_" } + ./.github/scripts/rows/fetch-browser.ps1 @fetch -Channel stable + } + $priorModule = (Get-Content $found | Where-Object { $_ -like 'module=*' } | Select-Object -First 1) -replace '^module=', '' + $priorModuleKey = ./tools/chrome-builds.exe pdb-key $priorModule + if (($priorModuleKey -split '/')[1] -ne $priorKey) { throw "The fetched $browser $priorVersion is not the build the prior row was made for" } + "prior_module=$priorModule" >> $env:GITHUB_OUTPUT + } - name: Make the row id: made + env: + NO_PDB: ${{ steps.symbols.outputs.no_pdb }} + PRIOR_MODULE: ${{ steps.symbols.outputs.prior_module }} + RECHECK: ${{ matrix.build.recheck }} + CHANNEL: ${{ matrix.build.channel }} + DOWNLOAD: ${{ matrix.build.download }} run: | trap { Add-Content (Join-Path $env:RUNNER_TEMP failure.txt) "Make the row: $_"; break } $temp = $env:RUNNER_TEMP - ./tools/chrome-builds.exe port "$temp/prior.json" "$temp/prior.pdb" "$temp/new.pdb" "${{ steps.build.outputs.module }}" "${{ matrix.build.version }}" "$temp/row.json" "$temp/report.json" - $code = $LASTEXITCODE - if ($code -ne 0 -and $code -ne 3) { throw "chrome-builds port exited with $code; report.json says why" } + $browser = "${{ matrix.build.browser }}" + $version = "${{ matrix.build.version }}" + if ($env:NO_PDB -eq 'true') { + # Every function is looked for by its fingerprint, and one that does not match exactly + # one place leaves the row for review. Offsets, slots and sizes are read as below. + ./tools/chrome-builds.exe find "$temp/prior.json" "$temp/prior.pdb" $env:PRIOR_MODULE "${{ steps.build.outputs.module }}" $version $env:CHANNEL "$env:DOWNLOAD" "$temp/row.json" "$temp/report.json" + $code = $LASTEXITCODE + if ($code -ne 0 -and $code -ne 3) { throw "chrome-builds find exited with $code" } + } else { + ./tools/chrome-builds.exe port "$temp/prior.json" "$temp/prior.pdb" "$temp/new.pdb" "${{ steps.build.outputs.module }}" $version "$temp/row.json" "$temp/report.json" + $code = $LASTEXITCODE + if ($code -ne 0 -and $code -ne 3) { throw "chrome-builds port exited with $code; report.json says why" } + } + if ($env:RECHECK) { + # The PDB is out now: the row made from it must agree with the row made without it. + ./tools/chrome-builds.exe pdb-check "$temp/stored-table.json" $browser $version "$temp/row.json" | Tee-Object "$temp/differences.txt" + $check = $LASTEXITCODE + if ($check -ne 0 -and $check -ne 4) { throw "chrome-builds pdb-check exited with $check" } + "checked=$(if ($check -eq 0) { 'agree' } else { 'differ' })" >> $env:GITHUB_OUTPUT + "for_review=recheck" >> $env:GITHUB_OUTPUT + Get-Content "$temp/report.json" + exit 0 + } # Fingerprints let a Rivet find these functions in a later build that has no row yet. A row - # whose addresses cannot all be fingerprinted is still a good row, just without them. - ./tools/chrome-builds.exe fingerprint "$temp/row.json" "${{ steps.build.outputs.module }}" "$temp/row.json" - if ($LASTEXITCODE -ne 0) { "The row has no fingerprints." } + # whose addresses cannot all be fingerprinted is still a good row, just without them. A row + # made without its PDB that is not vouched for still holds prior addresses, which would + # fingerprint the wrong code, so it gets none. + if ($env:NO_PDB -eq 'true' -and $code -ne 0) { + "A row made without its PDB that needs review gets no fingerprints." + } else { + ./tools/chrome-builds.exe fingerprint "$temp/row.json" "${{ steps.build.outputs.module }}" "$temp/row.json" + if ($LASTEXITCODE -ne 0) { "The row has no fingerprints." } + } "for_review=$($code -eq 3)".ToLower() >> $env:GITHUB_OUTPUT Get-Content "$temp/report.json" exit 0 @@ -307,10 +451,62 @@ jobs: ${{ runner.temp }}/session/features/traces/**/*.log ${{ runner.temp }}/session/features/*/*.png ${{ runner.temp }}/session/features/profiles/*/Crashpad/reports/*.dmp + ${{ runner.temp }}/differences.txt retention-days: 30 + - name: Tell a person that a row made without its PDB differs from it + if: steps.made.outputs.checked == 'differ' && !inputs.trial + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RECHECK: ${{ matrix.build.recheck }} + run: | + $temp = $env:RUNNER_TEMP + $browser = "${{ matrix.build.browser }}" + $version = "${{ matrix.build.version }}" + $reasons = Join-Path $temp "differences-reasons.txt" + Set-Content $reasons "The row in $env:RECHECK was made from fingerprints, and the row its PDB gives differs from it:" + Get-Content "$temp/differences.txt" | Add-Content $reasons + ./.github/scripts/rows/report-failure.ps1 ` + -Title "Row differs from its PDB: $browser $version" ` + -Reasons $reasons -Files "${{ steps.files.outputs.artifact-url }}" + + - name: Confirm a row made without its PDB + if: steps.made.outputs.checked == 'agree' && !inputs.trial + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RECHECK: ${{ matrix.build.recheck }} + run: | + $temp = $env:RUNNER_TEMP + $browser = "${{ matrix.build.browser }}" + $version = "${{ matrix.build.version }}" + git config user.name "rivet-browser-rows" + git config user.email "rivet-browser-rows@users.noreply.github.com" + # A row on its own branch is confirmed there. A row in develop is confirmed by a pull request. + if ($env:RECHECK -eq 'develop') { + $branch = "feat/row-pdb-$($browser -replace ' ','-')-$version".ToLower() + git switch -c $branch + } else { + $branch = $env:RECHECK + git fetch --quiet --depth 1 origin $branch + git switch -c $branch FETCH_HEAD + } + ./tools/chrome-builds.exe pdb-check rivet/runtime/windows/rows/table.json $browser $version "$temp/row.json" + if ($LASTEXITCODE -ne 0) { throw "chrome-builds pdb-check exited with $LASTEXITCODE on $branch" } + git add rivet/runtime/windows/rows/table.json + git commit -m "Fix: Confirm the row of $browser $version with its PDB" + git push origin "HEAD:$branch" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $note = "The row of $browser $version was made from fingerprints because its PDB was not published. The PDB is out now, the row made from it agrees, and the mark is removed in run $env:GITHUB_RUN_ID." + if ($env:RECHECK -eq 'develop') { + gh pr create --base develop --head $branch --title "Fix: Confirm the row of $browser $version with its PDB" --body $note + gh workflow run test.yml --ref $branch + } else { + gh pr comment $branch --body $note + $global:LASTEXITCODE = 0 + } + - name: Tell a person that this build failed - if: failure() + if: failure() && !inputs.trial env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -321,7 +517,7 @@ jobs: -Files "${{ steps.files.outputs.artifact-url }}" - name: Ask a person about a row the generator cannot vouch for - if: steps.made.outputs.for_review == 'true' + if: steps.made.outputs.for_review == 'true' && !inputs.trial env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -337,7 +533,11 @@ jobs: git commit -m "Feat: Propose a row for ${{ matrix.build.browser }} ${{ matrix.build.version }}" git push -u origin $branch $report = Get-Content -Raw "$temp/report.json" | ConvertFrom-Json - $body = "The generator could not vouch for this row, so it was not published.`n`n" + (($report.review | ForEach-Object { "- $_" }) -join "`n") + $body = "The generator could not vouch for this row, so it was not published.`n`n" + if ($report.made_without_pdb) { + $body += "The symbol server had no PDB for this build, so the row was made from the fingerprints of the $($report.prior_version) row: $($report.fingerprinted.Count) of $($report.addresses) addresses matched exactly one place. When the PDB appears, the row job makes the row again from it and compares the two.`n`n" + } + $body += (($report.review | ForEach-Object { "- $_" }) -join "`n") gh pr create --draft --base develop --head $branch --title "Feat: Propose a row for ${{ matrix.build.browser }} ${{ matrix.build.version }}" --body $body gh workflow run test.yml --ref $branch @@ -345,7 +545,7 @@ jobs: name: Publish the signed table needs: row # A row that failed its proof holds up nothing else: the others have passed theirs. - if: ${{ !cancelled() && needs.row.result != 'skipped' }} + if: ${{ !cancelled() && needs.row.result != 'skipped' && !inputs.trial }} runs-on: windows-latest defaults: run: diff --git a/rivet/platform/windows/src/bin/chrome-builds.rs b/rivet/platform/windows/src/bin/chrome-builds.rs index c278196..f3fb64b 100644 --- a/rivet/platform/windows/src/bin/chrome-builds.rs +++ b/rivet/platform/windows/src/bin/chrome-builds.rs @@ -8,6 +8,13 @@ //! `chrome-builds port ` //! makes a whole row for a new build from the row of a prior build, and exits 3 when the row needs //! a person's review. +//! `chrome-builds find ` +//! makes a candidate row for a new build whose PDB is not published, from the fingerprints of the +//! prior row: those it stored, or else those read now from the prior build's module. It marks the +//! row as made without its PDB and exits 3 when the row needs a person's review. +//! `chrome-builds pdb-check ` compares a row made +//! without its PDB with the row its PDB gives. It removes the mark when they agree and exits 4, +//! printing each difference, when they do not. //! `chrome-builds pending ` //! prints the builds that have no row yet, most urgent first. //! `chrome-builds table-key` prints a new signing key pair. @@ -65,6 +72,36 @@ fn main() -> ExitCode { } } } + ["find", prior, old_pdb, old_module, module, version, channel, download, row, report] => { + return match find( + Path::new(prior), + Path::new(old_pdb), + old_module, + Path::new(module), + version, + channel, + download, + Path::new(row), + Path::new(report), + ) { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::from(3), + Err(error) => { + eprintln!("chrome-builds: {error}"); + ExitCode::FAILURE + } + } + } + ["pdb-check", table, browser, version, row] => { + return match pdb_check(Path::new(table), browser, version, Path::new(row)) { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::from(4), + Err(error) => { + eprintln!("chrome-builds: {error}"); + ExitCode::FAILURE + } + } + } ["pending", stable, beta, edge, brave, known] => { pending(Path::new(stable), Path::new(beta), Path::new(edge), Path::new(brave), Path::new(known)) } @@ -86,7 +123,7 @@ fn main() -> ExitCode { ["table-withdraw", published, signed, builds @ ..] if !builds.is_empty() => { table_withdraw(Path::new(published), Path::new(signed), builds) } - _ => Err("usage: chrome-builds pdb-key | extract [frame-class] | port | pending | gate | fingerprint | release-sign ... | release-check | table-key | table-add | table-read | table-merge ... | table-sign [published-file] | table-withdraw ...".into()), + _ => Err("usage: chrome-builds pdb-key | extract
[frame-class] | port | find | pdb-check | pending | gate | fingerprint | release-sign ... | release-check | table-key | table-add | table-read | table-merge ... | table-sign [published-file] | table-withdraw ...".into()), }; match result { Ok(()) => ExitCode::SUCCESS, @@ -209,6 +246,134 @@ fn port( Ok(report.review.is_empty()) } +/// Makes a candidate row for a build with no published PDB from the prior row's fingerprints. The +/// prior PDB names each prior address, so a person sees which function a review line is about. +#[allow(clippy::too_many_arguments)] +fn find( + prior_path: &Path, + old_pdb: &Path, + old_module: &str, + module: &Path, + version: &str, + channel: &str, + download: &str, + row_path: &Path, + report_path: &Path, +) -> Result { + let prior = json_file(prior_path)?; + let stored = prior["fingerprints"].as_object(); + let prior_bytes = match old_module { + "-" => None, + path => Some(read(Path::new(path))?), + }; + let prior_pe = match &prior_bytes { + Some(bytes) => { + Some(PeFile::parse(bytes).ok_or("the prior module is not a 64-bit PE image")?) + } + None => None, + }; + let prints = match (stored, &prior_pe) { + (Some(stored), None) => stored.clone(), + (None, None) => { + return Err("the prior row has no fingerprints, and no prior module was given to read them from".into()); + } + (_, Some(pe)) => { + // Fingerprints read from another build would find the wrong functions. + let start = prior["frame"]["hwnd_for_view"] + .as_u64() + .and_then(|address| u32::try_from(address).ok()); + let prologue = prior["frame"]["hwnd_for_view_prologue"] + .as_str() + .and_then(chrome_build::hex); + match (start, prologue) { + (Some(start), Some(prologue)) + if pe.read(start, prologue.len()) == Some(prologue.as_slice()) => {} + _ => return Err("the prior module is not the build of the prior row".into()), + } + match stored { + Some(stored) => stored.clone(), + None => { + let (prints, missing) = row_port::fingerprints_of(&prior, pe); + if !missing.is_empty() { + println!( + "the prior build gives no fingerprint for {}", + missing.join(", ") + ); + } + prints + } + } + } + }; + // A fingerprint taken at a function start counts only function starts of the new build, and + // what fingerprints leave open is looked for by reading the prior build next to the new one. + let prior_starts = prior_pe + .as_ref() + .map(PeFile::function_starts) + .unwrap_or_default(); + let mut old = Vec::new(); + symbols::publics(old_pdb, &mut |address, name| { + old.push((address, name.to_owned())) + })?; + old.sort(); + let mut targets = row_port::addresses_of(&prior["build"]); + for key in ["frame_view_vtable", "client_view_bounds", "hwnd_for_view"] { + targets.extend(prior["frame"][key].as_u64().map(|value| value as u32)); + } + let named = row_port::name_addresses(&targets, &old); + drop(old); + let bytes = read(module)?; + let pe = PeFile::parse(&bytes).ok_or("not a 64-bit PE image")?; + let (mut row, mut report) = + row_port::find_candidate(&prior, &prints, prior_pe.as_ref(), version, &pe); + // No symbols name `screen_origin` here: a prior row without it gives a row without it. + row_port::seed_screen_origin(&mut row, &[], &pe, &mut report); + report.review.extend(row_port::hand_only_review(&prior)); + row_port::name_review_lines(&mut report.review, &named); + let key = pe + .pdb_key() + .ok_or("the new module has no CodeView record")?; + row_port::mark_without_pdb(&mut row, &key, &report.prior_version, channel, download); + let text = |value: &dyn erased::Json| value.pretty(); + std::fs::write(row_path, text(&row)).map_err(|error| error.to_string())?; + std::fs::write(report_path, text(&report)).map_err(|error| error.to_string())?; + println!( + "exception tables: {} function starts in the prior build, {} in the new one", + prior_starts.len(), + pe.function_starts().len() + ); + println!( + "{} {} without its PDB: {} of {} addresses found by fingerprint, {} settled by two methods, {} values read, {} carried, {} for review", + report.browser, + report.version, + report.fingerprinted.len(), + report.addresses, + report.settled.len(), + report.derived.len(), + report.carried.len(), + report.review.len() + ); + Ok(report.review.is_empty()) +} + +/// Compares a row made without its PDB with the row the PDB gives, and removes the mark when they +/// agree. +fn pdb_check(table_path: &Path, browser: &str, version: &str, row: &Path) -> Result { + let mut table = json_file(table_path)?; + let from_pdb = json_file(row)?; + let differences = row_port::check_against_pdb(&mut table, browser, version, &from_pdb)?; + for line in &differences { + println!("{line}"); + } + if !differences.is_empty() { + return Ok(false); + } + let text = serde_json::to_string_pretty(&table).map_err(|error| error.to_string())? + "\n"; + std::fs::write(table_path, text).map_err(|error| error.to_string())?; + println!("{browser} {version}: the row made from fingerprints agrees with its PDB"); + Ok(true) +} + mod erased { pub trait Json { fn pretty(&self) -> String; diff --git a/rivet/platform/windows/src/chrome_build.rs b/rivet/platform/windows/src/chrome_build.rs index 04db929..7328df3 100644 --- a/rivet/platform/windows/src/chrome_build.rs +++ b/rivet/platform/windows/src/chrome_build.rs @@ -68,6 +68,7 @@ pub struct PeFile<'a> { sections: Vec<(u32, u32, u32)>, code_sections: Vec<(u32, u32)>, debug_directory: (u32, u32), + exception_directory: (u32, u32), } fn u16_at(bytes: &[u8], offset: usize) -> Option { @@ -107,6 +108,11 @@ impl<'a> PeFile<'a> { u32_at(bytes, optional + 112 + 48)?, u32_at(bytes, optional + 116 + 48)?, ); + // Data directory 3 is the exception table, which lists where functions start. + let exception_directory = ( + u32_at(bytes, optional + 112 + 24)?, + u32_at(bytes, optional + 116 + 24)?, + ); let table = optional + optional_size; // IMAGE_SCN_MEM_EXECUTE. let code_sections = (0..sections) @@ -137,6 +143,7 @@ impl<'a> PeFile<'a> { sections, code_sections, debug_directory, + exception_directory, }) } @@ -210,6 +217,45 @@ impl<'a> PeFile<'a> { None } + /// The start of every function the exception table lists. A function that needs no unwind + /// data, such as a leaf that touches no stack, is not listed. + pub fn function_starts(&self) -> std::collections::BTreeSet { + let (directory, size) = self.exception_directory; + let mut starts = std::collections::BTreeSet::new(); + let Some(table) = self.offset(directory).filter(|_| directory != 0) else { + return starts; + }; + // Each entry is the function's start, its end and its unwind data. + for index in 0..(size / 12) as usize { + match u32_at(self.bytes, table + index * 12) { + Some(start) => starts.insert(start), + None => break, + }; + } + starts + } + + /// Every part of a function the exception table lists, as start and end. The linker moves cold + /// parts out of a function, and each such part has its own entry. + pub fn function_ranges(&self) -> std::collections::BTreeMap { + let (directory, size) = self.exception_directory; + let mut ranges = std::collections::BTreeMap::new(); + let Some(table) = self.offset(directory).filter(|_| directory != 0) else { + return ranges; + }; + for index in 0..(size / 12) as usize { + let entry = table + index * 12; + match (u32_at(self.bytes, entry), u32_at(self.bytes, entry + 4)) { + (Some(start), Some(end)) if end > start => { + ranges.insert(start, end); + } + (Some(_), Some(_)) => {} + _ => break, + } + } + ranges + } + /// Finds the slot of `function` in the vtable at `vtable`, scanning a bounded number of slots. pub fn vtable_slot(&self, vtable: u32, function: u32) -> Option { (0..512u32).find(|slot| self.pointer_rva(vtable + slot * 8) == Some(function)) diff --git a/rivet/platform/windows/src/lib.rs b/rivet/platform/windows/src/lib.rs index 27f8313..88c1962 100644 --- a/rivet/platform/windows/src/lib.rs +++ b/rivet/platform/windows/src/lib.rs @@ -11,6 +11,7 @@ pub mod focus; pub mod hold; pub mod row_port; pub mod row_recipe; +pub mod row_resolve; pub mod row_watch; pub mod signed_table; pub mod start; diff --git a/rivet/platform/windows/src/row_port.rs b/rivet/platform/windows/src/row_port.rs index 33b0dbd..79487ee 100644 --- a/rivet/platform/windows/src/row_port.rs +++ b/rivet/platform/windows/src/row_port.rs @@ -19,6 +19,7 @@ use serde::Serialize; use serde_json::{json, Value}; use crate::chrome_build::{hex, PeFile}; +use crate::row_resolve; /// The lowest value taken for an address. Offsets and slots lie below it. const LOWEST_ADDRESS: u64 = 0x10_0000; @@ -64,6 +65,12 @@ pub struct PortReport { pub carried: Vec, /// Optional parts the new build could not supply. The runtime does without them. pub left_out: Vec, + /// The new build had no symbols, so every address was looked for by the prior row's fingerprints. + pub made_without_pdb: bool, + /// Addresses a fingerprint found in exactly one place of the new build. + pub fingerprinted: Vec, + /// Values that are Rivet's settings rather than facts about the build, kept from the prior row. + pub settings: Vec, } fn is_address(value: u64) -> bool { @@ -159,14 +166,48 @@ fn candidates( votes } -/// A function's first instructions, read as far as this needs them. +/// A function's instructions, read as far as this needs them. pub mod x64 { - /// One decoded instruction: its length and where its RIP-relative displacement or relative - /// branch target is, if it has one. + /// One decoded instruction: its length, where its RIP-relative displacement or relative branch + /// target is, where a displacement from a register is, as in `[rcx + 0x558]`, and where its + /// immediate value is. Each place is `(offset, size)` within the instruction. #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct Instruction { pub length: usize, pub relative: Option<(usize, usize)>, + pub displacement: Option<(usize, usize)>, + pub immediate: Option<(usize, usize)>, + } + + impl Instruction { + fn plain(length: usize) -> Self { + Self { + length, + relative: None, + displacement: None, + immediate: None, + } + } + + /// An instruction whose last `size` bytes, from `at`, are an immediate value. + fn with_immediate(at: usize, size: usize) -> Self { + Self { + length: at + size, + relative: None, + displacement: None, + immediate: Some((at, size)), + } + } + + /// A branch whose last `size` bytes, from `at`, are the distance to its target. + fn branch(at: usize, size: usize) -> Self { + Self { + length: at + size, + relative: Some((at, size)), + displacement: None, + immediate: None, + } + } } fn modrm(bytes: &[u8], at: usize, immediate: usize) -> Option { @@ -174,10 +215,12 @@ pub mod x64 { let (mode, rm) = (byte >> 6, byte & 7); let mut length = at + 1; let mut relative = None; + let mut displacement = None; if mode != 3 && rm == 4 { let sib = *bytes.get(length)?; length += 1; if mode == 0 && sib & 7 == 5 { + displacement = Some((length, 4)); length += 4; } } @@ -186,19 +229,39 @@ pub mod x64 { relative = Some((length, 4)); length += 4; } - 1 => length += 1, - 2 => length += 4, + 1 => { + displacement = Some((length, 1)); + length += 1; + } + 2 => { + displacement = Some((length, 4)); + length += 4; + } _ => {} } Some(Instruction { length: length + immediate, relative, + displacement, + immediate: (immediate > 0).then_some((length, immediate)), }) } /// Decodes the instruction at the start of `bytes`, or `None` for one this does not know. The /// list covers what function starts are made of, and anything else goes to a person. pub fn decode(bytes: &[u8]) -> Option { + decode_with(bytes, false) + } + + /// Decodes as `decode` does, and also the instructions met inside a function body: lock and + /// segment prefixes, string and x87 instructions, VEX encoded vector instructions and more of + /// the two and three byte opcodes. It reads bodies to follow calls and operands, and never + /// judges a guard, so what a hook may copy stays as `decode` says. + pub fn decode_body(bytes: &[u8]) -> Option { + decode_with(bytes, true) + } + + fn decode_with(bytes: &[u8], body: bool) -> Option { let mut at = 0; let mut wide_operand = false; let mut rex_w = false; @@ -206,6 +269,7 @@ pub mod x64 { match *bytes.get(at)? { 0x66 => wide_operand = true, 0xF2 | 0xF3 => {} + 0xF0 | 0x26 | 0x2E | 0x36 | 0x3E | 0x64 | 0x65 if body => {} _ => break, } at += 1; @@ -217,15 +281,9 @@ pub mod x64 { let opcode = *bytes.get(at)?; at += 1; let word = if wide_operand { 2 } else { 4 }; - let plain = |length| { - Some(Instruction { - length, - relative: None, - }) - }; match opcode { - 0x50..=0x5F | 0x90 | 0xC3 | 0xCC | 0x99 => plain(at), - 0x98 => plain(at), + 0x50..=0x5F | 0x90 | 0xC3 | 0xCC | 0x99 => Some(Instruction::plain(at)), + 0x98 => Some(Instruction::plain(at)), 0x00..=0x03 | 0x08..=0x0B | 0x10..=0x13 @@ -242,10 +300,15 @@ pub mod x64 { | 0xFE | 0xFF => modrm(bytes, at, 0), 0x04 | 0x0C | 0x14 | 0x1C | 0x24 | 0x2C | 0x34 | 0x3C | 0x6A | 0xA8 | 0xB0..=0xB7 => { - plain(at + 1) + Some(Instruction::with_immediate(at, 1)) + } + 0x05 | 0x0D | 0x15 | 0x1D | 0x25 | 0x2D | 0x35 | 0x3D | 0x68 | 0xA9 => { + Some(Instruction::with_immediate(at, word)) } - 0x05 | 0x0D | 0x15 | 0x1D | 0x25 | 0x2D | 0x35 | 0x3D | 0x68 | 0xA9 => plain(at + word), - 0xB8..=0xBF => plain(at + if rex_w { 8 } else { word }), + 0xB8..=0xBF => Some(Instruction::with_immediate( + at, + if rex_w { 8 } else { word }, + )), 0x80 | 0x83 | 0xC0 | 0xC1 | 0xC6 | 0x6B => modrm(bytes, at, 1), 0x81 | 0xC7 | 0x69 => modrm(bytes, at, word), 0xF6 => { @@ -260,22 +323,13 @@ pub mod x64 { }; modrm(bytes, at, immediate) } - 0x70..=0x7F | 0xEB => Some(Instruction { - length: at + 1, - relative: Some((at, 1)), - }), - 0xE8 | 0xE9 => Some(Instruction { - length: at + 4, - relative: Some((at, 4)), - }), + 0x70..=0x7F | 0xEB => Some(Instruction::branch(at, 1)), + 0xE8 | 0xE9 => Some(Instruction::branch(at, 4)), 0x0F => { let second = *bytes.get(at)?; at += 1; match second { - 0x80..=0x8F => Some(Instruction { - length: at + 4, - relative: Some((at, 4)), - }), + 0x80..=0x8F => Some(Instruction::branch(at, 4)), 0x10..=0x17 | 0x1E | 0x1F @@ -291,9 +345,81 @@ pub mod x64 { | 0xBF | 0xD6 | 0xEF => modrm(bytes, at, 0), + _ if body => body_two_byte(bytes, at, second), _ => None, } } + _ if body => body_one_byte(bytes, at, opcode), + _ => None, + } + } + + /// One byte opcodes that only a function body holds. `at` is just past the opcode. + fn body_one_byte(bytes: &[u8], at: usize, opcode: u8) -> Option { + match opcode { + 0x91..=0x97 | 0x9C..=0x9F | 0xA4..=0xA7 | 0xAA..=0xAF | 0xC9 | 0xF4 | 0xF5 => { + Some(Instruction::plain(at)) + } + 0xF8..=0xFD => Some(Instruction::plain(at)), + 0xC2 => Some(Instruction::with_immediate(at, 2)), + 0xCD => Some(Instruction::with_immediate(at, 1)), + 0xD8..=0xDF => modrm(bytes, at, 0), + 0xE0..=0xE3 => Some(Instruction::branch(at, 1)), + // Two byte VEX: one payload byte, then an opcode of the 0F map. + 0xC5 => { + let opcode = *bytes.get(at + 1)?; + modrm(bytes, at + 2, vex_immediate(1, opcode)) + } + // Three byte VEX: the map in the first payload byte, then one more payload byte. + 0xC4 => { + let map = bytes.get(at)? & 0x1F; + let opcode = *bytes.get(at + 2)?; + modrm(bytes, at + 3, vex_immediate(map, opcode)) + } + _ => None, + } + } + + /// Whether a VEX instruction of `map` ends in an immediate byte. + fn vex_immediate(map: u8, opcode: u8) -> usize { + match map { + 3 => 1, + 1 => usize::from(matches!(opcode, 0x70..=0x73 | 0xC2 | 0xC4..=0xC6)), + _ => 0, + } + } + + /// Two byte opcodes that only a function body holds. `at` is just past the second byte. + fn body_two_byte(bytes: &[u8], at: usize, second: u8) -> Option { + match second { + 0x0B | 0x31 | 0x77 | 0xA2 | 0xC8..=0xCF => Some(Instruction::plain(at)), + 0x0D + | 0x18..=0x1D + | 0x74..=0x76 + | 0x7C + | 0x7D + | 0xA3 + | 0xA5 + | 0xAB + | 0xAD + | 0xAE + | 0xB0 + | 0xB1 + | 0xB3 + | 0xB8 + | 0xBB + | 0xBC + | 0xBD + | 0xC0 + | 0xC1 + | 0xC3 + | 0xC7 + | 0xD0..=0xD5 + | 0xD7..=0xEE + | 0xF0..=0xFE => modrm(bytes, at, 0), + 0x70..=0x73 | 0xA4 | 0xAC | 0xBA | 0xC2 | 0xC4..=0xC6 => modrm(bytes, at, 1), + 0x38 => modrm(bytes, at + 1, 0), + 0x3A => modrm(bytes, at + 1, 1), _ => None, } } @@ -350,6 +476,23 @@ pub mod x64 { } Some(false) } + + /// An instruction's bytes with its relative value, displacement and immediate cleared: what + /// stays the same when code moves and members, slots and constants change. + pub fn skeleton(bytes: &[u8], instruction: &Instruction) -> Vec { + let mut out = bytes[..instruction.length.min(bytes.len())].to_vec(); + let fields = [ + instruction.relative, + instruction.displacement, + instruction.immediate, + ]; + for (offset, size) in fields.into_iter().flatten() { + for index in offset..(offset + size).min(out.len()) { + out[index] = 0; + } + } + out + } } /// A byte field with no address field of the same name, and where it was read from. @@ -499,7 +642,7 @@ fn settle(target: u32, choices: &[(u32, usize)], build: &Value, pe: &PeFile) -> } /// The bytes of a function short enough to read to its end: its first `ret` with no branch before. -fn whole_function(pe: &PeFile, address: u32) -> Option> { +pub(crate) fn whole_function(pe: &PeFile, address: u32) -> Option> { let bytes = pe.read(address, 48)?; let mut at = 0; while at < 32 { @@ -838,7 +981,7 @@ pub fn port_row( named: &BTreeMap)>, new: &NewBuild, ) -> (Value, PortReport) { - let (row, mut report) = begin_row(prior, new.version); + let (row, mut report) = begin_row(prior, new.version, true); let mut moved: BTreeMap = BTreeMap::new(); let targets = addresses_of(&row); report.addresses = targets.len(); @@ -846,9 +989,15 @@ pub fn port_row( finish_row(row, &moved, new.pe, report) } -/// A copy of the prior row with the new version, and frame addresses in the `0x` form. -fn begin_row(prior: &Value, version: &str) -> (Value, PortReport) { +/// A copy of the prior row with the new version, and frame addresses in the `0x` form. `carries` +/// says the values with no recipe stay as the prior row has them, so a new milestone stops the +/// row; without it they are read from the new build later, each with its own review line. +fn begin_row(prior: &Value, version: &str, carries: bool) -> (Value, PortReport) { let mut row = prior.clone(); + if let Some(object) = row.as_object_mut() { + // The mark tells how the prior row was made, not this one. + object.remove(WITHOUT_PDB); + } let prior_version = prior["frame"]["version"] .as_str() .unwrap_or_default() @@ -872,7 +1021,8 @@ fn begin_row(prior: &Value, version: &str) -> (Value, PortReport) { .push(format!("{} is not a four part version", version)); } report.carried = crate::row_recipe::without_recipe(&prior["build"]); - if prior_version.split('.').next() != version.split('.').next() { + report.settings = crate::row_recipe::settings(&prior["build"]); + if carries && prior_version.split('.').next() != version.split('.').next() { report.review.push(format!( "the milestone changed from {prior_version} to {version}: {} values have no recipe and were carried over unchecked: {}", report.carried.len(), @@ -1037,7 +1187,7 @@ fn fingerprint_targets(row: &Value) -> BTreeSet { } /// The start of a function with its displacements cleared, cut at an instruction end. -fn code_print(pe: &PeFile, address: u32) -> Option> { +pub(crate) fn code_print(pe: &PeFile, address: u32) -> Option> { if !pe.is_code(address) { return None; } @@ -1067,6 +1217,17 @@ fn code_print(pe: &PeFile, address: u32) -> Option> { /// known by a slot that holds one of those functions. An address that is neither cannot be found, /// so then the row gets no fingerprints at all. pub fn add_fingerprints(row: &mut Value, pe: &PeFile) -> Result<(), String> { + let (prints, missing) = fingerprints_of(row, pe); + if !missing.is_empty() { + return Err(format!("no fingerprint for {}", missing.join(", "))); + } + row["fingerprints"] = Value::Object(prints); + Ok(()) +} + +/// The fingerprint of each address of a row that has one, read from the row's own module, and the +/// addresses that have none. A row made from them leaves those addresses unvouched. +pub fn fingerprints_of(row: &Value, pe: &PeFile) -> (serde_json::Map, Vec) { let targets = fingerprint_targets(row); let mut prints = serde_json::Map::new(); let mut missing = Vec::new(); @@ -1093,15 +1254,11 @@ pub fn add_fingerprints(row: &mut Value, pe: &PeFile) -> Result<(), String> { None => missing.push(key), } } - if !missing.is_empty() { - return Err(format!("no fingerprint for {}", missing.join(", "))); - } - row["fingerprints"] = Value::Object(prints); - Ok(()) + (prints, missing) } /// Every place each function fingerprint matches, in one pass over the code. -fn find_prints(pe: &PeFile, prints: &BTreeMap>) -> BTreeMap> { +pub(crate) fn find_prints(pe: &PeFile, prints: &BTreeMap>) -> BTreeMap> { let mut by_anchor: HashMap<[u8; ANCHOR], Vec> = HashMap::new(); let mut first = [false; 256]; for (target, shape) in prints { @@ -1137,7 +1294,7 @@ fn find_prints(pe: &PeFile, prints: &BTreeMap>) -> BTreeMap Vec { +pub(crate) fn find_table(pe: &PeFile, slot: u32, function: u32) -> Vec { let value = (pe.image_base() + u64::from(function)).to_le_bytes(); let mut found = Vec::new(); for (base, bytes, _) in pe.sections() { @@ -1161,11 +1318,43 @@ pub fn find_row(prior: &Value, version: &str, pe: &PeFile) -> Result, + prior_pe: Option<&PeFile>, + version: &str, + pe: &PeFile, +) -> (Value, PortReport) { + let (row, mut report) = begin_row(prior, version, prior_pe.is_none()); + report.made_without_pdb = true; + let prior_starts = prior_pe.map(PeFile::function_starts).unwrap_or_default(); let targets = fingerprint_targets(prior); report.addresses = targets.len(); let mut functions = BTreeMap::new(); let mut tables = BTreeMap::new(); + // Each address its fingerprint did not pin down, with its review line, waits for the methods + // that read the prior build next to the new one. + let mut wanted: BTreeMap = BTreeMap::new(); + let want = |kind, candidates: Option>| row_resolve::Wanted { kind, candidates }; for target in &targets { match stored.get(&format!("{target:#x}")) { Some(Value::String(text)) => match hex(text).filter(|shape| shape.len() >= ANCHOR) { @@ -1190,38 +1379,134 @@ pub fn find_row(prior: &Value, version: &str, pe: &PeFile) -> Result report.review.push(format!("{target:#x}: no fingerprint")), + _ => { + let line = format!("{target:#x}: no fingerprint"); + match prior_pe { + Some(prior_pe) => { + let kind = if prior_pe.is_code(*target) { + row_resolve::Kind::Function + } else { + row_resolve::Kind::Table + }; + wanted.insert(*target, (line, want(kind, None))); + } + None => report.review.push(line), + } + } } } let found = find_prints(pe, &functions); + let new_starts = if prior_starts.is_empty() { + BTreeSet::new() + } else { + pe.function_starts() + }; let mut moved = BTreeMap::new(); for target in functions.keys() { - match found.get(target).map(Vec::as_slice) { - Some([only]) => { + let all = found.get(target).map_or(&[][..], Vec::as_slice); + let places: Vec = all + .iter() + .copied() + .filter(|place| { + // The linker starts whole functions on 16 bytes, and the exception table also + // lists the parts it moves out of a function, which start anywhere. + !prior_starts.contains(target) + || (new_starts.contains(place) && (target % 16 != 0 || *place % 16 == 0)) + }) + .collect(); + let function = row_resolve::Kind::Function; + match places.as_slice() { + [only] => { moved.insert(*target, *only); + // Says which matches the exception tables checked, for the person reading the report. + let checked = if prior_starts.contains(target) { + " at a function start" + } else { + "" + }; + report + .fingerprinted + .push(format!("{target:#x} -> {only:#x}{checked}")); + } + [] if !all.is_empty() => { + let line = format!( + "{target:#x} matches no function start, of {} places", + all.len() + ); + wanted.insert(*target, (line, want(function, None))); + } + [] => { + let line = format!("{target:#x} matches no place"); + wanted.insert(*target, (line, want(function, None))); + } + many => { + let line = format!("{target:#x} matches {} places", many.len()); + wanted.insert(*target, (line, want(function, Some(many.to_vec())))); } - Some(many) => report - .review - .push(format!("{target:#x} matches {} places", many.len())), - None => report.review.push(format!("{target:#x} matches no place")), } } for (target, (slot, holds)) in &tables { + let table = row_resolve::Kind::Table; let Some(function) = moved.get(holds).copied() else { - report.review.push(format!( - "{target:#x}: the function in slot {slot} was not found" - )); + let line = format!("{target:#x}: the function in slot {slot} was not found"); + wanted.insert(*target, (line, want(table, None))); continue; }; match find_table(pe, *slot, function).as_slice() { [only] => { moved.insert(*target, *only); + report + .fingerprinted + .push(format!("{target:#x} -> {only:#x}")); } - many => report.review.push(format!( - "{target:#x}: {} tables hold {function:#x} in slot {slot}", - many.len() - )), + many => { + let line = format!( + "{target:#x}: {} tables hold {function:#x} in slot {slot}", + many.len() + ); + let candidates = (!many.is_empty()).then(|| many.to_vec()); + wanted.insert(*target, (line, want(table, candidates))); + } + } + } + let mut followed = BTreeMap::new(); + match prior_pe { + Some(prior_pe) if !wanted.is_empty() => { + let table_targets: BTreeSet = tables + .keys() + .copied() + .chain( + wanted + .iter() + .filter(|(_, (_, want))| want.kind == row_resolve::Kind::Table) + .map(|(target, _)| *target), + ) + .collect(); + let asked: BTreeMap = wanted + .iter() + .map(|(target, (_, want))| (*target, want.clone())) + .collect(); + let resolution = row_resolve::resolve(prior_pe, pe, &moved, &asked, &table_targets); + for (target, (line, _)) in &wanted { + match resolution.places.get(target) { + Some(place) => { + moved.insert(*target, *place); + } + None => { + let reason = resolution + .reasons + .get(target) + .map_or("no method was run", String::as_str); + report.review.push(format!("{line}; not settled: {reason}")); + } + } + } + report.settled.extend(resolution.settled); + followed = resolution.followed; } + _ => report + .review + .extend(wanted.into_values().map(|(line, _)| line)), } let places: BTreeSet = moved.values().copied().collect(); if places.len() != moved.len() { @@ -1229,15 +1514,231 @@ pub fn find_row(prior: &Value, version: &str, pe: &PeFile) -> Result(value: &'a Value, path: &str) -> Option<&'a Value> { + let mut current = value; + for part in path.split('.') { + let (key, indexes) = part.split_at(part.find('[').unwrap_or(part.len())); + current = current.get(key)?; + for index in indexes.split(']').filter(|index| !index.is_empty()) { + current = current.get(index.trim_start_matches('[').parse::().ok()?)?; + } + } + Some(current) +} + +fn value_at_mut<'a>(value: &'a mut Value, path: &str) -> Option<&'a mut Value> { + let mut current = value; + for part in path.split('.') { + let (key, indexes) = part.split_at(part.find('[').unwrap_or(part.len())); + current = current.get_mut(key)?; + for index in indexes.split(']').filter(|index| !index.is_empty()) { + current = current.get_mut(index.trim_start_matches('[').parse::().ok()?)?; + } + } + Some(current) +} + +/// Reads each value the recipes leave, a member offset, slot, size or id, from the instructions +/// that use it in the prior build's functions and the same instructions in the new build. A value +/// that cannot be read keeps a review line with the reason and stays listed as carried, so the row +/// is never published with a value of the prior build. +fn read_carried( + prior: &Value, + row: &mut Value, + moved: &BTreeMap, + followed: &BTreeMap, + prior_pe: &PeFile, + pe: &PeFile, + report: &mut PortReport, +) { + let mut values = Vec::new(); + let mut unreadable = Vec::new(); + for path in std::mem::take(&mut report.carried) { + let items: Vec<(String, &Value)> = match value_at(&prior["build"], &path) { + Some(Value::Array(items)) => items + .iter() + .enumerate() + .map(|(index, item)| (format!("{path}[{index}]"), item)) + .collect(), + Some(item) => vec![(path.clone(), item)], + None => vec![], + }; + if items.is_empty() { + unreadable.push(path); + } + for (path, item) in items { + match crate::row_recipe::number(item) { + Some(value) => values.push((path, value)), + None => unreadable.push(path), + } + } + } + // Each part of the row reads its values from its own functions. + let mut sections: BTreeMap> = BTreeMap::new(); + for (section, part) in prior["build"].as_object().into_iter().flatten() { + let mut part = part.clone(); + let mut pairs = Vec::new(); + walk_addresses("", &mut part, &mut |_, value| { + if let Some(address) = read_address(value).filter(|value| is_address(*value)) { + let address = address as u32; + if let Some(place) = moved.get(&address).filter(|_| prior_pe.is_code(address)) { + pairs.push((address, *place)); + } + } + }); + pairs.sort_unstable(); + pairs.dedup(); + if !pairs.is_empty() { + sections.insert(section.clone(), pairs); + } + } + for reading in row_resolve::read_values(prior_pe, pe, §ions, followed, &values) { + match reading.value { + Ok((value, how)) => { + if let Some(place) = value_at_mut(&mut row["build"], &reading.path) { + *place = if place.is_string() { + write_address(value) + } else { + json!(value) + }; + } + report.derived.push(if value == reading.prior { + format!("{}: {value:#x}, {how}", reading.path) + } else { + format!("{}: {:#x} -> {value:#x}, {how}", reading.path, reading.prior) + }); + } + Err(reason) => { + report.review.push(format!( + "{} was not read from the new build: {reason}", + reading.path + )); + report.carried.push(reading.path); + } + } + } + for path in unreadable { + report + .review + .push(format!("{path} is not a number this generator can read")); + report.carried.push(path); + } +} + +/// The key of a row made without its build's symbols. It holds what the row job needs to make the +/// row again from the symbols once they are published, and to compare the two. The runtime reads +/// only `frame` and `build`, so the mark changes nothing in a browser. +pub const WITHOUT_PDB: &str = "made_without_pdb"; + +/// Marks a row made from fingerprints with the symbol key of its module, the prior row it was made +/// from, and how the row job fetches the build again. +pub fn mark_without_pdb( + row: &mut Value, + pdb_key: &str, + prior_version: &str, + channel: &str, + download: &str, +) { + row[WITHOUT_PDB] = json!({ + "pdb_key": pdb_key, + "prior_version": prior_version, + "channel": channel, + "download": download, + }); +} + +/// Where a row made from fingerprints differs from the row the build's own symbols give. A value +/// the fingerprint row left out is not a difference, since the runtime does without it. +pub fn differences_from_pdb_row(found: &Value, from_pdb: &Value) -> Vec { + fn walk(path: &str, found: &Value, pdb: &Value, out: &mut Vec) { + match (found, pdb) { + (Value::Null, _) => {} + (Value::Object(left), Value::Object(right)) => { + for (key, value) in left { + let other = right.get(key).unwrap_or(&Value::Null); + walk(&format!("{path}.{key}"), value, other, out); + } + } + (Value::Array(left), Value::Array(right)) if left.len() == right.len() => { + for (index, (a, b)) in left.iter().zip(right).enumerate() { + walk(&format!("{path}[{index}]"), a, b, out); + } + } + _ if found != pdb => { + out.push(format!( + "{path}: {found} from fingerprints, {pdb} from the PDB" + )); + } + _ => {} + } + } + let mut out = vec![]; + for part in ["frame", "build"] { + walk(part, &found[part], &from_pdb[part], &mut out); + } + out +} + +/// Compares the marked row of `browser` `version` in `table` with the row made from the build's +/// symbols. When they agree the mark is removed, since the symbols now vouch for the row. The +/// differences are returned, and the table is left as it was when there are any. +pub fn check_against_pdb( + table: &mut Value, + browser: &str, + version: &str, + from_pdb: &Value, +) -> Result, String> { + let row = table["rows"] + .as_array_mut() + .ok_or("the table has no rows")? + .iter_mut() + .find(|row| row["frame"]["browser"] == browser && row["frame"]["version"] == version) + .ok_or_else(|| format!("the table has no row for {browser} {version}"))?; + if row.get(WITHOUT_PDB).is_none() { + return Err(format!( + "the row of {browser} {version} was not made without its PDB" + )); + } + let differences = differences_from_pdb_row(row, from_pdb); + if differences.is_empty() { + if let Some(object) = row.as_object_mut() { + object.remove(WITHOUT_PDB); + } + } + Ok(differences) +} + +/// Adds to each review line that starts with a prior address the public name that address lies +/// in, from the prior build's symbols, so a person sees which function was not found. +pub fn name_review_lines(review: &mut [String], named: &BTreeMap)>) { + for line in review { + let Some(digits) = line.strip_prefix("0x") else { + continue; + }; + let end = digits + .find(|c: char| !c.is_ascii_hexdigit()) + .unwrap_or(digits.len()); + let Ok(address) = u32::from_str_radix(&digits[..end], 16) else { + continue; + }; + if let Some((distance, names)) = named.get(&address) { + if let Some(name) = names.first() { + line.push_str(&format!(" ({name}+{distance:#x} in the prior build)")); + } + } + } } /// A folded function that fills a known table slot is the one in that slot of the new table. @@ -1802,6 +2303,33 @@ mod tests { assert_eq!(x64::shape(&[0x56, 0xCF]), None); } + #[test] + fn nfr6_the_operands_of_a_body_instruction_are_located() { + // mov rax, [rcx + 0x558]: a displacement of four bytes from the third byte. + let load = x64::decode(&[0x48, 0x8B, 0x81, 0x58, 0x05, 0x00, 0x00]).unwrap(); + assert_eq!(load.displacement, Some((3, 4))); + assert_eq!(load.immediate, None); + // call [rax + 0x2b8]: a table slot read through its displacement. + let call = x64::decode(&[0xFF, 0x90, 0xB8, 0x02, 0x00, 0x00]).unwrap(); + assert_eq!(call.displacement, Some((2, 4))); + // mov edx, 0x9c45, and cmp byte [rcx + 0x30], 1. + let id = x64::decode(&[0xBA, 0x45, 0x9C, 0x00, 0x00]).unwrap(); + assert_eq!(id.immediate, Some((1, 4))); + let compare = x64::decode(&[0x80, 0x79, 0x30, 0x01]).unwrap(); + assert_eq!(compare.displacement, Some((2, 1))); + assert_eq!(compare.immediate, Some((3, 1))); + // lock cmpxchg and a VEX move decode only as body instructions. + let lock = [0xF0, 0x48, 0x0F, 0xB1, 0x51, 0x08]; + assert_eq!(x64::decode(&lock), None); + assert_eq!(x64::decode_body(&lock).unwrap().displacement, Some((5, 1))); + let vex = [0xC5, 0xF8, 0x10, 0x41, 0x10]; + assert_eq!(x64::decode_body(&vex).unwrap().length, 5); + assert_eq!( + x64::skeleton(&[0x48, 0x8B, 0x81, 0x58, 0x05, 0x00, 0x00], &load), + [0x48, 0x8B, 0x81, 0, 0, 0, 0] + ); + } + /// A verified row whose functions sit at 0x101000 in its module. fn verified_row() -> Value { json!({ @@ -1936,6 +2464,291 @@ mod tests { assert_eq!(reasons, ["the row has no fingerprints"]); } + /// The fingerprints of the verified row, read from its own module as the row job reads them + /// when the prior row stored none. + fn prints_of_prior() -> serde_json::Map { + let bytes = module(&code_refs(&shifted_code(0, 0x10))); + let pe = PeFile::parse(&bytes).unwrap(); + let (prints, missing) = fingerprints_of(&verified_row(), &pe); + assert!(missing.is_empty(), "{missing:?}"); + prints + } + + fn candidate( + prints: &serde_json::Map, + code: &[(u32, Vec)], + version: &str, + ) -> (Value, PortReport) { + let bytes = module(&code_refs(code)); + let pe = PeFile::parse(&bytes).unwrap(); + find_candidate(&verified_row(), prints, None, version, &pe) + } + + /// `module` with an exception table at 0x10f000 that lists `starts`. + fn with_starts(code: &[(u32, Vec)], starts: &[u32]) -> Vec { + let ranges: Vec<(u32, u32)> = starts.iter().map(|start| (*start, start + 0x20)).collect(); + with_ranges(code, &ranges) + } + + /// `module` with an exception table at 0x10f000 that lists each `(start, end)`. + fn with_ranges(code: &[(u32, Vec)], ranges: &[(u32, u32)]) -> Vec { + let mut table = vec![]; + for (start, end) in ranges { + table.extend_from_slice(&start.to_le_bytes()); + table.extend_from_slice(&end.to_le_bytes()); + table.extend_from_slice(&0u32.to_le_bytes()); + } + let mut code = code.to_vec(); + code.push((0x10f000, table)); + let mut bytes = module(&code_refs(&code)); + let directory = 0x98 + 112 + 24; + bytes[directory..directory + 4].copy_from_slice(&0x10f000u32.to_le_bytes()); + bytes[directory + 4..directory + 8] + .copy_from_slice(&(12 * ranges.len() as u32).to_le_bytes()); + bytes + } + + /// The verified row's paint function, which reads `view_layer` with `loads` past the bytes its + /// fingerprint holds. + fn paint_reading(loads: &str) -> Vec { + let mut bytes = hex("4157415641554154565755534881eca0000000").unwrap(); + bytes.resize(32, 0x90); + bytes.extend(hex(loads).unwrap()); + bytes.push(0xC3); + bytes + } + + /// A candidate row for a build whose paint function reads `view_layer` with `new_loads`, made + /// from the verified row and its build, whose paint function reads it with `prior_loads`. + fn read_view_layer(prior_loads: &str, new_loads: &str) -> (Value, PortReport) { + let ranges = |shift: u32, paint: usize| { + vec![ + (0x101100 + shift, 0x101120 + shift), + (0x101200 + shift, 0x101220 + shift), + (0x101300 + shift, 0x101300 + shift + paint as u32), + ] + }; + let mut old = shifted_code(0, 0x10); + old[3].1 = paint_reading(prior_loads); + let prior_bytes = with_ranges(&old, &ranges(0, old[3].1.len())); + let prior_pe = PeFile::parse(&prior_bytes).unwrap(); + let (prints, missing) = fingerprints_of(&verified_row(), &prior_pe); + assert!(missing.is_empty(), "{missing:?}"); + let mut new = shifted_code(0x2000, 0x44); + new[3].1 = paint_reading(new_loads); + let bytes = with_ranges(&new, &ranges(0x2000, new[3].1.len())); + let pe = PeFile::parse(&bytes).unwrap(); + find_candidate(&verified_row(), &prints, Some(&prior_pe), "1.0.0.3", &pe) + } + + #[test] + fn nfr6_a_carried_value_is_read_from_the_instructions_that_use_it() { + // mov rax, [rcx + 0x20]; mov rdx, [rcx + 0x20], and the new build moved the member to 0x28. + let (row, report) = + read_view_layer("488b8120000000488b9120000000", "488b8128000000488b9128000000"); + assert!(report.review.is_empty(), "{:?}", report.review); + assert_eq!(row["build"]["toolbar"]["view_layer"], "0x28"); + assert!(report.carried.is_empty(), "{:?}", report.carried); + assert_eq!( + report.derived, + ["toolbar.view_layer: 0x20 -> 0x28, read at 2 uses in 1 of its functions"] + ); + } + + #[test] + fn nfr6_a_value_no_instruction_gives_stops_the_row() { + // The new build reads the member with a short displacement: not the same instruction. + let (_, report) = read_view_layer("488b8120000000488b9120000000", "488b4128488b5128"); + assert_eq!(report.carried, ["toolbar.view_layer"]); + assert!(report.derived.is_empty(), "{:?}", report.derived); + assert!( + report.review.iter().any(|line| line + .starts_with("toolbar.view_layer was not read from the new build: its 2 uses")), + "{:?}", + report.review + ); + // One use alone is not read either, and neither are uses that disagree. + let (_, report) = read_view_layer("488b8120000000", "488b8128000000"); + assert!( + report + .review + .iter() + .any(|line| line.contains("a second reading is needed")), + "{:?}", + report.review + ); + let (_, report) = + read_view_layer("488b8120000000488b9120000000", "488b8128000000488b9130000000"); + assert!( + report + .review + .iter() + .any(|line| line.contains("its uses disagree")), + "{:?}", + report.review + ); + } + + #[test] + fn nfr6_a_function_start_is_found_only_at_a_function_start() { + // The paint function's first bytes also open a block inside another function at 0x105300. + let mut code = shifted_code(0x2000, 0x44); + code.push(( + 0x105300, + hex("4157415641554154565755534881eca0000000").unwrap(), + )); + let prior_bytes = with_starts(&shifted_code(0, 0x10), &[0x101100, 0x101200, 0x101300]); + let prior_pe = PeFile::parse(&prior_bytes).unwrap(); + let prior_starts = prior_pe.function_starts(); + assert_eq!(prior_starts.len(), 3); + // The row holds no value without a recipe, so only its addresses are looked for. + let mut verified = verified_row(); + verified["build"]["toolbar"] + .as_object_mut() + .unwrap() + .remove("view_layer"); + let (prints, _) = fingerprints_of(&verified, &prior_pe); + let bytes = with_starts(&code, &[0x103100, 0x103200, 0x103300]); + let pe = PeFile::parse(&bytes).unwrap(); + let (row, report) = find_candidate(&verified, &prints, Some(&prior_pe), "1.0.0.3", &pe); + assert!(report.review.is_empty(), "{:?}", report.review); + assert_eq!(row["build"]["toolbar"]["paint"], "0x103300"); + // A part moved out of another function starts where it may, and a whole function does not. + let mut split = shifted_code(0x2000, 0x44); + split.push(( + 0x105308, + hex("4157415641554154565755534881eca0000000").unwrap(), + )); + let bytes = with_starts(&split, &[0x103100, 0x103200, 0x103300, 0x105308]); + let pe = PeFile::parse(&bytes).unwrap(); + let (row, report) = find_candidate(&verified, &prints, Some(&prior_pe), "1.0.0.3", &pe); + assert!(report.review.is_empty(), "{:?}", report.review); + assert_eq!(row["build"]["toolbar"]["paint"], "0x103300"); + // Bytes that match only inside another function are no place at all. + let bytes = with_starts(&code, &[0x103100, 0x103200]); + let pe = PeFile::parse(&bytes).unwrap(); + let (_, report) = find_candidate(&verified, &prints, Some(&prior_pe), "1.0.0.3", &pe); + assert!( + report + .review + .iter() + .any(|line| line + .starts_with("0x101300 matches no function start, of 2 places; not settled:")), + "{:?}", + report.review + ); + } + + #[test] + fn nfr6_a_build_without_symbols_gets_a_row_from_the_prior_builds_fingerprints() { + let (row, report) = candidate(&prints_of_prior(), &shifted_code(0x2000, 0x44), "1.0.0.3"); + assert!(report.review.is_empty(), "{:?}", report.review); + assert!(report.made_without_pdb); + assert_eq!(report.fingerprinted.len(), 4, "{:?}", report.fingerprinted); + assert_eq!(row["build"]["toolbar"]["paint"], "0x103300"); + assert_eq!(row["frame"]["frame_view_vtable"], 0x103000); + assert_eq!(row["frame"]["hwnd_for_view"], 0x103200); + // Values that are not addresses are left to the recipes, and nothing of the prior build stays. + assert_eq!(row["build"]["toolbar"]["view_layer"], "0x20"); + assert!(row.get("fingerprints").is_none()); + } + + #[test] + fn nfr6_an_address_its_fingerprint_does_not_pin_down_is_not_vouched_for() { + // Two places match the paint function: that part keeps its prior value and stops the row, + // and the rest of the row is still found. + let mut code = shifted_code(0x2000, 0x44); + code.push(( + 0x105300, + hex("4157415641554154565755534881eca0000000").unwrap(), + )); + let (row, report) = candidate(&prints_of_prior(), &code, "1.0.0.3"); + assert!( + report + .review + .iter() + .any(|line| line.contains("0x101300 matches 2 places")), + "{:?}", + report.review + ); + assert!(report + .review + .iter() + .any(|line| line.contains("no new place"))); + assert_eq!(row["build"]["toolbar"]["paint"], "0x101300"); + assert_eq!(row["frame"]["hwnd_for_view"], 0x103200); + // An address the prior build gave no fingerprint is not looked for at all. + let mut prints = prints_of_prior(); + prints.remove("0x101200"); + let (_, report) = candidate(&prints, &shifted_code(0x2000, 0x44), "1.0.0.3"); + assert!( + report + .review + .iter() + .any(|line| line == "0x101200: no fingerprint"), + "{:?}", + report.review + ); + } + + #[test] + fn nfr6_a_row_made_without_its_pdb_is_checked_against_the_pdb_row() { + let (mut row, _) = candidate(&prints_of_prior(), &shifted_code(0x2000, 0x44), "1.0.0.3"); + mark_without_pdb(&mut row, "ABC1", "1.0.0.2", "stable", ""); + assert_eq!(row[WITHOUT_PDB]["prior_version"], "1.0.0.2"); + let mut from_pdb = row.clone(); + from_pdb.as_object_mut().unwrap().remove(WITHOUT_PDB); + // A part the fingerprint row left out is no difference. + row["build"]["screen_origin"] = Value::Null; + from_pdb["build"]["screen_origin"] = json!(["0x103400", "5657"]); + let table = json!({"rows": [row]}); + let mut agreed = table.clone(); + assert_eq!( + check_against_pdb(&mut agreed, "B", "1.0.0.3", &from_pdb).unwrap(), + Vec::::new() + ); + assert!(agreed["rows"][0].get(WITHOUT_PDB).is_none()); + // A different address is a difference, and the row keeps its mark. + from_pdb["build"]["toolbar"]["paint"] = json!("0x103310"); + let mut differs = table.clone(); + let differences = check_against_pdb(&mut differs, "B", "1.0.0.3", &from_pdb).unwrap(); + assert_eq!(differences.len(), 1, "{differences:?}"); + assert!( + differences[0].starts_with("build.toolbar.paint:"), + "{differences:?}" + ); + assert_eq!(differs, table); + // A row made from symbols has nothing to check. + let mut plain = json!({"rows": [from_pdb.clone()]}); + assert!(check_against_pdb(&mut plain, "B", "1.0.0.3", &from_pdb).is_err()); + } + + #[test] + fn nfr6_a_row_made_from_a_marked_prior_does_not_inherit_the_mark() { + let mut prior = prior(); + mark_without_pdb(&mut prior, "ABC1", "1.0.0.0", "stable", ""); + let code = new_code(); + let (row, _) = port(&prior, NEW_SYMBOLS, &code_refs(&code), "1.0.0.2"); + assert!(row.get(WITHOUT_PDB).is_none()); + } + + #[test] + fn nfr6_a_review_line_names_the_prior_function_it_is_about() { + let named = BTreeMap::from([(0x101300, (0x10, vec!["?Paint@V@@".to_owned()]))]); + let mut review = vec![ + "0x101300 matches no place".to_owned(), + "0x101400: no fingerprint".to_owned(), + "the milestone changed".to_owned(), + ]; + name_review_lines(&mut review, &named); + assert_eq!( + review[0], + "0x101300 matches no place (?Paint@V@@+0x10 in the prior build)" + ); + assert_eq!(review[1], "0x101400: no fingerprint"); + assert_eq!(review[2], "the milestone changed"); + } + #[test] fn nfr3_an_address_that_is_neither_code_nor_a_table_gets_no_fingerprints() { let bytes = module(&code_refs(&shifted_code(0, 0x10))); diff --git a/rivet/platform/windows/src/row_recipe.rs b/rivet/platform/windows/src/row_recipe.rs index 327383c..a14d276 100644 --- a/rivet/platform/windows/src/row_recipe.rs +++ b/rivet/platform/windows/src/row_recipe.rs @@ -415,11 +415,43 @@ pub fn derive_row(build: &mut Value, pe: &PeFile) -> Derived { derived } +/// Values Rivet chooses rather than reads from a build, such as how long it waits before it opens +/// a new tab. The row carries them for the runtime, so they are kept from the prior row. +pub const SETTINGS: &[&str] = &["tabs.new_tab_delay_ms"]; + +/// Lists of `[id, colour]` pairs: the id is a value of the build, and the colour is Rivet's choice. +pub const ID_COLOUR_PAIRS: &[&str] = &["toolbar.dev.colours"]; + +/// The values of a row that are Rivet's settings rather than facts about its build. +pub fn settings(build: &Value) -> Vec { + let mut out: Vec = SETTINGS + .iter() + .filter(|path| number(at_path(build, path)).is_some()) + .map(|path| (*path).to_owned()) + .collect(); + for list in ID_COLOUR_PAIRS { + if let Some(pairs) = at_path(build, list).as_array() { + out.extend((0..pairs.len()).map(|index| format!("{list}[{index}][1]"))); + } + } + out +} + /// Every number of a row below the address range that no recipe reads: member offsets, slots, -/// sizes, ids and constants that a port can only carry over. +/// sizes, ids and constants that a port can only carry over. Rivet's own settings are not listed. pub fn without_recipe(build: &Value) -> Vec { fn walk(path: &str, value: &Value, out: &mut Vec) { match value { + Value::Array(items) + if items.len() == 2 + && path + .rsplit_once('[') + .is_some_and(|(list, _)| ID_COLOUR_PAIRS.contains(&list)) => + { + if number(&items[0]).is_some_and(|id| id < LOWEST_ADDRESS) { + out.push(format!("{path}[0]")); + } + } Value::Object(object) => { for (key, item) in object { let next = if path.is_empty() { @@ -459,6 +491,7 @@ pub fn without_recipe(build: &Value) -> Vec { path != "version" && !path.starts_with("version[") && !FIELDS.iter().any(|field| field.path == path) + && !SETTINGS.contains(&path.as_str()) }); out } @@ -637,4 +670,29 @@ mod tests { vec!["layout.caption_container_offset", "layout.frame_ids"] ); } + + #[test] + fn nfr6_rivet_settings_are_not_values_of_the_build() { + let build = json!({ + "tabs": {"new_tab_delay_ms": 0, "tab_contents": "0x0"}, + "toolbar": {"dev": {"colours": [[316, 4280953905u32], [2082, 0]], "console": 40005}}, + }); + assert_eq!( + without_recipe(&build), + vec![ + "tabs.tab_contents", + "toolbar.dev.colours[0][0]", + "toolbar.dev.colours[1][0]", + "toolbar.dev.console" + ] + ); + assert_eq!( + settings(&build), + vec![ + "tabs.new_tab_delay_ms", + "toolbar.dev.colours[0][1]", + "toolbar.dev.colours[1][1]" + ] + ); + } } diff --git a/rivet/platform/windows/src/row_resolve.rs b/rivet/platform/windows/src/row_resolve.rs new file mode 100644 index 0000000..f6d0bf3 --- /dev/null +++ b/rivet/platform/windows/src/row_resolve.rs @@ -0,0 +1,1278 @@ +//! Finds, in a build whose symbols are not published, the functions and class tables that the +//! prior row's fingerprints do not pin down to one place. +//! +//! Each such address is looked for by several independent methods, and a place is taken only when +//! two different methods name the same place and no method names another. The methods read the +//! prior build, whose row a person verified, next to the new build: +//! +//! - calls into it: a function known in both builds calls or loads it from the same spot of its +//! body in both builds; +//! - its calls: in the prior build it calls functions that their own fingerprints find in the new +//! build, and exactly one candidate makes the same calls from the same spots; +//! - its table slot: a class table known in both builds holds it, at the slot that lines up once +//! the slots around it are compared, since a milestone can add or remove a virtual function; +//! - a text it uses: it refers to a text the new build holds once, and exactly one function there +//! refers to that text; +//! - for a class table, references to it (a function known in both builds loads it from the same +//! spot) and its slots (exactly one candidate holds the same functions in the same slots). +//! +//! Anything else stays unresolved with its reason, and its row goes to a person. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fmt; +use std::rc::Rc; + +use crate::chrome_build::PeFile; +use crate::row_port::{code_print, find_prints, find_table, whole_function, x64}; + +/// How many bytes of a function body are read at most. +const BODY_LIMIT: u32 = 0x4000; +/// How far a function the exception table does not list, a leaf, is read at most. +const LEAF_LIMIT: u32 = 0x200; +/// How many instructions before a site name it within its body. +const CONTEXT: usize = 3; +/// How many levels of calls are followed from the functions known in both builds. +const FOLLOW_DEPTH: usize = 3; +/// How many functions are followed at most. +const FOLLOW_LIMIT: usize = 4000; +/// How many slots of a class table are read at most. +const TABLE_SLOTS: u32 = 512; +/// How many slots of a class table are compared to tell one table from another. +const TABLE_SEEN: u32 = 0x80; +/// How many slots on each side of a slot are compared to line two tables up. +const ALIGN_REACH: u32 = 6; +/// How far a milestone may move a slot. +const ALIGN_SHIFT: i64 = 2; +/// How many slots must agree for two tables, or two stretches of them, to be the same. +const ALIGN_LEAST: usize = 3; +/// The shortest text taken as a sign of the function that uses it. +const TEXT_LEAST: usize = 8; +/// The longest text read. +const TEXT_MOST: usize = 256; +/// How many candidates a function with no fingerprint match may have. +const CANDIDATES_MOST: usize = 256; +/// How many times the methods run again with what the last round found. +const ROUNDS: usize = 4; + +/// One instruction of a function body, as far as the methods need it. +#[derive(Debug, Clone)] +pub struct Step { + pub rva: u32, + /// The bytes with the relative value, displacement and immediate cleared. + pub skeleton: Vec, + /// Where a direct call or jump goes. + pub call: Option, + /// What a RIP-relative operand points at: data, a table or a function passed as a callback. + pub reference: Option, + /// A displacement from a register, as `[rcx + 0x558]` holds. + pub displacement: Option, + pub immediate: Option, + /// A call or jump through memory, as `call [rax + 0x2b8]` calls a table slot. + pub indirect: bool, + /// Its operand is the stack: a local variable, or the frame's size. Neither is a value of a row. + pub stack: bool, +} + +/// Whether a memory operand's base is the stack or frame pointer, which hold locals, not members. +fn stack_based(raw: &[u8], displacement: Option<(usize, usize)>) -> bool { + let Some((offset, size)) = displacement else { + return false; + }; + let opcode = opcode_at(raw); + // REX.B names r12 or r13 where the base field says rsp or rbp. + let extended = opcode > 0 && raw[opcode - 1] & 0xF0 == 0x40 && raw[opcode - 1] & 1 != 0; + let mode = if size == 1 { 1 } else { 2 }; + let before = offset.checked_sub(1).and_then(|at| raw.get(at)).copied(); + let base = match before { + // The ModRM byte itself, with no SIB byte after it. + Some(modrm) if modrm >> 6 == mode && modrm & 7 != 4 => modrm & 7, + // A SIB byte, whose low bits name the base. + Some(sib) => sib & 7, + None => return false, + }; + !extended && matches!(base, 4 | 5) +} + +fn signed(bytes: &[u8], field: (usize, usize)) -> Option { + let (offset, size) = field; + let part = bytes.get(offset..offset + size)?; + Some(match size { + 1 => i64::from(part[0] as i8), + 2 => i64::from(i16::from_le_bytes([part[0], part[1]])), + 4 => i64::from(i32::from_le_bytes([part[0], part[1], part[2], part[3]])), + 8 => i64::from_le_bytes(part.try_into().ok()?), + _ => return None, + }) +} + +/// Where the opcode of an instruction is, past its prefixes. +fn opcode_at(raw: &[u8]) -> usize { + let mut at = 0; + while raw.get(at).is_some_and(|byte| { + matches!( + byte, + 0x66 | 0xF2 | 0xF3 | 0xF0 | 0x26 | 0x2E | 0x36 | 0x3E | 0x64 | 0x65 + ) + }) { + at += 1; + } + if raw.get(at).is_some_and(|byte| byte & 0xF0 == 0x40) { + at += 1; + } + at +} + +fn step(raw: &[u8], instruction: &x64::Instruction, rva: u32) -> Step { + let next = i64::from(rva) + instruction.length as i64; + let opcode = opcode_at(raw); + let first = raw.get(opcode).copied(); + let second = raw.get(opcode + 1).copied(); + let branch = matches!(first, Some(0x70..=0x7F | 0xEB | 0xE0..=0xE3 | 0xE8 | 0xE9)) + || (first == Some(0x0F) && matches!(second, Some(0x80..=0x8F))); + let mut call = None; + let mut reference = None; + if let Some(field) = instruction.relative { + let target = signed(raw, field) + .map(|distance| next + distance) + .and_then(|target| u32::try_from(target).ok()); + if matches!(first, Some(0xE8 | 0xE9)) { + call = target; + } else if !branch { + reference = target; + } + } + Step { + rva, + skeleton: x64::skeleton(raw, instruction), + call, + reference, + displacement: instruction + .displacement + .and_then(|field| signed(raw, field)), + immediate: instruction.immediate.and_then(|field| signed(raw, field)), + indirect: first == Some(0xFF) + && second.is_some_and(|modrm| matches!((modrm >> 3) & 7, 2 | 4)), + // `add rsp, n` and `sub rsp, n` size the frame. + stack: stack_based(raw, instruction.displacement) + || (matches!(first, Some(0x81 | 0x83)) && matches!(second, Some(0xC4 | 0xEC))), + } +} + +/// Reads a function body from its start, as far as its exception table entry or, for a leaf, its +/// padding, and stops at the first instruction it cannot read. +fn read_body(pe: &PeFile, ranges: &BTreeMap, start: u32) -> Vec { + if !pe.is_code(start) { + return vec![]; + } + let listed = ranges.get(&start).copied(); + let length = match listed { + Some(end) => end.saturating_sub(start).min(BODY_LIMIT), + None => LEAF_LIMIT, + } as usize; + let Some(bytes) = pe.read(start, length).or_else(|| pe.read(start, 16)) else { + return vec![]; + }; + let mut steps = Vec::new(); + let mut at = 0; + while at < bytes.len() { + let rest = &bytes[at..]; + if listed.is_none() && rest.starts_with(&[0xCC, 0xCC]) { + break; + } + let Some(instruction) = x64::decode_body(rest) else { + break; + }; + if instruction.length == 0 || instruction.length > rest.len() { + break; + } + steps.push(step(&rest[..instruction.length], &instruction, start + at as u32)); + at += instruction.length; + } + steps +} + +/// One build: its module, its function ranges and the bodies read so far. +pub struct Side<'a> { + pub pe: &'a PeFile<'a>, + ranges: BTreeMap, + bodies: HashMap>>, +} + +impl<'a> Side<'a> { + pub fn new(pe: &'a PeFile<'a>) -> Self { + Self { + pe, + ranges: pe.function_ranges(), + bodies: HashMap::new(), + } + } + + pub fn body(&mut self, start: u32) -> Rc> { + if let Some(body) = self.bodies.get(&start) { + return Rc::clone(body); + } + let body = Rc::new(read_body(self.pe, &self.ranges, start)); + self.bodies.insert(start, Rc::clone(&body)); + body + } + + /// The start of the function part that holds an address. + fn enclosing(&self, rva: u32) -> Option { + let (start, end) = self.ranges.range(..=rva).next_back()?; + (rva < *end).then_some(*start) + } + + fn is_start(&self, rva: u32) -> bool { + self.ranges.contains_key(&rva) + } +} + +/// Names the instruction at `index` by the instructions around it, as they stay when code moves. +pub fn context(steps: &[Step], index: usize, before: usize, after: usize) -> Vec { + let first = index.saturating_sub(before); + let last = (index + after).min(steps.len().saturating_sub(1)); + let mut key = vec![(index - first) as u8]; + for step in &steps[first..=last] { + key.push(step.skeleton.len() as u8); + key.extend_from_slice(&step.skeleton); + } + key +} + +/// Where each call or reference of a body points, named by the instructions before it, in order. +fn named_sites(steps: &[Step]) -> HashMap, Vec> { + let mut sites: HashMap, Vec> = HashMap::new(); + for (index, step) in steps.iter().enumerate() { + if let Some(target) = step.call.or(step.reference) { + sites + .entry(context(steps, index, CONTEXT, 0)) + .or_default() + .push(target); + } + } + sites +} + +/// Pairs what the two bodies of one function point at: sites named the same way, which each body +/// has the same number of, in order. +fn pair_sites(old: &HashMap, Vec>, new: &HashMap, Vec>) -> Vec<(u32, u32)> { + let mut pairs = Vec::new(); + for (key, targets) in old { + if let Some(others) = new.get(key).filter(|others| others.len() == targets.len()) { + pairs.extend(targets.iter().copied().zip(others.iter().copied())); + } + } + pairs.sort_unstable(); + pairs +} + +/// Addresses of the prior build with their places in the new one, found by following the calls +/// and references of functions known in both. An address reached two ways that disagree has none. +#[derive(Debug, Default)] +pub struct Followed { + pub places: BTreeMap, + conflicts: BTreeSet, +} + +impl Followed { + /// Records a pair, and says whether it is new. + fn add(&mut self, old: u32, new: u32) -> bool { + if self.conflicts.contains(&old) { + return false; + } + match self.places.get(&old) { + Some(place) if *place == new => false, + Some(_) => { + self.places.remove(&old); + self.conflicts.insert(old); + false + } + None => { + self.places.insert(old, new); + true + } + } + } + + fn get(&self, old: u32) -> Option { + self.places.get(&old).copied() + } +} + +/// Follows calls and references from the functions known in both builds, a few levels deep. +pub fn follow(old: &mut Side, new: &mut Side, known: &BTreeMap) -> Followed { + let mut followed = Followed::default(); + let mut frontier: Vec<(u32, u32)> = known + .iter() + .map(|(from, to)| (*from, *to)) + .filter(|(from, to)| old.pe.is_code(*from) && new.pe.is_code(*to)) + .collect(); + let mut visited = BTreeSet::new(); + for _ in 0..FOLLOW_DEPTH { + let mut next = Vec::new(); + for (from_old, from_new) in frontier { + if visited.len() >= FOLLOW_LIMIT || !visited.insert(from_old) { + continue; + } + let old_sites = named_sites(&old.body(from_old)); + let new_sites = named_sites(&new.body(from_new)); + for (to_old, to_new) in pair_sites(&old_sites, &new_sites) { + if known.contains_key(&to_old) { + continue; + } + if followed.add(to_old, to_new) && old.pe.is_code(to_old) && new.pe.is_code(to_new) + { + next.push((to_old, to_new)); + } + } + } + frontier = next; + } + followed +} + +/// The calls a function makes, as its prior body names them, to functions whose place in the new +/// build is known: `(site, rank among sites named the same, how many such sites, new place)`. +fn expected_calls( + old: &mut Side, + target: u32, + anchors: &BTreeMap, +) -> Vec<(Vec, usize, usize, u32)> { + let body = old.body(target); + let mut expected = Vec::new(); + for (key, callees) in named_sites(&body) { + for (rank, callee) in callees.iter().enumerate() { + if let Some(place) = anchors.get(callee) { + expected.push((key.clone(), rank, callees.len(), *place)); + } + } + } + expected.sort(); + expected +} + +/// The one candidate that makes the expected calls from the same spots, and contradicts none. +fn by_its_calls( + new: &mut Side, + expected: &[(Vec, usize, usize, u32)], + candidates: &[u32], +) -> Vec { + if expected.is_empty() { + return vec![]; + } + let least = expected.len().min(2); + let mut passing = Vec::new(); + for candidate in candidates { + let theirs = named_sites(&new.body(*candidate)); + let (mut hits, mut misses) = (0, 0); + for (key, rank, count, place) in expected { + match theirs.get(key) { + Some(found) if found.len() == *count => { + if found[*rank] == *place { + hits += 1; + } else { + misses += 1; + } + } + _ => {} + } + } + if misses == 0 && hits >= least { + passing.push(*candidate); + } + } + // Two candidates that both pass cannot be told apart: neither is an answer. + if passing.len() == 1 { + passing + } else { + vec![] + } +} + +/// The functions of a build that call any of `wanted` directly, by the function part each call is in. +fn calls_into(side: &Side, wanted: &BTreeSet) -> BTreeMap> { + let mut found: BTreeMap> = BTreeMap::new(); + if wanted.is_empty() { + return found; + } + for (base, bytes, is_code) in side.pe.sections() { + if !is_code { + continue; + } + for at in 0..bytes.len().saturating_sub(4) { + if !matches!(bytes[at], 0xE8 | 0xE9) { + continue; + } + let distance = i32::from_le_bytes([bytes[at + 1], bytes[at + 2], bytes[at + 3], bytes[at + 4]]); + let site = base + at as u32; + let Ok(target) = u32::try_from(i64::from(site) + 5 + i64::from(distance)) else { + continue; + }; + if wanted.contains(&target) { + if let Some(start) = side.enclosing(site) { + found.entry(target).or_default().insert(start); + } + } + } + } + found +} + +/// The functions the prior bodies of `targets` call that their own fingerprints find in exactly one +/// place of the new build. They do not depend on the calls followed from known functions. +fn callee_anchors(old: &mut Side, new: &Side, targets: &[u32]) -> BTreeMap { + let mut prints = BTreeMap::new(); + for target in targets { + let body = old.body(*target); + for step in body.iter() { + let callee = step + .call + .or(step.reference.filter(|place| old.pe.is_code(*place))); + if let Some(callee) = callee { + if let Some(print) = code_print(old.pe, callee) { + prints.insert(callee, print); + } + } + } + } + let mut anchors = BTreeMap::new(); + for (callee, places) in find_prints(new.pe, &prints) { + // A print taken at a function start counts only function starts, as for the row itself. + let places: Vec = places + .into_iter() + .filter(|place| !old.is_start(callee) || new.is_start(*place)) + .collect(); + if let [only] = places.as_slice() { + anchors.insert(callee, *only); + } + } + anchors +} + +/// A function as the slots of a class table show it: its fingerprint, or the whole of a short one. +fn entry_shape(pe: &PeFile, function: u32) -> Option> { + code_print(pe, function).or_else(|| whole_function(pe, function)) +} + +fn same_function(old: &Side, new: &Side, known: &BTreeMap, from: u32, to: u32) -> bool { + if let Some(place) = known.get(&from) { + return *place == to; + } + match (entry_shape(old.pe, from), entry_shape(new.pe, to)) { + (Some(left), Some(right)) => left == right, + _ => false, + } +} + +/// The function in a slot of a class table, or `None` past the table's end. +fn entry(pe: &PeFile, table: u32, slot: i64) -> Option { + let slot = u32::try_from(slot).ok()?; + let at = table.checked_add(slot.checked_mul(8)?)?; + pe.pointer_rva(at).filter(|function| pe.is_code(*function)) +} + +/// How far slot `slot` of the old table moved in the new one, judged by the slots around it. +fn line_up( + old: &Side, + new: &Side, + known: &BTreeMap, + tables: (u32, u32), + slot: u32, +) -> Option { + let mut scores = Vec::new(); + for shift in -ALIGN_SHIFT..=ALIGN_SHIFT { + let mut score = 0; + for near in slot.saturating_sub(ALIGN_REACH)..=slot + ALIGN_REACH { + if near == slot { + continue; + } + let from = entry(old.pe, tables.0, i64::from(near)); + let to = entry(new.pe, tables.1, i64::from(near) + shift); + if let (Some(from), Some(to)) = (from, to) { + if same_function(old, new, known, from, to) { + score += 1; + } + } + } + scores.push((score, shift)); + } + scores.sort_by(|left, right| right.0.cmp(&left.0)); + match scores.as_slice() { + [(top, shift), (second, _), ..] if *top >= ALIGN_LEAST && top > second => Some(*shift), + _ => None, + } +} + +/// The places the class tables known in both builds give a function by its slot. +fn by_table_slot( + old: &Side, + new: &Side, + known: &BTreeMap, + tables: &[(u32, u32)], + target: u32, +) -> BTreeSet { + let mut found = BTreeSet::new(); + for pair in tables { + for slot in 0..TABLE_SLOTS { + let Some(function) = entry(old.pe, pair.0, i64::from(slot)) else { + break; + }; + if function != target { + continue; + } + if let Some(shift) = line_up(old, new, known, *pair, slot) { + if let Some(place) = entry(new.pe, pair.1, i64::from(slot) + shift) { + found.insert(place); + } + } + } + } + found +} + +/// The candidate tables whose slots hold the same functions as the old table, and how many agree. +fn by_slots( + old: &Side, + new: &Side, + known: &BTreeMap, + target: u32, + candidates: &[u32], +) -> Vec { + let mut passing = Vec::new(); + for candidate in candidates { + let (mut seen, mut agreed) = (0, 0); + for slot in 0..TABLE_SEEN { + let Some(function) = entry(old.pe, target, i64::from(slot)) else { + break; + }; + seen += 1; + let agrees = (-1..=1).any(|shift| { + entry(new.pe, *candidate, i64::from(slot) + shift) + .is_some_and(|place| same_function(old, new, known, function, place)) + }); + if agrees { + agreed += 1; + } + } + if agreed >= ALIGN_LEAST && agreed * 2 >= seen { + passing.push(*candidate); + } + } + passing +} + +/// Tables of the new build that hold, near the same slot, a function the old table holds and +/// whose place is known. +fn table_candidates(old: &Side, new: &Side, known: &BTreeMap, target: u32) -> Vec { + let mut found = BTreeSet::new(); + let mut tried = 0; + for slot in 0..TABLE_SEEN { + let Some(function) = entry(old.pe, target, i64::from(slot)) else { + break; + }; + let Some(place) = known.get(&function).filter(|place| new.pe.is_code(**place)) else { + continue; + }; + for shifted in slot.saturating_sub(1)..=slot + 1 { + found.extend(find_table(new.pe, shifted, *place)); + } + tried += 1; + if tried == 2 { + break; + } + } + found.into_iter().take(CANDIDATES_MOST).collect() +} + +/// A table a function known in both builds loads from the same spot, or a place a few slots into +/// one, which keeps its distance from the table's start. +fn by_reference(followed: &Followed, target: u32) -> Option { + let (from, to) = followed + .places + .range(target.saturating_sub(0x40)..=target) + .next_back()?; + to.checked_add(target - from) +} + +/// The text at an address of a data section, with its terminating zero, when it is plain text. +fn text_at(pe: &PeFile, rva: u32) -> Option> { + if pe.is_code(rva) { + return None; + } + let window = pe + .read(rva, TEXT_MOST + 1) + .or_else(|| pe.read(rva, TEXT_LEAST + 1))?; + let end = window.iter().position(|byte| *byte == 0)?; + let text = &window[..end]; + if text.len() < TEXT_LEAST || !text.iter().all(|byte| (0x20..0x7F).contains(byte)) { + return None; + } + Some(window[..=end].to_vec()) +} + +/// For each text, the functions of a build that load it, when the build holds it exactly once. +fn functions_using(side: &Side, texts: &BTreeSet>) -> BTreeMap, BTreeSet> { + let mut by_head: HashMap<[u8; TEXT_LEAST], Vec<&Vec>> = HashMap::new(); + let mut first = [false; 256]; + for text in texts { + let head: [u8; TEXT_LEAST] = text[..TEXT_LEAST].try_into().expect("checked length"); + first[head[0] as usize] = true; + by_head.entry(head).or_default().push(text); + } + let mut places: BTreeMap<&Vec, Vec> = BTreeMap::new(); + for (base, bytes, is_code) in side.pe.sections() { + if is_code { + continue; + } + for at in 0..bytes.len().saturating_sub(TEXT_LEAST) { + if !first[bytes[at] as usize] { + continue; + } + let head: [u8; TEXT_LEAST] = bytes[at..at + TEXT_LEAST].try_into().expect("in range"); + for text in by_head.get(&head).map_or(&[][..], Vec::as_slice) { + if bytes[at..].starts_with(text) { + places.entry(*text).or_default().push(base + at as u32); + } + } + } + } + let at_address: HashMap> = places + .iter() + .filter_map(|(text, found)| match found.as_slice() { + [only] => Some((*only, *text)), + _ => None, + }) + .collect(); + let mut users: BTreeMap, BTreeSet> = BTreeMap::new(); + for (base, bytes, is_code) in side.pe.sections() { + if !is_code { + continue; + } + // `lea` or `mov` of a register from [rip + disp32]: REX, 8D or 8B, a ModRM of 00 reg 101. + for at in 3..bytes.len().saturating_sub(4) { + if bytes[at - 1] & 0xC7 != 0x05 + || !matches!(bytes[at - 2], 0x8D | 0x8B) + || bytes[at - 3] & 0xF0 != 0x40 + { + continue; + } + let distance = i32::from_le_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]]); + let site = base + at as u32; + let Ok(target) = u32::try_from(i64::from(site) + 4 + i64::from(distance)) else { + continue; + }; + if let Some(text) = at_address.get(&target) { + if let Some(start) = side.enclosing(site) { + users.entry((*text).clone()).or_default().insert(start); + } + } + } + } + users +} + +/// Which method gave a place. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Method { + CallsInto, + ItsCalls, + TableSlot, + Text, + References, + Slots, +} + +impl fmt::Display for Method { + fn fmt(&self, out: &mut fmt::Formatter) -> fmt::Result { + out.write_str(match self { + Method::CallsInto => "calls into it", + Method::ItsCalls => "its calls", + Method::TableSlot => "its table slot", + Method::Text => "a text it uses", + Method::References => "references to it", + Method::Slots => "its slots", + }) + } +} + +fn names(methods: &BTreeSet) -> String { + methods + .iter() + .map(Method::to_string) + .collect::>() + .join(" and ") +} + +/// Takes a place only when two different methods name it, no method names another, and it is one +/// of the fingerprint's places where there are any. +pub fn decide( + answers: &[(Method, u32)], + candidates: Option<&[u32]>, +) -> Result<(u32, BTreeSet), String> { + let places: BTreeSet = answers.iter().map(|(_, place)| *place).collect(); + let methods: BTreeSet = answers.iter().map(|(method, _)| *method).collect(); + match places.iter().copied().collect::>().as_slice() { + [] => Err("no method found it".to_owned()), + [place] => { + if candidates.is_some_and(|candidates| !candidates.contains(place)) { + Err(format!( + "{} found {place:#x}, which is not one of its fingerprint's places", + names(&methods) + )) + } else if methods.len() < 2 { + Err(format!( + "only {} found it, at {place:#x}, and a second method is needed", + names(&methods) + )) + } else { + Ok((*place, methods)) + } + } + _ => { + let mut said: Vec = answers + .iter() + .map(|(method, place)| format!("{method} {place:#x}")) + .collect(); + said.sort(); + said.dedup(); + Err(format!("the methods disagree: {}", said.join(", "))) + } + } +} + +/// What an address of the prior row is. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Function, + Table, +} + +/// An address the fingerprints did not pin down, with the places they matched, if any. +#[derive(Debug, Clone)] +pub struct Wanted { + pub kind: Kind, + pub candidates: Option>, +} + +/// What the methods found. +#[derive(Debug, Default)] +pub struct Resolution { + /// The new place of each address two methods agree on. + pub places: BTreeMap, + /// For the report: each place and the methods that found it. + pub settled: Vec, + /// Why each other address has no place. + pub reasons: BTreeMap, + /// Functions of the prior build with their places in the new one, as the calls of known + /// functions pair them, for reading values from the same instructions in both builds. + pub followed: BTreeMap, +} + +/// Looks for each wanted address with every method, in rounds, since a place found in one round +/// is known in the next. `known` holds the places the fingerprints found, and `tables` names the +/// prior row's addresses that are class tables. +pub fn resolve( + old_pe: &PeFile, + new_pe: &PeFile, + known: &BTreeMap, + wanted: &BTreeMap, + tables: &BTreeSet, +) -> Resolution { + let mut old = Side::new(old_pe); + let mut new = Side::new(new_pe); + let mut known = known.clone(); + let mut resolution = Resolution::default(); + let functions: Vec = wanted + .iter() + .filter(|(_, want)| want.kind == Kind::Function) + .map(|(target, _)| *target) + .collect(); + + // What does not depend on the known places is read once: the callees found by their own + // fingerprints, the functions that call them, and the texts each function uses. + let anchors = callee_anchors(&mut old, &new, &functions); + let mut texts_of: BTreeMap>> = BTreeMap::new(); + for target in &functions { + let body = old.body(*target); + let texts: BTreeSet> = body + .iter() + .filter_map(|step| step.reference) + .filter_map(|place| text_at(old.pe, place)) + .collect(); + texts_of.insert(*target, texts); + } + let all_texts: BTreeSet> = texts_of.values().flatten().cloned().collect(); + let users = functions_using(&new, &all_texts); + let mut seen_anchors = anchors.clone(); + for (from, to) in &known { + if old.pe.is_code(*from) { + seen_anchors.insert(*from, *to); + } + } + let mut expected = BTreeMap::new(); + let mut called = BTreeSet::new(); + for target in &functions { + let calls = expected_calls(&mut old, *target, &seen_anchors); + called.extend(calls.iter().map(|call| call.3)); + expected.insert(*target, calls); + } + let callers = calls_into(&new, &called); + + let mut pending: BTreeSet = wanted.keys().copied().collect(); + for _ in 0..ROUNDS { + let followed = follow(&mut old, &mut new, &known); + let table_pairs: Vec<(u32, u32)> = known + .iter() + .filter(|(from, _)| tables.contains(from)) + .map(|(from, to)| (*from, *to)) + .collect(); + let mut found_any = false; + for target in pending.clone() { + let want = &wanted[&target]; + let mut answers = Vec::new(); + match want.kind { + Kind::Function => { + if let Some(place) = followed.get(target) { + answers.push((Method::CallsInto, place)); + } + let calls = &expected[&target]; + let candidates: Vec = match &want.candidates { + Some(list) => list.clone(), + None => calls + .iter() + .flat_map(|call| callers.get(&call.3).into_iter().flatten().copied()) + .collect::>() + .into_iter() + .take(CANDIDATES_MOST) + .collect(), + }; + for place in by_its_calls(&mut new, calls, &candidates) { + answers.push((Method::ItsCalls, place)); + } + for place in by_table_slot(&old, &new, &known, &table_pairs, target) { + answers.push((Method::TableSlot, place)); + } + if let Some(place) = by_text(&texts_of[&target], &users) { + answers.push((Method::Text, place)); + } + } + Kind::Table => { + if let Some(place) = by_reference(&followed, target) { + answers.push((Method::References, place)); + } + let candidates = match &want.candidates { + Some(list) => list.clone(), + None => table_candidates(&old, &new, &known, target), + }; + for place in by_slots(&old, &new, &known, target, &candidates) { + answers.push((Method::Slots, place)); + } + } + } + match decide(&answers, want.candidates.as_deref()) { + Ok((place, methods)) => { + known.insert(target, place); + resolution.places.insert(target, place); + resolution.settled.push(format!( + "{target:#x} -> {place:#x} by {}", + names(&methods) + )); + resolution.reasons.remove(&target); + pending.remove(&target); + found_any = true; + } + Err(reason) => { + resolution.reasons.insert(target, reason); + } + } + } + resolution.followed = followed.places; + if !found_any || pending.is_empty() { + break; + } + } + resolution +} + +/// The one function that uses every text of a target the new build holds once. +fn by_text(texts: &BTreeSet>, users: &BTreeMap, BTreeSet>) -> Option { + let mut common: Option> = None; + for text in texts { + let Some(found) = users.get(text) else { + continue; + }; + common = Some(match common { + Some(common) => common.intersection(found).copied().collect(), + None => found.clone(), + }); + } + let common = common?; + match common.iter().copied().collect::>().as_slice() { + [only] => Some(*only), + _ => None, + } +} + +/// How many instructions on each side of a use name it within its body. +const VALUE_CONTEXT: usize = 2; + +/// Which operand of an instruction holds a value. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Operand { + Displacement, + Immediate, +} + +/// A value of the prior row read again from the new build, or why it could not be. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reading { + pub path: String, + pub prior: u64, + /// The new value and how it was read. + pub value: Result<(u64, String), String>, +} + +fn keys(steps: &[Step]) -> Vec> { + (0..steps.len()) + .map(|index| context(steps, index, VALUE_CONTEXT, VALUE_CONTEXT)) + .collect() +} + +/// Whether an instruction uses `wanted`: as a member displacement or an immediate for a plain +/// value, or as the displacement of a call through a table for a slot, which is eight bytes each. +fn use_of(step: &Step, wanted: i64, slot: bool) -> Option { + if step.stack { + return None; + } + if slot { + return (step.indirect && step.displacement == Some(wanted)).then_some(Operand::Displacement); + } + if step.displacement == Some(wanted) { + Some(Operand::Displacement) + } else if step.immediate == Some(wanted) { + Some(Operand::Immediate) + } else { + None + } +} + +/// Reads one value from the instructions that use it. Each use of `prior` in a prior function body +/// is named by the instructions around it, with their own operands cleared, and the same +/// instruction is looked up in the function's new body. A value is taken only when at least two +/// uses are read and every use read gives the same value. +fn read_value( + old: &mut Side, + new: &mut Side, + pairs: &[(u32, u32)], + prior: u64, + slot: bool, +) -> Result<(u64, String), String> { + if prior == 0 { + return Err("zero is used too widely in code to be told apart".to_owned()); + } + let wanted = if slot { prior.checked_mul(8) } else { Some(prior) } + .and_then(|value| i64::try_from(value).ok()) + .ok_or_else(|| format!("{prior:#x} is too large to be an operand"))?; + let mut found: Vec<(u64, u32)> = Vec::new(); + let mut uses = 0; + for (from, to) in pairs { + let old_body = old.body(*from); + let new_body = new.body(*to); + let old_keys = keys(&old_body); + let new_keys = keys(&new_body); + for (index, step) in old_body.iter().enumerate() { + let Some(operand) = use_of(step, wanted, slot) else { + continue; + }; + uses += 1; + let key = &old_keys[index]; + let rank = old_keys[..index].iter().filter(|other| *other == key).count(); + let count = old_keys.iter().filter(|other| *other == key).count(); + let theirs: Vec = new_keys + .iter() + .enumerate() + .filter(|(_, other)| *other == key) + .map(|(at, _)| at) + .collect(); + if theirs.len() != count { + continue; + } + let other = &new_body[theirs[rank]]; + let read = match operand { + Operand::Displacement => other.displacement, + Operand::Immediate => other.immediate, + }; + let Some(read) = read.and_then(|read| u64::try_from(read).ok()) else { + continue; + }; + if slot && read % 8 != 0 { + continue; + } + found.push((if slot { read / 8 } else { read }, *from)); + } + } + let values: BTreeSet = found.iter().map(|(value, _)| *value).collect(); + let functions: BTreeSet = found.iter().map(|(_, function)| *function).collect(); + match values.iter().copied().collect::>().as_slice() { + [] if uses == 0 => Err(format!( + "no instruction of the {} functions known in both builds uses {prior:#x}", + pairs.len() + )), + [] => Err(format!( + "its {uses} uses in the prior build were not found in the new one" + )), + [value] if found.len() >= 2 => Ok(( + *value, + format!( + "read at {} uses in {} of its functions", + found.len(), + functions.len() + ), + )), + [value] => Err(format!( + "read at only one use, as {value:#x}, and a second reading is needed" + )), + _ => Err(format!("its uses disagree: {values:#x?}")), + } +} + +/// Reads each value of the prior row that has no recipe from the instructions that use it in the +/// functions of its part of the row, and in the functions they call, in both builds. `sections` +/// pairs the prior and new places of each part's functions, and `followed` pairs the functions +/// the calls of known functions reach. A value's part is the first name of its path, and a value +/// whose name ends in `_slot` is a slot of a class table. +pub fn read_values( + old_pe: &PeFile, + new_pe: &PeFile, + sections: &BTreeMap>, + followed: &BTreeMap, + values: &[(String, u64)], +) -> Vec { + let mut old = Side::new(old_pe); + let mut new = Side::new(new_pe); + let mut pairs_of: BTreeMap<&str, Vec<(u32, u32)>> = BTreeMap::new(); + for (section, pairs) in sections { + let mut all: BTreeSet<(u32, u32)> = pairs.iter().copied().collect(); + for (from, _) in pairs { + let body = old.body(*from); + for step in body.iter() { + if let Some(place) = step.call.and_then(|callee| followed.get(&callee)) { + all.insert((step.call.expect("checked"), *place)); + } + } + } + pairs_of.insert(section.as_str(), all.into_iter().collect()); + } + let mut readings = Vec::new(); + for (path, prior) in values { + let section = path.split('.').next().unwrap_or_default(); + let name = path.rsplit('.').next().unwrap_or_default(); + let slot = name.split('[').next().unwrap_or_default().ends_with("_slot"); + let pairs = pairs_of.get(section).map_or(&[][..], Vec::as_slice); + readings.push(Reading { + path: path.clone(), + prior: *prior, + value: read_value(&mut old, &mut new, pairs, *prior, slot), + }); + } + readings +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + pub const IMAGE_BASE: u64 = 0x1_8000_0000; + /// Where a fixture keeps its exception table, in its data section. + const EXCEPTIONS: u32 = 0x20f000; + + fn place(rva: u32) -> usize { + if rva >= 0x200000 { + 0x10400 + (rva - 0x200000) as usize + } else { + 0x400 + (rva - 0x100000) as usize + } + } + + /// A 64-bit module with a code section at 0x100000 and a data section at 0x200000, holding + /// the given bytes and an exception table that lists `ranges`. + pub fn module(parts: &[(u32, Vec)], ranges: &[(u32, u32)]) -> Vec { + let mut bytes = vec![0u8; 0x400]; + bytes[0x3c..0x40].copy_from_slice(&0x80u32.to_le_bytes()); + bytes[0x80..0x84].copy_from_slice(b"PE\0\0"); + bytes[0x86..0x88].copy_from_slice(&2u16.to_le_bytes()); + bytes[0x94..0x96].copy_from_slice(&240u16.to_le_bytes()); + bytes[0x98..0x9a].copy_from_slice(&0x20bu16.to_le_bytes()); + bytes[0x98 + 24..0x98 + 32].copy_from_slice(&IMAGE_BASE.to_le_bytes()); + let sections = [ + (0x100000u32, 0x400u32, 0x2000_0000u32), + (0x200000, 0x10400, 0x4000_0000), + ]; + for (index, (rva, raw, flags)) in sections.iter().enumerate() { + let header = 0x98 + 240 + 40 * index; + bytes[header + 8..header + 12].copy_from_slice(&0x10000u32.to_le_bytes()); + bytes[header + 12..header + 16].copy_from_slice(&rva.to_le_bytes()); + bytes[header + 16..header + 20].copy_from_slice(&0x10000u32.to_le_bytes()); + bytes[header + 20..header + 24].copy_from_slice(&raw.to_le_bytes()); + bytes[header + 36..header + 40].copy_from_slice(&flags.to_le_bytes()); + } + bytes.resize(0x400 + 0x20000, 0); + bytes[0x400..0x10400].fill(0xCC); + let mut table = vec![]; + for (start, end) in ranges { + table.extend_from_slice(&start.to_le_bytes()); + table.extend_from_slice(&end.to_le_bytes()); + table.extend_from_slice(&0u32.to_le_bytes()); + } + let directory = 0x98 + 112 + 24; + bytes[directory..directory + 4].copy_from_slice(&EXCEPTIONS.to_le_bytes()); + bytes[directory + 4..directory + 8].copy_from_slice(&(table.len() as u32).to_le_bytes()); + let at = place(EXCEPTIONS); + bytes[at..at + table.len()].copy_from_slice(&table); + for (rva, data) in parts { + let at = place(*rva); + bytes[at..at + data.len()].copy_from_slice(data); + } + bytes + } + + pub fn call(from: u32, to: u32) -> Vec { + let mut bytes = vec![0xE8]; + bytes.extend_from_slice(&((i64::from(to) - i64::from(from) - 5) as i32).to_le_bytes()); + bytes + } + + pub fn table(functions: &[u32]) -> Vec { + functions + .iter() + .flat_map(|function| (IMAGE_BASE + u64::from(*function)).to_le_bytes()) + .collect() + } + + /// sub rsp, 28h; call `to`; add rsp, 28h; ret. + fn caller(at: u32, to: u32) -> Vec { + let mut bytes = vec![0x48, 0x83, 0xEC, 0x28]; + bytes.extend(call(at + 4, to)); + bytes.extend([0x48, 0x83, 0xC4, 0x28, 0xC3]); + bytes + } + + /// push rsi; sub rsp, 20h; mov rsi, rcx; add rsp, 20h; pop rsi; ret. + const TARGET: [u8; 14] = [ + 0x56, 0x48, 0x83, 0xEC, 0x20, 0x48, 0x89, 0xCE, 0x48, 0x83, 0xC4, 0x20, 0x5E, 0xC3, + ]; + + /// `mov eax, n; ret`: a different small function for each slot of a table. + fn small(n: u8) -> Vec { + vec![0xB8, n, 0, 0, 0, 0xC3] + } + + const OLD_CALLER: u32 = 0x101000; + const OLD_TARGET: u32 = 0x101100; + const OLD_TABLE: u32 = 0x200100; + const NEW_CALLER: u32 = 0x103000; + const NEW_TARGETS: [u32; 2] = [0x103100, 0x103200]; + const NEW_TABLE: u32 = 0x200800; + + /// The prior build: a caller of the target, and a table that holds it in slot 3. + fn old_module() -> Vec { + let mut parts = vec![ + (OLD_CALLER, caller(OLD_CALLER, OLD_TARGET)), + (OLD_TARGET, TARGET.to_vec()), + ]; + let mut slots = vec![]; + for n in 0..6u8 { + let at = 0x101400 + 0x10 * u32::from(n); + parts.push((at, small(n))); + slots.push(at); + } + slots.insert(3, OLD_TARGET); + parts.push((OLD_TABLE, table(&slots))); + module( + &parts, + &[(OLD_CALLER, OLD_CALLER + 14), (OLD_TARGET, OLD_TARGET + 14)], + ) + } + + /// The new build: the target's bytes in two places, a caller of the first, and a table that + /// holds `in_table` in slot 3. + fn new_module(in_table: u32) -> Vec { + let mut parts = vec![ + (NEW_CALLER, caller(NEW_CALLER, NEW_TARGETS[0])), + (NEW_TARGETS[0], TARGET.to_vec()), + (NEW_TARGETS[1], TARGET.to_vec()), + ]; + let mut slots = vec![]; + for n in 0..6u8 { + let at = 0x103400 + 0x10 * u32::from(n); + parts.push((at, small(n))); + slots.push(at); + } + slots.insert(3, in_table); + parts.push((NEW_TABLE, table(&slots))); + let ranges = [ + (NEW_CALLER, NEW_CALLER + 14), + (NEW_TARGETS[0], NEW_TARGETS[0] + 14), + (NEW_TARGETS[1], NEW_TARGETS[1] + 14), + ]; + module(&parts, &ranges) + } + + fn run(new_bytes: &[u8], known_table: bool) -> Resolution { + let old_bytes = old_module(); + let old = PeFile::parse(&old_bytes).unwrap(); + let new = PeFile::parse(new_bytes).unwrap(); + let mut known = BTreeMap::from([(OLD_CALLER, NEW_CALLER)]); + if known_table { + known.insert(OLD_TABLE, NEW_TABLE); + } + let wanted = BTreeMap::from([( + OLD_TARGET, + Wanted { + kind: Kind::Function, + candidates: Some(NEW_TARGETS.to_vec()), + }, + )]); + resolve(&old, &new, &known, &wanted, &BTreeSet::from([OLD_TABLE])) + } + + #[test] + fn nfr6_a_function_with_several_places_takes_the_one_two_methods_agree_on() { + let resolution = run(&new_module(NEW_TARGETS[0]), true); + assert_eq!(resolution.places.get(&OLD_TARGET), Some(&NEW_TARGETS[0])); + assert_eq!( + resolution.settled, + ["0x101100 -> 0x103100 by calls into it and its table slot"] + ); + assert!(resolution.reasons.is_empty(), "{:?}", resolution.reasons); + } + + #[test] + fn nfr6_methods_that_disagree_leave_the_function_unresolved() { + let resolution = run(&new_module(NEW_TARGETS[1]), true); + assert!(resolution.places.is_empty()); + let reason = &resolution.reasons[&OLD_TARGET]; + assert!(reason.starts_with("the methods disagree: "), "{reason}"); + assert!(reason.contains("calls into it 0x103100"), "{reason}"); + assert!(reason.contains("its table slot 0x103200"), "{reason}"); + } + + #[test] + fn nfr6_one_method_alone_does_not_place_a_function() { + let resolution = run(&new_module(NEW_TARGETS[0]), false); + assert!(resolution.places.is_empty()); + let reason = &resolution.reasons[&OLD_TARGET]; + assert!(reason.starts_with("only calls into it found it"), "{reason}"); + } + + #[test] + fn nfr6_a_place_outside_the_fingerprint_matches_is_refused() { + let answers = [(Method::CallsInto, 0x103300), (Method::TableSlot, 0x103300)]; + let refused = decide(&answers, Some(&NEW_TARGETS[..])).unwrap_err(); + assert!(refused.contains("not one of its fingerprint's places"), "{refused}"); + let (place, methods) = decide(&answers, None).unwrap(); + assert_eq!(place, 0x103300); + assert_eq!(methods.len(), 2); + assert_eq!(decide(&[], None).unwrap_err(), "no method found it"); + } +} diff --git a/scripts/tests/browser-rows.test.sh b/scripts/tests/browser-rows.test.sh index 6834e3d..2e38aa8 100755 --- a/scripts/tests/browser-rows.test.sh +++ b/scripts/tests/browser-rows.test.sh @@ -25,6 +25,12 @@ expect "proves a row on the real browser" "windows-row-proof.ps1" expect "does not publish a row that needs review" "steps.made.outputs.for_review == 'false'" expect "publishes a signed table" "table-sign" expect "signs with a secret" "secrets.RIVET_TABLE_SIGNING_KEY" +expect "makes a row from fingerprints when the new PDB is missing" "chrome-builds.exe find" +expect "tells a missing PDB from a failed download" "catch [System.IO.FileNotFoundException]" +expect "compares a row made without its PDB once the PDB appears" "chrome-builds.exe pdb-check" +expect "reports a row that differs from its PDB" "Row differs from its PDB" +expect "a trial run publishes nothing" "needs.row.result != 'skipped' && !inputs.trial" +expect "a trial run proposes nothing" "steps.made.outputs.for_review == 'true' && !inputs.trial" if [[ -e "$repo_root/.github/workflows/chrome-support.yml" ]]; then echo "chrome-support.yml covers Chrome only and is replaced by browser-rows.yml" >&2