From 54a2719ee62c3c6c4ecfe651719c8301081c7589 Mon Sep 17 00:00:00 2001 From: Matthew Feroz <136640686+MatthewFeroz@users.noreply.github.com> Date: Wed, 9 Sep 2026 04:14:46 +0000 Subject: [PATCH] Prepare headless chart rendering and screenshot update workflow --- .github/workflows/screenshots.yml | 50 +++++++++++++++++++++++++++++++ README.md | 27 +++++++++++++++++ corporate-travel/main.py | 9 ++++-- render.py | 46 ++++++++++++++++++++++++++++ requirements.txt | 4 +-- test_render.py | 36 ++++++++++++++++++++++ 6 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/screenshots.yml create mode 100644 render.py create mode 100644 test_render.py diff --git a/.github/workflows/screenshots.yml b/.github/workflows/screenshots.yml new file mode 100644 index 0000000..9f5ab1d --- /dev/null +++ b/.github/workflows/screenshots.yml @@ -0,0 +1,50 @@ +name: Update chart screenshots + +on: + workflow_dispatch: + inputs: + corporate_travel: + description: Include corporate-travel (requires the external source to respond) + type: boolean + default: true + +permissions: + contents: write + pull-requests: write + +concurrency: + group: chart-screenshots + cancel-in-progress: false + +jobs: + render: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + cache: pip + - run: python -m pip install -r requirements.txt + - name: Render charts + env: + INCLUDE_CORPORATE_TRAVEL: ${{ inputs.corporate_travel }} + run: | + if [ "$INCLUDE_CORPORATE_TRAVEL" = "true" ]; then + python render.py + else + python render.py --charts pay food richest-by-age store-sales + fi + - uses: peter-evans/create-pull-request@v8 + with: + branch: automated/chart-screenshots + commit-message: Update chart screenshots + title: Update chart screenshots + body: | + Regenerated with render.py on Python 3.12. + Corporate travel included: ${{ inputs.corporate_travel }}. + Please review the image changes before merging. + add-paths: | + */screenshot.png + pay/zoomed.png diff --git a/README.md b/README.md index 666d701..eec242a 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,30 @@ Company Data + +## Regenerate screenshots + +Use Python 3.12, install `requirements.txt`, then run `python render.py`. +This saves PNGs without opening a window, including the pay detail view. +The scripts still support interactive use when run individually. + +`corporate-travel` fetches 100 company pages. Requests have timeouts and HTTP +or missing-data failures stop the run before any existing screenshots are +replaced. The source can deny automated requests (HTTP 403); resolve source +access before retrying. To explicitly render only local datasets, use: + +``` +python render.py --charts pay food richest-by-age store-sales +``` + +Use `--output-dir /path/to/previews` to inspect outputs elsewhere first. +The charts retain their original historical datasets; regeneration does not +make them current. The pay detail view uses x=85..753 and y=63..248 to approximate +the existing screenshot's visible region. + +In GitHub Actions, manually run **Update chart screenshots**. It renders all +charts by default; uncheck `corporate_travel` to leave that image unchanged. +Successful runs create or update a PR on `automated/chart-screenshots` when +images differ. Enable **Allow GitHub Actions to create and approve pull +requests** in the repository's Actions settings. This workflow opens PRs; +it does not approve or merge them. diff --git a/corporate-travel/main.py b/corporate-travel/main.py index 597b288..c82a1bf 100644 --- a/corporate-travel/main.py +++ b/corporate-travel/main.py @@ -110,9 +110,14 @@ for line in s.splitlines(): if 'McKinsey' in line: line += ' ' - response = SESSION.get("https://www.businesstravelnews.com/Corporate-Travel-100/2018/" + re.sub('\s*&\s*', '-', line).replace(' ', '-').replace('.', '').replace('Intel', 'Intel1')).text + response = SESSION.get("https://www.businesstravelnews.com/Corporate-Travel-100/2018/" + re.sub(r'\s*&\s*', '-', line).replace(' ', '-').replace('.', '').replace('Intel', 'Intel1'), timeout=(10, 30)) + response.raise_for_status() + response = response.text volumeStrings = [line for line in response.splitlines() if "Volume" in line] - volumePerCompany[line] = float(re.search("\$(.*?) million", volumeStrings[0]).group(1)) + match = re.search(r'\$(.*?) million', volumeStrings[0]) if volumeStrings else None + if match is None: + raise ValueError(f'No air volume found for {line}') + volumePerCompany[line] = float(match.group(1)) plt.figure(figsize=(8, 15)) plt.barh(*zip(*OrderedDict(sorted(volumePerCompany.items(), key=itemgetter(1))).items())) diff --git a/render.py b/render.py new file mode 100644 index 0000000..041160f --- /dev/null +++ b/render.py @@ -0,0 +1,46 @@ +"""Render charts without a display; publish files only after every chart succeeds.""" +import argparse +from pathlib import Path +import runpy +import shutil +import tempfile + +import matplotlib + +matplotlib.use('Agg') +import matplotlib.pyplot as plt + +ROOT = Path(__file__).resolve().parent +CHARTS = ('pay', 'food', 'richest-by-age', 'store-sales', 'corporate-travel') + + +def render(charts, output): + with tempfile.TemporaryDirectory() as directory: + staged = Path(directory) + for chart in charts: + target = staged / chart + target.mkdir(exist_ok=True) + try: + runpy.run_path(str(ROOT / chart / 'main.py'), run_name='__main__') + plt.savefig(target / 'screenshot.png') + if chart == 'pay': + # Reproduce the detail view in the existing pay/zoomed.png. + plt.gcf().set_size_inches(20.48, 10.24) + plt.xlim(85, 753) + plt.ylim(63, 248) + plt.savefig(target / 'zoomed.png') + finally: + plt.close('all') + for source in sorted(staged.glob('*/*.png')): + destination = output / source.relative_to(staged) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, destination) + print(destination) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--charts', nargs='+', choices=CHARTS, default=CHARTS) + parser.add_argument('--output-dir', type=Path, default=ROOT) + args = parser.parse_args() + render(args.charts, args.output_dir) diff --git a/requirements.txt b/requirements.txt index d781d3e..987192e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -requests==2.22.0 -matplotlib==3.1.1 +requests==2.34.2 +matplotlib==3.11.1 diff --git a/test_render.py b/test_render.py new file mode 100644 index 0000000..e409eae --- /dev/null +++ b/test_render.py @@ -0,0 +1,36 @@ +import tempfile +from pathlib import Path +from unittest.mock import patch + +import matplotlib.pyplot as plt + +from render import render + + +def test_failed_chart_preserves_existing_outputs(): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + previous = output / 'pay' / 'screenshot.png' + previous.parent.mkdir() + previous.write_bytes(b'previous screenshot') + + def run_chart(path, **kwargs): + if 'food' in path: + raise ValueError('source unavailable') + plt.plot([1, 2], [3, 4]) + + with patch('render.runpy.run_path', side_effect=run_chart): + try: + render(('pay', 'food'), output) + except ValueError as error: + assert str(error) == 'source unavailable' + else: + raise AssertionError('Failed chart must fail the rendering run') + assert previous.read_bytes() == b'previous screenshot' + assert not (output / 'pay' / 'zoomed.png').exists() + assert not (output / 'food').exists() + + +if __name__ == '__main__': + test_failed_chart_preserves_existing_outputs() + print('Failed chart preserves existing outputs: passed')