Adding linting, running tests, and check links to the pipeline

.github/workflows/deploy.yml

@@ -9,7 +9,85 @@ on:
     branches:
       - main
 
+# Every job except close_pull_request_job carries the same `if`: run on pushes
+# to main and on open PRs, but only build the Crowdin `localization` branch
+# once a review is requested. GitHub Actions has no way to share a job
+# condition, so the expression is repeated verbatim; keep the copies in sync.
+#
+# Job layout:
+#   lint_job, doctest_job      independent checks on the repo sources
+#   build_job                  full multi-language site build
+#   link_check_job             needs the built site from build_job
+#   deploy_job                 waits for all of the above, so a failing check
+#                              blocks the deploy (production and PR previews)
 jobs:
+  lint_job:
+    if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
+    runs-on: ubuntu-latest
+    name: Lint build scripts
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Set up uv
+        # `./run scripts check` runs ruff and mypy through uvx.
+        uses: astral-sh/setup-uv@v10.0.1
+      - name: Install shell tooling
+        # shellcheck ships on the ubuntu runner image; shfmt does not, so it
+        # is pinned and downloaded (formatting rules can drift between shfmt
+        # versions, and a moving version would make the check flaky).
+        run: |
+          command -v shellcheck >/dev/null || sudo apt-get install -y shellcheck
+          curl -sSfL "https://github.com/mvdan/sh/releases/download/${SHFMT_VERSION}/shfmt_${SHFMT_VERSION}_linux_amd64" -o /usr/local/bin/shfmt
+          chmod +x /usr/local/bin/shfmt
+          shellcheck --version | head -2
+          shfmt --version
+        env:
+          SHFMT_VERSION: v3.14.0
+      - name: Check scripts
+        # ruff lint + format diff + mypy --strict on the qualified Python
+        # sources (pyproject.toml); shellcheck + shfmt diff on the run scripts.
+        run: ./run scripts check
+
+  doctest_job:
+    if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
+    runs-on: ubuntu-latest
+    name: Doctest C# code blocks
+    env:
+      # Optional repository secret: a direct download URL for the Linux x64
+      # Tabular Editor CLI archive (te-linux-x64.tar.gz). The CLI is gated
+      # behind sign-in, so there is no public URL to fetch it from. When the
+      # secret is unset, only the te-free annotation validation runs.
+      TE_CLI_URL: ${{ secrets.TE_CLI_DOWNLOAD_URL }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Validate doctest annotations
+        # Grammar and coverage of the ```csharp {compile} / {run ...} fences
+        # in every annotated content/*.md; no te needed.
+        run: ./run doctest validate
+      - name: Install Tabular Editor CLI
+        if: env.TE_CLI_URL != ''
+        run: |
+          mkdir -p "$RUNNER_TEMP/te"
+          curl -sSfL "$TE_CLI_URL" | tar -xz -C "$RUNNER_TEMP/te"
+          chmod +x "$RUNNER_TEMP/te/te"
+          echo "$RUNNER_TEMP/te" >> "$GITHUB_PATH"
+          "$RUNNER_TEMP/te/te" --version
+      - name: Run doctests
+        if: env.TE_CLI_URL != ''
+        # Compile and execute every annotated block against a throwaway model
+        # and diff its Output() against the documented **Output** fence.
+        run: ./run doctest
+      - name: Doctest execution skipped
+        if: env.TE_CLI_URL == ''
+        run: echo "::notice title=Doctests not executed::Set the TE_CLI_DOWNLOAD_URL repository secret to a te-linux-x64.tar.gz download URL to compile and run the annotated C# blocks in CI. Only annotation validation ran."
+
   build_job:
     if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
     runs-on: windows-latest
@@ -56,9 +134,51 @@ jobs:
           path: _site
           retention-days: 7
 
+  link_check_job:
+    needs: build_job
+    runs-on: ubuntu-latest
+    name: Check links
+    # A full run fetches every unique external URL once (a few thousand), so
+    # it takes minutes; the cap keeps a stalled host from holding the deploy.
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Download build artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: site
+          path: _site
+      - name: Check links
+        # Internal (own-site) links are errors and fail this job: local files,
+        # anchors, and old root-style paths (those are resolved against the
+        # live site's redirects). External links are warnings only: they are
+        # listed in the report and never fail the run, since third-party
+        # outages and bot-blocking are outside our control.
+        shell: bash
+        run: python build_scripts/check_links.py validate stats 2>&1 | tee link-check-report.txt
+      - name: Publish link report
+        # The report (including the external URLs to verify by hand) goes to
+        # the job summary, so warnings are visible without opening the log.
+        if: always()
+        shell: bash
+        run: |
+          {
+            echo '## Link check report'
+            echo
+            echo 'Internal broken links fail this job. External URLs listed under "WARNINGS" are informational; verify them by hand.'
+            echo
+            echo '```'
+            head -c 900000 link-check-report.txt
+            echo '```'
+          } >> "$GITHUB_STEP_SUMMARY"
+
   deploy_job:
     if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
-    needs: build_job
+    needs: [build_job, lint_job, doctest_job, link_check_job]
     runs-on: ubuntu-latest
     name: Deploy to Azure
     steps:

README.md

@@ -83,6 +83,34 @@ swa start _site
 9. **Injects SEO tags** - Adds hreflang and canonical tags to HTML files
 10. **Generates SWA config** - Creates `staticwebapp.config.json` for Azure Static Web Apps routing
 
+# Continuous integration
+
+Every pull request against `main` (and every push to `main`) runs the workflow in [`.github/workflows/deploy.yml`](.github/workflows/deploy.yml).
+The deploy step waits for all checks, so a failing check blocks both the production deploy and the PR preview site.
+
+| Job | What it runs | Fails when |
+|-----|--------------|------------|
+| Lint build scripts | `./run scripts check`: ruff, ruff format, mypy `--strict` on the qualified Python sources (see `pyproject.toml`); shellcheck and shfmt on `run` and `build_scripts/run_scripts/*.sh` | any lint, type, or formatting finding |
+| Doctest C# code blocks | `./run doctest validate` always; `./run doctest` (compile, run, and compare `**Output**` blocks) when a te CLI is available, see below | malformed annotation; with te: compile error, runtime error, or output mismatch |
+| Build Documentation | `python build-docs.py --all --skip-gen` on Windows, uploads `_site` | DocFX build failure or English DocFX warnings |
+| Check links | `python build_scripts/check_links.py validate stats` against the built `_site` | any broken **internal** link: missing file, missing anchor, or an old root-style path that no longer redirects on the live site |
+| Deploy to Azure | Azure Static Web Apps upload | only after all of the above pass |
+
+**External links are warnings only.**
+The link check fetches every unique external URL once, but a bad status or a network error on a third-party site never fails the run:
+the site is outside our control, and outages, bot-blocking, and link rot should not stop docs from shipping.
+The full report, including the list of external URLs to verify by hand, is attached to the job summary of the "Check links" job.
+Fix or remove genuinely dead external links as part of normal maintenance.
+
+**Executing the doctests needs the te CLI.**
+The CLI download is gated behind sign-in, so the workflow cannot fetch it from a public URL.
+Set the repository secret `TE_CLI_DOWNLOAD_URL` to a direct download URL for `te-linux-x64.tar.gz` (for example a private blob with a SAS token);
+the doctest job then installs it and runs the full `./run doctest`.
+Without the secret, the job validates the annotations only and posts a notice.
+Use a te build aligned with the current Tabular Editor 3 release, otherwise the compare step reports false drift.
+
+Run the same checks locally before pushing: `./run scripts check`, `./run doctest`, and `./run build` followed by `./run check-links`.
+
 # Project Structure
 
 ```

build_scripts/README.md

@@ -42,6 +42,7 @@ Both need `te` on PATH.
 
 Current state only validates semantic bridge docs.
 The code block annotations can be used in any doc.
+CI runs `./run doctest validate` on every PR, and the full `./run doctest` when the `TE_CLI_DOWNLOAD_URL` repository secret provides a te CLI; see [Continuous integration](../README.md#continuous-integration).
 
 #### `te_script_runner.py` -- generic runner
 
@@ -181,6 +182,7 @@ Fragment/text failures keep their `#anchor` / `:~:text=` in that list (the bare
 the fragment is what broke); a wholly unreachable URL is listed bare.
 
 Exit codes: `1` if any internal (own-site) reference is broken, else `0`. External warnings never fail the run, so third-party link rot will not break CI.
+CI runs `validate stats` against the built site on every PR and attaches the report to the job summary; see [Continuous integration](../README.md#continuous-integration).
 
 Options to modify output:
 

build_scripts/run_scripts/README.md

@@ -143,7 +143,7 @@ and the authoring guidance below that section.
 `./run scripts` and its nested commands are the gateway to run scripts about run scripts.
 
 - Check the script-development tools are installed: `./run scripts setup`
-- Lint and verify formatting, without modifying anything (the check a PR should pass):
+- Lint and verify formatting, without modifying anything (CI runs this on every PR; see the [Continuous integration](../../README.md#continuous-integration) section of the project README):
   - `./run scripts check`: everything (all shell tooling, the qualified Python sources)
   - `./run scripts check file1.sh file2.py ...`: specific files, routed by kind
 - Apply the formatters (writes files): `./run scripts format [file ...]`

content/features/dax-query.md

@@ -63,7 +63,7 @@ The "Apply" option syncs the DAX expression for all measures, columns or tables
 
 The "Apply Selection" and "Apply Selection & Sync" will only apply the measures, columns or tables within the current selection of the query editor.
 
-Unlike the [DAX Script feature](xrefid:dax-scripts), only the expression property of a measure can be updated this way, as the DAX query syntax does not support specifying other properties, such as Description, Display Folder, etc.
+Unlike the [DAX Script feature](xref:dax-scripts), only the expression property of a measure can be updated this way, as the DAX query syntax does not support specifying other properties, such as Description, Display Folder, etc.
 
 The "Apply" option has also been added to the right-click context menu.
 
@@ -114,7 +114,7 @@ Customers
 
 ## Debugging DAX Query
 
-DAX queries are one of the two places where it is possible to run the [DAX Debugger](xrefid:dax-debugger), the other being the Pivot Grid.
+DAX queries are one of the two places where it is possible to run the [DAX Debugger](xref:dax-debugger), the other being the Pivot Grid.
 
 The DAX debugger unlocks the ability to understand how the DAX works inside a single cell. To start the debugger simply right click on the desired cell and choose 'Debug cell', which will start the debugger in the context of the chosen cell.
 

content/how-tos/Master-model-pattern.md

@@ -284,7 +284,7 @@ start /wait /d "c:\Program Files (x86)\Tabular Editor" TabularEditor.exe Model.b
 This assumes that you are executing the command line within the directory of your Model.bim file (or Database.json file if using the "Save to Folder"-functionality). The -S switch instructs Tabular Editor to apply the supplied script to the model, and the -D switch performs the deployment. The -O switch allows overwriting an existing database with the same name, and the -R switch indicates that we also want to overwrite roles of the target database.
 
 ## Master model processing
-If you have a dedicated processing server and large amounts of data overlap between the individual models, it may make sense for you to process the data into the master model first, before splitting it up. This way, you can avoid processing the same data several times, into individual models. **This assumes, however, that you are not processing any tables where the partition query has been changed between versions, as shown in [this section](/xref:Master-model-pattern#altering-partition-queries).** The recipe for this is outlined below:
+If you have a dedicated processing server and large amounts of data overlap between the individual models, it may make sense for you to process the data into the master model first, before splitting it up. This way, you can avoid processing the same data several times, into individual models. **This assumes, however, that you are not processing any tables where the partition query has been changed between versions, as shown in [this section](#altering-partition-queries).** The recipe for this is outlined below:
 
 1. (Optional - in case there were metadata changes) Deploy your master model to your processing server
 2. Perform the processing you need on your master model (do not process tables that have version-specific partition queries).

content/references/Roadmap2-h.md

@@ -81,7 +81,7 @@ The layout and structure of the Model.bim file, makes it horrible for purposes o
 
 For better release management workflows with Tabular Models, it would be interesting if Tabular Editor could save/load a Model.bim file as a folder structure with individual files for measures, calculated columns, etc. There should be command-line options available for exporting/importing Model.bim files from/to this format, and it should be possible to deploy directly from this format (in cases where you don't need the Model.bim file itself). These individual files should contain the same JSON as the Model.bim file, but without the "ModifiedTime" information, so that they can easily be used in a version control system, allowing multiple developers to work on the same model at once.
 
-**Update**: [Available in 2.2](/Advanced-features#folder-serialization).
+**Update**: [Available in 2.2](xref:folder-serialization).
 
 **Update**: As of 2.3, options exist to store Perspective and Translation metadata as annotations on the individual objects. This is useful for source control scenarios with multiple developers, to avoid having single files that gets lots of edits when developers change translations, perspective memberships, etc.
 
@@ -95,4 +95,4 @@ Today, it is already possible to connect Tabular Editor to a model hosted by Pow
 
 This is a standard feature in SSDT, which would be useful to have in Tabular Editor as well.
 
-**Update**: [Available in 2.2](/Advanced-features#import-export-translations).
+**Update**: [Available in 2.2](xref:import-export-translations).

content/references/preferences.md

@@ -830,4 +830,4 @@ List of addresses that should bypass the proxy (e.g., `localhost;*.company.local
 
 ## Next Steps
 
-For a user-friendly guide to the most commonly adjusted preferences, see the getting started guide (Personalizing TE3)[xrefid: personalizing-te3].
+For a user-friendly guide to the most commonly adjusted preferences, see the getting started guide [Personalizing TE3](xref:personalizing-te3).