One-line installers (install.sh, install.ps1), winget manifests, install test

.github/workflows/install-test.yml

@@ -0,0 +1,68 @@
+name: Install test
+
+# Runs every documented one-line install on real Windows, macOS and Linux
+# runners against the published release, then runs the installed command.
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+  push:
+    branches: [main]
+    paths: ["install.sh", "install.ps1", ".github/workflows/install-test.yml"]
+
+jobs:
+  script-unix:
+    name: install.sh on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04, ubuntu-latest, macos-14, macos-15-intel]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install with the one-liner (from this commit's install.sh)
+        run: sh install.sh
+      - name: Run it
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          token-visualizer --version
+
+  script-windows:
+    name: install.ps1 on Windows
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install with the one-liner (from this commit's install.ps1)
+        shell: powershell
+        run: Get-Content -Raw install.ps1 | Invoke-Expression
+      - name: Run it
+        shell: powershell
+        run: |
+          $env:Path = [Environment]::GetEnvironmentVariable('Path', 'User') + ';' + $env:Path
+          token-visualizer --version
+
+  homebrew:
+    name: Homebrew on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-14, macos-15-intel, ubuntu-latest]
+    steps:
+      - uses: Homebrew/actions/setup-homebrew@master
+      - run: brew install mattbusel/tap/token-visualizer
+      - run: brew test mattbusel/tap/token-visualizer
+      - run: token-visualizer --version
+
+  scoop:
+    name: Scoop on Windows
+    runs-on: windows-latest
+    steps:
+      - name: Install Scoop, add the bucket, install
+        shell: pwsh
+        run: |
+          iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
+          $env:Path = "$env:USERPROFILE\scoop\shims;$env:Path"
+          scoop bucket add mattbusel https://github.com/Mattbusel/scoop-bucket
+          scoop install mattbusel/token-visualizer
+          token-visualizer --version

install.ps1

@@ -0,0 +1,65 @@
+# Install Token Visualizer on Windows from the latest GitHub release.
+#
+#   irm https://raw.githubusercontent.com/Mattbusel/Token-Visualizer/main/install.ps1 | iex
+#
+# It downloads the Windows zip, checks it against the release's SHA256SUMS.txt,
+# puts token-visualizer.exe in %LOCALAPPDATA%\Programs\token-visualizer and adds that folder to
+# your user PATH. To remove it later: delete that folder and the PATH entry.
+# Set $env:INSTALL_VERSION = 'v1.2.3' first to install a specific release.
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue'
+[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
+
+$Repo   = 'Mattbusel/Token-Visualizer'
+$Prefix = 'token-visualizer'
+$Bin    = 'token-visualizer'
+$Cmd    = 'token-visualizer'
+$Dest   = Join-Path $env:LOCALAPPDATA "Programs\$Cmd"
+
+if ($env:INSTALL_VERSION) {
+    $tag = $env:INSTALL_VERSION
+} else {
+    try {
+        $tag = (Invoke-RestMethod -UseBasicParsing "https://api.github.com/repos/$Repo/releases/latest").tag_name
+    } catch {
+        throw "Could not look up the latest release of $Repo ($($_.Exception.Message)). Check your connection, or set `$env:INSTALL_VERSION = 'vX.Y.Z' and retry."
+    }
+}
+
+$name = "$Prefix-$tag-windows-x86_64"
+$base = "https://github.com/$Repo/releases/download/$tag"
+$tmp  = Join-Path ([IO.Path]::GetTempPath()) ("$Cmd-install-" + [Guid]::NewGuid().ToString('N'))
+New-Item -ItemType Directory -Path $tmp | Out-Null
+try {
+    Write-Host "Downloading $name.zip ..."
+    Invoke-WebRequest -UseBasicParsing "$base/$name.zip" -OutFile "$tmp\$name.zip"
+    Invoke-WebRequest -UseBasicParsing "$base/SHA256SUMS.txt" -OutFile "$tmp\SHA256SUMS.txt"
+
+    $line = Get-Content "$tmp\SHA256SUMS.txt" | Where-Object { $_ -match "\s\*?$([regex]::Escape("$name.zip"))$" } | Select-Object -First 1
+    if (-not $line) { throw "$name.zip is not listed in SHA256SUMS.txt; not installing." }
+    $expected = ($line -split '\s+')[0].ToLower()
+    $actual = (Get-FileHash -Algorithm SHA256 "$tmp\$name.zip").Hash.ToLower()
+    if ($expected -ne $actual) { throw "Checksum mismatch for $name.zip (expected $expected, got $actual); not installing." }
+    Write-Host "Checksum OK."
+
+    Expand-Archive -Path "$tmp\$name.zip" -DestinationPath $tmp -Force
+    $exe = Join-Path $tmp "$name\$Bin.exe"
+    if (-not (Test-Path $exe)) { throw "The archive did not contain $Bin.exe." }
+
+    New-Item -ItemType Directory -Force -Path $Dest | Out-Null
+    Copy-Item $exe (Join-Path $Dest "$Cmd.exe") -Force
+} finally {
+    Remove-Item -Recurse -Force $tmp -ErrorAction SilentlyContinue
+}
+
+$userPath = [Environment]::GetEnvironmentVariable('Path', 'User')
+if (-not $userPath) { $userPath = '' }
+if (($userPath -split ';') -notcontains $Dest) {
+    [Environment]::SetEnvironmentVariable('Path', (($userPath.TrimEnd(';') + ";$Dest").TrimStart(';')), 'User')
+    Write-Host "Added $Dest to your user PATH."
+}
+if (($env:Path -split ';') -notcontains $Dest) { $env:Path = "$env:Path;$Dest" }
+
+Write-Host "Installed $Cmd $tag to $Dest\$Cmd.exe"
+Write-Host "Try it: token-visualizer --help   (open a new terminal if the command is not found)"

install.sh

@@ -0,0 +1,101 @@
+#!/bin/sh
+# Install Token Visualizer from the latest GitHub release.
+#
+#   curl -fsSL https://raw.githubusercontent.com/Mattbusel/Token-Visualizer/main/install.sh | sh
+#
+# Options (environment variables):
+#   INSTALL_DIR=/some/dir   where to put the binary (default: ~/.local/bin)
+#   INSTALL_VERSION=v1.2.3  install a specific release instead of the latest
+#
+# It downloads the archive for your OS and CPU, checks it against the release's
+# SHA256SUMS.txt, and copies one file (token-visualizer) into INSTALL_DIR. Nothing else.
+set -eu
+
+REPO="Mattbusel/Token-Visualizer"
+PREFIX="token-visualizer"      # archive name prefix
+BIN="token-visualizer"            # file name inside the archive
+CMD="token-visualizer"            # command name you will type
+INSTALL_DIR="${INSTALL_DIR:-$HOME/.local/bin}"
+
+say() { printf '%s\n' "$*"; }
+die() { printf 'error: %s\n' "$*" >&2; exit 1; }
+need() { command -v "$1" >/dev/null 2>&1 || die "this installer needs '$1'; install it and try again"; }
+
+need curl
+need tar
+need uname
+
+os="$(uname -s)"
+arch="$(uname -m)"
+ext="tar.gz"
+exe=""
+case "$os" in
+  Linux)
+    case "$arch" in
+      x86_64|amd64) target="linux-x86_64" ;;
+      *) die "no prebuilt Linux binary for $arch yet; use: pipx install git+https://github.com/$REPO" ;;
+    esac ;;
+  Darwin)
+    case "$arch" in
+      arm64|aarch64) target="macos-arm64" ;;
+      x86_64) target="macos-x86_64" ;;
+      *) die "unsupported macOS CPU: $arch" ;;
+    esac ;;
+  MINGW*|MSYS*|CYGWIN*)
+    target="windows-x86_64"; ext="zip"; exe=".exe"; need unzip ;;
+  *) die "unsupported OS: $os (on Windows PowerShell use install.ps1 instead)" ;;
+esac
+
+if [ -n "${INSTALL_VERSION:-}" ]; then
+  tag="$INSTALL_VERSION"
+else
+  # The /releases/latest page redirects to /releases/tag/<tag>; no API token needed.
+  url="$(curl -fsSLI -o /dev/null -w '%{url_effective}' "https://github.com/$REPO/releases/latest")" \
+    || die "could not reach github.com; check your connection"
+  tag="${url##*/}"
+  case "$tag" in v*) ;; *) die "could not find the latest release of $REPO" ;; esac
+fi
+
+name="$PREFIX-$tag-$target"
+base="https://github.com/$REPO/releases/download/$tag"
+tmp="$(mktemp -d 2>/dev/null || mktemp -d -t "$CMD")"
+trap 'rm -rf "$tmp"' EXIT INT TERM
+
+say "Downloading $name.$ext ..."
+curl -fsSL -o "$tmp/$name.$ext" "$base/$name.$ext" || die "download failed: $base/$name.$ext"
+curl -fsSL -o "$tmp/SHA256SUMS.txt" "$base/SHA256SUMS.txt" || die "could not download SHA256SUMS.txt"
+
+expected="$(grep " \*\{0,1\}$name.$ext\$" "$tmp/SHA256SUMS.txt" | cut -d' ' -f1)"
+[ -n "$expected" ] || die "$name.$ext is not listed in SHA256SUMS.txt"
+if command -v sha256sum >/dev/null 2>&1; then
+  actual="$(sha256sum "$tmp/$name.$ext" | cut -d' ' -f1)"
+elif command -v shasum >/dev/null 2>&1; then
+  actual="$(shasum -a 256 "$tmp/$name.$ext" | cut -d' ' -f1)"
+else
+  die "need sha256sum or shasum to verify the download"
+fi
+[ "$expected" = "$actual" ] || die "checksum mismatch for $name.$ext (expected $expected, got $actual); not installing"
+say "Checksum OK."
+
+if [ "$ext" = "zip" ]; then
+  unzip -q "$tmp/$name.$ext" -d "$tmp"
+else
+  tar xzf "$tmp/$name.$ext" -C "$tmp"
+fi
+[ -f "$tmp/$name/$BIN$exe" ] || die "archive did not contain $BIN$exe"
+
+mkdir -p "$INSTALL_DIR"
+cp "$tmp/$name/$BIN$exe" "$INSTALL_DIR/$CMD$exe"
+chmod +x "$INSTALL_DIR/$CMD$exe"
+if [ "$os" = "Darwin" ] && command -v xattr >/dev/null 2>&1; then
+  xattr -d com.apple.quarantine "$INSTALL_DIR/$CMD$exe" 2>/dev/null || true
+fi
+
+say "Installed $CMD $tag to $INSTALL_DIR/$CMD$exe"
+case ":$PATH:" in
+  *":$INSTALL_DIR:"*) say "Try it: token-visualizer --help" ;;
+  *) say ""
+     say "$INSTALL_DIR is not on your PATH yet. Add this line to ~/.bashrc or ~/.zshrc:"
+     say "  export PATH=\"$INSTALL_DIR:\$PATH\""
+     say "Then open a new terminal and run: token-visualizer --help" ;;
+esac

packaging/winget/Mattbusel.TokenVisualizer.installer.yaml

@@ -0,0 +1,15 @@
+# yaml-language-server: $schema=https://aka.ms/winget-manifest.installer.1.6.0.schema.json
+
+PackageIdentifier: Mattbusel.TokenVisualizer
+PackageVersion: 0.3.0
+InstallerType: zip
+NestedInstallerType: portable
+NestedInstallerFiles:
+- RelativeFilePath: token-visualizer-v0.3.0-windows-x86_64\token-visualizer.exe
+  PortableCommandAlias: token-visualizer
+Installers:
+- Architecture: x64
+  InstallerUrl: https://github.com/Mattbusel/Token-Visualizer/releases/download/v0.3.0/token-visualizer-v0.3.0-windows-x86_64.zip
+  InstallerSha256: 45D0A5AF8B7DBECC7B22098B1946BF9E5EFB7F2963B5527EF99AF00B0CD9668A
+ManifestType: installer
+ManifestVersion: 1.6.0

packaging/winget/Mattbusel.TokenVisualizer.locale.en-US.yaml

@@ -0,0 +1,23 @@
+# yaml-language-server: $schema=https://aka.ms/winget-manifest.defaultLocale.1.6.0.schema.json
+
+PackageIdentifier: Mattbusel.TokenVisualizer
+PackageVersion: 0.3.0
+PackageLocale: en-US
+Publisher: Mattbusel
+PublisherUrl: https://github.com/Mattbusel
+PublisherSupportUrl: https://github.com/Mattbusel/Token-Visualizer/issues
+PackageName: Token Visualizer
+PackageUrl: https://github.com/Mattbusel/Token-Visualizer
+License: MIT
+LicenseUrl: https://github.com/Mattbusel/Token-Visualizer/blob/main/LICENSE
+ShortDescription: "See how a tokenizer splits your prompt and what to cut, with measured savings"
+Description: "Token Visualizer shows how an LLM tokenizer splits your prompt into tokens, which lines cost the most, and which wordy phrases to cut, then re-tokenizes to measure the real savings. GPT tokenizers are built in and work offline."
+Tags:
+- tokens
+- tiktoken
+- llm
+- prompt
+- cli
+ReleaseNotesUrl: https://github.com/Mattbusel/Token-Visualizer/releases/tag/v0.3.0
+ManifestType: defaultLocale
+ManifestVersion: 1.6.0

packaging/winget/Mattbusel.TokenVisualizer.yaml

@@ -0,0 +1,7 @@
+# yaml-language-server: $schema=https://aka.ms/winget-manifest.version.1.6.0.schema.json
+
+PackageIdentifier: Mattbusel.TokenVisualizer
+PackageVersion: 0.3.0
+DefaultLocale: en-US
+ManifestType: version
+ManifestVersion: 1.6.0