diff --git a/.devcontainer/.gitignore b/.devcontainer/.gitignore new file mode 100644 index 00000000000..8ab6b2d9a06 --- /dev/null +++ b/.devcontainer/.gitignore @@ -0,0 +1,2 @@ +# User-specific devcontainer directory with merged personal preferences +user/ diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000000..f4163e57045 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,39 @@ +# Dev container config + +This folder contains the configuration to work in a [dev container](https://containers.dev/); +an isolated, reproducible development environment that runs inside a container, typically Docker. + +The configuration skeleton is generated by the Guardian's [devenv](https://github.com/guardian/devenv) config tool. + +## Contents + +### `README.md` + +Edit this file as you see fit. It is created on the first run of `devenv init` but it won't be overwritten +by subsequent runs unless you delete it entirely. + +### `devenv.yaml` + +The source-of-truth configuration for this project's dev container. Edit this file to add/remove devenv modules, +or any other config settings. Then run: +```bash +devenv generate +``` +to regenerate the shared `devcontainer.json` file. + +### `shared/devcontainer.json` + +The main devcontainer configuration, committed to the repository and shared by all developers. +It is generated from `devenv.yaml`. + +> [!IMPORTANT] +> Do not edit this file directly! Instead, edit `devenv.yaml` and then run `devenv generate`. + +### `user/devcontainer.json` + +A personal copy of the devcontainer config, **git-ignored** (see `.gitignore`). +This file is intended for your own overrides: +extra VS Code extensions or Intellij plugins, personal settings or tweaks that shouldn't affect other contributors. +Modify this file freely without worrying about polluting shared config. +> [!IMPORTANT] +> But be aware that `devenv generate` will overwrite it! diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml new file mode 100644 index 00000000000..44306cce15c --- /dev/null +++ b/.devcontainer/devenv.yaml @@ -0,0 +1,47 @@ +# Devenv project configuration +# Edit this file to configure your project's devcontainer +# and then run `devenv generate` to create the devcontainer.json files + +name: "grid" + +# Modules: Built-in functionality +modules: + - mise + - docker-in-docker + - scala + - github-copilot + +features: + ghcr.io/devcontainers/features/aws-cli:1.1.4: {} + +# Optional: Ports to forward +forwardPorts: + - 9001 + - 9002 + - 9003 + - 9004 + - 9005 + - 9006 + - 9007 + - 9008 + - 9009 + - 9010 + - 9011 + - 9012 + - 4566 + - 6080 + +# Optional: Mount directories from host to container +# mounts: +# - "source=${localEnv:HOME}/.gu/example,target=/home/vscode/.gu/example,readonly,type=bind,consistency=cached" + +# Optional: IDE plugins - shared project plugins like language support +# plugins: +# vscode: [] +# intellij: [] + +postCreateCommand: + - cmd: "docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev ." + workingDirectory: "." + - cmd: 'PATH="$HOME/.local/share/mise/shims:$PATH" npm ci' + workingDirectory: "./e2e-tests" diff --git a/.devcontainer/shared/devcontainer-lock.json b/.devcontainer/shared/devcontainer-lock.json new file mode 100644 index 00000000000..20a580bcf17 --- /dev/null +++ b/.devcontainer/shared/devcontainer-lock.json @@ -0,0 +1,14 @@ +{ + "features": { + "ghcr.io/devcontainers/features/aws-cli:1.1.4": { + "version": "1.1.4", + "resolved": "ghcr.io/devcontainers/features/aws-cli@sha256:1f93c8315b7a6d76982ebb2269f8b0d50413fc0f965c032edf4aee0caceb73ef", + "integrity": "sha256:1f93c8315b7a6d76982ebb2269f8b0d50413fc0f965c032edf4aee0caceb73ef" + }, + "ghcr.io/devcontainers/features/docker-in-docker:3": { + "version": "3.1.0", + "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:62e04ec83d944da4fd2830f68a677d1f5466c0654aea1e4baa30232d16029ac1", + "integrity": "sha256:62e04ec83d944da4fd2830f68a677d1f5466c0654aea1e4baa30232d16029ac1" + } + } +} diff --git a/.devcontainer/shared/devcontainer.json b/.devcontainer/shared/devcontainer.json new file mode 100644 index 00000000000..1bc566af490 --- /dev/null +++ b/.devcontainer/shared/devcontainer.json @@ -0,0 +1,72 @@ +{ + "name" : "grid", + "customizations" : { + "vscode" : { + "extensions" : [ + "hverlin.mise-vscode", + "scala-lang.scala", + "scalameta.metals", + "GitHub.copilot" + ] + }, + "jetbrains" : { + "plugins" : [ + "com.github.l34130.mise", + "org.intellij.scala", + "com.github.copilot" + ] + } + }, + "forwardPorts" : [ + 9001, + 9002, + 9003, + 9004, + 9005, + 9006, + 9007, + 9008, + 9009, + 9010, + 9011, + 9012, + 4566, + 6080 + ], + "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", + "remoteUser" : "vscode", + "features" : { + "ghcr.io/devcontainers/features/docker-in-docker:3" : { + "version" : "latest", + "moby" : false, + "dockerDashComposeVersion" : "v2" + }, + "ghcr.io/devcontainers/features/aws-cli:1.1.4" : { + + } + }, + "mounts" : [ + { + "source" : "devenv-coursier-cache-volume", + "target" : "/mnt/coursier-cache", + "type" : "volume" + }, + { + "source" : "devenv-ivy-cache-volume", + "target" : "/mnt/ivy-cache", + "type" : "volume" + } + ], + "containerEnv" : { + "DEVENV_IVY_CACHE_MOUNT_DIR" : "/mnt/ivy-cache", + "DEVENV_COURSIER_CACHE_MOUNT_DIR" : "/mnt/coursier-cache" + }, + "capAdd" : [ + "SYS_ADMIN" + ], + "securityOpt" : [ + "seccomp=unconfined" + ], + "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd ./e2e-tests && PATH=\"$HOME/.local/share/mise/shims:$PATH\" npm ci && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" +} \ No newline at end of file diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json new file mode 100644 index 00000000000..3c366bf3cdc --- /dev/null +++ b/.devcontainer/user/devcontainer.json @@ -0,0 +1,77 @@ +{ + "name" : "grid", + "customizations" : { + "vscode" : { + "extensions" : [ + "hverlin.mise-vscode", + "scala-lang.scala", + "scalameta.metals", + "GitHub.copilot" + ] + }, + "jetbrains" : { + "plugins" : [ + "com.github.l34130.mise", + "org.intellij.scala", + "com.github.copilot" + ] + } + }, + "forwardPorts" : [ + 9001, + 9002, + 9003, + 9004, + 9005, + 9006, + 9007, + 9008, + 9009, + 9010, + 9011, + 9012, + 4566, + 6080 + ], + "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", + "remoteUser" : "vscode", + "features" : { + "ghcr.io/devcontainers/features/docker-in-docker:3" : { + "version" : "latest", + "moby" : false, + "dockerDashComposeVersion" : "v2" + }, + "ghcr.io/devcontainers/features/aws-cli:1.1.4" : { + + } + }, + "mounts" : [ + { + "source" : "devenv-coursier-cache-volume", + "target" : "/mnt/coursier-cache", + "type" : "volume" + }, + { + "source" : "devenv-ivy-cache-volume", + "target" : "/mnt/ivy-cache", + "type" : "volume" + } + ], + "containerEnv" : { + "DEVENV_IVY_CACHE_MOUNT_DIR" : "/mnt/ivy-cache", + "DEVENV_COURSIER_CACHE_MOUNT_DIR" : "/mnt/coursier-cache" + }, + "capAdd" : [ + "SYS_ADMIN" + ], + "securityOpt" : [ + "seccomp=unconfined" + ], + "runArgs" : [ + "--memory=16g", + "--cpus=8", + "--shm-size=512m" + ], + "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd ./e2e-tests && PATH=\"$HOME/.local/share/mise/shims:$PATH\" npm ci && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" +} \ No newline at end of file diff --git a/.github/workflows/check-labels.yaml b/.github/workflows/check-labels.yaml index 75a4a1445bd..bc6ed7677bf 100644 --- a/.github/workflows/check-labels.yaml +++ b/.github/workflows/check-labels.yaml @@ -6,4 +6,4 @@ on: types: [opened, labeled, unlabeled, synchronize, reopened, edited] jobs: require-label: - uses: guardian/.github/.github/workflows/require-label.yaml@4cb5024736632ffcc564b7f4b772c38b8e5ce739 # v2.0.0 + uses: guardian/.github/.github/workflows/require-label.yaml@3522b599ee9e7d77089f505fed04f084c8b5fac2 # v3.1.2 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ae66882420..88821815689 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,16 @@ jobs: pull-requests: write services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.29 - # Wait for elasticsearch to report healthy before continuing. + image: docker.elastic.co/elasticsearch/elasticsearch:8.18.3 + # These options should mimic those set in ElasticSearchDockerBase.scala, because this is + # the CI version of the TestContainers approach we use for running tests locally. + # `xpack.security.enabled=false` is needed since v8 for tests to use plain + # unauthenticated HTTP, because in v7 => v8 they changed the default to `true`: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.18/release-notes-8.0.0.html#:~:text=Set%20xpack.security.enabled%20to%20true%20for%20all%20licenses + # + # `--health-` settings make it wait for es to report healthy before continuing. # see https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml#L28 - options: -e "discovery.type=single-node" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 + options: -e "discovery.type=single-node" -e "xpack.security.enabled=false" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 ports: - 9200:9200 localstack: @@ -36,9 +42,9 @@ jobs: --health-timeout 5s --health-retries 10 steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@e8d4307400f9427dba7cb98e488d6ab85f1cec5f # v7.0.0 - name: Setup Node - uses: actions/setup-node@v6 + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version-file: ".nvmrc" cache: npm diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 00000000000..3ff622adaae --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,63 @@ +name: Playwright Tests +on: + push: + branches: [ main ] + pull_request: + workflow_dispatch: +permissions: + contents: read +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + defaults: + run: + working-directory: e2e-tests + steps: + - uses: actions/checkout@e8d4307400f9427dba7cb98e488d6ab85f1cec5f # v7.0.0 + - uses: actions/setup-node@v6 + with: + node-version: lts/* + # Cache the npm download cache, keyed on the e2e-tests lockfile. + cache: npm + cache-dependency-path: e2e-tests/package-lock.json + - name: Install dependencies + run: npm ci + # Cache the downloaded Playwright browser binaries. The cache key is tied to + # the resolved Playwright version in the lockfile, so a version bump busts it. + - name: Cache Playwright browsers + id: playwright-cache + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('e2e-tests/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-playwright- + - name: Install Playwright Browsers + # On a cache hit the browsers are already present, so only install the OS + # dependencies; otherwise download the browsers too. + run: | + if [ "${{ steps.playwright-cache.outputs.cache-hit }}" = "true" ]; then + npx playwright install-deps chromium + else + npx playwright install --with-deps chromium + fi + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build CI image + uses: docker/build-push-action@v6 + with: + context: . + file: e2e-tests/images/Dockerfile + target: ci + tags: grid-e2e-ci + push: false + load: true + - name: Run Playwright tests + run: npm test + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report + path: e2e-tests/playwright-report/ + retention-days: 30 diff --git a/.github/workflows/push-pr.yml b/.github/workflows/push-pr.yml index 0b204133620..8b68dc8ee39 100644 --- a/.github/workflows/push-pr.yml +++ b/.github/workflows/push-pr.yml @@ -18,7 +18,7 @@ jobs: contents: write actions: write steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@e8d4307400f9427dba7cb98e488d6ab85f1cec5f # v7.0.0 - name: push-pr env: PR: ${{ inputs.prNumber }} diff --git a/.github/workflows/sbt-dependency-graph.yaml b/.github/workflows/sbt-dependency-graph.yaml index 68729d8a7be..ed60dd47ee8 100644 --- a/.github/workflows/sbt-dependency-graph.yaml +++ b/.github/workflows/sbt-dependency-graph.yaml @@ -10,10 +10,10 @@ jobs: steps: - name: Checkout branch id: checkout - uses: actions/checkout@0c366fd6a839edf440554fa01a7085ccba70ac98 # v6.0.1 + uses: actions/checkout@e8d4307400f9427dba7cb98e488d6ab85f1cec5f # v7.0.0 - name: Install Java id: java - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + uses: actions/setup-java@0f481fcb613427c0f801b606911222b5b6f3083a # v5.5.0 with: distribution: corretto java-version: 17 diff --git a/.gitignore b/.gitignore index c8d01fbf7a9..5b6dcd2b913 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,11 @@ image-embedder.zip # Integration test data (cached S3 downloads and outputs) image-embedder-lambda/__tests__/embedder/integration/test-data/ image-embedder-lambda/__tests__/embedder/test-data + +# Playwright +/e2e-tests/.features-gen/ +/e2e-tests/test-results/ +/e2e-tests/playwright-report/ +/e2e-tests/blob-report/ +/e2e-tests/playwright/.cache/ +/e2e-tests/playwright/.auth/ diff --git a/.tool-versions b/.tool-versions index b60a0e5cfc6..b1348086897 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,4 @@ java corretto-11.0.24.8.1 +nodejs 22.12.0 +github:guardian/devenv 20260721-123412 +sbt 1 diff --git a/build.sbt b/build.sbt index 571b0bc9e53..dad2f1577af 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ ThisBuild / packageOptions += FixedTimestamp(Package.keepTimestamps) ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-java8-compat" % VersionScheme.Always -lazy val jacksonVersion = "2.21.1" +lazy val jacksonVersion = "2.21.4" lazy val jacksonAnnotationsVersion = "2.21" lazy val jacksonOverrides = Seq( "com.fasterxml.jackson.core" % "jackson-core", @@ -77,7 +77,7 @@ Global / concurrentRestrictions := Seq( ) val awsSdkVersion = "1.12.797" -val awsSdkV2Version = "2.44.13" +val awsSdkV2Version = "2.54.13" val elastic4sVersion = "8.19.1" val awsKclVersion = "3.4.3" val okHttpVersion = "3.12.1" @@ -91,24 +91,21 @@ val maybeBBCLib: Option[sbt.ProjectReference] = if(bbcBuildProcess) Some(bbcProj lazy val commonLib = project("common-lib").settings( libraryDependencies ++= Seq( - "com.gu" %% "editorial-permissions-client" % "4.0.0", + "com.gu" %% "editorial-permissions-client" % "7.0.0", "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0", - "com.amazonaws" % "aws-java-sdk-iam" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-sts" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion, + "software.amazon.awssdk" % "iam" % awsSdkV2Version, "software.amazon.awssdk" % "s3" % awsSdkV2Version, + "software.amazon.awssdk" % "sns" % awsSdkV2Version, + "software.amazon.awssdk" % "sqs" % awsSdkV2Version, + "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, + "software.amazon.awssdk" % "imds" % awsSdkV2Version, "nl.gn0s1s" %% "elastic4s-core" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-client-esjava" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-domain" % elastic4sVersion, "com.gu" %% "thrift-serializer" % "5.0.2", "org.scalaz" %% "scalaz-core" % "7.3.8", "org.im4java" % "im4java" % "1.4.0", - "com.gu" % "kinesis-logback-appender" % "1.4.4", + "com.gu" % "kinesis-logback-appender" % "2.1.3", "net.logstash.logback" % "logstash-logback-encoder" % "5.0", logback, // play-logback; needed when running the scripts "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2", @@ -127,7 +124,7 @@ lazy val commonLib = project("common-lib").settings( "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, ws, - "org.testcontainers" % "elasticsearch" % "1.21.4" % Test + "org.testcontainers" % "elasticsearch" % "1.21.4" % Test, ), dependencyOverrides += "ch.qos.logback" % "logback-classic" % "1.2.13" % Test ) @@ -194,7 +191,7 @@ lazy val thrall = playProject("thrall", 9002) lazy val usage = playProject("usage", 9009).settings( libraryDependencies ++= Seq( "com.gu" %% "content-api-client-default" % "32.0.0", - "com.gu" %% "content-api-client-aws" % "0.7.6", + "com.gu" %% "content-api-client-aws" % "1.0.1", "io.reactivex" %% "rxscala" % "0.27.0", "software.amazon.kinesis" % "amazon-kinesis-client" % awsKclVersion, // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl diff --git a/cdk/package-lock.json b/cdk/package-lock.json index f798c6935f8..f4a94597b83 100644 --- a/cdk/package-lock.json +++ b/cdk/package-lock.json @@ -7936,9 +7936,9 @@ "license": "MIT" }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "dev": true, "license": "MIT", "dependencies": { diff --git a/collections/app/store/CollectionsStore.scala b/collections/app/store/CollectionsStore.scala index fe7416c27c3..5bf8c1f1a61 100644 --- a/collections/app/store/CollectionsStore.scala +++ b/collections/app/store/CollectionsStore.scala @@ -18,7 +18,7 @@ case class Record(id: String, collection: Collection) class CollectionsStore(config: CollectionsConfig) extends DynamoHelpers { override val tableName: FieldName = config.collectionsTable - lazy val client: DynamoDbAsyncClient = config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build() + lazy val client: DynamoDbAsyncClient = config.withAWSCredentials(DynamoDbAsyncClient.builder()).build() import org.scanamo.generic.semiauto._ implicit val dateTimeFormat: Typeclass[DateTime] = DynamoFormat.coercedXmap[DateTime, String, IllegalArgumentException](DateTime.parse, _.toString) diff --git a/collections/app/store/ImageCollectionsStore.scala b/collections/app/store/ImageCollectionsStore.scala index e0dcb611c12..1e837e36cb2 100644 --- a/collections/app/store/ImageCollectionsStore.scala +++ b/collections/app/store/ImageCollectionsStore.scala @@ -19,7 +19,7 @@ case class ImageRecord(id: String, collections: List[Collection]) class ImageCollectionsStore(config: CollectionsConfig) extends DynamoHelpers { override val tableName = config.imageCollectionsTable - lazy val client: DynamoDbAsyncClient = config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build() + lazy val client: DynamoDbAsyncClient = config.withAWSCredentials(DynamoDbAsyncClient.builder()).build() import org.scanamo.generic.semiauto._ implicit val dateTimeFormat: Typeclass[DateTime] = diff --git a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala index c9058ea2850..bf017e31473 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala @@ -8,7 +8,7 @@ import com.gu.mediaservice.model.leases.LeasesByMedia import com.gu.mediaservice.model.usage.Usage import com.typesafe.scalalogging.LazyLogging import play.api.http.HeaderNames -import play.api.libs.json.{JsArray, JsObject, JsValue, Json, Reads} +import play.api.libs.json.{JsArray, JsObject, JsTrue, JsValue, Json, Reads} import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{ExecutionContext, Future} @@ -38,8 +38,8 @@ object ClientResponse { case class ClientErrorMessages(errorMessage: String, downstreamErrorMessage: String) object GridClient extends LazyLogging { - def apply(services: Services)(implicit wsClient: WSClient): GridClient = - new GridClient(services) + def apply(services: Services, originUri: String)(implicit wsClient: WSClient): GridClient = + new GridClient(services, originUri) sealed trait Response { def status: Int @@ -96,7 +96,7 @@ object GridClient extends LazyLogging { } -class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLogging { +class GridClient(services: Services, originDomain: String)(implicit wsClient: WSClient) extends LazyLogging { /* * `requestTimeout` will set the max duration of the request before timing out. You may also want to increase the @@ -260,6 +260,11 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo authorisedRequest.post(Json.obj("data" -> data)).map { response => validateResponse(response, url)} } + def putArchived(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext) = { + val url = new URL(s"${services.metadataBaseUri}/metadata/$mediaId/archived") + val request = authFn(wsClient.url(url.toString)) + request.put(Json.obj("data" -> JsTrue)).map { response => validateResponse(response, url)} + } } class DownstreamApiInBadStateException(message: String, downstreamMessage: String) extends IllegalStateException(message) { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/AttributeValueConverterProvider.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/AttributeValueConverterProvider.scala new file mode 100644 index 00000000000..035cbc20f9b --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/AttributeValueConverterProvider.scala @@ -0,0 +1,18 @@ +package com.gu.mediaservice.lib + +import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverter, AttributeConverterProvider, AttributeValueType, EnhancedType} +import software.amazon.awssdk.services.dynamodb.model.AttributeValue + +object AttributeValueConverterProvider extends AttributeConverterProvider { + private val converter = new AttributeConverter[AttributeValue] { + override def transformFrom(input: AttributeValue): AttributeValue = input + override def transformTo(input: AttributeValue): AttributeValue = input + override def attributeValueType(): AttributeValueType = AttributeValueType.M + override def `type`(): EnhancedType[AttributeValue] = EnhancedType.of(classOf[AttributeValue]) + } + + override def converterFor[T](enhancedType: EnhancedType[T]): AttributeConverter[T] = { + if (enhancedType.rawClass() == classOf[AttributeValue]) converter.asInstanceOf[AttributeConverter[T]] + else null + } +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala index 0aeb698cccb..9c6aa257b5c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala @@ -5,6 +5,7 @@ import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import org.joda.time.DateTime +import software.amazon.awssdk.services.s3.model.{GetObjectRequest, ListObjectsV2Request} import java.util.concurrent.atomic.AtomicReference import java.io.InputStream @@ -25,15 +26,17 @@ abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonCon protected def getS3Object(key: String): Option[String] = s3.getObjectAsString(bucket, key) protected def getLatestS3Stream: Option[InputStream] = { - val objects = s3.client - .listObjects(bucket).getObjectSummaries.asScala - .filterNot(_.getKey == "AMAZON_SES_SETUP_NOTIFICATION") + val objects = s3.client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) + .contents().asScala.toList + .filterNot(_.key() == "AMAZON_SES_SETUP_NOTIFICATION") if (objects.nonEmpty) { - val obj = objects.maxBy(_.getLastModified) - logger.info(s"Latest key ${obj.getKey} in bucket $bucket") + val obj = objects.maxBy(_.lastModified()) + logger.info(s"Latest key ${obj.key} in bucket $bucket") - val stream = s3.client.getObject(bucket, obj.getKey).getObjectContent + val stream = s3.client.getObject( + GetObjectRequest.builder().key(obj.key()).bucket(bucket).build() + ) Some(stream) } else { logger.error(s"Bucket $bucket is empty") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 0fdd57bb676..9483290c4fe 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -1,6 +1,5 @@ package com.gu.mediaservice.lib -import com.amazonaws.services.s3.model.{DeleteObjectsRequest, MultiObjectDeleteException} import java.io.File import com.gu.mediaservice.lib.config.CommonConfig @@ -8,6 +7,8 @@ import com.gu.mediaservice.lib.aws.S3Object import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.{MimeType, Png} import org.joda.time.DateTime +import software.amazon.awssdk.core.exception.SdkClientException +import software.amazon.awssdk.services.s3.model.{Delete, DeleteObjectsRequest, HeadObjectRequest, NoSuchKeyException, ObjectIdentifier} import scala.concurrent.Future import scala.jdk.CollectionConverters._ @@ -45,24 +46,24 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config storeImage(imageBucket, optimisedPngKeyFromId(storableImage.id), storableImage.file, Some(storableImage.mimeType), overwrite = true) + private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match { case Nil => Future.successful(Map.empty) case _ => Future { - try { - client.deleteObjects( - new DeleteObjectsRequest(bucket).withKeys(keys: _*) - ) - keys.map { key => - key -> true - }.toMap - } catch { - case partialFailure: MultiObjectDeleteException => - logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}") - val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet - keys.map { key => - key -> !errorKeys.contains(key) - }.toMap - } + val objects = keys.map { key => + ObjectIdentifier.builder() + .key(key) + .build() + }.asJava + val response = client.deleteObjects( + DeleteObjectsRequest.builder().bucket(bucket) + .delete(Delete.builder().objects(objects).build()) + .build() + ) + val errorKeys = response.errors().asScala.toList.map(_.key()) + keys.map { key => + key -> !errorKeys.contains(key) + }.toMap } } @@ -73,8 +74,10 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config def deletePNG(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id)) def deletePNGs(ids: Set[String]) = bulkDelete(imageBucket, ids.map(optimisedPngKeyFromId).toList) - def doesOriginalExist(id: String): Boolean = - client.doesObjectExist(imageBucket, fileKeyFromId(id)) + def doesOriginalExist(id: String): Boolean = { + this.doesObjectExist(imageBucket, fileKeyFromId(id)) + } + } sealed trait ImageWrapper { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 7e164f876bb..e477a3762d2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -5,6 +5,7 @@ import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.MimeType import org.slf4j.LoggerFactory +import software.amazon.awssdk.services.s3.model.{DeleteObjectRequest, HeadObjectRequest, ListObjectsV2Request} import java.io.File import scala.jdk.CollectionConverters._ @@ -27,20 +28,25 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage eventualObject } - def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - client.deleteObject(bucket, id) - logger.info(logMarker, s"Deleted image $id from bucket $bucket") + def deleteImage(bucket: String, key: String)(implicit logMarker: LogMarker) = Future { + logger.info(logMarker, s"Deleted image $key from bucket $bucket") + client.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(key).build()) } def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val objectVersion = client.getObjectMetadata(bucket, id).getVersionId - client.deleteVersion(bucket, id, objectVersion) + val objectVersion = client.headObject(HeadObjectRequest.builder().bucket(bucket).key(id).build()).versionId() + client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(id).versionId(objectVersion).build()) logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)") } def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val files = client.listObjects(bucket, id).getObjectSummaries.asScala - files.foreach(file => client.deleteObject(bucket, file.getKey)) + val files = client.listObjectsV2( + ListObjectsV2Request.builder().bucket(bucket).prefix(id).build() + ).contents().asScala.toList + files.foreach(file => client.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(file.key()).build() + )) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala new file mode 100644 index 00000000000..81ff0bedd7d --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -0,0 +1,80 @@ +package com.gu.mediaservice.lib + +object VectorUtils { + def dotProduct(vectorOne: List[Double], vectorTwo: List[Double]): Double = { + require(vectorOne.length == vectorTwo.length, "Vectors must be the same dimensionality") + vectorOne.zip(vectorTwo).map(component => component._1 * component._2).sum + } + + def magnitude(vector: List[Double]): Double = { + math.sqrt(vector.map(math.pow(_, 2)).sum) + } + + def cosineSimilarity(vectorOne: List[Double], vectorTwo: List[Double]): Option[Double] = { + // Cosine similarity is undefined when the vectors differ in dimensionality + // (the dot product is meaningless) or when either has zero magnitude (the + // angle to the origin is meaningless), so we return None in those cases and + // let the caller choose a domain-appropriate fallback. + if (vectorOne.length != vectorTwo.length) None + else { + val magnitudeProduct = magnitude(vectorOne) * magnitude(vectorTwo) + if (magnitudeProduct == 0.0) None + else Some(dotProduct(vectorOne, vectorTwo) / magnitudeProduct) + } + } + + // =============================================================================== + // The below functions are primarily for constructing artificial vectors for tests + // =============================================================================== + private def oneHotVector(dims: Int, hotIndex: Int): List[Double] = + List.tabulate(dims)(i => if (i == hotIndex) 1.0 else 0.0) + + // The first standard basis vector e0: (1, 0, 0, ...). + def firstBasisVector(dims: Int): List[Double] = oneHotVector(dims, 0) + + private def vectorWithComponents(dims: Int, components: (Int, Double)*): List[Double] = { + val arr = Array.fill(dims)(0.0) + components.foreach { case (i, v) => arr(i) = v } + arr.toList + } + + // Builds a vector whose cosine similarity with the first basis vector equals + // the requested `similarity` (in [-1, 1], where 1 is identical and 0 is orthogonal). + // This is specifically for use in tests where we need to create test fixtures with + // specific semantic scores. + def vectorWithCosineSimilarity(dims: Int, similarity: Double): List[Double] = { + require(dims > 1, "Dimensions must be at least 2") + require(similarity >= -1.0 && similarity <= 1.0, "Cosine similarity must be between -1 and 1") + if (similarity == 0.0) { + // A cosine similarity of 0 means orthogonal to the first basis vector. + // That vector is one-hot at index 0, so any vector with a zero + // first component is orthogonal to it. (0, 1, 0, 0, ...) is the simplest choice. + oneHotVector(dims, 1) + } else { + val absVec2D = { + // Let's pretend we're in just 2 dimensions and think about triangles. + // + // /| + // / | + // n / | opposite = sqrt(n^2 - 1) + // / | + // /θ___| + // 1 + // (adjacent) + // + // Our queryEmbedding lies flat on the x axis. + // cos(θ) = adjacent / hypotenuse = 1 / n + // If we want cos θ to be 1/n, e.g. 1/2, + // then the adjacent side must be 1, and the hypotenuse n. + // We want to find the opposite side + // n^2 = 1^2 + adj^2 + // opposite = sqrt(n^2 - 1) + // The two components of our vector are therefore (1, sqrt(n^2 - 1)) + val n = 1 / Math.abs(similarity) + (1, Math.sqrt(Math.pow(n, 2) - 1)) + } + val vec2D = if (similarity < 0.0) (-absVec2D._1, -absVec2D._2) else absVec2D + vectorWithComponents(dims, 0 -> vec2D._1, 1 -> vec2D._2) + } + } +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/argo/model/CollectionResponse.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/argo/model/CollectionResponse.scala index bb8af295b12..568382c2a49 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/argo/model/CollectionResponse.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/argo/model/CollectionResponse.scala @@ -19,8 +19,16 @@ case class ExtraCount( subCounts: Option[Map[String, Long]] = None ) +// Describes an AI search that had no text/similar-image query to rank by, so the +// client can tell the user how many images are in the pool a query would rank over +// (filteredPool) and prompt them to add a query. +case class FilterPoolCounts( + filteredPool: Long +) + case class ExtraCounts( - tickerCounts: Map[String, ExtraCount] + tickerCounts: Map[String, ExtraCount], + filterPoolCounts: Option[FilterPoolCounts] = None ) case class CollectionResponse[T]( @@ -37,6 +45,7 @@ case class CollectionResponse[T]( object CollectionResponse extends WriteHelpers { implicit val extraCountWrites: Writes[ExtraCount] = Json.writes[ExtraCount] + implicit val filterPoolCountsWrites: Writes[FilterPoolCounts] = Json.writes[FilterPoolCounts] implicit val extraCountsWrites: Writes[ExtraCounts] = Json.writes[ExtraCounts] implicit def collectionResponseWrites[T: Writes]: Writes[CollectionResponse[T]] = ( diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index d3c90bc739e..0f008241bfd 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -2,6 +2,7 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.BaseStore import com.gu.mediaservice.lib.config.CommonConfig +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request import scala.jdk.CollectionConverters._ import scala.concurrent.ExecutionContext @@ -18,7 +19,9 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte } private def fetchAll: Map[String, ApiAccessor] = { - val keys = s3.client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey) + val contents = s3.client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) + .contents().asScala.toList + val keys = contents.map(_.key()) keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV2BuilderUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientBuilderUtils.scala similarity index 60% rename from common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV2BuilderUtils.scala rename to common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientBuilderUtils.scala index e73d8d7408a..14040d0c33b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV2BuilderUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientBuilderUtils.scala @@ -7,16 +7,16 @@ import software.amazon.awssdk.regions.Region import java.net.URI -trait AwsClientV2BuilderUtils extends GridLogging { +trait AwsClientBuilderUtils extends GridLogging { def awsLocalEndpointUri: Option[URI] def isDev: Boolean - def awsRegionV2: Region = Region.EU_WEST_1 + def awsRegion: Region = Region.EU_WEST_1 - def awsCredentialsV2: AwsCredentialsProvider = DefaultCredentialsProvider.builder().profileName("media-service").build() + def awsCredentials: AwsCredentialsProvider = DefaultCredentialsProvider.builder().profileName("media-service").build() - final def withAWSCredentialsV2[T, S <: AwsClientBuilder[S, T]](builder: AwsClientBuilder[S, T], localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S = { - val credentialedBuilder = builder.credentialsProvider(awsCredentialsV2).region(maybeRegionOverride.getOrElse(awsRegionV2)) + final def withAWSCredentials[T, S <: AwsClientBuilder[S, T]](builder: AwsClientBuilder[S, T], localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S = { + val credentialedBuilder = builder.credentialsProvider(awsCredentials).region(maybeRegionOverride.getOrElse(awsRegion)) awsLocalEndpointUri match { case Some(endpoint) if localstackAware => diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV1BuilderUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV1BuilderUtils.scala deleted file mode 100644 index cdd4ec04d92..00000000000 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/AwsClientV1BuilderUtils.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.gu.mediaservice.lib.aws - -import com.amazonaws.auth.profile.ProfileCredentialsProvider -import com.amazonaws.auth.{AWSCredentialsProvider, AWSCredentialsProviderChain, InstanceProfileCredentialsProvider} -import com.amazonaws.client.builder.AwsClientBuilder -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration -import com.gu.mediaservice.lib.logging.GridLogging - -trait AwsClientV1BuilderUtils extends GridLogging { - def awsLocalEndpoint: Option[String] - def isDev: Boolean - - def awsRegion: String = "eu-west-1" - - def awsCredentials: AWSCredentialsProvider = new AWSCredentialsProviderChain( - new ProfileCredentialsProvider("media-service"), - InstanceProfileCredentialsProvider.getInstance() - ) - - final def awsEndpointConfiguration: Option[EndpointConfiguration] = awsLocalEndpoint match { - case Some(endpoint) if isDev => Some(new EndpointConfiguration(endpoint, awsRegion)) - case _ => None - } - - final def withAWSCredentials[T, S <: AwsClientBuilder[S, T]](builder: AwsClientBuilder[S, T], localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): S = { - awsEndpointConfiguration match { - case Some(endpointConfiguration) if localstackAware => { - logger.info(s"creating aws client with local endpoint $endpointConfiguration") - builder.withCredentials(awsCredentials).withEndpointConfiguration(endpointConfiguration) - } - case _ => builder.withCredentials(awsCredentials).withRegion(maybeRegionOverride.getOrElse(awsRegion)) - } - } -} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index bb1a8dfaabb..930ff508033 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -25,7 +25,7 @@ object Bedrock { } class Bedrock(config: CommonConfig) - extends AwsClientV2BuilderUtils { + extends AwsClientBuilderUtils { // TODO: figure out what the more usual pattern for turning off localstack behaviour is override def awsLocalEndpointUri: Option[URI] = None @@ -33,7 +33,7 @@ class Bedrock(config: CommonConfig) override def isDev: Boolean = config.isDev val client: BedrockRuntimeClient = { - withAWSCredentialsV2(BedrockRuntimeClient.builder()) + withAWSCredentials(BedrockRuntimeClient.builder()) .build() } @@ -76,14 +76,14 @@ class Bedrock(config: CommonConfig) } } - def createTextEmbedding(inputData: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { + def createTextEmbedding(inputData: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Double]] = { val requestBody = createRequestBody(inputData) val bedrockFuture = Future { sendBedrockEmbeddingRequest(requestBody) } bedrockFuture.map { response => val responseBody = response.body().asUtf8String() val json = Json.parse(responseBody) // Extract the embedding array (first element since it's an array of arrays) - val embedding = (json \ "embeddings" \ "float")(0).as[List[Float]] + val embedding = (json \ "embeddings" \ "float")(0).as[List[Double]] logger.info( logMarker, s"Successfully extracted text embedding. Vector size: ${embedding.size}" diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index 63e763fcaa6..11de358f687 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -1,24 +1,16 @@ package com.gu.mediaservice.lib.aws -import java.util -import com.amazonaws.AmazonServiceException -import com.amazonaws.services.dynamodbv2.document.spec.{DeleteItemSpec, GetItemSpec, PutItemSpec, QuerySpec, UpdateItemSpec} -import com.amazonaws.services.dynamodbv2.document.utils.ValueMap -import com.amazonaws.services.dynamodbv2.document.{DynamoDB => AwsDynamoDB, _} -import com.amazonaws.services.dynamodbv2.model.{AttributeValue, DeleteItemRequest, KeysAndAttributes, ReturnValue} -import com.amazonaws.services.dynamodbv2.{AmazonDynamoDBAsync, AmazonDynamoDBAsyncClientBuilder} -import com.gu.mediaservice.lib.aws.DynamoDB.{deleteExpr, setExpr} +import com.gu.mediaservice.lib.aws.DynamoDB.{deleteExpr, jsonWithNullAsEmptyString, setExpr} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import org.joda.time.DateTime import play.api.libs.json._ import software.amazon.awssdk.enhanced.dynamodb._ import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument -import software.amazon.awssdk.enhanced.dynamodb.model +import software.amazon.awssdk.enhanced.dynamodb.model.{BatchGetItemEnhancedRequest, ReadBatch} import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, ReturnValue => ReturnValueV2} +import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, QueryRequest => QueryRequestV2, ReturnValue => ReturnValueV2} -import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters._ @@ -32,41 +24,29 @@ object NoItemFound extends Throwable("item not found") * @tparam T The type of this table */ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Option[String] = None) extends GridLogging { - lazy val client2: DynamoDbClient = config.withAWSCredentialsV2(DynamoDbClient.builder()).build() - lazy val dynamo2: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client2).build() + lazy val client: DynamoDbClient = config.withAWSCredentials(DynamoDbClient.builder()).build() + lazy val dynamo: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client).build() lazy val tableSchema = TableSchema.documentSchemaBuilder() .addIndexPartitionKey(TableMetadata.primaryIndexName(), IdKey, AttributeValueType.S) .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) .build() - lazy val table2 = dynamo2.table(tableName, tableSchema) - - lazy val client: AmazonDynamoDBAsync = config.withAWSCredentials(AmazonDynamoDBAsyncClientBuilder.standard()).build() - lazy val dynamo = new AwsDynamoDB(client) - lazy val table: Table = dynamo.getTable(tableName) + lazy val table = dynamo.table(tableName, tableSchema) private val IdKey = "id" private def itemKey(key: String) = Key.builder().partitionValue(key).build() - def getV2(id: String)(implicit ex: ExecutionContext): Future[JsObject] = Future { - table2.getItem(itemKey(id)) + def get(id: String)(implicit ex: ExecutionContext): Future[JsObject] = Future { + table.getItem(itemKey(id)) } flatMap docOrNotFound map asJsObject - private def getV2(id: String, attribute: String)(implicit ex: ExecutionContext): Future[EnhancedDocument] = Future { - Option(table2.getItem(itemKey(id))).flatMap(doc => Option.when(doc.isPresent(attribute))(doc)) + private def get(id: String, attribute: String)(implicit ex: ExecutionContext): Future[EnhancedDocument] = Future { + Option(table.getItem(itemKey(id))).flatMap(doc => Option.when(doc.isPresent(attribute))(doc)) } flatMap { case Some(doc) => Future.successful(doc) case None => Future.failed(NoItemFound) } - private def get(id: String, attribute: String)(implicit ex: ExecutionContext): Future[Item] = Future { - table.getItem( - new GetItemSpec() - .withPrimaryKey(IdKey, id) - .withAttributesToGet(attribute) - ) - } flatMap itemOrNotFound - private def docOrNotFound(docOrNull: EnhancedDocument): Future[EnhancedDocument] = { Option(docOrNull) match { case Some(doc) => Future.successful(doc) @@ -74,131 +54,144 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti } } - private def itemOrNotFound(itemOrNull: Item): Future[Item] = { - Option(itemOrNull) match { - case Some(item) => Future.successful(item) - case None => Future.failed(NoItemFound) - } + def removeKey(id: String, key: String)(implicit ex: ExecutionContext) = Future{ + update(id, DynamoDB.removeExpr(key, lastModifiedKey)) } - def removeKey(id: String, key: String) - (implicit ex: ExecutionContext): Future[JsObject] = - update( - id, - s"REMOVE $key" - ) - - def removeKeyV2(id: String, key: String)(implicit ex: ExecutionContext) = Future{ - updateV2(id, DynamoDB.removeExpr(key, lastModifiedKey)) - } def deleteItem(id: String)(implicit ex: ExecutionContext): Future[Unit] = Future { - table.deleteItem(new DeleteItemSpec().withPrimaryKey(IdKey, id)) - } - - def deleteItemV2(id: String)(implicit ex: ExecutionContext): Future[Unit] = Future { - table2.deleteItem( + table.deleteItem( Key.builder().partitionValue(id).build() ) } - def booleanGetV2(id: String, key: String) - (implicit ex: ExecutionContext): Future[Boolean] = { - getV2(id, key).map(_.getBoolean(key).booleanValue()) + def booleanGet(id: String, key: String) + (implicit ex: ExecutionContext): Future[Boolean] = { + get(id, key).map(_.getBoolean(key).booleanValue()) } - def booleanSetV2(id: String, key: String, value: Boolean) + def booleanSet(id: String, key: String, value: Boolean) (implicit ex: ExecutionContext): Future[JsObject] = Future { - updateV2( + update( id, DynamoDB.setExpr(key, lastModifiedKey), AttributeValueV2.fromBool(value) ) } - def booleanSetOrRemoveV2(id: String, key: String, value: Boolean) + def booleanSetOrRemove(id: String, key: String, value: Boolean) (implicit ex: ExecutionContext): Future[JsObject] = - if (value) booleanSetV2(id, key, value) - else removeKeyV2(id, key) + if (value) booleanSet(id, key, value) + else removeKey(id, key) - def stringSetV2(id: String, key: String, value: String)(implicit ex: ExecutionContext): Future[JsObject] = Future { - updateV2(id, DynamoDB.setExpr(key, lastModifiedKey), AttributeValueV2.fromS(value)) + def stringSet(id: String, key: String, value: String)(implicit ex: ExecutionContext): Future[JsObject] = Future { + update(id, DynamoDB.setExpr(key, lastModifiedKey), AttributeValueV2.fromS(value)) } - def setGetV2(id: String, key: String) - (implicit ex: ExecutionContext): Future[Set[String]] = { - getV2(id, key).map(_.getStringSet(key).asScala.toSet) + def setGet(id: String, key: String) + (implicit ex: ExecutionContext): Future[Set[String]] = { + get(id, key).map(_.getStringSet(key).asScala.toSet) } - def setAddV2(id: String, key: String, value: List[String])(implicit ex: ExecutionContext): Future[JsObject] = Future { - updateV2(id, DynamoDB.addExpr(key, lastModifiedKey), AttributeValueV2.fromSs(value.asJava)) + def setAdd(id: String, key: String, value: List[String])(implicit ex: ExecutionContext): Future[JsObject] = Future { + update(id, DynamoDB.addExpr(key, lastModifiedKey), AttributeValueV2.fromSs(value.asJava)) } - def batchGet(ids: List[String], attributeKey: String) - (implicit ex: ExecutionContext, rjs: Reads[T]): Future[Map[String, T]] = { - val keyChunkList = ids - .map(k => Map(IdKey -> new AttributeValue(k)).asJava) - .grouped(100) - - Future.traverse(keyChunkList) { keyChunk => { - val keysAndAttributes: KeysAndAttributes = new KeysAndAttributes().withKeys(keyChunk.asJava) - - @tailrec - def nextPageOfBatch(request: java.util.Map[String, KeysAndAttributes], acc: List[(String, T)]) - (implicit ex: ExecutionContext, rjs: Reads[T]): List[(String, T)] = { - if (request.isEmpty) acc - else { - logger.info(s"Fetching records for $request") - val response = client.batchGetItem(request) - val responses = response.getResponses - logger.info(s"Got responses of $responses") - val results = responses.get(tableName).asScala.toList - .flatMap(att => { - val attributes: util.Map[String, AnyRef] = ItemUtils.toSimpleMapValue(att) - logger.info(s"Obtained attributes of $attributes from response $att") - val json = asJsObject(Item.fromMap(attributes)) - val maybeT = (json \ attributeKey).asOpt[T] + + def batchGet(ids: List[String], attributeKey: String)(implicit ex: ExecutionContext, rjs: Reads[T]): Future[Map[String, T]] = { + val chunks = + ids.grouped(100).toList.zipWithIndex + + Future + .traverse(chunks) { case (chunk, idx) => + logger.info(s"Fetching records for chunk $idx of ${chunks.size}") + Future { + + val readBatchBuilder = + ReadBatch.builder(classOf[EnhancedDocument]) + .mappedTableResource(table) + + chunk.foreach { id => + readBatchBuilder.addGetItem( + Key.builder() + .partitionValue(id) + .build() + ) + } + + val results = + dynamo.batchGetItem( + BatchGetItemEnhancedRequest.builder() + .readBatches(readBatchBuilder.build()) + .build() + ) + + results + .resultsForTable(table) + .asScala + .toList + .flatMap { doc => + + logger.info(s"Obtained document $doc") + + val json = asJsObject(doc) + + val maybeT = + (json \ attributeKey).asOpt[T] + logger.info(s"Obtained a T of $maybeT from json $json") + maybeT.map( - attributes.get(IdKey).toString -> _ + doc.getString(IdKey) -> _ ) - }) - logger.info(s"Got $results for request") - nextPageOfBatch(response.getUnprocessedKeys, acc ::: results) + } + .toMap } } - - Future { - nextPageOfBatch(Map(tableName -> keysAndAttributes).asJava, Nil).toMap - } - }} - .map(chunkIterator => chunkIterator.fold(Map.empty)((acc, result) => acc ++ result)) + .map(_.foldLeft(Map.empty[String, T])(_ ++ _)) } - // We cannot update, so make sure you send over the WHOLE document - def jsonAddV2(id: String, key: String, value: Map[String, JsValue]) + def jsonAdd(id: String, key: String, value: Map[String, JsValue]) (implicit ex: ExecutionContext): Future[JsObject] = Future { - updateV2( + update( id, setExpr(key, lastModifiedKey), AttributeValueV2.fromM(value.view.mapValues(DynamoDB.jsonToAttributeValue).toMap.asJava) ) } - def setDeleteV2(id: String, key: String, value: String) + def setDelete(id: String, key: String, value: String) (implicit ex: ExecutionContext): Future[JsObject] = Future { - updateV2(id, deleteExpr(key, lastModifiedKey), AttributeValueV2.fromSs(List(value).asJava)) + update(id, deleteExpr(key, lastModifiedKey), AttributeValueV2.fromSs(List(value).asJava)) } - def scanForId(indexName: String, keyname: String, key: String)(implicit ex: ExecutionContext) = Future { - val index = table.getIndex(indexName) - - val spec = new QuerySpec() - .withKeyConditionExpression(s"$keyname = :key") - .withValueMap(new ValueMap() - .withString(":key", key)) - - val items: List[Item] = index.query(spec).iterator.asScala.toList - items map (a => a.getString("id")) - } + def scanForId( + indexName: String, + keyName: String, + key: String + )(implicit ex: ExecutionContext): Future[List[String]] = + Future { + + val response = + client.query( + QueryRequestV2.builder() + .tableName(tableName) + .indexName(indexName) + .keyConditionExpression(s"$keyName = :key") + .expressionAttributeValues( + Map( + ":key" -> + AttributeValueV2.builder() + .s(key) + .build() + ).asJava + ) + .projectionExpression("id") + .build() + ) + + response.items().asScala.toList.map { item => + item.get("id").s() + } + } private def updateRequestBuilder(id: String, expression: String) = { UpdateItemRequest.builder() @@ -208,95 +201,30 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti .tableName(tableName) } - def updateV2(id: String, expression: String, attribute: AttributeValueV2): JsObject = { - updateV2(id, expression, Map(":value" -> attribute)) + def update(id: String, expression: String, attribute: AttributeValueV2): JsObject = { + update(id, expression, Map(":value" -> attribute)) } - def updateV2(id: String, expression: String): JsObject = { - updateV2(id, expression, Map.empty[String, AttributeValueV2]) + def update(id: String, expression: String): JsObject = { + update(id, expression, Map.empty[String, AttributeValueV2]) } - private def updateV2(id: String, expression: String, baseValuesMap: Map[String, AttributeValueV2]) = { + private def update(id: String, expression: String, baseValuesMap: Map[String, AttributeValueV2]) = { val valuesMap = lastModifiedKey.fold(baseValuesMap)(key => baseValuesMap ++ Map(s":${key}" -> AttributeValueV2.fromS(DateTime.now().toString))) val updateRequest = updateRequestBuilder(id, expression) .expressionAttributeValues(valuesMap.asJava) .build() - val updateItemResponse = client2.updateItem(updateRequest) + val updateItemResponse = client.updateItem(updateRequest) val jsonString = EnhancedDocument.fromAttributeValueMap(updateItemResponse.attributes()).toJson Json.parse(jsonString).as[JsObject] } - def update(id: String, expression: String, valueMap: ValueMap) - (implicit ex: ExecutionContext): Future[JsObject] = - update(id, expression, Some(valueMap)) - - def update(id: String, expression: String, valueMap: Option[ValueMap] = None) - (implicit ex: ExecutionContext): Future[JsObject] = Future { - - val baseUpdateSpec = new UpdateItemSpec(). - withPrimaryKey(IdKey, id). - withUpdateExpression(expression). - withReturnValues(ReturnValue.ALL_NEW). - withValueMap(valueMap.orNull) - - val updateSpec = lastModifiedKey.map { key => - DynamoDB.addLastModifiedUpdate(baseUpdateSpec, key, DateTime.now) - }.getOrElse(baseUpdateSpec) - - table.updateItem(updateSpec) - } map asJsObject - - - // FIXME: surely there must be a better way to convert? - def asJsObject(item: Item): JsObject = - jsonWithNullAsEmptyString(Json.parse(item.toJSON)).as[JsObject] - IdKey - def asJsObject(doc: EnhancedDocument): JsObject = jsonWithNullAsEmptyString(Json.parse(doc.toJson)).as[JsObject] - IdKey - def asJsObject(outcome: UpdateItemOutcome): JsObject = - Option(outcome.getItem) map asJsObject getOrElse Json.obj() - - // FIXME: Dynamo accepts `null`, but not `""`. This is a well documented issue - // around the community. This guard keeps the introduction of `null` fairly - // fenced in this Dynamo play area. `null` is continual and big annoyance with AWS libs. - // see: https://forums.aws.amazon.com/message.jspa?messageID=389032 - // see: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html - def mapJsValue(jsValue: JsValue)(f: JsValue => JsValue): JsValue = jsValue match { - case JsObject(items) => JsObject(items.map{ case (k, v) => k -> mapJsValue(v)(f) }) - case JsArray(items) => JsArray(items.map(f)) - case value => f(value) - } - - def jsonWithNullAsEmptyString(jsValue: JsValue): JsValue = mapJsValue(jsValue) { - case JsNull => JsString("") - case value => value - } - } object DynamoDB { - def jsonToValueMap(json: JsObject): ValueMap = { - val valueMap = new ValueMap() - json.value map { case (key, value) => - value match { - case v: JsString => valueMap.withString(key, v.value) - case v: JsBoolean => valueMap.withBoolean(key, v.value) - case v: JsNumber => valueMap.withNumber(key, v.value) - case v: JsObject => valueMap.withMap(key, jsonToValueMap(v)) - - // TODO: Lists of different Types? JsArray is not type safe (because json lists aren't) - // so this leaves us in a bit of a pickle when converting them. So for now we only support - // List[String] - case v: JsArray => valueMap.withList(key, v.value.map { - case i: JsString => i.value - case i: JsValue => i.toString - }.asJava) - case _ => valueMap - } - } - valueMap - } def jsonToAttributeValue(json: JsValue): AttributeValueV2 = { json match { @@ -314,34 +242,6 @@ object DynamoDB { def caseClassToMap[T](caseClass: T)(implicit tjs: Writes[T]): Map[String, JsValue] = Json.toJson[T](caseClass).as[JsObject].as[Map[String, JsValue]] - def addLastModifiedUpdate(update: UpdateItemSpec, lastModifiedKey: String, lastModifiedDate: DateTime): UpdateItemSpec = { - val expression = update.getUpdateExpression - val valueMap: ValueMap = { - val m = new ValueMap() - Option(update.getValueMap).foreach { vm => - m.putAll(vm) - } - m - } - - val newExpression = { - val keyUpdate: String = s"$lastModifiedKey = :$lastModifiedKey" - if (expression.contains("SET ")) { - // add to existing clause - expression.replace("SET ", s"SET ${keyUpdate}, ") - } else { - // add SET clause to existing expression - s"SET $keyUpdate ${expression}" - } - } - - valueMap.put(s":$lastModifiedKey", lastModifiedDate.toString) - - update - .withUpdateExpression(newExpression) - .withValueMap(valueMap) - } - def setExpr[T](key: String, lastModifiedKey: Option[String]) = { val baseExpression = s"SET $key = :value" lastModifiedKey.fold(baseExpression)(lastModifiedKey => s"$baseExpression, $lastModifiedKey = :$lastModifiedKey") @@ -362,4 +262,20 @@ object DynamoDB { def generateExpression(baseExpression: String, lastModifiedKey: Option[String]) = { lastModifiedKey.fold(baseExpression)(lastModifiedKey => s"$baseExpression SET $lastModifiedKey = :$lastModifiedKey") } + + // FIXME: Dynamo accepts `null`, but not `""`. This is a well documented issue + // around the community. This guard keeps the introduction of `null` fairly + // fenced in this Dynamo play area. `null` is continual and big annoyance with AWS libs. + // see: https://forums.aws.amazon.com/message.jspa?messageID=389032 + // see: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html + def mapJsValue(jsValue: JsValue)(f: JsValue => JsValue): JsValue = jsValue match { + case JsObject(items) => JsObject(items.map{ case (k, v) => k -> mapJsValue(v)(f) }) + case JsArray(items) => JsArray(items.map(f)) + case value => f(value) + } + def jsonWithNullAsEmptyString(jsValue: JsValue): JsValue = mapJsValue(jsValue) { + case JsNull => JsString("") + case value => value + } + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/EC2MetadataUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/EC2MetadataUtils.scala new file mode 100644 index 00000000000..1249441b240 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/EC2MetadataUtils.scala @@ -0,0 +1,13 @@ +package com.gu.mediaservice.lib.aws + +import software.amazon.awssdk.imds.Ec2MetadataClient + +import scala.util.Using + +object EC2MetadataUtils { + + def getInstanceId: Option[String] = Using(Ec2MetadataClient.create()) { client => + client.get("/latest/meta-data/instance-id").asString() + }.toOption + +} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index f3a5050fed0..f706b9e2137 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -1,10 +1,10 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.sqs.model.SendMessageResult import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.{Jpeg, MimeType, Png, Tiff} import play.api.libs.json.{Json, OFormat} import software.amazon.awssdk.services.s3vectors.model.QueryVectorsResponse import software.amazon.awssdk.services.s3vectors.model.{QueryOutputVector, QueryVectorsResponse, VectorData} +import software.amazon.awssdk.services.sqs.model.SendMessageResponse import java.nio.file.{Files, Path} import scala.concurrent.{ExecutionContext, Future} @@ -18,7 +18,7 @@ object EmbedderMessage { class Embedder(bedrock: Bedrock, sqs: SimpleSqsMessageConsumer)(implicit ec: ExecutionContext) extends GridLogging { - def createQueryEmbedding(query: String)(implicit logMarker: LogMarker): Future[List[Float]] = { + def createQueryEmbedding(query: String)(implicit logMarker: LogMarker): Future[List[Double]] = { logger.info(logMarker, s"Creating text embedding for query: $query") for { embedding <- bedrock.createTextEmbedding(query) @@ -27,7 +27,7 @@ class Embedder(bedrock: Bedrock, sqs: SimpleSqsMessageConsumer)(implicit ec: Exe def queueImageToEmbed(message: EmbedderMessage)(implicit logMarker: LogMarker) = { val messageBody = Json.stringify(Json.toJson(message)) - val result: SendMessageResult = sqs.sendMessage(messageBody) - logger.info(logMarker, s"Queued image for embedding with message ID: ${result.getMessageId}") + val result: SendMessageResponse = sqs.sendMessage(messageBody) + logger.info(logMarker, s"Queued image for embedding with message ID: ${result.messageId()}") } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala index 811bfe64007..bcc1e0a1093 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala @@ -1,32 +1,36 @@ package com.gu.mediaservice.lib.aws +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.core.SdkBytes +import software.amazon.awssdk.regions.Region + import java.nio.ByteBuffer import java.util.UUID -import com.amazonaws.services.kinesis.model.PutRecordRequest -import com.amazonaws.services.kinesis.{AmazonKinesis, AmazonKinesisClientBuilder} +import software.amazon.awssdk.services.kinesis.model.PutRecordRequest +import software.amazon.awssdk.services.kinesis.KinesisClient import com.gu.mediaservice.lib.json.JsonByteArrayUtil import com.gu.mediaservice.model.usage.UsageNotice import net.logstash.logback.marker.{LogstashMarker, Markers} import play.api.libs.json.{JodaWrites, Json, Writes} -import com.amazonaws.auth.AWSCredentialsProvider import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import org.joda.time.DateTime +import java.net.URI + case class KinesisSenderConfig( - override val awsRegion: String, - override val awsCredentials: AWSCredentialsProvider, - override val awsLocalEndpoint: Option[String], + override val awsRegion: Region, + override val awsCredentials: AwsCredentialsProvider, + override val awsLocalEndpointUri: Option[URI], override val isDev: Boolean, streamName: String -) extends AwsClientV1BuilderUtils +) extends AwsClientBuilderUtils class Kinesis(config: KinesisSenderConfig) extends GridLogging{ - private val builder = AmazonKinesisClientBuilder.standard() - private def getKinesisClient: AmazonKinesis = config.withAWSCredentials(builder).build() + private def getKinesisClient = config.withAWSCredentials(KinesisClient.builder()).build() - private lazy val kinesisClient: AmazonKinesis = getKinesisClient + private lazy val kinesisClient: KinesisClient = getKinesisClient def publish[T <: LogMarker](message: T)(implicit messageWrites: Writes[T]): Unit = { val partitionKey = UUID.randomUUID().toString @@ -40,10 +44,10 @@ class Kinesis(config: KinesisSenderConfig) extends GridLogging{ logger.info(markers, "Publishing message to kinesis") val data = ByteBuffer.wrap(payload) - val request = new PutRecordRequest() - .withStreamName(config.streamName) - .withPartitionKey(partitionKey) - .withData(data) + val request = PutRecordRequest.builder() + .streamName(config.streamName) + .partitionKey(partitionKey).data(SdkBytes.fromByteBuffer(data)) + .build() try { val result = kinesisClient.putRecord(request) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index da4e1eec97d..ebfd333496e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -1,18 +1,21 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.s3.model.{Region => _, _} -import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder, model} -import com.amazonaws.util.IOUtils -import com.amazonaws.{AmazonServiceException, ClientConfiguration} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ -import org.joda.time.{DateTime, Duration} +import org.joda.time.{DateTime, DateTimeZone} +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region -import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} +import software.amazon.awssdk.services.s3.model.{GetObjectRequest, GetObjectResponse, HeadObjectRequest, HeadObjectResponse, ListObjectsV2Request, NoSuchKeyException, PutObjectRequest} +import software.amazon.awssdk.services.s3.presigner.S3Presigner +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest import java.io.File import java.net.URI +import java.nio.charset.StandardCharsets +import java.time.Duration import scala.jdk.CollectionConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -48,13 +51,13 @@ object S3Object { case class S3Metadata(userMetadata: Map[String, String], objectMetadata: S3ObjectMetadata) object S3Metadata { - def apply(meta: ObjectMetadata): S3Metadata = { + def apply(meta: HeadObjectResponse): S3Metadata = { S3Metadata( - meta.getUserMetadata.asScala.toMap, + meta.metadata().asScala.toMap, S3ObjectMetadata( - contentType = Option(meta.getContentType).filterNot(_.toLowerCase == "application/octet-stream").map(MimeType.apply), - cacheControl = Option(meta.getCacheControl), - lastModified = Option(meta.getLastModified).map(new DateTime(_)) + contentType = Option(meta.contentType()).filterNot(_.toLowerCase == "application/octet-stream").map(MimeType.apply), + cacheControl = Option(meta.cacheControl()), + lastModified = Option(meta.lastModified()).map(l => new DateTime(l.toEpochMilli).withZone(DateTimeZone.UTC)) ) ) } @@ -67,76 +70,103 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with type Key = String type UserMetadata = Map[String, String] - lazy val client: AmazonS3 = S3Ops.buildS3Client(config) - - def signUrl(bucket: Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) + lazy val client: S3Client = S3Ops.buildS3Client(config) + lazy val presigner = S3Ops.buildPresignerClientV2(config) + def signUrl( + bucket: Bucket, + url: URI, + image: Image, + expiration: DateTime = cachableExpiration(), + imageType: ImageFileType = Source + ): String = { + // Fix key extraction (use stripPrefix to avoid corrupting relative paths) + val key: Key = url.getPath.stripPrefix("/") val contentDisposition = getContentDisposition(image, imageType, config.shortenDownloadFilename) - val headers = new ResponseHeaderOverrides().withContentDisposition(contentDisposition) + val nowMillis = System.currentTimeMillis() + val targetExpirationMillis = expiration.getMillis + val remainingSeconds = Math.max(1, (targetExpirationMillis - nowMillis) / 1000) + + val getObjectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(key) + .responseContentDisposition(contentDisposition) + .build() - val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) - client.generatePresignedUrl(request).toExternalForm + val getObjectPresignRequest = GetObjectPresignRequest.builder() + .getObjectRequest(getObjectRequest) + .signatureDuration(Duration.ofSeconds(remainingSeconds)) + .build() + + val req = presigner.presignGetObject(getObjectPresignRequest) + req.url().toExternalForm } - def getObject(bucket: Bucket, url: URI): model.S3Object = { + def getObject(bucket: Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { // get path and remove leading `/` val key: Key = url.getPath.drop(1) - client.getObject(new GetObjectRequest(bucket, key)) + client.getObject(GetObjectRequest.builder().key(key).bucket(bucket).build()) + } + + def getObject(bucket: Bucket, key: String): ResponseInputStream[GetObjectResponse] = { + client.getObject(GetObjectRequest.builder().key(key).bucket(bucket).build()) } def getObjectAsString(bucket: Bucket, key: String): Option[String] = { - val content = client.getObject(new GetObjectRequest(bucket, key)) - val stream = content.getObjectContent try { - Some(IOUtils.toString(stream).trim) + val stream = client.getObject(GetObjectRequest.builder().key(key).bucket(bucket).build()); + Some(new String(stream.readAllBytes(), StandardCharsets.UTF_8)) } catch { - case e: AmazonServiceException if e.getErrorCode == "NoSuchKey" => + case e: NoSuchKeyException => logger.warn(s"Cannot find key: $key in bucket: $bucket") None } - finally { - stream.close() - } + } + + def putString(bucket: String, key: String, fileContents: String) = { + client.putObject(PutObjectRequest.builder().bucket(bucket).key(key).build(), RequestBody.fromString(fileContents)) } def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = Future { - val metadata = new ObjectMetadata - mimeType.foreach(m => metadata.setContentType(m.name)) - cacheControl.foreach(metadata.setCacheControl) - metadata.setUserMetadata(meta.asJava) val fileMarkers = Map( "bucket" -> bucket, - "fileName" -> id, - "mimeType" -> mimeType.getOrElse("none"), ) val markers = logMarker ++ fileMarkers - val req = new PutObjectRequest(bucket, id, file).withMetadata(metadata) + val reqBuilder = PutObjectRequest.builder().key(id).bucket(bucket) + cacheControl.foreach(c => reqBuilder.cacheControl(c)) + mimeType.foreach(m => reqBuilder.contentType(m.name)) + reqBuilder.metadata(meta.asJava) + val req = reqBuilder.build() + Stopwatch(s"S3 client.putObject ($req)"){ - client.putObject(req) + client.putObject(req, RequestBody.fromFile(file)) // once we've completed the PUT read back to ensure that we are returning reality - val metadata = client.getObjectMetadata(bucket, id) - S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata)) + val metadata = client.headObject( + HeadObjectRequest.builder().key(id).bucket(bucket).build() + ) + + S3Object(bucket, id, metadata.contentLength(), S3Metadata(metadata)) }(markers) } def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = { - Future{ - Some(client.getObjectMetadata(bucket, id)) + Future { + Some(client.headObject( + HeadObjectRequest.builder().key(id).bucket(bucket).build() + )) }.recover { // translate this exception into the object not existing - case as3e:AmazonS3Exception if as3e.getStatusCode == 404 => None + case _: NoSuchKeyException => None }.flatMap { - case Some(objectMetadata) => + case Some(metadata) => logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket $bucket") - Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata))) + Future.successful(S3Object(bucket, id, metadata.contentLength(), S3Metadata(metadata))) case None => store(bucket, id, file, mimeType, meta, cacheControl) } @@ -145,28 +175,33 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with def list(bucket: Bucket, prefixDir: String) (implicit ex: ExecutionContext): Future[List[S3Object]] = Future { - val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixDir/") - val listing = client.listObjects(req) - val summaries = listing.getObjectSummaries.asScala - summaries.map(summary => (summary.getKey, summary)).foldLeft(List[S3Object]()) { - case (memo: List[S3Object], (key: String, summary: S3ObjectSummary)) => - S3Object(bucket, key, summary.getSize, getMetadata(bucket, key)) :: memo - } + val req = ListObjectsV2Request.builder().bucket(bucket).prefix(s"$prefixDir/").build() + val listing = client.listObjectsV2(req) + val s3Objects = listing.contents().asScala.toList + s3Objects.map(s3Object => { + S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + }) } def getMetadata(bucket: Bucket, key: Key): S3Metadata = { - val meta = client.getObjectMetadata(bucket, key) + val meta = client.headObject(HeadObjectRequest.builder().key(key).bucket(bucket).build()) S3Metadata(meta) } - def getUserMetadata(bucket: Bucket, key: Key): Map[Bucket, Bucket] = - client.getObjectMetadata(bucket, key).getUserMetadata.asScala.toMap - def syncFindKey(bucket: Bucket, prefixName: String): Option[Key] = { - val req = new ListObjectsRequest().withBucketName(bucket).withPrefix(s"$prefixName-") - val listing = client.listObjects(req) - val summaries = listing.getObjectSummaries.asScala - summaries.headOption.map(_.getKey) + val req = ListObjectsV2Request.builder().bucket(bucket).prefix(s"$prefixName-").build() + val objects = client.listObjectsV2(req).contents().asScala.toList + objects.headOption.map(_.key()) + } + def doesObjectExist(bucket: Bucket, key: String) = { + try { + client.headObject( + HeadObjectRequest.builder().key(key).bucket(bucket).build() + ) + true + } catch { + case _: NoSuchKeyException => false + } } } @@ -176,26 +211,28 @@ object S3Ops { // TODO: Make this region aware - i.e. RegionUtils.getRegion(region).getServiceEndpoint(AmazonS3.ENDPOINT_PREFIX) val s3Endpoint = "s3.amazonaws.com" - def buildS3Client(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[String] = None): AmazonS3 = { + def buildS3Client(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S3Client = { val builder = config.awsLocalEndpoint match { case Some(_) if config.isDev => - // TODO revise closer to the time of deprecation https://aws.amazon.com/blogs/aws/amazon-s3-path-deprecation-plan-the-rest-of-the-story/ - // `withPathStyleAccessEnabled` for localstack - // see https://github.com/localstack/localstack/issues/1512 - AmazonS3ClientBuilder.standard().withPathStyleAccessEnabled(true) - case _ => AmazonS3ClientBuilder.standard() + S3Client.builder().forcePathStyle(true) + case _ => S3Client.builder() } config.withAWSCredentials(builder, localstackAware, maybeRegionOverride).build() } - def buildS3ClientV2(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S3Client = { - val builder = config.awsLocalEndpoint match { - case Some(_) if config.isDev => - S3Client.builder().forcePathStyle(true) - case _ => S3Client.builder() + def buildPresignerClientV2(config: CommonConfig, localstackAware: Boolean = true, maybeRegionOverride: Option[Region] = None): S3Presigner = { + val builder = S3Presigner.builder() + .credentialsProvider(config.awsCredentials) + .region(config.awsRegion) + + config.awsLocalPresigningEndpointUri match { + case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint) + .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build() + case _ => builder.build() + } - config.withAWSCredentialsV2(builder, localstackAware, maybeRegionOverride).build() } + } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala index 824e8a798d1..5c8a0922510 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3Vectors.scala @@ -28,7 +28,7 @@ object S3Vectors { } class S3Vectors(config: CommonConfig)(implicit ec: ExecutionContext) - extends AwsClientV2BuilderUtils { + extends AwsClientBuilderUtils { import S3Vectors.DeletionStatus @@ -38,10 +38,10 @@ class S3Vectors(config: CommonConfig)(implicit ec: ExecutionContext) override def isDev: Boolean = config.isDev // The S3 Vector Store is not yet available in eu-west-1, so we are using eu-central-1 because it's closest to us. - override def awsRegionV2: Region = Region.EU_CENTRAL_1 + override def awsRegion: Region = Region.EU_CENTRAL_1 val client: S3VectorsClient = { - withAWSCredentialsV2(S3VectorsClient.builder()) + withAWSCredentials(S3VectorsClient.builder()) .build() } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SNS.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SNS.scala index 588e1664202..d405ee83688 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SNS.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SNS.scala @@ -1,16 +1,15 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.sns.model.PublishRequest -import com.amazonaws.services.sns.{AmazonSNS, AmazonSNSClientBuilder} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import play.api.libs.json.{JsValue, Json} - +import software.amazon.awssdk.services.sns.SnsClient +import software.amazon.awssdk.services.sns.model.PublishRequest class SNS(config: CommonConfig, topicArn: String) extends GridLogging { - lazy val client: AmazonSNS = config.withAWSCredentials(AmazonSNSClientBuilder.standard()).build() + lazy val client = config.withAWSCredentials(SnsClient.builder()).build() def publish(message: JsValue, subject: String): Unit = { - val result = client.publish(new PublishRequest(topicArn, Json.stringify(message), subject)) + val result = client.publish(PublishRequest.builder().topicArn(topicArn).message(Json.stringify(message)).subject(subject).build()) logger.info(s"Published message: $result") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala index 0af9faa80f8..8c680b1916e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala @@ -1,42 +1,59 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.config.CommonConfig -import com.amazonaws.services.sqs.AmazonSQS -import com.amazonaws.services.sqs.AmazonSQSClientBuilder -import com.amazonaws.services.sqs.model.{ChangeMessageVisibilityRequest, DeleteMessageRequest, ReceiveMessageRequest, SendMessageRequest, SendMessageResult, Message => SQSMessage} +import software.amazon.awssdk.services.sqs.SqsClient +import software.amazon.awssdk.services.sqs.model.{ChangeMessageVisibilityRequest, DeleteMessageRequest, GetQueueAttributesRequest, MessageSystemAttributeName, QueueAttributeName, ReceiveMessageRequest, SendMessageRequest, SendMessageResponse, Message => SQSMessage} import scala.jdk.CollectionConverters._ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { - lazy val client: AmazonSQS = config.withAWSCredentials(AmazonSQSClientBuilder.standard()).build() + lazy val client= config.withAWSCredentials(SqsClient.builder()).build() - def getNextMessage(attributeNames: String*): Option[SQSMessage] = + def getNextMessage(attributeNames: MessageSystemAttributeName*): Option[SQSMessage] = client.receiveMessage( - new ReceiveMessageRequest(queueUrl) - .withWaitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html - .withMaxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation - .withMessageSystemAttributeNames(attributeNames: _*) - ).getMessages.asScala.headOption + ReceiveMessageRequest.builder().queueUrl(queueUrl) + .waitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html + .maxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation + .messageSystemAttributeNames(attributeNames: _*) + .build() + ).messages().asScala.headOption def deleteMessage(message: SQSMessage): Unit = - client.deleteMessage(new DeleteMessageRequest(queueUrl, message.getReceiptHandle)) + client.deleteMessage( + DeleteMessageRequest.builder() + .queueUrl(queueUrl) + .receiptHandle(message.receiptHandle()) + .build() + ) def makeMessageVisible(message: SQSMessage): Unit = - client.changeMessageVisibility(new ChangeMessageVisibilityRequest(queueUrl, message.getReceiptHandle, 0)) + client.changeMessageVisibility( + ChangeMessageVisibilityRequest.builder() + .queueUrl(queueUrl) + .receiptHandle(message.receiptHandle()) + .visibilityTimeout(0) + .build() + ) def getStatus: Map[String, String] = { - client.getQueueAttributes(queueUrl, List( - "ApproximateNumberOfMessagesDelayed", - "ApproximateNumberOfMessages", - "ApproximateNumberOfMessagesNotVisible" - ).asJava).getAttributes.asScala.toMap + client.getQueueAttributes( + GetQueueAttributesRequest.builder(). + queueUrl(queueUrl) + .attributeNames( + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED, + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE + ) + .build() + ).attributes().asScala.toMap.map({ case(k, v) => (k.name(), v)}) } - def sendMessage(messageBody: String): SendMessageResult = { - val sendMessageRequest: SendMessageRequest = new SendMessageRequest() - .withQueueUrl(queueUrl) - .withMessageBody(messageBody) + def sendMessage(messageBody: String): SendMessageResponse = { + val sendMessageRequest: SendMessageRequest = SendMessageRequest.builder() + .queueUrl(queueUrl) + .messageBody(messageBody) + .build() client.sendMessage(sendMessageRequest) } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsHelpers.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsHelpers.scala index 288c58cd0f6..919753c0876 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsHelpers.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsHelpers.scala @@ -1,6 +1,6 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.sqs.model.{Message => SQSMessage} +import software.amazon.awssdk.services.sqs.model.{MessageSystemAttributeName, Message => SQSMessage} import com.gu.mediaservice.lib.net.URI.{decode => uriDecode} import play.api.libs.json.Json @@ -8,9 +8,9 @@ import scala.util.Try trait SqsHelpers { - val attrApproximateReceiveCount = "ApproximateReceiveCount" + val attrApproximateReceiveCount = MessageSystemAttributeName.APPROXIMATE_RECEIVE_COUNT def getApproximateReceiveCount(message: SQSMessage): Int = - Try(message.getAttributes.get(attrApproximateReceiveCount).toInt).toOption.getOrElse(-1) + Try(message.attributes().get(attrApproximateReceiveCount).toInt).toOption.getOrElse(-1) def extractS3KeyFromSqsMessage(message: SQSMessage): Try[String] = Try { @@ -21,7 +21,7 @@ trait SqsHelpers { // note that we don't bother reading the bucket name currently, but attempt the read elsewhere based on the // configured bucket value, this could be checked here if we're concerned (s3Records.head \ "bucket" \ "name") - (Json.parse(message.getBody) \ "Records" \\ "s3").toList match { + (Json.parse(message.body()) \ "Records" \\ "s3").toList match { case head :: Nil => uriDecode((head \ "object" \ "key").as[String]) case s3Records => throw new Exception(s"Expected 1 record containing 's3' in message body, got ${s3Records.size}") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsViaSnsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsViaSnsMessageConsumer.scala index ede4d897360..d9398cee525 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsViaSnsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsViaSnsMessageConsumer.scala @@ -4,7 +4,7 @@ import java.util.concurrent.Executors import _root_.play.api.libs.functional.syntax._ import _root_.play.api.libs.json._ import org.apache.pekko.actor.ActorSystem -import com.amazonaws.services.sqs.model.{Message => SQSMessage} +import software.amazon.awssdk.services.sqs.model.{Message => SQSMessage} import com.gu.mediaservice.lib.ImageId import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.json.PlayJsonHelpers._ @@ -57,7 +57,7 @@ abstract class SqsViaSnsMessageConsumer(queueUrl: String, config: CommonConfig, f.foreach { _ => deleteMessage(msg) } private def extractSNSMessage(sqsMessage: SQSMessage): Option[SNSMessage] = { - val result = Json.fromJson[SNSMessage](Json.parse(sqsMessage.getBody)) + val result = Json.fromJson[SNSMessage](Json.parse(sqsMessage.body())) logParseErrors(result) result.asOpt } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala index c55b9cf1b1f..a6bbcc588e1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala @@ -69,10 +69,12 @@ object PhotographerRenamer extends MetadataCleaner { "Aurelien Laudy" -> "Aurélien Laudy", "Aurelien Meunier" -> "Aurélien Meunier", "Aurelien Morissard" -> "Aurélien Morissard", + "Aurore Marechal" -> "Aurore Maréchal", "Aykut Unlupinar" -> "Aykut Ünlüpınar", "Aytac Unal" -> "Aytaç Ünal", "Azael Rodriguez" -> "Azael Rodríguez", "Balazs Mohai" -> "Balázs Mohai", + "Balint Szentgallay" -> "Bálint Szentgallay", "Bartlomiej Wojtowicz" -> "Bartłomiej Wojtowicz", "Bartlomiej Zborowski" -> "Bartłomiej Zborowski", "Bartosz Banka" -> "Bartosz Bańka", @@ -131,6 +133,7 @@ object PhotographerRenamer extends MetadataCleaner { "Daniel Garcia" -> "Daniel García", "Daniel Mihailescu" -> "Daniel Mihăilescu", "Daniel Muñoz" -> "Daniel Muñoz", + "Dante Fernandez" -> "Dante Fernández", "Darko Bandic" -> "Darko Bandić", "Darko Vojinovic" -> "Darko Vojinović", "Dave M Benett" -> "David M Benett", @@ -152,6 +155,7 @@ object PhotographerRenamer extends MetadataCleaner { "Eduardo Munoz" -> "Eduardo Muñoz", "Eduardo Munoz Alvarez" -> "Eduardo Muñoz", "Edwin Bercian" -> "Edwin Bercián", + "Efekan Akyuz" -> "Efekan Akyüz", "Egle Kazdailyte" -> "Eglė Každailytė", "Elif Ozturk" -> "Elif Öztürk", "Elin Hoyland" -> "Elin Høyland", @@ -399,6 +403,7 @@ object PhotographerRenamer extends MetadataCleaner { "Mario Arturo Martinez" -> "Mario Arturo Martínez", "Mario Cruz" -> "Mário Cruz", "Mario Vazquez" -> "Mario Vázquez", + "Marko Djokovic" -> "Marko Đoković", "Marko Djurica" -> "Marko Đurica", "Marko Drobnjakovic" -> "Marko Drobnjaković", "Martin Mejia" -> "Martín Mejía", @@ -426,6 +431,7 @@ object PhotographerRenamer extends MetadataCleaner { "Mitar Mitrovic" -> "Mitar Mitrović", "Moises Castillo" -> "Moisés Castillo", "Morne de Klerk" -> "Morné de Klerk", + "Muhammed Enes Yildirim" -> "Muhammed Enes Yıldırım", "Murat Ozgur Guvendik" -> "Murat Özgür Güvendik", "Mustafa Ciftci" -> "Mustafa Çiftçi", "Mustafa Ozturk" -> "Mustafa Öztürk", @@ -436,6 +442,7 @@ object PhotographerRenamer extends MetadataCleaner { "Niklas Halle’N" -> "Niklas Halle’n", "Nuno Andre Ferreira" -> "Nuno André Ferreira", "Octavio Passos" -> "Octávio Passos", + "Oisin Keniry" -> "Oisín Keniry", "Omer Urer" -> "Ömer Ürer", "Onur Coban" -> "Onur Çoban", "Orhan Seref Akkanat" -> "Orhan Şeref Akkanat", @@ -626,6 +633,7 @@ object PhotographerRenamer extends MetadataCleaner { "Zorana Jevtic" -> "Zorana Jevtić", "Zsolt Czegledi" -> "Zsolt Czeglédi", "Zsolt Szigetvary" -> "Zsolt Szigetváry", + "Zuzana Gogova" -> "Zuzana Gogová", "Zvonimir Barisin" -> "Zvonimir Barišin" ) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala index 87b4f28ccd3..4c897f2f857 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala @@ -22,9 +22,9 @@ class SupplierProcessors(resources: ImageProcessorResources) ReutersParser, RexParser, RonaldGrantParser, - new PhotographerParser(resources.commonConfiguration.usageRightsConfig), AllstarSportsphotoParser, AllStarParser, + new PhotographerParser(resources.commonConfiguration.usageRightsConfig), UsageRightsToMetadataParser(resources) //This should come after processors that assign usage rights ) @@ -52,21 +52,24 @@ case class UsageRightsToMetadataParser(resources: ImageProcessorResources) exten * Guardian specific logic to correctly identify Guardian and Observer photographers and their contracts */ class PhotographerParser(photographersConfig: UsageRightsConfigProvider) extends ImageProcessor { - def apply(image: Image): Image = { - image.metadata.byline.flatMap { byline => - photographersConfig.getPhotographer(byline, image.metadata.dateTaken.getOrElse(image.uploadTime)).map{ - case p: StaffPhotographer => image.copy( - usageRights = p, - metadata = image.metadata.copy(credit = Some(p.publication), byline = Some(p.photographer)) - ) - case p: ContractPhotographer => image.copy( - usageRights = p, - metadata = image.metadata.copy(credit = p.publication, byline = Some(p.photographer)) - ) - case _ => image - } - } - }.getOrElse(image) + def apply(image: Image): Image = image.usageRights match { + // only attempt to set Photographer Usagerights if no other processor has already set usage rights + case NoRights => + image.metadata.byline.flatMap { byline => + photographersConfig.getPhotographer(byline, image.metadata.dateTaken.getOrElse(image.uploadTime)).map{ + case p: StaffPhotographer => image.copy( + usageRights = p, + metadata = image.metadata.copy(credit = Some(p.publication), byline = Some(p.photographer)) + ) + case p: ContractPhotographer => image.copy( + usageRights = p, + metadata = image.metadata.copy(credit = p.publication, byline = Some(p.photographer)) + ) + case _ => image + } + }.getOrElse(image) + case _ => image + } } object AapParser extends ImageProcessor { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 2cdf3edea3f..669e49490c8 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -1,18 +1,19 @@ package com.gu.mediaservice.lib.config -import com.gu.mediaservice.lib.aws.{AwsClientV1BuilderUtils, AwsClientV2BuilderUtils, KinesisSenderConfig} +import com.gu.mediaservice.lib.aws.{AwsClientBuilderUtils, KinesisSenderConfig} import com.gu.mediaservice.model.UsageRightsSpec import com.typesafe.config.Config import com.typesafe.scalalogging.StrictLogging import play.api.{ConfigLoader, Configuration} import scalaz.NonEmptyList +import software.amazon.awssdk.regions.Region import java.net.URI import java.util.UUID import scala.jdk.CollectionConverters._ import scala.util.Try -abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1BuilderUtils with AwsClientV2BuilderUtils with StrictLogging { +abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBuilderUtils with StrictLogging { val configuration: Configuration = resources.configuration final val stackName = "media-service" @@ -24,11 +25,21 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B val isProd: Boolean = stage == "PROD" override val isDev: Boolean = stage == "DEV" - override val awsRegion: String = stringDefault("aws.region", "eu-west-1") + override val awsRegion: Region = Region.of(stringDefault("aws.region", "eu-west-1")) - override val awsLocalEndpoint: Option[String] = if(isDev) stringOpt("aws.local.endpoint").filter(_.nonEmpty) else None + val awsLocalEndpoint: Option[String] = if(isDev) stringOpt("aws.local.endpoint").filter(_.nonEmpty) else None override val awsLocalEndpointUri: Option[URI] = awsLocalEndpoint.map(new URI(_)) + // Endpoint baked into presigned URLs handed to the browser. Distinct from `aws.local.endpoint` + // (used by the S3 client for container-internal calls) because the browser cannot resolve the + // container-network host. Falls back to `aws.local.endpoint` when unset. + val awsLocalPresigningEndpointUri: Option[URI] = + if (isDev) stringOpt("aws.local.presigningEndpoint") + .filter(_.nonEmpty) + .orElse(awsLocalEndpoint) + .map(new URI(_)) + else None + val useLocalAuth: Boolean = isDev && boolean("auth.useLocal") val localLogShipping: Boolean = sys.env.getOrElse("LOCAL_LOG_SHIPPING", "false").toBoolean @@ -47,6 +58,12 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B val shouldDisplayOrgOwnedCountAndFilterCheckbox: Boolean = boolean("filters.shouldDisplayOrgOwnedCountAndFilterCheckbox") + // Defaults to false so orgs that haven't wired up the Bedrock/SQS/vector-ES + // infrastructure required for AI search don't have it enabled by staying up to date with main. + // Shared by media-api (gates the search endpoint) and kahuna (gates the tickbox), so it only + // needs setting in one config key. + val aiSearchEnabled: Boolean = booleanOpt("ai.search.enabled").getOrElse(false) + val systemName: String = stringOpt("branding.systemName").filterNot(_.isEmpty).getOrElse("the Grid") lazy val softDeletedMetadataTable: String = string("dynamo.table.softDelete.metadata") @@ -218,8 +235,9 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B val agencyPicksIngredients: Option[Map[String, Seq[String]]] = configuration.getOptional[Map[String, Seq[String]]]("agencyPicks.ingredients") val agencyPicksColour: String = stringDefault("agencyPicks.colour", "#7d0068") + val enforceUploadImagesPermission: Boolean = booleanOpt("enforceUploadImagesPermission").getOrElse(false) - private def getKinesisConfigForStream(streamName: String) = KinesisSenderConfig(awsRegion, awsCredentials, awsLocalEndpoint, isDev, streamName) + private def getKinesisConfigForStream(streamName: String) = KinesisSenderConfig(awsRegion, awsCredentials, awsLocalEndpointUri, isDev, streamName) final def getOptionalStringSet(key: String): Option[Set[String]] = Try { configuration.getOptional[Seq[String]](key) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/FieldAlias.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/FieldAlias.scala index a50f590ae2f..fa4468ed556 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/FieldAlias.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/FieldAlias.scala @@ -10,7 +10,15 @@ case class FieldAlias(elasticsearchPath: String, displayInAdditionalMetadata: Boolean, displaySearchHint: Boolean, alias: String, - searchHintOptions: List[String]) + searchHintOptions: List[String], + // Some fields are only ever indexed when "true" (e.g. a field that is present + // only to support `has:`/`-has:` queries, such as fileMetadata.c2pa.isAvailable). + // For these fields a literal `alias:false` term query can never match, since "false" + // is never actually indexed - it is only ever implied by the field's absence. + // Setting this flag translates `alias:true`/`alias:false` search queries into + // exists/not-exists queries instead of literal term matches, so both values behave + // intuitively for users (e.g. via a searchHintOptions dropdown of "true"/"false"). + matchViaExistence: Boolean = false) object FieldAlias { implicit val FieldAliasWrites: Writes[FieldAlias] = @@ -26,6 +34,8 @@ object FieldAlias { config.getBoolean("displaySearchHint") else false val searchHintOptions = if (config.hasPath("searchHintOptions")) config.getStringList("searchHintOptions").asScala.toList.filter(_.nonEmpty) else List.empty + val matchViaExistence = if (config.hasPath("matchViaExistence")) + config.getBoolean("matchViaExistence") else false FieldAlias( config.getString("elasticsearchPath"), @@ -33,7 +43,8 @@ object FieldAlias { displayInAdditionalMetadata, displaySearchHint, config.getString("alias"), - searchHintOptions + searchHintOptions, + matchViaExistence ) } ).toSeq diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/discovery/EC2.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/discovery/EC2.scala deleted file mode 100644 index 7ae37625a2b..00000000000 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/discovery/EC2.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.gu.mediaservice.lib.discovery - -import com.amazonaws.services.ec2.AmazonEC2 -import com.amazonaws.services.ec2.model.{DescribeInstancesRequest, Filter, InstanceStateName} -import com.gu.mediaservice.lib.logging.GridLogging - -import scala.jdk.CollectionConverters._ -import scala.util.Random - -object EC2 extends GridLogging { - - @annotation.tailrec - def findElasticsearchHostByTags(client: AmazonEC2, tags: Map[String, Seq[String]]): String = { - val instances = client.describeInstances(new DescribeInstancesRequest().withFilters( - new Filter("instance-state-name", List(InstanceStateName.Running.toString).asJava) +: - tagFilters(tags): _* - )) - - val hosts = instances.getReservations.asScala - .flatMap(_.getInstances.asScala) - .map(_.getPublicDnsName) - logger.info(s"Available Elasticsearch hosts in EC2: [${hosts.mkString(", ")}]") - - Random.shuffle(hosts).headOption match { - case None => - logger.warn("Could not find an Elasticsearch host. Trying again...") - Thread.sleep(1000) - findElasticsearchHostByTags(client, tags) - case Some(host) => - logger.info(s"Using Elasticsearch host $host") - host - } - } - - def tagFilters(tags: Map[String, Seq[String]]): List[Filter] = - for ((key, values) <- tags.toList) yield new Filter(s"tag:$key", values.asJava) - -} diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/dynamo/DynamoValue.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/dynamo/DynamoValue.scala new file mode 100644 index 00000000000..689ae61cb20 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/dynamo/DynamoValue.scala @@ -0,0 +1,35 @@ +package com.gu.mediaservice.lib.dynamo +import software.amazon.awssdk.services.dynamodb.model.AttributeValue +import scala.jdk.CollectionConverters._ + +sealed trait DynamoElement { + def toAttrValue: AttributeValue +} + + +case class DbString(value: String) extends DynamoElement { + def toAttrValue: AttributeValue = AttributeValue.builder().s(value).build() +} + +case class DbLong(value: Long) extends DynamoElement { + def toAttrValue: AttributeValue = AttributeValue.builder().n(value.toString).build() +} + +case class DbInt(value: Int) extends DynamoElement { + def toAttrValue: AttributeValue = AttributeValue.builder().n(value.toString).build() +} + +case class DbNestedMap(value: Map[String, DynamoElement]) extends DynamoElement { + def toAttrValueRec(value: Map[String, DynamoElement], acc: Map[String, AttributeValue]): Map[String, AttributeValue] = { + value.flatMap { case (k, v) => + v match { + case DbString(_) | DbLong(_) | DbInt(_) => acc.updated(k, v.toAttrValue) + case DbNestedMap(m) => toAttrValueRec(m, acc) + } + } + } + val javaMap = toAttrValueRec(value, Map[String, AttributeValue]()).asJava + + def toAttrValue = AttributeValue.builder().m(javaMap).build() +} + diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala index 6fbb23682f3..17321133d91 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala @@ -41,8 +41,15 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { lazy val client = { logger.info("Connecting to Elastic 8: " + url) - val client = JavaClient(ElasticProperties(url)) - ElasticClient(client) + val hosts = ElasticProperties(url).endpoints.collect { + case ElasticNodeEndpoint(protocol, host, port, _) => + new org.apache.http.HttpHost(host, port, protocol) + } + val restClient = org.elasticsearch.client.RestClient + .builder(hosts: _*) + .setCompressionEnabled(true) // request gzip from ES; reduces response body ~5-6× + .build() + ElasticClient(JavaClient.fromRestClient(restClient)) } //TODO: this function should fail and cause healthcheck fails diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala index f4935c04c53..45550775985 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala @@ -109,7 +109,8 @@ object MappingTest { icc = Map("icc1" -> "value1"), getty = Map("getty1" -> "value1"), colourModel = Some("my-model"), - colourModelInformation = Map("colourModel1" -> "value1") + colourModelInformation = Map("colourModel1" -> "value1"), + c2pa = FileMetadata.C2paAvailable ), userMetadata = Some(Edits( archived = true, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala index 3bdeb73b3dd..e5fe3318a3e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/guardian/GuardianUsageRightsConfig.scala @@ -81,7 +81,6 @@ object GuardianUsageRightsConfig extends UsageRightsConfigProvider { )), PublicationPhotographers(GuardianPublication, List( PublicationPhotographer("Alicia Canter"), - PublicationPhotographer("Antonio Olmos"), PublicationPhotographer("Christopher Thomond"), PublicationPhotographer("David Levene"), PublicationPhotographer("Eamonn McCabe"), diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/logging/LogConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/logging/LogConfig.scala index 7f2f170f36a..edcb16c97db 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/logging/LogConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/logging/LogConfig.scala @@ -1,13 +1,12 @@ package com.gu.mediaservice.lib.logging import java.net.InetSocketAddress - import ch.qos.logback.classic.spi.ILoggingEvent import ch.qos.logback.classic.{LoggerContext, Logger => LogbackLogger} import net.logstash.logback.appender.LogstashTcpSocketAppender -import com.amazonaws.util.EC2MetadataUtils import com.gu.logback.appender.kinesis.KinesisAppender import com.gu.mediaservice.lib.config.CommonConfig +import com.gu.mediaservice.lib.aws.EC2MetadataUtils import net.logstash.logback.encoder.LogstashEncoder import net.logstash.logback.layout.LogstashLayout import org.slf4j.{LoggerFactory, Logger => SLFLogger} @@ -27,8 +26,7 @@ object LogConfig { case class KinesisAppenderConfig(stream: String, region: String, roleArn: String, bufferSize: Int) private def makeCustomFields(config: CommonConfig): String = { - val instanceId = Option(EC2MetadataUtils.getInstanceId).getOrElse("unknown") - + val instanceId: String = if(config.isDev) "unknown" else EC2MetadataUtils.getInstanceId.getOrElse("unknown") Json.toJson(Map( "stack" -> config.stackName, "stage" -> config.stage.toUpperCase, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/SoftDeletedMetadataTable.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/SoftDeletedMetadataTable.scala index e20edf694a5..78310cf7b70 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/SoftDeletedMetadataTable.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/metadata/SoftDeletedMetadataTable.scala @@ -1,6 +1,5 @@ package com.gu.mediaservice.lib.metadata -import com.amazonaws.services.dynamodbv2.model.BatchWriteItemResult import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.model.ImageStatusRecord import org.scanamo._ @@ -11,7 +10,7 @@ import software.amazon.awssdk.services.dynamodb.DynamoDbAsyncClient import scala.concurrent.{ExecutionContext, Future} class SoftDeletedMetadataTable(config: CommonConfig) { - val client = config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build() + val client = config.withAWSCredentials(DynamoDbAsyncClient.builder()).build() private val softDeletedMetadataTable = Table[ImageStatusRecord](config.softDeletedMetadataTable) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala index da56dd2b823..6a5af78c4f6 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/metrics/CloudWatchMetrics.scala @@ -44,7 +44,7 @@ abstract class CloudWatchMetrics( protected def toDatum(value: Long, dimensions: List[Dimension]): MetricDatum = datum(StandardUnit.MILLISECONDS, value.toDouble, dimensions) } - private val client = config.withAWSCredentialsV2(CloudWatchClient.builder()).build() + private val client = config.withAWSCredentials(CloudWatchClient.builder()).build() private val random = new Random() private[CloudWatchMetrics] val metricsActor = actorSystem.actorOf(MetricsActor.props(namespace, client), s"metricsactor-${random.alphanumeric.take(8).mkString}") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala index eb3194bfe5b..882030a4907 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala @@ -1,112 +1,119 @@ package com.gu.mediaservice.lib.usage import java.net.URI -import com.amazonaws.services.dynamodbv2.document.Item + import com.gu.mediaservice.model.usage._ import org.joda.time.DateTime import org.joda.time.format.ISODateTimeFormat +import software.amazon.awssdk.enhanced.dynamodb.{DefaultAttributeConverterProvider, EnhancedType} +import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument +import software.amazon.awssdk.services.dynamodb.model.AttributeValue import scala.jdk.CollectionConverters._ import scala.util.Try object ItemToMediaUsage { - - def transform(item: Item): MediaUsage = { + def transform(doc: EnhancedDocument): MediaUsage = { MediaUsage( - UsageId(item.getString("usage_id")), - item.getString("grouping"), - item.getString("media_id"), - UsageType(item.getString("usage_type")), - item.getString("media_type"), - UsageStatus(item.getString("usage_status")), - Option(item.getMap[Any]("print_metadata")) - .map(_.asScala.toMap).flatMap(buildPrint), - Option(item.getMap[Any]("digital_metadata")) - .map(_.asScala.toMap).flatMap(buildDigital), - Option(item.getMap[Any]("syndication_metadata")) - .map(_.asScala.toMap).flatMap(buildSyndication), - Option(item.getMap[Any]("front_metadata")) - .map(_.asScala.toMap).flatMap(buildFront), - Option(item.getMap[Any]("download_metadata")) - .map(_.asScala.toMap).flatMap(buildDownload), - Option(item.getMap[Any]("child_metadata")) - .map(_.asScala.toMap).flatMap(buildChild), - new DateTime(item.getLong("last_modified")), - Try { - item.getLong("date_added") - }.toOption.map(new DateTime(_)), - Try { - item.getLong("date_removed") - }.toOption.map(new DateTime(_)) + UsageId(doc.getString("usage_id")), + doc.getString("grouping"), + doc.getString("media_id"), + UsageType(doc.getString("usage_type")), + doc.getString("media_type"), + UsageStatus(doc.getString("usage_status")), + Option(doc.getMap( + "print_metadata", + EnhancedType.of(classOf[String]), + EnhancedType.of(classOf[AttributeValue]) + )) + .map(_.asScala.toMap) + .flatMap(buildPrintFromAttr), + Option(doc.getMap("digital_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) + .map(_.asScala.toMap) + .flatMap(buildDigital), + Option(doc.getMap("syndication_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) + .map(_.asScala.toMap) + .flatMap(buildSyndication), + Option(doc.getMap("front_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) + .map(_.asScala.toMap) + .flatMap(buildFront), + Option(doc.getMap("download_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) + .map(_.asScala.toMap) + .flatMap(buildDownload), + Option(doc.getMap("child_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) + .map(_.asScala.toMap) + .flatMap(buildChild), + new DateTime(doc.getNumber("last_modified").longValue()), + Try(doc.getNumber("date_added").longValue()).toOption.map(new DateTime(_)), + Try(doc.getNumber("date_removed").longValue()).toOption.map(new DateTime(_)) ) } - private def buildFront(metadataMap: Map[String, Any]): Option[FrontUsageMetadata] = { + + private def buildFront(metadataMap: Map[String, String]): Option[FrontUsageMetadata] = { Try { FrontUsageMetadata( - metadataMap("addedBy").asInstanceOf[String], - metadataMap("front").asInstanceOf[String] + metadataMap("addedBy"), + metadataMap("front") ) }.toOption } - private def buildSyndication(metadataMap: Map[String, Any]): Option[SyndicationUsageMetadata] = { + private def buildSyndication(metadataMap: Map[String, String]): Option[SyndicationUsageMetadata] = { Try { SyndicationUsageMetadata( - metadataMap("partnerName").asInstanceOf[String], - metadataMap.get("syndicatedBy").map(x => x.asInstanceOf[String]) + metadataMap("partnerName"), + metadataMap.get("syndicatedBy") ) }.toOption } - private def buildDigital(metadataMap: Map[String, Any]): Option[DigitalUsageMetadata] = { + private def buildDigital(metadataMap: Map[String, String]): Option[DigitalUsageMetadata] = { Try { DigitalUsageMetadata( - URI.create(metadataMap("webUrl").asInstanceOf[String]), - metadataMap("webTitle").asInstanceOf[String], - metadataMap("sectionId").asInstanceOf[String], - metadataMap.get("composerUrl").map(x => URI.create(x.asInstanceOf[String])) + URI.create(metadataMap("webUrl")), + metadataMap("webTitle"), + metadataMap("sectionId"), + metadataMap.get("composerUrl").map(x => URI.create(x)) ) }.toOption } - private def buildPrint(metadataMap: Map[String, Any]): Option[PrintUsageMetadata] = { - type JStringNumMap = java.util.LinkedHashMap[String, java.math.BigDecimal] + private def buildPrintFromAttr(metadataMap: Map[String, AttributeValue]): Option[PrintUsageMetadata] = { Try { PrintUsageMetadata( - sectionName = metadataMap.apply("sectionName").asInstanceOf[String], - issueDate = metadataMap.get("issueDate").map(_.asInstanceOf[String]) + sectionName = metadataMap.apply("sectionName").s(), + issueDate = metadataMap.get("issueDate").map(_.s()) .map(ISODateTimeFormat.dateTimeParser().parseDateTime).get, - pageNumber = metadataMap.apply("pageNumber").asInstanceOf[java.math.BigDecimal].intValue, - storyName = metadataMap.apply("storyName").asInstanceOf[String], - publicationCode = metadataMap.apply("publicationCode").asInstanceOf[String], - publicationName = metadataMap.apply("publicationName").asInstanceOf[String], - layoutId = metadataMap.get("layoutId").map(_.asInstanceOf[java.math.BigDecimal].intValue), - edition = metadataMap.get("edition").map(_.asInstanceOf[java.math.BigDecimal].intValue), - size = metadataMap.get("size") - .map(_.asInstanceOf[JStringNumMap]) - .map(m => PrintImageSize(m.get("x").intValue, m.get("y").intValue)), - orderedBy = metadataMap.get("orderedBy").map(_.asInstanceOf[String]), - sectionCode = metadataMap.apply("sectionCode").asInstanceOf[String], - notes = metadataMap.get("notes").map(_.asInstanceOf[String]), - source = metadataMap.get("source").map(_.asInstanceOf[String]) + pageNumber = metadataMap.apply("pageNumber").n().toInt, + storyName = metadataMap.apply("storyName").s(), + publicationCode = metadataMap.apply("publicationCode").s(), + publicationName = metadataMap.apply("publicationName").s(), + layoutId = metadataMap.get("layoutId").map(_.n()).map(BigDecimal(_)).map(_.intValue), + edition = metadataMap.get("edition").map(_.n()).map(BigDecimal(_)).map(_.intValue), + size = metadataMap.get("size").map(_.m()) + .map(m => PrintImageSize(m.get("x").n().toInt, m.get("y").n().toInt)), + orderedBy = metadataMap.get("orderedBy").map(_.s()), + sectionCode = metadataMap.apply("sectionCode").s(), + notes = metadataMap.get("notes").map(_.s()), + source = metadataMap.get("source").map(_.s()) ) }.toOption } - private def buildDownload(metadataMap: Map[String, Any]): Option[DownloadUsageMetadata] = { + private def buildDownload(metadataMap: Map[String, String]): Option[DownloadUsageMetadata] = { Try { DownloadUsageMetadata( - metadataMap("downloadedBy").asInstanceOf[String] + metadataMap("downloadedBy") ) }.toOption } - private def buildChild(metadataMap: Map[String, Any]): Option[ChildUsageMetadata] = { + private def buildChild(metadataMap: Map[String, String]): Option[ChildUsageMetadata] = { Try { ChildUsageMetadata( - metadataMap("addedBy").asInstanceOf[String], - metadataMap("childMediaId").asInstanceOf[String], + metadataMap("addedBy"), + metadataMap("childMediaId"), ) }.toOption } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala index b465e689fc2..9e1dba21d0c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala @@ -15,7 +15,8 @@ case class FileMetadata( icc: Map[String, String] = Map(), getty: Map[String, String] = Map(), colourModel: Option[String] = None, - colourModelInformation: Map[String, String] = Map() + colourModelInformation: Map[String, String] = Map(), + c2pa: Map[String, JsValue] = Map() ) { def toLogMarker: LogMarker = { val fieldCountMarkers = Map ( @@ -31,7 +32,7 @@ case class FileMetadata( val totalFieldCount = fieldCountMarkers.foldLeft(0)(_ + _._2) val markers = fieldCountMarkers + ("totalFieldCount" -> totalFieldCount) - MarkerMap(markers) + MarkerMap(markers + ("c2paMetadataAvailable" -> c2pa.keys.nonEmpty)) } def readXmpHeadStringProp: (String) => Option[String] = (name: String) => { @@ -46,6 +47,9 @@ case class FileMetadata( } object FileMetadata { + val NoC2PA: Map[String, JsValue] = Map() + val C2paAvailable: Map[String, JsValue] = Map("isAvailable" -> JsBoolean(true)) + // TODO: reindex all images to make the getty map always present // for data consistency, so we can fallback to use the default Reads implicit val ImageMetadataReads: Reads[FileMetadata] = ( @@ -56,7 +60,8 @@ object FileMetadata { (__ \ "icc").readNullable[Map[String,String]].map(_ getOrElse Map.empty) ~ (__ \ "getty").readNullable[Map[String,String]].map(_ getOrElse Map.empty) ~ (__ \ "colourModel").readNullable[String] ~ - (__ \ "colourModelInformation").readNullable[Map[String,String]].map(_ getOrElse Map.empty) + (__ \ "colourModelInformation").readNullable[Map[String,String]].map(_ getOrElse Map.empty) ~ + (__ \ "c2pa").readNullable[Map[String,JsValue]].map(_ getOrElse Map.empty) )(FileMetadata.apply _) @@ -68,6 +73,7 @@ object FileMetadata { (JsPath \ "icc").write[Map[String,String]] and (JsPath \ "getty").write[Map[String,String]] and (JsPath \ "colourModel").writeNullable[String] and - (JsPath \ "colourModelInformation").write[Map[String,String]] + (JsPath \ "colourModelInformation").write[Map[String,String]] and + (JsPath \ "c2pa").write[Map[String,JsValue]] )(unlift(FileMetadata.unapply)) } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala index 2db517e51d9..05c0c2f133b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala @@ -158,7 +158,7 @@ case object NoRights val category = "" val defaultCost = None val restrictions = None - def name(commonConfig: CommonConfig) = "No Rights" + def name(commonConfig: CommonConfig) = "Unknown Rights" def description(commonConfig: CommonConfig) = "Images which we do not currently have the rights to use." diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/ChildUsageMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/ChildUsageMetadata.scala index 8daaaeab2fe..91b3dd2bf5d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/ChildUsageMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/ChildUsageMetadata.scala @@ -1,5 +1,6 @@ package com.gu.mediaservice.model.usage +import com.gu.mediaservice.lib.dynamo.{DbString, DynamoElement} import play.api.libs.json.{Json, Reads, Writes} case class ChildUsageMetadata( @@ -11,6 +12,11 @@ case class ChildUsageMetadata( "addedBy" -> addedBy, "childMediaId" -> childMediaId ) + + override def toDynamoMap: Map[String, DynamoElement] = Map( + "addedBy" -> DbString(addedBy), + "childMediaId" -> DbString(childMediaId) + ) } object ChildUsageMetadata { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DigitalUsageMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DigitalUsageMetadata.scala index df6337a5f89..17dc30d31c0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DigitalUsageMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DigitalUsageMetadata.scala @@ -1,5 +1,7 @@ package com.gu.mediaservice.model.usage +import com.gu.mediaservice.lib.dynamo.{DbString, DynamoElement} + import java.net.URI import play.api.libs.json._ import com.gu.mediaservice.syntax._ @@ -18,6 +20,12 @@ case class DigitalUsageMetadata ( "webTitle" -> dynamoSafeWebTitle, "sectionId" -> sectionId ) ++ composerUrl.map("composerUrl" -> _.toString) + + override def toDynamoMap: Map[String, DynamoElement] = Map( + "webUrl" -> DbString(webUrl.toString), + "webTitle" -> DbString(dynamoSafeWebTitle), + "sectionId" -> DbString(sectionId) + ) ++ composerUrl.map(c => "composerUrl" -> DbString(c.toString)) } object DigitalUsageMetadata { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DownloadUsageMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DownloadUsageMetadata.scala index 019968846db..fcc281166a0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DownloadUsageMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/DownloadUsageMetadata.scala @@ -1,5 +1,6 @@ package com.gu.mediaservice.model.usage +import com.gu.mediaservice.lib.dynamo.{DbString, DynamoElement} import play.api.libs.json.{Json, Reads, Writes} case class DownloadUsageMetadata( @@ -8,6 +9,10 @@ case class DownloadUsageMetadata( override def toMap: Map[String, Any] = Map( "downloadedBy" -> downloadedBy ) + + override def toDynamoMap: Map[String, DynamoElement] = Map( + "downloadedBy" -> DbString(downloadedBy) + ) } object DownloadUsageMetadata { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/FrontUsageMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/FrontUsageMetadata.scala index 28edaa86cfb..5ce130a5cc7 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/FrontUsageMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/FrontUsageMetadata.scala @@ -1,5 +1,6 @@ package com.gu.mediaservice.model.usage +import com.gu.mediaservice.lib.dynamo.{DbString, DynamoElement} import play.api.libs.json._ case class FrontUsageMetadata( @@ -10,6 +11,11 @@ case class FrontUsageMetadata( "addedBy" -> addedBy, "front" -> front ) + + override def toDynamoMap: Map[String, DynamoElement] = Map( + "addedBy" -> DbString(addedBy), + "front" -> DbString(front) + ) } object FrontUsageMetadata { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/PrintUsageMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/PrintUsageMetadata.scala index 9cab119fc68..3bfe49f0ad2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/PrintUsageMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/PrintUsageMetadata.scala @@ -1,5 +1,6 @@ package com.gu.mediaservice.model.usage +import com.gu.mediaservice.lib.dynamo.{DbInt, DbLong, DbNestedMap, DbString, DynamoElement} import play.api.libs.json._ import org.joda.time.DateTime @@ -13,6 +14,7 @@ case class PrintImageSize( "x" -> x, "y" -> y ) + override def toDynamoMap: Map[String, DynamoElement] = Map("x" -> DbInt(x), "y" -> DbInt(y)) } object PrintImageSize { implicit val reads: Reads[PrintImageSize] = Json.reads[PrintImageSize] @@ -54,6 +56,22 @@ case class PrintUsageMetadata( edition.foldLeft[IntElement](Nil)((_,i) => List("edition" -> i)) ++ notes.foldLeft[StringElement](Nil)((_,s) => if(s.isEmpty) Nil else List("notes" -> s)) ++ source.foldLeft[StringElement](Nil)((_,s) => if(s.isEmpty) Nil else List("source" -> s)) + + override def toDynamoMap: Map[String, DynamoElement] = Map( + "sectionName" -> DbString(sectionName), + "issueDate" -> DbString(issueDate.toString), + "pageNumber" -> DbInt(pageNumber), + "storyName" -> DbString(storyName), + "publicationCode" -> DbString(publicationCode), + "publicationName" -> DbString(publicationName), + "sectionCode" -> DbString(sectionCode) + ) ++ size.map(s => "size" -> DbNestedMap(s.toDynamoMap)) ++ + orderedBy.map(o => "orderedBy" -> DbString(o)) ++ + layoutId.map(l => "layoutId" -> DbLong(l)) ++ + edition.map(e => "edition" -> DbInt(e)) ++ + notes.map(n => "notes" -> DbString(n)) ++ + source.map(s => "source" -> DbString(s)) + } object PrintUsageMetadata { import JodaWrites._ diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/SyndicationUsageMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/SyndicationUsageMetadata.scala index 2804b828841..c90a63a725d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/SyndicationUsageMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/SyndicationUsageMetadata.scala @@ -1,14 +1,19 @@ package com.gu.mediaservice.model.usage +import com.gu.mediaservice.lib.dynamo.{DbString, DynamoElement} import play.api.libs.json._ case class SyndicationUsageMetadata( partnerName: String, syndicatedBy: Option[String] = None ) extends UsageMetadata { - override def toMap: Map[String, Any] = Map( + override def toMap: Map[String, String] = Map( "partnerName" -> partnerName ) ++ syndicatedBy.map("syndicatedBy" -> _) + + override def toDynamoMap: Map[String, DynamoElement] = Map( + "partnerName" -> DbString(partnerName) + ) ++ syndicatedBy.map(s => "syndicatedBy" -> DbString(s)) } object SyndicationUsageMetadata { diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageMetadata.scala index fc2ee1fdb5f..4ce11c1c533 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/usage/UsageMetadata.scala @@ -1,5 +1,8 @@ package com.gu.mediaservice.model.usage +import com.gu.mediaservice.lib.dynamo.DynamoElement + trait UsageMetadata { def toMap: Map[String, Any] + def toDynamoMap: Map[String, DynamoElement] } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala new file mode 100644 index 00000000000..1a3ce4087ca --- /dev/null +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -0,0 +1,143 @@ +package com.gu.mediaservice.lib + +import org.scalatest.Inspectors +import org.scalatest.OptionValues +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import org.scalacheck.Gen +import VectorUtils.{dotProduct, magnitude, cosineSimilarity, firstBasisVector, vectorWithCosineSimilarity} + + +class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with OptionValues { + val tolerance = 1e-9 + + describe("dotProduct") { + it ("should compute the dot product of two vectors") { + dotProduct(List(1.0, 2.0, 3.0), List(4.0, 5.0, 6.0)) shouldBe 32.0 + } + + it ("should compute a dot product of zero for orthogonal vectors") { + dotProduct(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 + } + + it ("should return zero for the dot product of empty vectors") { + dotProduct(List.empty, List.empty) shouldBe 0.0 + } + + it ("should reject vectors of differing dimensionality") { + an [IllegalArgumentException] should be thrownBy dotProduct(List(1.0, 2.0), List(1.0, 2.0, 3.0)) + } + } + + describe("magnitude") { + it ("should compute the magnitude of a vector") { + magnitude(List(3.0, 4.0)) shouldBe 5.0 + } + + it ("should return a magnitude of zero for a zero vector") { + magnitude(List(0.0, 0.0, 0.0)) shouldBe 0.0 + } + } + + describe("cosineSimilarity") { + it ("should return a cosine similarity of 1 for identical vectors") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)).value shouldBe 1.0 +- tolerance + } + + it ("should return a cosine similarity of 1 for parallel vectors") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)).value shouldBe 1.0 +- tolerance + } + + it ("should return a cosine similarity of 0 for orthogonal vectors") { + cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)).value shouldBe 0.0 +- tolerance + } + + it ("should return a cosine similarity of -1 for opposite vectors") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)).value shouldBe -1.0 +- tolerance + } + + it ("should be undefined (None) when the first vector has zero magnitude") { + cosineSimilarity(List(0.0, 0.0, 0.0), List(1.0, 2.0, 3.0)) shouldBe None + } + + it ("should be undefined (None) when the second vector has zero magnitude") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(0.0, 0.0, 0.0)) shouldBe None + } + + it ("should be undefined (None) when both vectors have zero magnitude") { + cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe None + } + + it ("should be undefined (None) when the vectors differ in dimensionality") { + cosineSimilarity(List(1.0, 2.0), List(1.0, 2.0, 3.0)) shouldBe None + } + } + + // This function is specifically for use in tests, + // where we need to create test fixtures with specific semantic scores. + describe("vectorWithCosineSimilarity") { + val dims = 256 + + it ("should create a vector at the requested cosine similarity across a fine sweep of the valid range") { + val similarities = (-1000 to 1000).map(_ / 1000.0) + forAll(similarities) { s => + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, s)).value shouldBe s +- tolerance + } + } + + it ("should create a vector at the requested cosine similarity for randomly generated similarities") { + val similarityGen = Gen.chooseNum(-1.0, 1.0) + val samples = Gen.listOfN(500, similarityGen).sample.getOrElse(Nil) + samples should not be empty + forAll(samples) { s => + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, s)).value shouldBe s +- tolerance + } + } + + it ("should work across a range of dimensions") { + forAll(List(2, 3, 8, 16, 128, 256, 1024)) { n => + cosineSimilarity(firstBasisVector(n), vectorWithCosineSimilarity(n, 0.42)).value shouldBe 0.42 +- tolerance + } + } + + it ("should return a vector identical to the basis vector for a similarity of 1") { + vectorWithCosineSimilarity(dims, 1.0) shouldBe firstBasisVector(dims) + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, 1.0)).value shouldBe 1.0 +- tolerance + } + + it ("should return a vector opposite to the basis vector for a similarity of -1") { + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, -1.0)).value shouldBe -1.0 +- tolerance + } + + it ("should return a vector orthogonal to the basis vector for a similarity of 0") { + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, 0.0)).value shouldBe 0.0 +- tolerance + } + + it ("should negate the vector when the similarity is negated") { + val positive = vectorWithCosineSimilarity(dims, 0.25) + val negative = vectorWithCosineSimilarity(dims, -0.25) + forAll(positive.zip(negative)) { case (p, n) => n shouldBe (-p) +- tolerance } + } + + it ("should produce only finite components for valid similarities") { + forAll(List(-1.0, -0.5, -1e-3, 0.0, 1e-3, 0.5, 1.0)) { s => + forAll(vectorWithCosineSimilarity(dims, s)) { component => + component.isNaN shouldBe false + component.isInfinite shouldBe false + } + } + } + + it ("should reject a similarity greater than 1") { + an [IllegalArgumentException] should be thrownBy vectorWithCosineSimilarity(dims, 1.0001) + } + + it ("should reject a similarity less than -1") { + an [IllegalArgumentException] should be thrownBy vectorWithCosineSimilarity(dims, -1.0001) + } + + it ("should reject fewer than 2 dimensions") { + an [IllegalArgumentException] should be thrownBy vectorWithCosineSimilarity(1, 0.5) + } + } +} diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/DynamoDBTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/DynamoDBTest.scala index 88c1686b9b1..8e1b6c2cba0 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/DynamoDBTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/DynamoDBTest.scala @@ -1,121 +1,13 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.dynamodbv2.document.spec.UpdateItemSpec -import com.amazonaws.services.dynamodbv2.document.utils.ValueMap -import com.amazonaws.services.dynamodbv2.model.ReturnValue -import com.gu.mediaservice.model.{ActionData, Collection} -import org.joda.time.DateTime import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers -import play.api.libs.json.{Format, JsObject, Json} +import play.api.libs.json.{Format, Json} import scala.jdk.CollectionConverters._ class DynamoDBTest extends AnyFunSpec with Matchers { - describe("jsonToValueMap") { - it ("should convert a simple JsObject to a valueMap") { - val json = Json.toJson(SimpleDynamoDBObj("this is a string", 100, true, List("list"))).as[JsObject] - val valueMap = DynamoDB.jsonToValueMap(json) - - - // This is the only way to get stuff type safely out of the valueMap - // It's not a problem as we shoulnd't be doing this anywhere else - val s: String = valueMap.get("s").asInstanceOf[String] - val d: BigDecimal = valueMap.get("d").asInstanceOf[java.math.BigDecimal] - val b: Boolean = valueMap.get("b").asInstanceOf[Boolean] - val a: List[String] = valueMap.get("a").asInstanceOf[java.util.ArrayList[String]].toArray().toList.map(_.asInstanceOf[String]) - - - s should be ("this is a string") - d should be (100) - b should equal(true) - a should equal(List("list")) - } - - it ("should convert a nested JsObject to a valueMap") { - val nestedObj = NestedDynamoDBObj("string", 100, false, SimpleDynamoDBObj("strang", 500, true, List("list"))) - val json = Json.toJson(nestedObj).as[JsObject] - val valueMap = DynamoDB.jsonToValueMap(json) - - val ss: String = valueMap.get("ss").asInstanceOf[String] - val dd: BigDecimal = valueMap.get("dd").asInstanceOf[java.math.BigDecimal] - val bb: Boolean = valueMap.get("bb").asInstanceOf[Boolean] - - val simpleMap = valueMap.get("simple").asInstanceOf[ValueMap] - - val s: String = simpleMap.get("s").asInstanceOf[String] - val d: BigDecimal = simpleMap.get("d").asInstanceOf[java.math.BigDecimal] - val b: Boolean = simpleMap.get("b").asInstanceOf[Boolean] - val a: List[String] = simpleMap.get("a").asInstanceOf[java.util.ArrayList[String]].toArray().toList.map(_.asInstanceOf[String]) - - - ss should be ("string") - dd should be (100) - bb should equal(false) - - s should be ("strang") - d should be (500) - b should equal(true) - a should equal(List("list")) - } - - it ("should convert a Collection to ValueMap") { - val collection = Collection.build(List("g2", "art", "batik"), ActionData("mighty.mouse@guardian.co.uk", DateTime.now)) - val json = Json.toJson(collection).as[JsObject] - val valueMap = DynamoDB.jsonToValueMap(json) - - val path = valueMap.get("path").asInstanceOf[java.util.ArrayList[String]].toArray.toList.asInstanceOf[List[String]] - val pathId = valueMap.get("pathId").asInstanceOf[String] - val actionData = valueMap.get("actionData").asInstanceOf[ValueMap] - val author = actionData.get("author").asInstanceOf[String] - - pathId should be (collection.pathId) - author should be (collection.actionData.author) - path should be (collection.path) - } - } - - describe("addLastModifiedUpdate") { - val testTime = new DateTime(2021, 3, 22, 14, 58) - - it ("should prefix a lastModified update to existing SET expression") { - val input = new UpdateItemSpec() - .withUpdateExpression("SET anotherKey = :anotherValue") - val output = DynamoDB.addLastModifiedUpdate(input, "lm", testTime) - output.getUpdateExpression shouldBe "SET lm = :lm, anotherKey = :anotherValue" - } - - it ("should prefix a lastModified SET clause to existing ADD expression") { - val input = new UpdateItemSpec() - .withUpdateExpression("ADD anotherKey :anotherValue") - val output = DynamoDB.addLastModifiedUpdate(input, "lm", testTime) - output.getUpdateExpression shouldBe "SET lm = :lm ADD anotherKey :anotherValue" - } - - it ("should add a date time to an existing value map") { - val valueMap = new ValueMap() - valueMap.put(":anotherValue", "anotherValue") - val input = new UpdateItemSpec() - .withUpdateExpression("banana") - .withValueMap(valueMap) - val output = DynamoDB.addLastModifiedUpdate(input, "lm", testTime) - output.getValueMap.asScala shouldBe Map( - ":anotherValue" -> "anotherValue", - ":lm" -> testTime.toString - ) - } - - it ("should add a date time when there is no value map") { - val input = new UpdateItemSpec() - .withUpdateExpression("banana") - val output = DynamoDB.addLastModifiedUpdate(input, "lm", testTime) - output.getValueMap.asScala shouldBe Map( - ":lm" -> testTime.toString - ) - } - } - describe("removeExpr") { it("should generate the correction expression when the lastModified is set") { DynamoDB.removeExpr("key", Some("lastModifiedKey")) shouldBe "REMOVE key SET lastModifiedKey = :lastModifiedKey" diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala index c08c6d3c098..6bf2c57564b 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala @@ -1,7 +1,7 @@ package com.gu.mediaservice.lib.aws import org.scalatest.funsuite.AnyFunSuiteLike -import com.amazonaws.services.sqs.model.{Message => SQSMessage} +import software.amazon.awssdk.services.sqs.model.Message import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import scala.io.Source @@ -12,8 +12,7 @@ class SqsHelpersTest extends AnyFunSuiteLike with SqsHelpers { test("extractS3KeyFromSqsMessage") { - val message = new SQSMessage() - message.setBody(Source.fromResource("s3SqsMessage.json").mkString) + val message = Message.builder().body(Source.fromResource("s3SqsMessage.json").mkString).build() extractS3KeyFromSqsMessage(message) shouldBe Success("bill.bloggs@example.co.uk/0cd747d665e2c59e86d256e6f45c369664e558bd") } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala index 3cb7da390a6..bb75dca2bd8 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala @@ -68,6 +68,18 @@ class SupplierProcessorsTest extends AnyFunSpec with Matchers with MetadataHelpe processedImage.usageRights should be(ContractPhotographer("Murdo MacLeod", Option("The Guardian"))) processedImage.metadata.byline should be(Some("Murdo MacLeod")) } + + it("should only assign Photographer rights if agency rights not already assigned") { + val image = createImageFromMetadata( + "byline" -> "Murdo MacLeod", + "credit" -> "PA Wire/PA Images" + ) + val processedImage = applyProcessors(image) + // UsageRights assigned to PA agency rather than ContractPhotographer("Murdo MacLeod") as PA + // runs first, and Staff/ContractPhotographer only assign rights if nothing else has first. + processedImage.usageRights should be(Agency("PA")) + processedImage.metadata.byline should be(Some("Murdo MacLeod")) + } } describe("AAP") { diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala new file mode 100644 index 00000000000..e109364e7b6 --- /dev/null +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala @@ -0,0 +1,178 @@ +package com.gu.mediaservice.lib.usage + +import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, DefaultAttributeConverterProvider, EnhancedType} +import com.gu.mediaservice.model.usage.{ChildUsageMetadata, DigitalUsageMetadata, DownloadUsageMetadata, FrontUsageMetadata, MediaUsage, PrintImageSize, PrintUsageMetadata, SyndicationUsageMetadata, UsageId, UsageStatus, UsageType} +import com.gu.mediaservice.lib.AttributeValueConverterProvider +import org.joda.time.DateTime +import org.scalatest.funsuite.AnyFunSuiteLike +import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper +import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument +import software.amazon.awssdk.services.dynamodb.model.AttributeValue + +import scala.jdk.CollectionConverters.MapHasAsJava +import java.math.BigDecimal +import java.net.URI + +class ItemToMediaUsageTest extends AnyFunSuiteLike { + + val lastModifiedMillis = 1700000000000L + val dateAddedMillis = 1700000100000L + val dateRemovedMillis = 1700000200000L + val printSizeMap = new java.util.LinkedHashMap[String, java.math.BigDecimal]() + printSizeMap.put("x", new BigDecimal(100)) + printSizeMap.put("y", new BigDecimal(200)) + + val printMetadata = Map[String, Any]( + "sectionName" -> "News", + "issueDate" -> "2026-07-20T10:00:00.000Z", + "pageNumber" -> new BigDecimal(5), + "storyName" -> "Lead Story", + "publicationCode" -> "GND", + "publicationName" -> "The Guardian", + "layoutId" -> new BigDecimal(12), + "edition" -> new BigDecimal(1), + "size" -> printSizeMap, + "orderedBy" -> "editor1", + "sectionCode" -> "GND-NEWS", + "notes" -> "Top priority", + "source" -> "Desk" + ).asJava + + val printMetadataAttr = Map[String, AttributeValue]( + "sectionName" -> AttributeValue.builder().s("News").build(), + "issueDate" -> AttributeValue.builder().s("2026-07-20T10:00:00.000Z").build(), + "pageNumber" -> AttributeValue.builder().n(new BigDecimal(5).toString).build(), + "storyName" -> AttributeValue.builder().s("Lead Story").build(), + "publicationCode" -> AttributeValue.builder().s("GND").build(), + "publicationName" -> AttributeValue.builder().s("The Guardian").build(), + "layoutId" -> AttributeValue.builder().n(new BigDecimal(12).toString).build(), + "edition" -> AttributeValue.builder().n(new BigDecimal(1).toString).build(), + "size" -> AttributeValue.builder().m(Map( + "x" -> AttributeValue.builder().n(new BigDecimal(100).toString).build(), + "y" -> AttributeValue.builder().n(new BigDecimal(200).toString).build() + ).asJava).build(), + "orderedBy" -> AttributeValue.builder.s("editor1").build(), + "sectionCode" -> AttributeValue.builder.s("GND-NEWS").build(), + "notes" -> AttributeValue.builder.s("Top priority").build(), + "source" -> AttributeValue.builder.s("Desk").build() + ).asJava + + val digitalMetadata = Map[String, String]( + "webUrl" -> "https://www.theguardian.com/world/2026/jul/20/article", + "webTitle" -> "Breaking News", + "sectionId" -> "world", + "composerUrl" -> "https://composer.gutools.co.uk/content/123" + ).asJava + + val syndicationMetadata = Map[String, String]( + "partnerName" -> "Reuters", + "syndicatedBy" -> "syndication_user" + ).asJava + + val frontMetadata = Map[String, String]( + "addedBy" -> "front_editor", + "front" -> "uk-news" + ).asJava + + val downloadMetadata = Map[String, String]( + "downloadedBy" -> "designer123" + ).asJava + + val childMetadata = Map[String, String]( + "addedBy" -> "parent_editor", + "childMediaId" -> "child-media-999" + ).asJava + + test("testTransform dynamo v2 with simple fields") { + val enchancedDoc = EnhancedDocument.builder() + .attributeConverterProviders(DefaultAttributeConverterProvider.create()) + .putString("usage_id", "usage-123") + .putString("grouping","group-abc") + .putString("media_id", "media-789") + .putString("usage_type", "print") + .putString("media_type", "image") + .putString("usage_status", "pending") + .putNumber("last_modified", lastModifiedMillis) + .putNumber("date_added", dateAddedMillis) + .putNumber("date_removed", dateRemovedMillis) + .build() + + val mediaUsage = ItemToMediaUsage.transform(enchancedDoc) + + mediaUsage shouldEqual MediaUsage( + UsageId("usage-123"), + "group-abc", + "media-789", + UsageType("print"), + "image", + UsageStatus("pending"), + None, + None, + None, + None, + None, + None, + new DateTime(lastModifiedMillis), + Some(new DateTime(dateAddedMillis)), + Some(new DateTime(dateRemovedMillis)) + ) + } + + test("testTransform dynamo v2 with complex fields") { + val enchancedDoc = EnhancedDocument.builder() + .attributeConverterProviders( + AttributeValueConverterProvider, + AttributeConverterProvider.defaultProvider() + ) + .putString("usage_id", "usage-123") + .putString("grouping","group-abc") + .putString("media_id", "media-789") + .putString("usage_type", "print") + .putString("media_type", "image") + .putString("usage_status", "pending") + .putNumber("last_modified", lastModifiedMillis) + .putNumber("date_added", dateAddedMillis) + .putNumber("date_removed", dateRemovedMillis) + .putMap("digital_metadata", digitalMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .putMap("syndication_metadata", syndicationMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .putMap("front_metadata", frontMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .putMap("print_metadata", printMetadataAttr, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[AttributeValue])) + .putMap("download_metadata", downloadMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .putMap("child_metadata", childMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .build() + + val mediaUsage = ItemToMediaUsage.transform(enchancedDoc) + + mediaUsage.downloadUsageMetadata shouldEqual Some(DownloadUsageMetadata("designer123")) + mediaUsage.childUsageMetadata shouldEqual Some(ChildUsageMetadata("parent_editor", "child-media-999")) + mediaUsage.printUsageMetadata shouldEqual Some(PrintUsageMetadata( + sectionName = "News", + issueDate = new DateTime("2026-07-20T10:00:00.000Z"), + pageNumber = 5, + storyName = "Lead Story", + publicationCode = "GND", + publicationName = "The Guardian", + layoutId = Some(12), + edition = Some(1), + size = Some(PrintImageSize(100, 200)), + orderedBy = Some("editor1"), + sectionCode = "GND-NEWS", + notes = Some("Top priority"), + source = Some("Desk") + )) + mediaUsage.syndicationUsageMetadata shouldEqual Some( + SyndicationUsageMetadata("Reuters", Some("syndication_user")) + ) + mediaUsage.frontUsageMetadata shouldEqual Some(FrontUsageMetadata("front_editor", "uk-news")) + mediaUsage.digitalUsageMetadata shouldEqual Some( + DigitalUsageMetadata( + URI.create("https://www.theguardian.com/world/2026/jul/20/article"), + "Breaking News", + "world", + Some(URI.create("https://composer.gutools.co.uk/content/123")) + ) + ) + + } + +} diff --git a/common-lib/src/test/scala/com/gu/mediaservice/model/FileMetadataTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/model/FileMetadataTest.scala new file mode 100644 index 00000000000..f6a5f48b81f --- /dev/null +++ b/common-lib/src/test/scala/com/gu/mediaservice/model/FileMetadataTest.scala @@ -0,0 +1,49 @@ +package com.gu.mediaservice.model + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import play.api.libs.json.{JsObject, JsSuccess, Json} + +class FileMetadataTest extends AnyFunSpec with Matchers { + + private def baseJson(extraFields: (String, Json.JsValueWrapper)*): JsObject = Json.obj( + "iptc" -> Json.obj(), + "exif" -> Json.obj(), + "exifSub" -> Json.obj(), + "xmp" -> Json.obj() + ) ++ Json.obj(extraFields: _*) + + describe("c2pa Reads") { + it("should read an available C2PA manifest") { + val json = baseJson("c2pa" -> Json.obj("isAvailable" -> true)) + + json.validate[FileMetadata] shouldBe JsSuccess(FileMetadata(c2pa = FileMetadata.C2paAvailable)) + } + + it("should read an explicitly empty c2pa object as unavailable") { + val json = baseJson("c2pa" -> Json.obj()) + + json.validate[FileMetadata] shouldBe JsSuccess(FileMetadata(c2pa = FileMetadata.NoC2PA)) + } + + it("should default to unavailable when the c2pa key is absent entirely (e.g. a legacy document indexed before this field existed)") { + val json = baseJson() + + json.validate[FileMetadata] shouldBe JsSuccess(FileMetadata(c2pa = FileMetadata.NoC2PA)) + } + } + + describe("c2pa Writes") { + it("should write an available C2PA manifest") { + val fileMetadata = FileMetadata(c2pa = FileMetadata.C2paAvailable) + + (Json.toJson(fileMetadata) \ "c2pa").as[JsObject] shouldBe Json.obj("isAvailable" -> true) + } + + it("should write an unavailable C2PA manifest as an empty object") { + val fileMetadata = FileMetadata(c2pa = FileMetadata.NoC2PA) + + (Json.toJson(fileMetadata) \ "c2pa").as[JsObject] shouldBe Json.obj() + } + } +} diff --git a/dev/oidc-provider/package-lock.json b/dev/oidc-provider/package-lock.json index 5a792a9a130..52a18029516 100644 --- a/dev/oidc-provider/package-lock.json +++ b/dev/oidc-provider/package-lock.json @@ -3134,16 +3134,15 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "5.1.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.11.tgz", - "integrity": "sha512-v+KEsUv2ps74PaSKv0gHTxTCgMXOIfBEbaqa6w6ISIGC7ZsvHN4N9oJ8d4cmf0n5oTzQz2SLmThbQWhjd/8eKg==", + "version": "5.1.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.16.tgz", + "integrity": "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], - "license": "MIT", "bin": { "nanoid": "bin/nanoid.js" }, @@ -3384,10 +3383,11 @@ "dev": true }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, + "license": "MIT", "engines": { "node": ">=8.6" }, @@ -6362,9 +6362,9 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "nanoid": { - "version": "5.1.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.11.tgz", - "integrity": "sha512-v+KEsUv2ps74PaSKv0gHTxTCgMXOIfBEbaqa6w6ISIGC7ZsvHN4N9oJ8d4cmf0n5oTzQz2SLmThbQWhjd/8eKg==" + "version": "5.1.16", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.16.tgz", + "integrity": "sha512-kVrnsrJqMR8+oLJnGEmSWw9BivK5mt7H3FZatVRjrc5wGqFYuBxX1yG7+A7Gi5AefkX6t/oCkizcQgpu0cY1dQ==" }, "natural-compare": { "version": "1.4.0", @@ -6535,9 +6535,9 @@ "dev": true }, "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true }, "pirates": { diff --git a/dev/script/create-root-collection.sh b/dev/script/create-root-collection.sh index 6ae21d714ef..dbb42e1dc7c 100755 --- a/dev/script/create-root-collection.sh +++ b/dev/script/create-root-collection.sh @@ -5,18 +5,54 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROOT_DIR=${DIR}/../.. -COLLECTION_NAME=${1:-TEST} +while [[ $# -gt 0 ]] && [[ "$1" == "--"* ]] ; +do + opt="$1"; + shift; + case "$opt" in + "--api-key" ) + api_key="$1"; shift;; + "--domain" ) + domain="$1"; shift;; + "--name" ) + name="$1"; shift;; + * ) + echo >&2 "Invalid option: $@"; exit 1;; + esac +done -# load values from .env into environment variables -# see https://stackoverflow.com/a/30969768/3868241 -set -o allexport -# shellcheck source=../.env -source "$ROOT_DIR/dev/.env" -set +o allexport +if [[ -z "$api_key" && -z "$domain" ]]; then + echo >&2 "API key and domain unset, setting from dotenv file" + # load values from .env into environment variables + # see https://stackoverflow.com/a/30969768/3868241 + set -o allexport + # shellcheck source=../.env + source "$ROOT_DIR/dev/.env" + set +o allexport +fi -echo "creating root collection $COLLECTION_NAME" +if [[ -z "$api_key" || -z "$domain" || -z "$name" ]]; then + echo >&2 "Missing setting(s). -curl -s -X POST -H "X-Gu-Media-Key: $API_KEY" \ +Usage: $0 --api-key --domain --name + + --api-key API key for authenticating grid access + --domain Root domain for grid instance (eg. local.dev-gutools.co.uk) + --name Name of new root collection" + exit 1 +fi + +echo "creating root collection $name" + +curl -s -X POST -H "X-Gu-Media-Key: $api_key" \ -H "Content-Type: application/json" \ - -d '{ "data": "'"$COLLECTION_NAME"'" }' \ - "https://media-collections.$DOMAIN/collections" + -d '{ "data": "'"$name"'" }' \ + "https://media-collections.$domain/collections" + +outcome="$?" + +if [[ "$outcome" == "0" ]]; then + echo "collection created" +else + echo "creation failed..." +fi diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js index 2424dac951f..b5820bba78e 100644 --- a/dev/script/generate-config/service-config.js +++ b/dev/script/generate-config/service-config.js @@ -10,7 +10,27 @@ function getCorsAllowedOriginString(config) { } function getCommonConfig(config) { - const isNoAuth = process.env.NO_AUTH === "true"; + // AI search depends on Bedrock/vector-ES infrastructure that isn't wired up for every org, + // so it defaults to false (see CommonConfig.scala) and is only turned on here for Guardian's + // own local dev environment. Shared by media-api and kahuna via the common config key. + const aiSearchEnabled = process.env.BUILD_ORG ? false : true; + + // `NO_AUTH` is shorthand that disables both authentication and authorisation. + // The two can also be controlled independently via `NO_AUTHENTICATION` and + // `NO_AUTHORISATION`. + // + // Each flag can be supplied on the `config` object or via the equivalent + // environment variable (used by the dev `setup.sh` flow). + const flag = (name) => config[name] === true || process.env[name] === "true"; + const isNoAuth = flag("NO_AUTH"); + const isNoAuthentication = isNoAuth || flag("NO_AUTHENTICATION"); + const isNoAuthorisation = isNoAuth || flag("NO_AUTHORISATION"); + + // When the real authorisation provider is in use and a permissions bucket has been + // provisioned locally (localstack), point the provider at it and enable the local-auth + // code path so it reads `permissions.json` from that bucket instead of Guardian infra. + const useLocalPermissions = !isNoAuthorisation && Boolean(config.coreStackProps.PermissionsBucket); + return `domain.root="${config.DOMAIN}" |authentication.providers.machine.config.authKeyStoreBucket="${config.coreStackProps.KeyBucket}" |aws.local.endpoint="https://localstack.media.${config.DOMAIN}" @@ -18,12 +38,15 @@ function getCommonConfig(config) { |thrall.kinesis.lowPriorityStream.name="${config.coreStackProps.ThrallLowPriorityMessageStream}" |es.index.aliases.current="Images_Current" |es.index.aliases.migration="Images_Migration" - |image.record.download=false + |image.record.download=true |defaultShouldBlurGraphicImages=true |filters.shouldDisplayOrgOwnedCountAndFilterCheckbox=true + |ai.search.enabled=${aiSearchEnabled} |dynamo.table.softDelete.metadata="SoftDeletedMetadataTable" - ${isNoAuth ? '|authentication.providers.user="com.gu.mediaservice.lib.auth.provider.LocalAuthenticationProvider"' : ''} - ${isNoAuth ? '|authorisation.provider="com.gu.mediaservice.lib.auth.provider.LocalAuthorisationProvider"' : ''} + ${isNoAuthentication ? '|authentication.providers.user="com.gu.mediaservice.lib.auth.provider.LocalAuthenticationProvider"' : ''} + ${isNoAuthorisation ? '|authorisation.provider="com.gu.mediaservice.lib.auth.provider.LocalAuthorisationProvider"' : ''} + ${useLocalPermissions ? '|auth.useLocal=true' : ''} + ${useLocalPermissions ? `|permissions.bucket="${config.coreStackProps.PermissionsBucket}"` : ''} |sqs.ingest.queue.url="${config.coreStackProps.IngestSqsQueue.replace("http://localhost:4576", `https://localstack.media.${config.DOMAIN}`)}" |s3.ingest.bucket="${config.coreStackProps.IngestQueueBucket}" |s3.fail.bucket="${config.coreStackProps.IngestQueueFailBucket}" diff --git a/docs/02-running/01-running-locally.md b/docs/02-running/01-running-locally.md index 99593002620..25e707d5a56 100644 --- a/docs/02-running/01-running-locally.md +++ b/docs/02-running/01-running-locally.md @@ -59,6 +59,13 @@ If you haven't used the `--with-local-auth` flag, you'll begin a Google authenti If you have used the `--no-auth` flag you'll be signed in automatically as John Doe (with no oauth flow) and will have ALL permissions. +### Auth Failed Errors +Sometimes you will see a red notification relating to a failure to auth. Visiting: + +https://media.test.dev-gutools.co.uk/search + +and refreshing the page will resolve this. + ## Cerebro To inspect the elasticsearch database, cerebro is included as part of the grids local stack. diff --git a/docs/03-apis/02-collections.md b/docs/03-apis/02-collections.md index 227c1018b89..72e2fd2c462 100644 --- a/docs/03-apis/02-collections.md +++ b/docs/03-apis/02-collections.md @@ -14,10 +14,11 @@ ROOT COLLECTION 2 ``` ## Root collections + Currently, root collections can only be created via the API. That is, Kahuna doesn't have a UI for it. We can use [create-root-collection.sh](../../dev/script/create-root-collection.sh) to create a root collection: ```shell script -./dev/script/create-root-collection.sh COLLECTION_NAME +./dev/script/create-root-collection.sh --api-key [api key] --domain [root domain] --name [collection name] ``` diff --git a/e2e-tests/README.md b/e2e-tests/README.md new file mode 100644 index 00000000000..29ccd1cbc7e --- /dev/null +++ b/e2e-tests/README.md @@ -0,0 +1,81 @@ +# End-to-end tests + +Playwright + [`playwright-bdd`](https://vitalets.github.io/playwright-bdd/) e2e tests that +run against a full Grid stack. Scenarios live in [`features/`](features) (Gherkin) with the +step definitions in [`steps/`](steps). + +## Running the tests + +### 1. CI (production mode) + +This is what [`.github/workflows/playwright.yml`](../.github/workflows/playwright.yml) does, +and how you reproduce a CI failure locally: + +```bash +# From the repo root: build the production image the harness runs. +DOCKER_BUILDKIT=1 docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci . + +cd e2e-tests +npm ci +npx playwright install --with-deps chromium +npm test +``` + +`grid-e2e-ci` ([`images/Dockerfile`](images/Dockerfile)) stages the services with `sbt stage` +and runs them under a production JRE, so it exercises the same code paths as a deployed +Grid. There is no dev-nginx in CI, so when `CI=true` `global-setup` also starts a bundled +Caddy reverse proxy on `:443` that replays dev-nginx's subdomain routing for the +`https://*.media.` browser origins. + +### 2. Dev (live recompilation) + +For iterating on failing tests you can run the services from source with live reload using +the dev image, which runs +`sbt /run` (recompiling on change) and rebuilds Kahuna's webpack bundle via +`npm run watch`. See [`images/README.md`](images/README.md) for how to build and run it. + +Useful commands: + +```bash +npm run test:headed # run with a visible browser +npm run test:report # open the last HTML report +npm run test:ui # open browser and test suite, run tests at your leisure +``` + +Traces are captured `on-first-retry` (see [`playwright.config.ts`](playwright.config.ts)), +so a failed test on CI leaves a trace you can open with `npx playwright show-trace`. + +## Running the stack without the tests + +`npm run dev:e2e` boots exactly the same stack the tests use, prints the service URLs and +holds it open until you press Ctrl-C, which tears it all down. + +```bash +npm run dev:e2e # uses your local dev-nginx for the https://*.media. domains +GRID_PROXY=true npm run dev:e2e # no dev-nginx? start the bundled Caddy proxy on :443 instead +``` + +The stack binds fixed host ports, so two stacks cannot run +at once. Starting a second one fails immediately rather than timing out. + +### Running the tests against a stack you already started + +The test commands reuse a running stack instead of booting their own, which turns a +multi-minute boot into a couple of seconds. Leave `npm run dev:e2e` running in one +terminal, then use `npm test`, `npm run test:ui` or any of the others as normal — they +attach automatically and leave the stack running when they finish. + +If only some services are up (usually because the stack is still booting), the run stops +straight away and names the ports it is waiting on. + +| Variable | Effect | +| --- | --- | +| `GRID_RESEED=true` | Reload the Elasticsearch fixtures into the reused stack. | + +**Watch out for stale provisioning.** A reused stack picks up Scala changes (the repo is +bind-mounted and services run under `sbt run`), but *not* changes to anything applied at +boot: generated service config, the CloudFormation template, bucket contents, permissions +or the Elasticsearch fixtures. After changing any of those, restart `dev:e2e`. + +Reuse also means state carries over between runs. The current suite is read-only, so this +is harmless today, but a test that uploads or edits an image will want a fresh stack. diff --git a/e2e-tests/features/kahuna.feature b/e2e-tests/features/kahuna.feature new file mode 100644 index 00000000000..d99ec3ece8a --- /dev/null +++ b/e2e-tests/features/kahuna.feature @@ -0,0 +1,12 @@ +Feature: Kahuna service availability + + Smoke checks that the Kahuna service, booted by Testcontainers in global-setup, + is serving requests against the provisioned infrastructure. + + Scenario: Healthcheck responds OK + When I request the Kahuna healthcheck endpoint + Then the response is successful + + Scenario: Kahuna serves the application + When I request the Kahuna root path without following redirects + Then the response status is below 500 diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature new file mode 100644 index 00000000000..c9be5cf33a4 --- /dev/null +++ b/e2e-tests/features/upload.feature @@ -0,0 +1,427 @@ +# Uploads mutate the single shared e2e user's upload history, so scenarios must not race. +@mode:serial +Feature: Uploading images to the Grid + This lets an authorised user get images into the Grid by selecting files, + dragging and dropping files or URLs, importing Witness contributions, and + then completing the required metadata while tracking each upload's progress. + + Background: + Given the application stack is running + And I have opened the image upload page + + # --------------------------------------------------------------------------- + # Upload page shell (view.html + controller.js) + # --------------------------------------------------------------------------- + + Scenario: An authorised user sees the upload tools + Given I am permitted to upload images + When the upload page loads + Then I should see the file upload prompt + And I should see my past 50 uploads + And the drag-and-drop uploader should be active + And I should not see the current uploads section + # Evidence: kahuna/public/js/upload/view.html lines 11-12, 19-27, 31 + # Evidence: kahuna/public/js/upload/controller.js lines 40-45 + + # Needs a stack booted with enforceUploadImagesPermission=true. + @todo + Scenario: An unauthorised user is told they cannot upload + Given I am not permitted to upload images + When the upload page loads + Then I should see a message that I am not authorised to upload images + And I should see a link to email support + And I should not see the file upload prompt + And I should not see the drag-and-drop uploader + # Evidence: kahuna/public/js/upload/view.html lines 11, 29 + # Evidence: kahuna/public/js/upload/controller.js lines 36-41 + + Scenario: Returning to search from the upload page + Given I had searched for "cats" before opening the upload page + When I choose "Back to search" from the top bar + Then I should be taken to the image search page + And my previous search should be intact + # Evidence: kahuna/public/js/upload/view.html lines 3-6 + + Scenario: The current uploads section appears while an upload is running + Given I have an upload in progress + When the upload page loads + Then I should see my current uploads section + # Evidence: kahuna/public/js/upload/view.html lines 14-17 + # Evidence: kahuna/public/js/upload/controller.js lines 45 + + Scenario: Viewing all of my uploads + When I choose "View all your uploads" + Then I should be taken to a search filtered to images I uploaded + # Evidence: kahuna/public/js/upload/view.html lines 22-23 + + # Currently unreachable: none of these controls reach the controller's own confirm. + # "Back to search" is a ui-sref, and ui-router destroys the UploadCtrl scope before + # broadcasting $locationChangeStart; "Home" and "View all your uploads" are plain hrefs, + # so the browser shows its own generic beforeunload prompt instead. + @todo + Scenario: Warning before leaving the page with uploads in progress + Given I have an upload in progress + When I try to navigate away from the upload page via the following buttons: + | Back to search | + | Home | + | View all your uploads | + Then I should be warned that uploads are in progress and asked to confirm + # Evidence: kahuna/public/js/upload/controller.js lines 18-34, 47-54 + + # --------------------------------------------------------------------------- + # File upload prompt (prompt/prompt.html + prompt.js) + # --------------------------------------------------------------------------- + + Scenario: The prompt explains how to upload + When the upload page loads + Then I should see a message telling me to drag and drop or click to upload to the system + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 1-3 + # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 20 + + @todo + Scenario: The prompt suggests an example label when no labels are applied + Given I have not applied any preset labels + When the upload page loads + Then I should see a suggested example label to apply to all uploads + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 + # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 21 + + @todo + Scenario: Preset labels are applied to all uploads + When I add a preset label via the 'apply label to all uploads' button + Then that label should be applied to all my uploads + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 43, 113-114 + + @todo + Scenario: Preset labels are stored in local storage + When I add preset label(s) in the prompt + Then those label(s) should already be selected when I reload the page + + # --------------------------------------------------------------------------- + # Select-files uploader (file-uploader.html + file-uploader.js) + # --------------------------------------------------------------------------- + + Scenario: Opening the file picker from the Upload button + When I click the "Upload" button + Then the system file picker should open + # Evidence: kahuna/public/js/upload/file-uploader.html lines 9-15 + # Evidence: kahuna/public/js/upload/file-uploader.js lines 41-42 + + Scenario: Selecting files queues them for upload + When I select one or more image files to upload + Then those files should be queued for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/file-uploader.html lines 4-7 + # Evidence: kahuna/public/js/upload/file-uploader.js lines 20-26 + # Evidence: kahuna/public/js/upload/manager.js lines 68-83 + + Scenario: Files above the size limit are skipped with a warning + Given an upload size limit is configured + When I select a file that is larger than the size limit + Then I should be warned that the oversized file will be skipped + And only the files within the limit should be queued for upload + # Evidence: kahuna/public/js/upload/manager.js lines 36-47 + + # --------------------------------------------------------------------------- + # Drag-and-drop uploader (dnd-uploader.html + dnd-uploader.js) + # --------------------------------------------------------------------------- + + @todo + Scenario: Dragging valid content over the page shows the dropzone + When I drag files over the upload page + Then the dropzone overlay should appear with an explanation + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1-6 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 113, 116, 158-171 + + @todo + Scenario: The dropzone hides when I stop dragging + Given the dropzone overlay is showing + When I drag away from the upload page + Then the dropzone overlay should disappear + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 117, 173-180 + + @todo + Scenario: Dropping files uploads them + When I drop one or more image files onto the page + Then those files should be queued for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 33-37, 182-205 + # Evidence: kahuna/public/js/upload/manager.js lines 68-83 + + @todo + Scenario: Dropping a URL loads the image from that URL + When I drop an image URL onto the page + Then the image at that URL should be loaded for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 64-66, 219-220 + # Evidence: kahuna/public/js/upload/manager.js lines 86-96 + + # can remove this functionality in another PR + @todo + Scenario: Dropping a Witness contribution imports it + When I drop a Witness contribution URL onto the page + Then the importing overlay should be shown + And the Witness image should be imported with its metadata and usage rights + And I should be taken to the imported image's page + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 8-12 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 40-62, 69-78, 206-217 + + # can remove this functionality in another PR + @todo + Scenario: A failed Witness import is reported + Given I drop a Witness contribution URL onto the page + When the Witness import fails + Then I should see an alert that importing the Witness contribution failed + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 79-85 + + # not implemented afaics + @todo + Scenario: Dropping invalid content is rejected + When I drop something that is not a valid file or URL + Then I should see an alert that I must drop valid files or URLs + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 222-231 + + @todo + Scenario: Dragging a Grid image back onto the page is ignored + When I drag an image that is already in the Grid over the page + Then the dropzone overlay should not appear + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 135-150, 158-160 + + # --------------------------------------------------------------------------- + # Current uploads list (jobs/upload-jobs.html + upload-jobs.js) + # --------------------------------------------------------------------------- + + @todo + Scenario: Current uploads show how many remain + Given I have several uploads in progress + When I view my current uploads + Then I should see a count of how many uploads remain + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 2 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 45, 79, 88 + + @todo + Scenario: An uploading job shows a preview with its name and size + Given a file is uploading + When I view my current uploads + Then I should see a preview thumbnail with the file name and size + And I should see the job's status + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 9-31 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 61-72 + # Evidence: kahuna/public/js/upload/manager.js lines 16-21 + + @todo + Scenario: A failed upload shows the error and can be removed + Given an upload has failed + When I view my current uploads + Then the job should be marked as an upload error with the error message + And I should be able to remove the failed job after confirming + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 24-38 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 77-78, 179-185 + + @todo + Scenario: An unsupported file type gives a helpful error + When I upload a file that is not a JPG, PNG or TIFF + Then the job should show an error explaining only JPG, PNG and TIFF are supported + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 155-163 + + @todo + Scenario: A completed upload becomes an editable image + Given an upload has completed + When I view my current uploads + Then the job should switch to the image metadata editor + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 40-48 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 84-92 + + @todo + Scenario: A missing description defaults to the file name + Given an uploaded image has no description + When the upload completes + Then the description should default to the file name without its extension + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 105-110 + + @todo + Scenario: Deleting an image removes it from current uploads + Given an uploaded image is shown in my current uploads + When the image is deleted using the delete button at the bottom of the job form + Then it should be removed from my current uploads + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + + @todo + Scenario: Uploading a previously deleted image displays it for undeletion + Given an image is uploaded + And then deleted using the delete button at the bottom of the job form + And the same image is uploaded again + And I have delete permission + Then it should be present in my current uploads + And I should be able to undelete it + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + + # Existing bug: the UI still presents this button even if permission to delete is not present + @todo + Scenario: Uploading a previously deleted image uploaded by a different user does not display undeletion button without the appropriate permission + Given an image is uploaded by a user that is not me + And then deleted + And the same image is uploaded again by me + And I do not have delete permissions + Then it should be present in current uploads + And I should see the undelete button + And the undelete button is disabled + And there should be a message indicating that I do not have permission to undelete + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + + @todo + Scenario: A failed deletion is reported + Given an uploaded image is shown in my current uploads + When deleting the image fails + Then I should see an alert explaining the deletion failed + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 197-203 + + # --------------------------------------------------------------------------- + # Required metadata editor (jobs/required-metadata-editor.html + .js) + # --------------------------------------------------------------------------- + + @todo + Scenario: Editing required metadata for an uploaded image + Given an uploaded image is shown in the metadata editor + When I fill in the description, byline and credit + Then the metadata should be saved automatically + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 1, 30-56, 61-72, 85-104 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 33, 40-65 + + @todo + Scenario: Description and credit are required + Given an uploaded image is shown in the metadata editor + When I leave the description or credit empty + Then those fields should be marked as required + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 36, 96 + + @todo + Scenario: The description placeholder gives guidance + Given an uploaded image with no description + When I view the description field + Then I should see placeholder guidance about who, what, where, when and why + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 34-40 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 157-161 + + # How to verify writes? + @todo + Scenario: Choosing an image type when image types are configured + Given image types are configured + When I view the metadata editor + Then I should be able to choose an image type from a dropdown + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 5-27 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 38 + + @todo + Scenario: The credit field suggests existing values + Given an uploaded image is shown in the metadata editor + When I type into the credit field + Then I should see suggestions matching existing credits + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 87-104 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 67-71 + + # How to verify writes? + @todo + Scenario: Metadata only shows when it was already present for some fields + Given an uploaded image that already has metadata values for the following fields: + | byline | + | credit | + | copyright | + | specialInstructions | + | description | + | domainMetadata | + | usageInstructions | + | imageType | + When I view the metadata editor + Then I should see the metadata values in the appropriate fields + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 114-134 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 36 + + # This also applies to any textual fields — we should pull them from img meta + @todo + Scenario: Existing usage instructions are shown with room for more + Given an uploaded image that already has usage instructions + When I view the metadata editor + Then I should see the existing usage instructions + And I should be able to add further special instructions + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 140-167 + + @todo + Scenario: Applying a field value to all current uploads in a batch + Given I am uploading more than one image + And I am permitted to edit + When I apply the following field values to all current uploads: + | Leases | + | Image type | + | Description | + | Byline | + | Credit | + | Special instructions | + | Collections | + | Labels | + | Keywords | + | Photoshoot | + Then that value should be applied to the same field on every current upload + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 20-27, 49-53, 76-80, 107-111, 129-133, 156-160, 179-183 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 108-119 + + @todo + Scenario: Metadata editing is disabled without edit permission + Given I am not permitted to edit the image, as it has been uploaded by another user and I do not have edit_metadata permission + When I view the metadata editor for an image I did not upload + Then the metadata fields should be disabled + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 14, 44, 71, 101, 124, 152, 173 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 27-29 + + # This requires configuration, which is already present in CODE + @todo + Scenario: Applying a metadata template makes fields read-only + Given an uploaded image is shown in the metadata editor + When a metadata template is selected + Then the affected fields should be populated and made read-only + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 15, 45, 72, 102, 125, 153, 175 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 84-98 + + @todo + Scenario: Removing a metadata template restores previously edited fields + Given an uploaded image is shown in the metadata editor + And a field is edited + When a metadata template is selected and then removed + Then the previously overridden fields are restored + + # --------------------------------------------------------------------------- + # Recent uploads (recent/recent-uploads.html + recent-uploads.js) + # --------------------------------------------------------------------------- + + @todo + Scenario: Past uploads are loading + When my past uploads have not yet loaded + Then I should see a loading message + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 1-2 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 34 + + @todo + Scenario: I have not uploaded anything yet + Given I have never uploaded an image + When my past uploads load + Then I should see a message that I haven't uploaded anything yet + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 4-5 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-34 + + @todo + Scenario: My past uploads are listed + Given I have uploaded images before + When my past uploads load + Then I should see each of my past uploaded images + And I should be able to delete an image I am permitted to delete + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 7-13 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-51 + + @todo + Scenario: A failed deletion of a past upload is reported + Given a past uploaded image is listed + When deleting the image fails + Then I should see an alert explaining the deletion failed + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 63-69 diff --git a/e2e-tests/fixtures/elasticsearch/images.json b/e2e-tests/fixtures/elasticsearch/images.json new file mode 100644 index 00000000000..242ed82c53f --- /dev/null +++ b/e2e-tests/fixtures/elasticsearch/images.json @@ -0,0 +1,2810 @@ +{ + "status": 200, + "body": { + "took": 7, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 10000, + "relation": "gte" + }, + "max_score": 1, + "hits": [ + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "f6bfd2f0e54c232fd642237dfe773bb14275b3b3", + "_score": 1, + "_source": { + "id": "f6bfd2f0e54c232fd642237dfe773bb14275b3b3", + "uploadTime": "2020-03-04T10:41:37.204Z", + "uploadedBy": "stingray", + "lastModified": "2020-03-04T10:55:06.059Z", + "identifiers": {}, + "uploadInfo": { + "filename": "Swans_Nesting_004.JPG" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/f/6/b/f/d/2/f6bfd2f0e54c232fd642237dfe773bb14275b3b3", + "size": 587773, + "mimeType": "image/jpeg", + "dimensions": { + "width": 3156, + "height": 1728 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/f/6/b/f/d/2/f6bfd2f0e54c232fd642237dfe773bb14275b3b3", + "size": 14830, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 140 + } + }, + "fileMetadata": { + "iptc": { + "Copyright Notice": "© Richard Austin", + "Coded Character Set": "UTF-8", + "Application Record Version": "4", + "Caption/Abstract": "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \rThis week will see the staff putting out bundles of reeds for the swans to build their nests.\rThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\rThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \rAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \rThe colony can number over 600 swans with around 150 nesting pairs. \rWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\r© Richard Austin\rTel: 07831 566005", + "Credit": "Richard Austin", + "Source": "Richard Austin", + "Keywords": "richardaustinimages.com", + "By-line": "Richard Austin", + "Caption Writer/Editor": "Richard Austin", + "Date Created": "2020:03:04" + }, + "exif": { + "Image Description": "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \nThis week will see the staff putting out bundles of reeds for the swans to build their nests.\nThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\nThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \nAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \nThe colony can number over 600 swans with around 150 nesting pairs. \nWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\n© Richard Austin\nTel: 07831 566005", + "X Resolution": "350 dots per inch", + "Software": "Digital Photo Professional", + "Photometric Interpretation": "RGB", + "Make": "Canon", + "Image Width": "5184 pixels", + "YCbCr Positioning": "Center of pixel array", + "Samples Per Pixel": "3 samples/pixel", + "Copyright": "© Richard Austin", + "Date/Time": "2020:03:03 19:10:38", + "Model": "Canon EOS-1D X", + "Orientation": "Top, left side (Horizontal / normal)", + "Resolution Unit": "Inch", + "Bits Per Sample": "8 8 8 bits/component/pixel", + "Y Resolution": "350 dots per inch", + "Artist": "Richard Austin", + "Image Height": "3456 pixels" + }, + "exifSub": { + "Exif Version": "2.30", + "Exposure Mode": "Auto exposure", + "ISO Speed Ratings": "400", + "Lens Specification": "24-70mm", + "Body Serial Number": "063012002171", + "Custom Rendered": "Normal process", + "Exif Image Height": "1728 pixels", + "Lens Serial Number": "5903000717", + "Flash": "Flash did not fire", + "Focal Length": "61 mm", + "Date/Time Original": "2020:03:03 14:54:13", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/789 sec", + "Exif Image Width": "3156 pixels", + "Focal Plane Y Resolution": "49/172800 inches", + "Sub-Sec Time Original": "82", + "Exposure Time": "1/800 sec", + "Metering Mode": "Multi-segment", + "Exposure Program": "Shutter priority", + "Exposure Bias Value": "2/3 EV", + "F-Number": "f/10.0", + "Color Space": "sRGB", + "FlashPix Version": "1.00", + "Components Configuration": "YCbCr", + "Lens Model": "EF24-70mm f/4L IS USM", + "Date/Time Digitized": "2016:07:28 15:57:54", + "Recommended Exposure Index": "400", + "Date/Time Original Composite": "2020-03-03T14:54:13.820Z", + "Aperture Value": "f/9.9", + "Sensitivity Type": "Recommended Exposure Index", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "Inches", + "Sub-Sec Time Digitized": "76", + "Focal Plane X Resolution": "731/2592000 inches", + "Max Aperture Value": "f/4.0" + }, + "xmp": { + "dc:format": "image/jpeg", + "dc:description": [ + "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \nThis week will see the staff putting out bundles of reeds for the swans to build their nests.\nThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\nThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \nAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \nThe colony can number over 600 swans with around 150 nesting pairs. \nWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\n© Richard Austin\nTel: 07831 566005", + [ + "{'xml:lang':'x-default'}" + ] + ], + "xmp:MetadataDate": "2020-03-04T10:04:59.000Z", + "aux:LensSerialNumber": "5903000717", + "aux:ImageNumber": "0", + "dc:subject": [ + "richardaustinimages.com" + ], + "photoshop:CaptionWriter": "Richard Austin", + "aux:FlashCompensation": "0/1", + "aux:Lens": "EF24-70mm f/4L IS USM", + "xmp:ModifyDate": "2020-03-03T19:10:38.000Z", + "xmpMM:OriginalDocumentID": "B5CA1C6D9E598ABFD1488E67E2EC3F78", + "xmp:Rating": "0", + "photoshop:DateCreated": "2020-03-04T00:00:00.000Z", + "photoshop:Credit": "Richard Austin", + "xmp:CreateDate": "2016-07-28T15:57:54.760Z", + "aux:LensID": "504", + "aux:ApproximateFocusDistance": "595/100", + "dc:rights": [ + "© Richard Austin", + [ + "{'xml:lang':'x-default'}" + ] + ], + "xmp:CreatorTool": "Digital Photo Professional", + "aux:LensInfo": "24/1 70/1 0/0 0/0", + "xmpMM:DocumentID": "B5CA1C6D9E598ABFD1488E67E2EC3F78", + "xmpRights:Marked": "True", + "dc:creator": [ + "Richard Austin" + ], + "Iptc4xmpCore:CreatorContactInfo": [ + "{'Iptc4xmpCore:CiUrlWork':'richardaustinimages.com'}", + "{'Iptc4xmpCore:CiEmailWork':'r_austin@btconnect.com'}", + "{'Iptc4xmpCore:CiTelWork':'07831 566005'}", + "{'Iptc4xmpCore:CiAdrCtry':'UK'}", + "{'Iptc4xmpCore:CiAdrPcode':'DT7 3PW'}" + ], + "aux:Firmware": "2.0.3", + "aux:SerialNumber": "063012002171", + "xmpMM:History": [ + [ + "{'stEvt:softwareAgent':'Adobe Photoshop CC 2015.5 (Macintosh)'}", + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-03-03T19:10:38Z'}", + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:e8c994fa-f18f-4333-815c-aa852b3beced'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 9.6'}", + "{'stEvt:instanceID':'xmp.iid:e9307528-6f0e-451c-830a-0580bd9c73a2'}", + "{'stEvt:changed':'/metadata'}", + "{'stEvt:when':'2020-03-03T21:20:25Z'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-03-04T10:04:59Z'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 9.6.1 (Macintosh)'}", + "{'stEvt:changed':'/metadata'}", + "{'stEvt:instanceID':'xmp.iid:f013e587-41b4-4496-a7c8-5be3c1495b5c'}" + ] + ], + "photoshop:Source": "Richard Austin", + "xmpMM:InstanceID": "xmp.iid:f013e587-41b4-4496-a7c8-5be3c1495b5c" + }, + "icc": { + "Chromatic Adaptation": "sf32 (0x73663332): 44 bytes", + "Profile Size": "3140", + "Media White Point": "(0.9642, 1, 0.8249)", + "Device model": "Z009", + "Red Colorant": "(0.436, 0.2224, 0.0139)", + "Blue TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Version": "2.4.0", + "Tag Count": "14", + "Profile Description": "sRGB v1.31 (Canon)", + "Green Colorant": "(0.3851, 0.7169, 0.0971)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "CANO", + "Device Mfg Description": "Canon Inc.", + "Green TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Red TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Blue Colorant": "(0.1431, 0.0606, 0.7139)", + "Color space": "RGB", + "Primary Platform": "Microsoft Corporation", + "Profile Connection Space": "XYZ", + "CMM Type": "UCCM", + "Class": "Display Device", + "Profile Date/Time": "2003:04:04 00:00:00", + "Device Model Description": "sRGB v1.31 (Canon)", + "Profile Copyright": "Copyright (c) 2003, Canon Inc. All rights reserved.", + "Technology": "CRT" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "photometricInterpretation": "RGB", + "bitsPerSample": "8" + } + }, + "metadata": { + "dateTaken": "2020-03-03T14:54:13.820Z", + "description": "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \nThis week will see the staff putting out bundles of reeds for the swans to build their nests.\nThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\nThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \nAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \nThe colony can number over 600 swans with around 150 nesting pairs. \nWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\n© Richard Austin\nTel: 07831 566005", + "credit": "Richard Austin", + "byline": "Richard Austin", + "copyright": "© Richard Austin", + "source": "Richard Austin", + "keywords": [ + "richardaustinimages.com" + ], + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2020-03-03T14:54:13.820Z", + "description": "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \nThis week will see the staff putting out bundles of reeds for the swans to build their nests.\nThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\nThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \nAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \nThe colony can number over 600 swans with around 150 nesting pairs. \nWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\n© Richard Austin\nTel: 07831 566005", + "credit": "Richard Austin", + "byline": "Richard Austin", + "copyright": "© Richard Austin", + "source": "Richard Austin", + "keywords": [ + "richardaustinimages.com" + ], + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [ + { + "path": [ + "Daily Edit", + "Daily edit home" + ], + "pathId": "daily edit/daily edit home", + "description": "Daily edit home", + "actionData": { + "author": "paul.bellsham@guardian.co.uk", + "date": "2020-03-04T10:55:06.059+00:00" + } + } + ], + "userMetadataLastModified": "2020-03-04T10:55:06.059Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "aa22008a915bf22184136ac94a3553977a8921bc", + "_score": 1, + "_source": { + "id": "aa22008a915bf22184136ac94a3553977a8921bc", + "uploadTime": "2019-01-24T16:21:51.712Z", + "uploadedBy": "karin.andreasson@guardian.co.uk", + "lastModified": "2019-01-25T11:23:46.597Z", + "identifiers": {}, + "uploadInfo": { + "filename": "GettyImages-143581630.jpg" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/a/a/2/2/0/0/aa22008a915bf22184136ac94a3553977a8921bc", + "size": 5698095, + "mimeType": "image/jpeg", + "dimensions": { + "width": 5200, + "height": 3454 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/a/a/2/2/0/0/aa22008a915bf22184136ac94a3553977a8921bc", + "size": 10107, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 170 + } + }, + "fileMetadata": { + "iptc": { + "By-line Title": "Contributor", + "Country/Primary Location Name": "United Kingdom", + "Country/Primary Location Code": "GBR", + "Copyright Notice": "Gary Tack Moral Rights Asserted", + "Credit": "Getty Images", + "Source": "Oxford Scientific RM", + "City": "Preston", + "Keywords": "Blackbird,Songbird,Animal,", + "By-line": "Gary Tack", + "Special Instructions": "Not Released (NR)", + "Headline": "Blackbird in winter English garden", + "Province/State": "Lancashire", + "Object Name": "143581630", + "Date Created": "2012:02:15" + }, + "exif": { + "Date/Time Digitized": "2012-02-15T00:00:00.000Z", + "Copyright": "Gary Tack Moral Rights Asserted", + "Orientation": "Top, left side (Horizontal / normal)" + }, + "exifSub": {}, + "xmp": { + "GettyImagesGIFT:ImageRank": "3", + "photoshop:CopyrightFlag": "true", + "photoshop:Headline": "Blackbird in winter English garden", + "photoshop:Instructions": "Not Released (NR)", + "photoshop:AuthorsPosition": "Contributor", + "dc:subject": [ + "Blackbird", + "Songbird", + "Animal" + ], + "photoshop:URL": "http://www.gettyimages.com", + "photoshop:City": "Preston", + "photoshop:DateCreated": "2012-02-15T00:00:00.000Z", + "photoshop:Credit": "Getty Images", + "photoshop:State": "Lancashire", + "plus:LicsensorURL": "http://www.gettyimages.com", + "dc:Rights": "Gary Tack Moral Rights Asserted", + "dc:creator": [ + "Gary Tack" + ], + "dc:title": [ + "143581630", + [ + "{'xml:lang':'x-default'}" + ] + ], + "GettyImagesGIFT:Dlref": "WKdcV+rIBAMIAwr1jFqQgg==", + "Iptc4xmpCore:CountryCode": "GBR", + "photoshop:Country": "United Kingdom", + "photoshop:Source": "Oxford Scientific RM", + "GettyImagesGIFT:AssetID": "143581630" + }, + "icc": { + "Profile Size": "560", + "Media White Point": "(0.9505, 1, 1.0891)", + "Red Colorant": "(0.6097, 0.3111, 0.0195)", + "Blue TRC": "0.0085908", + "Version": "2.1.0", + "Tag Count": "10", + "Profile Description": "Adobe RGB (1998)", + "Green Colorant": "(0.2053, 0.6257, 0.0609)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "none", + "Green TRC": "0.0085908", + "Red TRC": "0.0085908", + "Blue Colorant": "(0.1492, 0.0632, 0.7446)", + "Color space": "RGB", + "Primary Platform": "Apple Computer, Inc.", + "Profile Connection Space": "XYZ", + "CMM Type": "ADBE", + "Rendering Intent": "Media-Relative Colorimetric", + "Class": "Display Device", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1999:06:03 00:00:00", + "Profile Copyright": "Copyright 1999 Adobe Systems Incorporated" + }, + "getty": { + "Image Rank": "3", + "Asset ID": "143581630" + }, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "GettyImages-143581630" + }, + "lastModified": "2019-01-25T11:23:46.597Z" + }, + "metadata": { + "dateTaken": "2012-02-15T00:00:00.000Z", + "description": "GettyImages-143581630", + "credit": "Getty Images", + "byline": "Gary Tack", + "bylineTitle": "Contributor", + "title": "Blackbird in winter English garden", + "copyright": "Gary Tack Moral Rights Asserted", + "suppliersReference": "143581630", + "source": "Oxford Scientific RM", + "specialInstructions": "Not Released (NR)", + "keywords": [ + "Blackbird", + "Songbird", + "Animal" + ], + "city": "Preston", + "state": "Lancashire", + "country": "United Kingdom", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2012-02-15T00:00:00.000Z", + "credit": "Getty Images", + "byline": "Gary Tack", + "bylineTitle": "Contributor", + "title": "Blackbird in winter English garden", + "copyright": "Gary Tack Moral Rights Asserted", + "suppliersReference": "143581630", + "source": "Oxford Scientific RM", + "specialInstructions": "Not Released (NR)", + "keywords": [ + "Blackbird", + "Songbird", + "Animal" + ], + "city": "Preston", + "state": "Lancashire", + "country": "United Kingdom", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": { + "category": "agency", + "supplier": "Getty Images", + "suppliersCollection": "Oxford Scientific RM" + }, + "originalUsageRights": { + "category": "agency", + "supplier": "Getty Images", + "suppliersCollection": "Oxford Scientific RM" + }, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2019-01-25T11:23:46.597Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "02aa01b0ad57aa2794a5765dab3010d53387e677", + "_score": 1, + "_source": { + "id": "02aa01b0ad57aa2794a5765dab3010d53387e677", + "uploadTime": "2020-03-04T07:23:14.490Z", + "uploadedBy": "stingray", + "lastModified": "2020-03-04T10:47:49.175Z", + "identifiers": {}, + "uploadInfo": { + "filename": "PHT_China_Medicinal_herb_Plant_5.JPG" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/0/2/a/a/0/1/02aa01b0ad57aa2794a5765dab3010d53387e677", + "size": 2609472, + "mimeType": "image/jpeg", + "dimensions": { + "width": 3000, + "height": 1902 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/0/2/a/a/0/1/02aa01b0ad57aa2794a5765dab3010d53387e677", + "size": 22684, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 162 + } + }, + "fileMetadata": { + "iptc": { + "Country/Primary Location Name": "CHINA", + "Category": "MAC", + "Country/Primary Location Code": "CHN", + "Copyright Notice": "Avalon.red. All rights reserved.", + "Supplemental Category(s)": "Macro economics", + "Application Record Version": "2", + "Caption/Abstract": "(200304) -- MIANXIAN, March 4, 2020 (Xinhua) -- Aerial photo taken on March 3, 2020 shows villagers planting Ophiopogon japonicus, a traditional Chinese medicinal (TCM) herb, in the fields of Chunfeng Village in Mianxian County, northwest China's Shaanxi Province.\r\n Since 2018, the Chunfeng Village of Mianxian County has tried planting Ophiopogon japonicus, a TCM herb newly introduced into the village, and made a success. The village then established professional planting and processing cooperatives to extend its production chain of the TCM herb. \r\n Up till now, a 400 mu (about 26.7 hectares) standardized Ophiopogon japonicus breeding and planting base has been set up here with an annual output value of more than 7.2 million yuan (about 1 million U.S. dollars), creating jobs for local villagers and at the same time helping them increase their incomes. (Xinhua/Tao Ming)", + "Credit": "Avalon.red", + "Source": "Avalon.red", + "City": "MIANXIAN", + "Keywords": "cn;PHT_WIRE;Sent to UK Press on 04.03.2020 at 07:21:40", + "By-line": "Tao Ming/Avalon.red", + "Urgency": "53", + "Special Instructions": "Supplied by AVALON - Fee Payable Upon Reproduction - For queries contact Avalon - sales@avalon.red London: +44 (0) 20 7421 6000 Los Angeles: +1 (310) 822 0419", + "Headline": "China Medicinal herb Planting", + "Date Created": "2020:03:03" + }, + "exif": {}, + "exifSub": {}, + "xmp": {}, + "icc": { + "Profile Size": "3144", + "Media White Point": "(0.9505, 1, 1.0891)", + "Device model": "sRGB", + "Red Colorant": "(0.4361, 0.2225, 0.0139)", + "Blue TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Viewing Conditions Description": "Reference Viewing Condition in IEC61966-2.1", + "Luminance": "(76.0365, 80, 87.1246)", + "Viewing Conditions": "view (0x76696577): 36 bytes", + "Version": "2.1.0", + "Tag Count": "17", + "Profile Description": "sRGB IEC61966-2.1", + "Green Colorant": "(0.3851, 0.7169, 0.0971)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "IEC", + "Device Mfg Description": "IEC http://www.iec.ch", + "Green TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Red TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Blue Colorant": "(0.1431, 0.0606, 0.7141)", + "Color space": "RGB", + "Primary Platform": "Microsoft Corporation", + "Profile Connection Space": "XYZ", + "CMM Type": "Lino", + "Class": "Display Device", + "Measurement": "1931 2° Observer, Backing (0, 0, 0), Geometry Unknown, Flare 1%, Illuminant D65", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1998:02:09 06:49:00", + "Device Model Description": "IEC 61966-2.1 Default RGB colour space - sRGB", + "Profile Copyright": "Copyright (c) 1998 Hewlett-Packard Company", + "Technology": "CRT" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "metadata": { + "description": "(200304) -- MIANXIAN, March 4, 2020 (Xinhua) -- Aerial photo taken on March 3, 2020 shows villagers planting Ophiopogon japonicus, a traditional Chinese medicinal (TCM) herb, in the fields of Chunfeng Village in Mianxian County, northwest China's Shaanxi Province.\r\n Since 2018, the Chunfeng Village of Mianxian County has tried planting Ophiopogon japonicus, a TCM herb newly introduced into the village, and made a success. The village then established professional planting and processing cooperatives to extend its production chain of the TCM herb. \r\n Up till now, a 400 mu (about 26.7 hectares) standardized Ophiopogon japonicus breeding and planting base has been set up here with an annual output value of more than 7.2 million yuan (about 1 million U.S. dollars), creating jobs for local villagers and at the same time helping them increase their incomes. (Xinhua/Tao Ming)", + "credit": "Avalon.red", + "byline": "Tao Ming", + "title": "China Medicinal herb Planting", + "copyright": "Avalon.red. All rights reserved.", + "source": "Avalon.red", + "specialInstructions": "Supplied by AVALON - Fee Payable Upon Reproduction - For queries contact Avalon - sales@avalon.red London: +44 (0) 20 7421 6000 Los Angeles: +1 (310) 822 0419", + "keywords": [ + "cn", + "PHT_WIRE", + "Sent to UK Press on 04.03.2020 at 07:21:40" + ], + "city": "Mianxian", + "country": "China", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "description": "(200304) -- MIANXIAN, March 4, 2020 (Xinhua) -- Aerial photo taken on March 3, 2020 shows villagers planting Ophiopogon japonicus, a traditional Chinese medicinal (TCM) herb, in the fields of Chunfeng Village in Mianxian County, northwest China's Shaanxi Province.\r\n Since 2018, the Chunfeng Village of Mianxian County has tried planting Ophiopogon japonicus, a TCM herb newly introduced into the village, and made a success. The village then established professional planting and processing cooperatives to extend its production chain of the TCM herb. \r\n Up till now, a 400 mu (about 26.7 hectares) standardized Ophiopogon japonicus breeding and planting base has been set up here with an annual output value of more than 7.2 million yuan (about 1 million U.S. dollars), creating jobs for local villagers and at the same time helping them increase their incomes. (Xinhua/Tao Ming)", + "credit": "Avalon.red", + "byline": "Tao Ming", + "title": "China Medicinal herb Planting", + "copyright": "Avalon.red. All rights reserved.", + "source": "Avalon.red", + "specialInstructions": "Supplied by AVALON - Fee Payable Upon Reproduction - For queries contact Avalon - sales@avalon.red London: +44 (0) 20 7421 6000 Los Angeles: +1 (310) 822 0419", + "keywords": [ + "cn", + "PHT_WIRE", + "Sent to UK Press on 04.03.2020 at 07:21:40" + ], + "city": "Mianxian", + "country": "China", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [ + { + "path": [ + "Daily Edit", + "Daily edit foreign" + ], + "pathId": "daily edit/daily edit foreign", + "description": "Daily edit foreign", + "actionData": { + "author": "paul.bellsham@guardian.co.uk", + "date": "2020-03-04T10:47:49.175+00:00" + } + } + ], + "userMetadataLastModified": "2020-03-04T10:47:49.175Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "dafbac97097c9438bd3cbad8d7a04d2eaadee25e", + "_score": 1, + "_source": { + "id": "dafbac97097c9438bd3cbad8d7a04d2eaadee25e", + "uploadTime": "2015-06-17T12:59:31.000Z", + "uploadedBy": "Picdar Export", + "lastModified": "2015-06-17T23:00:00.000Z", + "identifiers": { + "picdarurn": "GD*53549404" + }, + "uploadInfo": {}, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/d/a/f/b/a/c/dafbac97097c9438bd3cbad8d7a04d2eaadee25e", + "size": 808161, + "mimeType": "image/jpeg", + "dimensions": { + "width": 1661, + "height": 2403 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/d/a/f/b/a/c/dafbac97097c9438bd3cbad8d7a04d2eaadee25e", + "size": 13890, + "mimeType": "image/jpeg", + "dimensions": { + "width": 177, + "height": 256 + } + }, + "fileMetadata": { + "iptc": {}, + "exif": { + "Subject Distance Range": "Macro", + "Rating": "0", + "Exposure Mode": "Auto exposure", + "Custom Rendered": "Normal process", + "Make": "EASTMAN KODAK COMPANY", + "White Balance Mode": "Manual white balance", + "YCbCr Positioning": "Center of pixel array", + "Gain Control": "Low gain down", + "Date/Time": "2011:01:07 12:03:59", + "Model": "KODAK EASYSHARE Z915 DIGITAL CAMERA", + "Sharpness": "None", + "Orientation": "Top, left side (Horizontal / normal)", + "Digital Zoom Ratio": "Digital zoom not used", + "Resolution Unit": "Inch", + "Focal Length 35": "35 mm", + "Contrast": "None", + "Scene Capture Type": "Standard", + "Saturation": "None" + }, + "exifSub": { + "Exif Version": "2.21", + "ISO Speed Ratings": "400", + "Exif Image Height": "2736 pixels", + "Flash": "Flash did not fire", + "Related Sound File": "RelatedSound", + "White Balance": "Daylight", + "Focal Length": "6.2 mm", + "Date/Time Original": "2011:01:07 12:04:00", + "Shutter Speed Value": "1/20 sec", + "Exif Image Width": "3648 pixels", + "Sensing Method": "One-chip color area sensor", + "Scene Type": "Directly photographed image", + "Sub-Sec Time Original": "59", + "Brightness Value": "1.0", + "Makernote": "[40960 values]", + "Exposure Time": "0.05 sec", + "Metering Mode": "Center weighted average", + "Exposure Program": "Program normal", + "Exposure Bias Value": "0 EV", + "F-Number": "f/3.5", + "Color Space": "sRGB", + "FlashPix Version": "1.00", + "Exposure Index": "400", + "Components Configuration": "YCbCr", + "Date/Time Digitized": "2011:01:07 12:04:00", + "Date/Time Original Composite": "2011-01-07T12:04:00.590Z", + "Aperture Value": "f/3.4", + "File Source": "Digital Still Camera (DSC)", + "Max Aperture Value": "f/3.5" + }, + "xmp": {}, + "icc": {}, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "Daphne Du Maurier - Rebecca" + } + }, + "metadata": { + "dateTaken": "2011-01-07T12:04:00.590Z", + "description": "Daphne Du Maurier - Rebecca", + "keywords": [], + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2011-01-07T12:04:00.590Z", + "keywords": [], + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [ + { + "id": "print/b01ff63186ac8b771d8d589227c5a3c9_140a9429c07583239faf0c7e4b1e256d", + "references": [ + { + "type": "indesign", + "name": "2015-06-17, The Guardian, g2f, Page 2" + } + ], + "platform": "print", + "media": "image", + "status": "published", + "dateAdded": "2015-06-17T23:00:00.000Z", + "lastModified": "2015-06-17T23:00:00.000Z", + "printUsageMetadata": { + "sectionName": "g2f", + "issueDate": "2015-06-17T23:00:00.000Z", + "pageNumber": 2, + "storyName": "scMWkatemo_02_g2f_0618_002", + "publicationCode": "gdn", + "publicationName": "The Guardian", + "edition": 1, + "sectionCode": "g2f", + "notes": "Andrew Stocks", + "source": "picdar" + } + } + ], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2015-06-17T23:00:00.000Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "ad61e5bee9ceb7e47350d1380ee6c6d7500ebe52", + "_score": 1, + "_source": { + "id": "ad61e5bee9ceb7e47350d1380ee6c6d7500ebe52", + "uploadTime": "2015-07-28T15:10:22.000Z", + "uploadedBy": "DEV Ingest Script", + "lastModified": "2014-01-07T05:01:26.000Z", + "identifiers": { + "picdarurn": "GD*40548955" + }, + "uploadInfo": {}, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/a/d/6/1/e/5/ad61e5bee9ceb7e47350d1380ee6c6d7500ebe52", + "size": 1653855, + "mimeType": "image/jpeg", + "dimensions": { + "width": 4928, + "height": 3280 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/a/d/6/1/e/5/ad61e5bee9ceb7e47350d1380ee6c6d7500ebe52", + "size": 41245, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 170 + } + }, + "fileMetadata": { + "iptc": { + "Country/Primary Location Name": "England", + "Copyright Notice": "Lucy Young", + "Application Record Version": "3", + "Caption/Abstract": "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges Ð including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\rBy Lucy Young \r07799118984\rlucyyounguk@gmail.com\r\rwww.lucyyoungphotos.co.uk", + "Credit": "Lucy Young", + "City": "London", + "Time Created": "12:43:54+0000", + "By-line": "Lucy Young", + "Special Instructions": "From: Lucy Young ", + "Headline": "A Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel.", + "Date Time Created Composite": "2014-01-02T12:43:54.000Z", + "Date Created": "2014:01:02" + }, + "exif": { + "X Resolution": "300 dots per inch", + "Software": "Adobe Photoshop CS6 (Macintosh)", + "Photometric Interpretation": "RGB", + "Make": "NIKON CORPORATION", + "Image Width": "4928 pixels", + "Samples Per Pixel": "3 samples/pixel", + "Copyright": "Lucy Young", + "Date/Time": "2014:01:02 20:18:33", + "Model": "NIKON D4", + "Orientation": "Top, left side (Horizontal / normal)", + "Resolution Unit": "Inch", + "Bits Per Sample": "8 8 8 bits/component/pixel", + "Y Resolution": "300 dots per inch", + "Artist": "Lucy Young", + "Image Height": "3280 pixels" + }, + "exifSub": { + "CFA Pattern": "[Red,Green][Green,Blue]", + "Exif Version": "2.21", + "Subject Distance Range": "Unknown", + "Exposure Mode": "Auto exposure", + "ISO Speed Ratings": "400", + "Lens Specification": "14-24mm f/2.8", + "Body Serial Number": "2055843", + "Custom Rendered": "Normal process", + "Exif Image Height": "3280 pixels", + "Flash": "Flash did not fire", + "White Balance": "Unknown", + "Focal Length": "14 mm", + "Date/Time Original": "2014:01:02 12:43:54", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/499 sec", + "Exif Image Width": "4928 pixels", + "Gain Control": "Low gain up", + "Sensing Method": "One-chip color area sensor", + "Scene Type": "Directly photographed image", + "Focal Plane Y Resolution": "32768/4485575 unknown (4)", + "Sub-Sec Time Original": "10", + "Exposure Time": "1/500 sec", + "Sharpness": "None", + "Metering Mode": "Multi-segment", + "Exposure Program": "Shutter priority", + "Digital Zoom Ratio": "1", + "Exposure Bias Value": "0 EV", + "F-Number": "f/9.0", + "Color Space": "Undefined", + "Sub-Sec Time": "10", + "Lens Model": "14.0-24.0 mm f/2.8", + "Focal Length 35": "14 mm", + "Date/Time Digitized": "2014:01:02 12:43:54", + "Date/Time Original Composite": "2014-01-02T12:43:54.100Z", + "Aperture Value": "f/9.0", + "Sensitivity Type": "Recommended Exposure Index", + "Contrast": "None", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "Unknown (4)", + "File Source": "Digital Still Camera (DSC)", + "Sub-Sec Time Digitized": "10", + "Focal Plane X Resolution": "32768/4485575 unknown (4)", + "Saturation": "None", + "Max Aperture Value": "f/2.8" + }, + "xmp": { + "crs:ParametricDarks": "0", + "dc:format": "image/jpeg", + "crs:DefringeGreenHueLo": "40", + "crs:SaturationAdjustmentPurple": "0", + "crs:HasSettings": "True", + "crs:GreenHue": "0", + "photoshop:ColorMode": "3", + "dc:description": [ + "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges – including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\nBy Lucy Young \n07799118984\nlucyyounguk@gmail.com\n\nwww.lucyyoungphotos.co.uk", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:SaturationAdjustmentAqua": "0", + "crs:UprightVersion": "134217728", + "crs:SharpenDetail": "25", + "crs:GrainAmount": "0", + "xmp:MetadataDate": "2014-01-02T20:18:33.000Z", + "crs:CameraProfile": "Adobe Standard", + "crs:WhiteBalance": "As Shot", + "crs:ColorNoiseReductionDetail": "50", + "crs:Blacks2012": "0", + "crs:SaturationAdjustmentBlue": "0", + "xmpMM:DerivedFrom": [ + "{'stRef:instanceID':'xmp.iid:F63F7CEC112068118083C60981F28C85'}", + "{'stRef:documentID':'xmp.did:bfa36716-faee-4eb2-9e22-02910f56286a'}", + "{'stRef:originalDocumentID':'4CC35D0B77263D9E2D74FAAEEEB7E101'}" + ], + "crs:UprightPreview": "False", + "aux:ImageNumber": "19408", + "crs:SplitToningShadowSaturation": "0", + "crs:SplitToningShadowHue": "0", + "photoshop:Headline": "A Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel.", + "crs:ParametricHighlightSplit": "75", + "crs:PerspectiveRotate": "0.0", + "crs:PerspectiveUpright": "0", + "photoshop:ICCProfile": "Adobe RGB (1998)", + "crs:DefringePurpleHueLo": "30", + "crs:ToneCurveName2012": "Linear", + "crs:ToneCurvePV2012": [ + "0, 0", + "255, 255" + ], + "crs:HueAdjustmentPurple": "0", + "crs:VignetteAmount": "0", + "crs:Exposure2012": "-0.25", + "crs:ParametricShadowSplit": "25", + "crs:LuminanceAdjustmentAqua": "0", + "crs:LuminanceAdjustmentPurple": "0", + "photomechanic:PMVersion": "PM5", + "crs:SplitToningBalance": "0", + "crs:SaturationAdjustmentMagenta": "0", + "crs:Sharpness": "25", + "crs:HueAdjustmentAqua": "0", + "crs:BlueHue": "0", + "crs:DefringeGreenHueHi": "60", + "crs:SaturationAdjustmentOrange": "0", + "crs:HueAdjustmentYellow": "0", + "crs:SplitToningHighlightHue": "0", + "crs:PerspectiveScale": "100", + "crs:HueAdjustmentGreen": "0", + "photomechanic:Prefs": "0:3:0:019408", + "crs:Temperature": "5050", + "crs:ConvertToGrayscale": "False", + "crs:RawFileName": "DSC_9706.NEF", + "crs:ParametricLights": "0", + "aux:Lens": "14.0-24.0 mm f/2.8", + "crs:UprightFocalMode": "0", + "photomechanic:Tagged": "False", + "crs:ToneCurvePV2012Green": [ + "0, 0", + "255, 255" + ], + "crs:ParametricHighlights": "0", + "crs:SharpenEdgeMasking": "0", + "crs:SaturationAdjustmentRed": "0", + "crs:Whites2012": "0", + "crs:Clarity2012": "+14", + "crs:UprightFocalLength35mm": "35", + "xmp:ModifyDate": "2014-01-02T20:18:33.000Z", + "crs:UprightTransformCount": "0", + "crs:UprightCenterNormX": "0.5", + "photoshop:City": "London", + "xmpMM:OriginalDocumentID": "4CC35D0B77263D9E2D74FAAEEEB7E101", + "xmp:Rating": "0", + "crs:ToneCurvePV2012Blue": [ + "0, 0", + "255, 255" + ], + "crs:LuminanceAdjustmentRed": "0", + "crs:DefringeGreenAmount": "0", + "crs:PerspectiveVertical": "0", + "crs:HueAdjustmentBlue": "0", + "crs:LensManualDistortionAmount": "0", + "crs:AutoLateralCA": "0", + "crs:LuminanceSmoothing": "0", + "crs:CameraProfileDigest": "939B4E1427152B8FC65E6BBE46CF1813", + "crs:Contrast2012": "0", + "photoshop:DateCreated": "2014-01-02T12:43:54.000Z", + "photoshop:Credit": "Lucy Young", + "crs:SaturationAdjustmentYellow": "0", + "xmp:CreateDate": "2014-01-02T12:43:54.000Z", + "crs:LensProfileSetup": "LensDefaults", + "crs:Shadows2012": "0", + "crs:UprightCenterMode": "0", + "crs:LensProfileEnable": "0", + "aux:LensID": "146", + "aux:ApproximateFocusDistance": "4294967295/1", + "crs:Tint": "-1", + "crs:ParametricMidtoneSplit": "50", + "crs:PerspectiveHorizontal": "0", + "crs:ShadowTint": "0", + "crs:UprightCenterNormY": "0.5", + "dc:rights": [ + "Lucy Young", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:HueAdjustmentMagenta": "0", + "xmp:CreatorTool": "Adobe Photoshop CS6 (Macintosh)", + "aux:LensInfo": "140/10 240/10 28/10 28/10", + "crs:LuminanceAdjustmentMagenta": "0", + "photoshop:LegacyIPTCDigest": "F9B58E32E365BE2D442491E7BA5AEE2F", + "photomechanic:ColorClass": "3", + "crs:BlueSaturation": "0", + "crs:LuminanceAdjustmentYellow": "0", + "crs:AlreadyApplied": "True", + "crs:DefringePurpleHueHi": "70", + "crs:PostCropVignetteAmount": "0", + "xmp:Label": "Superior", + "xmpMM:DocumentID": "xmp.did:bfa36716-faee-4eb2-9e22-02910f56286a", + "crs:PerspectiveAspect": "0", + "crs:LuminanceAdjustmentGreen": "0", + "crs:RedSaturation": "0", + "xmpRights:Marked": "True", + "crs:SaturationAdjustmentGreen": "0", + "dc:creator": [ + "Lucy Young" + ], + "crs:LuminanceAdjustmentOrange": "0", + "crs:Version": "8.1", + "crs:ToneCurvePV2012Red": [ + "0, 0", + "255, 255" + ], + "Iptc4xmpCore:CreatorContactInfo": [ + "{'Iptc4xmpCore:CiUrlWork':'www.lucyyoungphotos.co.uk'}", + "{'Iptc4xmpCore:CiEmailWork':'lucyyounguk@gmail.com'}", + "{'Iptc4xmpCore:CiAdrCtry':'England'}", + "{'Iptc4xmpCore:CiTelWork':'+44 (0)7799118984'}" + ], + "crs:ParametricShadows": "0", + "crs:HueAdjustmentRed": "0", + "crs:GreenSaturation": "0", + "crs:RedHue": "0", + "crs:SplitToningHighlightSaturation": "0", + "aux:SerialNumber": "2055843", + "crs:DefringePurpleAmount": "0", + "crs:Saturation": "+17", + "crs:HueAdjustmentOrange": "0", + "crs:LuminanceAdjustmentBlue": "0", + "xmpMM:History": [ + [ + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 8.1 (Macintosh)'}", + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2014-01-02T15:37:39Z'}", + "{'stEvt:changed':'/metadata'}", + "{'stEvt:instanceID':'xmp.iid:8f376438-551e-40c7-8867-759ff1c1e0c8'}" + ], + [ + "{'stEvt:parameters':'converted from image/x-nikon-nef to image/tiff'}", + "{'stEvt:action':'derived'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2014-01-02T15:39:41Z'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 8.1 (Macintosh)'}", + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:bfa36716-faee-4eb2-9e22-02910f56286a'}" + ], + [ + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:F63F7CEC112068118083C60981F28C85'}", + "{'stEvt:when':'2014-01-02T15:41:22Z'}", + "{'stEvt:softwareAgent':'Adobe Photoshop CS6 (Macintosh)'}", + "{'stEvt:action':'saved'}" + ], + [ + "{'stEvt:parameters':'from image/tiff to image/jpeg'}", + "{'stEvt:action':'converted'}" + ], + [ + "{'stEvt:action':'derived'}", + "{'stEvt:parameters':'converted from image/tiff to image/jpeg'}" + ], + [ + "{'stEvt:instanceID':'xmp.iid:F73F7CEC112068118083C60981F28C85'}", + "{'stEvt:action':'saved'}", + "{'stEvt:changed':'/'}", + "{'stEvt:softwareAgent':'Adobe Photoshop CS6 (Macintosh)'}", + "{'stEvt:when':'2014-01-02T15:41:22Z'}" + ], + [ + "{'stEvt:when':'2014-01-02T20:18:33Z'}", + "{'stEvt:instanceID':'xmp.iid:1D7ECFD70A2068118C1496C090CBA674'}", + "{'stEvt:changed':'/'}", + "{'stEvt:softwareAgent':'Adobe Photoshop CS6 (Macintosh)'}", + "{'stEvt:action':'saved'}" + ] + ], + "crs:Highlights2012": "0", + "photoshop:Country": "England", + "crs:ProcessVersion": "6.7", + "crs:Vibrance": "+14", + "crs:SharpenRadius": "+1.0", + "xmpMM:InstanceID": "xmp.iid:1D7ECFD70A2068118C1496C090CBA674", + "crs:ColorNoiseReduction": "25" + }, + "icc": { + "Profile Size": "560", + "Media White Point": "(0.9505, 1, 1.0891)", + "Red Colorant": "(0.6097, 0.3111, 0.0195)", + "Blue TRC": "0.0085908", + "Version": "2.1.0", + "Tag Count": "10", + "Profile Description": "Adobe RGB (1998)", + "Green Colorant": "(0.2053, 0.6257, 0.0609)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "none", + "Green TRC": "0.0085908", + "Red TRC": "0.0085908", + "Blue Colorant": "(0.1492, 0.0632, 0.7446)", + "Color space": "RGB", + "Primary Platform": "Apple Computer, Inc.", + "Profile Connection Space": "XYZ", + "CMM Type": "ADBE", + "Class": "Display Device", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1999:06:03 00:00:00", + "Profile Copyright": "Copyright 1999 Adobe Systems Incorporated" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "photometricInterpretation": "RGB", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of 11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\nBy Lucy Young \n07799118984\nlucyyounguk@gmail.com\nwww.lucyyoungphotos.co.uk", + "title": "A Sunseeker 115 Sport Super Yacht, which has a starting price of 11.5 million exc tax. arrives for The London Boat Show at ExCel." + } + }, + "metadata": { + "dateTaken": "2014-01-02T12:43:54.100Z", + "description": "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of 11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\nBy Lucy Young \n07799118984\nlucyyounguk@gmail.com\nwww.lucyyoungphotos.co.uk", + "credit": "Lucy Young", + "byline": "Lucy Young", + "title": "A Sunseeker 115 Sport Super Yacht, which has a starting price of 11.5 million exc tax. arrives for The London Boat Show at ExCel.", + "copyright": "Lucy Young", + "specialInstructions": "From: Lucy Young ", + "keywords": [], + "city": "London", + "country": "England", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2014-01-02T12:43:54.100Z", + "description": "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges – including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\nBy Lucy Young \n07799118984\nlucyyounguk@gmail.com\n\nwww.lucyyoungphotos.co.uk", + "credit": "Lucy Young", + "byline": "Lucy Young", + "title": "A Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel.", + "copyright": "Lucy Young", + "specialInstructions": "From: Lucy Young ", + "keywords": [], + "city": "London", + "country": "England", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [ + { + "id": "print/90dab991a7b8d6dc95066ea91694c9e0_26d733c73fae0793858104574cc9df69", + "references": [ + { + "type": "indesign", + "name": "2014-01-07, The Guardian, frt, Page 3" + } + ], + "platform": "print", + "media": "image", + "status": "published", + "dateAdded": "2014-01-07T05:01:26.000Z", + "lastModified": "2014-01-07T05:01:26.000Z", + "printUsageMetadata": { + "sectionName": "frt", + "issueDate": "2014-01-07T00:00:00.000Z", + "pageNumber": 3, + "storyName": "yachts_04_frt_0107_003", + "publicationCode": "gdn", + "publicationName": "The Guardian", + "edition": 1, + "sectionCode": "frt", + "notes": "Martin Nicholls", + "source": "picdar" + } + } + ], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2014-01-07T05:01:26.000Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "1a5b08c19cd3a68ac496c46bdd761f8dc05313aa", + "_score": 1, + "_source": { + "id": "1a5b08c19cd3a68ac496c46bdd761f8dc05313aa", + "uploadTime": "2020-01-23T15:10:16.577Z", + "uploadedBy": "getty", + "lastModified": "2020-06-06T14:14:16.513Z", + "identifiers": {}, + "uploadInfo": { + "filename": "1195421274.jpg" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/1/a/5/b/0/8/1a5b08c19cd3a68ac496c46bdd761f8dc05313aa", + "size": 5413118, + "mimeType": "image/jpeg", + "dimensions": { + "width": 5472, + "height": 3648 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/1/a/5/b/0/8/1a5b08c19cd3a68ac496c46bdd761f8dc05313aa", + "size": 15062, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 171 + } + }, + "fileMetadata": { + "iptc": { + "By-line Title": "Contributor", + "Country/Primary Location Name": "Russia", + "Category": "I", + "Country/Primary Location Code": "RUS", + "Supplemental Category(s)": "FIN", + "Coded Character Set": "UTF-8", + "Application Record Version": "4", + "Caption/Abstract": "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)", + "Enveloped Record Version": "4", + "Credit": "Donat Sorokin/TASS", + "Source": "TASS", + "City": "Yekaterinburg", + "Keywords": "medicine;coronavirus;drug;healthcare;corona virus;wuhan virus;china virus;urfu;ural federal university;treatment;remedy", + "By-line": "Donat Sorokin", + "Urgency": "51", + "Special Instructions": "0", + "Headline": "Ural Federal University researchers propose using Triazavirin against China coronavirus", + "Object Name": "1195421274", + "Caption Writer/Editor": "DF", + "Original Transmission Reference": "1195421274", + "Date Created": "20200123T00:00:00Z" + }, + "exif": { + "Image Description": "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)" + }, + "exifSub": {}, + "xmp": { + "GettyImagesGIFT:ImageRank": "3", + "GettyImagesGIFT:OriginalFilename": "ts0cb425.jpg", + "dc:description": [ + "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)", + [ + "{'xml:lang':'x-default'}" + ] + ], + "photoshop:Headline": "Ural Federal University researchers propose using Triazavirin against China coronavirus", + "photoshop:TransmissionReference": "0", + "photoshop:Instructions": "0", + "photoshop:AuthorsPosition": "Contributor", + "dc:subject": [ + "medicine", + "coronavirus", + "drug", + "healthcare", + "corona virus", + "wuhan virus", + "china virus", + "urfu", + "ural federal university", + "treatment", + "remedy" + ], + "photoshop:CaptionWriter": "DF", + "photoshop:SupplementalCategories": [ + "FIN" + ], + "plus:ImageSupplierImageId": "ts0cb425", + "photoshop:City": "Yekaterinburg", + "GettyImagesGIFT:ExclusiveCoverage": "False", + "photoshop:DateCreated": "2020-01-23T00:00:00.000Z", + "photoshop:Credit": "Donat Sorokin/TASS", + "plus:LicsensorURL": "http://www.gettyimages.com", + "GettyImagesGIFT:OriginalCreateDateTime": "1970-01-01T00:00:02.000Z", + "dc:creator": [ + "Donat Sorokin" + ], + "GettyImagesGIFT:AssetId": "1195421274", + "dc:title": [ + "1195421274", + [ + "{'xml:lang':'x-default'}" + ] + ], + "Iptc4xmpCore:CountryCode": "RUS", + "GettyImagesGIFT:CallForImage": "False", + "photoshop:Country": "Russia", + "photoshop:Source": "TASS", + "photoshop:Category": "I" + }, + "icc": {}, + "getty": { + "Call For Image": "False", + "Image Rank": "3", + "Asset ID": "1195421274", + "Original Filename": "ts0cb425.jpg", + "Exclusive Coverage": "False", + "Original Create Date Time": "1970-01-01T00:00:02.000Z" + }, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "metadata": { + "dateTaken": "2020-01-23T00:00:00.000Z", + "description": "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)", + "credit": "TASS", + "byline": "Donat Sorokin", + "bylineTitle": "Contributor", + "title": "Ural Federal University researchers propose using Triazavirin against China coronavirus", + "suppliersReference": "1195421274", + "source": "TASS", + "specialInstructions": "0", + "keywords": [ + "ural federal university", + "treatment", + "remedy", + "coronavirus", + "china virus", + "medicine", + "corona virus", + "healthcare", + "urfu", + "drug", + "wuhan virus" + ], + "city": "Yekaterinburg", + "country": "Russia", + "subjects": [ + "finance", + "news" + ], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2020-01-23T00:00:00.000Z", + "description": "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)", + "credit": "TASS", + "byline": "Donat Sorokin", + "bylineTitle": "Contributor", + "title": "Ural Federal University researchers propose using Triazavirin against China coronavirus", + "suppliersReference": "1195421274", + "source": "TASS", + "specialInstructions": "0", + "keywords": [ + "ural federal university", + "treatment", + "remedy", + "coronavirus", + "china virus", + "medicine", + "corona virus", + "healthcare", + "urfu", + "drug", + "wuhan virus" + ], + "city": "Yekaterinburg", + "country": "Russia", + "subjects": [ + "finance", + "news" + ], + "peopleInImage": [] + }, + "usageRights": { + "category": "agency", + "supplier": "Getty Images", + "suppliersCollection": "TASS" + }, + "originalUsageRights": { + "category": "agency", + "supplier": "Getty Images", + "suppliersCollection": "TASS" + }, + "exports": [], + "usages": [ + { + "id": "print/1ac0407e35d79a4b71bf9cb05d7da3e8_106ebc81cf0cde3c657607647b34fc93", + "references": [ + { + "type": "indesign", + "name": "2020-06-07, Observer, UK News (Observer), Page 13" + } + ], + "platform": "print", + "media": "image", + "status": "pending", + "dateAdded": "2020-06-06T14:14:16.513Z", + "lastModified": "2020-06-06T14:14:16.513Z", + "printUsageMetadata": { + "sectionName": "UK News (Observer)", + "issueDate": "2020-06-07T00:00:00.000Z", + "pageNumber": 13, + "storyName": "HOMRMCovid7", + "publicationCode": "obs", + "publicationName": "Observer", + "layoutId": 15284077, + "edition": 1, + "size": { + "x": 240, + "y": 240 + }, + "orderedBy": "Liz Boulter", + "sectionCode": "2hn" + } + } + ], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2020-06-06T14:14:16.513Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "ea763a169dd6574c6ac1ccaf3443d363e011533f", + "_score": 1, + "_source": { + "id": "ea763a169dd6574c6ac1ccaf3443d363e011533f", + "uploadTime": "2015-07-26T14:50:49.000Z", + "uploadedBy": "DEV Ingest Script", + "lastModified": "2006-03-31T23:00:00.000Z", + "identifiers": { + "picdarurn": "PD*9095009" + }, + "uploadInfo": {}, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/e/a/7/6/3/a/ea763a169dd6574c6ac1ccaf3443d363e011533f", + "size": 495188, + "mimeType": "image/jpeg", + "dimensions": { + "width": 2482, + "height": 1793 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/e/a/7/6/3/a/ea763a169dd6574c6ac1ccaf3443d363e011533f", + "size": 27755, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 185 + } + }, + "fileMetadata": { + "iptc": { + "Country/Primary Location Name": "United Kingdom", + "Category": "S", + "Supplemental Category(s)": "Rugby Union", + "Application Record Version": "2", + "Caption/Abstract": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\rNewcastle Falcons' Toby Flood prepears to take a penalty\rMandatory Credit: Action Images / Lee Smith\rLivepic", + "Credit": "Action Images", + "Source": "Action Images", + "City": "Newcastle", + "Keywords": "2006", + "By-line": "Lee Smith", + "Special Instructions": "01", + "Caption Writer/Editor": "tp", + "Original Transmission Reference": "GCLY-6LBGXV", + "Date Created": "2006:03:31" + }, + "exif": { + "Image Description": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\rNewcastle Falcons' Toby Flood prepears to take a penalty\rMandatory Credit: Action Images / Lee Smith\rLivepic", + "Primary Chromaticities": "64/100 33/100 21/100 71/100 15/100 6/100", + "X Resolution": "72 dots per inch", + "Software": "Adobe Photoshop 7.0", + "Make": "Canon", + "YCbCr Coefficients": "299/1000 587/1000 114/1000", + "YCbCr Positioning": "Datum point", + "Date/Time": "2006:03:31 20:44:35", + "Model": "Canon EOS-1D Mark II", + "Orientation": "Top, left side (Horizontal / normal)", + "White Point": "313/1000 329/1000", + "Resolution Unit": "Inch", + "Y Resolution": "72 dots per inch", + "Artist": "Lee Smith" + }, + "exifSub": { + "Exif Version": "2.21", + "Exposure Mode": "Auto exposure", + "ISO Speed Ratings": "3200", + "Custom Rendered": "Normal process", + "Exif Image Height": "1793 pixels", + "Flash": "Flash did not fire", + "Focal Length": "300 mm", + "Date/Time Original": "2006:03:31 19:04:17", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/511 sec", + "Exif Image Width": "2482 pixels", + "Focal Plane Y Resolution": "189/585500 inches", + "Exposure Time": "1/500 sec", + "Metering Mode": "Multi-segment", + "Exposure Program": "Shutter priority", + "Gamma": "2.2", + "Exposure Bias Value": "1/3 EV", + "F-Number": "f/2.8", + "Color Space": "Undefined", + "FlashPix Version": "1.00", + "Components Configuration": "YCbCr", + "Date/Time Digitized": "2006:03:31 19:04:17", + "Date/Time Original Composite": "2006-03-31T19:04:17.000Z", + "Aperture Value": "f/2.8", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "Inches", + "Focal Plane X Resolution": "71/220000 inches" + }, + "xmp": { + "dc:description": [ + "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\rNewcastle Falcons' Toby Flood prepears to take a penalty\rMandatory Credit: Action Images / Lee Smith\rLivepic", + [ + "{'xml:lang':'x-default'}" + ] + ], + "photoshop:TransmissionReference": "GCLY-6LBGXV", + "photoshop:Instructions": "01", + "dc:subject": [ + "2006" + ], + "photoshop:CaptionWriter": "tp", + "photoshop:SupplementalCategories": [ + "Rugby Union" + ], + "photoshop:City": "Newcastle", + "photoshop:DateCreated": "2006-03-31T00:00:00.000Z", + "photoshop:Credit": "Action Images", + "xmpMM:DocumentID": "adobe:docid:photoshop:bb6e8fa1-c0e8-11da-8611-8b5221e90d2f", + "xmpRights:Marked": "True", + "dc:creator": [ + "Lee Smith" + ], + "photoshop:Country": "United Kingdom", + "photoshop:Source": "Action Images", + "photoshop:Category": "S", + "xmpMM:InstanceID": "uuid:e3938d28-c0ed-11da-81f9-cc201dca07e4" + }, + "icc": {}, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\nNewcastle Falcons' Toby Flood prepears to take a penalty\nMandatory Credit: Action Images / Lee Smith\nLivepic" + }, + "usageRights": { + "category": "agency", + "supplier": "Corbis" + } + }, + "metadata": { + "dateTaken": "2006-03-31T19:04:17.000Z", + "description": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\nNewcastle Falcons' Toby Flood prepears to take a penalty\nMandatory Credit: Action Images / Lee Smith\nLivepic", + "credit": "Action Images/Reuters", + "byline": "Lee Smith", + "suppliersReference": "GCLY-6LBGXV", + "source": "Action Images", + "specialInstructions": "01", + "keywords": [ + "2006" + ], + "city": "Newcastle", + "country": "United Kingdom", + "subjects": [ + "sport" + ], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2006-03-31T19:04:17.000Z", + "description": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\rNewcastle Falcons' Toby Flood prepears to take a penalty\rMandatory Credit: Action Images / Lee Smith\rLivepic", + "credit": "Action Images/Reuters", + "byline": "Lee Smith", + "suppliersReference": "GCLY-6LBGXV", + "source": "Action Images", + "specialInstructions": "01", + "keywords": [ + "2006" + ], + "city": "Newcastle", + "country": "United Kingdom", + "subjects": [ + "sport" + ], + "peopleInImage": [] + }, + "usageRights": { + "category": "agency", + "supplier": "Corbis" + }, + "originalUsageRights": { + "category": "agency", + "supplier": "Action Images" + }, + "exports": [], + "usages": [ + { + "id": "print/e02f7633cbd760e61a9f1c4cbef78699_d6848c99a477f095de6dab5660465049", + "references": [ + { + "type": "indesign", + "name": "2006-03-31, The Guardian, Guardian Sport, Page 10" + } + ], + "platform": "print", + "media": "image", + "status": "published", + "dateAdded": "2006-03-31T23:00:00.000Z", + "lastModified": "2006-03-31T23:00:00.000Z", + "printUsageMetadata": { + "sectionName": "Guardian Sport", + "issueDate": "2006-03-31T23:00:00.000Z", + "pageNumber": 10, + "storyName": "flood01", + "publicationCode": "gdn", + "publicationName": "The Guardian", + "sectionCode": "Guardian Sport", + "notes": "45mm wide", + "source": "picdar" + } + } + ], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2006-03-31T23:00:00.000Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "82c4d17a78148cc7ab4006d5a1d8ddbf9810da72", + "_score": 1, + "_source": { + "id": "82c4d17a78148cc7ab4006d5a1d8ddbf9810da72", + "uploadTime": "2020-06-12T14:51:12.323Z", + "uploadedBy": "stingray", + "lastModified": "2020-06-12T15:04:11.817Z", + "identifiers": {}, + "uploadInfo": { + "filename": "pxl_churchill_mandela_statues_westminster_001.JPG" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/8/2/c/4/d/1/82c4d17a78148cc7ab4006d5a1d8ddbf9810da72", + "size": 6411306, + "mimeType": "image/jpeg", + "dimensions": { + "width": 6720, + "height": 4480 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/8/2/c/4/d/1/82c4d17a78148cc7ab4006d5a1d8ddbf9810da72", + "size": 20696, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 171 + } + }, + "fileMetadata": { + "iptc": { + "Country/Primary Location Name": "United Kingdom", + "Country/Primary Location Code": "GBR", + "Copyright Notice": "Gavin Rodgers/ Pixel8000", + "Application Record Version": "3", + "Caption/Abstract": "Pic shows: What would Mandela do?\n\nNelson Mandela statue appears to be either removing of placing the box on Winston Churchill’s statue in Parliament Square today\n\nafter it was actually covered Sadiq Khan ordered the council to cover the memorial due to Black Lives Matter protests\n\n\n\n\npicture by Gavin Rodgers/ Pixel8000", + "Credit": "gavin rodgers/pixel8000", + "City": "London", + "Digital Date Created": "2020:06:12", + "Keywords": "Sadiq Khan, statue, Nelson Mandela, Westminster, Parliament Square, London, protests, race relations", + "Time Created": "14:01:30+0100", + "By-line": "gavin rodgers/pixel8000", + "Urgency": "49", + "Date Time Created Composite": "2020-06-12T13:01:30.000Z", + "Date Created": "2020:06:12" + }, + "exif": { + "X Resolution": "300 dots per inch", + "Software": "PhotoShelter http://www.photoshelter.com", + "Make": "Canon", + "Date/Time": "2020:06:12 15:39:59", + "Model": "Canon EOS 5D Mark IV", + "Orientation": "Top, left side (Horizontal / normal)", + "Resolution Unit": "Inch", + "Y Resolution": "300 dots per inch" + }, + "exifSub": { + "Exif Version": "2.30", + "Exposure Mode": "Auto exposure", + "ISO Speed Ratings": "640", + "Lens Specification": "100-400mm", + "Body Serial Number": "018021000814", + "Custom Rendered": "Normal process", + "Exif Image Height": "4480 pixels", + "Lens Serial Number": "9720000826", + "Flash": "Flash did not fire", + "Focal Length": "110 mm", + "Date/Time Original": "2020:06:12 14:01:30", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/499 sec", + "Exif Image Width": "6720 pixels", + "Focal Plane Y Resolution": "32768/61166933 cm", + "Sub-Sec Time Original": "16", + "Exposure Time": "1/500 sec", + "Metering Mode": "Multi-segment", + "Exposure Program": "Shutter priority", + "Exposure Bias Value": "0 EV", + "F-Number": "f/5.0", + "Color Space": "Undefined", + "Sub-Sec Time": "16", + "Lens Model": "EF100-400mm f/4.5-5.6L IS II USM", + "Date/Time Digitized": "2020:06:12 14:01:30", + "Recommended Exposure Index": "640", + "Date/Time Original Composite": "2020-06-12T14:01:30.160Z", + "Aperture Value": "f/5.0", + "Sensitivity Type": "Recommended Exposure Index", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "cm", + "Sub-Sec Time Digitized": "16", + "Focal Plane X Resolution": "32768/61166933 cm", + "Max Aperture Value": "f/4.6" + }, + "xmp": { + "crs:ParametricDarks": "0", + "dc:format": "image/jpeg", + "crs:DefringeGreenHueLo": "40", + "crs:SaturationAdjustmentPurple": "0", + "crs:HasSettings": "True", + "crs:GreenHue": "0", + "photoshop:ColorMode": "3", + "dc:description": [ + "Pic shows: What would Mandela do?\n\nNelson Mandela statue appears to be either removing of placing the box on Winston Churchill’s statue in Parliament Square today\n\nafter it was actually covered Sadiq Khan ordered the council to cover the memorial due to Black Lives Matter protests\n\n\n\n\npicture by Gavin Rodgers/ Pixel8000", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:SaturationAdjustmentAqua": "0", + "crs:UprightVersion": "151388160", + "crs:SharpenDetail": "25", + "crs:GrainAmount": "0", + "xmp:MetadataDate": "2020-06-12T14:39:59.000Z", + "crs:CameraProfile": "Adobe Standard", + "aux:LensSerialNumber": "9720000826", + "crs:WhiteBalance": "As Shot", + "crs:ToneCurveRed": [ + "0, 0", + "255, 255" + ], + "crs:ColorNoiseReductionDetail": "50", + "crs:Blacks2012": "-33", + "crs:SaturationAdjustmentBlue": "0", + "xmpMM:DerivedFrom": [ + "{'stRef:instanceID':'xmp.iid:5eb5ca6f-073b-4823-a9e6-f689444584d7'}", + "{'stRef:documentID':'xmp.did:c2407944-aa6a-45df-9cd6-e408cba2e9e9'}", + "{'stRef:originalDocumentID':'E83BCB4532750F9B267DF41F7CC37811'}" + ], + "crs:UprightPreview": "False", + "crs:ToneCurveName": "Medium Contrast", + "aux:ImageNumber": "6280", + "dc:date": [ + "2020-06-12T14:01:30.000Z" + ], + "crs:SplitToningShadowSaturation": "0", + "crs:SplitToningShadowHue": "0", + "crs:ParametricHighlightSplit": "75", + "crs:PerspectiveRotate": "0.0", + "crs:PerspectiveUpright": "0", + "photoshop:ICCProfile": "Adobe RGB (1998)", + "crs:PerspectiveX": "0.00", + "crs:DefringePurpleHueLo": "30", + "crs:ToneCurveName2012": "Linear", + "crs:ToneCurvePV2012": [ + "0, 0", + "255, 255" + ], + "crs:HueAdjustmentPurple": "0", + "crs:VignetteAmount": "0", + "crs:Exposure2012": "+0.25", + "crs:ParametricShadowSplit": "25", + "crs:LuminanceAdjustmentAqua": "0", + "crs:LuminanceAdjustmentPurple": "0", + "photomechanic:PMVersion": "PM6", + "crs:SplitToningBalance": "0", + "crs:SaturationAdjustmentMagenta": "0", + "crs:Sharpness": "40", + "crs:HueAdjustmentAqua": "0", + "dc:subject": [ + "Sadiq Khan, statue, Nelson Mandela, Westminster, Parliament Square, London, protests, race relations" + ], + "crs:ColorNoiseReductionSmoothness": "50", + "crs:BlueHue": "0", + "crs:DefringeGreenHueHi": "60", + "crs:SaturationAdjustmentOrange": "0", + "crs:HueAdjustmentYellow": "0", + "crs:SplitToningHighlightHue": "0", + "crs:PerspectiveScale": "100", + "crs:HueAdjustmentGreen": "0", + "aux:FlashCompensation": "0/1", + "photomechanic:Prefs": "0:1:0:006280", + "crs:Temperature": "5200", + "crs:ConvertToGrayscale": "False", + "crs:ParametricLights": "0", + "aux:Lens": "EF100-400mm f/4.5-5.6L IS II USM", + "crs:UprightFocalMode": "0", + "photomechanic:Tagged": "False", + "crs:ToneCurvePV2012Green": [ + "0, 0", + "255, 255" + ], + "crs:ParametricHighlights": "0", + "crs:OverrideLookVignette": "False", + "crs:SharpenEdgeMasking": "0", + "crs:ToneCurve": [ + "0, 0", + "32, 22", + "64, 56", + "128, 128", + "192, 196", + "255, 255" + ], + "crs:SaturationAdjustmentRed": "0", + "crs:Whites2012": "+46", + "crs:Clarity2012": "+14", + "crs:UprightFocalLength35mm": "35", + "xmp:ModifyDate": "2020-06-12T14:39:59.000Z", + "crs:UprightTransformCount": "6", + "crs:UprightCenterNormX": "0.5", + "photoshop:City": "London", + "xmpMM:OriginalDocumentID": "E83BCB4532750F9B267DF41F7CC37811", + "xmp:Rating": "0", + "crs:AutoToneDigestNoSat": "06984843E46C78312DAF35D2B764C928", + "crs:ToneCurvePV2012Blue": [ + "0, 0", + "255, 255" + ], + "crs:Look": [ + "{'crs:Parameters':['{'crs:ToneCurvePV2012Blue':['0, 0','255, 255']}','{'crs:ToneCurvePV2012Red':['0, 0','255, 255']}','{'crs:ToneCurvePV2012':['0, 0','22, 16','40, 35','127, 127','224, 230','240, 246','255, 255']}','{'crs:ToneCurvePV2012Green':['0, 0','255, 255']}']}", + "{'crs:Group':['Profiles']}", + "{'crs:Parameters':['{'crs:LookTable':'E1095149FDB39D7A057BAB208837E2E1'}','{'crs:CameraProfile':'Adobe Standard'}','{'crs:ConvertToGrayscale':'False'}','{'crs:Version':'12.2.1'}','{'crs:ProcessVersion':'11.0'}']}", + "{'crs:SupportsMonochrome':'false'}", + "{'crs:UUID':'B952C231111CD8E0ECCF14B86BAA7077'}", + "{'crs:Amount':'1.000000'}", + "{'crs:SupportsOutputReferred':'false'}", + "{'crs:SupportsAmount':'false'}", + "{'crs:Copyright':'© 2018 Adobe Systems, Inc.'}", + "{'crs:Name':'Adobe Color'}", + "{'crs:Group':[['{'xml:lang':'x-default'}']]}" + ], + "crs:LuminanceAdjustmentRed": "0", + "crs:DefringeGreenAmount": "0", + "crs:PerspectiveVertical": "0", + "crs:HueAdjustmentBlue": "0", + "crs:PerspectiveY": "0.00", + "crs:LensManualDistortionAmount": "0", + "crs:AutoLateralCA": "0", + "crs:LuminanceSmoothing": "0", + "crs:CameraProfileDigest": "661433344C8532AFA5A1E9091401E43C", + "crs:Contrast2012": "+17", + "photoshop:DateCreated": "2020-06-12T14:01:30.000Z", + "photoshop:Credit": "gavin rodgers/pixel8000", + "crs:SaturationAdjustmentYellow": "0", + "xmp:CreateDate": "2020-06-12T14:01:30.000Z", + "crs:LensProfileSetup": "LensDefaults", + "crs:Shadows2012": "+45", + "crs:UprightCenterMode": "0", + "crs:LensProfileEnable": "0", + "aux:LensID": "747", + "aux:ApproximateFocusDistance": "192/10", + "crs:Tint": "+6", + "crs:ParametricMidtoneSplit": "50", + "crs:PerspectiveHorizontal": "0", + "crs:ShadowTint": "0", + "crs:UprightCenterNormY": "0.5", + "dc:rights": [ + "Gavin Rodgers/ Pixel8000", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:HueAdjustmentMagenta": "0", + "crs:AutoToneDigest": "47AB1A2F4B28A8F46A86BB6CA1E72290", + "aux:LensInfo": "100/1 400/1 0/0 0/0", + "crs:ToneCurveBlue": [ + "0, 0", + "255, 255" + ], + "crs:LuminanceAdjustmentMagenta": "0", + "crs:ToneCurveGreen": [ + "0, 0", + "255, 255" + ], + "photomechanic:ColorClass": "1", + "crs:BlueSaturation": "0", + "crs:LuminanceAdjustmentYellow": "0", + "crs:AlreadyApplied": "True", + "crs:DefringePurpleHueHi": "70", + "crs:PostCropVignetteAmount": "0", + "xmp:Label": "Winner", + "xmpMM:DocumentID": "adobe:docid:photoshop:e34c6534-8d64-9842-81e1-3a10bd6f8fae", + "crs:PerspectiveAspect": "0", + "crs:LuminanceAdjustmentGreen": "0", + "crs:Dehaze": "+4", + "crs:RedSaturation": "0", + "xmpRights:Marked": "True", + "crs:SaturationAdjustmentGreen": "0", + "exifEX:LensModel": "EF100-400mm f/4.5-5.6L IS II USM", + "photoshop:Urgency": "1", + "dc:creator": [ + "gavin rodgers/pixel8000" + ], + "crs:LuminanceAdjustmentOrange": "0", + "crs:Version": "12.2.1", + "crs:ToneCurvePV2012Red": [ + "0, 0", + "255, 255" + ], + "crs:Texture": "0", + "crs:ToneMapStrength": "0", + "crs:UprightFourSegmentsCount": "0", + "crs:ParametricShadows": "0", + "crs:HueAdjustmentRed": "0", + "aux:Firmware": "1.0.1", + "crs:GreenSaturation": "0", + "crs:RedHue": "0", + "crs:SplitToningHighlightSaturation": "0", + "aux:SerialNumber": "018021000814", + "crs:DefringePurpleAmount": "0", + "crs:Saturation": "+2", + "crs:HueAdjustmentOrange": "0", + "crs:LuminanceAdjustmentBlue": "0", + "xmpMM:History": [ + [ + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 12.2.1 (Macintosh)'}", + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-06-12T15:38:35+01:00'}", + "{'stEvt:changed':'/metadata'}", + "{'stEvt:instanceID':'xmp.iid:940c3657-87c0-44a2-81e0-0a01312ff71e'}" + ], + [ + "{'stEvt:parameters':'converted from image/x-canon-cr2 to image/tiff'}", + "{'stEvt:action':'derived'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-06-12T15:38:36+01:00'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 12.2.1 (Macintosh)'}", + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:c2407944-aa6a-45df-9cd6-e408cba2e9e9'}" + ], + [ + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:5eb5ca6f-073b-4823-a9e6-f689444584d7'}", + "{'stEvt:when':'2020-06-12T15:39:59+01:00'}", + "{'stEvt:softwareAgent':'Adobe Photoshop 21.1 (Macintosh)'}", + "{'stEvt:action':'saved'}" + ], + [ + "{'stEvt:parameters':'from image/tiff to image/jpeg'}", + "{'stEvt:action':'converted'}" + ], + [ + "{'stEvt:action':'derived'}", + "{'stEvt:parameters':'converted from image/tiff to image/jpeg'}" + ], + [ + "{'stEvt:instanceID':'xmp.iid:7789b156-9aaf-4034-b0d3-9ba1356bb1d1'}", + "{'stEvt:action':'saved'}", + "{'stEvt:changed':'/'}", + "{'stEvt:softwareAgent':'Adobe Photoshop 21.1 (Macintosh)'}", + "{'stEvt:when':'2020-06-12T15:39:59+01:00'}" + ] + ], + "Iptc4xmpCore:CountryCode": "GBR", + "crs:Highlights2012": "-64", + "photoshop:Country": "United Kingdom", + "crs:ProcessVersion": "11.0", + "crs:Vibrance": "+15", + "crs:SharpenRadius": "+1.0", + "xmpMM:InstanceID": "xmp.iid:7789b156-9aaf-4034-b0d3-9ba1356bb1d1", + "crs:ColorNoiseReduction": "25" + }, + "icc": { + "Profile Size": "560", + "Media White Point": "(0.9505, 1, 1.0891)", + "Red Colorant": "(0.6097, 0.3111, 0.0195)", + "Blue TRC": "0.0085908", + "Version": "2.1.0", + "Tag Count": "10", + "Profile Description": "Adobe RGB (1998)", + "Green Colorant": "(0.2053, 0.6257, 0.0609)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "none", + "Green TRC": "0.0085908", + "Red TRC": "0.0085908", + "Blue Colorant": "(0.1492, 0.0632, 0.7446)", + "Color space": "RGB", + "Primary Platform": "Apple Computer, Inc.", + "Profile Connection Space": "XYZ", + "CMM Type": "ADBE", + "Class": "Display Device", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1999:06:03 00:00:00", + "Profile Copyright": "Copyright 1999 Adobe Systems Incorporated" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "metadata": { + "dateTaken": "2020-06-12T14:01:30.160Z", + "description": "Pic shows: What would Mandela do?\n\nNelson Mandela statue appears to be either removing of placing the box on Winston Churchill’s statue in Parliament Square today\n\nafter it was actually covered Sadiq Khan ordered the council to cover the memorial due to Black Lives Matter protests\n\n\n\n\npicture by Gavin Rodgers/ Pixel8000", + "credit": "pixel8000", + "byline": "Gavin Rodgers", + "copyright": "Gavin Rodgers/ Pixel8000", + "keywords": [ + "Sadiq Khan, statue, Nelson Mandela, Westminster, Parliament Square, London, protests, race relations" + ], + "city": "London", + "country": "United Kingdom", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2020-06-12T14:01:30.160Z", + "description": "Pic shows: What would Mandela do?\n\nNelson Mandela statue appears to be either removing of placing the box on Winston Churchill’s statue in Parliament Square today\n\nafter it was actually covered Sadiq Khan ordered the council to cover the memorial due to Black Lives Matter protests\n\n\n\n\npicture by Gavin Rodgers/ Pixel8000", + "credit": "pixel8000", + "byline": "Gavin Rodgers", + "copyright": "Gavin Rodgers/ Pixel8000", + "keywords": [ + "Sadiq Khan, statue, Nelson Mandela, Westminster, Parliament Square, London, protests, race relations" + ], + "city": "London", + "country": "United Kingdom", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [ + { + "path": [ + "Daily Edit", + "Daily edit home" + ], + "pathId": "daily edit/daily edit home", + "description": "Daily edit home", + "actionData": { + "author": "fiona.shields@guardian.co.uk", + "date": "2020-06-12T15:04:11.817+00:00" + } + } + ], + "userMetadataLastModified": "2020-06-12T15:04:11.817Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "119960415d35d0f29d86b1513266296535b4d636", + "_score": 1, + "_source": { + "id": "119960415d35d0f29d86b1513266296535b4d636", + "uploadTime": "2020-02-24T09:02:54.358Z", + "uploadedBy": "guy.lane@guardian.co.uk", + "lastModified": "2020-02-24T09:08:18.715Z", + "identifiers": {}, + "uploadInfo": { + "filename": "_9DS9316.jpg" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/1/1/9/9/6/0/119960415d35d0f29d86b1513266296535b4d636", + "size": 8349532, + "mimeType": "image/jpeg", + "dimensions": { + "width": 4788, + "height": 3196 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/1/1/9/9/6/0/119960415d35d0f29d86b1513266296535b4d636", + "size": 9913, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 171 + } + }, + "fileMetadata": { + "iptc": { + "By-line Title": "photographer", + "Country/Primary Location Name": "spain", + "Country/Primary Location Code": "ESP", + "Copyright Notice": "Didier Bizet", + "Coded Character Set": "UTF-8", + "Application Record Version": "4", + "Caption/Abstract": "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + "Credit": "Didier Bizet", + "Source": "hans lucas", + "City": "Valencia", + "Digital Date Created": "2019:04:27", + "Keywords": "reborn;reborn doll;reborn dolls;dolls;baby;bebes;nouveau ne;new born;doll;poupee;phenomene;phenomenon;realism;hyper-realistic;reborn artist;mother;parents;mère;parents;fabrication;reborn workshop;silicon;reborn silicon", + "Time Created": "19:46:03+0000", + "By-line": "Didier Bizet", + "Object Name": "baby boom", + "Date Time Created Composite": "2019-04-27T19:46:03.000Z", + "Caption Writer/Editor": "photographer", + "Original Transmission Reference": "babyboom_bizet", + "Date Created": "2019:04:27" + }, + "exif": { + "Image Description": "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + "X Resolution": "300 dots per inch", + "Software": "Adobe Photoshop CC 2019 (Macintosh)", + "Photometric Interpretation": "RGB", + "Make": "NIKON CORPORATION", + "Image Width": "4788 pixels", + "Samples Per Pixel": "3 samples/pixel", + "Copyright": "Didier Bizet", + "Date/Time": "2020:01:02 17:46:06", + "Model": "NIKON D750", + "Orientation": "Top, left side (Horizontal / normal)", + "Resolution Unit": "Inch", + "Bits Per Sample": "8 8 8 bits/component/pixel", + "Y Resolution": "300 dots per inch", + "Artist": "Didier Bizet", + "Image Height": "3196 pixels" + }, + "exifSub": { + "CFA Pattern": "[Red,Green][Green,Blue]", + "Exif Version": "2.31", + "Subject Distance Range": "Unknown", + "Exposure Mode": "Manual exposure", + "ISO Speed Ratings": "2000", + "Lens Specification": "35mm f/1.8", + "Body Serial Number": "6085734", + "Custom Rendered": "Normal process", + "Exif Image Height": "3196 pixels", + "Flash": "Flash did not fire", + "Time Zone": "+02:00", + "White Balance": "Unknown", + "Focal Length": "35 mm", + "Date/Time Original": "2019:04:27 19:46:03", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/199 sec", + "Exif Image Width": "4788 pixels", + "Gain Control": "Low gain down", + "Sensing Method": "One-chip color area sensor", + "Scene Type": "Directly photographed image", + "Focal Plane Y Resolution": "32768/54886891 cm", + "Sub-Sec Time Original": "32", + "Exposure Time": "1/200 sec", + "Sharpness": "None", + "Metering Mode": "Center weighted average", + "Exposure Program": "Manual control", + "Digital Zoom Ratio": "1", + "Exposure Bias Value": "0 EV", + "F-Number": "f/4.0", + "Color Space": "Undefined", + "Lens Model": "35.0 mm f/1.8", + "Focal Length 35": "35 mm", + "Date/Time Digitized": "2019:04:27 19:46:03", + "Date/Time Original Composite": "2019-04-27T19:46:03.320Z", + "Aperture Value": "f/4.0", + "Sensitivity Type": "Recommended Exposure Index", + "Contrast": "None", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "cm", + "File Source": "Digital Still Camera (DSC)", + "Sub-Sec Time Digitized": "32", + "Focal Plane X Resolution": "32768/54886891 cm", + "Saturation": "None", + "Max Aperture Value": "f/1.7" + }, + "xmp": { + "crs:ParametricDarks": "0", + "dc:format": "image/jpeg", + "crs:DefringeGreenHueLo": "40", + "crs:SaturationAdjustmentPurple": "0", + "crs:CropConstrainToWarp": "0", + "crs:HasSettings": "True", + "crs:GreenHue": "0", + "photoshop:ColorMode": "3", + "dc:description": [ + "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:SaturationAdjustmentAqua": "0", + "crs:UprightVersion": "151388160", + "crs:SharpenDetail": "25", + "crs:GrainAmount": "0", + "xmp:MetadataDate": "2020-01-02T16:46:06.000Z", + "crs:CameraProfile": "Adobe Standard", + "crs:WhiteBalance": "Custom", + "crs:ToneCurveRed": [ + "0, 0", + "255, 255" + ], + "crs:ColorNoiseReductionDetail": "50", + "crs:Blacks2012": "+17", + "crs:SaturationAdjustmentBlue": "0", + "xmpMM:DerivedFrom": [ + "{'stRef:documentID':'8ACD7691E7D0DB5C6CB40855D7469AAE'}", + "{'stRef:originalDocumentID':'8ACD7691E7D0DB5C6CB40855D7469AAE'}" + ], + "crs:UprightPreview": "False", + "crs:ToneCurveName": "Linear", + "aux:ImageNumber": "46087", + "crs:SplitToningShadowSaturation": "0", + "crs:SplitToningShadowHue": "0", + "photoshop:TransmissionReference": "babyboom_bizet", + "crs:ParametricHighlightSplit": "75", + "crs:PerspectiveRotate": "0.0", + "crs:PerspectiveUpright": "0", + "crs:LuminanceNoiseReductionContrast": "0", + "crs:PerspectiveX": "0.00", + "crs:DefringePurpleHueLo": "30", + "crs:ToneCurveName2012": "Linear", + "crs:ToneCurvePV2012": [ + "0, 0", + "255, 255" + ], + "crs:HueAdjustmentPurple": "0", + "crs:VignetteAmount": "0", + "crs:Exposure2012": "0.00", + "photoshop:AuthorsPosition": "photographer", + "crs:ParametricShadowSplit": "25", + "crs:LuminanceAdjustmentAqua": "0", + "crs:CropAngle": "0", + "crs:LuminanceAdjustmentPurple": "0", + "crs:SplitToningBalance": "0", + "crs:SaturationAdjustmentMagenta": "0", + "crs:Sharpness": "40", + "crs:HueAdjustmentAqua": "0", + "dc:subject": [ + "reborn", + "reborn doll", + "reborn dolls", + "dolls", + "baby", + "bebes", + "nouveau ne", + "new born", + "doll", + "poupee", + "phenomene", + "phenomenon", + "realism", + "hyper-realistic", + "reborn artist", + "mother", + "parents", + "mère", + "parents", + "fabrication", + "reborn workshop", + "silicon", + "reborn silicon" + ], + "photoshop:CaptionWriter": "photographer", + "crs:ColorNoiseReductionSmoothness": "50", + "crs:BlueHue": "0", + "crs:HasCrop": "True", + "crs:DefringeGreenHueHi": "60", + "crs:SaturationAdjustmentOrange": "0", + "crs:HueAdjustmentYellow": "0", + "crs:SplitToningHighlightHue": "0", + "crs:PerspectiveScale": "100", + "crs:HueAdjustmentGreen": "0", + "crs:Temperature": "3301", + "crs:ConvertToGrayscale": "False", + "crs:RawFileName": "_9DS9316.NEF", + "crs:ParametricLights": "0", + "aux:Lens": "35.0 mm f/1.8", + "crs:UprightFocalMode": "0", + "crs:ToneCurvePV2012Green": [ + "0, 0", + "255, 255" + ], + "crs:LuminanceNoiseReductionDetail": "50", + "crs:ParametricHighlights": "0", + "crs:OverrideLookVignette": "False", + "crs:SharpenEdgeMasking": "32", + "crs:ToneCurve": [ + "0, 0", + "255, 255" + ], + "crs:SaturationAdjustmentRed": "0", + "crs:Whites2012": "0", + "crs:Clarity2012": "0", + "crs:UprightFocalLength35mm": "35", + "xmp:ModifyDate": "2020-01-02T16:46:06.000Z", + "crs:UprightTransformCount": "6", + "crs:UprightCenterNormX": "0.5", + "xmpRights:UsageTerms": [ + "All Rights reserved", + [ + "{'xml:lang':'x-default'}" + ] + ], + "photoshop:City": "Valencia", + "xmpMM:OriginalDocumentID": "8ACD7691E7D0DB5C6CB40855D7469AAE", + "xmp:Rating": "5", + "crs:ToneCurvePV2012Blue": [ + "0, 0", + "255, 255" + ], + "crs:Look": [ + "{'crs:Parameters':['{'crs:ToneCurvePV2012Blue':['0, 0','255, 255']}','{'crs:ToneCurvePV2012Red':['0, 0','255, 255']}','{'crs:ToneCurvePV2012':['0, 0','22, 16','40, 35','127, 127','224, 230','240, 246','255, 255']}','{'crs:ToneCurvePV2012Green':['0, 0','255, 255']}']}", + "{'crs:Group':['Profiles']}", + "{'crs:Parameters':['{'crs:LookTable':'E1095149FDB39D7A057BAB208837E2E1'}','{'crs:CameraProfile':'Adobe Standard'}','{'crs:ConvertToGrayscale':'False'}','{'crs:Version':'11.3'}','{'crs:ProcessVersion':'11.0'}']}", + "{'crs:SupportsMonochrome':'false'}", + "{'crs:UUID':'B952C231111CD8E0ECCF14B86BAA7077'}", + "{'crs:Amount':'1.000000'}", + "{'crs:SupportsOutputReferred':'false'}", + "{'crs:Name':'Adobe Color'}", + "{'crs:SupportsAmount':'false'}", + "{'crs:Group':[['{'xml:lang':'x-default'}']]}" + ], + "crs:LuminanceAdjustmentRed": "0", + "crs:DefringeGreenAmount": "0", + "crs:PerspectiveVertical": "0", + "crs:HueAdjustmentBlue": "0", + "crs:PerspectiveY": "0.00", + "crs:LensManualDistortionAmount": "0", + "crs:AutoLateralCA": "0", + "crs:LuminanceSmoothing": "17", + "crs:CameraProfileDigest": "5F02E85A544BE2CCC1DFAA73D311C57C", + "crs:Contrast2012": "0", + "photoshop:DateCreated": "2019-04-27T19:46:03.032Z", + "photoshop:Credit": "Didier Bizet", + "crs:SaturationAdjustmentYellow": "0", + "xmp:CreateDate": "2019-04-27T19:46:03.000Z", + "crs:LensProfileSetup": "LensDefaults", + "crs:Shadows2012": "+19", + "crs:UprightCenterMode": "0", + "crs:CropRight": "0.884268", + "crs:LensProfileEnable": "0", + "aux:LensID": "165", + "aux:ApproximateFocusDistance": "79/100", + "crs:Tint": "+11", + "crs:ParametricMidtoneSplit": "50", + "xmpRights:WebStatement": "hanslucas.com", + "crs:PerspectiveHorizontal": "0", + "crs:ShadowTint": "0", + "crs:UprightCenterNormY": "0.5", + "dc:rights": [ + "Didier Bizet", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:HueAdjustmentMagenta": "0", + "xmp:CreatorTool": "Adobe Photoshop CC 2019 (Macintosh)", + "aux:LensInfo": "350/10 350/10 18/10 18/10", + "crs:ToneCurveBlue": [ + "0, 0", + "255, 255" + ], + "crs:LuminanceAdjustmentMagenta": "0", + "photoshop:LegacyIPTCDigest": "CFA151707B37962499516E28485C8B15", + "crs:ToneCurveGreen": [ + "0, 0", + "255, 255" + ], + "crs:BlueSaturation": "0", + "crs:LuminanceAdjustmentYellow": "0", + "crs:AlreadyApplied": "True", + "crs:DefringePurpleHueHi": "70", + "crs:PostCropVignetteAmount": "0", + "xmpMM:DocumentID": "xmp.did:d080270a-0a88-4df3-b158-8c47f73340ef", + "crs:PerspectiveAspect": "0", + "crs:LuminanceAdjustmentGreen": "0", + "crs:Dehaze": "0", + "crs:RedSaturation": "0", + "xmpRights:Marked": "True", + "crs:SaturationAdjustmentGreen": "0", + "dc:creator": [ + "Didier Bizet" + ], + "crs:LuminanceAdjustmentOrange": "0", + "crs:Version": "11.3", + "crs:ToneCurvePV2012Red": [ + "0, 0", + "255, 255" + ], + "crs:Texture": "0", + "Iptc4xmpCore:CreatorContactInfo": [ + "{'Iptc4xmpCore:CiUrlWork':'http://hanslucas.com/dbizet/photo ; http://didierbizet.com'}", + "{'Iptc4xmpCore:CiEmailWork':'dbizet@orange.fr'}", + "{'Iptc4xmpCore:CiTelWork':'0033675952033'}", + "{'Iptc4xmpCore:CiAdrCity':'Paris'}", + "{'Iptc4xmpCore:CiAdrPcode':'75019'}", + "{'Iptc4xmpCore:CiAdrCtry':'France'}", + "{'Iptc4xmpCore:CiAdrExtadr':'30 rue clavel'}" + ], + "crs:ToneMapStrength": "0", + "crs:UprightFourSegmentsCount": "0", + "crs:CropTop": "0.177282", + "crs:ParametricShadows": "0", + "crs:HueAdjustmentRed": "0", + "crs:GreenSaturation": "0", + "crs:CropLeft": "0.088464", + "crs:RedHue": "0", + "dc:title": [ + "baby boom", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:SplitToningHighlightSaturation": "0", + "aux:SerialNumber": "6085734", + "crs:DefringePurpleAmount": "0", + "crs:Saturation": "0", + "crs:HueAdjustmentOrange": "0", + "crs:LuminanceAdjustmentBlue": "0", + "xmpMM:PreservedFileName": "_9DS9316.NEF", + "xmpMM:History": [ + [ + "{'stEvt:parameters':'converted from image/x-nikon-nef to image/jpeg, saved to new location'}", + "{'stEvt:action':'derived'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Lightroom Classic 8.3 (Macintosh)'}", + "{'stEvt:instanceID':'xmp.iid:d080270a-0a88-4df3-b158-8c47f73340ef'}", + "{'stEvt:changed':'/'}", + "{'stEvt:when':'2019-06-14T13:15:17+02:00'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-01-02T17:46:06+01:00'}", + "{'stEvt:softwareAgent':'Adobe Photoshop CC 2019 (Macintosh)'}", + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:15644f3f-1e61-4033-a456-8b43605ed09b'}" + ] + ], + "Iptc4xmpCore:CountryCode": "ESP", + "crs:Highlights2012": "0", + "crs:CropBottom": "0.973086", + "photoshop:Country": "spain", + "crs:ProcessVersion": "11.0", + "photoshop:Source": "hans lucas", + "crs:Vibrance": "0", + "crs:SharpenRadius": "+1.0", + "xmpMM:InstanceID": "xmp.iid:15644f3f-1e61-4033-a456-8b43605ed09b", + "crs:ColorNoiseReduction": "25" + }, + "icc": {}, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "photometricInterpretation": "RGB", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [ + "reborn" + ], + "usageRights": { + "category": "chargeable", + "restrictions": "PHOTO ESSAY ONLY" + }, + "lastModified": "2020-02-24T09:08:18.715Z" + }, + "metadata": { + "dateTaken": "2019-04-27T19:46:03.320Z", + "description": "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + "credit": "Didier Bizet", + "byline": "Didier Bizet", + "bylineTitle": "photographer", + "copyright": "Didier Bizet", + "suppliersReference": "babyboom_bizet", + "source": "hans lucas", + "keywords": [ + "mother", + "fabrication", + "reborn dolls", + "phenomenon", + "doll", + "poupee", + "dolls", + "parents", + "hyper-realistic", + "reborn workshop", + "mère", + "realism", + "phenomene", + "reborn artist", + "reborn", + "new born", + "silicon", + "bebes", + "nouveau ne", + "baby", + "reborn doll", + "reborn silicon" + ], + "city": "Valencia", + "country": "Spain", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2019-04-27T19:46:03.320Z", + "description": "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + "credit": "Didier Bizet", + "byline": "Didier Bizet", + "bylineTitle": "photographer", + "copyright": "Didier Bizet", + "suppliersReference": "babyboom_bizet", + "source": "hans lucas", + "keywords": [ + "mother", + "fabrication", + "reborn dolls", + "phenomenon", + "doll", + "poupee", + "dolls", + "parents", + "hyper-realistic", + "reborn workshop", + "mère", + "realism", + "phenomene", + "reborn artist", + "reborn", + "new born", + "silicon", + "bebes", + "nouveau ne", + "baby", + "reborn doll", + "reborn silicon" + ], + "city": "Valencia", + "country": "Spain", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": { + "category": "chargeable", + "restrictions": "PHOTO ESSAY ONLY" + }, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2020-02-24T09:08:18.715Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "1e8491b56759fdf85c28dd0b02c593ba468ba3b9", + "_score": 1, + "_source": { + "id": "1e8491b56759fdf85c28dd0b02c593ba468ba3b9", + "uploadTime": "2015-07-27T17:37:11.000Z", + "uploadedBy": "DEV Ingest Script", + "identifiers": { + "picdarurn": "PD*1693653" + }, + "uploadInfo": {}, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/1/e/8/4/9/1/1e8491b56759fdf85c28dd0b02c593ba468ba3b9", + "size": 301115, + "mimeType": "image/jpeg", + "dimensions": { + "width": 1534, + "height": 2160 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/1/e/8/4/9/1/1e8491b56759fdf85c28dd0b02c593ba468ba3b9", + "size": 17159, + "mimeType": "image/jpeg", + "dimensions": { + "width": 182, + "height": 256 + } + }, + "fileMetadata": { + "iptc": { + "By-line Title": "Atlas Photography", + "Country/Primary Location Name": "United States", + "Category": "ACE", + "Country/Primary Location Code": "USA", + "Copyright Notice": "Atlas Photography", + "Supplemental Category(s)": "ENT", + "Application Record Version": "2", + "Caption/Abstract": "Casey Chaos, lead singert with the group Amen on the Nautica Stage, in Cleveland Ohio, USA.\r\n\r\n\"Amen's more pissed off than we ever were.\" So says former Sex Pistol Steve Jones of L.A. punk band Amen. The group took shape in the mid-'90s when frontman Casey Chaos hooked up with guitarist Paul Fig and began experimenting with their raw, visceral sound and in-your-face lyrics. Later adding Larkin on drums, Tumor on bass and ex-Snot guitarist Sonny Mayo on guitar, the lineup of Amen was complete. \r\n\r\nThe band signed to Ross Robinson's I Am Records, an imprint of Roadrunner, to record their self -- titled debut. \"Amen\", produced by Robinson, was released in 1999, followed by Amen's sophomore effort, We Have Come For Your Parents, the following year. \r\n\r\nDate Taken: 19/08/2001\r\nPicture Credit: Amy L Weiser/Atlas Photography\r\n\r\nAtlas Photography\r\n20 The Limes\r\nBurniston\r\nNorth Yorkshire\r\nYO13 OEE - England\r\n\r\nTel: +44(0)1723 870659\r\nEmail: atlasphotography@ukonline.co.uk", + "Credit": "Amy L Weiser/Atlas Photography", + "Source": "Atlas Photography", + "City": "Cleveland", + "Keywords": "controversial;vertical;upright;thrash;stage;singing;rock;portrait;playing;performing;outrageous;music;metal;loud;live;heavy", + "By-line": "Atlas Photography", + "Urgency": "48", + "Headline": "Casey Chaos of the group Amen", + "Province/State": "Ohio", + "Object Name": "Casey Chaos of the group Amen", + "Caption Writer/Editor": "SN", + "Date Created": "2001:08:19" + }, + "exif": {}, + "exifSub": {}, + "xmp": {}, + "icc": { + "Profile Size": "3144", + "Media White Point": "(0.9505, 1, 1.0891)", + "Device model": "sRGB", + "Red Colorant": "(0.4361, 0.2225, 0.0139)", + "Blue TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Viewing Conditions Description": "Reference Viewing Condition in IEC61966-2.1", + "Luminance": "(76.0365, 80, 87.1246)", + "Viewing Conditions": "view (0x76696577): 36 bytes", + "Version": "2.1.0", + "Tag Count": "17", + "Profile Description": "sRGB IEC61966-2.1", + "Green Colorant": "(0.3851, 0.7169, 0.0971)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "IEC", + "Device Mfg Description": "IEC http://www.iec.ch", + "Green TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Red TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Blue Colorant": "(0.1431, 0.0606, 0.7141)", + "Color space": "RGB", + "Primary Platform": "Microsoft Corporation", + "Profile Connection Space": "XYZ", + "CMM Type": "Lino", + "Class": "Display Device", + "Measurement": "1931 2° Observer, Backing (0, 0, 0), Geometry Unknown, Flare 1%, Illuminant D65", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1998:02:09 06:49:00", + "Device Model Description": "IEC 61966-2.1 Default RGB colour space - sRGB", + "Profile Copyright": "Copyright (c) 1998 Hewlett-Packard Company", + "Technology": "CRT" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "Casey Chaos, lead singert with the group Amen on the Nautica Stage, in Cleveland Ohio, USA.\n \n \"Amen's more pissed off than we ever were.\" So says former Sex Pistol Steve Jones of L.A. punk band Amen. The group took shape in the mid-'90s when frontman Casey Chaos hooked up with guitarist Paul Fig and began experimenting with their raw, visceral sound and in-your-face lyrics. Later adding Larkin on drums, Tumor on bass and ex-Snot guitarist Sonny Mayo on guitar, the lineup of Amen was complete. \n \n The band signed to Ross Robinson's I Am Records, an imprint of Roadrunner, to record their self -- titled debut. \"Amen\", produced by Robinson, was released in 1999, followed by Amen's sophomore effort, We Have Come For Your Parents, the following year. \n \n Date Taken: 19/08/2001\n Picture Credit: Amy L Weiser/Atlas Photography\n \n Atlas Photography\n 20 The Limes\n Burniston\n North Yorkshire\n YO13 OEE - England\n \n Tel: +44(0)1723 870659\n Email: atlasphotography@ukonline.co.uk\n " + } + }, + "metadata": { + "description": "Casey Chaos, lead singert with the group Amen on the Nautica Stage, in Cleveland Ohio, USA.\n \n \"Amen's more pissed off than we ever were.\" So says former Sex Pistol Steve Jones of L.A. punk band Amen. The group took shape in the mid-'90s when frontman Casey Chaos hooked up with guitarist Paul Fig and began experimenting with their raw, visceral sound and in-your-face lyrics. Later adding Larkin on drums, Tumor on bass and ex-Snot guitarist Sonny Mayo on guitar, the lineup of Amen was complete. \n \n The band signed to Ross Robinson's I Am Records, an imprint of Roadrunner, to record their self -- titled debut. \"Amen\", produced by Robinson, was released in 1999, followed by Amen's sophomore effort, We Have Come For Your Parents, the following year. \n \n Date Taken: 19/08/2001\n Picture Credit: Amy L Weiser/Atlas Photography\n \n Atlas Photography\n 20 The Limes\n Burniston\n North Yorkshire\n YO13 OEE - England\n \n Tel: +44(0)1723 870659\n Email: atlasphotography@ukonline.co.uk\n ", + "credit": "Amy L Weiser", + "byline": "Atlas Photography", + "bylineTitle": "Atlas Photography", + "title": "Casey Chaos of the group Amen", + "copyright": "Atlas Photography", + "suppliersReference": "Casey Chaos of the group Amen", + "source": "Atlas Photography", + "keywords": [ + "rock", + "stage", + "loud", + "music", + "portrait", + "performing", + "vertical", + "heavy", + "live", + "outrageous", + "playing", + "thrash", + "metal", + "controversial", + "singing", + "upright" + ], + "city": "Cleveland", + "state": "Ohio", + "country": "United States", + "subjects": [ + "arts" + ], + "peopleInImage": [] + }, + "originalMetadata": { + "description": "Casey Chaos, lead singert with the group Amen on the Nautica Stage, in Cleveland Ohio, USA.\r\n\r\n\"Amen's more pissed off than we ever were.\" So says former Sex Pistol Steve Jones of L.A. punk band Amen. The group took shape in the mid-'90s when frontman Casey Chaos hooked up with guitarist Paul Fig and began experimenting with their raw, visceral sound and in-your-face lyrics. Later adding Larkin on drums, Tumor on bass and ex-Snot guitarist Sonny Mayo on guitar, the lineup of Amen was complete. \r\n\r\nThe band signed to Ross Robinson's I Am Records, an imprint of Roadrunner, to record their self -- titled debut. \"Amen\", produced by Robinson, was released in 1999, followed by Amen's sophomore effort, We Have Come For Your Parents, the following year. \r\n\r\nDate Taken: 19/08/2001\r\nPicture Credit: Amy L Weiser/Atlas Photography\r\n\r\nAtlas Photography\r\n20 The Limes\r\nBurniston\r\nNorth Yorkshire\r\nYO13 OEE - England\r\n\r\nTel: +44(0)1723 870659\r\nEmail: atlasphotography@ukonline.co.uk", + "credit": "Amy L Weiser", + "byline": "Atlas Photography", + "bylineTitle": "Atlas Photography", + "title": "Casey Chaos of the group Amen", + "copyright": "Atlas Photography", + "suppliersReference": "Casey Chaos of the group Amen", + "source": "Atlas Photography", + "keywords": [ + "rock", + "stage", + "loud", + "music", + "portrait", + "performing", + "vertical", + "heavy", + "live", + "outrageous", + "playing", + "thrash", + "metal", + "controversial", + "singing", + "upright" + ], + "city": "Cleveland", + "state": "Ohio", + "country": "United States", + "subjects": [ + "arts" + ], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [] + } + } + ] + } + } +} diff --git a/e2e-tests/fixtures/images/test-card-f.jpg b/e2e-tests/fixtures/images/test-card-f.jpg new file mode 100644 index 00000000000..a57b764b234 Binary files /dev/null and b/e2e-tests/fixtures/images/test-card-f.jpg differ diff --git a/e2e-tests/fixtures/images/test.jpg b/e2e-tests/fixtures/images/test.jpg new file mode 100644 index 00000000000..ed01477ed6b Binary files /dev/null and b/e2e-tests/fixtures/images/test.jpg differ diff --git a/e2e-tests/fixtures/permissions/permissions.json b/e2e-tests/fixtures/permissions/permissions.json new file mode 100644 index 00000000000..b6176cd6c8e --- /dev/null +++ b/e2e-tests/fixtures/permissions/permissions.json @@ -0,0 +1,74 @@ +[ + { + "permission": { + "name": "grid_access", + "app": "grid", + "defaultValue": true + }, + "overrides": [{ + "userId": "e2e.user@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] + }, + { + "permission": { + "name": "edit_metadata", + "app": "grid", + "defaultValue": true + }, + "overrides": [{ + "userId": "e2e.user@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] + }, + { + "permission": { + "name": "delete_image", + "app": "grid", + "defaultValue": true + }, + "overrides": [{ + "userId": "e2e.user@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] + }, + { + "permission": { + "name": "delete_crops_or_usages", + "app": "grid", + "defaultValue": true + }, + "overrides": [{ + "userId": "e2e.user@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] + }, + { + "permission": { + "name": "delete_crops", + "app": "grid", + "defaultValue": true + }, + "overrides": [{ + "userId": "e2e.user@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] + }, + { + "permission": { + "name": "show_paid", + "app": "grid", + "defaultValue": true + }, + "overrides": [{ + "userId": "e2e.user@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] + } +] diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile new file mode 100644 index 00000000000..579c271a5b4 --- /dev/null +++ b/e2e-tests/images/Dockerfile @@ -0,0 +1,187 @@ +# syntax=docker/dockerfile:1 + +# Single-container image that builds and runs all Grid Play services under test +# (auth, collections, cropper, image-loader, kahuna, leases, media-api, +# metadata-editor, thrall) together with Kahuna's frontend. +# +# This one Dockerfile produces two images, selected with `--target`: +# +# * ci (grid-e2e-ci) — CI/production style. Stages pre-compiled artefacts and +# runs them under a slim JRE. docker build --target ci +# -f e2e-tests/images/Dockerfile -t grid-e2e-ci . +# +# * dev (grid-e2e-dev) — local development. Runs the services under +# `sbt /run` (Play dev mode) with the repo +# bind-mounted over /build so host edits recompile live. +# docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . +# +# `--target` is required: a plain `docker build` hits the `default` guard stage +# below and fails with a reminder. See images/README.md for details. +# +# Runtime configuration is not baked into the ci image. Mount per-environment +# config (and a stage marker) into /etc/grid at run time, e.g.: +# docker run --rm -v "$PWD/my-config:/etc/grid" grid-e2e-ci + +# Shared build args (single source of truth for both images). +ARG SBT_VERSION=1.10.3 +ARG NODE_VERSION=22.12.0 +# Native tools used by the image processing services (see repo Brewfile). Kept +# as one arg so the ci and dev stages install the same set. +ARG NATIVE_PKGS="graphicsmagick imagemagick pngquant libimage-exiftool-perl libgd3" + +# Stage: build the Kahuna frontend (webpack production bundle). Uses the official +# Node image because this throwaway build stage only needs Node + npm. +FROM node:${NODE_VERSION}-bookworm AS frontend + +WORKDIR /build/kahuna + +# Install dependencies first for better layer caching. +COPY kahuna/package.json kahuna/package-lock.json ./ +RUN npm ci + +# Build the production bundle into kahuna/public/dist. +COPY kahuna/ ./ +RUN npm run dist + +# Stage: shared JDK + sbt toolchain used by both `backend` (prod) and `dev`. +FROM eclipse-temurin:11-jdk-jammy AS toolchain + +ARG SBT_VERSION + +# Install sbt from the official release tarball. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends curl ca-certificates; \ + curl -fLo /tmp/sbt.tgz "https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz"; \ + tar -xzf /tmp/sbt.tgz -C /opt; \ + ln -s /opt/sbt/bin/sbt /usr/local/bin/sbt; \ + rm -f /tmp/sbt.tgz; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +# Stage: compile and stage the Play services with sbt (prod path). +FROM toolchain AS backend + +# Copy the full repository (build context is the repo root). A .dockerignore +# keeps target/, node_modules/ and logs/ out of the context. +COPY --exclude=e2e-tests/images . . + +# Drop in the pre-built frontend so `kahuna/stage` packages the bundled assets. +COPY --from=frontend /build/kahuna/public/dist ./kahuna/public/dist + +# Stage every requested service in a single sbt invocation to share compilation. +RUN sbt \ + "auth/stage" \ + "collections/stage" \ + "cropper/stage" \ + "image-loader/stage" \ + "kahuna/stage" \ + "leases/stage" \ + "media-api/stage" \ + "metadata-editor/stage" \ + "thrall/stage" + +# Stage: slim ci image with native image tooling and all services (grid-e2e-ci). +FROM eclipse-temurin:11-jre-jammy AS ci + +ARG NATIVE_PKGS + +# Native tools used by the image processing services (see repo Brewfile). +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends ${NATIVE_PKGS}; \ + rm -rf /var/lib/apt/lists/* + +# Staged apps expect their config at /usr/share//conf/application.conf +# and write gc logs to /var/log// (see build.sbt javaOptions). +COPY --from=backend /build/auth/target/universal/stage /usr/share/auth +COPY --from=backend /build/collections/target/universal/stage /usr/share/collections +COPY --from=backend /build/cropper/target/universal/stage /usr/share/cropper +COPY --from=backend /build/image-loader/target/universal/stage /usr/share/image-loader +COPY --from=backend /build/kahuna/target/universal/stage /usr/share/kahuna +COPY --from=backend /build/leases/target/universal/stage /usr/share/leases +COPY --from=backend /build/media-api/target/universal/stage /usr/share/media-api +COPY --from=backend /build/metadata-editor/target/universal/stage /usr/share/metadata-editor +COPY --from=backend /build/thrall/target/universal/stage /usr/share/thrall + +RUN set -eux; \ + for service in auth collections cropper image-loader kahuna leases media-api metadata-editor thrall; do \ + mkdir -p "/var/log/$service"; \ + done + +COPY e2e-tests/images/entrypoint.common.sh /usr/local/bin/entrypoint.common.sh +COPY e2e-tests/images/entrypoint.ci.sh /usr/local/bin/entrypoint.ci.sh +RUN chmod +x /usr/local/bin/entrypoint.ci.sh + +# The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. +ENV AWS_CBOR_DISABLE=true + +# media-api, thrall, image-loader, kahuna, cropper, metadata-editor, collections, auth, leases +EXPOSE 9001 9002 9003 9005 9006 9007 9010 9011 9012 + +ENTRYPOINT ["/usr/local/bin/entrypoint.ci.sh"] + +# Stage: development image (grid-e2e-dev). JDK + sbt + Node + native tooling in a +# single stage with warmed caches, running services under `sbt /run`. +FROM toolchain AS dev + +ARG NODE_VERSION +ARG NATIVE_PKGS +# Provided automatically by BuildKit (e.g. amd64, arm64). +ARG TARGETARCH + +# xz-utils is needed to unpack the Node tarball; the native tools mirror the +# ci stage. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends xz-utils ${NATIVE_PKGS}; \ + rm -rf /var/lib/apt/lists/* + +# Install Node (pinned to match kahuna's required version) from the official +# tarball so `npm run watch` matches local development. +RUN set -eux; \ + case "${TARGETARCH:-amd64}" in \ + amd64) node_arch=x64 ;; \ + arm64) node_arch=arm64 ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + curl -fLo /tmp/node.tar.xz "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz"; \ + tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner; \ + rm -f /tmp/node.tar.xz; \ + node --version; \ + npm --version + +# Warm the sbt dependency cache (~/.ivy2, ~/.sbt, ~/.cache/coursier). These live +# under $HOME, so a runtime bind-mount over /build does not shadow them: the +# first runtime compile only needs to compile sources, not re-resolve deps. +# Copying just the build definition keeps this layer cached across source edits. +COPY build.sbt ./ +COPY project/ ./project/ +RUN sbt update + +# Warm Kahuna's node_modules. The dockerignore keeps node_modules out of the +# build context, so this install is not clobbered by the COPY below. +COPY kahuna/package.json kahuna/package-lock.json ./kahuna/ +RUN cd kahuna && npm ci + +# Copy the rest of the source so the image is runnable standalone; a runtime +# bind-mount of the repo over /build overrides this for live editing. +COPY . . + +COPY e2e-tests/images/entrypoint.dev.sh /usr/local/bin/entrypoint.dev.sh +RUN chmod +x /usr/local/bin/entrypoint.dev.sh + +# The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. +ENV AWS_CBOR_DISABLE=true + +# media-api, thrall, image-loader, kahuna, cropper, metadata-editor, collections, auth, leases +# plus the JDWP debug port used when GRID_DEBUG is set. +EXPOSE 9001 9002 9003 9005 9006 9007 9010 9011 9012 5005 + +ENTRYPOINT ["/usr/local/bin/entrypoint.dev.sh"] + +# Stage: default guard. This is the last stage, so a plain `docker build` (no +# --target) builds it and fails, forcing an explicit `--target ci|dev`. +FROM eclipse-temurin:11-jre-jammy AS default +RUN echo 'ERROR: --target is required. Pass --target ci (grid-e2e-ci) or --target dev (grid-e2e-dev) when building the image' >&2; exit 1 diff --git a/e2e-tests/images/Dockerfile.dockerignore b/e2e-tests/images/Dockerfile.dockerignore new file mode 100644 index 00000000000..f28462df9be --- /dev/null +++ b/e2e-tests/images/Dockerfile.dockerignore @@ -0,0 +1,10 @@ +# BuildKit uses this file (named .dockerignore) to trim the build +# context for images/Dockerfile. Keeps build artefacts and caches out of the +# context sent to the daemon. +**/target +**/node_modules +**/logs +**/.git +.git +**/.DS_Store +dev/.localstack diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md new file mode 100644 index 00000000000..c23ff06cd75 --- /dev/null +++ b/e2e-tests/images/README.md @@ -0,0 +1,70 @@ +c# Grid E2E test images + +A single `Dockerfile` produces two single-container images that run all +Grid Play services under test, plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter: +- **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. +- **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. + +`--target` is required — a plain `docker build` fails with a reminder. + +## Build + +Build from the **repository root** (the build context must be the repo root): + +```bash +DOCKER_BUILDKIT=1 docker build --target -f e2e-tests/images/Dockerfile -t grid-e2e- . +``` + +## Runtime configuration + +Configuration is not baked into the image. At runtime each service loads +`/etc/grid/common.conf` and `/etc/grid/.conf` (and reads the stage +from `/etc/grid/stage`; defaults to `DEV`). Mount your environment's config with: + +```bash +docker run --rm \ + -v "$PWD/my-config:/etc/grid:ro" \ + -p 9001:9001 -p 9002:9002 -p 9003:9003 -p 9005:9005 -p 9006:9006 \ + -p 9007:9007 -p 9010:9010 -p 9011:9011 -p 9012:9012 \ + grid-e2e-[ci|dev] +``` + +## Development image (live reload) + +The `--target dev` build is for local development. Instead of staging compiled +artefacts it runs the services under `sbt /run` (Play dev mode), so changed Scala sources are recompiled on the next request. When `kahuna` is selected its webpack bundle is rebuilt continuously via `npm run watch`. + +Build from the repository root: + +```bash +DOCKER_BUILDKIT=1 docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . +``` + +Run with the repo bind-mounted over `/build` so host edits are picked up live, +and your DEV config mounted at `/root/.grid`: + +```bash +docker run --rm \ + -v "$PWD:/build" \ + -v "$HOME/.grid:/root/.grid:ro" \ + -p 9001:9001 -p 9005:9005 -p 9011:9011 \ + grid-e2e-dev +``` + +### Options + +All configurable via environment variables: + +| Variable | Default | Purpose | +| ----------------------- | -------------------- | -------------------------------------------------------------- | +| `GRID_DEBUG` | _(unset)_ | If set, opens a debug server on port `5005` (also `EXPOSE`d). | + +```bash +docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" \ + -e GRID_DEBUG=1 \ + -p 9001:9001 -p 9005:9005 -p 5005:5005 \ + grid-e2e-dev +``` + +The services still need their backing infrastructure reachable (see the root +`docker-compose.yml`). The image also works against the e2e-tests testcontainers harness: mount the generated config the same way the CI image does (to`/root/.grid` and/or `/etc/grid`) and join the same network. diff --git a/e2e-tests/images/entrypoint.ci.sh b/e2e-tests/images/entrypoint.ci.sh new file mode 100644 index 00000000000..84c4ef5d5e9 --- /dev/null +++ b/e2e-tests/images/entrypoint.ci.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# Supervises the Grid Play services inside a single container. +# +# Each staged application defaults to port 9000 in production, so the correct +# port is passed explicitly per service via -Dhttp.port. Services run in the +# background; if any one exits the container stops, and SIGTERM/SIGINT are +# forwarded for a clean shutdown. + +set -euo pipefail + +# Shared service list, port map, and shutdown helper. +source "$(dirname "$0")/entrypoint.common.sh" + +pids=() + +trap 'shutdown_children "${pids[@]}"' TERM INT + +for svc in $SERVICES; do + port="${PORTS[$svc]:-}" + if [[ -z "$port" ]]; then + echo "Unknown service '$svc' (no port mapping); skipping." >&2 + continue + fi + + bin="/usr/share/$svc/bin/$svc" + if [[ ! -x "$bin" ]]; then + echo "Executable for service '$svc' not found at $bin; skipping." >&2 + continue + fi + + echo "Starting $svc on port $port" + "$bin" -Dhttp.port="$port" & + pids+=("$!") +done + +# Block until the first service exits. During a healthy run the services stay up, +# so this only returns if one crashed; when that happens, tear the rest down and +# exit non-zero so the failure surfaces instead of leaving a half-running stack +# that still looks alive. +wait -n || true +echo "A Grid service exited unexpectedly; shutting down the rest." >&2 +for pid in "${pids[@]}"; do + kill -TERM "$pid" 2>/dev/null || true +done +wait 2>/dev/null || true +exit 1 diff --git a/e2e-tests/images/entrypoint.common.sh b/e2e-tests/images/entrypoint.common.sh new file mode 100644 index 00000000000..f4c9666ceba --- /dev/null +++ b/e2e-tests/images/entrypoint.common.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Shared definitions for the Grid container entrypoints (ci and dev). +# +# Sourced by entrypoint.ci.sh and entrypoint.dev.sh to provide a single source of +# truth for the service list, their http ports, and the graceful-shutdown trap. + +# service -> http port +declare -A PORTS=( + [media-api]=9001 + [thrall]=9002 + [image-loader]=9003 + [kahuna]=9005 + [cropper]=9006 + [metadata-editor]=9007 + [collections]=9010 + [auth]=9011 + [leases]=9012 +) + +export SERVICES="auth collections cropper image-loader kahuna leases media-api metadata-editor thrall" + +# Install a TERM/INT trap that kills the given child pids, waits for them, and +# exits cleanly. Pass the pids to tear down as arguments, e.g. +# trap 'shutdown_children "${pids[@]}"' TERM INT +shutdown_children() { + echo "Shutting down Grid services..." + local pid + for pid in "$@"; do + kill -TERM "$pid" 2>/dev/null || true + done + wait 2>/dev/null || true + exit 0 +} diff --git a/e2e-tests/images/entrypoint.dev.sh b/e2e-tests/images/entrypoint.dev.sh new file mode 100644 index 00000000000..b480faac53b --- /dev/null +++ b/e2e-tests/images/entrypoint.dev.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# +# Development entrypoint for the Grid all-in-one image. +# +# Runs the selected Play services under `sbt /run` (Play dev mode), which +# recompiles changed Scala sources on the next request. When kahuna is selected, +# its webpack bundle is rebuilt continuously via `npm run watch`. +# +# The repository is expected to be bind-mounted at /build so host edits are +# picked up live, e.g.: +# docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" grid-dev +# +# Environment: +# GRID_DEBUG if non-empty, opens a JDWP debug server on port 5005 + +set -euo pipefail + +REPO=/build +cd "$REPO" + +source "$REPO/e2e-tests/images/entrypoint.common.sh" + +# Default to the full production service list; GRID_SERVICES can narrow it. +SERVICES="${GRID_SERVICES:-$SERVICES}" + +# --- Kahuna frontend watcher +watch_pid="" +if [[ " $SERVICES " == *" kahuna "* ]]; then + echo "Starting Kahuna webpack watcher..." + ( + cd "$REPO/kahuna" + # node_modules may be shadowed by the host bind-mount; install if absent. + if [[ ! -d node_modules ]]; then + npm install + fi + npm run watch + ) & + watch_pid=$! +fi + +# --- Assemble the sbt run command +run_tasks="" +for svc in $SERVICES; do + if [[ -z "${PORTS[$svc]:-}" ]]; then + echo "Unknown service '$svc' (no port mapping); skipping." >&2 + continue + fi + echo "Will run $svc on port ${PORTS[$svc]}" + run_tasks="$run_tasks ${svc}/run" +done + +if [[ -z "$run_tasks" ]]; then + echo "No valid services selected in GRID_SERVICES='$SERVICES'." >&2 + exit 1 +fi + +# `all` runs the per-service `run` tasks in parallel from a single sbt session. +SBT_COMMAND="all${run_tasks}" + +# --- sbt / JVM options +SBT_OPTS="${SBT_OPTS:-}" + +trap 'shutdown_children "$watch_pid" "${sbt_pid:-}"' TERM INT + +echo "Running: sbt $SBT_OPTS \"$SBT_COMMAND\"" + +# Play's dev-mode `run` blocks reading stdin and stops on EOF/Enter. Keep stdin +# open (via a never-ending source) so the services stay up in a non-interactive +# container; shutdown is driven by the SIGTERM/SIGINT trap above. +tail -f /dev/null | sbt $SBT_OPTS "$SBT_COMMAND" & +sbt_pid=$! +wait "$sbt_pid" diff --git a/e2e-tests/package-lock.json b/e2e-tests/package-lock.json new file mode 100644 index 00000000000..2512b809255 --- /dev/null +++ b/e2e-tests/package-lock.json @@ -0,0 +1,3518 @@ +{ + "name": "e2e-tests", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "e2e-tests", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@aws-sdk/client-cloudformation": "^3.658.0", + "@aws-sdk/client-dynamodb": "^3.1127.0", + "@aws-sdk/client-kinesis": "^3.1127.0", + "@aws-sdk/client-s3": "^3.658.0", + "@playwright/test": "^1.61.1", + "@testcontainers/localstack": "^10.13.2", + "@types/node": "^26.1.1", + "json5": "^2.2.3", + "listr2": "^10.1.2", + "playwright-bdd": "^9.2.0", + "testcontainers": "^10.13.2" + }, + "engines": { + "node": ">=22.12" + } + }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.18.tgz", + "integrity": "sha512-IImkbEyXdV6/uaF5r6Wkk+8718mQw1ll83j0a4a30R3JM/rHVFdWAiT4jtJpFjJiIwM/oJ6SxIxr0z2TaQUGqw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-cloudformation": { + "version": "3.1089.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudformation/-/client-cloudformation-3.1089.0.tgz", + "integrity": "sha512-56kyaFKOnU0rE1DPTxIsmegpMb9z8uJJJpKjg+/6vynPbnGVHOAoxBxbVjCHhSLuozezVEM/k0T9g3fRzxy2tA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-node": "^3.972.70", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb": { + "version": "3.1127.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1127.0.tgz", + "integrity": "sha512-rV9TC7gqtwekWR9jXBdzuvCG65fnv7jphnYdJmFBfLX1b0Q+ma3xijzZaGGeN+US2pKw/XaQiZapJX/nZqtBJw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-node": "^3.972.82", + "@aws-sdk/dynamodb-codec": "^3.973.44", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.30", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-kinesis": { + "version": "3.1127.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis/-/client-kinesis-3.1127.0.tgz", + "integrity": "sha512-8uAuIE20oxRFocQfD71v8xBhrIKaFO8xYsnuKcAuw0OP+5DsXUmDg4lOv8bsdYJPi7tJxE9HkSz7cHxEw+Y5jA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-node": "^3.972.82", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1089.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1089.0.tgz", + "integrity": "sha512-Sc+JOtNeP+v+XdP9Rb4Hh+uhiNO2hh/CZQbKYooNakhOIgK6/K0cjoDCEGeFmkG0vf2DopTmSctzKlcKwy1BPw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.18", + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-node": "^3.972.70", + "@aws-sdk/middleware-sdk-s3": "^3.972.64", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.977.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.9.tgz", + "integrity": "sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.5", + "@aws-sdk/xml-builder": "^3.972.40", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.33.3", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.70.tgz", + "integrity": "sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.72", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.72.tgz", + "integrity": "sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.15", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.15.tgz", + "integrity": "sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-login": "^3.972.77", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.77", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.77.tgz", + "integrity": "sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.82", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.82.tgz", + "integrity": "sha512-znDkEOGXB8W3kG1LJUKP3foBZY/9qLM0eil/DxWXSp37XsdsRLQHE/d/OaCGGVgKpA6znR38h/+INk8do1FjiA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-ini": "^3.973.15", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.70.tgz", + "integrity": "sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.14", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.14.tgz", + "integrity": "sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/token-providers": "3.1116.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.76.tgz", + "integrity": "sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/dynamodb-codec": { + "version": "3.973.44", + "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.44.tgz", + "integrity": "sha512-eo27HNeBqMAfy9JBQug6ErU6DeG8OmOH6ou8+KmX3/fWdTmWUsHsfisz2tEQOG+GJ57bBC7kC6AjKAjpcxU4QA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.972.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", + "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.972.30", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.30.tgz", + "integrity": "sha512-0hmD7/NG2NoVOVHvUb6rtY5POQYr+/9gdHvrYhIBA1zmCqKOBd5Gz3dL+iZ15FHOJbs/5kLplGoePc8PHTlzYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/endpoint-cache": "^3.972.11", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.64", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.64.tgz", + "integrity": "sha512-RBi43anhDBUv+HCfxCOXwGOE7GmT4n7ChV04Mwr22RhXTNcamW/iWnJlOotDPCZSrJ4dEvhZSiWWQMwLX+ZhFA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.44", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.44.tgz", + "integrity": "sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/signature-v4-multi-region": "^3.996.46", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.46", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.46.tgz", + "integrity": "sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.5", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1116.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1116.0.tgz", + "integrity": "sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.5.tgz", + "integrity": "sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.40", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.40.tgz", + "integrity": "sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@colors/colors": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", + "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@cucumber/ci-environment": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/ci-environment/-/ci-environment-13.0.0.tgz", + "integrity": "sha512-cs+3NzfNkGbcmHPddjEv4TKFiBpZRQ6WJEEufB9mw+ExS22V/4R/zpDSEG+fsJ/iSNCd6A2sATdY8PFOyY3YnA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@cucumber/cucumber-expressions": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/cucumber-expressions/-/cucumber-expressions-19.0.0.tgz", + "integrity": "sha512-4FKoOQh2Uf6F6/Ln+1OxuK8LkTg6PyAqekhf2Ix8zqV2M54sH+m7XNJNLhOFOAW/t9nxzRbw2CcvXbCLjcvHZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "regexp-match-indices": "1.0.2" + } + }, + "node_modules/@cucumber/gherkin": { + "version": "39.1.0", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-39.1.0.tgz", + "integrity": "sha512-pqmSO2bUWxJm3TbNrKXlDaHjL6c77+ez9kWmfCd9oRPeTRPEVH3spZvpAqdXYWOZYSNYwWFCAAeZ4RGpkauNoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/messages": ">=31.0.0 <33" + } + }, + "node_modules/@cucumber/gherkin-utils": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin-utils/-/gherkin-utils-11.0.0.tgz", + "integrity": "sha512-LJ+s4+TepHTgdKWDR4zbPyT7rQjmYIcukTwNbwNwgqr6i8Gjcmzf6NmtbYDA19m1ZFg6kWbFsmHnj37ZuX+kZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/gherkin": "^38.0.0", + "@cucumber/messages": "^32.0.0", + "@teppeis/multimaps": "3.0.0", + "commander": "14.0.2", + "source-map-support": "^0.5.21" + }, + "bin": { + "gherkin-utils": "bin/gherkin-utils" + } + }, + "node_modules/@cucumber/gherkin-utils/node_modules/@cucumber/gherkin": { + "version": "38.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-38.0.0.tgz", + "integrity": "sha512-duEXK+KDfQUzu3vsSzXjkxQ2tirF5PRsc1Xrts6THKHJO6mjw4RjM8RV+vliuDasmhhrmdLcOcM7d9nurNTJKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/messages": ">=31.0.0 <33" + } + }, + "node_modules/@cucumber/gherkin-utils/node_modules/commander": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", + "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/@cucumber/html-formatter": { + "version": "23.1.0", + "resolved": "https://registry.npmjs.org/@cucumber/html-formatter/-/html-formatter-23.1.0.tgz", + "integrity": "sha512-DcCSFoGs6jbwzXPgX1CwgJKEE+ZMcIEzq/0Memg0o24maNn9NJizBFHmoFWG4iv/OxHza+mvc+56cTHetfHndw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@cucumber/messages": ">=18" + } + }, + "node_modules/@cucumber/junit-xml-formatter": { + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/@cucumber/junit-xml-formatter/-/junit-xml-formatter-0.13.3.tgz", + "integrity": "sha512-w9ujOxiuKDtU6fLzJz+wp4Sgp5Xu6ba7ls00LHJccVmQU0Ba7zs+AHnv3iIgPjKZAQe1w8x93dr8Gaubh7Vqkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/query": "^15.0.1", + "@teppeis/multimaps": "^3.0.0", + "luxon": "^3.5.0", + "xmlbuilder": "^15.1.1" + }, + "peerDependencies": { + "@cucumber/messages": "*" + } + }, + "node_modules/@cucumber/messages": { + "version": "32.3.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-32.3.1.tgz", + "integrity": "sha512-yNQq1KoXRYaEKrWMFmpUQX7TdeQuU9jeGgJAZ3dArTsC/T4NpJ6DnqaJIIgwPnz/wtQIQTNX7/h0rOuF5xY4qQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "class-transformer": "0.5.1", + "reflect-metadata": "0.2.2" + } + }, + "node_modules/@cucumber/query": { + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/query/-/query-15.0.1.tgz", + "integrity": "sha512-FMfT3orJblRsOxvU2doECBvQmauizYlj+5JsM8atAKKPbnQTj7v2/OrnuykvQpfZNBf19DYbRq1e832vllRP/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@teppeis/multimaps": "3.0.0", + "lodash.sortby": "^4.7.0" + }, + "peerDependencies": { + "@cucumber/messages": "*" + } + }, + "node_modules/@cucumber/tag-expressions": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/@cucumber/tag-expressions/-/tag-expressions-9.1.0.tgz", + "integrity": "sha512-bvHjcRFZ+J1TqIa9eFNO1wGHqwx4V9ZKV3hYgkuK/VahHx73uiP4rKV3JVrvWSMrwrFvJG6C8aEwnCWSvbyFdQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@smithy/core": { + "version": "3.33.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.3.tgz", + "integrity": "sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz", + "integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", + "integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.3.tgz", + "integrity": "sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.18.0.tgz", + "integrity": "sha512-CgB6HHWer/vrKps24ulRIbpcpb7K4xAU7SkZ7YHzBPlwHsvsrCJFEXK421s+cJzX+ZrqtA/TuU5w1HzI7k9N8A==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@teppeis/multimaps": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@teppeis/multimaps/-/multimaps-3.0.0.tgz", + "integrity": "sha512-ID7fosbc50TbT0MK0EG12O+gAP3W3Aa/Pz4DaTtQtEvlc9Odaqi0de+xuZ7Li2GtK4HzEX7IuRWS/JmZLksR3Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@testcontainers/localstack": { + "version": "10.28.0", + "resolved": "https://registry.npmjs.org/@testcontainers/localstack/-/localstack-10.28.0.tgz", + "integrity": "sha512-3izFtLEklezU5gmqSYtT0xKsLvdXV2AusNRXeO7hMWTb9JypoztT+Nf0dza22qczqpRi6mDKgqrgla5YoIIjJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "testcontainers": "^10.28.0" + } + }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "3.3.47", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.3.47.tgz", + "integrity": "sha512-ShM1mz7rCjdssXt7Xz0u1/R2BJC7piWa3SJpUBiVjCf2A3XNn4cP6pUVaD8bLanpPVVn4IKzJuw3dOvkJ8IbYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/node": { + "version": "26.1.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", + "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2-streams": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/ssh2-streams/-/ssh2-streams-0.1.13.tgz", + "integrity": "sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ssh2/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/ansi-escapes": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", + "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true, + "license": "MIT" + }, + "node_modules/async-lock": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.1.tgz", + "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/bare-events": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.1.tgz", + "integrity": "sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.7.4", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.4.tgz", + "integrity": "sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.1.1.tgz", + "integrity": "sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/bare-stream": { + "version": "2.13.3", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.3.tgz", + "integrity": "sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.8.1", + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.5.tgz", + "integrity": "sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", + "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/buildcheck": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/byline": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, + "license": "ISC" + }, + "node_modules/class-transformer": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", + "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-cursor": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", + "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "restore-cursor": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-table3": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.5.tgz", + "integrity": "sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^4.2.0" + }, + "engines": { + "node": "10.* || >= 12.*" + }, + "optionalDependencies": { + "@colors/colors": "1.5.0" + } + }, + "node_modules/cli-table3/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-table3/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-table3/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-table3/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-truncate": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-5.2.0.tgz", + "integrity": "sha512-xRwvIOMGrfOAnM1JYtqQImuaNtDEv9v6oIYAs4LIHwTiKee8uwvIi363igssOC0O5U04i4AlENs79LQLu9tEMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "slice-ansi": "^8.0.0", + "string-width": "^8.2.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-truncate/node_modules/string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz", + "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/docker-compose": { + "version": "0.24.8", + "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-0.24.8.tgz", + "integrity": "sha512-plizRs/Vf15H+GCVxq2EUvyPK7ei9b/cVesHvjnX4xaXjM9spHe2Ytq0BitndFgvTJ3E3NljPNUEl7BAN43iZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "yaml": "^2.2.2" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/docker-modem": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.15.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/docker-modem/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.12.tgz", + "integrity": "sha512-/bCZd6KlGcjZO8Buqmi/vXuqEGVEZ0PNjx/biBNqJD3MhK9DmdiAuKxqfNhflgDESDIiBz3qF+0e55+CpnrUcw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@grpc/grpc-js": "^1.11.1", + "@grpc/proto-loader": "^0.7.13", + "docker-modem": "^5.0.7", + "protobufjs": "^7.3.2", + "tar-fs": "^2.1.4", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode/node_modules/tar-fs": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", + "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/dockerode/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/environment": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", + "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-port": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.2.0.tgz", + "integrity": "sha512-afP4W205ONCuMoPBqcR6PSXnzX35KTcJygfJfcp+QY+uwm3p20p1YczWXhlICIzGMCxYBQcySEcOgsJcrkyobg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/listr2": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/listr2/-/listr2-10.1.2.tgz", + "integrity": "sha512-ENXj5KYVtdZigbaWY0+lUCzRZNvKDEbxree+93oaCeSz8GICxgJyOMi6U9mHuiFd3hJ9y0RIDQh8RYttBKOnaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "cli-truncate": "^5.2.0", + "colorette": "^2.0.20", + "eventemitter3": "^5.0.1", + "log-update": "^6", + "rfdc": "^1.4.1", + "wrap-ansi": "^10.0.0" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/listr2/node_modules/string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/listr2/node_modules/wrap-ansi": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-10.0.1.tgz", + "integrity": "sha512-M0N4xzyzosiIok3svYlEo1sdLZts/8FPgYH/GPC3wvlmPoRvnoManGMrE54waYj3tISA8w6lsdesfVv67qSr8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.3", + "string-width": "^8.2.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==", + "dev": true, + "license": "MIT" + }, + "node_modules/log-update": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz", + "integrity": "sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-escapes": "^7.0.0", + "cli-cursor": "^5.0.0", + "slice-ansi": "^7.1.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "dev": true, + "license": "MIT" + }, + "node_modules/log-update/node_modules/is-fullwidth-code-point": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", + "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/slice-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.2.tgz", + "integrity": "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "is-fullwidth-code-point": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/log-update/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/mimic-function": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", + "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, + "license": "MIT" + }, + "node_modules/mnemonist": { + "version": "0.38.3", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "obliterator": "^1.6.1" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nan": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.28.0.tgz", + "integrity": "sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/obliterator": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", + "dev": true, + "license": "MIT" + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", + "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-function": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-bdd": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/playwright-bdd/-/playwright-bdd-9.2.0.tgz", + "integrity": "sha512-1tBTmo4DpOhLsc+A6PB4isWO3DHKb4BQ3Tzw5+ze/PmgBW9W2m9c+nc0TPy2nByWJtw0gKSfMoexyBR+y82+pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/ci-environment": "^13.0.0", + "@cucumber/cucumber-expressions": "19.0.0", + "@cucumber/gherkin": "^39.1.0", + "@cucumber/gherkin-utils": "^11.0.0", + "@cucumber/html-formatter": "^23.1.0", + "@cucumber/junit-xml-formatter": "^0.13.3", + "@cucumber/messages": "^32.3.1", + "@cucumber/query": "^15.0.1", + "@cucumber/tag-expressions": "^9.1.0", + "cli-table3": "0.6.5", + "commander": "^13.1.0", + "mime-types": "^3.0.2", + "tinyglobby": "0.2.17" + }, + "bin": { + "bddgen": "dist/cli/index.js" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/vitalets" + }, + "peerDependencies": { + "@playwright/test": ">=1.44" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/properties-reader": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/properties-reader/-/properties-reader-2.3.0.tgz", + "integrity": "sha512-z597WicA7nDZxK12kZqHr2TcvwNU1GCfA5UwfDY/HDp3hXPoPlb5rlEx9bwGTiJnc0OqbBTkU975jDToth8Gxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mkdirp": "^1.0.4" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/properties?sponsor=1" + } + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "dev": true, + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/reflect-metadata": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", + "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/regexp-match-indices": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regexp-match-indices/-/regexp-match-indices-1.0.2.tgz", + "integrity": "sha512-DwZuAkt8NF5mKwGGER1EGh2PRqyvhRhhLviH+R8y8dIuaQROlUfXjt4s9ZTXstIsSkptf06BSvwcEmmfheJJWQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "regexp-tree": "^0.1.11" + } + }, + "node_modules/regexp-tree": { + "version": "0.1.27", + "resolved": "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.27.tgz", + "integrity": "sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==", + "dev": true, + "license": "MIT", + "bin": { + "regexp-tree": "bin/regexp-tree" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/restore-cursor": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", + "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", + "dev": true, + "license": "MIT", + "dependencies": { + "onetime": "^7.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/rfdc": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", + "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", + "dev": true, + "license": "MIT" + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/slice-ansi": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-8.0.0.tgz", + "integrity": "sha512-stxByr12oeeOyY2BlviTNQlYV5xOj47GirPr4yA1hE9JCtxfQN0+tVbkxwCtYDQWhEKWFHsEK48ORg5jrouCAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.3", + "is-fullwidth-code-point": "^5.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", + "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ssh-remote-port-forward": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ssh-remote-port-forward/-/ssh-remote-port-forward-1.0.4.tgz", + "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ssh2": "^0.5.48", + "ssh2": "^1.4.0" + } + }, + "node_modules/ssh-remote-port-forward/node_modules/@types/ssh2": { + "version": "0.5.52", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-0.5.52.tgz", + "integrity": "sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2-streams": "*" + } + }, + "node_modules/ssh2": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.23.0" + } + }, + "node_modules/streamx": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.0.tgz", + "integrity": "sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==", + "dev": true, + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.3.tgz", + "integrity": "sha512-/hU4AXnIdZu+Gvl1pk0oI5f5HxWsCJRtY2aFaJdk9VvyL48DWU6iU5WAIPG+wIi1YvWA6eTJvIviP/tMAZZNwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz", + "integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/testcontainers": { + "version": "10.28.0", + "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-10.28.0.tgz", + "integrity": "sha512-1fKrRRCsgAQNkarjHCMKzBKXSJFmzNTiTbhb5E/j5hflRXChEtHvkefjaHlgkNUjfw92/Dq8LTgwQn6RDBFbMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@types/dockerode": "^3.3.35", + "archiver": "^7.0.1", + "async-lock": "^1.4.1", + "byline": "^5.0.0", + "debug": "^4.3.5", + "docker-compose": "^0.24.8", + "dockerode": "^4.0.5", + "get-port": "^7.1.0", + "proper-lockfile": "^4.1.2", + "properties-reader": "^2.3.0", + "ssh-remote-port-forward": "^1.0.4", + "tar-fs": "^3.0.7", + "tmp": "^0.2.3", + "undici": "^5.29.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "dev": true, + "license": "Unlicense" + }, + "node_modules/undici": { + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/xmlbuilder": { + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", + "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + } + } +} diff --git a/e2e-tests/package.json b/e2e-tests/package.json new file mode 100644 index 00000000000..439d0420e30 --- /dev/null +++ b/e2e-tests/package.json @@ -0,0 +1,34 @@ +{ + "name": "e2e-tests", + "version": "1.0.0", + "description": "", + "main": "index.js", + "type": "module", + "scripts": { + "dev:e2e": "node --experimental-strip-types --disable-warning=ExperimentalWarning setup/dev.ts", + "test": "bddgen && playwright test --reporter=list,html", + "test:headed": "bddgen && playwright test --headed --reporter=list,html", + "test:report": "playwright show-report", + "test:ui": "bddgen && playwright test --ui-port 6080", + "test:log": "bddgen && DEBUG=testcontainers* playwright test --ui-port 6080" + }, + "engines": { + "node": ">=22.12" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@aws-sdk/client-cloudformation": "^3.658.0", + "@aws-sdk/client-dynamodb": "^3.1127.0", + "@aws-sdk/client-kinesis": "^3.1127.0", + "@aws-sdk/client-s3": "^3.658.0", + "@playwright/test": "^1.61.1", + "@testcontainers/localstack": "^10.13.2", + "@types/node": "^26.1.1", + "json5": "^2.2.3", + "listr2": "^10.1.2", + "playwright-bdd": "^9.2.0", + "testcontainers": "^10.13.2" + } +} diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts new file mode 100644 index 00000000000..b2506c2ed07 --- /dev/null +++ b/e2e-tests/playwright.config.ts @@ -0,0 +1,54 @@ +import { defineConfig, devices } from '@playwright/test'; +import { defineBddConfig } from 'playwright-bdd'; +import { KAHUNA_PORT } from './setup/constants.ts'; + +/* Generate Playwright test files from the Gherkin feature files and step definitions. + Run via `bddgen` (see package.json scripts) before `playwright test`. */ +const testDir = defineBddConfig({ + features: './features/**/*.feature', + steps: './steps/**/*.ts', + /* @todo marks scenarios that are specified but not yet implemented. Excluding them keeps + bddgen's "undefined step" failures meaningful for the scenarios we do implement. */ + tags: 'not @todo', +}); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir, + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Boot the Grid stack with Testcontainers before tests, and tear it down after. */ + globalSetup: './setup/global-setup.ts', + globalTeardown: './setup/global-teardown.ts', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL (Kahuna) to use in actions like `await page.goto('/')`. Set by global-setup. */ + baseURL: process.env.GRID_BASE_URL ?? `http://localhost:${KAHUNA_PORT}`, + + /* The Grid service domains are served over https by the developer's dev-nginx, which + terminates TLS with a locally-generated (self-signed) certificate. Ignore cert + validation so browser tests can load the https://*.media. origins. */ + ignoreHTTPSErrors: true, + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/e2e-tests/setup/config.ts b/e2e-tests/setup/config.ts new file mode 100644 index 00000000000..68c8bbcb966 --- /dev/null +++ b/e2e-tests/setup/config.ts @@ -0,0 +1,106 @@ +/** + * Reuses the existing dev config generator (`dev/script/generate-config/service-config.js`) + * to produce per-service Grid config from live CloudFormation stack resources, then + * rewrites the LocalStack/Elasticsearch endpoints so that they resolve via the + * Testcontainers network aliases instead of the Guardian dev domains / localhost. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { createRequire } from 'module'; +import JSON5 from 'json5'; +import { + DOMAIN, + EMAIL_DOMAIN, + ELASTICSEARCH_ALIAS, + ELASTICSEARCH_PORT, + LOCALSTACK_ALIAS, + LOCALSTACK_PORT, + REGION, + REPO_ROOT, + SERVICE_PORTS, +} from './constants.ts'; + +/** `service-config.js` is CommonJS and lives outside this package, so load it via require. */ +const require = createRequire(import.meta.url); + +const GENERATE_CONFIG_DIR = path.join(REPO_ROOT, 'dev', 'script', 'generate-config'); + +const GRID_SERVICES = Object.keys(SERVICE_PORTS); + +type StackProps = Record; + +/** + * Rewrite the Guardian dev endpoints baked in by `service-config.js` so that the + * app container reaches the infrastructure containers over the shared network. + */ +function rewriteEndpoints(conf: string): string { + const localstackUrl = `http://${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`; + // Endpoints baked in by `service-config.js` that must be redirected to the container. + const guardianLocalstackUrl = `https://${LOCALSTACK_ALIAS}.media.${DOMAIN}`; + const legacyLocalstackUrl = 'http://localhost:4576'; + const localLocalstackUrl = `http://localhost:${LOCALSTACK_PORT}`; + + const rewrittenConf = conf + .split(guardianLocalstackUrl) + .join(localstackUrl) + .split(legacyLocalstackUrl) + .join(localstackUrl) + .split(localLocalstackUrl) + .join(localstackUrl); + + // The S3 client reaches LocalStack over the container network (localstack:4566), but + // presigned URLs are handed to the browser, which can only reach LocalStack via the + // `localstack.media.` vanity domain (dev-nginx locally, the Caddy proxy in CI). + // Sign against that host so the URLs resolve outside the container network. + return `${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`; +} + +/** + * Generate all service config files into `configDir`. + */ +export function generateServiceConfig(configDir: string, coreStackProps: StackProps): void { + const ServiceConfig = require(path.join(GENERATE_CONFIG_DIR, 'service-config.js')); + const defaultConfig = JSON5.parse( + fs.readFileSync(path.join(GENERATE_CONFIG_DIR, 'config.json5'), 'utf8'), + ); + + const config = { + ...defaultConfig, + DOMAIN, + EMAIL_DOMAIN, + AWS_DEFAULT_REGION: REGION, + // NO_AUTHENTICATION makes `getCommonConfig` emit the Local authentication provider, so + // we don't need pan-domain / OIDC infrastructure. Authorisation is left as the real + // (S3-backed) provider, which reads `permissions.json` from the provisioned bucket. + NO_AUTHENTICATION: true, + coreStackProps, + es6: { + ...defaultConfig.es6, + url: `http://${ELASTICSEARCH_ALIAS}:${ELASTICSEARCH_PORT}`, + }, + }; + + const serviceConfigs: Record = ServiceConfig.getCoreConfigs(config); + + fs.mkdirSync(configDir, { recursive: true }); + + // Mark the stage as DEV so services load `~/.grid/.conf` (see GridConfigLoader). + fs.writeFileSync(path.join(configDir, 'stage'), 'DEV'); + + // The staged apps run in Play prod mode, which requires an application secret (>= 256 + // bits). GridConfigLoader loads `common.conf` for every service, so set it once here + // rather than injecting it as a JVM option in the container. + fs.writeFileSync( + path.join(configDir, 'common.conf'), + 'play.http.secret.key = "testcontainers-e2e-application-secret-0123456789"\n', + ); + + for (const service of GRID_SERVICES) { + const conf = serviceConfigs[service]; + if (!conf) { + throw new Error(`service-config.js did not produce config for '${service}'`); + } + + fs.writeFileSync(path.join(configDir, `${service}.conf`), rewriteEndpoints(conf)); + } +} diff --git a/e2e-tests/setup/constants.ts b/e2e-tests/setup/constants.ts new file mode 100644 index 00000000000..5c001d5e77f --- /dev/null +++ b/e2e-tests/setup/constants.ts @@ -0,0 +1,69 @@ +import * as os from 'os'; +import * as path from 'path'; + +export const REPO_ROOT = path.resolve(import.meta.dirname, '..', '..'); + +export const DOMAIN = 'local.dev-gutools.co.uk'; +export const EMAIL_DOMAIN = 'guardian.co.uk'; +export const REGION = 'eu-west-1'; + +export const CORE_STACK_NAME = 'grid-dev-core'; + +/** + * Bucket holding `permissions.json` for the real authorisation provider. Not part of the + * core CloudFormation stack, so it is created directly during provisioning. The name + * matches the code default in `PermissionsAuthorisationProvider`. + */ +export const PERMISSIONS_BUCKET = 'permissions-cache'; + +/** Pre-built application image (see e2e-tests/images/Dockerfile). Assumed to exist. */ +export const GRID_IMAGE = process.env.CI ? 'grid-e2e-ci' : 'grid-e2e-dev'; +export const ELASTICSEARCH_IMAGE = 'docker.elastic.co/elasticsearch/elasticsearch:8.18.3'; +export const LOCALSTACK_IMAGE = 'localstack/localstack:4.5.0'; +/** Reverse proxy used in CI to stand in for the developer's dev-nginx (see global-setup). */ +export const PROXY_IMAGE = 'caddy:2.8-alpine'; + +/** + * imgops: standalone nginx on-the-fly image resizer, built from dev/imgops at setup time. + * Its nginx.conf proxies to the `localstack` alias on 4566, so it shares the stack network. + * Published on IMGOPS_PORT, the fixed host port dev-nginx maps `media-imgops` to. + */ +export const IMGOPS_ALIAS = 'imgops'; +export const IMGOPS_PORT = 9008; +export const IMGOPS_CONTEXT = path.join(REPO_ROOT, 'dev', 'imgops'); +export const IMGOPS_NGINX_CONF = path.join(IMGOPS_CONTEXT, 'nginx.conf'); +/** Stable tag for the imgops image so it persists across runs and Docker reuses cached layers. */ +export const IMGOPS_IMAGE = 'grid-e2e-imgops'; + +/** Network aliases the app container uses to reach the infrastructure containers. */ +export const ELASTICSEARCH_ALIAS = 'elasticsearch'; +/** Fixed host port, so a stack started by `dev.ts` can be reached without a container handle. */ +export const ELASTICSEARCH_PORT = 9200; +export const LOCALSTACK_ALIAS = 'localstack'; +export const LOCALSTACK_PORT = 4566; +/** Network alias the CI reverse proxy uses to reach the grid-e2e-ci app container. */ +export const GRID_ALIAS = 'grid-e2e-ci'; + +/** service -> http port, from e2e-tests/images/entrypoint.ci.sh. */ +export const SERVICE_PORTS: Record = { + 'media-api': 9001, + thrall: 9002, + 'image-loader': 9003, + kahuna: 9005, + cropper: 9006, + 'metadata-editor': 9007, + collections: 9010, + auth: 9011, + leases: 9012, +}; + +export const KAHUNA_PORT = SERVICE_PORTS.kahuna; +export const MEDIA_API_PORT = SERVICE_PORTS['media-api']; + +/** + * Generated per-service config, bind-mounted into the app container. + */ +export const CONFIG_DIR = path.join(os.tmpdir(), 'grid-e2e-config'); + +/** The API key value uploaded to the KeyBucket (dev/.env API_KEY). */ +export const API_KEY = 'e2e-dev'; diff --git a/e2e-tests/setup/dev.ts b/e2e-tests/setup/dev.ts new file mode 100644 index 00000000000..c493652c7f8 --- /dev/null +++ b/e2e-tests/setup/dev.ts @@ -0,0 +1,88 @@ +/** + * Boots the Grid stack outside Playwright and holds it open until Ctrl-C. + * + * Set `GRID_PROXY=true` to also start the bundled Caddy reverse proxy, which + * serves the https://*.media. domains for anyone without dev-nginx + * running locally. + */ +import { + CONFIG_DIR, + DOMAIN, + ELASTICSEARCH_PORT, + LOCALSTACK_PORT, + SERVICE_PORTS, +} from './constants.ts'; +import { probeStack, startStack, stopStack } from './stack.ts'; +import type { GridEnvironment } from './state.ts'; + +function banner(elapsedMs: number): string { + const services = Object.keys(SERVICE_PORTS) + .sort() + .map((service) => ` ${service.padEnd(16)} http://localhost:${SERVICE_PORTS[service]}`) + .join('\n'); + + return [ + '', + ` Grid is up in ${Math.round(elapsedMs / 1000)}s at https://media.${DOMAIN}`, + '', + ' Services:', + services, + '', + ' Infrastructure:', + ` localstack http://localhost:${LOCALSTACK_PORT}`, + ` elasticsearch http://localhost:${ELASTICSEARCH_PORT}`, + '', + ` Config at ${CONFIG_DIR}`, + '', + ' Press Ctrl-C to tear everything down.', + '', + ].join('\n'); +} + +async function dev(): Promise { + let environment: GridEnvironment | undefined; + let shuttingDown = false; + + const shutdown = async (signal: NodeJS.Signals): Promise => { + // A second Ctrl-C exits immediately; Testcontainers' Ryuk reaper cleans up the rest. + if (shuttingDown) { + console.log('\nForcing exit.'); + process.exit(130); + } + shuttingDown = true; + + if (!environment) { + // Interrupted mid-boot: startStack owns the containers, so leave them to Ryuk. + console.log(`\nReceived ${signal} during startup; leaving cleanup to the Ryuk reaper.`); + process.exit(130); + } + + console.log(`\nReceived ${signal}, stopping the Grid stack...`); + await stopStack(environment); + console.log('Done.'); + process.exit(0); + }; + + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); + + // The host ports are fixed, so a second stack cannot coexist with the first. + const { state, healthy } = await probeStack(); + if (state !== 'none') { + throw new Error( + `Ports ${healthy.join(', ')} are already serving Grid. Stop that stack before starting another.`, + ); + } + + const startedAt = Date.now(); + environment = await startStack({ proxy: process.env.GRID_PROXY === 'true' }); + console.log(banner(Date.now() - startedAt)); + + // Hold the process open indefinitely. + setInterval(() => {}, 2_000_000_000); +} + +dev().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/e2e-tests/setup/global-setup.ts b/e2e-tests/setup/global-setup.ts new file mode 100644 index 00000000000..1952354d6bc --- /dev/null +++ b/e2e-tests/setup/global-setup.ts @@ -0,0 +1,15 @@ +/** + * Playwright global setup. Attaches to a running Grid stack if there is one, otherwise + * boots a fresh one (see `setup/stack.ts`), stashing it for `global-teardown.ts` + * and exposing the Kahuna base URL via `GRID_BASE_URL`. + */ +import { ensureStack } from './stack.ts'; +import { setEnvironment } from './state.ts'; + +async function globalSetup(): Promise { + const environment = await ensureStack(); + process.env.GRID_BASE_URL = environment.baseUrl; + setEnvironment(environment); +} + +export default globalSetup; diff --git a/e2e-tests/setup/global-teardown.ts b/e2e-tests/setup/global-teardown.ts new file mode 100644 index 00000000000..f6079c8d495 --- /dev/null +++ b/e2e-tests/setup/global-teardown.ts @@ -0,0 +1,11 @@ +/** + * Playwright global teardown. Stops the stack started in `global-setup.ts`. + */ +import { stopStack } from './stack.ts'; +import { getEnvironment } from './state.ts'; + +async function globalTeardown(): Promise { + await stopStack(getEnvironment()); +} + +export default globalTeardown; diff --git a/e2e-tests/setup/progress.ts b/e2e-tests/setup/progress.ts new file mode 100644 index 00000000000..6e8a6ac223b --- /dev/null +++ b/e2e-tests/setup/progress.ts @@ -0,0 +1,42 @@ +/** + * Renders long-running work as a listr2 task tree, so booting or tearing down the stack + * shows which container or gate is currently being waited on instead of a silent terminal. + * + * Interactive terminals get the animated renderer; CI and piped output fall back to the + * line-oriented `simple` renderer, which prints one line per task transition. + */ +import { Listr, PRESET_TIMER } from 'listr2'; +import type { ListrTask } from 'listr2'; + +export type { ListrTask }; + +export interface RunTasksOptions { + /** Keep going after a failed task. Used by teardown, where every step is best-effort. */ + exitOnError?: boolean; +} + +/** Run `tasks` against `context`, which they mutate as they go, and return it. */ +export async function runTasks( + tasks: ListrTask[], + context: Ctx, + options: RunTasksOptions = {}, +): Promise { + const runner = new Listr(tasks, { + ctx: context, + exitOnError: options.exitOnError ?? true, + // The animated renderer redraws in place, which CI logs record as a wall of escape codes. + fallbackRendererCondition: () => !!process.env.CI, + rendererOptions: { + timer: PRESET_TIMER, + // Keep finished containers and services on screen: the completed list is the summary. + collapseSubtasks: false, + collapseErrors: false, + }, + registerSignalListeners: false, + fallbackRendererOptions: { timer: PRESET_TIMER }, + }); + + return runner.run(); +} + +export const reportTo = (task: { output: string }) => (message: string) => { task.output = message; }; \ No newline at end of file diff --git a/e2e-tests/setup/provision.ts b/e2e-tests/setup/provision.ts new file mode 100644 index 00000000000..9b577b99d9a --- /dev/null +++ b/e2e-tests/setup/provision.ts @@ -0,0 +1,173 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { + CloudFormationClient, + CreateStackCommand, + DescribeStackResourcesCommand, + waitUntilStackCreateComplete, +} from '@aws-sdk/client-cloudformation'; +import { + CreateTableCommand, + DynamoDBClient, + PutItemCommand, + ScanCommand, + waitUntilTableExists, +} from '@aws-sdk/client-dynamodb'; +import { KinesisClient, ListShardsCommand } from '@aws-sdk/client-kinesis'; +import { CreateBucketCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { API_KEY as API_KEY_PATH, CORE_STACK_NAME, PERMISSIONS_BUCKET, REGION, REPO_ROOT } from './constants.ts'; + +const CREDENTIALS = { accessKeyId: 'test', secretAccessKey: 'test' }; + +export type StackProps = Record; + +export function provisioningClients(endpoint: string) { + const cfn = new CloudFormationClient({ endpoint, region: REGION, credentials: CREDENTIALS }); + const s3 = new S3Client({ + endpoint, + region: REGION, + credentials: CREDENTIALS, + forcePathStyle: true, + }); + const dynamo = new DynamoDBClient({ endpoint, region: REGION, credentials: CREDENTIALS }); + const kinesis = new KinesisClient({ endpoint, region: REGION, credentials: CREDENTIALS }); + return { cfn, s3, dynamo, kinesis }; +} + +/** + * Provisions the Grid core infrastructure inside LocalStack: applies the CloudFormation + * core stack, waits for completion, reads the created resource names, and seeds the + * buckets with the config files the services expect (similar to dev/script/setup.sh). + */ +export async function createCoreStack(cfn: CloudFormationClient): Promise { + const templateBody = fs.readFileSync( + path.join(REPO_ROOT, 'dev', 'cloudformation', 'grid-dev-core.yml'), + 'utf8', + ); + + await cfn.send(new CreateStackCommand({ StackName: CORE_STACK_NAME, TemplateBody: templateBody })); + + // LocalStack applies the stack in seconds; the SDK default would sit out its 30s minimum delay. + await waitUntilStackCreateComplete( + { client: cfn, maxWaitTime: 180, minDelay: 1, maxDelay: 1 }, + { StackName: CORE_STACK_NAME }, + ); + + const { StackResources = [] } = await cfn.send( + new DescribeStackResourcesCommand({ StackName: CORE_STACK_NAME }), + ); + + return Object.fromEntries( + StackResources.filter((r) => r.LogicalResourceId && r.PhysicalResourceId).map((r) => [ + r.LogicalResourceId as string, + r.PhysicalResourceId as string, + ]), + ); +} + +async function putObject( + s3: S3Client, + bucket: string, + key: string, + body: Buffer | string, +): Promise { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })); +} + +export async function seedBuckets(s3: S3Client, props: StackProps): Promise { + const devConfig = path.join(REPO_ROOT, 'dev', 'config'); + + await Promise.all([ + // API key used by the machine authentication provider. + putObject(s3, props.KeyBucket, API_KEY_PATH, 'DEV Key'), + // Static config consumed by the services. + ...['photographers.json', 'rcs-quota.json', 'usage_rights.json'].map((file) => + putObject(s3, props.ConfigBucket, file, fs.readFileSync(path.join(devConfig, file))), + ), + putObject( + s3, + props.UsageMailBucket, + 'usages.eml', + fs.readFileSync(path.join(devConfig, 'usages.eml')), + ), + ]); +} + +/** + * Create the permissions bucket (not part of the core stack) and seed it with the + * permissions fixture so the real authorisation provider can read `permissions.json`. + * Returns the bucket name so it can be added to the stack props map. + */ +export async function provisionPermissionsBucket(s3: S3Client): Promise { + await s3.send( + new CreateBucketCommand({ + Bucket: PERMISSIONS_BUCKET, + CreateBucketConfiguration: { LocationConstraint: REGION }, + }), + ); + + await putObject( + s3, + PERMISSIONS_BUCKET, + 'permissions.json', + fs.readFileSync(path.join(REPO_ROOT, 'e2e-tests', 'fixtures', 'permissions', 'permissions.json')), + ); + + return PERMISSIONS_BUCKET; +} + +/** + * Pre-create the KCL lease table for a stream, with an unowned lease per shard. + * + * Thrall consumes each stream with the Kinesis Client Library, which names its DynamoDB + * lease table after the KCL application name — and thrall sets that to the stream name. + * Left to itself on a cold stack, KCL spends ~70s before it reads a single record: + * + * - `Scheduler.initialize()` calls `shouldInitiateLeaseSync()`, which sleeps a random + * 1-30s (polling every 3s) for as long as the lease table is empty. It is anti-stampede + * jitter for a real fleet, is not configurable, and is pure cost for a single worker. + * - It then runs the initial shard sync synchronously, and `ShardSyncTask` finishes with + * `Thread.sleep(shardSyncIntervalMillis)` — 60s on KCL's defaults. + * + * Seeding the leases here makes `isLeaseTableEmpty()` false on the very first check, so + * neither the jitter nor the shard sync runs: the lease taker picks up the unowned leases + * on its first pass instead. + */ +export async function seedKclLeaseTable( + dynamo: DynamoDBClient, + kinesis: KinesisClient, + streamName: string, +): Promise { + await dynamo.send( + new CreateTableCommand({ + TableName: streamName, + KeySchema: [{ AttributeName: 'leaseKey', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'leaseKey', AttributeType: 'S' }], + BillingMode: 'PAY_PER_REQUEST', + }), + ); + await waitUntilTableExists({ client: dynamo, maxWaitTime: 60, minDelay: 1 }, { TableName: streamName }); + + const { Shards = [] } = await kinesis.send(new ListShardsCommand({ StreamName: streamName })); + + await Promise.all( + Shards.map((shard) => + dynamo.send( + new PutItemCommand({ + TableName: streamName, + Item: { + leaseKey: { S: shard.ShardId! }, + leaseCounter: { N: '0' }, + checkpoint: { S: 'TRIM_HORIZON' }, + checkpointSubSequenceNumber: { N: '0' }, + ownerSwitchesSinceCheckpoint: { N: '0' }, + // Without the hash range KCL's lease auditor reports the stream as having holes. + startingHashKey: { S: shard.HashKeyRange!.StartingHashKey! }, + endingHashKey: { S: shard.HashKeyRange!.EndingHashKey! }, + }, + }), + ), + ), + ); +} + diff --git a/e2e-tests/setup/seed-elasticsearch.ts b/e2e-tests/setup/seed-elasticsearch.ts new file mode 100644 index 00000000000..c950979d57b --- /dev/null +++ b/e2e-tests/setup/seed-elasticsearch.ts @@ -0,0 +1,115 @@ +/** + * Seeds Elasticsearch with image fixtures once the Grid stack is up. + * + * The Grid app (media-api/thrall) creates the `images` index with the correct mappings + * and assigns the `Images_Current` alias on startup. This module waits for that alias to + * exist, then bulk-inserts the documents from `fixtures/elasticsearch/images.json` into it + * so searches performed during the tests return them. + * + * The fixture is a raw Elasticsearch search response, so documents live at + * `hits.hits[]` as `{ _id, _source }`. Each document is (re)indexed under its `_id` — the + * stale `_index` recorded in the fixture is ignored in favour of the live alias. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { REPO_ROOT } from './constants.ts'; + +/** Alias the Grid app assigns to the live images index (dev service-config `es.index.aliases.current`). */ +const IMAGES_ALIAS = 'Images_Current'; + +const FIXTURE_PATH = path.join( + REPO_ROOT, + 'e2e-tests', + 'fixtures', + 'elasticsearch', + 'images.json', +); + +interface EsHit { + _id: string; + _source: Record; +} + +interface EsSearchFixture { + body?: { + hits?: { + hits?: EsHit[]; + }; + }; +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Poll the alias until the Grid app has created the index and assigned it, so the bulk + * insert below has a valid write target. Throws if it never appears within the timeout. + */ +async function waitForAlias(esBaseUrl: string, timeoutMs: number): Promise { + const deadline = Date.now() + timeoutMs; + let lastStatus = 'no response'; + while (Date.now() < deadline) { + try { + const response = await fetch(`${esBaseUrl}/${IMAGES_ALIAS}`); + if (response.ok) { + return; + } + lastStatus = `HTTP ${response.status}`; + } catch (error) { + lastStatus = error instanceof Error ? error.message : String(error); + } + await sleep(1000); + } + throw new Error( + `Elasticsearch alias '${IMAGES_ALIAS}' was not assigned within ${timeoutMs}ms (last: ${lastStatus})`, + ); +} + +/** + * Load the image fixtures into Elasticsearch. No-op if the fixture has no documents. + */ +export async function seedElasticsearch( + esBaseUrl: string, + startupTimeoutMs: number, + report: (message: string) => void, +): Promise { + const fixture = JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) as EsSearchFixture; + const hits = fixture.body?.hits?.hits ?? []; + + if (hits.length === 0) { + throw new Error('No image fixtures found to seed into Elasticsearch. This is unexpected - failing early'); + } + + report(`Waiting for the '${IMAGES_ALIAS}' alias`); + await waitForAlias(esBaseUrl, startupTimeoutMs); + + // NDJSON bulk body: an index action line (targeting the document's `_id`) followed by + // the source document, repeated for every hit. + const body = hits + .flatMap((hit) => [JSON.stringify({ index: { _id: hit._id } }), JSON.stringify(hit._source)]) + .join('\n') + // The bulk API requires a trailing newline. + .concat('\n'); + + // `refresh=wait_for` makes the documents searchable before this call resolves. + const response = await fetch(`${esBaseUrl}/${IMAGES_ALIAS}/_bulk?refresh=wait_for`, { + method: 'POST', + headers: { 'Content-Type': 'application/x-ndjson' }, + body, + }); + + if (!response.ok) { + throw new Error(`Elasticsearch bulk seed failed: HTTP ${response.status} ${await response.text()}`); + } + + const result = (await response.json()) as { + errors: boolean; + items: Array<{ index?: { error?: unknown } }>; + }; + + if (result.errors) { + const firstError = result.items.find((item) => item.index?.error)?.index?.error; + throw new Error(`Elasticsearch bulk seed reported errors: ${JSON.stringify(firstError)}`); + } + + report(`Seeded ${hits.length} image fixture(s) into Elasticsearch alias '${IMAGES_ALIAS}'`); +} diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts new file mode 100644 index 00000000000..f56e5328934 --- /dev/null +++ b/e2e-tests/setup/stack.ts @@ -0,0 +1,664 @@ +/** + * Boots and tears down the full local Grid stack with Testcontainers: + * 1. a shared network, + * 2. Elasticsearch + LocalStack + imgops (infrastructure), + * 3. the CloudFormation core stack + seeded buckets (provisioning), + * 4. generated per-service config (reusing dev/script/generate-config), + * 5. the pre-built `grid-e2e-ci` / `grid-e2e-dev` image running the Grid services. + * + * Used by Playwright's global setup/teardown and by `dev.ts`, which runs the same + * stack interactively outside the test runner. + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { LocalstackContainer } from '@testcontainers/localstack'; +import { GenericContainer, Network, Wait } from 'testcontainers'; +import type { StartedNetwork, StartedTestContainer } from 'testcontainers'; +import { + CONFIG_DIR, + DOMAIN, + ELASTICSEARCH_ALIAS, + ELASTICSEARCH_IMAGE, + ELASTICSEARCH_PORT, + GRID_ALIAS, + GRID_IMAGE, + IMGOPS_ALIAS, + IMGOPS_CONTEXT, + IMGOPS_IMAGE, + IMGOPS_NGINX_CONF, + IMGOPS_PORT, + KAHUNA_PORT, + LOCALSTACK_ALIAS, + LOCALSTACK_IMAGE, + LOCALSTACK_PORT, + MEDIA_API_PORT, + PROXY_IMAGE, + REGION, + REPO_ROOT, + SERVICE_PORTS, +} from './constants.ts'; +import { generateServiceConfig } from './config.ts'; +import { reportTo, runTasks } from './progress.ts'; +import type { ListrTask } from './progress.ts'; +import { + createCoreStack, + provisioningClients, + provisionPermissionsBucket, + seedBuckets, + seedKclLeaseTable, +} from './provision.ts'; +import type { StackProps } from './provision.ts'; +import { seedElasticsearch } from './seed-elasticsearch.ts'; +import type { GridEnvironment } from './state.ts'; +import type { ListrTaskFn } from 'listr2'; + +const LOCALSTACK_SERVICES = [ + "cloudformation", + "cloudwatch", + "dynamodb", + "kinesis", + "s3", + "sns", + "sqs", + "iam", +].join(","); + +export interface StartStackOptions { + /** + * Start the bundled Caddy reverse proxy instead of relying on the developer's + * dev-nginx. Defaults to true under CI, which has no dev-nginx. + */ + proxy?: boolean; + /** Seed Elasticsearch with the image fixtures. Defaults to true. */ + seed?: boolean; +} + +// Every service is bound to a fixed host port, so its URL is the same whether this process +// started the stack or attached to one already running. +const KAHUNA_URL = `http://localhost:${KAHUNA_PORT}`; +const MEDIA_API_URL = `http://localhost:${MEDIA_API_PORT}`; +const ELASTICSEARCH_URL = `http://localhost:${ELASTICSEARCH_PORT}`; + +/** How much of the stack is already listening on the fixed host ports. */ +type StackProbe = 'none' | 'healthy' | 'partial'; + +const PROBE_OUTCOMES: Record = { + none: 'No running Grid stack found', + healthy: `Found a Grid stack already running on ${KAHUNA_URL}`, + partial: 'Found a partially running Grid stack', +}; + +const PROBE_TIMEOUT_MS = 2_000; + +/** Names a container by its role in the stack, since Docker otherwise assigns a random one. */ +const ROLE_LABEL = 'uk.co.guardian.grid.role'; + +/** The subset of a started stack that teardown needs; a partially-booted stack also fits. */ +interface StoppableStack { + network?: StartedNetwork; + containers: StartedTestContainer[]; + configDir?: string; +} + +/** What the boot tasks build up. Each task mutates it in place for the ones that follow. */ +interface BootContext extends StoppableStack { + coreStackProps?: StackProps; +} + +/** + * Build a Caddyfile that reproduces the dev-nginx subdomain routing: each Grid service + * domain (https://.media.) reverse-proxies to the grid-e2e-ci container on + * its in-container port, and the S3 vanity domains proxy to localstack (prepending the + * real bucket to the path). `tls internal` serves a self-signed cert per site; Playwright + * runs with `ignoreHTTPSErrors`, so the internal CA does not need to be trusted. + * + * Example output: + * + * { + * auto_https disable_redirects + * } + * + * media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:9005 + * } + * + * api.media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:9001 + * } + * + * # ... etc for other similar services + * + * images.media.local.dev-gutools.co.uk { + * tls internal + * rewrite * /grid-dev-core-imagebucket-2b34bef1{uri} + * reverse_proxy localhost:4566 + * } + * + * public.media.local.dev-gutools.co.uk { + * tls internal + * rewrite * /grid-dev-core-imageoriginbucket-e8cd1fec{uri} + * reverse_proxy localhost:4566 + * } + * + * localstack.media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:4566 + * } + */ +function buildCaddyfile(coreStackProps: Record): string { + const appServices: Record = { + [`media.${DOMAIN}`]: SERVICE_PORTS.kahuna, + [`api.media.${DOMAIN}`]: SERVICE_PORTS['media-api'], + [`loader.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], + [`loader-projection.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], + [`cropper.media.${DOMAIN}`]: SERVICE_PORTS.cropper, + [`thrall.media.${DOMAIN}`]: SERVICE_PORTS.thrall, + [`media-metadata.${DOMAIN}`]: SERVICE_PORTS['metadata-editor'], + [`media-collections.${DOMAIN}`]: SERVICE_PORTS.collections, + [`media-leases.${DOMAIN}`]: SERVICE_PORTS.leases, + [`media-auth.${DOMAIN}`]: SERVICE_PORTS.auth, + }; + + // S3 vanity domains that omit the bucket -> localstack, with the bucket prepended. + const imageBuckets: Record = { + [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket, + [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket + }; + + const blocks: string[] = []; + + for (const [siteHost, port] of Object.entries(appServices)) { + blocks.push(`${siteHost} {\n\ttls internal\n\treverse_proxy ${GRID_ALIAS}:${port}\n}`); + } + + for (const [siteHost, bucket] of Object.entries(imageBuckets)) { + blocks.push( + `${siteHost} {\n\ttls internal\n\trewrite * /${bucket}{uri}\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, + ); + } + + // Thumbnails / direct S3 access already include the bucket in the path. + blocks.push( + `localstack.media.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, + ); + + // On-the-fly image resizing (optimised / full-screen views) -> the imgops container. + blocks.push( + `media-imgops.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${IMGOPS_ALIAS}:80\n}`, + ); + + return `${blocks.join('\n\n')}\n`; +} + +/** + * Set while this process owns the config directory, so the exit hook below never deletes + * the config of a stack started elsewhere and still running (see `attachToStack`). + */ +let ownedConfigDir: string | undefined; + +// Catches the exits that bypass `stopStack`, such as a Ctrl-C mid-boot in dev.ts. +// Must stay synchronous: async work in an `exit` handler never runs. +process.on('exit', () => { + if (ownedConfigDir) { + fs.rmSync(ownedConfigDir, { recursive: true, force: true }); + } +}); + +function elasticsearchContainer(network: StartedNetwork): GenericContainer { + return new GenericContainer(ELASTICSEARCH_IMAGE) + .withNetwork(network) + .withNetworkAliases(ELASTICSEARCH_ALIAS) + .withLabels({ [ROLE_LABEL]: 'Elasticsearch' }) + .withEnvironment({ + 'discovery.type': 'single-node', + 'xpack.security.enabled': 'false', + ES_JAVA_OPTS: '-Xms1024m -Xmx1024m', + }) + .withExposedPorts({ container: ELASTICSEARCH_PORT, host: ELASTICSEARCH_PORT }) + .withWaitStrategy( + Wait.forHttp('/_cluster/health', ELASTICSEARCH_PORT).forStatusCodeMatching((code) => code < 300), + ) + .withStartupTimeout(180_000); +} + +function localstackContainer(network: StartedNetwork): LocalstackContainer { + return new LocalstackContainer(LOCALSTACK_IMAGE) + .withNetwork(network) + .withNetworkAliases(LOCALSTACK_ALIAS) + .withLabels({ [ROLE_LABEL]: 'LocalStack' }) + // Pin to the fixed host port dev-nginx expects for the S3 vanity domains + // (images.media / public.media / localstack.media -> 4566). + .withExposedPorts({ container: LOCALSTACK_PORT, host: LOCALSTACK_PORT }) + .withEnvironment({ + SERVICES: LOCALSTACK_SERVICES, + DEFAULT_REGION: REGION, + KINESIS_ERROR_PROBABILITY: '0.0', + // Make resource URLs resolve via the network alias so the + // app container can reach them, and keep queue URLs path-style. + LOCALSTACK_HOST: `${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`, + SQS_ENDPOINT_STRATEGY: 'path', + }) + .withStartupTimeout(120_000); +} + +/** + * imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to + * the `localstack` alias on 4566, so it shares the stack network. + */ +function imgopsContainer(image: GenericContainer, network: StartedNetwork): GenericContainer { + return image + .withNetwork(network) + .withNetworkAliases(IMGOPS_ALIAS) + .withLabels({ [ROLE_LABEL]: 'imgops' }) + .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) + .withExposedPorts({ container: 80, host: IMGOPS_PORT }) + .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) + .withStartupTimeout(120_000); +} + +/** + * All Grid services under test run inside this single container and talk to each + * other over its localhost. Each is published on the fixed host port its + * dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's + * dev-nginx routes the https://*.media. domains straight into this + * container. + */ +function gridContainer( + network: StartedNetwork, + configDir: string, + startupTimeoutMs: number, +): GenericContainer { + const container = new GenericContainer(GRID_IMAGE) + .withNetwork(network) + .withNetworkAliases(GRID_ALIAS) + .withLabels({ [ROLE_LABEL]: 'the Grid services' }) + .withExposedPorts( + ...Object.values(SERVICE_PORTS).map((port) => ({ container: port, host: port })), + ) + .withBindMounts([ + // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. + { source: configDir, target: '/root/.grid', mode: 'ro' }, + { source: configDir, target: '/etc/grid', mode: 'ro' }, + // Outside CI the grid-e2e-dev image runs services under sbt; mount the repo + // over /build so host edits recompile live. + ...(process.env.CI ? [] : [{ source: REPO_ROOT, target: '/build', mode: 'rw' as const }]), + ]) + .withEnvironment({ + AWS_ACCESS_KEY_ID: 'test', + AWS_SECRET_ACCESS_KEY: 'test', + AWS_REGION: REGION, + AWS_DEFAULT_REGION: REGION, + AWS_CBOR_DISABLE: 'true', + }) + // Waiting on the healthchecks here would collapse every service into one opaque wait; + // the first line of output is enough to hand over to the per-service checks below. + .withWaitStrategy(Wait.forLogMessage(/./)) + .withStartupTimeout(10_000); + + if (!process.env.GRID_DEBUG) { + return container; + } + + const logStream = fs.createWriteStream(path.join(os.tmpdir(), 'grid-boot.log')); + return container.withLogConsumer((stream) => { + stream.on('data', (line) => logStream.write(line)); + stream.on('err', (line) => logStream.write(line)); + }); +} + +/** + * Locally, the browser reaches the https://*.media. domains via the developer's + * dev-nginx. CI has no dev-nginx, so a Caddy proxy replays the same subdomain routing and + * terminates TLS with a self-signed cert, published on the https port the domains resolve to. + */ +function proxyContainer(network: StartedNetwork, caddyfile: string): GenericContainer { + return new GenericContainer(PROXY_IMAGE) + .withNetwork(network) + .withLabels({ [ROLE_LABEL]: 'the reverse proxy' }) + .withExposedPorts({ container: 443, host: 443 }) + .withCopyContentToContainer([{ content: caddyfile, target: '/etc/caddy/Caddyfile' }]) + .withWaitStrategy(Wait.forListeningPorts()) + .withStartupTimeout(60_000); +} + +/** Poll a fixed host port until its healthcheck passes, reporting how long it has waited. */ +async function waitForHealthy( + port: number, + healthPath: string, + timeoutMs: number, + report: (message: string) => void, +): Promise { + const startedAt = Date.now(); + const deadline = startedAt + timeoutMs; + + for (; ;) { + const { healthy } = await isServiceHealthy(healthPath)(port); + if (healthy) { + return; + } + if (Date.now() > deadline) { + throw new Error(`localhost:${port}/${healthPath} did not pass its healthcheck within ${timeoutMs}ms`); + } + + report(`waiting on localhost:${port} (${Math.round((Date.now() - startedAt) / 1000)}s)`); + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } +} + +/** + * Start LocalStack and provision everything held in it. Sequential: each step needs the + * resources the one before it created. + */ +function localstackTasks(): ListrTask[] { + let clients: ReturnType; + + return [ + { + title: 'Start container', + task: async (ctx) => { + const localstack = await localstackContainer(ctx.network!).start(); + ctx.containers.push(localstack); + clients = provisioningClients(localstack.getConnectionUri()); + }, + }, + { + title: 'Apply CloudFormation template', + task: async (ctx) => { + ctx.coreStackProps = await createCoreStack(clients.cfn); + }, + }, + { + title: 'Seed config buckets', + task: (ctx) => seedBuckets(clients.s3, ctx.coreStackProps!), + }, + { + title: 'Create permissions bucket', + task: async (ctx) => { + ctx.coreStackProps!.PermissionsBucket = await provisionPermissionsBucket(clients.s3); + }, + }, + { + title: 'Seed KCL lease tables', + task: async (ctx) => { + await Promise.all( + [ + ctx.coreStackProps!.ThrallMessageStream, + ctx.coreStackProps!.ThrallLowPriorityMessageStream, + ].map((stream) => seedKclLeaseTable(clients.dynamo, clients.kinesis, stream)), + ); + }, + }, + ]; +} + +/** Start the whole stack, tearing down anything already started if a later step fails. */ +export async function startStack(options: StartStackOptions = {}): Promise { + const { proxy = !!process.env.CI, seed = true } = options; + + const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 120_000); + const context: BootContext = { containers: [] }; + + const tasks: ListrTask[] = [ + { + title: 'Create network', + task: async (ctx) => { + ctx.network = await new Network().start(); + }, + }, + { + // These three share only the network, and Elasticsearch is by far the slowest to come + // up, so provisioning LocalStack costs nothing beyond it. + title: 'Start infrastructure', + task: (_, task) => + task.newListr( + [ + { + title: 'Elasticsearch', + task: async (ctx) => { + ctx.containers.push(await elasticsearchContainer(ctx.network!).start()); + }, + }, + { + // nginx resolves the `localstack` alias per request, so this need not wait for it. + title: 'imgops', + task: (_, imgopsTask) => { + let image: GenericContainer; + + return imgopsTask.newListr( + [ + { + title: 'Build image', + task: async () => { + image = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build( + IMGOPS_IMAGE, + { deleteOnExit: false }, + ); + }, + }, + { + title: 'Start container', + task: async (ctx) => { + ctx.containers.push(await imgopsContainer(image, ctx.network!).start()); + }, + }, + ], + { concurrent: false }, + ); + }, + }, + { + title: 'LocalStack', + task: (_, task) => task.newListr(localstackTasks(), { concurrent: false }), + }, + ], + { concurrent: true }, + ), + }, + { + title: 'Generate service config', + task: (ctx) => { + // Reaching here means the probe found no live stack, so recreating the shared path is + // safe and clears anything a killed run left behind. + fs.rmSync(CONFIG_DIR, { recursive: true, force: true }); + fs.mkdirSync(CONFIG_DIR, { recursive: true }); + ctx.configDir = CONFIG_DIR; + ownedConfigDir = CONFIG_DIR; + generateServiceConfig(CONFIG_DIR, ctx.coreStackProps!); + }, + }, + { + title: 'Start Grid services', + task: (_, task) => + task.newListr([ + { + title: 'Start container', + task: async (ctx) => { + ctx.containers.push( + await gridContainer(ctx.network!, ctx.configDir!, startupTimeoutMs).start(), + ); + }, + }, + { + title: 'Wait for services', + task: (_, services) => { + const readiness: ListrTask[] = [ + ...Object.entries(SERVICE_PORTS).map(([service, port]): { title: string, task: ListrTaskFn } => ({ + title: service, + task: (_, serviceTask) => + waitForHealthy(port, 'management/healthcheck', startupTimeoutMs, reportTo(serviceTask)), + })), + { + // Waits for the `Images_Current` alias the app assigns on startup, so this + // only needs the container running, not every service healthy. + title: 'Seed Elasticsearch', + skip: () => !seed && 'seeding not requested', + task: async (_, seedTask) => { + await seedElasticsearch(ELASTICSEARCH_URL, startupTimeoutMs, reportTo(seedTask)); + }, + }, + ]; + + return services.newListr(readiness, { concurrent: true }); + }, + }, + ]), + }, + { + title: 'Start reverse proxy', + task: async (ctx) => { + const caddy = await proxyContainer(ctx.network!, buildCaddyfile(ctx.coreStackProps!)).start(); + ctx.containers.push(caddy); + }, + }, + ]; + + try { + await runTasks(tasks, context); + + return { + network: context.network, + containers: context.containers, + configDir: context.configDir, + baseUrl: KAHUNA_URL, + mediaApiUrl: MEDIA_API_URL, + }; + } catch (error) { + // Leave nothing running if we failed part-way through the boot. + await stopStack(context); + throw error; + } +} + +/** Is a service answering its healthcheck on this fixed host port? */ +const isServiceHealthy = (path: string) => async (port: number): Promise<{ port: number, healthy: boolean }> => { + try { + const response = await fetch(`http://localhost:${port}/${path}`, { + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + return { port, healthy: response.ok }; + } catch { + return { port, healthy: false }; + } +} + +/** + * Probe the fixed host ports for an already-running stack. + * + * The ports are pinned, so a live stack is always reachable at localhost:. + * That makes the probe, rather than the presence of the URLs file, the source of truth: + * the file outlives a `SIGKILL`ed run and would otherwise point tests at nothing. + */ +export async function probeStack(): Promise<{ state: StackProbe; healthy: number[]; ports: number[] }> { + const healthchecks = [ + ...Object.values(SERVICE_PORTS).map(isServiceHealthy('management/healthcheck')), + isServiceHealthy('_cluster/health')(ELASTICSEARCH_PORT), + isServiceHealthy('_localstack/health')(LOCALSTACK_PORT), + isServiceHealthy('_')(IMGOPS_PORT) + ]; + + const results = await Promise.all(healthchecks); + const healthy = results.filter(({ healthy }) => healthy).map(({ port }) => port); + const ports = results.map(({ port }) => port); + + if (healthy.length === 0) return { state: 'none', healthy, ports }; + if (healthy.length === ports.length) return { state: 'healthy', healthy, ports }; + return { state: 'partial', healthy, ports }; +} + +/** + * Attach to a running stack if there is a healthy one, otherwise boot a fresh one. + * + * The returned environment holds handles only for what this call created, so teardown + * never stops a stack started by `npm run dev:e2e`. + */ +export async function ensureStack(options: StartStackOptions = {}): Promise { + // Nothing pre-exists in CI, and silently attaching there would undermine the run. + const reuseAllowed = !process.env.CI; + + const { state, healthy, ports } = await probeStack(); + console.log(PROBE_OUTCOMES[state]); + + if (state === 'partial') { + const missing = ports.filter((port) => !healthy.includes(port)); + throw new Error( + `A partial Grid stack is running: ports ${healthy.join(', ')} are healthy but ` + + `${missing.join(', ')} are not. Stop it (or wait for it to finish booting) and retry.`, + ); + } + + if (state === 'healthy') { + if (!reuseAllowed) { + throw new Error( + 'A Grid stack is already running and reuse is disabled, as we are running in CI. ' + + 'Stop it before starting a fresh one; the fixed host ports cannot be shared.', + ); + } + return attachToStack(options); + } + + return startStack(options); +} + +/** + * Build an environment for a stack this process did not start. It owns no containers, + * so `stopStack` leaves everything running. + */ +async function attachToStack(options: StartStackOptions): Promise { + // Re-seeding is opt-in: whoever started the stack already seeded it, and the fixtures + // are only reloaded on request because tests may have since changed the data. + const reseed = options.seed === true || process.env.GRID_RESEED === 'true'; + + await runTasks( + [ + { + title: 'Seed Elasticsearch', + skip: () => !reseed && 'reseeding not requested', + task: async (_, task) => { + await seedElasticsearch(ELASTICSEARCH_URL, 60_000, reportTo(task)); + }, + }, + ], + {}, + ); + + return { containers: [], baseUrl: KAHUNA_URL, mediaApiUrl: MEDIA_API_URL }; +} + +/** Stop what this process started and delete what it wrote; anything else is left alone. */ +export async function stopStack(environment: StoppableStack | undefined): Promise { + if (!environment) { + return; + } + + const { containers, network, configDir } = environment; + + const tasks: ListrTask[] = [ + // Reverse start order, so nothing is stopped before whatever depends on it. + ...[...containers].reverse().map((container) => ({ + title: `Stop ${container.getLabels()[ROLE_LABEL] ?? container.getName()}`, + task: () => container.stop(), + })), + ...(network ? [{ title: 'Remove network', task: () => network.stop() }] : []), + ...(configDir + ? [ + { + title: 'Remove generated config', + task: () => { + fs.rmSync(configDir, { recursive: true, force: true }); + ownedConfigDir = undefined; + }, + }, + ] + : []), + ]; + + if (tasks.length === 0) { + return; + } + + // Every step is best-effort: a failure is reported against its task and the rest still run. + await runTasks(tasks, {}, { exitOnError: false }); +} diff --git a/e2e-tests/setup/state.ts b/e2e-tests/setup/state.ts new file mode 100644 index 00000000000..1c16a90b51f --- /dev/null +++ b/e2e-tests/setup/state.ts @@ -0,0 +1,35 @@ +/** + * Shared, process-scoped registry for the Testcontainers instances started by + * `global-setup.ts`. Playwright runs global setup and global teardown in the same + * Node process, so a module-level singleton is a reliable way to hand references + * from setup to teardown. + * + * Ownership is structural: this process only holds handles to resources it created, + * so teardown stops what it holds and deletes what it wrote. When attaching to a + * stack someone else started, these are empty and teardown is a no-op. + */ +import type { StartedTestContainer } from 'testcontainers'; +import type { StartedNetwork } from 'testcontainers'; + +export interface GridEnvironment { + /** Absent when attached to a stack this process did not start. */ + network?: StartedNetwork; + /** Every container this process started, in start order. Stopped in reverse on teardown. */ + containers: StartedTestContainer[]; + /** Absolute path to the generated config directory mounted into the app container. */ + configDir?: string; + /** Base URL of the Kahuna UI, e.g. http://localhost:9005 */ + baseUrl: string; + /** Base URL of the media-api, e.g. http://localhost:9001 */ + mediaApiUrl: string; +} + +let environment: GridEnvironment | undefined; + +export function setEnvironment(env: GridEnvironment): void { + environment = env; +} + +export function getEnvironment(): GridEnvironment | undefined { + return environment; +} diff --git a/e2e-tests/steps/common.steps.ts b/e2e-tests/steps/common.steps.ts new file mode 100644 index 00000000000..4a664e28f37 --- /dev/null +++ b/e2e-tests/steps/common.steps.ts @@ -0,0 +1,32 @@ +import { Given, KAHUNA_APP_URL, expect } from './fixtures.ts'; + +Given('the application stack is running', async ({ request }) => { + const response = await request.get('/management/healthcheck'); + expect(response.ok()).toBeTruthy(); +}); + +Given('I have opened the image upload page', async ({ page, testContext }) => { + // An unhandled dialog blocks the page, so record every one and answer it. Dismissing a + // confirm answers "no", which keeps us on the page under test. + page.on('dialog', (dialog) => { + testContext.dialogs.push(dialog.message()); + void dialog.dismiss(); + }); + + // Accept the default of blurring graphic images up front. Kahuna otherwise shows a + // first-run explainer overlay that covers the top bar (see services/graphic-image-blur.js). + await page.context().addCookies([ + { + name: 'SHOULD_BLUR_GRAPHIC_IMAGES', + value: 'true', + domain: `.${new URL(KAHUNA_APP_URL).hostname}`, + path: '/', + }, + ]); + + // Arrive from search rather than deep-linking, so the upload page has a same-document + // history entry behind it and back-navigation behaves as it does for a real user. + await page.goto(KAHUNA_APP_URL); + await page.getByRole('banner').getByRole('link', { name: 'My recent uploads' }).click(); + await page.waitForURL('**/upload'); +}); diff --git a/e2e-tests/steps/fixtures.ts b/e2e-tests/steps/fixtures.ts new file mode 100644 index 00000000000..15460542ceb --- /dev/null +++ b/e2e-tests/steps/fixtures.ts @@ -0,0 +1,34 @@ +import { expect } from '@playwright/test'; +import type { APIResponse, FileChooser, Response } from '@playwright/test'; +import { test as base, createBdd } from 'playwright-bdd'; +import { DOMAIN } from '../setup/constants.ts'; + +export const KAHUNA_APP_URL = `https://media.${DOMAIN}`; + +/** + * Mutable per-scenario state shared between steps (e.g. the response captured in a + * `When` step and asserted on in a `Then` step). + */ +interface TestContext { + response?: APIResponse; + mediaApiResponse?: Promise; + /** Messages from alerts and confirms, in the order the page raised them. */ + dialogs: string[]; + fileChooser?: Promise; + /** The query run on the search page before navigating to the upload page. */ + previousSearchQuery?: string; +} + +/** + * Browser steps should navigate to `KAHUNA_APP_URL` so the page origin is a real Grid + * domain (required for the services' CORS origins to match). API-level `request` tests + * use `baseURL` from the Playwright config, which points at Kahuna's fixed host port. + */ +export const test = base.extend<{ testContext: TestContext }>({ + testContext: async ({}, use) => { + await use({ dialogs: [] }); + }, +}); + +export const { Given, When, Then, Before, After } = createBdd(test); +export { expect }; diff --git a/e2e-tests/steps/kahuna.steps.ts b/e2e-tests/steps/kahuna.steps.ts new file mode 100644 index 00000000000..4833a4a23cf --- /dev/null +++ b/e2e-tests/steps/kahuna.steps.ts @@ -0,0 +1,19 @@ +import { When, Then, expect } from './fixtures.ts'; + +When('I request the Kahuna healthcheck endpoint', async ({ request, testContext }) => { + testContext.response = await request.get('/management/healthcheck'); +}); + +Then('the response is successful', async ({ testContext }) => { + expect(testContext.response?.ok()).toBeTruthy(); +}); + +When('I request the Kahuna root path without following redirects', async ({ request, testContext }) => { + // The root path may redirect to auth; any non-server-error response proves Kahuna + // is serving requests against the provisioned infrastructure. + testContext.response = await request.get('/', { maxRedirects: 0 }); +}); + +Then('the response status is below {int}', async ({ testContext }, status: number) => { + expect(testContext.response?.status()).toBeLessThan(status); +}); diff --git a/e2e-tests/steps/upload/setup.ts b/e2e-tests/steps/upload/setup.ts new file mode 100644 index 00000000000..68759793500 --- /dev/null +++ b/e2e-tests/steps/upload/setup.ts @@ -0,0 +1,52 @@ +import { statSync } from 'node:fs'; +import * as path from 'node:path'; +import type { Page } from '@playwright/test'; + +declare global { + interface Window { + _clientConfig: { + systemName: string; + maybeUploadLimitInBytes: number; + }; + } +} + +const FIXTURE_IMAGES = path.join(import.meta.dirname, '..', '..', 'fixtures', 'images'); + +export interface TestImage { + fileName: string; + path: string; + bytes: number; +} + +const testImage = (fileName: string): TestImage => { + const filePath = path.join(FIXTURE_IMAGES, fileName); + return { fileName, path: filePath, bytes: statSync(filePath).size }; +}; + +/** Sizes are read from disk so the size-limit scenario can pick a threshold between them. */ +export const testImages = { + smaller: testImage('test-card-f.jpg'), + larger: testImage('test.jpg'), +}; + +export const uploadPage = (page: Page) => { + const prompt = page.getByRole('region', { name: 'File upload' }); + + return { + prompt, + main: page.getByRole('main', { name: 'Image uploads' }), + currentUploads: page.getByRole('region', { name: 'Your current uploads' }), + pastUploads: page.getByRole('region', { name: 'Your past 50 uploads' }), + dragAndDropUploader: page.getByRole('region', { name: 'Drag and drop uploader' }), + fileInput: prompt.locator('input[name="files"]'), + /* The upload and back-to-search controls carry aria-labels that override their visible + text, so filter on the text the feature file names rather than the accessible name. */ + uploadButton: (label: string) => prompt.getByRole('button').filter({ hasText: label }), + topBarLink: (label: string) => page.getByRole('banner').getByRole('link').filter({ hasText: label }), + /** Any control that takes you off the upload page, wherever it sits on it. */ + leaveLink: (label: string) => page.getByRole('link').filter({ hasText: label }), + /** A queued or in-flight upload, before it becomes an editable image. */ + job: (fileName: string) => page.getByRole('region', { name: `${fileName} upload` }), + }; +}; diff --git a/e2e-tests/steps/upload/upload.steps.ts b/e2e-tests/steps/upload/upload.steps.ts new file mode 100644 index 00000000000..bd9665f13cb --- /dev/null +++ b/e2e-tests/steps/upload/upload.steps.ts @@ -0,0 +1,182 @@ +import type { DataTable } from 'playwright-bdd'; +import { Given, KAHUNA_APP_URL, Then, When, expect } from '../fixtures.ts'; +import { testImages, uploadPage } from './setup.ts'; + +const filesToUpload = [testImages.smaller, testImages.larger]; + +/** + * Upload page shell + */ + +Given('I am permitted to upload images', async ({ page }) => { + // Kahuna decides whether to show the upload tools by looking for a `loader` link on the + // media API root, which the API only emits for users holding the upload permission. + const mediaApiUri = await page.evaluate( + () => document.querySelector('link[rel="media-api-uri"]')?.getAttribute('href'), + ); + const response = await page.request.get(mediaApiUri!); + const { links } = (await response.json()) as { links: { rel: string }[] }; + + expect(links.map((link) => link.rel)).toContain('loader'); +}); + +When('the upload page loads', async ({ page }) => { + await expect(uploadPage(page).main).toBeVisible(); +}); + +Then('I should see the file upload prompt', async ({ page }) => { + await expect(uploadPage(page).prompt).toBeVisible(); +}); + +Then('I should see my past {int} uploads', async ({ page }, count: number) => { + await expect(page.getByRole('region', { name: `Your past ${count} uploads` })).toBeVisible(); +}); + +Then('the drag-and-drop uploader should be active', async ({ page }) => { + // The dropzone overlay only renders mid-drag, so "active" means mounted and listening. + await expect(uploadPage(page).dragAndDropUploader).toBeAttached(); +}); + +Then('I should not see the current uploads section', async ({ page }) => { + await expect(uploadPage(page).currentUploads).not.toBeVisible(); +}); + +Given( + 'I had searched for {string} before opening the upload page', + async ({ page, testContext }, query: string) => { + await page.goto(`${KAHUNA_APP_URL}/search?query=${encodeURIComponent(query)}`); + await expect(page.getByRole('main', { name: 'Image search results' })).toBeVisible(); + testContext.previousSearchQuery = query; + + await page.getByRole('banner').getByRole('link', { name: 'My recent uploads' }).click(); + await page.waitForURL('**/upload'); + }, +); + +When('I choose {string} from the top bar', async ({ page }, label: string) => { + await uploadPage(page).topBarLink(label).click(); +}); + +Then('I should be taken to the image search page', async ({ page }) => { + await expect(page).toHaveURL((url) => url.origin === KAHUNA_APP_URL && url.pathname === '/search'); +}); + +Then('my previous search should be intact', async ({ page, testContext }) => { + // The `search` state is a deep-state redirect, so entering it without params sends us + // back to the `search.results` params we left behind. + await expect(page).toHaveURL( + (url) => url.searchParams.get('query') === testContext.previousSearchQuery, + ); +}); + +Given('I have an upload in progress', async ({ page }) => { + // Hold the transfer to the ingest bucket open, otherwise the job reaches a terminal + // state within a second or so and is no longer "in progress" by the time we assert. + // This is long enough for a scenario's assertions, short enough not to drag out teardown. + const uploadHoldMs = 5_000; + await page.route( + (url) => url.hostname.startsWith('localstack.'), + async (route) => { + if (route.request().method() !== 'PUT') return route.fallback(); + await new Promise((resolve) => setTimeout(resolve, uploadHoldMs)); + await route.abort(); + }, + ); + + await uploadPage(page).fileInput.setInputFiles(testImages.smaller.path); + await expect(uploadPage(page).job(testImages.smaller.fileName)).toBeVisible(); +}); + +Then('I should see my current uploads section', async ({ page }) => { + await expect(uploadPage(page).currentUploads).toBeVisible(); +}); + +When('I choose {string}', async ({ page }, label: string) => { + await uploadPage(page).pastUploads.getByRole('link', { name: label }).click(); +}); + +Then('I should be taken to a search filtered to images I uploaded', async ({ page }) => { + await expect(page).toHaveURL((url) => url.searchParams.get('uploadedBy') === 'johndoe@example.com'); +}); + +When( + 'I try to navigate away from the upload page via the following buttons:', + async ({ page }, buttons: DataTable) => { + for (const [label] of buttons.raw()) { + await uploadPage(page).leaveLink(label).click(); + // Dismissing the confirm answers "no", so each button leaves us here for the next one. + await expect(page).toHaveURL(/\/upload/); + } + }, +); + +Then('I should be warned that uploads are in progress and asked to confirm', async ({ testContext }) => { + await expect + .poll(() => testContext.dialogs) + .toContain('You have uploads in progress. Are you sure you want to leave this page?'); +}); + +/** + * File upload prompt + */ + +Then('I should see a message telling me to drag and drop or click to upload to the system', async ({ page }) => { + const systemName = await page.evaluate(() => window._clientConfig.systemName); + + await expect(uploadPage(page).prompt).toContainText( + `Either drag 'n drop images onto this screen or click`, + ); + await expect(uploadPage(page).prompt).toContainText(`to get your images on ${systemName}.`); +}); + +/** + * Select-files uploader + */ + +When('I click the {string} button', async ({ page, testContext }, label: string) => { + testContext.fileChooser = page.waitForEvent('filechooser'); + await uploadPage(page).uploadButton(label).click(); +}); + +Then('the system file picker should open', async ({ testContext }) => { + const fileChooser = await testContext.fileChooser; + + expect(fileChooser?.isMultiple()).toBe(true); +}); + +When('I select one or more image files to upload', async ({ page }) => { + await uploadPage(page).fileInput.setInputFiles(filesToUpload.map(({ path }) => path)); +}); + +Then('those files should be queued for upload', async ({ page }) => { + for (const { fileName } of filesToUpload) { + await expect(uploadPage(page).job(fileName)).toBeVisible(); + } +}); + +Then('I should be taken to the upload progress view', async ({ page }) => { + await expect(uploadPage(page).currentUploads).toBeVisible(); +}); + +Given('an upload size limit is configured', async ({ page }) => { + // upload/manager.js reads the limit when files are selected, so setting it now is enough. + const limit = Math.round((testImages.smaller.bytes + testImages.larger.bytes) / 2); + + await page.evaluate((bytes) => { + window._clientConfig.maybeUploadLimitInBytes = bytes; + }, limit); +}); + +When('I select a file that is larger than the size limit', async ({ page }) => { + await uploadPage(page).fileInput.setInputFiles(filesToUpload.map(({ path }) => path)); +}); + +Then('I should be warned that the oversized file will be skipped', async ({ testContext }) => { + await expect.poll(() => testContext.dialogs.join('\n')).toContain('are above the size limit'); + expect(testContext.dialogs.join('\n')).toContain(testImages.larger.fileName); +}); + +Then('only the files within the limit should be queued for upload', async ({ page }) => { + await expect(uploadPage(page).job(testImages.smaller.fileName)).toBeVisible(); + await expect(uploadPage(page).job(testImages.larger.fileName)).toBeHidden(); +}); diff --git a/image-embedder-lambda/package-lock.json b/image-embedder-lambda/package-lock.json index 8bdb0a1d16f..743f28d4ca8 100644 --- a/image-embedder-lambda/package-lock.json +++ b/image-embedder-lambda/package-lock.json @@ -15,7 +15,7 @@ "@aws-sdk/client-s3vectors": "^3.1030.0", "@aws-sdk/client-sqs": "^3.1030.0", "@aws-sdk/config": "^3.1030.0", - "sharp": "^0.34.5", + "sharp": "^0.35.0", "source-map-support": "^0.5.21" }, "devDependencies": { @@ -1928,9 +1928,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", - "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", "license": "MIT", "optional": true, "dependencies": { @@ -2401,18 +2401,18 @@ } }, "node_modules/@img/colour": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", - "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", "license": "MIT", "engines": { "node": ">=18" } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.0.tgz", + "integrity": "sha512-ZgaYEwaj+lx/5n4W8GmZ2IYz0PQHjN5eqRcfijWGB+2Aq7ZInZGa0qJyAn6DEtyLuWHRSrmWOqT9q3qqTBvmUQ==", "cpu": [ "arm64" ], @@ -2422,19 +2422,19 @@ "darwin" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" + "@img/sharp-libvips-darwin-arm64": "1.3.0" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.0.tgz", + "integrity": "sha512-c1z9LFpKB0slQW3RchwBE8iSVzGp70TNjUUO9k4BZwwW4HH7JBGHeIy4b+kk4n/kcBASb9evKCE3/7Slmslgiw==", "cpu": [ "x64" ], @@ -2444,19 +2444,38 @@ "darwin" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" + "@img/sharp-libvips-darwin-x64": "1.3.0" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.0.tgz", + "integrity": "sha512-Li2KTev0H90kEtnJHkI9xQojXt1AqWmFBMXiPw5kqd1jQgP7gi5HVK/qC5Rmh/59NuAwUuPzzPITmX22NomYYQ==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.0" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.0.tgz", + "integrity": "sha512-EKbmBKtyTH+GPFDRw2TgK2oV6hyxxlJVIar4hoTYSNmIwipgMFdxPQqR392GmfdsPGWga0mCFN1cCKjRb9cljw==", "cpu": [ "arm64" ], @@ -2470,9 +2489,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.0.tgz", + "integrity": "sha512-Pl2OmOvrJ42adUllESxBsG54PfXLo1OYg9i3c5/5Ln/qJ0gZuTM9YMhQJPIbXqwidLRc/c2zuHt4RsrymmNv7A==", "cpu": [ "x64" ], @@ -2486,12 +2505,15 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.0.tgz", + "integrity": "sha512-A8UpHoUDW4DwnXoV6+q3C1s7QLRAHtPDEjWuNZjwHMyoCNZnm0GeNN8ls9f/bsEYTRQRW96C/n34XJQHJ2fT7A==", "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2502,12 +2524,15 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.0.tgz", + "integrity": "sha512-C0SqjoFKnszqa44EQ7xoaT48nnO0lOyXEULfXMWi8krrjOPGYkeK30Okzla6ATbBYsyZ0ySinK0FVkpv3DwzfQ==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2518,12 +2543,15 @@ } }, "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.0.tgz", + "integrity": "sha512-WOpkVxAjFd369iaIzEgNRreFD+gWdUMIGD5zplhNKNeqS6mm5dac3q2AFyCBmzYoAdouzZvRBgxy4z8QHZb4/A==", "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2534,12 +2562,15 @@ } }, "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.0.tgz", + "integrity": "sha512-DRWw0mOHusrCCuw2rqP87oLg6PGlkomVDFqw2hIwsSfwWpu4k3XLcBPaKKl6ct/GtL/cwNkgwjV/tc0Mqht3VA==", "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2550,12 +2581,15 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.0.tgz", + "integrity": "sha512-9APy+nFWhHS+kzLgWZfLcyrUd7YqnAQVa4BPOo4xkoHpdoktOAPG4cEr9+Jpl0TtqfVmcMJimNL5qNTyyOHZNA==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2566,12 +2600,15 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.0.tgz", + "integrity": "sha512-y9RNUYDe2A1UAdhLyfeOodGRszQdaEoe4nfOpp/sNVPl2CWIcUyFaDoCh4vPLPxu19803j2naLqZup2WxDXCLA==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2582,12 +2619,15 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.0.tgz", + "integrity": "sha512-cC1wkC0Mlucd0KSiGrLkJnB/ZqPvZCntc/Lk7ZnYO5ZSbF2euNek4Xvxafojq+wN1q/W0eprdpUIjUr/EV2PBg==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2598,12 +2638,15 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.0.tgz", + "integrity": "sha512-LiYMhUZicB1QG//+RvmYZpXJO8fYRENfp+MZUCnG9aw+AKvGAy9gPaCnuwsPcBFs8EV66M0NNxj9VHcNklE8zw==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2614,204 +2657,244 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.0.tgz", + "integrity": "sha512-VVlpEWwizEFIOom0zdoeKuO5nuTswzVE5uHcBNvHzmeHUpNFajY3HFfbQ+zIH4E2kVaZ/yVxmsShW56TtEy4uA==", "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" + "@img/sharp-libvips-linux-arm": "1.3.0" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.0.tgz", + "integrity": "sha512-4+4XHLNT5wDT0roYlHTEmH9lDKt0acf9Tv+3hM3iceOirkxrR404/3WjAYZ9F9CkHrxeRcGLJXbi4vluMZ9O+A==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" + "@img/sharp-libvips-linux-arm64": "1.3.0" } }, "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.0.tgz", + "integrity": "sha512-N3hzbEpUTJC8pWpPVJvgzGxM+so/MAXc8O2s/53B0LL9ZGpfXpME7Wizkc5d/8fRBlBtkDjzoZGDCqqNDHqLEw==", "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" + "@img/sharp-libvips-linux-ppc64": "1.3.0" } }, "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.0.tgz", + "integrity": "sha512-l6vmKVPnbS0RhVMbyxP5meAARsbhCnBN4fy31qz0+3a6Rv4jEqfzDrT89y6ZPkCi0AJGnwp2En528yXo401Hpw==", "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" + "@img/sharp-libvips-linux-riscv64": "1.3.0" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.0.tgz", + "integrity": "sha512-MYlMiPFiv/EKPAHnp3yNZ9AAWFsxga9c5Bkc6wkar6bqzHLlkGVJHRm0u1ei+VXnZxp3Mz9MG9ZIsI8vSOf3sQ==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" + "@img/sharp-libvips-linux-s390x": "1.3.0" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.0.tgz", + "integrity": "sha512-TYaItB5oj1ioXjhyn2xrR208vf+YuIIcHptQWRRaBmFhvIvL9D72DXN8w75xup0KXA8UdEAhQ9Qb2S49FD/9Cw==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" + "@img/sharp-libvips-linux-x64": "1.3.0" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.0.tgz", + "integrity": "sha512-DSTb6ijQzqe6DdAaOBVqJ/SYf1vO8EW5bK6X6LRXufEBebf2722VCdvBUtZ3rtV0x2ApfPNDy/p7LrrjaWjiyQ==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + "@img/sharp-libvips-linuxmusl-arm64": "1.3.0" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.0.tgz", + "integrity": "sha512-K7ykQ+26Rt6+4BTU80AuGgTPIYX86UxiAKT4rcXX/WNTo7k1ZxpKz+TguHnwVpCqQK3B5PK0vZ0ZBe6nz/ib1w==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + "@img/sharp-libvips-linuxmusl-x64": "1.3.0" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.0.tgz", + "integrity": "sha512-9woLIFORERCr+6cWu87dQ22J34EExkhc73U1kZW0c+RclQqWetoodByp4dWZ/hN8/KVmTRAx2HOnUwib8AwZdA==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.0" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.0.tgz", + "integrity": "sha512-t+kie1TOyaDM6Dho+f+y0VqIUNhYQaKCUahuZVi0E0frgdiaOaPsDxDW3wfKacUdaNBCnK/ZDBMg33ydvHj8uA==", "cpu": [ "wasm32" ], - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "license": "Apache-2.0", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.7.0" + "@img/sharp-wasm32": "0.35.0" }, "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.0.tgz", + "integrity": "sha512-M5eKxug0dabbaWgFKvPa3odNs2OpaP+81NASfGKkt4GcYXpNhSu7CaeYxWkLNV6vHmUp4hnCxnxrUyhUJhXbKA==", "cpu": [ "arm64" ], @@ -2821,16 +2904,16 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.0.tgz", + "integrity": "sha512-z0+pZ03QCDvdVN0Ez9IX/yjWC19ikMlXrmdYMwYNLTh2BLPx3hXWPvyqWfquZ0BTO9O6GVOjIVoTcyyacMnWlQ==", "cpu": [ "ia32" ], @@ -2840,16 +2923,16 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": "^20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.0.tgz", + "integrity": "sha512-feNnlz5ZHKr0MY1LPHvZQyJeBkbo4ctsn0D8FvA53VTw5TC63rfEL2UrWbkSBR19htSE7Mw78xYVwdJqoMWVHw==", "cpu": [ "x64" ], @@ -2859,7 +2942,7 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" @@ -10322,53 +10405,53 @@ } }, "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "hasInstallScript": true, + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.0.tgz", + "integrity": "sha512-BqvG5XbwPZ4NV0DK90d86leEECMsoa8bO0nqnKWlBDYxri4GJ7c4EDInaF6q20lTh/mATmnDIKWJFfXnoVfH5g==", "license": "Apache-2.0", "dependencies": { - "@img/colour": "^1.0.0", + "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", - "semver": "^7.7.3" + "semver": "^7.8.4" }, "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" + "@img/sharp-darwin-arm64": "0.35.0", + "@img/sharp-darwin-x64": "0.35.0", + "@img/sharp-freebsd-wasm32": "0.35.0", + "@img/sharp-libvips-darwin-arm64": "1.3.0", + "@img/sharp-libvips-darwin-x64": "1.3.0", + "@img/sharp-libvips-linux-arm": "1.3.0", + "@img/sharp-libvips-linux-arm64": "1.3.0", + "@img/sharp-libvips-linux-ppc64": "1.3.0", + "@img/sharp-libvips-linux-riscv64": "1.3.0", + "@img/sharp-libvips-linux-s390x": "1.3.0", + "@img/sharp-libvips-linux-x64": "1.3.0", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.0", + "@img/sharp-libvips-linuxmusl-x64": "1.3.0", + "@img/sharp-linux-arm": "0.35.0", + "@img/sharp-linux-arm64": "0.35.0", + "@img/sharp-linux-ppc64": "0.35.0", + "@img/sharp-linux-riscv64": "0.35.0", + "@img/sharp-linux-s390x": "0.35.0", + "@img/sharp-linux-x64": "0.35.0", + "@img/sharp-linuxmusl-arm64": "0.35.0", + "@img/sharp-linuxmusl-x64": "0.35.0", + "@img/sharp-webcontainers-wasm32": "0.35.0", + "@img/sharp-win32-arm64": "0.35.0", + "@img/sharp-win32-ia32": "0.35.0", + "@img/sharp-win32-x64": "0.35.0" } }, "node_modules/sharp/node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/image-embedder-lambda/package.json b/image-embedder-lambda/package.json index 35364bf9a72..b23b6e9daeb 100644 --- a/image-embedder-lambda/package.json +++ b/image-embedder-lambda/package.json @@ -39,7 +39,7 @@ "@aws-sdk/client-s3vectors": "^3.1030.0", "@aws-sdk/client-sqs": "^3.1030.0", "@aws-sdk/config": "^3.1030.0", - "sharp": "^0.34.5", + "sharp": "^0.35.0", "source-map-support": "^0.5.21" }, "prettier": "@guardian/prettier" diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index db2b9a059e3..be7748d9525 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -21,7 +21,7 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne logger.info(s" $index -> ${processor.description}") } - private val gridClient = GridClient(config.services)(wsClient) + private val gridClient = GridClient(config.services, config.services.loaderBaseUri)(wsClient) val store = new ImageLoaderStore(config) val maybeIngestQueue = config.maybeIngestSqsQueueUrl.map(queueUrl => new SimpleSqsMessageConsumer(queueUrl, config)) diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index ab44b57a88f..cde5129fe82 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -3,9 +3,7 @@ package controllers import org.apache.pekko.Done import org.apache.pekko.stream.Materializer import org.apache.pekko.stream.scaladsl.Source -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.sqs.model.{Message => SQSMessage} -import com.amazonaws.util.IOUtils +import software.amazon.awssdk.services.sqs.model.{Message => SQSMessage} import com.drew.imaging.ImageProcessingException import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.ImageIngestOperations.fileKeyFromId @@ -26,18 +24,23 @@ import lib.storage.{ImageLoaderStore, S3FileDoesNotExistException} import lib._ import model.upload.UploadRequest import model.{Projector, QuarantineUploader, S3FileExtractedMetadata, S3IngestObject, StatusType, UploadStatus, UploadStatusRecord, UploadStatusUri, Uploader} +import org.joda.time.{DateTime, DateTimeZone} import play.api.data.Form import play.api.data.Forms._ import play.api.inject.ApplicationLifecycle import play.api.libs.json.Json import play.api.mvc._ +import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.cloudwatch.model.Dimension +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.{GetObjectRequest, HeadObjectRequest, NoSuchKeyException} import java.io.{File, FileOutputStream} import java.net.URI -import java.time.Instant +import java.time.{Duration, Instant} import java.util.concurrent.atomic.AtomicReference import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters.MapHasAsScala import scala.util.control.NonFatal import scala.util.{Failure, Success, Try} @@ -84,7 +87,7 @@ class ImageLoaderController(auth: Authentication, activeProcessingMessage.set(Some(sqsMessage)) val logMarker: LogMarker = MarkerMap( "requestType" -> "handleMessageFromIngestBucket", - "requestId" -> sqsMessage.getMessageId, + "requestId" -> sqsMessage.messageId(), ) handleMessageFromIngestBucket(sqsMessage)(logMarker) .recover { @@ -237,6 +240,7 @@ class ImageLoaderController(auth: Authentication, def getPreSignedUploadUrlsAndTrack: Action[AnyContent] = AuthenticatedAndAuthorised.async { request => val expiration = DateTimeUtils.now().plusHours(1) + val signatureDuration = Duration.ofHours(1) val mediaIdToFilenameMap = request.body.asJson.get.as[Map[String, String]] @@ -246,7 +250,7 @@ class ImageLoaderController(auth: Authentication, mediaIdToFilenameMap.map{case (mediaId, filename) => - val preSignedUrl = store.generatePreSignedUploadUrl(filename, expiration, uploadedBy, mediaId) + val preSignedUrl = store.generatePreSignedUploadUrl(filename, signatureDuration, uploadedBy, mediaId) uploadStatusTable.setStatus(UploadStatusRecord( id = mediaId, @@ -558,7 +562,15 @@ class ImageLoaderController(auth: Authentication, } } - lazy val replicaS3: AmazonS3 = S3Ops.buildS3Client(config, maybeRegionOverride = Some("us-west-1")) + lazy val replicaS3: S3Client = S3Ops.buildS3Client(config, maybeRegionOverride = Some(Region.US_WEST_1)) + def doesObjectExist(bucket: String, key: String) = { + try { + replicaS3.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + true + } catch { + case _: NoSuchKeyException => false + } + } private case class RestoreFromReplicaForm(imageId: String) def restoreFromReplica: Action[AnyContent] = AuthenticatedAndAuthorised.async { implicit request => @@ -581,20 +593,24 @@ class ImageLoaderController(auth: Authentication, Future.successful(Conflict("Image already exists in main bucket")) case None => Future.successful(NotImplemented("No replica bucket configured")) - case Some(replicaBucket) if replicaS3.doesObjectExist(replicaBucket, fileKeyFromId(imageId)) => + case Some(replicaBucket) if doesObjectExist(replicaBucket, fileKeyFromId(imageId)) => val s3Key = fileKeyFromId(imageId) logger.info(logMarker, s"Restoring image $imageId from replica bucket $replicaBucket (key: $s3Key)") - val replicaObject = replicaS3.getObject(replicaBucket, s3Key) - val metadata = S3FileExtractedMetadata(replicaObject.getObjectMetadata) - val stream = replicaObject.getObjectContent + val replicaObject = replicaS3.getObject( + GetObjectRequest.builder().bucket(replicaBucket).key(s3Key).build() + ) + val lastModified = replicaObject.response().lastModified() + val metaMap = replicaObject.response().metadata().asScala.toMap + val metadata = S3FileExtractedMetadata.apply(new DateTime(lastModified.toEpochMilli).withZone(DateTimeZone.UTC), metaMap) val tempFile = createTempFile(s"restoringReplica-$imageId") val fos = new FileOutputStream(tempFile) try { - IOUtils.copy(stream, fos) + replicaObject.transferTo(fos) } finally { - stream.close() + replicaObject.close() + fos.close() } val future = uploader.restoreFile( diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index fefb71177c7..3758c583ff9 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -1,14 +1,17 @@ package lib.storage -import com.amazonaws.HttpMethod -import com.amazonaws.services.s3.model.{AmazonS3Exception, GeneratePresignedUrlRequest, S3Object} +import software.amazon.awssdk.services.s3.model.{CopyObjectRequest, DeleteObjectRequest, GetObjectRequest, GetObjectResponse, PutObjectRequest, S3Exception} + +import java.time.Duration +import scala.jdk.CollectionConverters.MapHasAsJava +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.services.s3.presigner.S3Presigner +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest import lib.ImageLoaderConfig import com.gu.mediaservice.lib import com.gu.mediaservice.lib.logging.LogMarker import java.io.File -import java.time.ZonedDateTime -import java.util.Date class S3FileDoesNotExistException extends Exception() @@ -18,15 +21,16 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati try { doWork } catch { - case e: AmazonS3Exception if e.getStatusCode == 404 || e.getStatusCode == 403 => + case e: S3Exception if e.statusCode() == 404 || e.statusCode() == 403 => loggingIfNotFound throw new S3FileDoesNotExistException case other: Throwable => throw other } } - def getS3Object(key: String)(implicit logMarker: LogMarker): S3Object = handleNotFound(key) { - client.getObject(config.maybeIngestBucket.get, key) + def getS3Object(key: String)(implicit logMarker: LogMarker): ResponseInputStream[GetObjectResponse] = handleNotFound(key) { + client.getObject( + GetObjectRequest.builder().bucket(config.maybeIngestBucket.get).key(key).build()) } { logger.error(logMarker, s"Attempted to read $key from ingest bucket, but it does not exist.") } @@ -40,30 +44,40 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati meta = s3Meta, ) } + def generatePreSignedUploadUrl(filename: String, duration: Duration, uploadedBy: String, mediaId: String): String = { - def generatePreSignedUploadUrl(filename: String, expiration: ZonedDateTime, uploadedBy: String, mediaId: String): String = { - val request = new GeneratePresignedUrlRequest( - config.maybeBucketForUIUploads.get, // bucket - s"$uploadedBy/$filename", // key - ) - .withMethod(HttpMethod.PUT) - .withExpiration(Date.from(expiration.toInstant)); - - // sent by the client in manager.js - request.putCustomRequestHeader("x-amz-meta-media-id", mediaId) + val putObjectRequest = PutObjectRequest.builder() + .bucket(config.maybeBucketForUIUploads.get).key(s"$uploadedBy/$filename").metadata(Map( + "media-id" -> mediaId).asJava) + .build() + val putObjectPresignRequest = + PutObjectPresignRequest.builder() + .putObjectRequest(putObjectRequest) + .signatureDuration(duration) + .build(); - client.generatePresignedUrl(request).toString + val req = presigner.presignPutObject(putObjectPresignRequest) + req.url().toExternalForm } def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){ - client.copyObject(config.maybeIngestBucket.get, key, config.maybeFailBucket.get, key) + client.copyObject( + CopyObjectRequest.builder() + .sourceBucket(config.maybeIngestBucket.get) + .sourceKey(key) + .destinationBucket(config.maybeFailBucket.get) + .destinationKey(key) + .build() + ) deleteObjectFromIngestBucket(key) } { logger.warn(logMarker, s"Attempted to copy $key from ingest bucket to fail bucket, but it does not exist.") } def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key) { - client.deleteObject(config.maybeIngestBucket.get,key) + client.deleteObject( + DeleteObjectRequest.builder().bucket(config.maybeIngestBucket.get).key(key).build()) + () } { logger.warn(logMarker, s"Attempted to delete $key from ingest bucket, but it does not exist.") } diff --git a/image-loader/app/lib/UploadStatusTable.scala b/image-loader/app/lib/UploadStatusTable.scala index c545eb045a9..d193b0a3e83 100644 --- a/image-loader/app/lib/UploadStatusTable.scala +++ b/image-loader/app/lib/UploadStatusTable.scala @@ -12,7 +12,7 @@ import scala.concurrent.Future class UploadStatusTable(config: ImageLoaderConfig) { - val client = config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build() + val client = config.withAWSCredentials(DynamoDbAsyncClient.builder()).build() val scanamo = ScanamoAsync(client) private val uploadStatusTable = Table[UploadStatusRecord](config.uploadStatusTable) diff --git a/image-loader/app/lib/imaging/C2PADetector.scala b/image-loader/app/lib/imaging/C2PADetector.scala new file mode 100644 index 00000000000..85d43f2038f --- /dev/null +++ b/image-loader/app/lib/imaging/C2PADetector.scala @@ -0,0 +1,88 @@ +package lib.imaging + +import com.drew.imaging.jpeg.{JpegSegmentReader, JpegSegmentType} +import com.drew.imaging.png.{PngChunkReader, PngChunkType} +import com.drew.imaging.tiff.TiffMetadataReader +import com.drew.lang.StreamReader +import com.gu.mediaservice.model.{Jpeg, MimeType, Png, Tiff} + +import java.io.{BufferedInputStream, File, FileInputStream} +import java.util.Collections +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal + +/** + * Lightweight detector for embedded C2PA (Content Credentials) manifests, built entirely on + * top of the `metadata-extractor` library already used elsewhere in Grid (see FileMetadataReader). + * + * This does NOT parse or validate the manifest itself - `metadata-extractor` has no support for + * C2PA/JUMBF (ISO/IEC 19566-5). It only checks for the presence of the container structures C2PA + * uses to embed a manifest in JPEG (an APP11 marker segment), PNG (a "caBX" ancillary chunk) and + * TIFF (a private IFD tag), as defined in the C2PA Technical Specification, Appendix A.2 + * "Embedding manifests into non-BMFF-based assets": + * https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_non_bmff_based_assets + * + * The mime type is taken from the caller (already known/validated by Grid at upload time) rather + * than re-detected from the file's bytes. + */ +object C2paDetector { + + // An APP11 segment payload carrying a JUMBF box starts with an 8-byte APP11 prefix (CI/EN/Z), + // followed by the JUMBF box's own LBox+TBox (8 bytes) and its description box's LBox+TBox + // (8 bytes) - after which comes a 16-byte UUID identifying what the box's content actually is. + // That puts the UUID we care about at a fixed offset of 24 bytes into the payload. A C2PA + // manifest store's UUID begins with the ASCII mnemonic "c2pa" - checking for that (rather than + // just the generic "JP" CI prefix at offset 0) is what tells an actual C2PA manifest apart from + // some other, non-C2PA use of the same JPEG APP11/JUMBF embedding mechanism. This matches the + // check the reference C2PA Rust SDK itself performs (see C2PA_MARKER in + // https://github.com/contentauth/c2pa-rs/blob/main/sdk/src/asset_handlers/jpeg_io.rs). + // Spec reference - https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_jpeg + private val JumbfUuidOffset = 24 + private val C2paManifestStoreUuidMnemonic: Array[Byte] = "c2pa".getBytes("US-ASCII") + + // PNG stores JUMBF/C2PA data in an ancillary chunk named "caBX". + // Spec reference - https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_png + private val PngC2PAChunkType = new PngChunkType("caBX") + + // TIFF stores the raw JUMBF box directly (no "JP" identifier) as the data of a tag with ID + // 52545 decimal / 0xCD41 hex, tag type 7. + // Spec reference - https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_tiff_based_assets + private val TiffC2PATag = 0xCD41 + + def hasC2paManifest(file: File, mimeType: MimeType): Boolean = { + try { + mimeType match { + case Jpeg => hasJpegApp11Jumbf(file) + case Png => hasPngC2PAChunk(file) + case Tiff => hasTiffC2PATag(file) + } + } catch { + case NonFatal(_) => false + } + } + + private def hasJpegApp11Jumbf(file: File): Boolean = { + val segments = JpegSegmentReader.readSegments(file, Collections.singletonList(JpegSegmentType.APPB)) + segments.getSegments(JpegSegmentType.APPB).asScala.exists { payload => + val mnemonicLength = C2paManifestStoreUuidMnemonic.length + val mnemonicEndOffset = JumbfUuidOffset + mnemonicLength + val isLongEnoughToContainMnemonic = payload.length >= mnemonicEndOffset + val mnemonicInPayload = payload.slice(JumbfUuidOffset, mnemonicEndOffset) + + isLongEnoughToContainMnemonic && mnemonicInPayload.sameElements(C2paManifestStoreUuidMnemonic) + } + } + + private def hasPngC2PAChunk(file: File): Boolean = { + val in = new BufferedInputStream(new FileInputStream(file)) + try { + val chunks = new PngChunkReader().extract(new StreamReader(in), Collections.singleton(PngC2PAChunkType)) + chunks.asScala.nonEmpty + } finally in.close() + } + + private def hasTiffC2PATag(file: File): Boolean = { + val metadata = TiffMetadataReader.readMetadata(file) + metadata.getDirectories.asScala.exists(_.containsTag(TiffC2PATag)) + } +} diff --git a/image-loader/app/lib/imaging/FileMetadataReader.scala b/image-loader/app/lib/imaging/FileMetadataReader.scala index f81b9ce3a7f..777ecf8dad3 100644 --- a/image-loader/app/lib/imaging/FileMetadataReader.scala +++ b/image-loader/app/lib/imaging/FileMetadataReader.scala @@ -67,7 +67,10 @@ object FileMetadataReader extends GridLogging { metadata <- readMetadata(image) colourModelInformation <- getColorModelInformation(image, metadata, mimeType) } - yield getMetadataWithIPTCHeaders(metadata, imageId).copy(colourModelInformation = colourModelInformation) + yield getMetadataWithIPTCHeaders(metadata, imageId).copy( + colourModelInformation = colourModelInformation, + c2pa = if (C2paDetector.hasC2paManifest(image, mimeType)) FileMetadata.C2paAvailable else FileMetadata.NoC2PA + ) private def getMetadataWithIPTCHeaders(metadata: Metadata, imageId:String): FileMetadata = FileMetadata( diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 20b222382a1..41dcf25fa00 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -1,13 +1,14 @@ package model +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.services.s3.model.GetObjectResponse + import java.io.{File, FileOutputStream} -import com.amazonaws.services.s3.AmazonS3 import com.gu.mediaservice.{GridClient, ImageDataMerger} import com.gu.mediaservice.lib.auth.Authentication -import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object} import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Ops} +import com.gu.mediaservice.lib.aws.{Embedder, S3, S3Object} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} @@ -19,9 +20,8 @@ import model.upload.UploadRequest import org.apache.commons.io.IOUtils import org.joda.time.{DateTime, DateTimeZone} import play.api.libs.ws.WSRequest -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse -import java.nio.file.Path +import java.time.Instant import scala.jdk.CollectionConverters._ import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future} @@ -31,7 +31,7 @@ object Projector { import Uploader.toImageUploadOpsCfg def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder])(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbedder) + = new Projector(toImageUploadOpsCfg(config), new S3(config), imageOps, processor, auth, maybeEmbedder) } case class S3FileExtractedMetadata( @@ -42,12 +42,7 @@ case class S3FileExtractedMetadata( ) object S3FileExtractedMetadata { - def apply(s3ObjectMetadata: ObjectMetadata): S3FileExtractedMetadata = { - val lastModified = new DateTime(s3ObjectMetadata.getLastModified) - val userMetadata = s3ObjectMetadata.getUserMetadata.asScala.toMap - apply(lastModified, userMetadata) - } - + def apply(lastModified: DateTime, userMetadata: Map[String, String]): S3FileExtractedMetadata = { val fileUserMetadata = userMetadata.map { case (key, value) => // Fix up the contents of the metadata. @@ -81,7 +76,7 @@ object S3FileExtractedMetadata { } class Projector(config: ImageUploadOpsCfg, - s3: AmazonS3, + s3: S3, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, @@ -95,7 +90,7 @@ class Projector(config: ImageUploadOpsCfg, import ImageIngestOperations.fileKeyFromId val s3Key = fileKeyFromId(imageId) - if (!s3.doesObjectExist(config.originalFileBucket, s3Key)) + if (!s3.doesObjectExist(config.originalFileBucket, s3Key)) throw new NoSuchImageExistsInS3(config.originalFileBucket, s3Key) val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket}/$s3Key to perform Image projection"){ @@ -104,8 +99,8 @@ class Projector(config: ImageUploadOpsCfg, try { val digestedFile = getSrcFileDigestForProjection(s3Source, imageId, tempFile) - val extractedS3Meta = S3FileExtractedMetadata(s3Source.getObjectMetadata) - + val lastModifiedInstant = s3Source.response().lastModified() + val extractedS3Meta = S3FileExtractedMetadata(new DateTime(lastModifiedInstant.toEpochMilli).withZone(DateTimeZone.UTC), s3Source.response().metadata().asScala.toMap) val finalImageFuture = projectImage(digestedFile, extractedS3Meta, gridClient, onBehalfOfFn) val finalImage = Await.result(finalImageFuture, Duration.Inf) @@ -116,10 +111,10 @@ class Projector(config: ImageUploadOpsCfg, } } - private def getSrcFileDigestForProjection(s3Src: AwsS3Object, imageId: String, tempFile: File) = { + private def getSrcFileDigestForProjection(s3Src: ResponseInputStream[GetObjectResponse], imageId: String, tempFile: File) = { val fos = new FileOutputStream(tempFile) try { - IOUtils.copy(s3Src.getObjectContent, fos) + IOUtils.copy(s3Src, fos) DigestedFile(tempFile, imageId) } finally { fos.close() @@ -159,7 +154,7 @@ class Projector(config: ImageUploadOpsCfg, class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, - s3: AmazonS3, + s3: S3, maybeEmbedder: Option[Embedder], ) extends GridLogging { @@ -216,10 +211,10 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, logger.warn(logMarker, s"image did not exist in bucket $bucket at key $key") Future.successful(None) // falls back to creating from original file case true => - val obj = s3.getObject(new GetObjectRequest(bucket, key)) + val obj = s3.getObject(bucket, key) val fos = new FileOutputStream(outFile) try { - IOUtils.copy(obj.getObjectContent, fos) + IOUtils.copy(obj, fos) } finally { fos.close() obj.close() diff --git a/image-loader/app/model/S3IngestObject.scala b/image-loader/app/model/S3IngestObject.scala index 7621ac2067e..c6179d83635 100644 --- a/image-loader/app/model/S3IngestObject.scala +++ b/image-loader/app/model/S3IngestObject.scala @@ -4,6 +4,7 @@ import com.gu.mediaservice.lib.ImageStorageProps import com.gu.mediaservice.lib.logging.LogMarker import lib.storage.ImageLoaderStore +import java.util.Date import scala.jdk.CollectionConverters._ case class S3IngestObject ( @@ -20,21 +21,19 @@ case class S3IngestObject ( object S3IngestObject { def apply (key: String, store: ImageLoaderStore)(implicit logMarker: LogMarker): S3IngestObject = { - val keyParts = key.split("/") val s3Object = store.getS3Object(key) - val metadata = s3Object.getObjectMetadata - + val metadata = s3Object.response().metadata() S3IngestObject( key, uploadedBy = keyParts.head, filename = keyParts.last, - maybeMediaIdFromUiUpload = metadata.getUserMetadata.asScala.get("media-id"), // set by the client in upload in manager.js - uploadTime = metadata.getLastModified, - contentLength = metadata.getContentLength, - getInputStream = () => s3Object.getObjectContent, - identifiers = metadata.getUserMetadata.asScala.collect{ + maybeMediaIdFromUiUpload = metadata.asScala.toMap.get("media-id"), // set by the client in upload in manager.js + uploadTime = new Date(s3Object.response().lastModified().toEpochMilli), + contentLength = s3Object.response().contentLength(), + getInputStream = () => s3Object, + identifiers = metadata.asScala.collect{ case (key, value) if key.startsWith(ImageStorageProps.identifierMetadataKeyPrefix) => key.stripPrefix(ImageStorageProps.identifierMetadataKeyPrefix) -> value }.toMap diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index aaa811e52c6..6e6c7acc26b 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -9,7 +9,7 @@ import java.nio.file.{Files, Path} import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.auth.Authentication import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, S3Vectors, UpdateMessage} +import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -25,12 +25,13 @@ import lib.storage.ImageLoaderStore import model.Uploader.{fromUploadRequestShared, toImageUploadOpsCfg} import model.upload.{OptimiseOps, OptimiseWithPngQuant, UploadRequest} import org.joda.time.DateTime -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.Json import play.api.libs.ws.WSRequest -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse +import software.amazon.awssdk.services.s3.model.CopyObjectRequest import scala.collection.compat._ import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters.MapHasAsScala case class ImageUpload(uploadRequest: UploadRequest, image: Image) @@ -320,7 +321,8 @@ object Uploader extends GridLogging { } def patchUploadRequestWithS3Metadata(request: UploadRequest, s3Object: S3Object): UploadRequest = { - val metadata = S3FileExtractedMetadata(s3Object.metadata.objectMetadata.lastModified.getOrElse(new DateTime), s3Object.metadata.userMetadata) + val lastModified = s3Object.metadata.objectMetadata.lastModified + val metadata = S3FileExtractedMetadata(lastModified.getOrElse(new DateTime), s3Object.metadata.userMetadata) request.copy( uploadTime = metadata.uploadTime, uploadedBy = metadata.uploadedBy, @@ -471,8 +473,12 @@ class Uploader( logger.info(logMarker, s"Copying $mediaId to lower environment queue bucket $lowerEnvironmentQueueBucket") try { store.client.copyObject( - config.imageBucket, fileKeyFromId(mediaId), - lowerEnvironmentQueueBucket, s"${uploadRequest.uploadedBy}/${uploadRequest.uploadInfo.filename.getOrElse(mediaId)}" + CopyObjectRequest.builder() + .sourceBucket(config.imageBucket) + .sourceKey(fileKeyFromId(mediaId)) + .destinationBucket(lowerEnvironmentQueueBucket) + .destinationKey(s"${uploadRequest.uploadedBy}/${uploadRequest.uploadInfo.filename.getOrElse(mediaId)}") + .build() ) } catch { case e: Throwable => diff --git a/image-loader/conf/routes b/image-loader/conf/routes index a76b80b5939..83a4684e970 100644 --- a/image-loader/conf/routes +++ b/image-loader/conf/routes @@ -1,10 +1,8 @@ GET / controllers.ImageLoaderController.index POST /prepare controllers.ImageLoaderController.getPreSignedUploadUrlsAndTrack POST /images controllers.ImageLoaderController.loadImage(uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String]) -+nocsrf POST /enqueueDerivativeImage controllers.ImageLoaderController.enqueueDerivativeImage(derivativeOfMediaIds: String, filename: String, uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String]) POST /imports controllers.ImageLoaderController.importImage(uri: String, uploadedBy: Option[String], identifiers: Option[String], uploadTime: Option[String], filename: Option[String]) -+nocsrf POST /images/restore controllers.ImageLoaderController.restoreFromReplica GET /images/project/:imageId controllers.ImageLoaderController.projectImageBy(imageId: String) diff --git a/image-loader/test/resources/c2pa/c2pa-absent.jpg b/image-loader/test/resources/c2pa/c2pa-absent.jpg new file mode 100644 index 00000000000..e114f92fd44 Binary files /dev/null and b/image-loader/test/resources/c2pa/c2pa-absent.jpg differ diff --git a/image-loader/test/resources/c2pa/c2pa-absent.png b/image-loader/test/resources/c2pa/c2pa-absent.png new file mode 100644 index 00000000000..a125401519c Binary files /dev/null and b/image-loader/test/resources/c2pa/c2pa-absent.png differ diff --git a/image-loader/test/resources/c2pa/c2pa-absent.tiff b/image-loader/test/resources/c2pa/c2pa-absent.tiff new file mode 100644 index 00000000000..228f691e21d Binary files /dev/null and b/image-loader/test/resources/c2pa/c2pa-absent.tiff differ diff --git a/image-loader/test/resources/c2pa/c2pa-present.jpg b/image-loader/test/resources/c2pa/c2pa-present.jpg new file mode 100644 index 00000000000..0976984646a Binary files /dev/null and b/image-loader/test/resources/c2pa/c2pa-present.jpg differ diff --git a/image-loader/test/resources/c2pa/c2pa-present.png b/image-loader/test/resources/c2pa/c2pa-present.png new file mode 100644 index 00000000000..87c9a3ccc2f Binary files /dev/null and b/image-loader/test/resources/c2pa/c2pa-present.png differ diff --git a/image-loader/test/resources/c2pa/c2pa-present.tiff b/image-loader/test/resources/c2pa/c2pa-present.tiff new file mode 100644 index 00000000000..19a83366539 Binary files /dev/null and b/image-loader/test/resources/c2pa/c2pa-present.tiff differ diff --git a/image-loader/test/scala/lib/imaging/C2paDetectorTest.scala b/image-loader/test/scala/lib/imaging/C2paDetectorTest.scala new file mode 100644 index 00000000000..0fe7fe792f3 --- /dev/null +++ b/image-loader/test/scala/lib/imaging/C2paDetectorTest.scala @@ -0,0 +1,60 @@ +package test.lib.imaging + +import com.gu.mediaservice.model.{Jpeg, Png, Tiff} +import lib.imaging.C2paDetector +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +import java.io.File + +class C2paDetectorTest extends AnyFunSpec with Matchers { + + import test.lib.ResourceHelpers._ + + describe("hasC2paManifest") { + it("should detect a C2PA manifest in a signed JPEG") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-present.jpg"), Jpeg) shouldBe true + } + + it("should not detect a C2PA manifest in an unsigned JPEG") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-absent.jpg"), Jpeg) shouldBe false + } + + it("should not detect a C2PA manifest in a JPEG with no APP11/JUMBF segment at all") { + C2paDetector.hasC2paManifest(fileAt("getty.jpg"), Jpeg) shouldBe false + } + + it("should detect a C2PA manifest in a signed PNG") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-present.png"), Png) shouldBe true + } + + it("should not detect a C2PA manifest in an unsigned PNG") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-absent.png"), Png) shouldBe false + } + + it("should not detect a C2PA manifest in a PNG with no caBX chunk at all") { + C2paDetector.hasC2paManifest(fileAt("IndexedColor.png"), Png) shouldBe false + } + + it("should detect a C2PA manifest in a signed TIFF") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-present.tiff"), Tiff) shouldBe true + } + + it("should not detect a C2PA manifest in an unsigned TIFF") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-absent.tiff"), Tiff) shouldBe false + } + + it("should not detect a C2PA manifest in a TIFF with no private C2PA tag at all") { + C2paDetector.hasC2paManifest(fileAt("flower.tif"), Tiff) shouldBe false + } + + it("should gracefully return false, rather than throw, when the file doesn't match the given mime type") { + // a PNG parsed as though it were a JPEG - the underlying metadata-extractor call will fail + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-present.png"), Jpeg) shouldBe false + } + + it("should gracefully return false, rather than throw, when the file doesn't exist") { + C2paDetector.hasC2paManifest(new File("/no/such/file.jpg"), Jpeg) shouldBe false + } + } +} diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index eccec5b65f0..4de3767ef3f 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -3,10 +3,9 @@ package model import java.io.File import java.net.URI import java.util.{Date, UUID} -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ObjectMetadata import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} @@ -22,6 +21,7 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Span} import org.scalatestplus.mockito.MockitoSugar import play.api.libs.json.{JsArray, JsString} +import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import test.lib.ResourceHelpers @@ -44,7 +44,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val maybeEmbedder = None - private val s3 = mock[AmazonS3] + private val s3 = mock[S3] private val auth = mock[Authentication] private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbedder) @@ -234,16 +234,12 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc "S3FileExtractedMetadata" - { "should extract URL encoded metadata" in { - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( + val result = S3FileExtractedMetadata(new DateTime(1613388118000L), Map( "file-name" -> "This%20photo%20was%20taken%20in%20%C5%81%C3%B3d%C5%BA.jpg", "uploaded-by" -> "s%C3%A9b.cevey%40theguardian.co.uk", "upload-time" -> "2021-02-01T12%3A52%3A34%2B09%3A00", "identifier!picdarurn" -> "12*543%5E25" - ).asJava) - - val result = S3FileExtractedMetadata(s3Metadata) + )) result.uploadFileName shouldBe Some("This photo was taken in Łódź.jpg") result.uploadedBy shouldBe "séb.cevey@theguardian.co.uk" result.uploadTime.toString shouldBe "2021-02-01T03:52:34.000Z" @@ -252,16 +248,13 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc } "should remap headers with underscores to dashes" in { - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( + + val result = S3FileExtractedMetadata(new DateTime(1613388118000L), Map( "file_name" -> "filename.jpg", "uploaded_by" -> "user", "upload_time" -> "2021-02-01T12%3A52%3A34%2B09%3A00", "identifier!picdarurn" -> "12*543" - ).asJava) - - val result = S3FileExtractedMetadata(s3Metadata) + )) result.uploadFileName shouldBe Some("filename.jpg") result.uploadedBy shouldBe "user" result.uploadTime.toString shouldBe "2021-02-01T03:52:34.000Z" @@ -270,16 +263,10 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc } "should correctly read in non URL encoded values" in { - // we have plenty of values in S3 that are not URL encoded - // and we must be able to read them correctly - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( + val result = S3FileExtractedMetadata(new DateTime(1613388118000L), Map( "uploaded_by" -> "user", "upload_time" -> "2019-12-11T01:12:10.427Z", - ).asJava) - - val result = S3FileExtractedMetadata(s3Metadata) + )) result.uploadedBy shouldBe "user" result.uploadTime.toString shouldBe "2019-12-11T01:12:10.427Z" } diff --git a/kahuna/app/controllers/KahunaController.scala b/kahuna/app/controllers/KahunaController.scala index 7674168c42a..816350e42e6 100644 --- a/kahuna/app/controllers/KahunaController.scala +++ b/kahuna/app/controllers/KahunaController.scala @@ -3,7 +3,7 @@ package controllers import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.auth.{Authentication, Authorisation, BaseControllerWithLoginRedirects} -import lib.{EnableAISearch, ExampleSwitch, FeatureSwitches, KahunaConfig, UseCqlChips} +import lib.{ExampleSwitch, FeatureSwitches, KahunaConfig, UseCqlChips} import play.api.mvc.ControllerComponents import play.api.libs.json._ @@ -38,7 +38,7 @@ class KahunaController( val isIFramed = request.headers.get("Sec-Fetch-Dest").contains("iframe") val featureSwitches = new FeatureSwitches( - List(ExampleSwitch, UseCqlChips, EnableAISearch) + List(ExampleSwitch, UseCqlChips) ) val featureSwitchesWithClientValues = featureSwitches.getClientSwitchValues(featureSwitches.getFeatureSwitchCookies(request.cookies.get)) val featureSwitchesJson = Json.stringify(Json.toJson(featureSwitches.getFeatureSwitchesToStringify(featureSwitchesWithClientValues))) diff --git a/kahuna/app/lib/FeatureSwitch.scala b/kahuna/app/lib/FeatureSwitch.scala index 67e78114b86..c6c8ac2c60c 100644 --- a/kahuna/app/lib/FeatureSwitch.scala +++ b/kahuna/app/lib/FeatureSwitch.scala @@ -16,12 +16,6 @@ object UseCqlChips extends FeatureSwitch( default = true ) -object EnableAISearch extends FeatureSwitch( - key = "enable-ai-search", - title = "Enable the use of AI search", - default = false -) - class FeatureSwitches(featureSwitches: List[FeatureSwitch]){ // Feature switches are defined here, but updated by setting a cookie following the pattern e.g. "feature-switch-my-key" // for a switch called "my-key". diff --git a/kahuna/app/lib/KahunaConfig.scala b/kahuna/app/lib/KahunaConfig.scala index bbcdae6b002..1daa8623d4e 100644 --- a/kahuna/app/lib/KahunaConfig.scala +++ b/kahuna/app/lib/KahunaConfig.scala @@ -32,6 +32,7 @@ class KahunaConfig(resources: GridConfigResources) extends CommonConfig(resource val usageRightsHelpLink: Option[String]= stringOpt("links.usageRightsHelp").filterNot(_.isEmpty) val invalidSessionHelpLink: Option[String]= stringOpt("links.invalidSessionHelp").filterNot(_.isEmpty) val supportEmail: Option[String]= stringOpt("links.supportEmail").filterNot(_.isEmpty) + val uploadImagesPermissionSupportEmail: Option[String] = stringOpt("links.uploadImagesPermissionSupportEmail").filterNot(_.isEmpty) val telemetryUri: Option[String] = stringOpt("links.telemetryUri").filterNot(_.isEmpty) val homeLinkHtml: Option[String] = stringOpt("branding.homeLinkHtml").filterNot(_.isEmpty) diff --git a/kahuna/app/views/main.scala.html b/kahuna/app/views/main.scala.html index d5c7800221b..b09de2ed432 100644 --- a/kahuna/app/views/main.scala.html +++ b/kahuna/app/views/main.scala.html @@ -59,6 +59,7 @@ usageRightsHelpLink: "@Html(kahunaConfig.usageRightsHelpLink.getOrElse(""))", invalidSessionHelpLink: "@Html(kahunaConfig.invalidSessionHelpLink.getOrElse(""))", supportEmail: "@Html(kahunaConfig.supportEmail.getOrElse(""))", + uploadImagesPermissionSupportEmail: "@Html(kahunaConfig.uploadImagesPermissionSupportEmail.getOrElse(""))", staffPhotographerOrganisation: "@Html(kahunaConfig.staffPhotographerOrganisation)", fieldAliases: @Html(fieldAliases), homeLinkHtml: '@Html(kahunaConfig.homeLinkHtml.getOrElse(""))', @@ -86,6 +87,7 @@ featureSwitches: @Html(featureSwitches), telemetryUri: '@kahunaConfig.telemetryUri.getOrElse("")', usePermissionsFilter: @kahunaConfig.usePermissionsFilter.getOrElse(false), + aiSearchEnabled: @kahunaConfig.aiSearchEnabled, aiSearchResultLimit: @kahunaConfig.aiSearchResultLimit, usageRightsSummary: @kahunaConfig.usageRightsSummary.getOrElse(false), interimFilterOptions: @Html(interimFilterOptions), diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index f1487a33969..d5b54597ed6 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "@babel/polyfill": "7.8.7", - "@guardian/cql": "1.8.4", + "@guardian/cql": "1.8.6", "@guardian/user-telemetry-client": "1.1.0", "@sentry/browser": "6.11.0", "@sentry/integrations": "6.11.0", @@ -23,7 +23,7 @@ "any-promise-angular": "0.1.1", "cropperjs": "1.5.12", "filehash": "1.0.0", - "immutable": "3.8.3", + "immutable": "4.3.9", "javascript-detect-element-resize": "0.5.3", "jszip": "3.8.0", "moment": "2.29.4", @@ -47,7 +47,7 @@ "uuid": "9.0.0" }, "devDependencies": { - "@babel/core": "7.28.0", + "@babel/core": "7.29.6", "@babel/eslint-parser": "7.16.5", "@babel/preset-env": "7.28.0", "@jest/globals": "30.0.4", @@ -80,32 +80,6 @@ "webpack-merge": "5.8.0" } }, - "node_modules/@ampproject/remapping": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", - "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", - "dev": true, - "dependencies": { - "@jridgewell/gen-mapping": "^0.1.0", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@ampproject/remapping/node_modules/@jridgewell/gen-mapping": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", - "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", - "dev": true, - "dependencies": { - "@jridgewell/set-array": "^1.0.0", - "@jridgewell/sourcemap-codec": "^1.4.10" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@asamuzakjp/css-color": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", @@ -128,13 +102,13 @@ "license": "ISC" }, "node_modules/@babel/code-frame": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", - "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.27.1", + "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, @@ -143,9 +117,9 @@ } }, "node_modules/@babel/compat-data": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.0.tgz", - "integrity": "sha512-60X7qkglvrap8mn1lh2ebxXdZYtUcpd7gsmy9kLaBJ4i/WdY8PqTSdxyA8qraikqKQK5C1KRBKXqznrVapyNaw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", "dev": true, "license": "MIT", "engines": { @@ -153,22 +127,22 @@ } }, "node_modules/@babel/core": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.0.tgz", - "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.0", - "@babel/helper-compilation-targets": "^7.27.2", - "@babel/helper-module-transforms": "^7.27.3", - "@babel/helpers": "^7.27.6", - "@babel/parser": "^7.28.0", - "@babel/template": "^7.27.2", - "@babel/traverse": "^7.28.0", - "@babel/types": "^7.28.0", + "version": "7.29.6", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.6.tgz", + "integrity": "sha512-QdxmAo/ikZqqRGA8s43ww8lcql6naWRvEz0FFrl6MIlc7Gi6TroXnSdWa5U/kq6fzcpqpHesicQxFZIieZbyIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.6", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.29.2", + "@babel/parser": "^7.29.3", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", @@ -242,14 +216,14 @@ } }, "node_modules/@babel/generator": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.0.tgz", - "integrity": "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.28.0", - "@babel/types": "^7.28.0", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -272,14 +246,14 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", - "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.27.2", - "@babel/helper-validator-option": "^7.27.1", + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" @@ -376,9 +350,9 @@ } }, "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", "dev": true, "license": "MIT", "engines": { @@ -400,29 +374,29 @@ } }, "node_modules/@babel/helper-module-imports": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", - "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/traverse": "^7.27.1", - "@babel/types": "^7.27.1" + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.27.3", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.27.3.tgz", - "integrity": "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-imports": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1", - "@babel/traverse": "^7.27.3" + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -505,9 +479,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -515,9 +489,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", - "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", "engines": { @@ -525,9 +499,9 @@ } }, "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", "dev": true, "license": "MIT", "engines": { @@ -550,27 +524,27 @@ } }, "node_modules/@babel/helpers": { - "version": "7.27.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.6.tgz", - "integrity": "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.27.2", - "@babel/types": "^7.27.6" + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.0.tgz", - "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.0" + "@babel/types": "^7.29.7" }, "bin": { "parser": "bin/babel-parser.js" @@ -1925,33 +1899,33 @@ } }, "node_modules/@babel/template": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", - "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/parser": "^7.27.2", - "@babel/types": "^7.27.1" + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.0.tgz", - "integrity": "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.0", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.28.0", - "@babel/template": "^7.27.2", - "@babel/types": "^7.28.0", + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", "debug": "^4.3.1" }, "engines": { @@ -1959,14 +1933,14 @@ } }, "node_modules/@babel/types": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.0.tgz", - "integrity": "sha512-jYnje+JyZG5YThjHiF28oT4SIZLnYOcSBb6+SDaFIyzDVSkXQmQQYclJ2R+YxcdmK0AX6x1E5OQNtuh3jHDrUg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -2269,9 +2243,9 @@ "license": "MIT" }, "node_modules/@guardian/cql": { - "version": "1.8.4", - "resolved": "https://registry.npmjs.org/@guardian/cql/-/cql-1.8.4.tgz", - "integrity": "sha512-cDhPd63XcAs6Xj4Rerstjbo9uO0aJL8DEyrZ6FX2tmlpm0WY6vVzQ6aKmXpej9hw9wka8jHE02ZvoHY0yfhbdQ==", + "version": "1.8.6", + "resolved": "https://registry.npmjs.org/@guardian/cql/-/cql-1.8.6.tgz", + "integrity": "sha512-R8UoHaSiUVCOfgbFOCBxfPUQmUNoasd/Rj0o1KF7ikO0g+amsX3Dro2mabuzeuszMjAT4kMD809kuS+w+igvXg==", "dependencies": { "@floating-ui/dom": "^1.6.11", "preact": "^10.25.4", @@ -3276,6 +3250,17 @@ "@jridgewell/trace-mapping": "^0.3.24" } }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", @@ -3285,15 +3270,6 @@ "node": ">=6.0.0" } }, - "node_modules/@jridgewell/set-array": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", - "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@jridgewell/source-map": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", @@ -8742,13 +8718,10 @@ "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" }, "node_modules/immutable": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.3.tgz", - "integrity": "sha512-AUY/VyX0E5XlibOmWt10uabJzam1zlYjwiEgQSDc5+UIkFNaF9WM0JxXKaNMGf+F/ffUF+7kRKXM9A7C0xXqMg==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } + "version": "4.3.9", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.9.tgz", + "integrity": "sha512-ObHy4YN7ycwZOUCLI1/6svfyAFu7vL8RhAvVu/bh/RZW9EPlOyDaQ9jDQWCtdqzaXUjgXZCW1migtHE7YI7UGQ==", + "license": "MIT" }, "node_modules/import-fresh": { "version": "3.3.1", @@ -12055,9 +12028,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", "dev": true, "funding": [ { @@ -12065,6 +12038,7 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -13290,9 +13264,9 @@ } }, "node_modules/postcss": { - "version": "8.4.38", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz", - "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==", + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", "dev": true, "funding": [ { @@ -13308,10 +13282,11 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "dependencies": { - "nanoid": "^3.3.7", - "picocolors": "^1.0.0", - "source-map-js": "^1.2.0" + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" }, "engines": { "node": "^10 || ^12 || >=14" @@ -14432,10 +14407,11 @@ } }, "node_modules/source-map-js": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", - "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "dev": true, + "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" } diff --git a/kahuna/package.json b/kahuna/package.json index 3f1a820d562..8aa0376247f 100644 --- a/kahuna/package.json +++ b/kahuna/package.json @@ -1,7 +1,7 @@ { "dependencies": { "@babel/polyfill": "7.8.7", - "@guardian/cql": "1.8.4", + "@guardian/cql": "1.8.6", "@guardian/user-telemetry-client": "1.1.0", "@sentry/browser": "6.11.0", "@sentry/integrations": "6.11.0", @@ -18,7 +18,7 @@ "any-promise-angular": "0.1.1", "cropperjs": "1.5.12", "filehash": "1.0.0", - "immutable": "3.8.3", + "immutable": "4.3.9", "javascript-detect-element-resize": "0.5.3", "jszip": "3.8.0", "moment": "2.29.4", @@ -42,7 +42,7 @@ "uuid": "9.0.0" }, "devDependencies": { - "@babel/core": "7.28.0", + "@babel/core": "7.29.6", "@babel/eslint-parser": "7.16.5", "@babel/preset-env": "7.28.0", "@jest/globals": "30.0.4", diff --git a/kahuna/public/js/components/gr-archiver/gr-archiver.js b/kahuna/public/js/components/gr-archiver/gr-archiver.js index d2a5b4d28c4..dd6633b53d3 100644 --- a/kahuna/public/js/components/gr-archiver/gr-archiver.js +++ b/kahuna/public/js/components/gr-archiver/gr-archiver.js @@ -27,6 +27,7 @@ module.controller('grArchiverCtrl', [ 'humanJoin', 'mediaApi', 'getDeletedState', + 'apiPoll', function ($scope, $window, archiveService, @@ -34,7 +35,8 @@ module.controller('grArchiverCtrl', [ imageLogic, humanJoin, mediaApi, - getDeletedState) { + getDeletedState, + apiPoll) { const ctrl = this; @@ -115,14 +117,31 @@ module.controller('grArchiverCtrl', [ reduce((all, items) => all.union(items)). toArray(); } + + function pollUndeleted(imageId) { + const findImage = () => mediaApi.find(imageId).then( + (i) => i.data?.softDeletedMetadata === undefined && i.data.userMetadata?.data?.archived + ? Promise.resolve() + : Promise.reject() + ); + return apiPoll(findImage); + } function undelete() { ctrl.undeleting = true; const imageId = ctrl.image.data.id; mediaApi.undelete(imageId) - .then( - ctrl.canUndelete = ctrl.isDeleted = false - ).catch(() => { + .then(() => pollUndeleted(imageId)) + .then(() => { + ctrl.canUndelete = false; + ctrl.isDeleted = false; + ctrl.image.softDeletedMetadata = undefined; + if (ctrl.image.userMetadata?.data !== undefined) { + ctrl.image.userMetadata.data.archived = true; + } + $scope.$emit('images-updated', [ctrl.image]); + }).catch((e) => { $window.alert('Failed to undelete image!, please try again.'); + console.error('Image undeletion failed', e); }).finally(() => { ctrl.undeleting = false; }); diff --git a/kahuna/public/js/components/gr-collection-overlay/gr-collection-overlay.css b/kahuna/public/js/components/gr-collection-overlay/gr-collection-overlay.css index f57453b833f..adb36b6c7b7 100644 --- a/kahuna/public/js/components/gr-collection-overlay/gr-collection-overlay.css +++ b/kahuna/public/js/components/gr-collection-overlay/gr-collection-overlay.css @@ -5,7 +5,7 @@ position: fixed; top: 0; left: 0; - z-index: 40; + z-index: 200; } .collection-overlay__button:hover { diff --git a/kahuna/public/js/components/gr-collections-panel/gr-collections-panel.js b/kahuna/public/js/components/gr-collections-panel/gr-collections-panel.js index 9a50ca7595d..f40d61316a3 100644 --- a/kahuna/public/js/components/gr-collections-panel/gr-collections-panel.js +++ b/kahuna/public/js/components/gr-collections-panel/gr-collections-panel.js @@ -183,8 +183,7 @@ grCollectionsPanel.controller('GrNodeCtrl', grCollectionTreeCtrl.onSelect({$collection: ctrl.node.data.data.path}); }; - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; - + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; }; ctrl.searchWithModifiers = searchWithModifiers; diff --git a/kahuna/public/js/components/gr-confirm-delete/gr-confirm-delete.css b/kahuna/public/js/components/gr-confirm-delete/gr-confirm-delete.css index 0b3b47bda1d..8d84a9848a8 100644 --- a/kahuna/public/js/components/gr-confirm-delete/gr-confirm-delete.css +++ b/kahuna/public/js/components/gr-confirm-delete/gr-confirm-delete.css @@ -8,6 +8,8 @@ gr-confirm-delete.gr-delete-image { line-height: inherit; width: 100%; height: 100%; + padding-top: 10px; + padding-bottom: 10px; } .gr-confirm-delete:hover { diff --git a/kahuna/public/js/components/gr-display-crops/gr-display-crops.css b/kahuna/public/js/components/gr-display-crops/gr-display-crops.css index b392fac8b07..5b9ec9b8c12 100644 --- a/kahuna/public/js/components/gr-display-crops/gr-display-crops.css +++ b/kahuna/public/js/components/gr-display-crops/gr-display-crops.css @@ -12,3 +12,6 @@ justify-content: flex-start; } +.gr-display-crops + dd { + width: 100%; +} \ No newline at end of file diff --git a/kahuna/public/js/components/gr-display-crops/gr-display-crops.html b/kahuna/public/js/components/gr-display-crops/gr-display-crops.html index 80db07d7aab..6d58b199db0 100644 --- a/kahuna/public/js/components/gr-display-crops/gr-display-crops.html +++ b/kahuna/public/js/components/gr-display-crops/gr-display-crops.html @@ -1,10 +1,10 @@
-
+
diff --git a/kahuna/public/js/components/gr-downloader/gr-downloader.html b/kahuna/public/js/components/gr-downloader/gr-downloader.html index dd4a277554b..2974341eaf6 100644 --- a/kahuna/public/js/components/gr-downloader/gr-downloader.html +++ b/kahuna/public/js/components/gr-downloader/gr-downloader.html @@ -26,6 +26,7 @@ href="{{ ctrl.firstImageUris.uris.downloadUri }}" download rel="noopener" target="_blank" aria-label="Download image" class="side-padded" + ng-click="ctrl.onSingleDownloadClicked()" > Download diff --git a/kahuna/public/js/components/gr-downloader/gr-downloader.js b/kahuna/public/js/components/gr-downloader/gr-downloader.js index 95e9c560c88..54de2ed5374 100644 --- a/kahuna/public/js/components/gr-downloader/gr-downloader.js +++ b/kahuna/public/js/components/gr-downloader/gr-downloader.js @@ -3,18 +3,24 @@ import './gr-downloader.css'; import template from './gr-downloader.html'; import '../../services/image/downloads'; +import '../../services/api/media-api'; + export const downloader = angular.module('gr.downloader', [ - 'gr.image-downloads.service' + 'gr.image-downloads.service', + 'kahuna.services.api.media' ]); downloader.controller('DownloaderCtrl', [ '$window', '$q', + '$rootScope', '$scope', 'inject$', 'imageDownloadsService', + 'apiPoll', + 'mediaApi', - function Controller($window, $q, $scope, inject$, imageDownloadsService) { + function Controller($window, $q, $rootScope, $scope, inject$, imageDownloadsService, apiPoll, mediaApi) { let ctrl = this; @@ -60,6 +66,23 @@ downloader.controller('DownloaderCtrl', [ inject$($scope, uris$, ctrl, 'firstImageUris'); + function pollForDownloadUsageUpdate(image, downloadedAt) { + return mediaApi.getSession().then(session => { + return apiPoll(() => image.get().then(updatedImage => { + const hasNewUsageByCurrentUser = updatedImage.data.usages.data.some( + u => u.data.platform === 'download' && + u.data.downloadUsageMetadata && + u.data.downloadUsageMetadata.downloadedBy === session.user.email && + new Date(u.data.dateAdded) > downloadedAt + ); + if (!hasNewUsageByCurrentUser) { + return $q.reject(); + } + $rootScope.$emit('images-updated', [updatedImage]); + })); + }); + } + ctrl.download = (downloadKey) => { ctrl.downloading = true; @@ -94,6 +117,12 @@ downloader.controller('DownloaderCtrl', [ throw e; }); }; + + ctrl.onSingleDownloadClicked = () => { + if (window._clientConfig.recordDownloadAsUsage && ctrl.firstImageUris && ctrl.firstImageUris.uris.downloadUri) { + pollForDownloadUsageUpdate(ctrl.imagesArray()[0], new Date()).catch(() => {}); + } + }; }; }]); diff --git a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html index a2d891cc287..876edd9eb64 100644 --- a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html +++ b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html @@ -28,7 +28,7 @@
- {{ctrl.usageCategory || 'None'}} + {{ctrl.usageCategory || 'Unknown'}}
-
+
diff --git a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.js b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.js index 8edc2647be2..f8488c697e9 100644 --- a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.js +++ b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.js @@ -413,7 +413,7 @@ module.controller('grImageMetadataCtrl', [ return storage.getJs(generateStoreName(key)).hidden; }; - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; function isUsefulMetadata(metadataKey) { return ignoredMetadata.indexOf(metadataKey) === -1; diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage-list.html b/kahuna/public/js/components/gr-image-usage/gr-image-usage-list.html index aae5504999e..ddcd7527f86 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage-list.html +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage-list.html @@ -1,9 +1,13 @@
- + -
    +
    • local_library phonelink diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage.css b/kahuna/public/js/components/gr-image-usage/gr-image-usage.css index df64db796ec..15927d6a3d8 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage.css +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage.css @@ -7,6 +7,14 @@ gr-image-usage-list .gr-image-usage__field_title { color: #999; vertical-align: top; text-align: left; + display: inline-flex; + align-items: center; + gap: 4px; +} + +gr-image-usage-list .gr-image-usage__field_title > button { + display: inline-flex; + align-items: center; } gr-image-usage-list .date-added { @@ -18,8 +26,9 @@ gr-image-usage-list .gr-image-usage__record { width: 90%; } -gr-image-usage-list .source-list__icon svg { - padding: 0 2px; +gr-image-usage-list .usage-list__icon { + padding-right: 4px; + font-size:1.4em; } gr-image-usage-list gr-frontend-icon svg { @@ -46,7 +55,6 @@ gr-image-usage-list .reference-list__icon:hover svg path { gr-image-usage-list gr-icon { position: relative; - top: -2px; color: #999; } diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage.html b/kahuna/public/js/components/gr-image-usage/gr-image-usage.html index 626ef8e3bc6..6e6ce00e6cb 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage.html +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage.html @@ -1,7 +1,7 @@
      - +
      diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage.js b/kahuna/public/js/components/gr-image-usage/gr-image-usage.js index ff7b23b7789..ab5e0c8226e 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage.js +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage.js @@ -22,21 +22,40 @@ export const module = angular.module('gr.imageUsage', [ module.controller('grImageUsageCtrl', [ '$scope', + '$rootScope', '$state', 'inject$', '$window', 'imageUsagesService', - function ($scope, $state, inject$, $window, imageUsagesService) { + function ($scope, $rootScope, $state, inject$, $window, imageUsagesService) { const ctrl = this; + const statusOrder = ['pending', 'published', 'unknown', 'removed', 'downloaded']; + + function bindUsages(image) { + const usages = imageUsagesService.getUsages(image); + const usages$ = usages.groupedByState$ + .map((grouped) => grouped + .map(list => list.sortBy(usage => usage.get('dateAdded')).reverse()) + .sortBy((_, status) => { + const index = statusOrder.indexOf(status); + return index === -1 ? statusOrder.length : index; + }) + .toJS() + ); + inject$($scope, usages$.do(grouped => { + ctrl.hasMultipleStatusGroups = Object.keys(grouped).length > 1; + }), ctrl, 'usages'); + inject$($scope, usages.count$, ctrl, 'usagesCount'); + inject$($scope, usages.hasSyndicationUsages$, ctrl, 'hasSyndicationUsages'); + } + ctrl.$onInit = () => { ctrl.showSendToPhotoSales = $window._clientConfig.showSendToPhotoSales; - const usages = imageUsagesService.getUsages(ctrl.image); - const usages$ = usages.groupedByState$.map((u) => u.toJS()); - const usagesCount$ = usages.count$; + bindUsages(ctrl.image); // TODO match on `platform` rather than `type` as `platform` includes more detail ctrl.usageTypeToName = (usageType) => { @@ -72,17 +91,18 @@ module.controller('grImageUsageCtrl', [ }; ctrl.onUsagesDeleted = () => { - // a bit nasty - but it updates the state of the page better than trying to do that in - // the client. $state.go('image', {imageId: ctrl.image.data.id, crop: undefined}, {reload: true}); }; - const hasSyndicationUsages$ = - imageUsagesService.getUsages(ctrl.image).hasSyndicationUsages$; + const freeImagesUpdateListener = $rootScope.$on('images-updated', (e, updatedImages) => { + const maybeUpdatedImage = updatedImages.find(u => u.data.id === ctrl.image.data.id); + if (maybeUpdatedImage) { + ctrl.image = maybeUpdatedImage; + bindUsages(ctrl.image); + } + }); - inject$($scope, usages$, ctrl, 'usages'); - inject$($scope, usagesCount$, ctrl, 'usagesCount'); - inject$($scope, hasSyndicationUsages$, ctrl, 'hasSyndicationUsages'); + $scope.$on('$destroy', freeImagesUpdateListener); }; }]); @@ -104,6 +124,16 @@ module.controller('grImageUsageListCtrl', [ function (imageUsagesService) { const ctrl = this; + const collapseThreshold = 10; + + ctrl.$onInit = () => { + ctrl.isCollapsed = ctrl.usages.length > collapseThreshold && ctrl.hasMultipleStatusGroups; + }; + + ctrl.toggle = () => { + ctrl.isCollapsed = !ctrl.isCollapsed; + }; + ctrl.formatTimestamp = (timestamp) => { return moment(timestamp).fromNow(); }; @@ -125,7 +155,8 @@ module.directive('grImageUsageList', [function () { bindToController: true, scope: { type: '=', - usages: '=' + usages: '=', + hasMultipleStatusGroups: '<' } }; }]); diff --git a/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.js b/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.js index f61c125f893..b6f24de0550 100644 --- a/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.js +++ b/kahuna/public/js/components/gr-more-like-this/gr-more-like-this.js @@ -1,7 +1,6 @@ import angular from 'angular'; import './gr-more-like-this.css'; import template from './gr-more-like-this.html'; -import {getFeatureSwitchActive} from '../gr-feature-switch-panel/gr-feature-switch-panel'; export const moreLikeThis = angular.module('gr.moreLikeThis', []); @@ -13,7 +12,7 @@ moreLikeThis.controller('MoreLikeThisCtrl', [ let ctrl = this; ctrl.$onInit = () => { - ctrl.showMoreLikeThis = getFeatureSwitchActive('enable-ai-search'); + ctrl.showMoreLikeThis = window._clientConfig.aiSearchEnabled; ctrl.getMoreLikeThisQuery = function() { return `similar:${ctrl.image.data.id}`; }; diff --git a/kahuna/public/js/components/gr-my-uploads/gr-my-uploads.tsx b/kahuna/public/js/components/gr-my-uploads/gr-my-uploads.tsx index ceb54ca44b8..2eb74d27bcc 100644 --- a/kahuna/public/js/components/gr-my-uploads/gr-my-uploads.tsx +++ b/kahuna/public/js/components/gr-my-uploads/gr-my-uploads.tsx @@ -17,10 +17,6 @@ export interface MyUploadsWrapperProps { props: MyUploadsProps; } -//-- logo click event -- -interface LogoClickEventDetail { showPaid: boolean } -interface LogoClickEvent extends CustomEvent {optional?: string} - //-- filter change event -- interface Filter { uploadedByMe: boolean } interface FilterChangeEventDetail { filter: Filter } @@ -38,6 +34,10 @@ const MyUploads: React.FC = ({ props }) => { const [myUploads, setMyUploads] = useState(props.myUploads); + useEffect(() => { + setMyUploads(prev => prev === props.myUploads ? prev : props.myUploads); + }, [props.myUploads]); + const handleCheckboxClick = () => { setMyUploads(prevChkd => { props.onChange(!prevChkd); @@ -61,17 +61,18 @@ const MyUploads: React.FC = ({ props }) => { } }; - const handleLogoClick = (event: LogoClickEvent) => { - setMyUploads(false); + const handleLogoClick = () => { + setMyUploads(prev => prev ? false : prev); }; const handleFilterChange = (event: FilterChangeEvent) => { - setMyUploads(event.detail.filter.uploadedByMe); + const next = event.detail.filter.uploadedByMe; + setMyUploads(prev => prev === next ? prev : next); }; const handleUploadedBy = (event: UploadedByEvent) => { const matches: boolean = (event.detail.userEmail === event.detail.uploadedBy); - setMyUploads(matches); + setMyUploads(prev => prev === matches ? prev : matches); }; useEffect(() => { diff --git a/kahuna/public/js/components/gr-permissions-filter/gr-permissions-filter.tsx b/kahuna/public/js/components/gr-permissions-filter/gr-permissions-filter.tsx index 1db69875ff1..170750c8a9a 100644 --- a/kahuna/public/js/components/gr-permissions-filter/gr-permissions-filter.tsx +++ b/kahuna/public/js/components/gr-permissions-filter/gr-permissions-filter.tsx @@ -106,8 +106,8 @@ const PermissionsFilter: React.FC = ({ props }) => { setIsChargeable(event.detail.showPaid); }; - const handleQueryChange = (event: QueryChangeEvent) => { - const newQuery = event.detail.query ? (" " + event.detail.query) : ""; + const syncSelectionFromQuery = (rawQuery: string | undefined) => { + const newQuery = rawQuery ? (" " + rawQuery) : ""; if (propsRef.current.query !== newQuery) { propsRef.current.query = newQuery; @@ -127,6 +127,10 @@ const PermissionsFilter: React.FC = ({ props }) => { } }; + const handleQueryChange = (event: QueryChangeEvent) => { + syncSelectionFromQuery(event.detail.query); + }; + useEffect(() => { window.addEventListener('queryChangeEvent', handleQueryChange); window.addEventListener('logoClick', handleLogoClick); @@ -134,7 +138,6 @@ const PermissionsFilter: React.FC = ({ props }) => { window.addEventListener('mouseup', autoHideListener); window.addEventListener('scroll', autoHideListener); window.addEventListener('keydown', autoHideListener); - setSelection(defPerms); // Clean up the event listener when the component unmounts return () => { @@ -148,6 +151,17 @@ const PermissionsFilter: React.FC = ({ props }) => { }; }, []); + useEffect(() => { + syncSelectionFromQuery(props.query); + }, [props.query]); + + useEffect(() => { + if (propsRef.current.chargeable !== props.chargeable) { + propsRef.current.chargeable = props.chargeable; + setIsChargeable(props.chargeable); + } + }, [props.chargeable]); + const handleOptionClick = (option: PermissionsDropdownOption) => { const payableDef = payableDefaults.filter(pd => pd.opt === option.value)[0]; if (payableDef.payable === 'false' || payableDef.payable === 'true') { @@ -162,6 +176,7 @@ const PermissionsFilter: React.FC = ({ props }) => { }; useEffect(() => { + propsRef.current.chargeable = isChargeable; props.onChargeable(isChargeable); }, [isChargeable]); diff --git a/kahuna/public/js/components/gr-photoshoot/gr-photoshoot.js b/kahuna/public/js/components/gr-photoshoot/gr-photoshoot.js index b5ba6c7dca1..a4f5a954f92 100644 --- a/kahuna/public/js/components/gr-photoshoot/gr-photoshoot.js +++ b/kahuna/public/js/components/gr-photoshoot/gr-photoshoot.js @@ -77,7 +77,7 @@ photoshoot.controller('GrPhotoshootCtrl', [ return photoshootService.batchRemove({ images: ctrl.images }); }; - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; if (Boolean(ctrl.withBatch)) { const batchApplyEvent = 'events:batch-apply:photoshoot'; diff --git a/kahuna/public/js/components/gr-preset-labels/gr-preset-labels.js b/kahuna/public/js/components/gr-preset-labels/gr-preset-labels.js index 5294f160c6a..d05a1e77310 100644 --- a/kahuna/public/js/components/gr-preset-labels/gr-preset-labels.js +++ b/kahuna/public/js/components/gr-preset-labels/gr-preset-labels.js @@ -62,7 +62,7 @@ presetLabels.controller('GrPresetLabelsCtrl', [ ctrl.newLabel = ''; } - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; } ]); diff --git a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx index 8881a026351..b68920c665a 100644 --- a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx @@ -5,7 +5,6 @@ import { DefaultSortOption, CollectionSortOption } from "./gr-sort-control-confi import "./gr-sort-control.css"; const SELECT_OPTION = "Select an option"; -const DEFAULT_OPTION = DefaultSortOption.value; const COLLECTION_OPTION = CollectionSortOption.value; const CONTROL_TITLE = "Sort by:"; const SORT_ORDER = "Sort order"; @@ -13,7 +12,7 @@ const PANEL_IDENTIFIER = "info"; const SCROLL_IDENTIFIER = "scroll"; const downArrowIcon = () => - + ; @@ -66,13 +65,13 @@ const hasClassInSelfOrParent = (node: Element | null, className: string): boolea }; export const BaseSortControl: React.FC = ({ - options, - startSelectedOption, - onSelect, - startHasCollection, - panelVisible, - isSimple - }) => { + options, + startSelectedOption, + onSelect, + startHasCollection, + panelVisible, + isSimple + }) => { const hasCollection = startHasCollection; const startSort:SortDropdownOption = startSelectedOption ? startSelectedOption : DefaultSortOption; @@ -82,11 +81,23 @@ export const BaseSortControl: React.FC = ({ const [currentIndex, setCurrentIndex] = useState(-1); const [isPanelVisible, setPanelVisible] = useState(panelVisible); + useEffect(() => { + const nextSort = startSelectedOption ? startSelectedOption : DefaultSortOption; + setSelection(nextSort); + if (!nextSort.isCollection) { + setPrevious(nextSort); + } + }, [startSelectedOption]); + + useEffect(() => { + setPanelVisible(panelVisible); + }, [panelVisible]); + const handleArrowKeys = (event:KeyboardEvent) => { if (event.key === 'ArrowDown' || - event.key === 'ArrowUp' || - event.key === 'Enter' || - event.code === 'Space') { + event.key === 'ArrowUp' || + event.key === 'Enter' || + event.code === 'Space') { event.preventDefault(); event.stopPropagation(); let rowCount = options.length; @@ -126,16 +137,16 @@ export const BaseSortControl: React.FC = ({ const handlePanelShow = (event: any) => { const panel = event.detail.panel; - if (panel === PANEL_IDENTIFIER) { - setPanelVisible(true); - } + if (panel === PANEL_IDENTIFIER) { + setPanelVisible(true); + } }; const handlePanelHide = (event: any) => { const panel = event.detail.panel; - if (panel === PANEL_IDENTIFIER || panel === SCROLL_IDENTIFIER) { - setPanelVisible(false); - } + if (panel === PANEL_IDENTIFIER || panel === SCROLL_IDENTIFIER) { + setPanelVisible(false); + } }; window.addEventListener("mouseup", autoHideListener); diff --git a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx index 95e3115d283..92a25e457cc 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx @@ -24,25 +24,44 @@ export interface ExtendedSortWrapperProps { const checkForCollection = (query:string): boolean => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); +const deriveExtendedSortState = ( + query: string, + orderBy: string, + sortOptions: SortDropdownOption[], + noTakenDateClause: string +) => { + let selectedSort = DefaultSortOption; + if (!query.includes(noTakenDateClause)) { + selectedSort = sortOptions.find(o => o.value === orderBy) || DefaultSortOption; + } + return { + selectedSort, + hasCollection: checkForCollection(query) + }; +}; + const ExtendedSortControl: React.FC = ({ props }) => { const noTakenDateClause = "-has:dateTaken"; const takenDateClause = "has:dateTaken"; const sortOptions = SortOptions; - const orderBy = props.orderBy; - const query = props.query; - - let startSortOption = DefaultSortOption; - if (!query.includes(noTakenDateClause) && (sortOptions.filter(o => o.value === orderBy)).length > 0) { - startSortOption = sortOptions.find(o => o.value === orderBy); - } - - const startHasCollection = checkForCollection(query); + const orderBy = props.orderBy || ""; + const query = props.query || ""; + const { selectedSort: startSortOption, hasCollection: startHasCollection } = + deriveExtendedSortState(query, orderBy, sortOptions, noTakenDateClause); const [selSortOption, setSortOption] = useState(startSortOption); const [userTakenSelect, setUserTakenSelect] = useState(props.userTakenSelect); const noTakenDateCount = props.noTakenDateCount; const [hasCollection, setHasCollection] = useState(startHasCollection); + useEffect(() => { + const { selectedSort, hasCollection: nextHasCollection } = + deriveExtendedSortState(query, orderBy, sortOptions, noTakenDateClause); + setSortOption(selectedSort); + setHasCollection(nextHasCollection); + setUserTakenSelect(Boolean(props.userTakenSelect)); + }, [query, orderBy, props.userTakenSelect]); + const onSortSelect = (selOption: SortDropdownOption) => { setSortOption(selOption); setUserTakenSelect(selOption.isTaken); @@ -67,11 +86,11 @@ const ExtendedSortControl: React.FC = ({ props }) => { }; const handleQueryChange = (e: any) => { - const newQuery = e.detail.query ? (" " + e.detail.query) : ""; - setHasCollection(checkForCollection(newQuery)); - if (userTakenSelect && !newQuery.includes(takenDateClause)) { - props.onSortSelect(DefaultSortOption, 'with', false); - } + const newQuery = e.detail.query ? (" " + e.detail.query) : ""; + setHasCollection(checkForCollection(newQuery)); + if (userTakenSelect && !newQuery.includes(takenDateClause)) { + props.onSortSelect(DefaultSortOption, 'with', false); + } }; window.addEventListener("logoClick", handleLogoClick); @@ -87,24 +106,23 @@ const ExtendedSortControl: React.FC = ({ props }) => { return (
      - - + +
      ); }; export const extendedSortControl = angular.module('gr.extendedSortControl', []) .component('extendedSortControl', react2angular(ExtendedSortControl, ["props"])); - diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css index 78b23df5a28..f7eab4465da 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css @@ -149,7 +149,7 @@ } } -@media screen and (max-width: 1380px) { +@media screen and (max-width: 1400px) { .sort-selection-title { display: none; } diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 4bef8d6132d..82c1ae71675 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -18,27 +18,36 @@ export interface SortWrapperProps { const checkForCollection = (query: string): boolean => /~"[a-zA-Z0-9 #\-_.:/]+"/.test(query); +const deriveSortState = (query: string, orderBy: string, sortOptions: SortDropdownOption[]) => { + const hasCollection = checkForCollection(query); + const matchedByOrderBy = sortOptions.find(o => o.value === orderBy); + + if (hasCollection && !matchedByOrderBy) { + // no explicit sort chosen yet for this collection query - use the collection default + const collectionSort = sortOptions.find(o => o.isCollection) || DefaultSortOption; + return { hasCollection, selectedSort: collectionSort }; + } + + return { hasCollection, selectedSort: matchedByOrderBy || DefaultSortOption }; +}; + const SortControl: React.FC = ({ props }) => { const sortOptions = SortOptions; - const orderBy = props.orderBy; - const query = props.query; - const startHasCollection = checkForCollection(query); - - let startSortOption = DefaultSortOption; - if (startHasCollection) { - if ((sortOptions.filter(o => o.isCollection)).length > 0) { - startSortOption = sortOptions.find(o => o.isCollection); - } - } else { - if ((sortOptions.filter(o => o.value === orderBy)).length > 0) { - startSortOption = sortOptions.find(o => o.value === orderBy); - } - } + const orderBy = props.orderBy || ""; + const query = props.query || ""; + const { hasCollection: startHasCollection, selectedSort: startSortOption } = + deriveSortState(query, orderBy, sortOptions); const [selSortOption, setSortOption] = useState(startSortOption); const [hasCollection, setHasCollection] = useState(startHasCollection); + useEffect(() => { + const { hasCollection: nextHasCollection, selectedSort } = deriveSortState(query, orderBy, sortOptions); + setHasCollection(nextHasCollection); + setSortOption(selectedSort); + }, [query, orderBy]); + const onSortSelect = (selOption: SortDropdownOption) => { setSortOption(selOption); props.onSortSelect(selOption); diff --git a/kahuna/public/js/components/gr-top-bar/gr-top-bar.css b/kahuna/public/js/components/gr-top-bar/gr-top-bar.css index 783f42ae2e0..404ac9e2c1c 100644 --- a/kahuna/public/js/components/gr-top-bar/gr-top-bar.css +++ b/kahuna/public/js/components/gr-top-bar/gr-top-bar.css @@ -16,9 +16,9 @@ gr-top-bar-nav { gr-top-bar-actions { display: flex; + flex-shrink: 0; flex-direction: row; justify-content: flex-end; - min-width: 278px; } diff --git a/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js b/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js index aeff5022cfe..d9f326f0d4c 100644 --- a/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js +++ b/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js @@ -7,27 +7,27 @@ export const undeleteImage = angular.module('gr.undeleteImage', [ ]); undeleteImage.controller('grUnDeleteImageCtrl', [ - '$rootScope', '$q', '$timeout', 'mediaApi', 'apiPoll', - function ($rootScope, $q, $timeout, mediaApi, apiPoll) { + '$rootScope', '$q', 'mediaApi', 'apiPoll', + function ($rootScope, $q, mediaApi, apiPoll) { var ctrl = this; ctrl.$onInit = () => { - function pollDeleted (image) { + function pollUndeleted (image) { const findImage = () => mediaApi.find(image.data.id).then( - () => $q.reject(), - // resolve when image cannot be found, i.e. image has been deleted. - () => $q.resolve() + (i) => i.data.softDeletedMetadata === undefined && i.data.userMetadata?.data?.archived + ? $q.resolve() + : $q.reject() ); - apiPoll(findImage); + return apiPoll(findImage); } ctrl.unDeleteImage = function (image) { return mediaApi.undelete(image.data.id) - .then(() => pollDeleted(image)) + .then(() => pollUndeleted(image)) .catch((err) => { - $rootScope.$emit('image-delete-failure', err, image); + $rootScope.$emit('image-undelete-failure', err, image); }); }; @@ -35,7 +35,7 @@ undeleteImage.controller('grUnDeleteImageCtrl', [ // HACK to wait for thrall to process the message so that when we // poll the api, it will be up to date. return $q.all(Array.from(ctrl.images.values()).map(image => ctrl.unDeleteImage(image))) - .then(() => $rootScope.$emit('images-deleted', ctrl.images)); + .then(() => $rootScope.$emit('images-undeleted', ctrl.images)); }; }; } diff --git a/kahuna/public/js/components/gr-usagerights-summary/gr-usagerights-summary.tsx b/kahuna/public/js/components/gr-usagerights-summary/gr-usagerights-summary.tsx index f1a36b2e0e4..1f47c05652d 100644 --- a/kahuna/public/js/components/gr-usagerights-summary/gr-usagerights-summary.tsx +++ b/kahuna/public/js/components/gr-usagerights-summary/gr-usagerights-summary.tsx @@ -107,7 +107,7 @@ const UsageRightsSummary: React.FC = ({ props }) => { return (
      -
      { category || 'None'}
      +
      { category || 'Unknown'}
      diff --git a/kahuna/public/js/components/gu-date-range/gu-date-range.css b/kahuna/public/js/components/gu-date-range/gu-date-range.css index 6ad383d8949..c106bb3a388 100644 --- a/kahuna/public/js/components/gu-date-range/gu-date-range.css +++ b/kahuna/public/js/components/gu-date-range/gu-date-range.css @@ -18,7 +18,7 @@ .gu-date-range__display__icon__toggle { display: inline-block; - margin-left: 5px; + margin-left: 3px; } .gu-date-range__icon { diff --git a/kahuna/public/js/components/gu-date-range/gu-date-range.js b/kahuna/public/js/components/gu-date-range/gu-date-range.js index 6f73a1986db..c043025dc3b 100644 --- a/kahuna/public/js/components/gu-date-range/gu-date-range.js +++ b/kahuna/public/js/components/gu-date-range/gu-date-range.js @@ -122,7 +122,12 @@ guDateRange.directive('guDateRange', [function () { pikaStart.show(); }); - $scope.$watch('ctrl.guEndDate', resetView); + // Watch both start and end dates: many presets/filters only set + // one of the two (e.g. 'since' with 'until' left undefined), so + // watching guEndDate alone can miss changes to guStartDate and + // leave the display summary out of sync with the actual filter + // (e.g. after using the logo click or browser back button). + $scope.$watchGroup(['ctrl.guStartDate', 'ctrl.guEndDate'], resetView); $scope.$on('$destroy', function() { pikaStart.destroy(); diff --git a/kahuna/public/js/crop/controller.js b/kahuna/public/js/crop/controller.js index 370278f599a..8f43516385b 100644 --- a/kahuna/public/js/crop/controller.js +++ b/kahuna/public/js/crop/controller.js @@ -5,12 +5,14 @@ import {radioList} from '../components/gr-radio-list/gr-radio-list'; import {cropUtil} from "../util/crop"; import {cropOptions} from "../util/constants/cropOptions"; import {storage as storageModule} from "../util/storage"; +import '../imgops/service'; const crop = angular.module('kahuna.crop.controller', [ 'gr.keyboardShortcut', radioList.name, cropUtil.name, - storageModule.name + storageModule.name, + 'kahuna.imgops' ]); crop.controller('ImageCropCtrl', [ @@ -21,7 +23,7 @@ crop.controller('ImageCropCtrl', [ 'mediaApi', 'mediaCropper', 'image', - 'optimisedImageUri', + 'imgops', 'keyboardShortcut', 'defaultCrop', 'cropSettings', @@ -37,7 +39,7 @@ crop.controller('ImageCropCtrl', [ mediaApi, mediaCropper, image, - optimisedImageUri, + imgops, keyboardShortcut, defaultCrop, cropSettings, @@ -50,6 +52,7 @@ crop.controller('ImageCropCtrl', [ const imageId = $stateParams.imageId; const circularMaskKey = 'crop.shouldUseCircularMask'; + ctrl.shouldUseCircularMask = true; try { const stored = storage.getJs(circularMaskKey); if (typeof stored === 'boolean') { @@ -83,7 +86,16 @@ crop.controller('ImageCropCtrl', [ ctrl.cropType = storageCropType || storageDefaultCropType || defaultCrop.key; ctrl.image = image; - ctrl.optimisedImageUri = optimisedImageUri; + ctrl.optimisedImageUri = null; + ctrl.imageLoading = true; + + imgops.getFullScreenUri(image).then(uri => { + ctrl.optimisedImageUri = uri; + }); + + ctrl.onCropperReady = () => { + ctrl.imageLoading = false; + }; ctrl.cropping = false; diff --git a/kahuna/public/js/crop/index.js b/kahuna/public/js/crop/index.js index ac11db7c785..216e094a1f9 100644 --- a/kahuna/public/js/crop/index.js +++ b/kahuna/public/js/crop/index.js @@ -44,9 +44,7 @@ crop.config(['$stateProvider', return $q.reject(error); } }); - }], - optimisedImageUri: ['image', 'imgops', - (image, imgops) => imgops.getFullScreenUri(image)] + }] } }); diff --git a/kahuna/public/js/crop/view.html b/kahuna/public/js/crop/view.html index 89aaa3e6775..ec131fac5e4 100644 --- a/kahuna/public/js/crop/view.html +++ b/kahuna/public/js/crop/view.html @@ -125,6 +125,9 @@ 'circular-mask': ctrl.shouldShowCircularGuideline && ctrl.shouldUseCircularMask }">
      +
      + Loading image… +
      scope.onReady()); } function update(c) { diff --git a/kahuna/public/js/edits/image-editor.html b/kahuna/public/js/edits/image-editor.html index 45658896e27..55cbe453030 100644 --- a/kahuna/public/js/edits/image-editor.html +++ b/kahuna/public/js/edits/image-editor.html @@ -147,7 +147,7 @@

      Image Rights

      - {{ctrl.usageRightsCategory || 'None'}} + {{ctrl.usageRightsCategory || 'Unknown'}} @@ -45,6 +46,7 @@ when the option is off --> {{ searchQuery.costFilterLabel }} @@ -64,7 +66,8 @@
      • @@ -72,7 +75,8 @@
      • diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index db7facc3a08..9bb85973c95 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -26,20 +26,22 @@ import { HAS_DATE_TAKEN, TAKEN_SORT } from "../components/gr-sort-control/gr-sort-control-config"; -import {getFeatureSwitchActive} from "../components/gr-feature-switch-panel/gr-feature-switch-panel"; + +const toNonFreeString = (val) => (val === true || val === 'true') ? 'true' : 'false'; +const isNonFreeString = (val) => val === 'true'; export var query = angular.module('kahuna.search.query', [ - // Note: temporarily disabled for performance reasons, see above - // 'ngAnimate', - eq.name, - guDateRange.name, - syntax.name, - grStructuredQuery.name, - 'util.storage', - 'gr.sortControl', - 'gr.extendedSortControl', - 'gr.permissionsFilter', - 'gr.myUploads' + // Note: temporarily disabled for performance reasons, see above + // 'ngAnimate', + eq.name, + guDateRange.name, + syntax.name, + grStructuredQuery.name, + 'util.storage', + 'gr.sortControl', + 'gr.extendedSortControl', + 'gr.permissionsFilter', + 'gr.myUploads' ]); query.controller('SearchQueryCtrl', [ @@ -47,10 +49,11 @@ query.controller('SearchQueryCtrl', [ '$scope', '$state', '$stateParams', + '$timeout', 'onValChange', 'storage', 'mediaApi', - function($rootScope, $scope, $state, $stateParams, onValChange, storage, mediaApi) { + function($rootScope, $scope, $state, $stateParams, $timeout, onValChange, storage, mediaApi) { const ctrl = this; ctrl.costFilterLabel = window._clientConfig.costFilterLabel; @@ -60,26 +63,29 @@ query.controller('SearchQueryCtrl', [ ctrl.maybeOrgOwnedValue = window._clientConfig.maybeOrgOwnedValue; ctrl.canUpload = false; mediaApi.canUserUpload().then(canUpload => { - ctrl.canUpload = canUpload; + ctrl.canUpload = canUpload; }); ctrl.ordering = { - orderBy: $stateParams.orderBy + orderBy: $stateParams.orderBy }; + let lastRequestedOrderBy = $stateParams.orderBy; + ctrl.filter = { - uploadedByMe: false + uploadedByMe: false }; ctrl.dateFilter = { - // filled in by the watcher below + // filled in by the watcher below }; ctrl.usePermissionsFilter = window._clientConfig.usePermissionsFilter; ctrl.filterMyUploads = false; + let lastUploadedByEventKey; ctrl.initialShowPaidEvent = ($stateParams.nonFree === undefined && ctrl.usePermissionsFilter) ? false : true; - ctrl.shouldDisplayAISearchOption = getFeatureSwitchActive("enable-ai-search"); + ctrl.shouldDisplayAISearchOption = window._clientConfig.aiSearchEnabled; if (!ctrl.shouldDisplayAISearchOption) { ctrl.useAISearch = false; ctrl.vecWeight = undefined; @@ -90,7 +96,7 @@ query.controller('SearchQueryCtrl', [ //--react - angular interop events-- function raisePayableImagesEvent(showPaid) { - const boolShowPaid = (showPaid === true || showPaid === "true") ? true : false; + const boolShowPaid = toNonFreeString(showPaid) === 'true'; const customEvent = new CustomEvent('setPayableImages', { detail: {showPaid: boolShowPaid}, bubbles: true @@ -116,8 +122,14 @@ query.controller('SearchQueryCtrl', [ function raiseUploadedByCheckEvent() { if (ctrl.user) { + const uploadedBy = ctrl.filter ? ctrl.filter.uploadedBy : undefined; + const eventKey = `${ctrl.user.email}|${uploadedBy || ''}`; + if (eventKey === lastUploadedByEventKey) { + return; + } + lastUploadedByEventKey = eventKey; const customEvent = new CustomEvent('uploadedByEvent', { - detail: { userEmail: ctrl.user.email, uploadedBy: $stateParams.uploadedBy }, + detail: { userEmail: ctrl.user.email, uploadedBy: uploadedBy }, bubbles: true }); window.dispatchEvent(customEvent); @@ -140,9 +152,9 @@ query.controller('SearchQueryCtrl', [ } else { if (sender === "selectMyUploads") { const shouldOverwriteUploadedBy = - !filter.uploadedBy || - filter.uploadedBy === (ctrl.user ? ctrl.user.email : undefined) || - ctrl.filterMyUploads; + !filter.uploadedBy || + filter.uploadedBy === (ctrl.user ? ctrl.user.email : undefined) || + ctrl.filterMyUploads; if (shouldOverwriteUploadedBy) { ctrl.filter.uploadedBy = (ctrl.user && ctrl.filterMyUploads) ? ctrl.user.email : undefined; ctrl.filter.uploadedByMe = ctrl.filterMyUploads; @@ -153,53 +165,104 @@ query.controller('SearchQueryCtrl', [ storage.setJs("isUploadedByMe", ctrl.filter.uploadedByMe, true); } - function manageDefaultNonFree(filter) { - const defaultNonFreeFilter = storage.getJs("defaultNonFreeFilter", true); - if (defaultNonFreeFilter && defaultNonFreeFilter.isDefault === true){ - let newNonFree = defaultNonFreeFilter.isNonFree ? "true" : undefined; - if (newNonFree !== filter.nonFree) { - storage.setJs("isNonFree", newNonFree ? newNonFree : false, true); - storage.setJs("defaultIsNonFree", newNonFree ? newNonFree : false, true); - storage.setJs("isUploadedByMe", false, true); - storage.setJs("defaultNonFreeFilter", {isDefault: false, isNonFree: false}, true); - ctrl.filter.orgOwned = false; - } - Object.assign(ctrl.filter, {nonFree: newNonFree, uploadedByMe: false, uploadedBy: undefined}); - raiseFilterChangeEvent(ctrl.filter); + // Guards the delayed disarming of the "pending default nonFree" flag, + // see manageDefaultNonFree() below for why this is needed. + let clearDefaultNonFreeFilterTimeout; + + function disarmDefaultNonFreeFilter() { + if (clearDefaultNonFreeFilterTimeout) { + $timeout.cancel(clearDefaultNonFreeFilterTimeout); + clearDefaultNonFreeFilterTimeout = undefined; + } + const defaultNonFreeFilter = storage.getJs("defaultNonFreeFilter", true); + if (defaultNonFreeFilter && defaultNonFreeFilter.isDefault === true) { + storage.setJs( + "defaultNonFreeFilter", + {isDefault: false, isNonFree: defaultNonFreeFilter.isNonFree}, + true + ); + } + } + + function manageDefaultNonFree() { + const defaultNonFreeFilter = storage.getJs("defaultNonFreeFilter", true); + if (defaultNonFreeFilter && defaultNonFreeFilter.isDefault === true){ + const newNonFree = toNonFreeString(defaultNonFreeFilter.isNonFree); + storage.setJs("isNonFree", newNonFree, true); + storage.setJs("defaultIsNonFree", newNonFree, true); + storage.setJs("isUploadedByMe", false, true); + ctrl.filterMyUploads = false; + if (ctrl.myUploadsProps) { + syncMyUploadsProps(); + } + ctrl.filter.orgOwned = false; + ctrl.useAISearch = false; + Object.assign(ctrl.filter, {nonFree: newNonFree, uploadedByMe: false, uploadedBy: undefined}); + raiseFilterChangeEvent(ctrl.filter); + + // IMPORTANT: don't disarm the "isDefault" flag immediately here. + // A single logo click triggers *several* back-to-back + // navigations/filter-change digests (onLogoClick's own $state.go, + // then gr-sort-control's handleLogoClick -> updateSortChips, and + // sometimes a delayed, spurious transition fired by + // ui-router-extras' Deep State Redirect mechanism that drops + // params such as `nonFree` back to their state defaults). + // If we disarm the flag on the very *first* of these passes + // (which may just be an incidental/harmless one - e.g. the echo + // of onLogoClick's own navigation - and not the actual rogue + // transition we need to correct), this function becomes a no-op + // for any later pass, so `nonFree` never gets corrected again if + // the rogue transition drops it afterwards, leaving it stuck at + // its non-default value. Instead, keep re-applying the default + // on every pass for a short grace period after being armed, and + // only disarm it once that period elapses. + if (clearDefaultNonFreeFilterTimeout) { + $timeout.cancel(clearDefaultNonFreeFilterTimeout); } + clearDefaultNonFreeFilterTimeout = $timeout(() => { + storage.setJs("defaultNonFreeFilter", {isDefault: false, isNonFree: newNonFree}, true); + clearDefaultNonFreeFilterTimeout = undefined; + }, 1500); + } } + $scope.$on('$destroy', () => { + if (clearDefaultNonFreeFilterTimeout) { + $timeout.cancel(clearDefaultNonFreeFilterTimeout); + } + }); + function manageOrgOwnedSetting(filter) { - const structuredQuery = structureQuery(filter.query) || []; - const orgOwnedIndexInQuery = structuredQuery.findIndex(item => item.value === ctrl.maybeOrgOwnedValue); - const queryHasOrgOwned = orgOwnedIndexInQuery >= 0; - if (ctrl.filter.orgOwned && !queryHasOrgOwned){ - // If the checkbox is ticked, ensure the chip is part of the search bar - const orgOwnedChip = { - type: "filter", - filterType: "inclusion", - key : "is", - value: ctrl.maybeOrgOwnedValue - }; - ctrl.filter.query = renderQuery([ - ...structuredQuery, - orgOwnedChip - ]); - } else if (!ctrl.filter.orgOwned && queryHasOrgOwned && !ctrl.usePermissionsFilter) { - // If the checkbox is unticked, ensure chip is no longer in the search bar - structuredQuery.splice(orgOwnedIndexInQuery, 1); - ctrl.filter.query = renderQuery(structuredQuery); - } + const structuredQuery = structureQuery(filter.query) || []; + const orgOwnedIndexInQuery = structuredQuery.findIndex(item => item.value === ctrl.maybeOrgOwnedValue); + const queryHasOrgOwned = orgOwnedIndexInQuery >= 0; + if (ctrl.filter.orgOwned && !queryHasOrgOwned){ + // If the checkbox is ticked, ensure the chip is part of the search bar + const orgOwnedChip = { + type: "filter", + filterType: "inclusion", + key : "is", + value: ctrl.maybeOrgOwnedValue + }; + ctrl.filter.query = renderQuery([ + ...structuredQuery, + orgOwnedChip + ]); + } else if (!ctrl.filter.orgOwned && queryHasOrgOwned && !ctrl.usePermissionsFilter) { + // If the checkbox is unticked, ensure chip is no longer in the search bar + structuredQuery.splice(orgOwnedIndexInQuery, 1); + ctrl.filter.query = renderQuery(structuredQuery); + } } function resetQuery() { - ctrl.filter.query = undefined; - ctrl.filter.orgOwned = false; + ctrl.filter.query = undefined; + ctrl.filter.orgOwned = false; } function checkForCollection(query) { return /~"[a-zA-Z0-9 #-_.://]+"/.test(query); - }; + } function storeCollection(query) { const match = query ? query.match(/~"[a-zA-Z0-9 #-_.://]+"/) : undefined; @@ -209,13 +272,11 @@ query.controller('SearchQueryCtrl', [ } function getCollection() { - const collection = storage.getJs("currentCollection") ? storage.getJs("currentCollection") : ""; - return collection; + return storage.getJs("currentCollection") ? storage.getJs("currentCollection") : ""; } function getPriorOrderBy() { - const prior = storage.getJs("priorOrderBy") ? storage.getJs("priorOrderBy") : ""; - return prior; + return storage.getJs("priorOrderBy") ? storage.getJs("priorOrderBy") : ""; } function setPriorOrderBy(priorOrderBy) { @@ -233,7 +294,7 @@ query.controller('SearchQueryCtrl', [ function priorRevisedOrderBy(collectionSearch, newCollection, oldCollection) { const priorOrderBy = getPriorOrderBy(); if (collectionSearch && ((oldCollection !== newCollection) || ("" !== priorOrderBy))) { - if (priorOrderBy != "") { + if (priorOrderBy !== "") { setPriorOrderBy(""); return priorOrderBy; } else { @@ -245,11 +306,29 @@ query.controller('SearchQueryCtrl', [ } } + function resolveNonFree() { + let nonFreeCheck = ctrl.filter.nonFree !== undefined + ? toNonFreeString(ctrl.filter.nonFree) + : undefined; + if (ctrl.usePermissionsFilter && nonFreeCheck === undefined) { + nonFreeCheck = toNonFreeString(storage.getJs("defaultIsNonFree", true)); + } else if (!ctrl.usePermissionsFilter && nonFreeCheck === 'false') { + nonFreeCheck = undefined; + } + ctrl.filter.nonFree = nonFreeCheck; + return nonFreeCheck; + } + + function emitQueryTelemetry() { + const nonFreeCheck = resolveNonFree(); + sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, ctrl.filter.uploadedByMe, ctrl.useAISearch); + } + // eslint-disable-next-line complexity function watchSearchChange(newFilter, sender) { - let showPaid = newFilter.nonFree ? newFilter.nonFree : false; - if (sender && sender == "filterChange" && !newFilter.nonFree) { - showPaid = ctrl.user.permissions.showPaid; + let showPaid = toNonFreeString(newFilter.nonFree); + if (ctrl.usePermissionsFilter && sender && sender === "filterChange" && newFilter.nonFree === undefined) { + showPaid = toNonFreeString(ctrl.user.permissions.showPaid); } storage.setJs("isNonFree", showPaid, true); @@ -260,7 +339,7 @@ query.controller('SearchQueryCtrl', [ const newCollection = storeCollection(newFilter.query); if (ctrl.usePermissionsFilter) { - if (sender && ctrl.ordering["orderBy"] != $stateParams.orderBy) { + if (sender && ctrl.ordering["orderBy"] !== $stateParams.orderBy) { ctrl.ordering["orderBy"] = $stateParams.orderBy; } if ($stateParams.orderBy && $stateParams.orderBy.includes(TAKEN_SORT) && (!newFilter.query || !newFilter.query.includes(HAS_DATE_TAKEN))) { @@ -275,34 +354,72 @@ query.controller('SearchQueryCtrl', [ //--update filter elements-- manageUploadedBy(newFilter, sender); - manageDefaultNonFree(newFilter); + manageDefaultNonFree(); manageOrgOwnedSetting(newFilter); - const { nonFree, uploadedByMe } = ctrl.filter; - let nonFreeCheck = nonFree; - if (ctrl.usePermissionsFilter && nonFreeCheck === undefined) { - const defaultShowPaid = storage.getJs("defaultIsNonFree", true); - nonFreeCheck = defaultShowPaid; - } else if (!ctrl.usePermissionsFilter && (nonFreeCheck === 'false' || nonFreeCheck === false)) { - nonFreeCheck = undefined; + emitQueryTelemetry(); + + function normaliseParamForComparison(key, val) { + if (key === 'nonFree' && !ctrl.usePermissionsFilter && val === 'false') { + return undefined; + } + // treat '' and undefined as equivalent to avoid false negatives + return val || undefined; + } + + function goParamsAlreadyCurrent(goParams) { + return Object.keys(goParams).every(key => { + const a = normaliseParamForComparison(key, goParams[key]); + const b = normaliseParamForComparison(key, $stateParams[key]); + return a === b; + }); + } + + function isOrderByOnlyCorrection(goParams) { + return Object.keys(goParams).every(key => { + if (key === 'orderBy') { return true; } + const a = normaliseParamForComparison(key, goParams[key]); + const b = normaliseParamForComparison(key, $stateParams[key]); + return a === b; + }); } - ctrl.filter.nonFree = nonFreeCheck; - sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, uploadedByMe, ctrl.useAISearch); if (ctrl.collectionSearch && !curCollectionSearch) { storage.setJs("orderBy", CollectionSortOption.value); ctrl.ordering["orderBy"] = CollectionSortOption.value; - raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, CollectionSortOption.value); - $state.go('search.results', {...ctrl.filter, ...{orderBy: CollectionSortOption.value}}); + // Record synchronously *before* the ctrl.ordering.orderBy $watch can + // run in this same digest, so it recognises this orderBy change as + // already handled and doesn't fire its own extra navigation. + lastRequestedOrderBy = CollectionSortOption.value; + const goParams1 = {...ctrl.filter, ...{orderBy: CollectionSortOption.value}}; + if (!goParamsAlreadyCurrent(goParams1)) { + raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, CollectionSortOption.value); + const options1 = isOrderByOnlyCorrection(goParams1) ? {location: 'replace'} : undefined; + $state.go('search.results', goParams1, options1); + } } else { - raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, ctrl.ordering["orderBy"]); - $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering["orderBy"]}}); + lastRequestedOrderBy = ctrl.ordering["orderBy"]; + const goParams2 = {...ctrl.filter, ...{orderBy: ctrl.ordering["orderBy"]}}; + if (!goParamsAlreadyCurrent(goParams2)) { + raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, ctrl.ordering["orderBy"]); + const options2 = isOrderByOnlyCorrection(goParams2) ? {location: 'replace'} : undefined; + $state.go('search.results', goParams2, options2); + } } } //-my uploads- + function syncMyUploadsProps() { + ctrl.myUploadsProps = { + ...ctrl.myUploadsProps, + myUploads: ctrl.filterMyUploads + }; + } + function selectMyUploads(myUploadsChecked) { + disarmDefaultNonFreeFilter(); ctrl.filterMyUploads = myUploadsChecked; + syncMyUploadsProps(); watchSearchChange(ctrl.filter, "selectMyUploads"); } @@ -316,39 +433,88 @@ query.controller('SearchQueryCtrl', [ function updateSortChips (sortSel) { ctrl.ordering['orderBy'] = manageSortSelection(sortSel.value); storage.setJs("orderBy", ctrl.ordering["orderBy"]); - $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering['orderBy']}}); + + lastRequestedOrderBy = ctrl.ordering['orderBy']; + $state.go('search.results', {orderBy: ctrl.ordering['orderBy']}); } ctrl.sortProps = { onSortSelect: updateSortChips, - query: ctrl.filter.query, + query: $stateParams.query, orderBy: ctrl.ordering ? ctrl.ordering.orderBy : "" }; + + // Keep the React sort-control's props in sync with the underlying model. + // A *new* object reference is required each time, since react2angular's + // one-way ('<') binding only re-renders when the bound reference changes. + $scope.$watch( + () => `${ctrl.ordering.orderBy}||${ctrl.filter.query}`, + onValChange(() => { + ctrl.sortProps = { + ...ctrl.sortProps, + query: ctrl.filter.query, + orderBy: ctrl.ordering.orderBy + }; + }) + ); //-end sort control- //-permissions filter- function updatePermissionsChips (permissionsSel, showChargeable) { + disarmDefaultNonFreeFilter(); ctrl.permissionsProps.selectedOption = permissionsSel; ctrl.filter.query = updateFilterChips(permissionsSel, ctrl.filter.query); - ctrl.filter.nonFree = showChargeable; + ctrl.filter.nonFree = toNonFreeString(showChargeable); watchSearchChange(ctrl.filter, "updatePermissionsChips"); } function chargeableChange (showChargeable) { - ctrl.filter.nonFree = showChargeable; + disarmDefaultNonFreeFilter(); + ctrl.filter.nonFree = toNonFreeString(showChargeable); watchSearchChange(ctrl.filter, "chargeableChange"); } + ctrl.onNonFreeCheckboxChange = function() { + disarmDefaultNonFreeFilter(); + }; + + ctrl.onUploadedByMeCheckboxChange = function() { + disarmDefaultNonFreeFilter(); + }; + + ctrl.onOrgOwnedCheckboxChange = function() { + disarmDefaultNonFreeFilter(); + }; + + ctrl.onAISearchCheckboxChange = function() { + disarmDefaultNonFreeFilter(); + }; + let pfOpts = PermissionsConf.permissionsOptions(); let defOptVal = PermissionsConf.permissionsDefaultOpt(); let pfDefPerm = pfOpts.filter(opt => opt.value == defOptVal)[0]; ctrl.permissionsProps = { options: pfOpts, - selectedOption: pfDefPerm, - onSelect: updatePermissionsChips, - onChargeable: chargeableChange, - chargeable: ctrl.filter.nonFree ? ctrl.filter.nonFree : ($stateParams.nonFree == "true"), - query: ctrl.filter.query - }; + selectedOption: pfDefPerm, + onSelect: updatePermissionsChips, + onChargeable: chargeableChange, + chargeable: toNonFreeString(ctrl.filter.nonFree || $stateParams.nonFree) === "true", + query: ctrl.filter.query + }; + + // Keep the React permissions-filter's props in sync with the underlying + // model (e.g. after a reset via onLogoClick or the back button), for the + // same reason as ctrl.sortProps above: react2angular's one-way binding + // only re-renders on a *new* object reference. + $scope.$watch( + () => `${ctrl.filter.nonFree}||${ctrl.filter.query}`, + onValChange(() => { + ctrl.permissionsProps = { + ...ctrl.permissionsProps, + chargeable: toNonFreeString(ctrl.filter.nonFree) === "true", + query: ctrl.filter.query + }; + }) + ); //-end permissions filter- ctrl.resetQuery = resetQuery; @@ -362,36 +528,36 @@ query.controller('SearchQueryCtrl', [ const pastYear = moment().subtract(1, 'years').toISOString(); ctrl.payTypeOptions = [ - {label: 'Free', value: 'free'}, - {label: 'Free and No Rights', value: 'maybe-free'}, - {label: 'All (inc. paid)', value: 'all'} + {label: 'Free', value: 'free'}, + {label: 'Free and No Rights', value: 'maybe-free'}, + {label: 'All (inc. paid)', value: 'all'} ]; ctrl.sinceOptions = [ - {label: 'Anytime'}, // value: undefined - {label: 'Today', value: lastMidnight}, - {label: 'Past 24 hours', value: past24Hours}, - {label: 'Past week', value: pastWeek}, - {label: 'Past 6 months', value: past6Months}, - {label: 'Past year', value: pastYear} + {label: 'Anytime'}, // value: undefined + {label: 'Today', value: lastMidnight}, + {label: 'Past 24 hours', value: past24Hours}, + {label: 'Past week', value: pastWeek}, + {label: 'Past 6 months', value: past6Months}, + {label: 'Past year', value: pastYear} ]; ctrl.filterDateFieldsOptions = [ - {label: 'Upload time', name: 'uploaded'}, // value: undefined - {label: 'Date taken', name: 'taken', value: 'taken'}, - {label: 'Last modified', name: 'modified', value: 'modified'} + {label: 'Upload time', name: 'uploaded'}, // value: undefined + {label: 'Date taken', name: 'taken', value: 'taken'}, + {label: 'Last modified', name: 'modified', value: 'modified'} ]; const dateFilterParams = [ - 'dateField', 'since', 'until', 'takenSince', 'takenUntil', - 'modifiedSince', 'modifiedUntil' + 'dateField', 'since', 'until', 'takenSince', 'takenUntil', + 'modifiedSince', 'modifiedUntil' ]; Object.keys($stateParams). - // Exclude date-related filters, managed separately in dateFilter - filter(key => dateFilterParams.indexOf(key) === -1). - // Exclude useAISearch, managed separately by its own dedicated watcher - filter(key => key !== 'useAISearch'). - forEach(setAndWatchParam); + // Exclude date-related filters, managed separately in dateFilter + filter(key => dateFilterParams.indexOf(key) === -1). + // Exclude useAISearch, managed separately by its own dedicated watcher + filter(key => key !== 'useAISearch'). + forEach(setAndWatchParam); // URL parameters are not decoded when taken out of the params. // Might be fixed with: https://github.com/angular-ui/ui-router/issues/1759 @@ -399,49 +565,67 @@ query.controller('SearchQueryCtrl', [ function valOrUndefined(str) { return str ? str : undefined; } function setAndWatchParam(key) { - //this value has been set on ctrl.order + //this value has been set on ctrl.order + if (key !== 'orderBy') { + ctrl.filter[key] = valOrUndefined($stateParams[key]); + } + + ctrl.collectionSearch = ctrl.filter.query ? checkForCollection(ctrl.filter.query) : false; + storeCollection(ctrl.filter.query); + + $scope.$watch(() => $stateParams[key], onValChange(newVal => { + // FIXME: broken for 'your uploads' + // FIXME: + they triggers filter $watch and $state.go (breaks history) if (key !== 'orderBy') { - ctrl.filter[key] = valOrUndefined($stateParams[key]); + // For nonFree, ensure boolean true from URL decode is normalised to string 'true' + const val = valOrUndefined(newVal); + ctrl.filter[key] = (key === 'nonFree' && val !== undefined) ? toNonFreeString(val) : val; + } else { + ctrl.ordering.orderBy = valOrUndefined(newVal); + } + + if (key === 'query') { + const sq = structureQuery(valOrUndefined(newVal)) || []; + ctrl.filter.orgOwned = sq.some(item => item.value === ctrl.maybeOrgOwnedValue); + } + + // When uploadedBy changes externally (e.g. back button), sync uploadedByMe + // and filterMyUploads so manageUploadedBy doesn't re-set uploadedBy. + if (key === 'uploadedBy') { + const isMyUploads = ctrl.user ? valOrUndefined(newVal) === ctrl.user.email : false; + ctrl.filter.uploadedByMe = isMyUploads; + ctrl.filterMyUploads = isMyUploads; + syncMyUploadsProps(); } - ctrl.collectionSearch = ctrl.filter.query ? checkForCollection(ctrl.filter.query) : false; - storeCollection(ctrl.filter.query); - - $scope.$watch(() => $stateParams[key], onValChange(newVal => { - // FIXME: broken for 'your uploads' - // FIXME: + they triggers filter $watch and $state.go (breaks history) - if (key !== 'orderBy') { - ctrl.filter[key] = valOrUndefined(newVal); - } - - // don't track changes to `query` as it would trigger on every keypress - if (key !== 'query') { - $rootScope.$emit( - 'track:event', 'Query', 'Change', 'Success', null, { field: key, value: newVal } - ); - } - })); + // don't track changes to `query` as it would trigger on every keypress + if (key !== 'query') { + $rootScope.$emit( + 'track:event', 'Query', 'Change', 'Success', null, { field: key, value: newVal } + ); + } + })); } // Init and apply date-related changes in $stateParams to ctrl.dateFilter $scope.$watchCollection(() => $stateParams, () => { - switch ($stateParams.dateField) { + switch ($stateParams.dateField) { case 'taken': - ctrl.dateFilter.since = $stateParams.takenSince; - ctrl.dateFilter.until = $stateParams.takenUntil; - ctrl.dateFilter.field = 'taken'; - break; + ctrl.dateFilter.since = $stateParams.takenSince; + ctrl.dateFilter.until = $stateParams.takenUntil; + ctrl.dateFilter.field = 'taken'; + break; case 'modified': - ctrl.dateFilter.since = $stateParams.modifiedSince; - ctrl.dateFilter.until = $stateParams.modifiedUntil; - ctrl.dateFilter.field = 'modified'; - break; + ctrl.dateFilter.since = $stateParams.modifiedSince; + ctrl.dateFilter.until = $stateParams.modifiedUntil; + ctrl.dateFilter.field = 'modified'; + break; default: // uploaded (default so represented as `undefined`) - ctrl.dateFilter.since = $stateParams.since; - ctrl.dateFilter.until = $stateParams.until; - ctrl.dateFilter.field = undefined; - break; - } + ctrl.dateFilter.since = $stateParams.since; + ctrl.dateFilter.until = $stateParams.until; + ctrl.dateFilter.field = undefined; + break; + } }); $scope.$watchCollection(() => ctrl.filter, onValChange(newFilter => { @@ -449,11 +633,27 @@ query.controller('SearchQueryCtrl', [ })); $scope.$watch(() => ctrl.ordering.orderBy, onValChange(newVal => { - $state.go('search.results', {...ctrl.filter, orderBy: newVal}); + if ($stateParams.orderBy === newVal || lastRequestedOrderBy === newVal) { + return; + } + lastRequestedOrderBy = newVal; + const stateGoParams = {...ctrl.filter, orderBy: newVal}; + $state.go('search.results', stateGoParams); })); + + let aiSearchInitialised = false; $scope.$watch(() => ctrl.useAISearch, () => { // Note: $watch expressions execute at least once during initialization, so this is executed on page refresh. - // This is the behaviour we want so that the URL is updated based on the AI search toggle + // This is the behaviour we want so that the URL is updated based on the AI search toggle. + // We only emit telemetry on an actual toggle though - the load-time search event is emitted + // exactly once via the getSession() -> watchSearchChange path, so emitting here on init would double-count. + if (aiSearchInitialised) { + emitQueryTelemetry(); + } else { + resolveNonFree(); + } + aiSearchInitialised = true; + if (ctrl.useAISearch) { $state.go('search.results', { ...ctrl.filter, @@ -461,81 +661,87 @@ query.controller('SearchQueryCtrl', [ vecWeight: ctrl.vecWeight }); } else { - $state.go('search.results', {...ctrl.filter, useAISearch: null}); + $state.go('search.results', {...ctrl.filter, useAISearch: undefined}); } }); $scope.$watchCollection(() => ctrl.dateFilter, onValChange(({field, since, until}) => { - // Translate dateFilter to actual state and query params - $state.go('search.results', {...ctrl.filter, ...{ - since: field === undefined ? since : null, - until: field === undefined ? until : null, - takenSince: field === 'taken' ? since : null, - takenUntil: field === 'taken' ? until : null, - modifiedSince: field === 'modified' ? since : null, - modifiedUntil: field === 'modified' ? until : null, - dateField: field + // Translate dateFilter to actual state and query params + $state.go('search.results', {...ctrl.filter, ...{ + since: field === undefined ? since : null, + until: field === undefined ? until : null, + takenSince: field === 'taken' ? since : null, + takenUntil: field === 'taken' ? until : null, + modifiedSince: field === 'modified' ? since : null, + modifiedUntil: field === 'modified' ? until : null, + dateField: field }}); })); // we can't user dynamic values in the ng:true-value see: // https://docs.angularjs.org/error/ngModel/constexpr mediaApi.getSession().then(session => { - //-uploaded by me- - const isUploadedByMe = storage.getJs("isUploadedByMe", true); - ctrl.user = session.user; - if (isUploadedByMe === null) { - ctrl.filter.uploadedByMe = ctrl.filter.uploadedBy === ctrl.user.email; + //-uploaded by me- + const isUploadedByMe = storage.getJs("isUploadedByMe", true); + ctrl.user = session.user; + if (ctrl.filter.uploadedBy === ctrl.user.email) { + ctrl.filter.uploadedByMe = true; + ctrl.filterMyUploads = true; + storage.setJs("isUploadedByMe", true); + } else if (isUploadedByMe === null) { + ctrl.filter.uploadedByMe = ctrl.filter.uploadedBy === ctrl.user.email; + ctrl.filterMyUploads = ctrl.filter.uploadedByMe; + storage.setJs("isUploadedByMe",ctrl.filter.uploadedByMe); + } else { + if ((ctrl.filter.uploadedBy === ctrl.user.email) && !isUploadedByMe ) { + ctrl.filter.uploadedByMe = true; ctrl.filterMyUploads = ctrl.filter.uploadedByMe; storage.setJs("isUploadedByMe",ctrl.filter.uploadedByMe); } else { - if ((ctrl.filter.uploadedBy === ctrl.user.email) && !isUploadedByMe ) { - ctrl.filter.uploadedByMe = true; - ctrl.filterMyUploads = ctrl.filter.uploadedByMe; - storage.setJs("isUploadedByMe",ctrl.filter.uploadedByMe); - } else { - ctrl.filter.uploadedByMe = isUploadedByMe; - ctrl.filterMyUploads = isUploadedByMe; - } + ctrl.filter.uploadedByMe = isUploadedByMe; + ctrl.filterMyUploads = isUploadedByMe; } + } - //-default non free- - const defNonFree = session.user.permissions ? session.user.permissions.showPaid : undefined; - storage.setJs("defaultIsNonFree", defNonFree ? defNonFree : false, true); - if (!ctrl.initialShowPaidEvent && (defNonFree === true || defNonFree === "true")) { - ctrl.initialShowPaidEvent = true; - raisePayableImagesEvent(defNonFree); - } + syncMyUploadsProps(); + raiseUploadedByCheckEvent(); + + //-default non free- + const defNonFree = session.user.permissions ? session.user.permissions.showPaid : undefined; + storage.setJs("defaultIsNonFree", toNonFreeString(defNonFree), true); + if (!ctrl.initialShowPaidEvent && toNonFreeString(defNonFree) === 'true') { + ctrl.initialShowPaidEvent = true; + raisePayableImagesEvent(defNonFree); + } + // If nonFree is provided in URL params, use that; otherwise use stored value + if ($stateParams.nonFree !== undefined) { + ctrl.filter.nonFree = toNonFreeString($stateParams.nonFree); + storage.setJs("isNonFree", ctrl.filter.nonFree, true); + } else { const isNonFree = storage.getJs("isNonFree", true); if (isNonFree === null) { - ctrl.filter.nonFree = $stateParams.nonFree; - storage.setJs("isNonFree", ctrl.filter.nonFree ? ctrl.filter.nonFree : (ctrl.usePermissionsFilter ? "false" : undefined), true); - } - else if (isNonFree === true || isNonFree === "true") { - ctrl.filter.nonFree = "true"; + ctrl.filter.nonFree = toNonFreeString($stateParams.nonFree); + storage.setJs("isNonFree", ctrl.filter.nonFree, true); } else { - ctrl.filter.nonFree = (ctrl.usePermissionsFilter ? "false" : undefined); + ctrl.filter.nonFree = isNonFreeString(isNonFree) ? 'true' : 'false'; } + } - //-org owned- - const structuredQuery = structureQuery(ctrl.filter.query); - const orgOwned = (structuredQuery.some(item => item.value === ctrl.maybeOrgOwnedValue)); - ctrl.filter.orgOwned = orgOwned; + //-org owned- + const structuredQuery = structureQuery(ctrl.filter.query); + const orgOwned = (structuredQuery.some(item => item.value === ctrl.maybeOrgOwnedValue)); + ctrl.filter.orgOwned = orgOwned; - watchSearchChange(ctrl.filter, "userPermissions"); + watchSearchChange(ctrl.filter, "userPermissions"); }); - - - const { nonFree, uploadedByMe } = ctrl.filter; - sendTelemetryForQuery(ctrl.filter.query, nonFree, uploadedByMe, ctrl.useAISearch); -}]); + }]); query.directive('searchQuery', [function() { - return { - restrict: 'E', - controller: 'SearchQueryCtrl as searchQuery', - template: template - }; + return { + restrict: 'E', + controller: 'SearchQueryCtrl as searchQuery', + template: template + }; }]); diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index 2c61e3c77bb..1d4e707d3b8 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -22,7 +22,11 @@
      - {{ctrl.totalResults | toLocaleString}} matches + + Best {{ctrl.aiResultsShown | toLocaleString}} of {{ctrl.totalResults | toLocaleString}} matches + {{ctrl.filterPoolCounts.filteredPool | toLocaleString}} matches + + {{ctrl.totalResults | toLocaleString}} matches
    -
    Too many results to display
    Please refine your search to limit the number of results
    -
    + + {{ctrl.filterPoolCounts.filteredPool | toLocaleString}} images match your filters. Add a query to see the best 200. + +
    + +
    - No matches, sorry! Try altering your search. - Please enter a query. You can include filters, but must include either some text or a similar:image-id term. + No matches, sorry! Try altering your search.
diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 6721bf53ba5..5e322d46807 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -233,6 +233,12 @@ results.controller('SearchResultsCtrl', [ // FIXME: https://github.com/argo-rest/theseus has forced us to co-opt the actions field for this ctrl.tickerCounts = images.$response?.$$state?.value?.actions?.tickerCounts; + // Present only when an AI search has no text/similar-image query to + // rank by. When set, we show how the filters have narrowed + // the pool of images and invite the user to add a query. + ctrl.filterPoolCounts = images.$response?.$$state?.value?.actions?.filterPoolCounts; + ctrl.filtersOnlyAiSearch = !!ctrl.filterPoolCounts; + ctrl.hasQuery = !!$stateParams.query; ctrl.initialSearchUri = images.uri; ctrl.embeddableUrl = window.location.href; @@ -248,6 +254,9 @@ results.controller('SearchResultsCtrl', [ function initialiseAiResults(images) { const totalLength = images.data.length; ctrl.imagesAll = new Array(totalLength); + // Number of results actually shown (the top k), so we can display + // "Best k of N matches" where N is ctrl.totalResults. + ctrl.aiResultsShown = totalLength; // AI search returns a single fixed result set rather than a paged/lazy-loaded one, // so we populate the full backing array up front to avoid placeholder rows. @@ -293,6 +302,8 @@ results.controller('SearchResultsCtrl', [ checkForNewImages(); } + ctrl.isAiSearch = isAiSearch; + updateLastSearchBoundary(); return images; @@ -827,24 +838,27 @@ results.controller('SearchResultsCtrl', [ }); }; - const freeImageDeleteListener = $rootScope.$on('images-deleted', (e, images) => { - images.forEach(image => { - // TODO: should not be needed here, the selection and - // results should listen to these events and update - // itself outside of any controller - ctrl.deselect(image); + const imageDeleteHandler = (_, images) => { + images.forEach(image => { + // TODO: should not be needed here, the selection and + // results should listen to these events and update + // itself outside of any controller + ctrl.deselect(image); - const indexAll = ctrl.imagesAll.findIndex(i => image.data.id === i.data.id); - results.removeAt(indexAll); + const indexAll = ctrl.imagesAll.findIndex(i => image.data.id === i.data.id); + results.removeAt(indexAll); - updateImageArray(ctrl.images, image); - updateImageArray(ctrl.imagesAll, image); + updateImageArray(ctrl.images, image); + updateImageArray(ctrl.imagesAll, image); - updatePositions(image); + updatePositions(image); - ctrl.totalResults--; - }); - }); + ctrl.totalResults--; + }); + }; + + const freeImageDeleteListener = $rootScope.$on('images-deleted', imageDeleteHandler); + const freeImageUndeleteListener = $rootScope.$on('images-undeleted', imageDeleteHandler); // Safer than clearing the timeout in case of race conditions // FIXME: nicer (reactive?) way to do this? @@ -913,6 +927,7 @@ results.controller('SearchResultsCtrl', [ } freeUpdatesListener(); freeImageDeleteListener(); + freeImageUndeleteListener(); scopeGone = true; }); } diff --git a/kahuna/public/js/services/image-list.js b/kahuna/public/js/services/image-list.js index 0e49f902c78..4a685311b15 100644 --- a/kahuna/public/js/services/image-list.js +++ b/kahuna/public/js/services/image-list.js @@ -18,8 +18,14 @@ imageList.factory("imageList", [ }, new Map()); } - function occurrencesToTuple(countsMap) { - return countsMap.map((count, data) => ({data, count})).toArray(); + function occurrencesToArray(countsMap) { + // Map is a Keyed collection, so toArray() on it returns [key, value] + // entry pairs rather than just the mapped values. Use valueSeq() first + // to get a flat array of the mapped {data, count} objects. + return countsMap + .map((count, data) => ({ data, count })) + .valueSeq() + .toArray(); } // TODO: a lot of these are boilerplate around imageAccessor, can @@ -54,7 +60,7 @@ imageList.factory("imageList", [ function getOccurrences(items) { const valueCounts = countOccurrences(items); - return occurrencesToTuple(valueCounts); + return occurrencesToArray(valueCounts); } function getSetOfProperties(objects) { diff --git a/kahuna/public/js/services/image/usages.js b/kahuna/public/js/services/image/usages.js index b94e1df5fc5..07519cb329e 100644 --- a/kahuna/public/js/services/image/usages.js +++ b/kahuna/public/js/services/image/usages.js @@ -87,7 +87,6 @@ imageUsagesService.factory('imageUsagesService', [function() { }); }); - const groupedByState$ = usages$ .map((usagesList) => usagesList.groupBy(usage => usage.get('status'))); diff --git a/kahuna/public/js/services/telemetry.ts b/kahuna/public/js/services/telemetry.ts index b6531608ec3..ffa41e96f17 100644 --- a/kahuna/public/js/services/telemetry.ts +++ b/kahuna/public/js/services/telemetry.ts @@ -49,11 +49,11 @@ const sendFilterTelemetryEvent = (key: string, value: string, searchUuid: string }, 1); }; -export const sendTelemetryForQuery = (query: string, nonFree?: boolean | string, uploadedByMe?: boolean, useAISearch?: boolean ) => { +export const sendTelemetryForQuery = (query: string, nonFree?: string, uploadedByMe?: boolean, useAISearch?: boolean ) => { const structuredQuery = structureQuery(query || ""); + const searchUuid = v4(); - // nonFree is unfortunately either a boolean, stringified boolean, or undefined - const freeToUseOnly = (!(nonFree === 'true' || nonFree === true)); + const freeToUseOnly = nonFree !== 'true'; const uploadedByMeOnly = (uploadedByMe); // Only log for true - matching how these filters work in Grid (only applied when true) @@ -74,11 +74,16 @@ export const sendTelemetryForQuery = (query: string, nonFree?: boolean | string, return `GRID_${type.toUpperCase()}`; }; + const searchKey = (type: string) => { + return type === 'text' ? { searchKey: `${getBrowserId()}::${queryComponent.value}` } : undefined; + }; + // In case search is empty, as with a search containing only filters sendTelemetryEvent(formattedType(type), { ...queryComponent, searchUuid: searchUuid, - useAISearch: useAISearch ?? false + useAISearch: useAISearch ?? false, + ...searchKey(type) }, 1); }); }; diff --git a/kahuna/public/js/upload/controller.css b/kahuna/public/js/upload/controller.css index 3cbbd621f28..bc1a76916bf 100644 --- a/kahuna/public/js/upload/controller.css +++ b/kahuna/public/js/upload/controller.css @@ -1,5 +1,5 @@ .unauthorised-message { text-align: center; font-size: 3rem; - margin-top: 50px; + margin: 50px 20px 0 20px; } \ No newline at end of file diff --git a/kahuna/public/js/upload/controller.js b/kahuna/public/js/upload/controller.js index 8ed693ce051..43fe44075ed 100644 --- a/kahuna/public/js/upload/controller.js +++ b/kahuna/public/js/upload/controller.js @@ -4,11 +4,13 @@ import './prompt/prompt'; import './recent/recent-uploads'; import '../services/scroll-position'; +const toNonFreeString = (val) => (val === true || val === 'true') ? 'true' : 'false'; + var upload = angular.module('kahuna.upload.controller', [ - 'kahuna.upload.prompt', - 'kahuna.upload.recent', - 'kahuna.services.scroll-position', - 'util.storage' + 'kahuna.upload.prompt', + 'kahuna.upload.recent', + 'kahuna.services.scroll-position', + 'util.storage' ]); upload.controller('UploadCtrl', ['uploadManager', 'mediaApi', 'scrollPosition', '$scope', 'storage', @@ -23,7 +25,7 @@ upload.controller('UploadCtrl', ['uploadManager', 'mediaApi', 'scrollPosition', $scope.$on("$locationChangeStart", function(event, _, current) { // handle route changes if (current.indexOf("/upload") > -1) { - ctrl.displayWarning(event); + ctrl.displayWarning(event); } }); @@ -34,10 +36,11 @@ upload.controller('UploadCtrl', ['uploadManager', 'mediaApi', 'scrollPosition', }; ctrl.supportEmailLink = window._clientConfig.supportEmail; + ctrl.uploadImagesPermissionSupportEmailLink = window._clientConfig.uploadImagesPermissionSupportEmail; ctrl.systemName = window._clientConfig.systemName; mediaApi.canUserUpload().then(canUpload => { - ctrl.canUpload = canUpload; + ctrl.canUpload = canUpload; }); // TODO: Show multiple jobs? @@ -57,14 +60,14 @@ upload.controller('UploadCtrl', ['uploadManager', 'mediaApi', 'scrollPosition', const showPaid = session.user.permissions.showPaid ? session.user.permissions.showPaid : undefined; const defaultNonFreeFilter = { isDefault: true, - isNonFree: showPaid ? showPaid : false + isNonFree: toNonFreeString(showPaid) }; storage.setJs("defaultNonFreeFilter", defaultNonFreeFilter, true); window.dispatchEvent(new CustomEvent("logoClick", { - detail: {showPaid: defaultNonFreeFilter.isNonFree}, + detail: {showPaid: defaultNonFreeFilter.isNonFree === 'true'}, bubbles: true })); scrollPosition.resetToTop(); }); }; -}]); + }]); diff --git a/kahuna/public/js/upload/jobs/required-metadata-editor.js b/kahuna/public/js/upload/jobs/required-metadata-editor.js index 7e609c21a5c..b0fb1ea12f9 100644 --- a/kahuna/public/js/upload/jobs/required-metadata-editor.js +++ b/kahuna/public/js/upload/jobs/required-metadata-editor.js @@ -59,7 +59,6 @@ jobs.controller('RequiredMetadataEditorCtrl', update(ctrl.resource, cleanMetadata, ctrl.image). then(resource => { ctrl.resource = resource; - setInterval(() => {console.log(ctrl.metadata);}, 5000); }). finally(() => ctrl.saving = false); }; diff --git a/kahuna/public/js/upload/view.html b/kahuna/public/js/upload/view.html index 59ead24bc90..dbe9cef750b 100644 --- a/kahuna/public/js/upload/view.html +++ b/kahuna/public/js/upload/view.html @@ -26,6 +26,6 @@

Your past 50 uploads

-
You are not authorised to upload images. Send an email to {{ctrl.systemName}} team if you think this is incorrect.
+
You are not authorised to upload images. Send an email to support if you think this is incorrect.
- + diff --git a/kahuna/public/js/window.ts b/kahuna/public/js/window.ts index 7d3871a6467..9dd257079cf 100644 --- a/kahuna/public/js/window.ts +++ b/kahuna/public/js/window.ts @@ -8,6 +8,7 @@ declare global { _clientConfig: { rootUri: string; telemetryUri: string; + aiSearchEnabled: boolean; aiSearchResultLimit: number; featureSwitches: Array; interimFilterOptions: Array; diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index d92bc3dcc3f..7b5716c24b2 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -736,7 +736,7 @@ textarea.ng-invalid { display: inline-flex; flex-direction: row; align-items: center; - border-left: 2px solid #565656; + border-left: 1px solid #565656; height: 50px; } @@ -745,7 +745,7 @@ textarea.ng-invalid { } .top-bar-item:last-child { - border-right: 2px solid #565656; + border-right: 1px solid #565656; } .top-bar-item__form { @@ -796,9 +796,6 @@ textarea.ng-invalid { min-width: inherit; } - gr-top-bar-actions { - min-width: 128px; - } .results-toolbar-item .gr-confirm-delete gr-icon-label { padding: 0 @@ -956,7 +953,7 @@ textarea.ng-invalid { border: 1px solid #999; box-sizing: border-box; - min-width: 300px; + min-width: 270px; flex-grow: 1; } @@ -983,7 +980,7 @@ textarea.ng-invalid { display: flex; flex-direction: row; justify-content: flex-start; - margin-left: 25px; + margin-left: 10px; align-items: center; /* vertical align all filters */ } @@ -1053,7 +1050,7 @@ textarea.ng-invalid { } } -@media screen and (max-width: 1380px) { +@media screen and (max-width: 1400px) { .search__modifier-toggle { display: flex; flex-direction: row; @@ -1195,10 +1192,13 @@ textarea.ng-invalid { .search__advanced-toggle { margin-left: 10px; + padding: 0 5px 3px; } .search__advanced-toggle:hover { color: white; + background-color: #666666; + border-radius: 2px; } .search__advanced-toggle:focus { @@ -2022,6 +2022,7 @@ FIXME: what to do with touch devices } .easel__canvas { + position: relative; height: calc(100vh - 50px); /* viewport - top-bar */ vertical-align: middle; overflow: hidden; @@ -2031,6 +2032,16 @@ FIXME: what to do with touch devices margin: 10px; } +.easel__loading { + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + color: #999; + font-size: 1.6rem; + white-space: nowrap; +} + .easel__image-container { /*style rules also affect the cropper preview*/ display: inline-block; @@ -2082,6 +2093,7 @@ FIXME: what to do with touch devices right: 0; padding: 10px; user-select: none; + z-index: 1; } @@ -2190,6 +2202,13 @@ FIXME: what to do with touch devices position: relative; } +.image-info__group--expandable { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 10px; +} + .image-info__title { color: #eee; } diff --git a/leases/app/lib/LeaseStore.scala b/leases/app/lib/LeaseStore.scala index 3c06a71f8dc..6e64fd4cbcb 100644 --- a/leases/app/lib/LeaseStore.scala +++ b/leases/app/lib/LeaseStore.scala @@ -11,7 +11,7 @@ import scala.concurrent.{ExecutionContext, Future} class LeaseStore(config: LeasesConfig) { - val client = config.withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build() + val client = config.withAWSCredentials(DynamoDbAsyncClient.builder()).build() implicit val dateTimeFormat: Typeclass[DateTime] = DynamoFormat.coercedXmap[DateTime, String, IllegalArgumentException](DateTime.parse, _.toString) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 12c99aaeeae..1f068099831 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -19,6 +19,7 @@ import com.gu.mediaservice.{GridClient, JsonDiff} import com.sksamuel.elastic4s.requests.searches.queries.Query import lib._ import lib.elasticsearch._ +import lib.querysyntax.Condition import org.apache.http.entity.ContentType import org.apache.pekko.stream.scaladsl.StreamConverters import org.http4s.UriTemplate @@ -34,6 +35,14 @@ import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} import scala.util.Try +object MediaApi { + val InDesignIdentity = "InDesign (testing)" + + def shouldSkipUsageRecording(uri: String, user: String): Boolean = + // We don't want to record usages for InDesign downloads for now + uri.contains("download") && user == InDesignIdentity +} + class MediaApi( auth: Authentication, messageSender: ThrallMessageSender, @@ -50,12 +59,12 @@ class MediaApi( )(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition { val services: Services = new Services(config.domainRoot, config.serviceHosts, Set.empty) - val gridClient: GridClient = GridClient(services)(ws) + val gridClient: GridClient = GridClient(services, services.apiBaseUri)(ws) // Process-local cache keyed on normalised query text. Stores the Bedrock Future so that // concurrent requests for the same query share a single in-flight Bedrock call, and // subsequent requests within the TTL window skip Bedrock entirely. - private val embeddingCache: AsyncLoadingCache[String, List[Float]] = Scaffeine() + private val embeddingCache: AsyncLoadingCache[String, List[Double]] = Scaffeine() .maximumSize(config.aiSearchEmbeddingCacheMaxSize) .buildAsyncFuture((normQuery: String) => embedder.createQueryEmbedding(normQuery)(MarkerMap()) @@ -309,8 +318,12 @@ class MediaApi( val maybeResult = for { export <- source.exports.find(_.id.contains(exportId)) asset <- export.assets.find(_.dimensions.exists(_.width == width)) - s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, asset.file)).toOption - file = StreamConverters.fromInputStream(() => s3Object.getObjectContent) + s3Res = Try(s3Client.getObject(config.imgPublishingBucket, asset.file)) + _ = s3Res.failed.foreach { ex => + logger.error("Failed to fetch S3 object", ex) + } + s3Object <- s3Res.toOption + file = StreamConverters.fromInputStream(() => s3Object) entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name)) result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset, config.shortenDownloadFilename)) } yield { @@ -404,15 +417,19 @@ class MediaApi( && ImageExtras.userMayUndeleteImage(request.user, image, authorisation) => logger.info(logMarker, s"undeleting image $id") - softDeletedMetadataTable.updateStatus(id, isDeleted = false) - .map { _ => - messageSender.publish( - UpdateMessage( - subject = UnSoftDeleteImage, - id = Some(id) - ) + for { + // Take an "undeletion" as a marker that the file should be kept and not reaped. + // Mark as archived before undeleting, to make sure the reaper doesn't immediately reap it. + _ <- gridClient.putArchived(id, auth.getOnBehalfOfPrincipal(request.user)) + _ <- softDeletedMetadataTable.updateStatus(id, isDeleted = false) + _ = messageSender.publish( + UpdateMessage( + subject = UnSoftDeleteImage, + id = Some(id) ) - }.map { _ => Accepted } + ) + } yield Accepted + case Some(image) if isVisibleToAccessor(request.user, image) => logger.info(logMarker, s"user ${request.user.accessor.identity} was not permitted to undelete image $id") Future.successful(ImageDeleteForbidden) @@ -433,7 +450,7 @@ class MediaApi( logger.info(logMarker, s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}") mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType) val s3Object = s3Client.getObject(config.imageBucket, image.source.file) - val file = StreamConverters.fromInputStream(() => s3Object.getObjectContent) + val file = StreamConverters.fromInputStream(() => s3Object) val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name)) if(config.recordDownloadAsUsage) { @@ -509,29 +526,33 @@ class MediaApi( user: String, partnerName: Option[String] = None, startPending: Option[String] = None, - )(implicit logMarker: LogMarker) = { - - val baseRequest = ws.url(uri) - .withHttpHeaders(Authentication.originalServiceHeaderName -> config.appName, - HttpHeaders.ORIGIN -> config.rootUri, - HttpHeaders.CONTENT_TYPE -> ContentType.APPLICATION_JSON.getMimeType) - - val request = onBehalfOfPrincipal(baseRequest) - - val usagesMetadata = uri match { - case s if s.contains("download") => Map("mediaId" -> mediaId, - "dateAdded" -> printDateTime(DateTime.now()), - "downloadedBy" -> user) - case s if s.contains("syndication") => Map("mediaId" -> mediaId, - "dateAdded" -> printDateTime(DateTime.now()), - "syndicatedBy" -> user, - "startPending" -> startPending.getOrElse("false"), - "partnerName" -> partnerName.getOrElse( - throw new IllegalArgumentException("partnerName required for SyndicationUsageRequest")) - ) + )(implicit logMarker: LogMarker): Future[Unit] = { + if (MediaApi.shouldSkipUsageRecording(uri, user)) { + logger.info(logMarker, s"Skipping usages request to $uri for $user") + Future.successful(()) + } else { + val baseRequest = ws.url(uri) + .withHttpHeaders(Authentication.originalServiceHeaderName -> config.appName, + HttpHeaders.ORIGIN -> config.rootUri, + HttpHeaders.CONTENT_TYPE -> ContentType.APPLICATION_JSON.getMimeType) + + val request = onBehalfOfPrincipal(baseRequest) + + val usagesMetadata = uri match { + case s if s.contains("download") => Map("mediaId" -> mediaId, + "dateAdded" -> printDateTime(DateTime.now()), + "downloadedBy" -> user) + case s if s.contains("syndication") => Map("mediaId" -> mediaId, + "dateAdded" -> printDateTime(DateTime.now()), + "syndicatedBy" -> user, + "startPending" -> startPending.getOrElse("false"), + "partnerName" -> partnerName.getOrElse( + throw new IllegalArgumentException("partnerName required for SyndicationUsageRequest")) + ) + } + logger.info(logMarker, s"Making usages request to $uri") + request.post(Json.toJson(Map("data" -> usagesMetadata))).map(_ => ()) //fire and forget } - logger.info(logMarker, s"Making usages request to $uri") - request.post(Json.toJson(Map("data" -> usagesMetadata))) //fire and forget } @@ -579,12 +600,10 @@ class MediaApi( case object TextSearch extends AiSearchMode case class SimilarSearch(imageId: String) extends AiSearchMode - def parseAiSearchMode(params: SearchParams): AiSearchMode = - params.aiQueryParts.similarImageId.map(SimilarSearch).getOrElse(TextSearch) - - def buildAiFilter(params: SearchParams): Option[Query] = { - val filterConditions = params.aiQueryParts.filterConditions + def parseAiSearchMode(parts: AiQueryParts): AiSearchMode = + parts.similarImageId.map(SimilarSearch).getOrElse(TextSearch) + def buildAiFilter(filterConditions: List[Condition], params: SearchParams): Option[Query] = { val chipFilterOpt = if (filterConditions.nonEmpty) Some(elasticSearch.queryBuilder.makeQuery(filterConditions)) else None @@ -601,107 +620,153 @@ class MediaApi( def emptyAiSearchResponse = Future.successful(respondCollection(List.empty[EmbeddedEntity[JsValue]], Some(0), Some(0), None, List())) + // Response for an AI search that only has filters (no text/similar-image query to + // rank by). Rather than erroring, we return an empty result set annotated with how + // many images are in the pool a query would rank over, so the client can prompt the + // user to add a query. + def filtersOnlyAiSearchResponse(filterConditions: List[Condition], params: SearchParams): Future[Result] = { + val filteredPoolFilter = buildAiFilter(filterConditions, params) + + for { + (filteredPool, extraCounts) <- elasticSearch.countMatchingFilterWithExtraCounts(filteredPoolFilter) + } yield respondCollection( + data = List.empty[EmbeddedEntity[JsValue]], + offset = Some(0), + total = Some(0), + maybeExtraCounts = Some(extraCounts.copy( + filterPoolCounts = Some(FilterPoolCounts(filteredPool = filteredPool)) + )), + links = Nil + ) + } + def aiSearchResponseFromResults(searchResults: SearchResults): Result = { val imageEntities = searchResults.hits map (hitToImageEntity _).tupled respondCollection( data = imageEntities, offset = Some(0), total = Some(searchResults.total), - maybeExtraCounts = None, + maybeExtraCounts = searchResults.extraCounts, links = Nil ) } - def semanticSearchByImage(imageId: String, k: Int, params: SearchParams): Future[SearchResults] = { - val filterOpt = buildAiFilter(params) + def semanticSearchByImage(imageId: String, k: Int, parts: AiQueryParts, params: SearchParams): Future[SearchResults] = { + val outerMarker = logMarker - for { - maybeImage <- elasticSearch.getImageById(imageId) - maybeEmbedding = maybeImage - .filter(image => isVisibleToAccessor(request.user, image)) - .flatMap(_.embedding) - .flatMap(_.cohereEmbedV4) - .map(_.image.map(_.toFloat)) - searchResults <- maybeEmbedding match { - // If we have an embedding, perform the KNN search. If not, return an empty result set. - case Some(embedding) => - elasticSearch.knnSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) - case None => - Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) - } - } yield searchResults + { + implicit val logMarker: LogMarker = outerMarker ++ Map( + "imageId" -> imageId, + "aiSearchType" -> "image" + ) + + val filterOpt = buildAiFilter(parts.filterConditions, params) + + // Compute the filtered pool total and ticker count badges in parallel with the + // KNN search, so similar-image results show the same "Best k of N matches" total + // and tickers as text/hybrid AI search. + val filterTotalAndCounts = elasticSearch.countMatchingFilterWithExtraCounts(filterOpt) + + for { + maybeImage <- elasticSearch.getImageById(imageId) + maybeEmbedding = maybeImage + .filter(image => isVisibleToAccessor(request.user, image)) + .flatMap(_.embedding) + .flatMap(_.cohereEmbedV4) + .map(_.image) + searchResults <- maybeEmbedding match { + // If we have an embedding, perform the KNN search. If not, return an empty result set. + case Some(embedding) => + elasticSearch.semanticSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) + case None => + Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) + } + (total, extraCounts) <- filterTotalAndCounts + } yield searchResults.copy(total = total, extraCounts = Some(extraCounts)) + } } - def semanticSearchByText(k: Int, params: SearchParams): Future[SearchResults] = { + def semanticSearchByText(k: Int, parts: AiQueryParts, params: SearchParams): Future[SearchResults] = { // Separate the chips from the main query text // So that we can embed just the query text - params.aiQueryParts.semanticQuery match { + parts.semanticQuery match { case None => logger.info(logMarker, s"No semantic query found in structured query; returning no AI results") Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) case Some(semanticQuery) => - // Normalise key so that "Dogs" and "dogs " share a cache entry. - val cacheKey = semanticQuery.trim.toLowerCase - val markers = logMarker ++ Map("query" -> semanticQuery) - - if (embeddingCache.getIfPresent(cacheKey).isDefined) { - logger.info(markers, s"AI search embedding cache hit query=$semanticQuery") - } else { - logger.info(markers, s"AI search embedding cache miss query=$semanticQuery") - } - - val weight = params.vecWeight.getOrElse(1.0) - - // cache.get(key) is atomic: if two requests race on the same key, only one - // load fires and both callers receive the same Future. - val embeddingFuture = embeddingCache.get(cacheKey) + val vecWeight = params.vecWeight.getOrElse(0.85) + val outerMarker = logMarker + + { + implicit val logMarker: LogMarker = outerMarker ++ Map( + "query" -> semanticQuery, + "vecWeight" -> vecWeight, + "aiSearchType" -> "hybrid" + ) - val filterOpt = buildAiFilter(params) + // Normalise key so that "Dogs" and "dogs " share a cache entry. + val cacheKey = semanticQuery.trim.toLowerCase + if (embeddingCache.getIfPresent(cacheKey).isDefined) { + logger.info(logMarker, s"AI search embedding cache hit query=$semanticQuery") + } else { + logger.info(logMarker, s"AI search embedding cache miss query=$semanticQuery") + } - logger.info(markers, s"vecWeight for query '$semanticQuery' is $weight") - for { - embedding <- embeddingFuture - searchResults <- elasticSearch.hybridSearch( - query = semanticQuery, - queryEmbedding = embedding, - k = k, - numCandidates = Math.max(k * 2, 100), - vecWeight = weight, - filterOpt = filterOpt - ) - } yield searchResults + // cache.get(key) is atomic: if two requests race on the same key, only one + // load fires and both callers receive the same Future. + val embeddingFuture = embeddingCache.get(cacheKey) + + val filterOpt = buildAiFilter(parts.filterConditions, params) + + for { + embedding <- embeddingFuture + searchResults <- elasticSearch.hybridSearch( + query = semanticQuery, + queryEmbedding = embedding, + k = k, + numCandidates = Math.max(k * 2, 100), + vecWeight = vecWeight, + filterOpt = filterOpt + ) + } yield searchResults + } } } def performAiSearchAndRespond(params: SearchParams): Future[Result] = { - params.aiQueryParts.validationError match { - case Left(errorMessage) => - logger.info(logMarker, s"Invalid AI search query: $errorMessage") - Future.successful(respondError(UnprocessableEntity, "invalid-ai-search", errorMessage)) - case scala.util.Right(_) => - val k = config.aiSearchResultLimit - val searchResultsFuture = parseAiSearchMode(params) match { - case SimilarSearch(imageId) => semanticSearchByImage(imageId, k, params) - case TextSearch => semanticSearchByText(k, params) + params.aiQueryParts match { + case scala.util.Right(parts) => + val k = Math.min(params.length, config.aiSearchResultLimit) + val searchResultsFuture = parseAiSearchMode(parts) match { + case SimilarSearch(imageId) => semanticSearchByImage(imageId, k, parts, params) + case TextSearch => semanticSearchByText(k, parts, params) } searchResultsFuture.map(aiSearchResponseFromResults) + + // No query to rank by, so we can't return ranked results. Instead return the + // size of the pool we'd be searching over, so the client can prompt the user to + // add a query. + case Left(AiQueryError.NoRankingSignal(filterConditions)) => + logger.info(logMarker, s"AI search with no query to rank by; returning filter pool counts: $filterConditions") + filtersOnlyAiSearchResponse(filterConditions, params) + + case Left(AiQueryError.ConflictingRankingSignals) => + logger.info(logMarker, "AI search combining a similar image with a text query; rankings can't be merged") + Future.successful(respondError( + UnprocessableEntity, + InvalidUriParams.errorKey, + AiQueryError.ConflictingRankingSignals.message + )) } } - if (_searchParams.useAISearch.contains(true)) { - _searchParams.query match { - // Short-circuit polling requests (length=0) and empty queries to avoid unnecessary Bedrock/KNN calls - case _ if _searchParams.length == 0 => - emptyAiSearchResponse - case Some(q) if !q.isBlank => - performAiSearchAndRespond(_searchParams) - // Empty queries do not make sense for AI search as we can - // only rank results once we have a meaningful vector to compare with. - // So return 0 results if the query was empty. - case _ => - emptyAiSearchResponse - } + if (_searchParams.useAISearch.contains(true) && config.aiSearchEnabled) { + // Short-circuit polling requests (length=0) to avoid unnecessary Bedrock/KNN calls. + // Everything else is handled by performAiSearchAndRespond, which falls back to + // filter-pool counts when there's no query to rank by. + if (_searchParams.length == 0) emptyAiSearchResponse + else performAiSearchAndRespond(_searchParams) } else { val searchParams = if (canViewDeletedImages) { _searchParams.copy(uploadedBy = Some(Authentication.getIdentity(request.user))) diff --git a/media-api/app/controllers/UsageController.scala b/media-api/app/controllers/UsageController.scala index 4db59c979be..61afccc559b 100644 --- a/media-api/app/controllers/UsageController.scala +++ b/media-api/app/controllers/UsageController.scala @@ -6,8 +6,11 @@ import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.lib.play.RequestLoggingFilter import com.gu.mediaservice.model.Agencies +import com.gu.mediaservice.model.usage.Usage import lib._ -import lib.elasticsearch.ElasticSearch +import lib.elasticsearch.{ElasticSearch, InvalidUriParams, SearchParams} +import lib.elasticsearch.SearchParams.parseIntFromQuery +import lib.querysyntax.Parser import play.api.mvc.Security.AuthenticatedRequest import play.api.mvc._ @@ -18,8 +21,6 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc override val controllerComponents: ControllerComponents)(implicit val ec: ExecutionContext) extends BaseController with ArgoHelpers { - val numberOfDayInPeriod = 30 - def bySupplier = auth.async { implicit request => implicit val logMarker: LogMarker = MarkerMap( "requestType" -> "usage-by-supplier", @@ -27,9 +28,9 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc ) ++ RequestLoggingFilter.loggablePrincipal(request.user) Future.sequence( - Agencies.all.keys.map(elasticSearch.usageForSupplier(_, numberOfDayInPeriod))) + Agencies.all.keys.map(elasticSearch.usageForSupplier(_, UsageStore.countPeriodInDays))) .map(_.toList) - .map((s: List[SupplierUsageSummary]) => respond(s)) + .map((s: List[SupplierQuotaCount]) => respond(s)) .recover { case e => respondError(InternalServerError, "unknown-error", e.toString) } @@ -42,15 +43,29 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc "imageId" -> id, ) ++ RequestLoggingFilter.loggablePrincipal(request.user) - elasticSearch.usageForSupplier(id, numberOfDayInPeriod) - .map((s: SupplierUsageSummary) => respond(s)) + elasticSearch.usageForSupplier(id, UsageStore.countPeriodInDays) + .map((s: SupplierQuotaCount) => respond(s)) .recover { case e => respondError(InternalServerError, "unknown-error", e.toString) } } - def usageStatusForImage(id: String)(implicit logMarker: LogMarker): Future[UsageStatus] = for { + def quotaCountForSupplier(id: String) = auth.async { implicit request => + implicit val logMarker: LogMarker = MarkerMap( + "requestType" -> "quota-count-for-supplier", + "requestId" -> RequestLoggingFilter.getRequestId(request), + "supplierId" -> id, + ) ++ RequestLoggingFilter.loggablePrincipal(request.user) + + elasticSearch.quotaCountBySupplier(id, UsageStore.countPeriodInDays) + .map((s: SupplierQuotaCount) => respond(s)) + .recover { + case e => respondError(InternalServerError, "unknown-error", e.toString) + } + } + + def usageStatusForImage(id: String)(implicit logMarker: LogMarker): Future[SupplierUsageStatus] = for { imageOption <- elasticSearch.getImageById(id) image <- Future { imageOption.get } @@ -69,7 +84,7 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc ) ++ RequestLoggingFilter.loggablePrincipal(request.user) usageStatusForImage(id) - .map((u: UsageStatus) => respond(u)) + .map((u: SupplierUsageStatus) => respond(u)) .recover { case e: ImageNotFound => respondError(NotFound, "image-not-found", e.toString) case e => respondError(InternalServerError, "unknown-error", e.toString) @@ -90,4 +105,29 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc respondError(InternalServerError, "unknown-error", e.toString) } } + + def imageUsagesBySupplier(id: String) = auth.async { implicit request => + implicit val logMarker: LogMarker = MarkerMap( + "requestType" -> "images-by-supplier", + "requestId" -> RequestLoggingFilter.getRequestId(request), + "supplierId" -> id, + ) ++ RequestLoggingFilter.loggablePrincipal(request.user) + + val structuredQuery = request.getQueryString("q").map(Parser.run).getOrElse(List.empty) + val searchParams = SearchParams(request) + + SearchParams.validate(searchParams) match { + case Left(errors) => + Future.successful(respondError(BadRequest, InvalidUriParams.errorKey, errors.map(_.message).mkString(", "))) + case Right(_) => + elasticSearch.imageUsagesBySupplier(id, structuredQuery, searchParams.offset, searchParams.length) + .map { result => + respondCollection(result.images, Some(searchParams.offset.toLong), Some(result.total)) + } + .recover { + case e: IllegalArgumentException => respondError(BadRequest, InvalidUriParams.errorKey, e.getMessage) + case e => respondError(InternalServerError, "unknown-error", e.toString) + } + } + } } diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala index fb50aa33511..04c1fa95e41 100644 --- a/media-api/app/lib/ImageResponse.scala +++ b/media-api/app/lib/ImageResponse.scala @@ -428,8 +428,10 @@ object ImageResponse { config.fieldAliasConfigs.flatMap { config => val parts = config.elasticsearchPath.split('.').toList.filter(_.nonEmpty) val lookupResult = nestedLookup(JsDefined(source.source), parts) - lookupResult.toOption.map { - config.alias -> _ + lookupResult.toOption match { + case Some(value) => Some(config.alias -> value) + case None if config.matchViaExistence => Some(config.alias -> JsBoolean(false)) + case None => None } } } diff --git a/media-api/app/lib/ImageUsagesBySupplier.scala b/media-api/app/lib/ImageUsagesBySupplier.scala new file mode 100644 index 00000000000..f3370643d55 --- /dev/null +++ b/media-api/app/lib/ImageUsagesBySupplier.scala @@ -0,0 +1,21 @@ +package lib + +import com.gu.mediaservice.model.{Agency, CommissionedAgency, Composite, UsageRights} +import com.gu.mediaservice.model.usage.Usage +import play.api.libs.json.{Json, OFormat} + +case class ImageUsagesBySupplier(id: String, supplier: String, usages: List[Usage]) +object ImageUsagesBySupplier { + def apply(id: String, usageRights: UsageRights, usages: List[Usage]): ImageUsagesBySupplier = { + val supplier = usageRights match { + case c: Composite => c.suppliers + case a: Agency => a.supplier + case ca: CommissionedAgency => ca.supplier + case u => "" + } + ImageUsagesBySupplier(id, supplier, usages) + } + implicit val jsonFormat: OFormat[ImageUsagesBySupplier] = Json.format[ImageUsagesBySupplier] +} + +case class ImageUsagesBySupplierResult(images: List[ImageUsagesBySupplier], total: Long) diff --git a/media-api/app/lib/UsageStore.scala b/media-api/app/lib/UsageStore.scala index ea999fb2f93..d4cb1e5546d 100644 --- a/media-api/app/lib/UsageStore.scala +++ b/media-api/app/lib/UsageStore.scala @@ -6,6 +6,7 @@ import java.util.Properties import com.gu.mediaservice.lib.BaseStore import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.model.{Agencies, Agency, UsageRights} +import com.gu.mediaservice.model.usage.{DigitalUsage, PrintUsage, PublishedUsageStatus, RemovedUsageStatus, UnknownUsageStatus, Usage, UsageStatus, UsageType} import javax.mail.Session import javax.mail.internet.{MimeBodyPart, MimeMultipart} import org.apache.commons.mail.util.MimeMessageUtils @@ -30,30 +31,30 @@ object SupplierUsageQuota { )(SupplierUsageQuota.apply _) } -case class SupplierUsageSummary(agency: Agency, count: Long) -object SupplierUsageSummary { - implicit val customReads: Reads[SupplierUsageSummary] = ( +case class SupplierQuotaCount(agency: Agency, count: Long) +object SupplierQuotaCount { + implicit val customReads: Reads[SupplierQuotaCount] = ( (__ \ "Supplier").read[String].map(Agency(_)) ~ (__ \ "Usage").read[Long] - )(SupplierUsageSummary.apply _) + )(SupplierQuotaCount.apply _) - implicit val writes: Writes[SupplierUsageSummary] = ( + implicit val writes: Writes[SupplierQuotaCount] = ( (__ \ "agency").write[String].contramap((a: Agency) => a.supplier) ~ (__ \ "count").write[Long] - )(unlift(SupplierUsageSummary.unapply)) + )(unlift(SupplierQuotaCount.unapply)) } -case class UsageStatus( +case class SupplierUsageStatus( exceeded: Boolean, fractionOfQuota: Float, - usage: SupplierUsageSummary, + usage: SupplierQuotaCount, quota: Option[SupplierUsageQuota] ) -object UsageStatus { - implicit val writes: Writes[UsageStatus] = Json.writes[UsageStatus] +object SupplierUsageStatus { + implicit val writes: Writes[SupplierUsageStatus] = Json.writes[SupplierUsageStatus] } -case class StoreAccess(store: Map[String, UsageStatus], lastUpdated: DateTime) +case class StoreAccess(store: Map[String, SupplierUsageStatus], lastUpdated: DateTime) object StoreAccess { import play.api.libs.json.JodaWrites._ @@ -61,6 +62,10 @@ object StoreAccess { } object UsageStore extends GridLogging { + val countQualifyingStatuses: Set[UsageStatus] = Set(PublishedUsageStatus, UnknownUsageStatus, RemovedUsageStatus) + val countQualifyingPlatforms: Set[UsageType] = Set(PrintUsage, DigitalUsage) + val countPeriodInDays: Int = 30 + def extractEmail(stream: InputStream): List[String] = { val s = Session.getDefaultInstance(new Properties()) val message = MimeMessageUtils.createMimeMessage(s, stream) @@ -88,7 +93,7 @@ object UsageStore extends GridLogging { } } - def csvParser(list: List[String]): List[SupplierUsageSummary] = { + def csvParser(list: List[String]): List[SupplierQuotaCount] = { def stripQuotes(s: String): String = s .stripSuffix("\"") .stripPrefix("\"") @@ -108,7 +113,7 @@ object UsageStore extends GridLogging { case Some("Cpro Name" :: "Id" :: Nil) => lines.tail.map { case supplier :: count :: Nil => - SupplierUsageSummary(Agency(supplier), count.toInt) + SupplierQuotaCount(Agency(supplier), count.toInt) case _ => logger.error("CSV body error. Expected 2 columns") @@ -132,10 +137,10 @@ class UsageStore( bucket: String, config: MediaApiConfig, quotaStore: QuotaStore -)(implicit val ec: ExecutionContext) extends BaseStore[String, UsageStatus](bucket, config) with GridLogging { +)(implicit val ec: ExecutionContext) extends BaseStore[String, SupplierUsageStatus](bucket, config) with GridLogging { import UsageStore._ - def getUsageStatusForUsageRights(usageRights: UsageRights): Future[UsageStatus] = { + def getUsageStatusForUsageRights(usageRights: UsageRights): Future[SupplierUsageStatus] = { usageRights match { case agency: Agency => Future.successful(store.get().getOrElse(agency.supplier, { throw NoUsageQuota() })) case _ => Future.failed(new Exception("Image is not supplied by Agency")) @@ -154,7 +159,7 @@ class UsageStore( store.set(fetchUsage) } - private def fetchUsage: Map[String, UsageStatus] = { + private def fetchUsage: Map[String, SupplierUsageStatus] = { logger.info("Updating usage store") val maybeLines: Option[List[String]] = getLatestS3Stream.map(extractEmail) @@ -163,9 +168,9 @@ class UsageStore( case None => Map.empty case Some(lines) => logger.info(s"Last usage file has ${lines.length} lines") - val summary: List[SupplierUsageSummary] = csvParser(lines) + val summary: List[SupplierQuotaCount] = csvParser(lines) - def copyAgency(supplier: SupplierUsageSummary, id: String) = Agencies.all.get(id) + def copyAgency(supplier: SupplierQuotaCount, id: String) = Agencies.all.get(id) .map(a => supplier.copy(agency = a)) .getOrElse(supplier) @@ -184,12 +189,12 @@ class UsageStore( .groupBy(_.agency.supplier) .view .mapValues(_.head) - .mapValues((summary: SupplierUsageSummary) => { + .mapValues((summary: SupplierQuotaCount) => { val quota = summary.agency.id.flatMap(id => supplierQuota.get(id)) val exceeded = quota.exists(q => summary.count > q.count) val fractionOfQuota: Float = quota.map(q => summary.count.toFloat / q.count).getOrElse(0F) - UsageStatus( + SupplierUsageStatus( exceeded, fractionOfQuota, summary, diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 0d890ba598a..35a514edc65 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -2,16 +2,19 @@ package lib.elasticsearch import org.apache.pekko.actor.Scheduler import com.gu.mediaservice.lib.ImageFields +import com.gu.mediaservice.lib.formatting.printDateTime import com.gu.mediaservice.lib.argo.model.{ExtraCount, ExtraCountConfig, ExtraCounts} import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.elasticsearch.{CompletionPreview, ElasticSearchClient, ElasticSearchConfig, MigrationStatusProvider, Running} -import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap} +import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap, Stopwatch, combineMarkers} import com.gu.mediaservice.lib.metrics.FutureSyntax import com.gu.mediaservice.model.{Agencies, Agency, AwaitingReviewForSyndication, Image} -import com.sksamuel.elastic4s.ElasticDsl +import com.gu.mediaservice.model.usage.{ComposerUsageReference, DigitalUsage, FrontUsageReference, InDesignUsageReference, PrintUsage, PublishedUsageStatus, RemovedUsageStatus, Usage, UnknownUsageStatus, UsageStatus, UsageType} +import com.sksamuel.elastic4s.{ElasticDsl, Hit} import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.requests.common.Operator.And +import com.sksamuel.elastic4s.requests.common.Operator +import com.sksamuel.elastic4s.requests.common.Operator.Or import com.sksamuel.elastic4s.requests.get.{GetRequest, GetResponse} import com.sksamuel.elastic4s.requests.script.{Script, ScriptField} import com.sksamuel.elastic4s.requests.searches._ @@ -20,11 +23,11 @@ import com.sksamuel.elastic4s.requests.searches.aggs.responses.Aggregations import com.sksamuel.elastic4s.requests.searches.aggs.responses.bucket.{DateHistogram, Terms} import com.sksamuel.elastic4s.requests.searches.queries.Query import com.sksamuel.elastic4s.requests.searches.knn.Knn -import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery import com.sksamuel.elastic4s.requests.searches.queries.matches.MultiMatchQueryBuilderType.BEST_FIELDS import com.sksamuel.elastic4s.requests.searches.queries.matches.{FieldWithOptionalBoost, MultiMatchQuery} -import lib.querysyntax.{HierarchyField, Match, Parser, Phrase} -import lib.{MediaApiConfig, MediaApiMetrics, SupplierUsageSummary} +import lib.elasticsearch.ResultSource.{Both, Lexical, Semantic} +import lib.querysyntax.{Condition, DateRange, HierarchyField, Match, Nested, Parser, Phrase, SingleField} +import lib.{MediaApiConfig, MediaApiMetrics, SupplierQuotaCount, ImageUsagesBySupplier, ImageUsagesBySupplierResult, UsageStore} import play.api.libs.json.{JsError, JsObject, JsSuccess, Json} import play.api.mvc.AnyContent import play.api.mvc.Security.AuthenticatedRequest @@ -179,96 +182,145 @@ class ElasticSearch( } } - def knnSearch(queryEmbedding: List[Float], k: Int, numCandidates: Int, filterOpt: Option[Query]) - (implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { - if (!includeDenseVectorMappings) { - logger.warn(logMarker, "knnSearch called but includeDenseVectorMappings=false, returning empty results") - Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) - } else { - val knn = Knn("embedding.cohereEmbedV4.image", filter = filterOpt) - .queryVector(queryEmbedding.map(_.toDouble)) - .k(k) - .numCandidates(numCandidates) - - val searchRequest = ElasticDsl.search(imagesCurrentAlias) - .knn(knn) - .size(k) - - executeAndLog(withSearchQueryTimeout(searchRequest), "knn search").map { r => - val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) - SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) - } - } - } - - private def createMultiMatchQuery(query: String, boost: Option[Double] = None): MultiMatchQuery = + private def createMultiMatchQuery(query: String, boost: Option[Double] = None, operator: Operator): MultiMatchQuery = MultiMatchQuery( text = query, fields = matchFields.map(field => FieldWithOptionalBoost(field, None)), `type` = Some(BEST_FIELDS), fuzziness = Some("AUTO"), maxExpansions = Some(50), - operator = Some(And), + operator = Some(operator), prefixLength = Some(1), boost = boost ) - // BM25 scores are unbounded [0,inf] and typically much larger in magnitude - // than cosine similarity (knn). So we get the max BM25 score for the query and use that to calculate - // the scaling factor for the lexical part of the query, so that BM25 and knn scores are both between 0-1 scale - // and can be effectively combined in a hybrid query. - private def fetchMaxBm25Score(query: String, filterOpt: Option[Query])(implicit ex: ExecutionContext, logMarker: LogMarker): Future[Double] = { - val baseQuery = createMultiMatchQuery(query) - val filteredQuery = filterOpt.map(filter => boolQuery().must(baseQuery).filter(filter)).getOrElse(baseQuery) + private def maybeWithFilter(query: Query, filterOpt: Option[Query]): Query = { + filterOpt.map { f => + boolQuery() must (query) filter f + }.getOrElse(query) + } - val maxScoreRequest = ElasticDsl.search(imagesCurrentAlias) - .query(filteredQuery) + private def lexicalRequest( + lexicalQuery: MultiMatchQuery, + k: Int, + filterOpt: Option[Query] + ): SearchRequest = + ElasticDsl + .search(imagesCurrentAlias) + .query(maybeWithFilter(lexicalQuery, filterOpt)) + .size(k) + + private def lexicalSearch( + query: String, + k: Int, + filterOpt: Option[Query] + )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { + val searchRequest = lexicalRequest(createMultiMatchQuery(query, operator = Or), k, filterOpt) - executeAndLog(withSearchQueryTimeout(maxScoreRequest), "max BM25 score").map { r => - logger.info(logMarker, s"Max BM25 score for query '$query' is ${r.result.hits.maxScore}") - if (r.result.hits.hits.isEmpty) 1.0 else r.result.hits.maxScore + executeAndLog(withSearchQueryTimeout(searchRequest), "lexical search").map { r => + val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) + SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) } } - private def makeHybridSearchRequest( - query: String, + private def semanticRequest( + queryEmbedding: List[Double], + k: Int, numCandidates: Int, + filterOpt: Option[Query] + ): SearchRequest = + ElasticDsl + .search(imagesCurrentAlias) + .knn(Knn("embedding.cohereEmbedV4.image", filter = filterOpt) + .queryVector(queryEmbedding) + .k(k) + .numCandidates(numCandidates) + ) + .size(k) + + def semanticSearch( queryEmbedding: List[Double], k: Int, numCandidates: Int, - vecWeight: Double, - maxScore: Double, filterOpt: Option[Query] - )(implicit logMarker: LogMarker): SearchRequest = { - val knn = Knn("embedding.cohereEmbedV4.image", filter = filterOpt) - .queryVector(queryEmbedding) - .k(k) - .numCandidates(numCandidates) - .boost(if (vecWeight > 0.0) 1.0 else 0.0) - - val lexicalWeight = 1.0 - vecWeight - - // KNN results are in [0,1], but BM25 scores are unbounded and typically much - // larger than cosine similarity, so we need to apply a scaling factor to the - // BM25 score to bring it to the same range as the cosine similarity. - val scalingFactor = if (maxScore > 0.0) 1.0 / maxScore else 1.0 - - // We want to apply only one boost if we can help it, so we scale the - // multi_match boost to be in line with the max_score and the desired - // lexical_weight/vec_weight balance - val multiMatchBoost = if (vecWeight > 0.0) (lexicalWeight / vecWeight) * scalingFactor else 1.0 + )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { + if (!includeDenseVectorMappings) { + // TODO: could we factor out this check into semanticRequest or elsewhere? + logger.warn(logMarker, "semanticSearch called but includeDenseVectorMappings=false, returning empty results") + Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) + } else { + val searchRequest = semanticRequest(queryEmbedding, k, numCandidates, filterOpt) - logger.info(logMarker, s"Scaling factor for BM25 score is $scalingFactor, multi-match boost is $multiMatchBoost") + executeAndLog(withSearchQueryTimeout(searchRequest), "semantic search").map { r => + val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) + SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + } + } + } - val multiMatchQuery = createMultiMatchQuery(query, boost = Some(multiMatchBoost)) + // Runs lexical and semantic searches in parallel, fills in the missing scores + // for each result clientside, then combines and re-ranks them. + // This approach was inspired by + // "An Analysis of Fusion Functions for Hybrid Retrieval" + // https://arxiv.org/pdf/2210.11934 + private def fusedLexicalAndSemanticSearch( + query: String, + queryEmbedding: List[Double], + k: Int, + numCandidates: Int, + vecWeight: Double, + filterOpt: Option[Query] + )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { + import HybridResult.{resolveHitAndFillInSemanticScore, fuseAndRank, renderRankedTable} + + val lexicalQuery = createMultiMatchQuery(query, operator = Or) + val lexicalSearchRequest = lexicalRequest(lexicalQuery, k, filterOpt) + + val semanticSearchRequest = semanticRequest(queryEmbedding, k, numCandidates, filterOpt) + .rescore(Rescore(lexicalQuery) + .window(k) + // We want to replace the knn score with the BM25 score, + // because we can calculate cosine similarity clientside, + // but can't do that for BM25. + .originalQueryWeight(0) + .rescoreQueryWeight(1) + ) - ElasticDsl.search(imagesCurrentAlias) - .bool(BoolQuery().should(Seq(multiMatchQuery, knn)).filter(filterOpt)) - .size(k) + // Assigning to vals here eagerly starts both requests, so they run in + // parallel. The for-comprehension below only sequences the *combination* + // of their results, not their execution. + val lexicalSearchResponse = executeAndLog(withSearchQueryTimeout(lexicalSearchRequest), "lexical side of hybrid AI search") + val semanticSearchResponse = executeAndLog(withSearchQueryTimeout(semanticSearchRequest), "semantic side of hybrid AI search") + + for { + lexical <- lexicalSearchResponse + semantic <- semanticSearchResponse + } yield { + val lexicalHits = lexical.result.hits.hits.toList + val semanticHits = semantic.result.hits.hits.toList + logger.info(logMarker, s"Hybrid AI search returned ${lexicalHits.length + semanticHits.length} initial hits: ${lexicalHits.length} lexical, ${semanticHits.length} semantic") + + // Resolve each side to images and fill in the client-side semantic score. + // We keep the two sides separate so that fuseAndRank can tag each result + // with where it originally came from (lexical, semantic, or both). + val lexicalResults = lexicalHits.flatMap(resolveHitAndFillInSemanticScore(_, queryEmbedding, resolveHit)) + val semanticResults = semanticHits.flatMap(resolveHitAndFillInSemanticScore(_, queryEmbedding, resolveHit)) + + val ranked = fuseAndRank(lexicalResults, semanticResults, vecWeight, k) + val counts = ranked.groupBy(_.source).view.mapValues(_.size).toMap.withDefaultValue(0) + logger.info(logMarker, s"Hybrid AI search returned ${ranked.length} hits (k=$k) after fusing and ranking, ${counts(Lexical)} from lexical, ${counts(Semantic)} from semantic, ${counts(Both)} from both") + // This log will be noisy and we can get rid of it at a later stage if we want, + // but I think it could be indispensable if we see weird results and we + // want to understand why they're there. An alternative would be putting it + // into the search response so we could see it in the network tab and even surface + // in the UI if we wanted, but that would be a bigger change. + logger.info(logMarker, s"Hybrid AI search ranked results:\n${renderRankedTable(ranked)}") + SearchResults(hits = ranked.map(r => (r.result.id, r.result.image)), total = ranked.length, extraCounts = None) + } } def hybridSearch( query: String, - queryEmbedding: List[Float], + queryEmbedding: List[Double], k: Int, numCandidates: Int, vecWeight: Double, @@ -281,28 +333,54 @@ class ElasticSearch( logger.warn(logMarker, "hybridSearch called but includeDenseVectorMappings=false, returning empty results") Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) } else { - val queryEmbeddingDouble: List[Double] = queryEmbedding.map(_.toDouble) - - for { - maxScore <- fetchMaxBm25Score(query, filterOpt) - searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore, filterOpt) - result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") - } yield { - val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) - SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + val stopwatch = Stopwatch.start + + // When the weighting is entirely on one side, short-circuit to that side + // alone rather than running both queries and fusing. + val searchResults = vecWeight match { + case 0.0 => lexicalSearch(query, k, filterOpt) + case 1.0 => semanticSearch(queryEmbedding, k, numCandidates, filterOpt) + case _ => fusedLexicalAndSemanticSearch(query, queryEmbedding, k, numCandidates, vecWeight, filterOpt) + } + + // Run in parallel: how many images match the filters in total (so we can + // show "Best k of N matches") along with the ticker count badges, both + // computed over the whole filtered set. + val filterTotalAndCounts = countMatchingFilterWithExtraCounts(filterOpt) + + (for { + results <- searchResults + (total, extraCounts) <- filterTotalAndCounts + } yield results.copy(total = total, extraCounts = Some(extraCounts))).andThen { case _ => + val elapsed = stopwatch.elapsed + logger.info( + combineMarkers(logMarker, elapsed), + s"Hybrid AI search completed in ${elapsed.toMillis} ms" + ) } } } + // How many images match the active filters in total (so we can show + // "Best k of N matches") along with the ticker count badges, both computed + // over the whole filtered set in a single request. + def countMatchingFilterWithExtraCounts(filterOpt: Option[Query])(implicit ex: ExecutionContext, logMarker: LogMarker): Future[(Long, ExtraCounts)] = { + val searchRequest = ElasticDsl.search(imagesCurrentAlias) + .query(filterOpt.getOrElse(matchAllQuery())) + .trackTotalHits(true) + .size(0) + .aggregations(extraCountAggregations) + + executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid AI search filter count and ticker counts").map { r => + (r.result.totalHits, extraCountsFrom(r.result.aggregations)) + } + } + def search(params: SearchParams)(implicit ex: ExecutionContext, request: AuthenticatedRequest[AnyContent, Principal], logMarker: LogMarker = MarkerMap()): Future[SearchResults] = { val query: Query = queryBuilder.makeQuery(params.structuredQuery) val filterOpt: Option[Query] = queryBuilder.buildFilterOpt(params, searchFilters, syndicationFilter) - val withFilter = filterOpt.map { f => - boolQuery() must (query) filter f - }.getOrElse(query) - val sort = params.orderBy match { case Some("dateAddedToCollection") => sorts.dateAddedToCollectionDescending case _ => sorts.createSort(params.orderBy) @@ -333,15 +411,12 @@ class ElasticSearch( Seq.empty } - val searchRequest = prepareSearch(withFilter) + val searchRequest = prepareSearch(maybeWithFilter(query, filterOpt)) .trackTotalHits(trackTotalHits) .runtimeMappings(runtimeMappings) .storedFields("_source") // this needs to be explicit when using script fields .scriptfields(graphicImagesScriptFields) - .aggregations(aggregationsNameToSearchClauseMap.map { - case (name, ExtraCountConfig(searchClause, _, maybeSubAggregation)) => - filterAgg(name, queryBuilder.makeQuery(Parser.run(searchClause))).subAggregations(maybeSubAggregation) - }) + .aggregations(extraCountAggregations) .from(params.offset) .size(params.length) .sortBy(sort) @@ -355,28 +430,37 @@ class ElasticSearch( SearchResults( hits = imageHits, total = if (trackTotalHits) r.result.totalHits else 0, - extraCounts = Some(ExtraCounts( - tickerCounts = aggregationsNameToSearchClauseMap.map { - case (name, ExtraCountConfig(searchClause, backgroundColour, maybeSubAggregation)) => - val aggResult = r.result.aggregations.filter(name) - val maybeSubAggResult = maybeSubAggregation.map(_.name).map(aggResult.result[Terms]) - name -> ExtraCount( - value = aggResult.docCount, - searchClause, - backgroundColour, - subCounts = maybeSubAggResult.map { termsAgg => - ListMap(termsAgg.buckets.sortBy(_.docCount).reverse.map { bucket => - (bucket.key, bucket.docCount) - }: _*) + ("other" -> termsAgg.otherDocCount) - }.filter(_.exists { case (_, count) => count > 0 }) - ) - } - )) + extraCounts = Some(extraCountsFrom(r.result.aggregations)) ) } } - def usageForSupplier(id: String, numDays: Int)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SupplierUsageSummary] = { + // The aggregations used to compute the ticker count badges ("GNM-owned", + // "agency picks" etc) shown above search results. + private def extraCountAggregations = aggregationsNameToSearchClauseMap.map { + case (name, ExtraCountConfig(searchClause, _, maybeSubAggregation)) => + filterAgg(name, queryBuilder.makeQuery(Parser.run(searchClause))).subAggregations(maybeSubAggregation) + } + + private def extraCountsFrom(aggregations: Aggregations): ExtraCounts = ExtraCounts( + tickerCounts = aggregationsNameToSearchClauseMap.map { + case (name, ExtraCountConfig(searchClause, backgroundColour, maybeSubAggregation)) => + val aggResult = aggregations.filter(name) + val maybeSubAggResult = maybeSubAggregation.map(_.name).map(aggResult.result[Terms]) + name -> ExtraCount( + value = aggResult.docCount, + searchClause, + backgroundColour, + subCounts = maybeSubAggResult.map { termsAgg => + ListMap(termsAgg.buckets.sortBy(_.docCount).reverse.map { bucket => + (bucket.key, bucket.docCount) + }: _*) + ("other" -> termsAgg.otherDocCount) + }.filter(_.exists { case (_, count) => count > 0 }) + ) + } + ) + + def usageForSupplier(id: String, numDays: Int)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SupplierQuotaCount] = { val supplier = Agencies.get(id) val supplierName = supplier.supplier @@ -397,7 +481,147 @@ class ElasticSearch( executeAndLog(search, s"$id usage search").map { r => import r.result logSearchQueryIfTimedOut(search, result) - SupplierUsageSummary(supplier, result.hits.total.value) + SupplierQuotaCount(supplier, result.hits.total.value) + } + } + + def quotaCountBySupplier(id: String, numDays: Int)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SupplierQuotaCount] = { + val supplier = Agencies.get(id) + val supplierName = supplier.supplier + + val haveQualifyingStatus = termsQuery("usages.status", UsageStore.countQualifyingStatuses.map(_.toString)) + // lt("now+1d/d") instead of lte("now") so the query is day-rounded and fully request-cacheable + val beInLastPeriod = rangeQuery("usages.dateAdded").gt(s"now-${numDays}d/d").lt("now+1d/d") + val haveQualifyingPlatform = termsQuery("usages.platform", UsageStore.countQualifyingPlatforms.map(_.toString)) + val haveQualifyingUsage = nestedQuery("usages", boolQuery().must(haveQualifyingStatus, haveQualifyingPlatform, beInLastPeriod)) + + val beSupplier = boolQuery().should( + termQuery("usageRights.supplier", supplierName), + matchQuery("usageRights.suppliers", supplierName) + ).minimumShouldMatch(1) + val query = boolQuery().must(matchAllQuery()).filter(boolQuery().must(beSupplier, haveQualifyingUsage)) + + + // Usage-level filters for counting inside the nested aggregation context + val composerUsageFilter = boolQuery().must( + haveQualifyingStatus, beInLastPeriod, + termQuery("usages.platform", DigitalUsage.toString), + termQuery("usages.references.type", ComposerUsageReference.toString) + ) + val frontsUsageFilter = boolQuery().must( + haveQualifyingStatus, beInLastPeriod, + termQuery("usages.platform", DigitalUsage.toString), + termQuery("usages.references.type", FrontUsageReference.toString) + ) + val printUsageFilter = boolQuery().must( + haveQualifyingStatus, beInLastPeriod, + termQuery("usages.platform", PrintUsage.toString) + ) + // Document-level filters: classify images by which quota bucket they fall into. + val hasQualifyingComposer = nestedQuery("usages", composerUsageFilter) + val hasQualifyingFronts = nestedQuery("usages", frontsUsageFilter) + val hasQualifyingPrint = nestedQuery("usages", printUsageFilter) + + val imagesWithComposer = hasQualifyingComposer + val imagesWithFrontsButNoComposer = boolQuery().must(hasQualifyingFronts).withNot(hasQualifyingComposer) + val imagesWithPrintButNoComposer = boolQuery().must(hasQualifyingPrint).withNot(hasQualifyingComposer) + + // Quota counting logic per image: + // - Each Composer usage counts as 1; if any exist, Fronts and Print on the same image are not counted additionally. + // - Multiple Fronts count as 1 in total. + // - Multiple Print usages each count separately. + // Three mutually exclusive aggregations: + // 1. Images with composer: sum of composer usage counts + // 2. Images with fronts, no composer: count of images (each image contributes 1) + // 3. Images with print, no composer: sum of print usage counts + val composerQuotaAgg = filterAgg("has_composer", imagesWithComposer) + .subAggregations( + nestedAggregation("usages_agg", "usages") + .subAggregations( + filterAgg("qualifying", composerUsageFilter) + ) + ) + val frontsQuotaAgg = filterAgg("fronts_no_composer", imagesWithFrontsButNoComposer) + val printQuotaAgg = filterAgg("print_no_composer", imagesWithPrintButNoComposer) + .subAggregations( + nestedAggregation("usages_agg", "usages") + .subAggregations( + filterAgg("qualifying", printUsageFilter) + ) + ) + + val search = prepareSearch(query).size(0).aggs(composerQuotaAgg, frontsQuotaAgg, printQuotaAgg) + + executeAndLog(search, s"$id quota count by supplier search").map { r => + import r.result + logSearchQueryIfTimedOut(search, result) + val aggs = result.aggregations + val composerQuota = aggs.filter("has_composer").nested("usages_agg").filter("qualifying").docCount.toInt + val frontsQuota = aggs.filter("fronts_no_composer").docCount.toInt + val printQuota = aggs.filter("print_no_composer").nested("usages_agg").filter("qualifying").docCount.toInt + logger.info(s"Quota count for supplier $supplierName in last $numDays days: composer=$composerQuota, fronts=$frontsQuota, print=$printQuota") + SupplierQuotaCount(supplier, composerQuota + frontsQuota + printQuota) + } + } + + def imageUsagesBySupplier( + id: String, + structuredQuery: List[Condition] = List.empty, + offset: Int = 0, + length: Int = 10 + )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[ImageUsagesBySupplierResult] = { + if (offset + length > 10000) + return Future.failed(new IllegalArgumentException(s"offset + length cannot exceed 10000 (Elasticsearch result window limit)")) + + val supplier = Agencies.get(id) + val supplierName = supplier.supplier + + val haveQualifyingStatus = termsQuery("usages.status", UsageStore.countQualifyingStatuses.map(_.toString)) + val haveQualifyingPlatform = termsQuery("usages.platform", UsageStore.countQualifyingPlatforms.map(_.toString)) + + // e.g. usages@added:2026-07-01 - each date bound is inclusive, + // and since they all apply within the same nested "usages" entry, multiple bounds combine into a range. + val dateAddedRanges = structuredQuery.collect { + case Nested(SingleField("usages"), SingleField("dateAdded"), DateRange(start, end)) => (start, end) + } + val maybeDateAddedRange = dateAddedRanges match { + case Nil => None + case ranges => + val from = ranges.map(_._1).maxBy(_.getMillis) + // ` rangeQuery("usages.dateAdded").gte(printDateTime(from)).lte(printDateTime(to)) } + val haveQualifyingUsage = nestedQuery("usages", boolQuery().must(qualifyingUsageClauses)) + + val beSupplier = boolQuery().should( + termQuery("usageRights.supplier", supplierName), + matchQuery("usageRights.suppliers", supplierName) + ).minimumShouldMatch(1) + + val query = boolQuery().must(matchAllQuery()).filter(boolQuery().must(beSupplier, haveQualifyingUsage)) + + val search = prepareSearch(query).trackTotalHits(true).from(offset).size(length) + + def isQualifyingUsage(usage: Usage): Boolean = + UsageStore.countQualifyingStatuses.contains(usage.status) && + UsageStore.countQualifyingPlatforms.contains(usage.platform) && + maybeDateAddedRange.forall { case (from, to) => + usage.dateAdded.exists(dateAdded => !dateAdded.isBefore(from) && !dateAdded.isAfter(to)) + } + + executeAndLog(search, s"$id image usages by supplier search").map { r => + import r.result + logSearchQueryIfTimedOut(search, result) + val images = result.hits.hits.toList + .flatMap(resolveHit) + .map(sourceWrapperImage => sourceWrapperImage.instance) + .map(image => ImageUsagesBySupplier(image.id, image.usageRights, image.usages.filter(isQualifyingUsage))) + .distinctBy(_.id) + ImageUsagesBySupplierResult(images, result.totalHits) } } diff --git a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala index e6dd8fed5b0..d984650a2bd 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala @@ -17,43 +17,43 @@ case class SearchResults(hits: Seq[(String, SourceWrapper[Image])], total: Long, case class AggregateSearchResults(results: Seq[BucketResult], total: Long) +// The reasons an AI search can't be run as a ranked query. These are distinct from +// each other so the caller can decide how to respond: an expected "no ranking signal" +// case is handled by reporting filter-pool counts, whereas a genuine error is surfaced +// to the user. +sealed trait AiQueryError +object AiQueryError { + // The query has no KNN ranking signal (no text and no similar image), so there's + // nothing to rank by. We still carry the parsed filter conditions so the caller can + // report how big the filtered pool is and prompt the user to add a query. + final case class NoRankingSignal(filterConditions: List[Condition]) extends AiQueryError + + // The query combines a similar image with a text query. The two rankings can't be + // merged, so this is a genuine error. + case object ConflictingRankingSignals extends AiQueryError { + val message: String = + "AI search can't combine a similar image search with a text query because the two rankings can't be merged. " + + "Please use either a text query or a similar image, not both." + } +} + case class AiQueryParts( semanticQuery: Option[String], filterConditions: List[Condition], similarImageId: Option[String] -) { - def hasSimilarAndSemanticText: Boolean = similarImageId.nonEmpty && semanticQuery.nonEmpty - - // A KNN bit is the only thing AI search can rank by, so a query must contain either - // a text query (text KNN) or a similar image (image KNN) to be valid. - def hasKnn: Boolean = similarImageId.nonEmpty || semanticQuery.nonEmpty - - // Returns Left(error message) if the AI query cannot be satisfied, otherwise Right(()). - def validationError: Either[String, Unit] = { - if (hasSimilarAndSemanticText) - Left( - "AI search can't combine a similar image search with a text query because the two rankings can't be merged. " + - "Please use either a text query or a similar image, not both." - ) - else if (!hasKnn) - Left( - "AI search needs something to rank by. " + - "Add a text query or a similar image alongside your filters." - ) - else - Right(()) - } -} +) object AiQueryParts { - def from(conditions: List[Condition]): AiQueryParts = { + // Parse the structured query into AI search parts. Returns Right with parts ready to + // rank by, or Left explaining why the search can't be run as a ranked query. + def from(conditions: List[Condition]): Either[AiQueryError, AiQueryParts] = { val initial = AiQueryParts( semanticQuery = None, filterConditions = List.empty, similarImageId = None, ) - conditions.foldLeft(initial) { + val parts = conditions.foldLeft(initial) { case (parts, Match(AnyField, Words(value))) => appendSemanticText(parts, value) case (parts, Match(AnyField, Phrase(value))) => @@ -65,6 +65,15 @@ object AiQueryParts { case (parts, condition) => parts.copy(filterConditions = parts.filterConditions :+ condition) } + + val hasText = parts.semanticQuery.nonEmpty + val hasSimilarImage = parts.similarImageId.nonEmpty + + // A KNN bit is the only thing AI search can rank by, so a query must contain exactly + // one of a text query (text KNN) or a similar image (image KNN) to be valid. + if (hasText && hasSimilarImage) Left(AiQueryError.ConflictingRankingSignals) + else if (!hasText && !hasSimilarImage) Left(AiQueryError.NoRankingSignal(parts.filterConditions)) + else Right(parts) } private def appendSemanticText(parts: AiQueryParts, value: String): AiQueryParts = { @@ -149,7 +158,7 @@ case class SearchParams( useAISearch: Option[Boolean] = None, vecWeight: Option[Double] = None ) { - lazy val aiQueryParts: AiQueryParts = AiQueryParts.from(structuredQuery) + lazy val aiQueryParts: Either[AiQueryError, AiQueryParts] = AiQueryParts.from(structuredQuery) } case class InvalidUriParams(message: String) extends Throwable diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala new file mode 100644 index 00000000000..61f3ff0d6f7 --- /dev/null +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -0,0 +1,154 @@ +package lib.elasticsearch + +import com.gu.mediaservice.lib.VectorUtils +import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} +import com.gu.mediaservice.model.Image +import com.sksamuel.elastic4s.requests.searches.SearchHit + +case class HybridResult( + id: String, + lexicalScore: Double, + semanticScore: Double, + image: SourceWrapper[Image] +) + + +sealed trait ResultSource +object ResultSource { + case object Lexical extends ResultSource + case object Semantic extends ResultSource + case object Both extends ResultSource + + def from(inLexical: Boolean, inSemantic: Boolean): ResultSource = + (inLexical, inSemantic) match { + case (true, true) => Both + case (true, false) => Lexical + case (false, true) => Semantic + case _ => throw new IllegalArgumentException("Result must be in either lexical or semantic") + } +} + +case class FusedScore( + normedLexicalScore: Double, + normedSemanticScore: Double, + fusedScore: Double +) + +case class RankedResult( + result: HybridResult, + source: ResultSource, + score: FusedScore +) + +object HybridResult extends GridLogging { + // Theoretical minimum cosine similarity (opposite vectors). + val CosineSimilarityTheoreticalMin: Double = -1.0 + // Theoretical minimum BM25 lexical score. + val Bm25TheoreticalMin: Double = 0.0 + + def resolveHitAndFillInSemanticScore( + hit: SearchHit, + queryEmbedding: List[Double], + resolveHit: SearchHit => Option[SourceWrapper[Image]] + )(implicit logMarker: LogMarker): Option[HybridResult] = + resolveHit(hit).map { image => + val semanticScore = image.instance.embedding + .flatMap(_.cohereEmbedV4) + // We can't use the dot product shortcut because image vectors + // are truncated 256-dim versions of a normalised 1536-dim vector, + // meaning they will not have magnitude 1. + // Note this is true cosine similarity from -1 to 1, + // *not* the ES-normalised score, but when we normalise + // later it will end up in the range 0-1. + .flatMap(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) + // cosineSimilarity is None when there's no usable semantic signal (a + // missing, mismatched, or zero-magnitude embedding), all of which fall + // through to the theoretical min. + .getOrElse { + logger.warn(logMarker, s"Could not calculate semantic score for image ${hit.id}. Falling back to minimum semantic score") + CosineSimilarityTheoreticalMin + } + HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) + } + + // This is the "theoretical min-max" normalisation chosen in + // https://arxiv.org/pdf/2210.11934 + def normalise(score: Double, max: Double, theoreticalMin: Double): Double = + if (max == theoreticalMin) 0.0 + else (score - theoreticalMin) / (max - theoreticalMin) + + def fuseScores( + result: HybridResult, + maxLexicalScore: Double, + maxSemanticScore: Double, + vecWeight: Double + ): FusedScore = { + val normedLexicalScore = normalise(result.lexicalScore, maxLexicalScore, theoreticalMin = Bm25TheoreticalMin) + // The semantic theoretical min of -1 means we do both the max-norming + // and the ES-score norming in one step. + val normedSemanticScore = normalise(result.semanticScore, maxSemanticScore, theoreticalMin = CosineSimilarityTheoreticalMin) + val fusedScore = (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) + FusedScore(normedLexicalScore, normedSemanticScore, fusedScore) + } + + def fuseAndRank( + lexicalResults: List[HybridResult], + semanticResults: List[HybridResult], + vecWeight: Double, + k: Int + ): List[RankedResult] = { + val lexicalIds = lexicalResults.map(_.id).toSet + val semanticIds = semanticResults.map(_.id).toSet + + // De-duplicating by id is safe because, at this point, every result (lexical + // or semantic) carries only the lexical (BM25) score from Elasticsearch, so + // the two copies of a shared result are identical. + val distinctResults = (lexicalResults ::: semanticResults).distinctBy(_.id) + + // Why compute the maxes ourselves rather than reading them from Elasticsearch's + // top result or hits.maxScore? Because for the semantic query we rescore by BM25, + // so neither maxScore nor the top result reflects the max semantic score. + // + // A side benefit is that this also accounts for the rare case in which KNN + // doesn't contain the true closest vector, and that true closest vector + // happens to be among the lexical-only results. + (for { + maxLexicalScore <- distinctResults.map(_.lexicalScore).maxOption + maxSemanticScore <- distinctResults.map(_.semanticScore).maxOption + } yield { + distinctResults + .map { result => + RankedResult( + result = result, + source = ResultSource.from(lexicalIds.contains(result.id), semanticIds.contains(result.id)), + score = fuseScores(result, maxLexicalScore, maxSemanticScore, vecWeight) + ) + } + .sortBy(_.score.fusedScore)(Ordering[Double].reverse) + .take(k) + }).getOrElse(List()) + } + + // Renders the full ranked output as an aligned table, ordered as given + // (i.e. by fused score descending), for logging/debugging. + def renderRankedTable(ranked: List[RankedResult]): String = { + val header = Seq("id", "source", "lexical", "semantic", "normedLex", "normedSem", "fused") + val rows = ranked.map { r => + Seq( + r.result.id, + r.source.toString, + f"${r.result.lexicalScore}%.4f", + f"${r.result.semanticScore}%.4f", + f"${r.score.normedLexicalScore}%.4f", + f"${r.score.normedSemanticScore}%.4f", + f"${r.score.fusedScore}%.4f" + ) + } + val widths = header.indices.map { i => + (header +: rows).map(_(i).length).max + } + (header +: rows) + .map(row => row.zip(widths).map { case (cell, w) => cell.padTo(w, ' ') }.mkString(" ")) + .mkString("\n") + } +} diff --git a/media-api/app/lib/elasticsearch/QueryBuilder.scala b/media-api/app/lib/elasticsearch/QueryBuilder.scala index ed8989282b1..b8932dbc11f 100644 --- a/media-api/app/lib/elasticsearch/QueryBuilder.scala +++ b/media-api/app/lib/elasticsearch/QueryBuilder.scala @@ -17,11 +17,25 @@ import scalaz.syntax.std.list._ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agency], config: MediaApiConfig) extends ImageFields with GridLogging { - def resolveFieldPath(field: String): String = { - config.fieldAliasConfigs.find(_.alias == field) match { - case Some(x) => x.elasticsearchPath - case None => getFieldPath(field) - } + // Users can search by either the friendly alias (e.g. "c2paMetadataAvailable") or the underlying + // Elasticsearch path it resolves to (e.g. "fileMetadata.c2pa.isAvailable"), so field alias configs + // are looked up by whichever of the two was typed. + private def findFieldAliasConfig(field: String) = + config.fieldAliasConfigs.find(fieldAlias => fieldAlias.alias == field || fieldAlias.elasticsearchPath == field) + + def resolveFieldPath(field: String): String = + findFieldAliasConfig(field).map(_.elasticsearchPath).getOrElse(getFieldPath(field)) + + private def isMatchViaExistence(field: String): Boolean = + findFieldAliasConfig(field).exists(_.matchViaExistence) + + private def isBooleanPhrase(value: String): Boolean = + value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false") + + private def booleanValueOf(value: Value): String = value match { + case Words(v) => v + case Phrase(v) => v + case _ => "" } // For some sad reason, there was no helpful alias for this in the ES library @@ -52,6 +66,12 @@ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agenc case AnyField => makeMultiQuery(condition.value, matchFields) case MultipleField(fields) => makeMultiQuery(condition.value, fields) case SingleField(field) => condition.value match { + // Some fields are only ever indexed when true (see FieldAlias.matchViaExistence) - for these, + // translate a literal true/false value query into an exists/not-exists query so both values + // behave intuitively, rather than a literal term match (which could never match "false"). + case v @ (Words(_) | Phrase(_)) if isMatchViaExistence(field) && isBooleanPhrase(booleanValueOf(v)) => + val existsQ = boolQuery().filter(existsQuery(resolveFieldPath(field))) + if (booleanValueOf(v).equalsIgnoreCase("true")) existsQ else boolQuery().not(existsQ) // Force AND operator else it will only require *any* of the words, not *all* case Words(value) => matchQuery(resolveFieldPath(field), value).operator(Operator.AND) diff --git a/media-api/conf/routes b/media-api/conf/routes index 2b22b7b6692..30f5196932f 100644 --- a/media-api/conf/routes +++ b/media-api/conf/routes @@ -28,7 +28,6 @@ DELETE /images/:id controllers. DELETE /images/:id/hard-delete controllers.MediaApi.hardDeleteImage(id: String) PUT /images/:id/undelete controllers.MediaApi.unSoftDeleteImage(id: String) GET /images controllers.MediaApi.imageSearch() - # completion GET /suggest/metadata/credit controllers.SuggestionController.suggestMetadataCredit(q: Option[String], size: Option[Int]) GET /suggest/metadata/photoshoot controllers.SuggestionController.suggestPhotoshoot(q: Option[String], size: Option[Int]) @@ -36,9 +35,10 @@ GET /suggest/metadata/photoshoot controllers. # usage quotas GET /usage/suppliers controllers.UsageController.bySupplier GET /usage/suppliers/:id controllers.UsageController.forSupplier(id: String) +GET /usage/suppliers/:id/images controllers.UsageController.imageUsagesBySupplier(id: String) +GET /usage/suppliers/:id/quotaCount controllers.UsageController.quotaCountForSupplier(id: String) GET /usage/quotas controllers.UsageController.quotas GET /usage/quotas/:id controllers.UsageController.quotaForImage(id: String) - # configuration GET /configuration/crop-variations controllers.ConfigurationController.cropVariations diff --git a/media-api/test/controllers/MediaApiTest.scala b/media-api/test/controllers/MediaApiTest.scala new file mode 100644 index 00000000000..4560b513be8 --- /dev/null +++ b/media-api/test/controllers/MediaApiTest.scala @@ -0,0 +1,37 @@ +package controllers + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +class MediaApiTest extends AnyFunSpec with Matchers { + + describe("MediaApi.shouldSkipUsageRecording") { + it("skips recording when the URI is a download and the user is the InDesign API key") { + MediaApi.shouldSkipUsageRecording( + uri = "https://usage.example.com/usages/download", + user = MediaApi.InDesignIdentity + ) shouldBe true + } + + it("does not skip recording for a download from a real user") { + MediaApi.shouldSkipUsageRecording( + uri = "https://usage.example.com/usages/download", + user = "some-real-user@guardian.co.uk" + ) shouldBe false + } + + it("does not skip recording for a syndication request, even from the InDesign API key") { + MediaApi.shouldSkipUsageRecording( + uri = "https://usage.example.com/usages/syndication", + user = MediaApi.InDesignIdentity + ) shouldBe false + } + + it("does not skip recording for a syndication request from a real user") { + MediaApi.shouldSkipUsageRecording( + uri = "https://usage.example.com/usages/syndication", + user = "some-real-user@guardian.co.uk" + ) shouldBe false + } + } +} diff --git a/media-api/test/lib/ImageResponseTest.scala b/media-api/test/lib/ImageResponseTest.scala index 2e16e8e6e86..d9e2e4c5f35 100644 --- a/media-api/test/lib/ImageResponseTest.scala +++ b/media-api/test/lib/ImageResponseTest.scala @@ -35,6 +35,13 @@ class ImageResponseTest extends AnyFunSpec with Matchers with Fixtures { "alias" -> "captionWriter", "label" -> "Caption Writer / Editor", "displaySearchHint" -> true + ), + Map( + "elasticsearchPath" -> "fileMetadata.c2pa.isAvailable", + "alias" -> "c2paMetadataAvailable", + "label" -> "C2PA Metadata Available", + "displaySearchHint" -> true, + "matchViaExistence" -> true ) ) ) ++ MOCK_CONFIG_KEYS.map(_ -> NOT_USED_IN_TEST).toMap), @@ -114,14 +121,17 @@ class ImageResponseTest extends AnyFunSpec with Matchers with Fixtures { val extractedFields = ImageResponse.extractAliasFieldValues(mediaApiConfig, sourceWrapper) extractedFields.nonEmpty shouldEqual true - extractedFields should have length 3 + // 3 regular fields plus c2paMetadataAvailable, which is always present for matchViaExistence + // fields (as an explicit `false` here, since this image has no c2pa data - see below) + extractedFields should have length 4 extractedFields.contains("orgProgrammeMaker" -> JsString("xmp programme maker")) shouldEqual true extractedFields.contains("auxLens" -> JsString("xmp aux lens")) shouldEqual true extractedFields.contains("captionWriter" -> JsString("the editor")) shouldEqual true + extractedFields.contains("c2paMetadataAvailable" -> JsBoolean(false)) shouldEqual true } - it("should return empty set of extract configured alias fields from sourcewrapper if fields do not exist in image") { + it("should return only the matchViaExistence alias fields (as false) if no other configured alias fields exist in image") { val image = createImage( id = "test-image-with-no-filemetadata", agency, @@ -132,6 +142,34 @@ class ImageResponseTest extends AnyFunSpec with Matchers with Fixtures { val extractedFields = ImageResponse.extractAliasFieldValues(mediaApiConfig, sourceWrapper) - extractedFields.isEmpty shouldEqual true + extractedFields shouldEqual Seq("c2paMetadataAvailable" -> JsBoolean(false)) + } + + it("should extract a matchViaExistence alias field as true when the underlying field is present") { + val image = createImage( + id = "test-image-with-c2pa", + agency, + fileMetadata = Some(FileMetadata(c2pa = FileMetadata.C2paAvailable)) + ) + val json = Json.toJson(image) + val sourceWrapper = SourceWrapper[Image](json, image, fromIndex="test_index") + + val extractedFields = ImageResponse.extractAliasFieldValues(mediaApiConfig, sourceWrapper) + + extractedFields.contains("c2paMetadataAvailable" -> JsBoolean(true)) shouldEqual true + } + + it("should extract a matchViaExistence alias field as false, rather than omitting it, when the underlying field is absent") { + val image = createImage( + id = "test-image-without-c2pa", + agency, + fileMetadata = Some(FileMetadata(c2pa = FileMetadata.NoC2PA)) + ) + val json = Json.toJson(image) + val sourceWrapper = SourceWrapper[Image](json, image, fromIndex="test_index") + + val extractedFields = ImageResponse.extractAliasFieldValues(mediaApiConfig, sourceWrapper) + + extractedFields.contains("c2paMetadataAvailable" -> JsBoolean(false)) shouldEqual true } } diff --git a/media-api/test/lib/UsageStoreTest.scala b/media-api/test/lib/UsageStoreTest.scala index 018f49a3fd8..e464a5dcc7f 100644 --- a/media-api/test/lib/UsageStoreTest.scala +++ b/media-api/test/lib/UsageStoreTest.scala @@ -16,7 +16,7 @@ class UsageStoreTest extends AnyFunSpec with Matchers { val list = UsageStore.csvParser(lines) - list.head should be (SupplierUsageSummary(Agency("Australian Associated Press Pty Limited (Stacey Shipton)"), 397)) + list.head should be (SupplierQuotaCount(Agency("Australian Associated Press Pty Limited (Stacey Shipton)"), 397)) } it("should parse non-ASCII RCS usage emails") { diff --git a/media-api/test/lib/elasticsearch/AiQueryPartsTest.scala b/media-api/test/lib/elasticsearch/AiQueryPartsTest.scala index 306b670fd68..a999e72dda3 100644 --- a/media-api/test/lib/elasticsearch/AiQueryPartsTest.scala +++ b/media-api/test/lib/elasticsearch/AiQueryPartsTest.scala @@ -19,111 +19,89 @@ class AiQueryPartsTest extends AnyFunSpec with Matchers with ImageFields { // Build the AiQueryParts the same way the controller does: parse the raw query // string with the production parser, then derive the AI search parts from it. - private def aiPartsFor(query: String): AiQueryParts = + private def aiPartsFor(query: String): Either[AiQueryError, AiQueryParts] = AiQueryParts.from(Parser.run(query)) + // Convenience for the valid cases, which are expected to parse into rankable parts. + private def validPartsFor(query: String): AiQueryParts = + aiPartsFor(query) match { + case Right(parts) => parts + case Left(error) => fail(s"expected a rankable AI query but got: $error") + } + describe("AiQueryParts validation") { describe("valid queries (each contains a KNN ranking signal)") { it("allows a similar-image-only query (image KNN)") { - val parts = aiPartsFor("similar:abc123") + val parts = validPartsFor("similar:abc123") parts.similarImageId should be(Some("abc123")) parts.semanticQuery should be(None) - parts.hasKnn should be(true) - parts.validationError should be(Right(())) } it("allows a text-only query (text KNN)") { - val parts = aiPartsFor("keir starmer") + val parts = validPartsFor("keir starmer") parts.semanticQuery should be(Some("keir starmer")) parts.similarImageId should be(None) - parts.hasKnn should be(true) - parts.validationError should be(Right(())) } it("allows chips/filters combined with a text query (pre-filter then text KNN)") { - val parts = aiPartsFor("fileType:jpeg keir starmer") + val parts = validPartsFor("fileType:jpeg keir starmer") parts.semanticQuery should be(Some("keir starmer")) parts.filterConditions should be(jpegFilter :: standardNegations) - parts.hasKnn should be(true) - parts.validationError should be(Right(())) } it("allows chips/filters combined with a similar image (pre-filter then image KNN)") { - val parts = aiPartsFor("similar:abc123 fileType:jpeg") + val parts = validPartsFor("similar:abc123 fileType:jpeg") parts.similarImageId should be(Some("abc123")) parts.filterConditions should be(jpegFilter :: standardNegations) - parts.hasKnn should be(true) - parts.validationError should be(Right(())) } it("allows a negative term combined with a text query (negative is a lexical pre-filter, then text KNN)") { - val parts = aiPartsFor("keir starmer -angela") + val parts = validPartsFor("keir starmer -angela") parts.semanticQuery should be(Some("keir starmer")) // The negation is treated as a filter condition, not a KNN ranking signal. parts.filterConditions should be(negatedAngela :: standardNegations) - parts.hasKnn should be(true) - parts.validationError should be(Right(())) } it("allows a negative term combined with a similar image (negative pre-filter, then image KNN)") { - val parts = aiPartsFor("similar:abc123 -angela") + val parts = validPartsFor("similar:abc123 -angela") parts.similarImageId should be(Some("abc123")) parts.filterConditions should be(negatedAngela :: standardNegations) - parts.hasKnn should be(true) - parts.validationError should be(Right(())) } } describe("invalid queries that combine two rankings") { it("rejects a similar image combined with a text query (rankings can't be merged)") { - val parts = aiPartsFor("similar:abc123 keir starmer") - - parts.similarImageId should be(Some("abc123")) - parts.semanticQuery should be(Some("keir starmer")) - parts.hasSimilarAndSemanticText should be(true) - parts.validationError.isLeft should be(true) + aiPartsFor("similar:abc123 keir starmer") should be(Left(AiQueryError.ConflictingRankingSignals)) } } describe("invalid queries that contain no KNN ranking signal") { it("rejects a chips/filters-only query") { - val parts = aiPartsFor("fileType:jpeg") - - parts.semanticQuery should be(None) - parts.similarImageId should be(None) - parts.filterConditions should be(jpegFilter :: standardNegations) - parts.hasKnn should be(false) - parts.validationError.isLeft should be(true) + aiPartsFor("fileType:jpeg") should be( + Left(AiQueryError.NoRankingSignal(jpegFilter :: standardNegations)) + ) } it("rejects a negative-only query") { - val parts = aiPartsFor("-angela") - - parts.semanticQuery should be(None) - parts.similarImageId should be(None) - parts.filterConditions should be(negatedAngela :: standardNegations) - parts.hasKnn should be(false) - parts.validationError.isLeft should be(true) + aiPartsFor("-angela") should be( + Left(AiQueryError.NoRankingSignal(negatedAngela :: standardNegations)) + ) } it("rejects a negative term combined with chips/filters") { - val parts = aiPartsFor("-angela fileType:jpeg") - - parts.semanticQuery should be(None) - parts.similarImageId should be(None) - parts.filterConditions should be(negatedAngela :: jpegFilter :: standardNegations) - parts.hasKnn should be(false) - parts.validationError.isLeft should be(true) + aiPartsFor("-angela fileType:jpeg") should be( + Left(AiQueryError.NoRankingSignal(negatedAngela :: jpegFilter :: standardNegations)) + ) } } } diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 9f333f9cea1..8380ae468f7 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -8,7 +8,7 @@ import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearc import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.DenySyndicationLease -import com.gu.mediaservice.model.usage.PublishedUsageStatus +import com.gu.mediaservice.model.usage.{PendingUsageStatus, PublishedUsageStatus, RemovedUsageStatus, SyndicationUsage, UnknownUsageStatus, ComposerUsageReference, DigitalUsage, FrontUsageReference, InDesignUsageReference, PrintUsage} import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ import lib.querysyntax._ @@ -152,12 +152,12 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast implicit val logMarker: LogMarker = MarkerMap() val publishedAgencyImages = images.filter(i => i.usageRights.isInstanceOf[Agency] && i.usages.exists(_.status == PublishedUsageStatus)) - publishedAgencyImages.size shouldBe 2 + publishedAgencyImages.size shouldBe 7 // Reporting date range is implemented as round down to last full day val withinReportedDateRange = publishedAgencyImages.filter(i => i.usages. exists(u => u.dateAdded.exists(_.isBefore(DateTime.now.withTimeAtStartOfDay())))) - withinReportedDateRange.size shouldBe 1 + withinReportedDateRange.size shouldBe 6 val results = Await.result(ES.usageForSupplier("ACME", 5), fiveSeconds) @@ -165,6 +165,263 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast } } + // Saves images for a single quota test and deletes them by ID afterwards, leaving the + // shared image set (loaded in beforeAll) untouched. + private def withQuotaImages[A](images: Seq[Image])(test: => A): A = { + implicit val logMarker: LogMarker = MarkerMap() + Await.ready(saveImages(images), 1.minute) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe expectedNumberOfImages + images.size) + try test finally { + val deletes = images.map(i => executeAndLog(deleteById(index, i.id), s"Deleting quota test image ${i.id}")) + Await.ready(Future.sequence(deletes), fiveSeconds) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe expectedNumberOfImages) + } + } + + describe("quotaCountBySupplier") { + // "quota-agency" is not in Agencies.all so Agencies.get("quota-agency").supplier falls back to "quota-agency" + val supplier = "quota-agency" + val inRange = DateTime.now.minusDays(15) + val numDays = 30 + + it("counts a single composer usage as 1") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composer-1", Agency(supplier), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("counts multiple composer usages on the same image individually") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composer-2", Agency(supplier), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(ComposerUsageReference, DigitalUsage, UnknownUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 2 + } + } + + it("counts multiple fronts usages on the same image as 1") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-fronts-multi", Agency(supplier), + usages = List( + createUsage(FrontUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(FrontUsageReference, DigitalUsage, RemovedUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("counts each qualifying print usage individually") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-print-multi", Agency(supplier), + usages = List( + createUsage(InDesignUsageReference, PrintUsage, PublishedUsageStatus, inRange), + createUsage(InDesignUsageReference, PrintUsage, RemovedUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 2 + } + } + + it("composer precedence: only counts composer usages when an image has both composer and fronts usages") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composer-and-fronts", Agency(supplier), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(FrontUsageReference, DigitalUsage, PublishedUsageStatus, inRange) + ))) + withQuotaImages(images) { + // fronts usage must not be counted — if it were, result would be 2 + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("composer precedence: only counts composer usages when an image has both composer and print usages") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composer-and-print", Agency(supplier), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(InDesignUsageReference, PrintUsage, PublishedUsageStatus, inRange) + ))) + withQuotaImages(images) { + // print usage must not be counted — if it were, result would be 2 + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("counts both fronts (as 1) and print (per usage) when there are no composer usages") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-fronts-and-print", Agency(supplier), + usages = List( + createUsage(FrontUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(InDesignUsageReference, PrintUsage, PublishedUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 2 + } + } + + it("returns 0 when all usages are outside the date range") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-out-of-range", Agency(supplier), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.now.minusDays(31))))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 0 + } + } + + it("excludes usages with a non-qualifying status") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-pending-status", Agency(supplier), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PendingUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 0 + } + } + + it("excludes usages with a non-qualifying platform") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-syndication-platform", Agency(supplier), + usages = List(createUsage(ComposerUsageReference, SyndicationUsage, PublishedUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 0 + } + } + + it("excludes images belonging to a different supplier") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-wrong-supplier", Agency("completely-different-agency"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 0 + } + } + + it("includes Composite images matched via the usageRights.suppliers field") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composite", Composite(s"$supplier, other-supplier"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("qualifies all three counting statuses: published, unknown, and removed") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-all-statuses", Agency(supplier), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(ComposerUsageReference, DigitalUsage, UnknownUsageStatus, inRange), + createUsage(ComposerUsageReference, DigitalUsage, RemovedUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 3 + } + } + } + + describe("images usages by supplier") { + it("only returns images with a qualifying usage status/platform for the given supplier, with the full usages list and supplier populated per item") { + implicit val logMarker: LogMarker = MarkerMap() + + val expectedIds = Set( + "usages-by-supplier-qualified-published-digital", + "usages-by-supplier-qualified-unknown-print", + "usages-by-supplier-qualified-removed-digital", + "usages-by-supplier-multi-usage", + "usages-by-supplier-out-of-date-range", + "usages-by-supplier-composite" + ) + + val result = Await.result(ES.imageUsagesBySupplier("test-wire", length = 100), fiveSeconds) + + result.total shouldBe expectedIds.size + result.images.map(_.id).toSet shouldBe expectedIds + result.images.filterNot(_.id == "usages-by-supplier-composite").foreach(_.supplier shouldBe "test-wire") + + // wrong supplier, non-qualifying status and non-qualifying platform are all excluded + result.images.map(_.id) should not contain "usages-by-supplier-wrong-supplier" + result.images.map(_.id) should not contain "usages-by-supplier-non-qualifying-status" + result.images.map(_.id) should not contain "usages-by-supplier-non-qualifying-platform" + + // distinctBy(_.id) collapses multiple qualifying usages into one result. Only the qualifying + // usages should be returned - the non-qualifying (pending status) usage on this image is excluded. + val multiUsageResult = result.images.find(_.id == "usages-by-supplier-multi-usage").get + multiUsageResult.usages should have size 2 + multiUsageResult.usages.map(_.status) should contain theSameElementsAs List(PublishedUsageStatus, RemovedUsageStatus) + } + + it("only returns usages within the requested date range on a matching image, excluding out-of-range usages on the same image") { + implicit val logMarker: LogMarker = MarkerMap() + + val dateRangeQuery = List(Nested( + SingleField("usages"), + SingleField("dateAdded"), + DateRange(DateTime.parse("2020-06-20"), DateTime.parse("2020-06-30")) + )) + + // the multi-usage image has one qualifying usage in range (2020-06-25, removed/print) and + // one qualifying usage out of range (2020-06-01, published/digital) - only the in-range one should be returned. + val result = Await.result(ES.imageUsagesBySupplier("test-wire", dateRangeQuery, length = 100), fiveSeconds) + + val multiUsageResult = result.images.find(_.id == "usages-by-supplier-multi-usage").get + multiUsageResult.usages should have size 1 + multiUsageResult.usages.head.status shouldBe RemovedUsageStatus + } + + it("applies an inclusive date range filter on usages.dateAdded and paginates the results") { + implicit val logMarker: LogMarker = MarkerMap() + + val dateRangeQuery = List(Nested( + SingleField("usages"), + SingleField("dateAdded"), + DateRange(DateTime.parse("2020-06-01"), DateTime.parse("2020-06-30")) + )) + + val filteredResult = Await.result(ES.imageUsagesBySupplier("test-wire", dateRangeQuery, length = 100), fiveSeconds) + filteredResult.images.map(_.id).toSet shouldBe Set( + "usages-by-supplier-qualified-published-digital", + "usages-by-supplier-qualified-unknown-print", + "usages-by-supplier-qualified-removed-digital", + "usages-by-supplier-multi-usage", + "usages-by-supplier-composite" + ) + + // pagination: paging through with a small length shouldn't drop or duplicate results + val pageSize = 2 + val pages = (0 until 3).map { page => + Await.result(ES.imageUsagesBySupplier("test-wire", offset = page * pageSize, length = pageSize), fiveSeconds) + } + pages.map(_.images.size) shouldBe Seq(2, 2, 2) + pages.foreach(_.total shouldBe 6) + pages.flatMap(_.images.map(_.id)).toSet shouldBe Set( + "usages-by-supplier-qualified-published-digital", + "usages-by-supplier-qualified-unknown-print", + "usages-by-supplier-qualified-removed-digital", + "usages-by-supplier-multi-usage", + "usages-by-supplier-out-of-date-range", + "usages-by-supplier-composite" + ) + } + + it("includes composite images whose suppliers field contains the given supplier name") { + implicit val logMarker: LogMarker = MarkerMap() + + val result = Await.result(ES.imageUsagesBySupplier("test-wire", length = 100), fiveSeconds) + + val compositeResult = result.images.find(_.id == "usages-by-supplier-composite") + compositeResult shouldBe defined + // supplier field is populated from usageRights.suppliers for composite images + compositeResult.get.supplier shouldBe "test-wire, other-supplier" + } + } + describe("aggregations") { it("can load date aggregations") { implicit val logMarker: LogMarker = MarkerMap() diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala index 466c604a659..3e43a63f071 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala @@ -1,6 +1,7 @@ package lib.elasticsearch import com.gu.mediaservice.model._ +import com.gu.mediaservice.model.usage._ import com.gu.mediaservice.testlib.ElasticSearchDockerBase import org.joda.time.DateTime import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout} @@ -144,6 +145,17 @@ trait ElasticSearchTestBase extends AnyFunSpec with ElasticSearchDockerBase with usages = List(createDigitalUsage(date = DateTime.now)) ), + // Agency image with an "unknown" status usage just now - unknown usages should still + // count towards the supplier usage summary, alongside "published" usages. + createImageForSyndication( + id = "test-image-agency-unknown-status", + rightsAcquired = false, + None, + None, + usageRights = agency, + usages = List(createUsage(ComposerUsageReference, DigitalUsage, UnknownUsageStatus, date = DateTime.now.minusDays(1))) + ), + // Screen grab with rights acquired, not eligible for syndication review createImageForSyndication( id = "test-image-11", @@ -187,5 +199,33 @@ trait ElasticSearchTestBase extends AnyFunSpec with ElasticSearchDockerBase with // ) // ) // ) + + // Fixtures for ES.imageUsagesBySupplier: a dedicated supplier ("test-wire" isn't in Agencies.all, + // so Agencies.get("test-wire").supplier falls back to == "test-wire"). + createImage("usages-by-supplier-qualified-published-digital", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-06-15")))), + createImage("usages-by-supplier-qualified-unknown-print", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, PrintUsage, UnknownUsageStatus, DateTime.parse("2020-06-16")))), + createImage("usages-by-supplier-qualified-removed-digital", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, RemovedUsageStatus, DateTime.parse("2020-06-30T23:59:59")))), + createImage("usages-by-supplier-non-qualifying-status", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PendingUsageStatus, DateTime.parse("2020-06-18")))), + createImage("usages-by-supplier-non-qualifying-platform", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, SyndicationUsage, PublishedUsageStatus, DateTime.parse("2020-06-19")))), + createImage("usages-by-supplier-wrong-supplier", Agency("other-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-06-20")))), + // Has two qualifying usages plus one non-qualifying usage (wrong status) - checks distinctBy(_.id) + // collapses to a single result, and that only the qualifying usages are returned (not the non-qualifying one). + createImage("usages-by-supplier-multi-usage", Agency("test-wire"), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-06-01")), + createUsage(ComposerUsageReference, PrintUsage, RemovedUsageStatus, DateTime.parse("2020-06-25")), + createUsage(ComposerUsageReference, DigitalUsage, PendingUsageStatus, DateTime.parse("2020-06-10")) + )), + createImage("usages-by-supplier-out-of-date-range", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-07-01")))), + // Composite image whose `suppliers` field contains "test-wire" alongside another supplier + createImage("usages-by-supplier-composite", Composite("test-wire, other-supplier"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-06-15")))) ) } diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala index 431329aeb32..57fe3c005de 100644 --- a/media-api/test/lib/elasticsearch/Fixtures.scala +++ b/media-api/test/lib/elasticsearch/Fixtures.scala @@ -56,6 +56,7 @@ trait Fixtures { usages: List[Usage] = Nil, fileMetadata: Option[FileMetadata] = None, softDeletedMetadata: Option[SoftDeletedMetadata] = None, + vector: Option[List[Double]] = None, ): Image = { Image( id = id, @@ -88,7 +89,7 @@ trait Fixtures { syndicationRights = syndicationRights, leases = leases.getOrElse(LeasesByMedia.build(Nil)), usages = usages, - embedding = None, + embedding = vector.map(v => Embedding(cohereEmbedV4 = Some(CohereV4Embedding(image = v)))), ) } @@ -152,7 +153,7 @@ trait Fixtures { out } - private def createUsage(t: UsageReferenceType, usageType: UsageType, status: Status, date: DateTime): Usage = { + def createUsage(t: UsageReferenceType, usageType: UsageType, status: Status, date: DateTime): Usage = { Usage( UUID.randomUUID().toString, List(UsageReference(t)), diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala new file mode 100644 index 00000000000..715e5d41db8 --- /dev/null +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -0,0 +1,352 @@ +package lib.elasticsearch + +import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} +import com.gu.mediaservice.model.{CohereV4Embedding, Embedding, Handout, Image} +import com.sksamuel.elastic4s.requests.searches.SearchHit +import org.scalactic.Tolerance +import org.scalatest.OptionValues +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import play.api.libs.json.Json + +class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with Tolerance with Fixtures { + + import HybridResult.{fuseAndRank, fuseScores, normalise, resolveHitAndFillInSemanticScore, Bm25TheoreticalMin, CosineSimilarityTheoreticalMin} + + private val tolerance = 1e-9 + + implicit val logMarker: LogMarker = MarkerMap() + + private def searchHit(id: String, score: Float): SearchHit = SearchHit( + id = id, + index = "test-index", + version = 1L, + seqNo = 0L, + primaryTerm = 0L, + score = score, + parent = None, + shard = None, + node = None, + routing = None, + explanation = None, + sort = None, + _source = Map.empty, + fields = Map.empty, + _highlight = None, + inner_hits = Map.empty, + matchedQueries = None + ) + + private def imageWithEmbedding(id: String, embedding: Option[List[Double]]): Image = + createImage(id, Handout()).copy( + embedding = embedding.map(vec => Embedding(cohereEmbedV4 = Some(CohereV4Embedding(image = vec)))) + ) + + private def sourceWrapperFor(image: Image): SourceWrapper[Image] = + SourceWrapper(Json.obj(), image, fromIndex = "test-index") + + private def resolveTo(image: Image): SearchHit => Option[SourceWrapper[Image]] = + _ => Some(sourceWrapperFor(image)) + + private val resolveToNothing: SearchHit => Option[SourceWrapper[Image]] = _ => None + + private def hybridResult(id: String, lexicalScore: Double, semanticScore: Double): HybridResult = + HybridResult( + id = id, + lexicalScore = lexicalScore, + semanticScore = semanticScore, + image = sourceWrapperFor(imageWithEmbedding(id, embedding = None)) + ) + + describe("resolveHitAndFillInSemanticScore") { + + it("returns None when the hit cannot be resolved to an image") { + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 3.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveToNothing + ) + result should be(None) + } + + it("carries the hit id and uses the hit score as the lexical score") { + val image = imageWithEmbedding("img-1", embedding = Some(List(1.0, 0.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 4.2f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.id should be("img-1") + result.lexicalScore should be(4.2f.toDouble +- tolerance) + } + + it("computes the semantic score as the cosine similarity between the image and query embeddings") { + // Orthogonal vectors -> cosine similarity 0. + val image = imageWithEmbedding("img-1", embedding = Some(List(0.0, 1.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(0.0 +- tolerance) + } + + it("returns a cosine similarity of 1 for identical (parallel) embeddings") { + val image = imageWithEmbedding("img-1", embedding = Some(List(3.0, 4.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(6.0, 8.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(1.0 +- tolerance) + } + + it("returns a cosine similarity of -1 for opposite embeddings") { + val image = imageWithEmbedding("img-1", embedding = Some(List(-1.0, 0.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(-1.0 +- tolerance) + } + + it("falls back to a semantic score of -1 when the image has no embedding") { + val image = imageWithEmbedding("img-1", embedding = None) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(CosineSimilarityTheoreticalMin +- tolerance) + } + + it("falls back to a semantic score of -1 when the image embedding has zero magnitude (cosine similarity is undefined)") { + val image = imageWithEmbedding("img-1", embedding = Some(List(0.0, 0.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(CosineSimilarityTheoreticalMin +- tolerance) + } + } + + describe("normalise") { + it("maps a score to its fraction of the way from the theoretical min to the max") { + // Halfway between theoreticalMin 0 and max 10. + normalise(score = 5.0, max = 10.0, theoreticalMin = Bm25TheoreticalMin) should be(0.5 +- tolerance) + } + + it("maps the max itself to 1") { + normalise(score = 10.0, max = 10.0, theoreticalMin = Bm25TheoreticalMin) should be(1.0 +- tolerance) + } + + it("maps the theoretical min itself to 0") { + normalise(score = -1.0, max = 1.0, theoreticalMin = CosineSimilarityTheoreticalMin) should be(0.0 +- tolerance) + } + + it("accounts for a non-zero theoretical min when normalising") { + // (0 - -1) / (1 - -1) = 0.5 + normalise(score = 0.0, max = 1.0, theoreticalMin = CosineSimilarityTheoreticalMin) should be(0.5 +- tolerance) + } + + it("returns 0 when the range collapses (max == theoreticalMin)") { + normalise(score = 0.0, max = 0.0, theoreticalMin = Bm25TheoreticalMin) should be(0.0 +- tolerance) + normalise(score = -1.0, max = -1.0, theoreticalMin = CosineSimilarityTheoreticalMin) should be(0.0 +- tolerance) + } + } + + describe("fuseScores") { + it("exposes the normalised component scores and blends them using vecWeight") { + // normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 + // fused = 0.25 * 0.5 + 0.75 * 0.5 = 0.5 + val score = fuseScores( + hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0), + maxLexicalScore = 4.0, + maxSemanticScore = 1.0, + vecWeight = 0.25 + ) + score.normedLexicalScore should be(0.5 +- tolerance) + score.normedSemanticScore should be(0.5 +- tolerance) + score.fusedScore should be(0.5 +- tolerance) + } + + it("uses only the lexical score when vecWeight is 0") { + val score = fuseScores( + hybridResult("lexical-only", lexicalScore = 2.0, semanticScore = 1.0), + maxLexicalScore = 4.0, + maxSemanticScore = 1.0, + vecWeight = 0.0 + ) + score.fusedScore should be(0.5 +- tolerance) + } + + it("uses only the semantic score when vecWeight is 1") { + val score = fuseScores( + hybridResult("semantic-only", lexicalScore = 2.0, semanticScore = 0.0), + maxLexicalScore = 4.0, + maxSemanticScore = 1.0, + vecWeight = 1.0 + ) + // normedSemantic = (0 + 1)/(1 + 1) = 0.5 + score.fusedScore should be(0.5 +- tolerance) + } + + it("falls back to the semantic score for a pure semantic search (no lexical matches, so max lexical score is 0)") { + val score = fuseScores( + hybridResult("no-lexical", lexicalScore = 0.0, semanticScore = 1.0), + maxLexicalScore = 0.0, + maxSemanticScore = 1.0, + vecWeight = 0.5 + ) + // normedLexical = 0, normedSemantic = (1 + 1)/(1 + 1) = 1 + // fused = 0.5 * 1 + 0.5 * 0 = 0.5 + score.normedLexicalScore should be(0.0 +- tolerance) + score.fusedScore should be(0.5 +- tolerance) + } + + it("falls back to the lexical score when no result has an embedding (so max semantic score is -1)") { + val score = fuseScores( + hybridResult("no-semantic", lexicalScore = 4.0, semanticScore = -1.0), + maxLexicalScore = 4.0, + maxSemanticScore = -1.0, + vecWeight = 0.5 + ) + // normedSemantic = 0, normedLexical = 4/4 = 1 + // fused = 0.5 * 0 + 0.5 * 1 = 0.5 + score.normedSemanticScore should be(0.0 +- tolerance) + score.fusedScore should be(0.5 +- tolerance) + } + } + + describe("fuseAndRank") { + import ResultSource.{Lexical, Semantic, Both} + + it("returns an empty list if both inputs are empty") { + fuseAndRank(lexicalResults = List(), semanticResults = List(), vecWeight = 0.0, k = 3) should be(List()) + } + + it("ranks purely by lexical score when vecWeight is 0") { + val lexical = List( + hybridResult("low-lexical", lexicalScore = 1.0, semanticScore = 1.0), + hybridResult("high-lexical", lexicalScore = 10.0, semanticScore = -1.0), + hybridResult("mid-lexical", lexicalScore = 5.0, semanticScore = 0.0) + ) + + val ranked = fuseAndRank(lexical, semanticResults = List(), vecWeight = 0.0, k = 3) + + ranked.map(_.result.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) + } + + it("ranks purely by lexical score when no result has an embedding (so max semantic score is -1.0)") { + val lexical = List( + hybridResult("low-lexical", lexicalScore = 1.0, semanticScore = -1.0), + hybridResult("high-lexical", lexicalScore = 10.0, semanticScore = -1.0), + hybridResult("mid-lexical", lexicalScore = 5.0, semanticScore = -1.0) + ) + + val ranked = fuseAndRank(lexical, semanticResults = List(), vecWeight = 0.5, k = 3) + + ranked.map(_.result.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) + } + + it("ranks purely by semantic score when vecWeight is 1") { + val semantic = List( + hybridResult("low-semantic", lexicalScore = 10.0, semanticScore = -1.0), + hybridResult("high-semantic", lexicalScore = 1.0, semanticScore = 1.0), + hybridResult("mid-semantic", lexicalScore = 5.0, semanticScore = 0.0) + ) + + val ranked = fuseAndRank(lexicalResults = List(), semantic, vecWeight = 1.0, k = 3) + + ranked.map(_.result.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) + } + + it("ranks purely by semantic score for a pure semantic search (no lexical matches, so max lexical score is 0)") { + val semantic = List( + hybridResult("low-semantic", lexicalScore = 0.0, semanticScore = -1.0), + hybridResult("high-semantic", lexicalScore = 0.0, semanticScore = 1.0), + hybridResult("mid-semantic", lexicalScore = 0.0, semanticScore = 0.0) + ) + + val ranked = fuseAndRank(lexicalResults = List(), semantic, vecWeight = 0.5, k = 3) + + ranked.map(_.result.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) + } + + it("only returns the top k results") { + val lexical = List( + hybridResult("a", lexicalScore = 1.0, semanticScore = 1.0), + hybridResult("b", lexicalScore = 10.0, semanticScore = 1.0), + hybridResult("c", lexicalScore = 5.0, semanticScore = 1.0) + ) + + val ranked = fuseAndRank(lexical, semanticResults = List(), vecWeight = 0.0, k = 2) + + ranked should have size 2 + ranked.map(_.result.id) should be(List("b", "c")) + } + + it("de-duplicates a result that appears in both sets, keeping it once") { + val lexical = List(hybridResult("shared", lexicalScore = 4.0, semanticScore = 1.0)) + val semantic = List(hybridResult("shared", lexicalScore = 4.0, semanticScore = 1.0)) + + val ranked = fuseAndRank(lexical, semantic, vecWeight = 0.5, k = 10) + + ranked.map(_.result.id) should be(List("shared")) + } + + it("tags each result with whether it came from lexical, semantic, or both") { + val lexical = List( + hybridResult("lexical-only", lexicalScore = 3.0, semanticScore = -1.0), + hybridResult("shared", lexicalScore = 4.0, semanticScore = 1.0) + ) + val semantic = List( + hybridResult("shared", lexicalScore = 4.0, semanticScore = 1.0), + hybridResult("semantic-only", lexicalScore = 0.0, semanticScore = 0.5) + ) + + val ranked = fuseAndRank(lexical, semantic, vecWeight = 0.5, k = 10) + + ranked.map(r => r.result.id -> r.source).toMap should be(Map( + "lexical-only" -> Lexical, + "semantic-only" -> Semantic, + "shared" -> Both + )) + } + + it("produces a ranked table of original scores, normed scores, source and fused score") { + // maxLexical = 4, maxSemantic = 1, vecWeight = 0.25. + val lexical = List( + hybridResult("top-lexical", lexicalScore = 4.0, semanticScore = -1.0), + hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0) + ) + val semantic = List( + hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0), + hybridResult("top-semantic", lexicalScore = 0.0, semanticScore = 1.0) + ) + + val ranked = fuseAndRank(lexical, semantic, vecWeight = 0.25, k = 10) + + // A flat projection of the full ranked output, ordered by fused score descending. + val table = ranked.map { r => + (r.result.id, r.source, r.result.lexicalScore, r.result.semanticScore, + r.score.normedLexicalScore, r.score.normedSemanticScore, r.score.fusedScore) + } + + // id source lexical semantic normedLex normedSem fused + table should be(List( + ("top-lexical", Lexical, 4.0, -1.0, 1.0, 0.0, 0.75), + ("blend", Both, 2.0, 0.0, 0.5, 0.5, 0.50), + ("top-semantic", Semantic, 0.0, 1.0, 0.0, 1.0, 0.25) + )) + } + } +} diff --git a/media-api/test/lib/elasticsearch/HybridSearchTest.scala b/media-api/test/lib/elasticsearch/HybridSearchTest.scala new file mode 100644 index 00000000000..61b129c9d42 --- /dev/null +++ b/media-api/test/lib/elasticsearch/HybridSearchTest.scala @@ -0,0 +1,162 @@ +package lib.elasticsearch + +import org.apache.pekko.actor.{ActorSystem, Scheduler} +import com.gu.mediaservice.lib.VectorUtils.{firstBasisVector, vectorWithCosineSimilarity} +import com.gu.mediaservice.lib.config.GridConfigResources +import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearchConfig, ElasticSearchExecutions} +import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} +import com.gu.mediaservice.model._ +import com.gu.mediaservice.testlib.ElasticSearchDockerBase +import com.sksamuel.elastic4s.ElasticDsl +import com.sksamuel.elastic4s.ElasticDsl._ +import lib.{MediaApiConfig, MediaApiMetrics} +import org.scalatest.concurrent.{Eventually, ScalaFutures} +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.mockito.MockitoSugar +import play.api.Configuration +import play.api.inject.ApplicationLifecycle +import play.api.libs.json.Json + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} + +class HybridSearchTest extends AnyFunSpec + with ElasticSearchDockerBase + with Matchers + with ScalaFutures + with Eventually + with ElasticSearchExecutions + with Fixtures + with MockitoSugar { + + private val index = "images" + + private val applicationLifecycle = new ApplicationLifecycle { + override def addStopHook(hook: () => Future[_]): Unit = {} + override def stop(): Future[_] = Future.successful(()) + } + + private val mediaApiConfig = new MediaApiConfig(GridConfigResources( + Configuration.from(USED_CONFIGS_IN_TEST ++ MOCK_CONFIG_KEYS.map(_ -> NOT_USED_IN_TEST).toMap), + null, + applicationLifecycle + )) + private val actorSystem: ActorSystem = ActorSystem() + private val mediaApiMetrics = new MediaApiMetrics(mediaApiConfig, actorSystem, applicationLifecycle) + private val elasticConfig = ElasticSearchConfig( + aliases = ElasticSearchAliases( + current = "Images_Current", + migration = "Images_Migration" + ), + url = esTestUrl, + shards = 1, + replicas = 0 + ) + + private lazy val ES = new ElasticSearch(mediaApiConfig, mediaApiMetrics, elasticConfig, () => List.empty, mock[Scheduler]) + lazy val client = ES.client + + private val oneHundredMilliseconds = Duration(100, MILLISECONDS) + private val fiveSeconds = Duration(5, SECONDS) + + // The vector we'll "search" with - represents the user's query embedding. + private val queryEmbedding: List[Double] = firstBasisVector(256) + + private def aiImage(id: String, title: String, vector: List[Double]): Image = { + val base = createImage(id = id, usageRights = Handout(), vector = Some(vector)) + base.copy(metadata = base.metadata.copy(title = Some(title))) + } + + // 256-dim vectors to match the `embedding.cohereEmbedV4.image` dense_vector + // mapping, each constructed to have a known cosine similarity to the queryEmbedding. + private def vectorWithScore(score: Double): List[Double] = { + vectorWithCosineSimilarity(256, score) + } + + // The full set of images shared by every spec below; indexed once in beforeAll. + private val testImages = Seq( + aiImage("a", title = "zero lexical", vector = vectorWithScore(1.0)), + aiImage("b", title = "zero lexical", vector = vectorWithScore(0.9)), + aiImage("c", title = "good lexical", vector = vectorWithScore(0.8)), + aiImage("d", title = "good good lexical", vector = vectorWithScore(0.7)) + ) + + override def beforeAll(): Unit = { + super.beforeAll() + ES.ensureIndexExistsAndAliasAssigned() + + implicit val logMarker: LogMarker = MarkerMap() + purgeTestImages + Await.ready(saveImages(testImages), 1.minute) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe testImages.size) + } + + describe("AI / hybrid search") { + implicit val logMarker: LogMarker = MarkerMap() + + it("orders purely by semantic similarity when vecWeight = 1.0") { + val semanticOnlyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 1.0, filterOpt = None), + fiveSeconds + ) + semanticOnlyResults.hits.map(_._1) shouldBe List("a", "b", "c", "d") + } + + it("runs only the lexical query when vecWeight = 0.0") { + val lexicalOnlyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 0.0, filterOpt = None), + fiveSeconds + ) + // Because we asked for k = 4 but only got 2 back, this confirms + // the short-circuiting behaviour at vecWeight 0.0, i.e. only the lexical query ran. + lexicalOnlyResults.hits.map(_._1) shouldBe List("d", "c") + } + + it("includes all results once vecWeight > 0") { + val weightedHeavilyLexicallyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 0.1, filterOpt = None), + fiveSeconds + ) + // We should now get all 4 because vecWeight > 0 + weightedHeavilyLexicallyResults.hits.map(_._1) shouldBe List("d", "c", "a", "b") + } + + it("promotes a score-filled result into the top 2 with equal weighting") { + // We've now proven that the lexical top 2 (d, c) are disjunct + // from the semantic top 2 (a, b). Therefore, without score filling, + // the top 2 hybrid results after theoretical min-max norming + // would have to be the top semantic and the top lexical, i.e. a and d, + // which both get a score of 1. But thanks to score filling, + // c can make it into the top 2. + val equalWeightingResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 2, numCandidates = 10, vecWeight = 0.5, filterOpt = None), + fiveSeconds + ) + equalWeightingResults.hits.map(_._1) shouldBe List("d", "c") + } + } + + private def saveImages(images: Seq[Image]) = { + implicit val logMarker: LogMarker = MarkerMap() + + Future.sequence(images.map { i => + executeAndLog(indexInto(index) id i.id source Json.stringify(Json.toJson(i)), s"Indexing test image") + }) + } + + private def totalImages: Long = Await.result(ES.client.execute(ElasticDsl.search(ES.imagesCurrentAlias)).map { + _.result.totalHits + }, oneHundredMilliseconds) + + private def purgeTestImages = { + implicit val logMarker: LogMarker = MarkerMap() + + def deleteImages = executeAndLog(deleteByQuery(index, matchAllQuery()), s"Deleting images") + + Await.result(deleteImages, fiveSeconds) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe 0) + } + +} diff --git a/media-api/test/lib/elasticsearch/QueryBuilderTest.scala b/media-api/test/lib/elasticsearch/QueryBuilderTest.scala index ace13ce3005..137cb560371 100644 --- a/media-api/test/lib/elasticsearch/QueryBuilderTest.scala +++ b/media-api/test/lib/elasticsearch/QueryBuilderTest.scala @@ -5,7 +5,7 @@ import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.requests.common.Operator import com.sksamuel.elastic4s.requests.searches.queries._ import com.sksamuel.elastic4s.requests.searches.queries.matches.{MatchPhraseQuery, MatchQuery, MultiMatchQuery, MultiMatchQueryBuilderType} -import lib.querysyntax.Negation +import lib.querysyntax.{Match, Negation, Phrase, SingleField} import com.gu.mediaservice.lib.config.GridConfigResources import com.sksamuel.elastic4s.handlers.searches.queries.QueryBuilderFn import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery @@ -183,6 +183,93 @@ class QueryBuilderTest extends AnyFunSpec with Matchers with ConditionFixtures w } } + describe("field alias matchViaExistence queries") { + // e.g. `fileMetadata.c2pa.isAvailable` is only ever indexed as `true`, so `:true`/`:false` are + // translated into exists/not-exists queries rather than literal term matches (see FieldAlias.matchViaExistence). + val c2paFieldAliasConfig = Map( + "elasticsearchPath" -> "fileMetadata.c2pa.isAvailable", + "alias" -> "c2paMetadataAvailable", + "label" -> "C2PA Metadata Available", + "matchViaExistence" -> true + ) + val configWithC2paAlias = new MediaApiConfig(GridConfigResources( + Configuration.from(commonConfigurations ++ Map("field.aliases" -> Seq(c2paFieldAliasConfig))), + null, + new ApplicationLifecycle { + override def addStopHook(hook: () => Future[_]): Unit = {} + override def stop(): Future[_] = Future.successful(()) + } + )) + val queryBuilderWithC2paAlias = new QueryBuilder(matchFields, () => Nil, configWithC2paAlias) + + def existsClauseOf(query: Query): ExistsQuery = + query.asInstanceOf[BoolQuery].filters.head.asInstanceOf[ExistsQuery] + + it("alias:true is expressed as an exists query on the underlying elasticsearch path") { + val condition = Match(SingleField("c2paMetadataAvailable"), Phrase("true")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + val existsClause = existsClauseOf(query.must.head) + existsClause.field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("alias:false is expressed as a not-exists query on the underlying elasticsearch path") { + val condition = Match(SingleField("c2paMetadataAvailable"), Phrase("false")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + query.must.size shouldBe 1 + val notClause = query.must.head.asInstanceOf[BoolQuery] + val existsClause = existsClauseOf(notClause.not.head) + existsClause.field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("the underlying elasticsearch path:true also resolves to the same exists query as the alias") { + val condition = Match(SingleField("fileMetadata.c2pa.isAvailable"), Phrase("true")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + val existsClause = existsClauseOf(query.must.head) + existsClause.field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("the underlying elasticsearch path:false also resolves to the same not-exists query as the alias") { + val condition = Match(SingleField("fileMetadata.c2pa.isAvailable"), Phrase("false")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + query.must.size shouldBe 1 + val notClause = query.must.head.asInstanceOf[BoolQuery] + val existsClause = existsClauseOf(notClause.not.head) + existsClause.field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("boolean values are matched case-insensitively, e.g. alias:TRUE and alias:False") { + val trueQuery = queryBuilderWithC2paAlias.makeQuery( + List(Match(SingleField("c2paMetadataAvailable"), Phrase("TRUE"))) + ).asInstanceOf[BoolQuery] + existsClauseOf(trueQuery.must.head).field shouldBe "fileMetadata.c2pa.isAvailable" + + val falseQuery = queryBuilderWithC2paAlias.makeQuery( + List(Match(SingleField("c2paMetadataAvailable"), Phrase("False"))) + ).asInstanceOf[BoolQuery] + falseQuery.must.size shouldBe 1 + val notClause = falseQuery.must.head.asInstanceOf[BoolQuery] + existsClauseOf(notClause.not.head).field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("a non-boolean value on a matchViaExistence field falls back to a literal phrase match, not an exists query") { + val condition = Match(SingleField("c2paMetadataAvailable"), Phrase("maybe")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + query.must.size shouldBe 1 + val phraseClause = query.must.head.asInstanceOf[MatchPhraseQuery] + phraseClause.field shouldBe "fileMetadata.c2pa.isAvailable" + phraseClause.value shouldBe "maybe" + } + + it("fields with no matching alias or elasticsearch path still fall back to getFieldPath, unaffected by the new lookup") { + queryBuilderWithC2paAlias.resolveFieldPath("someUnrelatedField") shouldBe "someUnrelatedField" + } + } + describe("is search filter") { it("should correctly construct an is owned photo query") { val query = queryBuilder.makeQuery(List(isOwnedPhotoCondition)).asInstanceOf[BoolQuery] diff --git a/metadata-editor/app/controllers/EditsController.scala b/metadata-editor/app/controllers/EditsController.scala index 44be486d4c1..53f682cca28 100644 --- a/metadata-editor/app/controllers/EditsController.scala +++ b/metadata-editor/app/controllers/EditsController.scala @@ -3,8 +3,6 @@ package controllers import java.net.URI import java.net.URLDecoder.decode - -import com.amazonaws.AmazonServiceException import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.argo.model._ @@ -22,6 +20,7 @@ import org.joda.time.DateTime import play.api.libs.json._ import play.api.libs.ws.WSClient import play.api.mvc.{BaseController, ControllerComponents} +import software.amazon.awssdk.awscore.exception.AwsServiceException import scala.concurrent.{ExecutionContext, Future} import scala.collection.compat._ @@ -59,7 +58,7 @@ class EditsController( import com.gu.mediaservice.lib.metadata.UsageRightsMetadataMapper.usageRightsToMetadata val services: Services = new Services(config.domainRoot, config.serviceHosts, Set.empty) - val gridClient: GridClient = GridClient(services)(ws) + val gridClient: GridClient = GridClient(services, services.metadataBaseUri)(ws) val metadataBaseUri = config.services.metadataBaseUri private val AuthenticatedAndAuthorised = auth andThen authorisation.CommonActionFilters.authorisedForArchive @@ -73,7 +72,7 @@ class EditsController( // TODO: Think about calling this `overrides` or something that isn't metadata def getAllMetadata(id: String) = auth.async { val emptyResponse = respond(Edits.getEmpty)(editsEntity(id)) - editsStore.getV2(id) map { dynamoEntry => + editsStore.get(id) map { dynamoEntry => dynamoEntry.asOpt[Edits] .map(respond(_)(editsEntity(id))) .getOrElse(emptyResponse) @@ -81,14 +80,14 @@ class EditsController( } def getEdits(id: String) = auth.async { - editsStore.getV2(id) map { dynamoEntry => + editsStore.get(id) map { dynamoEntry => val edits = dynamoEntry.asOpt[Edits] respond(data = edits) } recover { case NoItemFound => NotFound } } def getArchived(id: String) = auth.async { - editsStore.booleanGetV2(id, Edits.Archived) map { archived => + editsStore.booleanGet(id, Edits.Archived) map { archived => respond(archived) } recover { case NoItemFound => respond(false) @@ -100,21 +99,21 @@ class EditsController( errors => Future.successful(BadRequest(errors.toString())), archived => - editsStore.booleanSetOrRemoveV2(id, "archived", archived) + editsStore.booleanSetOrRemove(id, "archived", archived) .map(publish(id, UpdateImageUserMetadata)) .map(edits => respond(edits.archived)) ) } def unsetArchived(id: String) = auth.async { - editsStore.removeKeyV2(id, Edits.Archived) + editsStore.removeKey(id, Edits.Archived) .map(publish(id, UpdateImageUserMetadata)) .map(_ => respond(false)) } def getLabels(id: String) = auth.async { - editsStore.setGetV2(id, Edits.Labels) + editsStore.setGet(id, Edits.Labels) .map(labelsCollection(id, _)) .map {case (_, labels) => respondCollection(labels)} recover { case NoItemFound => respond(Array[String]()) @@ -127,17 +126,17 @@ class EditsController( Future.successful(BadRequest(errors.toString())), labels => editsStore - .setAddV2(id, Edits.Labels, labels) + .setAdd(id, Edits.Labels, labels) .map(publish(id, UpdateImageUserMetadata)) .map(edits => labelsCollection(id, edits.labels.toSet)) .map { case (uri, l) => respondCollection(l) } recover { - case _: AmazonServiceException => BadRequest + case _: AwsServiceException => BadRequest } ) } def removeLabel(id: String, label: String) = auth.async { - editsStore.setDeleteV2(id, Edits.Labels, decodeUriParam(label)) + editsStore.setDelete(id, Edits.Labels, decodeUriParam(label)) .map(publish(id, UpdateImageUserMetadata)) .map(edits => labelsCollection(id, edits.labels.toSet)) .map {case (uri, labels) => respondCollection(labels, uri=Some(uri))} @@ -145,8 +144,8 @@ class EditsController( def getMetadata(id: String) = auth.async { - editsStore.getV2(id).map { dynamoEntry => - val metadata = (dynamoEntry \ Edits.Metadata).as[ImageMetadata] + editsStore.get(id).map { dynamoEntry => + val metadata = (dynamoEntry \ Edits.Metadata).asOpt[ImageMetadata].getOrElse(Edits.emptyMetadata) respond(metadata) } recover { case NoItemFound => respond(Json.toJson(JsObject(Nil))) @@ -168,7 +167,7 @@ class EditsController( }) val validatedMetadata = metadata.copy(domainMetadata = validatedDomainMetadata) - editsStore.jsonAddV2(id, Edits.Metadata, metadataAsMap(validatedMetadata)) + editsStore.jsonAdd(id, Edits.Metadata, metadataAsMap(validatedMetadata)) .map(publish(id, UpdateImageUserMetadata)) .map(edits => respond(edits.metadata)) } @@ -176,7 +175,7 @@ class EditsController( } def setMetadataFromUsageRights(id: String) = (auth andThen authorisedForEditMetadataOrUploader(id)).async { req => - editsStore.getV2(id) flatMap { dynamoEntry => + editsStore.get(id) flatMap { dynamoEntry => gridClient.getMetadata(id, auth.getOnBehalfOfPrincipal(req.user)) flatMap { imageMetadata => val edits = dynamoEntry.as[Edits] val originalUserMetadata = edits.metadata @@ -191,7 +190,7 @@ class EditsController( imageType = metadata.imageType orElse originalUserMetadata.imageType ) - editsStore.jsonAddV2(id, Edits.Metadata, metadataAsMap(mergedMetadata)) + editsStore.jsonAdd(id, Edits.Metadata, metadataAsMap(mergedMetadata)) .map(publish(id, UpdateImageUserMetadata)) .map(edits => respond(edits.metadata, uri = Some(metadataUri(id)))) } getOrElse { @@ -206,7 +205,7 @@ class EditsController( } def getUsageRights(id: String) = auth.async { - editsStore.getV2(id).map { dynamoEntry => + editsStore.get(id).map { dynamoEntry => val usageRights = (dynamoEntry \ Edits.UsageRights).as[UsageRights] respond(usageRights) } recover { @@ -216,7 +215,7 @@ class EditsController( def setUsageRights(id: String) = auth.async(parse.json) { req => (req.body \ "data").asOpt[UsageRights].map(usageRight => { - editsStore.jsonAddV2(id, Edits.UsageRights, DynamoDB.caseClassToMap(usageRight)) + editsStore.jsonAdd(id, Edits.UsageRights, DynamoDB.caseClassToMap(usageRight)) .map(publish(id, UpdateImageUserMetadata)) .map(_ => respond(usageRight)) }).getOrElse(Future.successful(respondError(BadRequest, "invalid-form-data", "Invalid form data"))) @@ -224,7 +223,7 @@ class EditsController( //TODO - remove as a second step def deleteUsageRights(id: String) = auth.async { req => - editsStore.removeKeyV2(id, Edits.UsageRights).map(publish(id, UpdateImageUserMetadata)).map(edits => Accepted) + editsStore.removeKey(id, Edits.UsageRights).map(publish(id, UpdateImageUserMetadata)).map(edits => Accepted) } private def labelsCollection(id: String, labels: Set[String]): (URI, Seq[EmbeddedEntity[String]]) = diff --git a/metadata-editor/app/controllers/SyndicationController.scala b/metadata-editor/app/controllers/SyndicationController.scala index c0b771a3be0..e196bf548af 100644 --- a/metadata-editor/app/controllers/SyndicationController.scala +++ b/metadata-editor/app/controllers/SyndicationController.scala @@ -24,7 +24,7 @@ class SyndicationController(auth: Authentication, override val metadataBaseUri: String = config.services.metadataBaseUri def getPhotoshoot(id: String) = auth.async { - editsStore.getV2(id).map(dynamoEntry => { + editsStore.get(id).map(dynamoEntry => { (dynamoEntry \ Edits.Photoshoot).toOption match { case Some(photoshoot) => respond(photoshoot.as[Photoshoot]) case None => respondNotFound("No photoshoot found") diff --git a/metadata-editor/app/lib/Syndication.scala b/metadata-editor/app/lib/Syndication.scala index d8a6f767fbf..9ef65c96e2d 100644 --- a/metadata-editor/app/lib/Syndication.scala +++ b/metadata-editor/app/lib/Syndication.scala @@ -58,8 +58,8 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { (implicit ec: ExecutionContext): Future[Photoshoot] = { publishChangedSyndicationRightsForPhotoshoot[Photoshoot](id, photoshoot = Some(newPhotoshoot)) { () => for { - editsAsJsonResponse <- editsStore.jsonAddV2(id, Edits.Photoshoot, DynamoDB.caseClassToMap(newPhotoshoot)) - _ <- editsStore.stringSetV2(id, Edits.PhotoshootTitle, newPhotoshoot.title) // store - don't care about return + editsAsJsonResponse <- editsStore.jsonAdd(id, Edits.Photoshoot, DynamoDB.caseClassToMap(newPhotoshoot)) + _ <- editsStore.stringSet(id, Edits.PhotoshootTitle, newPhotoshoot.title) // store - don't care about return _ = publish(id, UpdateImagePhotoshootMetadata)(editsAsJsonResponse) } yield newPhotoshoot } @@ -68,7 +68,7 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { def deleteSyndicationAndPublish(id: String) (implicit ec: ExecutionContext): Future[Unit] = { publishChangedSyndicationRightsForPhotoshoot[Unit](id, unchangedPhotoshoot = true) { () => - syndicationStore.deleteItemV2(id) + syndicationStore.deleteItem(id) // Always publish, in case there is no photoshoot publish(Map(id -> None), UpdateImageSyndicationMetadata) } @@ -77,7 +77,7 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { def setSyndicationAndPublish(id: String, syndicationRight: SyndicationRights) (implicit ec: ExecutionContext): Future[SyndicationRights] = publishChangedSyndicationRightsForPhotoshoot[SyndicationRights](id, unchangedPhotoshoot = true) { () => - val result = syndicationStore.jsonAddV2(id, syndicationRightsFieldName, DynamoDB.caseClassToMap (syndicationRight)).map(_=>syndicationRight) + val result = syndicationStore.jsonAdd(id, syndicationRightsFieldName, DynamoDB.caseClassToMap (syndicationRight)).map(_=>syndicationRight) // Always publish, in case there is no photoshoot publish(Map(id -> Some(syndicationRight)), UpdateImageSyndicationMetadata) result @@ -85,7 +85,7 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { def getSyndicationForImage(id: String) (implicit ec: ExecutionContext): Future[Option[SyndicationRights]] = { - syndicationStore.getV2(id) + syndicationStore.get(id) // It's OK for the image to _not_ exist in the syndication store, so this needs to be recovered .recover { case NoItemFound => JsNull } .flatMap(dynamoEntry => (dynamoEntry \ syndicationRightsFieldName).toOption match { @@ -156,7 +156,7 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { def getPhotoshootForImage(id: String) (implicit ec: ExecutionContext): Future[Option[Photoshoot]] = - editsStore.getV2(id) + editsStore.get(id) .map(dynamoEntry => (dynamoEntry \ Edits.Photoshoot).toOption map { photoshootJson => photoshootJson.as[Photoshoot] }) diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala index 00f45b4690a..4e6687906ce 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala @@ -112,6 +112,22 @@ class PandaAuthenticationProvider( override def flushToken: Option[(RequestHeader, Result) => Result] = Some((rh, _) => processLogout(rh)) val PandaCookieKey: TypedKey[Cookie] = TypedKey[Cookie]("PandaCookie") + val OriginKey: TypedKey[String] = TypedKey[String]("Origin") + + private def withPersistedOrigin(request: Principal): WSRequest => WSRequest = { wsRequest => + request.attributes.get(OriginKey) match { + case Some(origin) => wsRequest.withHttpHeaders(("Origin", origin)) + case None => wsRequest + } + } + + private def withPersistedCookie(request: Principal): Either[String, WSRequest => WSRequest] = { + val cookieName = panDomainSettings.settings.cookieSettings.cookieName + request.attributes.get(PandaCookieKey) match { + case Some(cookie) => Right(wsRequest => wsRequest.addCookies(DefaultWSCookie(cookieName, cookie.value))) + case None => Left(s"Pan domain cookie $cookieName is missing in principal.") + } + } /** * A function that allows downstream API calls to be made using the credentials of the inflight request @@ -119,19 +135,13 @@ class PandaAuthenticationProvider( * @param request The request header of the inflight call * @return A function that adds appropriate data to a WSRequest */ - override def onBehalfOf(request: Principal): Either[String, WSRequest => WSRequest] = { - val cookieName = panDomainSettings.settings.cookieSettings.cookieName - request.attributes.get(PandaCookieKey) match { - case Some(cookie) => Right { wsRequest: WSRequest => - wsRequest.addCookies(DefaultWSCookie(cookieName, cookie.value)) - } - case None => Left(s"Pan domain cookie $cookieName is missing in principal.") - } - } + override def onBehalfOf(request: Principal): Either[String, WSRequest => WSRequest] = + withPersistedCookie(request).map(_.andThen(withPersistedOrigin(request))) private def gridUserFrom(pandaUser: User, request: RequestHeader): UserPrincipal = { val maybePandaCookie: Option[TypedEntry[Cookie]] = request.cookies.get(panDomainSettings.settings.cookieSettings.cookieName).map(TypedEntry[Cookie](PandaCookieKey, _)) - val attributes = TypedMap.empty + (maybePandaCookie.toSeq:_*) + val maybeOrigin = request.headers.get("Origin").map(TypedEntry[String](OriginKey, _)) + val attributes = TypedMap(Seq(maybePandaCookie, maybeOrigin).flatten:_*) UserPrincipal( firstName = pandaUser.firstName, lastName = pandaUser.lastName, @@ -147,7 +157,7 @@ class PandaAuthenticationProvider( system = providerConfiguration.getOptional[String]("panda.system").getOrElse("media-service"), bucketName = providerConfiguration.getOptional[String]("panda.bucketName").getOrElse("pan-domain-auth-settings"), settingsFileKey = providerConfiguration.getOptional[String]("panda.settingsFileKey").getOrElse(s"$domain.settings"), - s3Client = S3Ops.buildS3ClientV2(resources.commonConfig, localstackAware=resources.commonConfig.useLocalAuth) + s3Client = S3Ops.buildS3Client(resources.commonConfig, localstackAware=resources.commonConfig.useLocalAuth) ) } diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/Permissions.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/Permissions.scala index 2aefd806b7c..788752c0924 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/Permissions.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/Permissions.scala @@ -6,6 +6,7 @@ object Permissions { val app = "grid" val GridAccess: PermissionDefinition = PermissionDefinition("grid_access", app) + val UploadImages: PermissionDefinition = PermissionDefinition("upload_images", app) val EditMetadata: PermissionDefinition = PermissionDefinition("edit_metadata", app) val DeleteImage: PermissionDefinition = PermissionDefinition("delete_image", app) //FIXME I think this needs to be renamed ot `delete_crops_or_usages` - see https://github.com/guardian/grid/pull/3416 and https://github.com/guardian/permissions/pull/138 diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala index c374d5b6112..4aa072e88c9 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala @@ -25,7 +25,7 @@ class PermissionsAuthorisationProvider(configuration: Configuration, resources: provider case _ => val permissionsStage = if(config.isProd) { "PROD" } else { "CODE" } - PermissionsProvider(PermissionsConfig(permissionsStage, config.awsRegion, config.awsCredentials, permissionsBucket)) + PermissionsProvider(PermissionsConfig(permissionsStage, config.awsRegion.toString, config.awsCredentials, permissionsBucket)) } override def initialise(): Unit = { @@ -57,6 +57,7 @@ class PermissionsAuthorisationProvider(configuration: Configuration, resources: case DeleteCropsOrUsages => hasPermission(Permissions.DeleteCrops) case ShowPaid => hasPermission(Permissions.ShowPaid) case Pinboard => hasPermission(Permissions.Pinboard) + case UploadImages if config.enforceUploadImagesPermission => hasPermission(Permissions.UploadImages) case UploadImages => true case ArchiveImages => true } diff --git a/scripts/src/main/scala/com/gu/mediaservice/lib/UserCredentials.scala b/scripts/src/main/scala/com/gu/mediaservice/lib/UserCredentials.scala deleted file mode 100644 index 0a3d7a2b0e2..00000000000 --- a/scripts/src/main/scala/com/gu/mediaservice/lib/UserCredentials.scala +++ /dev/null @@ -1,25 +0,0 @@ -package com.gu.mediaservice.lib - -import java.nio.file.Paths -import java.util.Properties -import java.io.{File, FileInputStream} -import scala.jdk.CollectionConverters._ -import com.amazonaws.auth.{BasicAWSCredentials, AWSCredentials} - -object UserCredentials { - - def awsCredentials: AWSCredentials = { - val path = sys.env.getOrElse("AWS_CREDENTIAL_FILE", sys.error("Required environment variable AWS_CREDENTIAL_FILE is missing")) - val file = Paths.get(path).toFile - println(s"Reading AWS credentials from $path") - val props = properties(file) - new BasicAWSCredentials(props("AWSAccessKeyId"), props("AWSSecretKey")) - } - - def properties(file: File): Map[String, String] = { - val props = new Properties() - props.load(new FileInputStream(file)) - props.asScala.toMap - } - -} diff --git a/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala b/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala new file mode 100644 index 00000000000..22a8920e45d --- /dev/null +++ b/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala @@ -0,0 +1,112 @@ +package com.gu.mediaservice.scripts + +import play.api.libs.json.{JsValue, Json} + +import java.net.URI +import java.net.http.HttpRequest.BodyPublishers +import java.net.http.HttpResponse.BodyHandlers +import java.net.http.{HttpClient, HttpRequest} +import scala.io.Source + +object AlamyCleanUp extends App { + val GRIDKEY = sys.env.getOrElse("GRIDKEY", throw new RuntimeException("Must set a GRIDKEY env variable")) + val STAGE = sys.env.getOrElse("STAGE", throw new RuntimeException("Must set a STAGE env variable")) + val GRIDDOMAIN = if(STAGE == "PROD") "gutools.co.uk" else "test.dev-gutools.co.uk" + + println(s"Running for stage $STAGE with domain $GRIDDOMAIN") + + val supplierRefs = if (STAGE == "PROD") { + val csvPath = args.headOption.getOrElse { + throw new IllegalArgumentException("To run the script pass in a path to a csv file as an argument") + } + val source = Source.fromFile(csvPath) + try { + source + .getLines() + .drop(1) + .map(_.takeWhile(_ != ',').trim) + .filter(_.nonEmpty) + .toList + } finally { + source.close() + } + } else { + List("2BW0WK7", "AA63AP", "F208HC", "D9CNPG") + } + + if (supplierRefs.isEmpty) { + println("No supplier refs found in supplied csv, exiting script") + sys.exit(1) + } + + val ids = { + supplierRefs.zipWithIndex.flatMap({case (ref, index) => + println(s"Fetching grid ids, checking supplierRef ${index + 1} of ${supplierRefs.size}") + val client = HttpClient.newHttpClient() + val request = HttpRequest.newBuilder(new URI(s"https://api.media.$GRIDDOMAIN/images?q=suppliersReference%3A$ref")).headers("X-Gu-Media-Key", GRIDKEY).build() + val response = client.send(request, BodyHandlers.ofString()) + val json = Json.parse(response.body()) + (json \ "data").as[Seq[JsValue]].map(imageMetadata => (imageMetadata \ "data" \ "id").as[String]) + }) + } + + if (ids.isEmpty) { + println("No Grid ids found, exiting script") + sys.exit(1) + } + + println(s"Found ${ids.size} ids, now running deletion") + + val usagesBody = BodyPublishers.ofString("""{"data":{"restrictions":"No longer available from Alamy","category":"chargeable"}}""") + val labelBody = BodyPublishers.ofString("""{"data":["a2g"]}""") + val outcomes = ids.zipWithIndex.map({case (id, index) => + println(s"Running for id $id, ${index + 1} of ${ids.size}") + // Make an api request to try to delete the image, either successful or + val client = HttpClient.newHttpClient() + // attempt a hard delete + val request = HttpRequest.newBuilder(new URI(s"https://api.media.$GRIDDOMAIN/images/$id")).headers("X-Gu-Media-Key", GRIDKEY).DELETE().build() + val response = client.send(request, BodyHandlers.ofString()) + // check status response, if not 202 + // modify the usage rights + val success: Boolean = response.statusCode() match { + case 202 => { + println(s"Successfully deleted $id") + true + } + case 405 => { + println(s"Unable to delete $id, setting usages and labels instead") + val request = HttpRequest.newBuilder(new URI(s"https://media-metadata.$GRIDDOMAIN/metadata/$id/usage-rights")) + .headers("X-Gu-Media-Key", GRIDKEY, "Content-Type", "application/json").PUT(usagesBody).build() + val response = client.send(request, BodyHandlers.ofString()) + if(response.statusCode() == 200) { + println(s"Successfully set usage rights for $id") + val request = HttpRequest.newBuilder(new URI(s"https://media-metadata.$GRIDDOMAIN/metadata/$id/labels")) + .headers("X-Gu-Media-Key", GRIDKEY, "Content-Type", "application/json").POST(labelBody).build() + val response = client.send(request, BodyHandlers.ofString()) + if(response.statusCode() == 200) { + println(s"Successfully set a label for $id") + true + } + else { + println(s"Unable to set label for $id, received response ${response.statusCode()}") + false + } + } else { + println(s"Unable to set usage rights for $id, received response ${response.statusCode()}") + false + } + } + case _ => { + println(s"Got unexpected response ${response.statusCode()} calling delete for ${id}") + false + } + } + (success, id) + }) + + val (success, fail) = outcomes.partition({case (b, _) => b}) + println(s"Successfully processed ${success.size} of ${ids.size}") + if(fail.nonEmpty) { + println(s"Unable to process ${fail.size} ids: \n${fail.map(_._2).mkString("\n")}") + } +} diff --git a/scripts/src/main/scala/com/gu/mediaservice/scripts/BulkRemoveFromCollection.scala b/scripts/src/main/scala/com/gu/mediaservice/scripts/BulkRemoveFromCollection.scala new file mode 100644 index 00000000000..45615e14cfe --- /dev/null +++ b/scripts/src/main/scala/com/gu/mediaservice/scripts/BulkRemoveFromCollection.scala @@ -0,0 +1,49 @@ +package com.gu.mediaservice.scripts + +import java.net.URI +import java.net.http.{HttpClient, HttpRequest} +import java.net.http.HttpResponse.BodyHandlers +import scala.io.Source + +object BulkRemoveFromCollection extends App { + val GRIDKEY = sys.env.getOrElse("GRIDKEY", throw new RuntimeException("Must set a GRIDKEY env variable")) + val STAGE = sys.env.getOrElse("STAGE", throw new RuntimeException("Must set a STAGE env variable")) + val GRIDDOMAIN = if(STAGE == "PROD") "gutools.co.uk" else "test.dev-gutools.co.uk" + // Case-sensitive. For the nested collection Web News > test, the format would be: Web%20News/test + val COLLECTION = sys.env.getOrElse("COLLECTION", throw new RuntimeException("Must set a COLLECTION env variable")) + + val ids = if(STAGE == "PROD") { + val resource = Source.fromResource("remove-collection-grid-ids.csv") + resource.getLines().toList + } else { + // Add test IDs to a collection before running to test the script + List("f5f08f82492db26884fbb132bd25f007d0d97da7", "8dc44d419d2aa0ab4c8ebdfde9647888c1543f20", "dbaa480f3424cdd833a520bdff6b6f134282c524") + } + + println(s"Running for stage $STAGE with domain $GRIDDOMAIN with ${ids.size} ids") + + val outcomes = ids.zipWithIndex.map({case (id, index) => + println(s"Running for id $id, ${index + 1} of ${ids.size}") + // Make an api request to try to remove the image from the collection + val client = HttpClient.newHttpClient() + val request = HttpRequest.newBuilder(new URI(s"https://media-collections.$GRIDDOMAIN/images/$id/$COLLECTION")).headers("X-Gu-Media-Key", GRIDKEY).DELETE().build() + val response = client.send(request, BodyHandlers.ofString()) + val success: Boolean = response.statusCode() match { + case 200 => { + println(s"Successfully removed from collection: $id") + true + } + case _ => { + println(s"Got unexpected response ${response.statusCode()} trying to remove collection for ${id}") + false + } + } + (success, id) + }) + + val (success, fail) = outcomes.partition({case (b, _) => b}) + println(s"Successfully processed ${success.size} of ${ids.size}") + if(fail.nonEmpty) { + println(s"Unable to process ${fail.size} ids: \n${fail.map(_._2).mkString("\n")}") + } +} diff --git a/scripts/src/main/scala/com/gu/mediaservice/scripts/EnactS3Changes.scala b/scripts/src/main/scala/com/gu/mediaservice/scripts/EnactS3Changes.scala index c728ca354e5..26f5490e14f 100644 --- a/scripts/src/main/scala/com/gu/mediaservice/scripts/EnactS3Changes.scala +++ b/scripts/src/main/scala/com/gu/mediaservice/scripts/EnactS3Changes.scala @@ -1,14 +1,10 @@ package com.gu.mediaservice.scripts import java.io._ -import com.amazonaws.auth.profile.ProfileCredentialsProvider import org.apache.commons.compress.compressors.bzip2.{BZip2CompressorInputStream, BZip2CompressorOutputStream} import play.api.libs.json.{JsObject, JsValue, Json, OWrites} import scala.io.Source -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.AmazonS3ClientBuilder -import com.amazonaws.services.s3.model.AmazonS3Exception import org.joda.time.{DateTime, Duration} import com.gu.mediaservice.lib.JsonValueCodecJsValue.jsValueCodec import com.github.plokhotnyuk.jsoniter_scala.core._ diff --git a/scripts/src/main/scala/com/gu/mediaservice/scripts/LoadFromS3Bucket.scala b/scripts/src/main/scala/com/gu/mediaservice/scripts/LoadFromS3Bucket.scala index 7e7752f8b9c..3ff86bd075c 100644 --- a/scripts/src/main/scala/com/gu/mediaservice/scripts/LoadFromS3Bucket.scala +++ b/scripts/src/main/scala/com/gu/mediaservice/scripts/LoadFromS3Bucket.scala @@ -1,11 +1,12 @@ package com.gu.mediaservice.scripts -import com.amazonaws.auth.profile.ProfileCredentialsProvider -import com.amazonaws.auth.{AWSCredentialsProviderChain, InstanceProfileCredentialsProvider} -import com.amazonaws.services.s3.AmazonS3ClientBuilder import org.apache.http.client.methods.HttpPost import org.apache.http.entity.{ContentType, InputStreamEntity} import org.apache.http.impl.client.HttpClients +import software.amazon.awssdk.auth.credentials.{AwsCredentialsProviderChain, InstanceProfileCredentialsProvider, ProfileCredentialsProvider} +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.{GetObjectRequest, GetObjectResponse, ListObjectsV2Request} import scala.jdk.CollectionConverters._ @@ -18,24 +19,40 @@ object LoadFromS3Bucket { case _ => sys.error("Usage: LoadFromS3Bucket ") } - lazy val awsCredentials = new AWSCredentialsProviderChain( - new ProfileCredentialsProvider("media-service"), - InstanceProfileCredentialsProvider.getInstance() + lazy val awsCredentials = AwsCredentialsProviderChain.of( + ProfileCredentialsProvider.create("media-service"), + InstanceProfileCredentialsProvider.create() ) - val client = AmazonS3ClientBuilder.standard().withCredentials(awsCredentials).build() + val client: S3Client = S3Client.builder() + .credentialsProvider(awsCredentials) + .build() - val keys = client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey) + val keys = client.listObjectsV2( + ListObjectsV2Request.builder().bucket(bucket).build() + ).contents().asScala.map(_.key) val httpClient = HttpClients.createDefault for (key <- keys) { - val obj = client.getObject(bucket, key) + val getObjectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(key) + .build() + + val objStream: ResponseInputStream[GetObjectResponse] = client.getObject(getObjectRequest) + val length = objStream.response().contentLength() + val postReq = new HttpPost(loaderEndpoint) - val length = obj.getObjectMetadata.getContentLength - val entity = new InputStreamEntity(obj.getObjectContent, length, ContentType.DEFAULT_BINARY) + val entity = new InputStreamEntity(objStream, length, ContentType.DEFAULT_BINARY) postReq.setEntity(entity) - httpClient.execute(postReq).close() + + try { + httpClient.execute(postReq).close() + } finally { + objStream.close() + } + println(s"Loaded image $key") } diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala index 781ab62bace..cf9c5e75add 100644 --- a/thrall/app/ThrallComponents.scala +++ b/thrall/app/ThrallComponents.scala @@ -31,7 +31,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr es.ensureIndexExistsAndAliasAssigned() val services: Services = new Services(config.domainRoot, config.serviceHosts, Set.empty) - val gridClient: GridClient = GridClient(services)(wsClient) + val gridClient: GridClient = GridClient(services, services.thrallBaseUri)(wsClient) // before firing up anything to consume streams or say we are OK let's do the critical good to go check private val goodToGoCheckResult = Await.ready(GoodToGoCheck.run(es), 30 seconds) diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala index 42e454d025a..b5a62a64cc2 100644 --- a/thrall/app/controllers/ReaperController.scala +++ b/thrall/app/controllers/ReaperController.scala @@ -16,6 +16,8 @@ import org.joda.time.{DateTime, DateTimeZone} import play.api.libs.json.{JsValue, Json} import play.api.mvc.{Action, AnyContent, ControllerComponents} import scalaz.NonEmptyList +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, PutObjectRequest, DeleteObjectRequest} import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} @@ -66,7 +68,7 @@ class ReaperController( interval = INTERVAL, ){ () => try { - if (store.client.doesObjectExist(reaperBucket, CONTROL_FILE_NAME)) { + if (store.doesObjectExist(reaperBucket, CONTROL_FILE_NAME)) { logger.info("Reaper is paused") es.countTotalSoftReapable(isReapable).map(metrics.softReapable.increment(Nil, _)) es.countTotalHardReapable(isReapable, config.hardReapImagesAge).map(metrics.hardReapable.increment(Nil, _)) @@ -109,7 +111,7 @@ class ReaperController( case Some(reaperBucket) => doBatchDelete.map { json => val now = DateTime.now(DateTimeZone.UTC) val key = s"$deleteType/${s3DirNameFromDate(now)}/$deleteType-${now.toString()}.json" - store.client.putObject(reaperBucket, key, json.toString()) + store.putString(reaperBucket, key, json.toString()) json } } @@ -186,18 +188,24 @@ class ReaperController( case (None, _) => NotImplemented("'s3.reaper.bucket' not configured in thrall.conf") case (_, None) => NotImplemented("'reaper.countPerRun' not configured in thrall.conf") case (Some(reaperBucket), Some(countOfImagesToReap)) => - val isPaused = store.client.doesObjectExist(reaperBucket, CONTROL_FILE_NAME) + val isPaused = store.doesObjectExist(reaperBucket, CONTROL_FILE_NAME) val recentRecords = List(now, now.minusDays(1), now.minusDays(2)).flatMap { day => val s3DirName = s3DirNameFromDate(day) - store.client.listObjects(reaperBucket, s"soft/$s3DirName/").getObjectSummaries.asScala.toList ++ - store.client.listObjects(reaperBucket, s"hard/$s3DirName/").getObjectSummaries.asScala.toList - } + val softDeletes = store.client.listObjectsV2( + ListObjectsV2Request.builder().bucket(reaperBucket).prefix(s"soft/$s3DirName/").build() + ).contents().asScala.toList + + val hardDeletes = store.client.listObjectsV2( + ListObjectsV2Request.builder().bucket(reaperBucket).prefix(s"hard/$s3DirName/").build() + ).contents().asScala.toList + softDeletes ++ hardDeletes + } val recentRecordKeys = recentRecords - .filter(_.getLastModified after now.minusHours(48).toDate) - .sortBy(_.getLastModified) + .filter(_.lastModified() isAfter now.minusHours(48).toDate.toInstant) + .sortBy(_.lastModified()) .reverse - .map(_.getKey) + .map(_.key()) Ok(views.html.reaper(isPaused, INTERVAL.toString(), countOfImagesToReap, recentRecordKeys)) }} @@ -205,22 +213,23 @@ class ReaperController( def reaperRecord(key: String) = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - Ok( - store.client.getObjectAsString(reaperBucket, key) - ).as(JSON) + store.getObjectAsString(reaperBucket, key) match { + case Some(res) => Ok(res).as(JSON) + case None => NotFound + } }} def pauseReaper = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - store.client.putObject(reaperBucket, CONTROL_FILE_NAME, "") + store.putString(reaperBucket, CONTROL_FILE_NAME, "") Redirect(routes.ReaperController.index) }} def resumeReaper = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - store.client.deleteObject(reaperBucket, CONTROL_FILE_NAME) + store.client.deleteObject(DeleteObjectRequest.builder().bucket(reaperBucket).key(CONTROL_FILE_NAME).build()) Redirect(routes.ReaperController.index) }} diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala index 72be443c96a..f0ecea7fd6c 100644 --- a/thrall/app/controllers/ThrallController.scala +++ b/thrall/app/controllers/ThrallController.scala @@ -38,7 +38,7 @@ class ThrallController( private val numberFormatter: Long => String = java.text.NumberFormat.getIntegerInstance().format - def index = withLoginRedirectAsync { + def index = withLoginRedirectAsync { implicit request => val countDocsInIndex = OptionalFutureRunner.run(es.countImages) _ for { currentIndex <- es.getIndexForAlias(es.imagesCurrentAlias) @@ -101,7 +101,7 @@ class ThrallController( } } - def migrationFailures(filter: String, maybePage: Option[Int]): Action[AnyContent] = withLoginRedirectAsync { + def migrationFailures(filter: String, maybePage: Option[Int]): Action[AnyContent] = withLoginRedirectAsync { implicit request => Paging.withPaging(maybePage) { paging => es.migrationStatus match { case running: Running => diff --git a/thrall/app/lib/SyncChecker.scala b/thrall/app/lib/SyncChecker.scala index 959057ce581..112a04325f2 100644 --- a/thrall/app/lib/SyncChecker.scala +++ b/thrall/app/lib/SyncChecker.scala @@ -4,11 +4,11 @@ import org.apache.pekko.Done import org.apache.pekko.actor.ActorSystem import org.apache.pekko.stream.{Materializer, RestartSettings} import org.apache.pekko.stream.scaladsl.{RestartSource, Source} -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ListObjectsV2Request import com.gu.mediaservice.lib.elasticsearch.InProgress import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap} import lib.elasticsearch.ElasticSearch +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request import scala.jdk.CollectionConverters._ import scala.concurrent.duration.DurationInt @@ -20,7 +20,7 @@ case class Prefix(prefix: String) extends SyncCheckJob case object Other extends SyncCheckJob class SyncChecker( - s3: AmazonS3, + s3: S3Client, es: ElasticSearch, imageBucketName: String, actorSystem: ActorSystem @@ -36,15 +36,15 @@ class SyncChecker( z <- expectedAlphabet } yield Prefix(s"$x$y$z")) :+ Other - private def paginatedListObjects(request: ListObjectsV2Request, maybeContinuationToken: Option[String] = None): Future[Seq[String]] = { + private def paginatedListObjects(request: ListObjectsV2Request.Builder, maybeContinuationToken: Option[String] = None): Future[Seq[String]] = { Future { val fullRequest = maybeContinuationToken match { - case Some(token) => request.withContinuationToken(token) + case Some(token) => request.continuationToken(token) case None => request } - val result = s3.listObjectsV2(fullRequest) - val keys = result.getObjectSummaries.asScala.toSeq.map(_.getKey.split("/").last) - val nextContinuationToken = Option(result.getNextContinuationToken) + val result = s3.listObjectsV2(fullRequest.build()) + val keys = result.contents().asScala.toSeq.map(_.key().split("/").last) + val nextContinuationToken = Option(result.nextContinuationToken()) keys -> nextContinuationToken } flatMap { case (keys, None) => Future.successful(keys) @@ -54,15 +54,11 @@ class SyncChecker( } private def listFilesWithPrefix(prefix: String): Future[Seq[String]] = { - val request = new ListObjectsV2Request() - .withBucketName(imageBucketName) - .withPrefix(prefix) + val request = ListObjectsV2Request.builder().bucket(imageBucketName).prefix(prefix) paginatedListObjects(request) } private def listFilesAtRoot(): Future[Seq[String]] = { - val request = new ListObjectsV2Request() - .withBucketName(imageBucketName) - .withDelimiter("/") + val request = ListObjectsV2Request.builder().bucket(imageBucketName).delimiter("/") paginatedListObjects(request) } diff --git a/thrall/app/lib/ThrallConfig.scala b/thrall/app/lib/ThrallConfig.scala index a5eeab49d86..cd331ad9835 100644 --- a/thrall/app/lib/ThrallConfig.scala +++ b/thrall/app/lib/ThrallConfig.scala @@ -1,6 +1,6 @@ package lib -import com.gu.mediaservice.lib.aws.AwsClientV2BuilderUtils +import com.gu.mediaservice.lib.aws.AwsClientBuilderUtils import com.gu.mediaservice.lib.cleanup.ReapableEligibiltyResources import com.gu.mediaservice.lib.config.{CommonConfigWithElastic, GridConfigResources, ReapableEligibilityLoader} import com.gu.mediaservice.lib.elasticsearch.ReapableEligibility @@ -21,16 +21,16 @@ import scala.concurrent.duration.{DurationInt, FiniteDuration} import scala.language.postfixOps case class KinesisReceiverConfig( - override val awsRegionV2: Region, - override val awsCredentialsV2: AwsCredentialsProvider, + override val awsRegion: Region, + override val awsCredentials: AwsCredentialsProvider, override val awsLocalEndpointUri: Option[URI], override val isDev: Boolean, streamName: String, rewindFrom: Option[DateTime], metricsLevel: MetricsLevel = MetricsLevel.DETAILED -) extends AwsClientV2BuilderUtils { +) extends AwsClientBuilderUtils { lazy val kinesisClient: KinesisAsyncClient = { - val clientBuilder = withAWSCredentialsV2(KinesisAsyncClient.builder()) + val clientBuilder = withAWSCredentials(KinesisAsyncClient.builder()) if (isDev) { clientBuilder.httpClientBuilder(NettyNioAsyncHttpClient .builder() @@ -38,14 +38,14 @@ case class KinesisReceiverConfig( } clientBuilder.build() } - lazy val dynamoClient: DynamoDbAsyncClient = withAWSCredentialsV2(DynamoDbAsyncClient.builder()).build() - lazy val cloudwatchClient: CloudWatchAsyncClient = withAWSCredentialsV2(CloudWatchAsyncClient.builder()).build() + lazy val dynamoClient: DynamoDbAsyncClient = withAWSCredentials(DynamoDbAsyncClient.builder()).build() + lazy val cloudwatchClient: CloudWatchAsyncClient = withAWSCredentials(CloudWatchAsyncClient.builder()).build() } object KinesisReceiverConfig { def apply(streamName: String, rewindFrom: Option[DateTime], thrallConfig: ThrallConfig): KinesisReceiverConfig = KinesisReceiverConfig( - thrallConfig.awsRegionV2, - thrallConfig.awsCredentialsV2, + thrallConfig.awsRegion, + thrallConfig.awsCredentials, thrallConfig.awsLocalEndpointUri, thrallConfig.isDev, streamName, diff --git a/thrall/app/lib/elasticsearch/GoodToGoCheck.scala b/thrall/app/lib/elasticsearch/GoodToGoCheck.scala index 601d524a61e..7df65a1d500 100644 --- a/thrall/app/lib/elasticsearch/GoodToGoCheck.scala +++ b/thrall/app/lib/elasticsearch/GoodToGoCheck.scala @@ -1,16 +1,14 @@ package lib.elasticsearch -import com.amazonaws.util.EC2MetadataUtils +import com.gu.mediaservice.lib.aws.EC2MetadataUtils +import software.amazon.awssdk.imds.Ec2MetadataClient import com.gu.mediaservice.lib.elasticsearch.{MappingTest, Running} import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.Image -import com.sksamuel.elastic4s.Response -import com.sksamuel.elastic4s.requests.delete.{DeleteByQueryResponse, DeleteResponse} import com.typesafe.scalalogging.StrictLogging import org.joda.time.{DateTime, Period} import scala.concurrent.{ExecutionContext, Future} -import scala.util.Success /** * This implements a good to go check that validates when Thrall starts up that it can successfully write and read @@ -19,7 +17,7 @@ import scala.util.Success object GoodToGoCheck extends StrictLogging { def run(es: ElasticSearch)(implicit ex: ExecutionContext): Future[Unit] = { // make a test image with an ID that is unique to the node so that multiple nodes can run the check at once - val uniqueId = Option(EC2MetadataUtils.getInstanceId).map(_.stripPrefix("i-")).getOrElse("abcdef") + val uniqueId = EC2MetadataUtils.getInstanceId.map(_.stripPrefix("i-")).getOrElse("abcdef") val checkTime = DateTime.now val image: Image = MappingTest.testImage.copy( id = s"${MappingTest.testImage.id}$uniqueId", diff --git a/thrall/app/views/index.scala.html b/thrall/app/views/index.scala.html index bbc8134d8a3..45c935b0137 100644 --- a/thrall/app/views/index.scala.html +++ b/thrall/app/views/index.scala.html @@ -13,13 +13,14 @@ migrationIndexCount: String, migrationStatus: MigrationStatus, hasHistoricalIndex: Boolean -) +)(implicit request: RequestHeader) @migrationIndexCell = { @migrationStatus match { case running: Running => { @running.migrationIndexName @form(action = routes.ThrallController.migrateSingleImage){ + @CSRF.formField @@ -27,6 +28,7 @@ } case NotRunning => { @form(routes.ThrallController.startMigration) { + @CSRF.formField } @@ -72,12 +74,14 @@

Current elasticsearch indices

case CompletionPreview(_) => {} case Paused(_) => { @form(routes.ThrallController.resumeMigration) { + @CSRF.formField PAUSED } } case InProgress(_) => { @form(routes.ThrallController.pauseMigration) { + @CSRF.formField } } @@ -108,15 +112,18 @@

Complete Migration?

and we're happy that all images in failure can become inaccessible). @if(migrationStatus.isInstanceOf[CompletionPreview]) { @form(routes.ThrallController.unPreviewMigrationCompletion) { + @CSRF.formField (this will make grid search both indexes again) } @form(routes.ThrallController.completeMigration()) { + @CSRF.formField or fully complete the migration: } } else { @form(routes.ThrallController.previewMigrationCompletion) { + @CSRF.formField
This will make grid only search the migration index (both indexes will continue to be populated diff --git a/thrall/app/views/migrationFailures.scala.html b/thrall/app/views/migrationFailures.scala.html index ecc69cffa37..22b8ce9d5b0 100644 --- a/thrall/app/views/migrationFailures.scala.html +++ b/thrall/app/views/migrationFailures.scala.html @@ -1,6 +1,7 @@ @import lib.FailedMigrationSummary @import views.html.helper.form +@import views.html.helper.CSRF @( failures: FailedMigrationSummary, apiBaseUrl: String, @@ -8,7 +9,7 @@ filter: String, page: Int, shouldAllowReattempts: Boolean, -) +)(implicit request: RequestHeader) @navigation = { @if(page > 1) { @@ -33,6 +34,7 @@

@failures.totalFailed images failed with the above message!

@if(shouldAllowReattempts) { @form(action = routes.ThrallController.reattemptMigrationFailures(filter, page)){ + @CSRF.formField } } @@ -63,6 +65,7 @@

@failure.uploadTime @if(shouldAllowReattempts) { @form(action = routes.ThrallController.migrateSingleImage){ + @CSRF.formField diff --git a/thrall/app/views/restoreFromReplica.scala.html b/thrall/app/views/restoreFromReplica.scala.html index a512108a8fe..513b274fed7 100644 --- a/thrall/app/views/restoreFromReplica.scala.html +++ b/thrall/app/views/restoreFromReplica.scala.html @@ -22,6 +22,7 @@

Restore from replica

Useful if the image has been hard-deleted.

@form(Call("POST", imageLoaderEndpoint)) { + @CSRF.formField diff --git a/thrall/app/views/upsertProject.scala.html b/thrall/app/views/upsertProject.scala.html index 3ba607882af..7a9dd5f2e5c 100644 --- a/thrall/app/views/upsertProject.scala.html +++ b/thrall/app/views/upsertProject.scala.html @@ -24,6 +24,7 @@

Upsert image via projection

If you're happy, then enter the media ID below and hit 'submit'...

@form(routes.ThrallController.upsertProjectPage(None)) { + @CSRF.formField diff --git a/thrall/conf/routes b/thrall/conf/routes index 520e83b2ee9..977f323ab57 100644 --- a/thrall/conf/routes +++ b/thrall/conf/routes @@ -7,35 +7,23 @@ GET /upsertProject controllers.ThrallControll GET /restoreFromReplica controllers.ThrallController.restoreFromReplica GET /migrationFailuresOverview controllers.ThrallController.migrationFailuresOverview() GET /migrationFailures controllers.ThrallController.migrationFailures(filter: String, page: Option[Int]) -+nocsrf POST /migrationFailures/reattempt controllers.ThrallController.reattemptMigrationFailures(filter: String, page: Int) -+nocsrf POST /startMigration controllers.ThrallController.startMigration -+nocsrf POST /pauseMigration controllers.ThrallController.pauseMigration -+nocsrf POST /resumeMigration controllers.ThrallController.resumeMigration -+nocsrf POST /migrate controllers.ThrallController.migrateSingleImage POST /upsertProject controllers.ThrallController.upsertFromProjectionSingleImage -+nocsrf POST /completeMigration controllers.ThrallController.completeMigration() -+nocsrf POST /previewMigrationCompletion controllers.ThrallController.previewMigrationCompletion -+nocsrf POST /unPreviewMigrationCompletion controllers.ThrallController.unPreviewMigrationCompletion # reaper GET /reaper controllers.ReaperController.index GET /reaper/:key controllers.ReaperController.reaperRecord(key: String) -+nocsrf DELETE /doBatchSoftReap controllers.ReaperController.doBatchSoftReap(count: Int) -+nocsrf DELETE /doBatchHardReap controllers.ReaperController.doBatchHardReap(count: Int) -+nocsrf POST /pauseReaper controllers.ReaperController.pauseReaper -+nocsrf POST /resumeReaper controllers.ReaperController.resumeReaper # Management diff --git a/usage/app/lib/ContentApis.scala b/usage/app/lib/ContentApis.scala index 4ce51ce6909..3f5eaa12296 100644 --- a/usage/app/lib/ContentApis.scala +++ b/usage/app/lib/ContentApis.scala @@ -1,8 +1,12 @@ package lib -import com.amazonaws.auth.profile.ProfileCredentialsProvider -import com.amazonaws.auth.{AWSCredentialsProvider, AWSCredentialsProviderChain, STSAssumeRoleSessionCredentialsProvider} -import com.amazonaws.services.securitytoken.{AWSSecurityTokenService, AWSSecurityTokenServiceClientBuilder} +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest +import software.amazon.awssdk.services.sts.StsClient +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider + import com.gu.contentapi.client._ import com.gu.contentapi.client.model.{HttpResponse, ItemQuery} @@ -47,16 +51,21 @@ class PreviewContentApi(protected val config: UsageConfig)(implicit val ex: Sche trait IAMAuthContentApiClient extends ContentApiClient { protected val config: UsageConfig - lazy val sts: AWSSecurityTokenService = AWSSecurityTokenServiceClientBuilder.standard() - .withRegion(config.awsRegionName) + lazy val sts: StsClient = StsClient.builder() + .region(Region.of(config.awsRegionName)) .build() - lazy val capiCredentials: AWSCredentialsProvider = - config.capiPreviewRole.map( - new STSAssumeRoleSessionCredentialsProvider.Builder(_, "capi") - .withStsClient(sts) + private lazy val sessionId: String = "session-" + Math.random() + lazy val capiCredentials: AwsCredentialsProvider = + config.capiPreviewRole.map(arn => { + + val assumeRoleRequest = AssumeRoleRequest.builder().roleArn(arn).roleSessionName(sessionId).build() + + StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(assumeRoleRequest) + .stsClient(sts) .build() - ).getOrElse(new ProfileCredentialsProvider("capi")) // will be used if stream is ever run locally (unusual) + }).getOrElse(ProfileCredentialsProvider.create("capi")) // will be used if stream is ever run locally (unusual) abstract override def get( url: String, diff --git a/usage/app/lib/UsageConfig.scala b/usage/app/lib/UsageConfig.scala index 19f07e6e7e1..e2b7caa28ed 100644 --- a/usage/app/lib/UsageConfig.scala +++ b/usage/app/lib/UsageConfig.scala @@ -1,9 +1,10 @@ package lib -import com.amazonaws.services.identitymanagement._ import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.lib.net.URI.ensureSecure +import software.amazon.awssdk.core.exception.SdkClientException +import software.amazon.awssdk.services.iam.IamClient import scala.util.Try @@ -37,13 +38,13 @@ class UsageConfig(resources: GridConfigResources) extends CommonConfig(resources val awsRegionName = string("aws.region") - private val iamClient: AmazonIdentityManagement = withAWSCredentials(AmazonIdentityManagementClientBuilder.standard()).build() + private val iamClient: IamClient = withAWSCredentials(IamClient.builder()).build() val postfix: String = if (isDev) { try { - iamClient.getUser.getUser.getUserName + iamClient.getUser.user().userName() } catch { - case e:com.amazonaws.SdkClientException => + case e: SdkClientException => logger.warn("Unable to determine current IAM user, probably because you're using temp credentials. Usage may not be able to determine the live/preview app names", e) "tempcredentials" } diff --git a/usage/app/model/UsageRecord.scala b/usage/app/model/UsageRecord.scala index 69e286ea78e..61da9b2df6e 100644 --- a/usage/app/model/UsageRecord.scala +++ b/usage/app/model/UsageRecord.scala @@ -1,17 +1,19 @@ package model -import com.amazonaws.services.dynamodbv2.xspec.{ExpressionSpecBuilder, UpdateItemExpressionSpec} -import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder.{M, N, S} +import software.amazon.awssdk.services.dynamodb.model.AttributeValue +import scala.jdk.CollectionConverters._ import com.gu.mediaservice.model.usage._ -import scala.jdk.CollectionConverters._ +import com.gu.mediaservice.lib.dynamo.DynamoElement import org.joda.time.DateTime +import software.amazon.awssdk.enhanced.dynamodb.Expression sealed trait DateRemovedOperation case object ClearDateRemoved extends DateRemovedOperation case object LeaveDateRemovedUntouched extends DateRemovedOperation case class SetDateRemoved(dateRemoved: DateTime) extends DateRemovedOperation + case class UsageRecord( hashKey: String, rangeKey: String, @@ -29,28 +31,85 @@ case class UsageRecord( childUsageMetadata: Option[ChildUsageMetadata] = None, dateAdded: Option[DateTime] = None ) { - def toXSpec: UpdateItemExpressionSpec = { - val specBuilder = new ExpressionSpecBuilder - List( - mediaId.filter(_.nonEmpty).map(S("media_id").set(_)), - usageType.map(usageType => S("usage_type").set(usageType.toString)), - mediaType.filter(_.nonEmpty).map(S("media_type").set(_)), - lastModified.map(lastMod => N("last_modified").set(lastMod.getMillis)), - usageStatus.filter(_.nonEmpty).map(S("usage_status").set(_)), - printUsageMetadata.map(_.toMap).map(map => M("print_metadata").set(map.asJava)), - digitalUsageMetadata.map(_.toMap).map(map => M("digital_metadata").set(map.asJava)), - syndicationUsageMetadata.map(_.toMap).map(map => M("syndication_metadata").set(map.asJava)), - frontUsageMetadata.map(_.toMap).map(map => M("front_metadata").set(map.asJava)), - downloadUsageMetadata.map(_.toMap).map(map => M("download_metadata").set(map.asJava)), - childUsageMetadata.map(_.toMap).map(map => M("child_metadata").set(map.asJava)), - dateAdded.map(dateAdd => N("date_added").set(dateAdd.getMillis)), - dateRemovedOperation match { - case ClearDateRemoved => Some(N("date_removed").remove) - case LeaveDateRemovedUntouched => None - case SetDateRemoved(dateRemoved) => Some(N("date_removed").set(dateRemoved.getMillis)) - } - ).flatten.foreach(specBuilder.addUpdate) - specBuilder.buildForUpdate + def toAttributeValueMap(m: Map[String, DynamoElement]):java.util.Map[String, AttributeValue] = { + m.map { case(k, v) => + k -> v.toAttrValue + } + }.asJava + def toExpression: Expression = { + val setOps = scala.collection.mutable.ListBuffer.empty[(String, AttributeValue)] + val removeOps = scala.collection.mutable.ListBuffer.empty[String] + val names = scala.collection.mutable.Map.empty[String, String] + + def setS(field: String, value: String): Unit = { + val name = s"#$field" + val valueKey = s":$field" + names += name -> field + setOps += valueKey -> AttributeValue.builder().s(value).build() + } + + def setN(field: String, value: Long): Unit = { + val name = s"#$field" + val valueKey = s":$field" + names += name -> field + setOps += valueKey -> AttributeValue.builder().n(value.toString).build() + } + + def setM(field: String, value: java.util.Map[String, AttributeValue]): Unit = { + val name = s"#$field" + val valueKey = s":$field" + names += name -> field + setOps += valueKey -> AttributeValue.builder().m(value).build() + } + + mediaId.filter(_.nonEmpty).foreach(setS("media_id", _)) + + usageType.foreach(t => setS("usage_type", t.toString)) + + mediaType.filter(_.nonEmpty).foreach(setS("media_type", _)) + + lastModified.foreach(dt => setN("last_modified", dt.getMillis)) + + usageStatus.filter(_.nonEmpty).foreach(setS("usage_status", _)) + + printUsageMetadata.foreach(p => setM("print_metadata", toAttributeValueMap(p.toDynamoMap))) + digitalUsageMetadata.foreach(m => setM("digital_metadata", toAttributeValueMap(m.toDynamoMap))) + syndicationUsageMetadata.foreach(m => setM("syndication_metadata", toAttributeValueMap(m.toDynamoMap))) + frontUsageMetadata.foreach(m => setM("front_metadata", toAttributeValueMap(m.toDynamoMap))) + downloadUsageMetadata.foreach(m => setM("download_metadata", toAttributeValueMap(m.toDynamoMap))) + childUsageMetadata.foreach(m => setM("child_metadata", toAttributeValueMap(m.toDynamoMap))) + + dateAdded.foreach(dt => setN("date_added", dt.getMillis)) + + dateRemovedOperation match { + case ClearDateRemoved => + removeOps += "date_removed" + + case LeaveDateRemovedUntouched => + () + + case SetDateRemoved(dt) => + setN("date_removed", dt.getMillis) + } + + val setExpr = + if (setOps.nonEmpty) + "SET " + setOps.map { case (v, _) => s"#${v.drop(1)} = $v" }.mkString(", ") + else "" + + val removeExpr = + if (removeOps.nonEmpty) + "REMOVE " + removeOps.map(f => s"$f").mkString(", ") + else "" + + val expression = + Seq(setExpr, removeExpr).filter(_.nonEmpty).mkString(" ") + + Expression.builder() + .expression(expression) + .expressionValues(setOps.toMap.asJava) + .expressionNames(names.asJava) + .build() } } diff --git a/usage/app/model/UsageTable.scala b/usage/app/model/UsageTable.scala index 356c175b544..720faa7e73c 100644 --- a/usage/app/model/UsageTable.scala +++ b/usage/app/model/UsageTable.scala @@ -1,34 +1,53 @@ package model -import com.amazonaws.services.dynamodbv2.document.spec.{DeleteItemSpec, QuerySpec, UpdateItemSpec} -import com.amazonaws.services.dynamodbv2.document.{DeleteItemOutcome, KeyAttribute, RangeKeyCondition} -import com.amazonaws.services.dynamodbv2.model.ReturnValue -import com.gu.mediaservice.lib.aws.DynamoDB +import com.gu.mediaservice.lib.AttributeValueConverterProvider +import com.gu.mediaservice.lib.aws.DynamoDB.jsonWithNullAsEmptyString import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.lib.usage.ItemToMediaUsage import com.gu.mediaservice.model.usage.{MediaUsage, PendingUsageStatus, PublishedUsageStatus, UsageTableFullKey} import lib.{BadInputException, UsageConfig, WithLogMarker} import play.api.libs.json._ import rx.lang.scala.Observable +import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument +import software.amazon.awssdk.enhanced.dynamodb.model.{DeleteItemEnhancedRequest, QueryConditional, QueryEnhancedRequest, UpdateItemEnhancedRequest} +import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, Key, TableMetadata, TableSchema} +import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, ReturnValue, UpdateItemRequest} -import scala.jdk.CollectionConverters._ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future +import scala.jdk.CollectionConverters.{IterableHasAsScala, IteratorHasAsScala, MapHasAsJava, MapHasAsScala} -class UsageTable(config: UsageConfig) extends DynamoDB(config, config.usageRecordTable) with GridLogging { +class UsageTable(config: UsageConfig) extends GridLogging { val hashKeyName = "grouping" val rangeKeyName = "usage_id" val imageIndexName = "media_id" + + lazy val client: DynamoDbClient = config.withAWSCredentials(DynamoDbClient.builder()).build() + lazy val dynamo: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client).build() + lazy val tableSchema = TableSchema.documentSchemaBuilder() + .addIndexPartitionKey(TableMetadata.primaryIndexName(), hashKeyName, AttributeValueType.S) + .addIndexSortKey(TableMetadata.primaryIndexName(), rangeKeyName, AttributeValueType.S) + .addIndexPartitionKey( + imageIndexName, + imageIndexName, + AttributeValueType.S + ) + .attributeConverterProviders(AttributeValueConverterProvider, AttributeConverterProvider.defaultProvider()) + .build() + lazy val table = dynamo.table(config.usageRecordTable, tableSchema) + def queryByUsageId(id: String): Future[Option[MediaUsage]] = Future { UsageTableFullKey.build(id).flatMap((tableFullKey: UsageTableFullKey) => { - val keyAttribute: KeyAttribute = new KeyAttribute(hashKeyName, tableFullKey.hashKey) - val rangeKeyCondition: RangeKeyCondition = new RangeKeyCondition(rangeKeyName).eq(tableFullKey.rangeKey) - - val queryResult = table.query(keyAttribute, rangeKeyCondition) - queryResult.asScala.map(ItemToMediaUsage.transform).headOption + val key = Key.builder() + .partitionValue(tableFullKey.hashKey) + .sortValue(tableFullKey.rangeKey) + .build() + val queryResult = table.query(QueryConditional.keyEqualTo(key)) + queryResult.items().asScala.map(ItemToMediaUsage.transform).headOption }) } @@ -38,11 +57,18 @@ class UsageTable(config: UsageConfig) extends DynamoDB(config, config.usageRecor throw new BadInputException("Empty string received for image id") logger.info(logMarkerWithId, s"Querying usages table for $id") - val imageIndex = table.getIndex(imageIndexName) - val keyAttribute = new KeyAttribute(imageIndexName, id) - val queryResult = imageIndex.query(keyAttribute) - val unsortedUsages = queryResult.asScala.map(ItemToMediaUsage.transform).toList + val key = Key.builder() + .partitionValue(id) + .build() + + val queryResult = table.index(imageIndexName).query(QueryConditional.keyEqualTo(key)) + + val unsortedUsages = queryResult.iterator() + .asScala + .flatMap(_.items().asScala) + .map(ItemToMediaUsage.transform) + .toList logger.info(logMarkerWithId, s"Query of usages table for $id found ${unsortedUsages.size} results") @@ -86,13 +112,17 @@ class UsageTable(config: UsageConfig) extends DynamoDB(config, config.usageRecor val grouping = usageGroup.grouping logger.info(logMarker, s"Querying table for $grouping") - val queryResult = table.query( - new QuerySpec() - .withConsistentRead(true) - .withHashKey(new KeyAttribute("grouping", grouping)) - ) - - val usages = queryResult.asScala + val key = Key.builder() + .partitionValue(grouping) + .build() + val request = + QueryEnhancedRequest.builder() + .queryConditional(QueryConditional.keyEqualTo(key)) + .consistentRead(true) + .build() + val queryResult = table.query(request) + + val usages = queryResult.items().asScala .map(ItemToMediaUsage.transform) .toSet @@ -111,36 +141,41 @@ class UsageTable(config: UsageConfig) extends DynamoDB(config, config.usageRecor def markAsRemoved(mediaUsage: MediaUsage)(implicit logMarker: LogMarker): Observable[JsObject] = upsertFromRecord(UsageRecord.buildMarkAsRemovedRecord(mediaUsage)) - def deleteRecord(mediaUsage: MediaUsage)(implicit logMarker: LogMarker): DeleteItemOutcome = { + def deleteRecord(mediaUsage: MediaUsage)(implicit logMarker: LogMarker): EnhancedDocument = { logger.info(logMarker, s"deleting usage ${mediaUsage.usageId} for media id ${mediaUsage.mediaId}") - val deleteSpec = new DeleteItemSpec() - .withPrimaryKey( - hashKeyName, mediaUsage.grouping, - rangeKeyName, mediaUsage.usageId.toString - ) + val key = Key.builder() + .partitionValue(mediaUsage.grouping) + .sortValue(mediaUsage.usageId.toString) + .build() - table.deleteItem(deleteSpec) + table.deleteItem(DeleteItemEnhancedRequest.builder().key(key).build()) } def upsertFromRecord(record: UsageRecord)(implicit logMarker: LogMarker): Observable[JsObject] = Observable.from(Future { - - val updateSpec = new UpdateItemSpec() - .withPrimaryKey( - hashKeyName, - record.hashKey, - rangeKeyName, - record.rangeKey - ) - .withExpressionSpec(record.toXSpec) - .withReturnValues(ReturnValue.ALL_NEW) - - table.updateItem(updateSpec) + val key = Map( + hashKeyName -> AttributeValue.builder().s(record.hashKey).build(), + rangeKeyName -> AttributeValue.builder().s(record.rangeKey).build() + ).asJava + + val request = UpdateItemRequest.builder() + .tableName(table.tableName()) + .key(key) + .updateExpression(record.toExpression.expression()) + .expressionAttributeNames(record.toExpression.expressionNames()) + .expressionAttributeValues(record.toExpression.expressionValues()) + .returnValues(ReturnValue.ALL_NEW) + .build() + + client.updateItem(request) }) .onErrorResumeNext(e => { logger.error(logMarker, s"Dynamo update fail for $record!", e) Observable.error(e) }) - .map(asJsObject) + .map(updateResponse => { + val doc = EnhancedDocument.fromAttributeValueMap(updateResponse.attributes()) + jsonWithNullAsEmptyString(Json.parse(doc.toJson)).as[JsObject] + }) } diff --git a/usage/conf/routes b/usage/conf/routes index 8a1456cbd3f..ebed0d15332 100644 --- a/usage/conf/routes +++ b/usage/conf/routes @@ -8,7 +8,6 @@ POST /usages/print controllers.UsageApi.set POST /usages/syndication controllers.UsageApi.setSyndicationUsages() POST /usages/front controllers.UsageApi.setFrontUsages() POST /usages/download controllers.UsageApi.setDownloadUsages() -+nocsrf POST /usages/child controllers.UsageApi.setChildUsages() PUT /usages/status/update/:mediaId/*usageId controllers.UsageApi.updateUsageStatus(mediaId: String, usageId: String) GET /usages/digital/content/*contentId/reindex controllers.UsageApi.reindexForContent(contentId: String) diff --git a/usage/test/model/UsageRecordTest.scala b/usage/test/model/UsageRecordTest.scala new file mode 100644 index 00000000000..83aac5bd477 --- /dev/null +++ b/usage/test/model/UsageRecordTest.scala @@ -0,0 +1,387 @@ +package model + +import com.gu.mediaservice.lib.dynamo.{DbInt, DbNestedMap, DbString} +import com.gu.mediaservice.model.usage.{ChildUsageMetadata, DigitalUsageMetadata, DownloadUsageMetadata, FrontUsageMetadata, PrintImageSize, PrintUsage, PrintUsageMetadata, SyndicationUsageMetadata} +import software.amazon.awssdk.services.dynamodb.model.AttributeValue +import org.joda.time.DateTime +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +import java.net.URI +import scala.jdk.CollectionConverters.{CollectionHasAsScala, MapHasAsJava, MapHasAsScala} +class UsageRecordTest extends AnyFunSpec with Matchers { + + def getMapFromValues(rawValues: Iterable[AnyRef]) = { + rawValues.collectFirst { + case javaMap: java.util.Map[_, _] => + javaMap.asInstanceOf[java.util.Map[String, Any]].asScala.toMap + }.getOrElse(fail("Could not find the value map")) + } + + describe("toExpression") { + + it("should handle an empty record with default/None values correctly") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched + ) + val expression = record.toExpression + val names = expression.expressionNames().values().asScala + names shouldBe empty + + val values = expression.expressionValues().values().asScala + + values shouldBe empty + expression.expression() shouldBe "" + } + + it("should include standard String and UsageType updates when present") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + mediaId = Some("media-789"), + usageType = Some(PrintUsage), + mediaType = Some("image"), + usageStatus = Some("active") + ) + + val expression = record.toExpression + + val names = expression.expressionNames().values().asScala + + names.size shouldBe 4 + names should contain ("media_id") + names should contain ("usage_type") + names should contain ("media_type") + names should contain ("usage_status") + + val values = expression.expressionValues().values().asScala + values.size shouldBe 4 + values should contain (AttributeValue.builder.s("media-789").build()) + values should contain (AttributeValue.builder.s(PrintUsage.toString).build()) + values should contain (AttributeValue.builder.s("image").build()) + values should contain (AttributeValue.builder.s("active").build()) + + val setExp = "SET #media_id = :media_id, #usage_type = :usage_type," + + " #media_type = :media_type, #usage_status = :usage_status" + + expression.expression() shouldEqual setExp + } + describe("dateRemovedOperation") { + it("should generate a set block when SetDateRemoved is specified") { + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = SetDateRemoved(DateTime.now()) + ) + + val expression = record.toExpression + val names = expression.expressionNames().values().asScala + names.size shouldBe 1 + names should contain ("date_removed") + + expression.expression() shouldEqual "SET #date_removed = :date_removed" + } + it("should generate a REMOVE block when ClearDateRemoved is specified") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = ClearDateRemoved + ) + + val expression = record.toExpression + expression.expression() shouldEqual "REMOVE date_removed" + } + it("should handle a mix of remove and set updates when the relevant fields are set") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = ClearDateRemoved, + mediaId = Some("media-789") + ) + + val expression = record.toExpression + + val names = expression.expressionNames().values().asScala + names.size shouldBe 1 + names should contain ("media_id") + expression.expression() shouldEqual "SET #media_id = :media_id REMOVE date_removed" + } + } + describe("PrintUsageMetadata") { + it("should handle PrintUsageMetadata with empty values") { + val targetDate = DateTime.parse("2026-07-07T12:00:00Z") + val metadata = PrintUsageMetadata( + sectionName = "News", + issueDate = targetDate, + pageNumber = 5, + storyName = "Scala Breakthrough", + publicationCode = "PUB1", + publicationName = "The Tech Daily", + sectionCode = "SEC-A", + layoutId = None, + edition = None, + size = None, + orderedBy = None, + notes = None, + source = None + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + printUsageMetadata = Some(metadata) + ) + val expression = record.toExpression + val rawValues = expression.expressionValues().values().asScala + + val printMetadataMap = rawValues.head.m().asScala + + printMetadataMap("sectionName") shouldBe DbString("News").toAttrValue + printMetadataMap("issueDate") shouldBe DbString("2026-07-07T12:00:00.000Z").toAttrValue + printMetadataMap("pageNumber") shouldBe DbInt(5).toAttrValue + printMetadataMap("storyName") shouldBe DbString("Scala Breakthrough").toAttrValue + printMetadataMap("publicationCode") shouldBe DbString("PUB1").toAttrValue + printMetadataMap("publicationName") shouldBe DbString("The Tech Daily").toAttrValue + printMetadataMap("sectionCode") shouldBe DbString("SEC-A").toAttrValue + + printMetadataMap should have size 7 + } + it("should handle PrintUsageMetadata") { + val targetDate = DateTime.parse("2026-07-07T12:00:00Z") + val metadata = PrintUsageMetadata( + sectionName = "News", + issueDate = targetDate, + pageNumber = 5, + storyName = "Scala Breakthrough", + publicationCode = "PUB1", + publicationName = "The Tech Daily", + sectionCode = "SEC-A", + layoutId = Some(12345L), + edition = Some(2), + size = Some(PrintImageSize(1, 2)), + orderedBy = Some("Alice"), + notes = Some("Urgent print"), + source = Some("Internal") + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + printUsageMetadata = Some(metadata) + ) + val expression = record.toExpression + val rawValues = expression.expressionValues().values().asScala + + val printMetadataMap = rawValues.head.m().asScala + printMetadataMap("sectionName") shouldBe DbString("News").toAttrValue + printMetadataMap("issueDate") shouldBe DbString("2026-07-07T12:00:00.000Z").toAttrValue + printMetadataMap("pageNumber") shouldBe DbInt(5).toAttrValue + printMetadataMap("storyName") shouldBe DbString("Scala Breakthrough").toAttrValue + printMetadataMap("publicationCode") shouldBe DbString("PUB1").toAttrValue + printMetadataMap("publicationName") shouldBe DbString("The Tech Daily").toAttrValue + printMetadataMap("sectionCode") shouldBe DbString("SEC-A").toAttrValue + printMetadataMap("size") shouldBe DbNestedMap(Map("x" -> DbInt(1), "y" -> DbInt(2))).toAttrValue + printMetadataMap("layoutId") shouldBe DbInt(12345).toAttrValue + printMetadataMap("edition") shouldBe DbInt(2).toAttrValue + printMetadataMap("orderedBy") shouldBe DbString("Alice").toAttrValue + printMetadataMap("notes") shouldBe DbString("Urgent print").toAttrValue + printMetadataMap("source") shouldBe DbString("Internal").toAttrValue + + printMetadataMap should have size 13 + } + } + describe("digitalUsageMetadata") { + + it("should correctly compile DigitalUsageMetadata with all optional fields present") { + val metadata = DigitalUsageMetadata( + webUrl = new URI("https://www.theguardian.com/tech"), + webTitle = "Scala 3 adoption grows", + sectionId = "technology", + composerUrl = Some(new URI("https://composer.internal/123")) + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + digitalUsageMetadata = Some(metadata) + ) + + val expression = record.toExpression + val rawValues = expression.expressionValues().values().asScala + + val digitalMetadataMap = rawValues.head.m().asScala + digitalMetadataMap shouldBe Map( + "webUrl" -> AttributeValue.builder().s("https://www.theguardian.com/tech").build(), + "webTitle" -> AttributeValue.builder().s("Scala 3 adoption grows").build(), + "sectionId" -> AttributeValue.builder().s("technology").build(), + "composerUrl" -> AttributeValue.builder().s("https://composer.internal/123").build() + ) + } + + it("should fall back to placeholder when webTitle is empty and omit composerUrl when None") { + val metadata = DigitalUsageMetadata( + webUrl = new URI("https://www.theguardian.com/media"), + webTitle = "", + sectionId = "media", + composerUrl = None + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + digitalUsageMetadata = Some(metadata) + ) + + val expression = record.toExpression + val rawValues = expression.expressionValues().values().asScala + + val digitalMetadataMap = rawValues.head.m().asScala + digitalMetadataMap("webTitle") shouldBe AttributeValue.builder().s("No title given").build() + digitalMetadataMap.contains("composerUrl") shouldBe false + } + } + + it("should build a comprehensive Expression when all fields are populated") { + val printMetadata = PrintUsageMetadata( + sectionName = "News", + issueDate = DateTime.parse("2026-07-08T12:00:00Z"), + pageNumber = 5, + storyName = "Scala Breakthrough", + publicationCode = "PUB1", + publicationName = "The Tech Daily", + layoutId = Some(12345L), + edition = Some(2), + size = Some(PrintImageSize(10, 20)), + orderedBy = Some("Alice"), + sectionCode = "SEC-A", + notes = Some("Urgent print"), + source = Some("Internal") + ) + + val digitalMetadata = DigitalUsageMetadata( + webUrl = new URI("https://www.theguardian.com/tech"), + webTitle = "Scala adoption grows", + sectionId = "technology", + composerUrl = Some(new URI("https://composer.internal/123")) + ) + + val syndicationMetadata = SyndicationUsageMetadata( + partnerName = "Reuters", + syndicatedBy = Some("Bob") + ) + + val frontMetadata = FrontUsageMetadata( + addedBy = "Charlie", + front = "Main Front" + ) + + val downloadMetadata = DownloadUsageMetadata( + downloadedBy = "David" + ) + + val childMetadata = ChildUsageMetadata( + addedBy = "Eve", + childMediaId = "child-999" + ) + + val fixedLastModified = DateTime.parse("2026-07-08T15:00:00Z") + val fixedDateAdded = DateTime.parse("2026-07-08T10:00:00Z") + val fixedDateRemoved = DateTime.parse("2026-07-08T18:00:00Z") + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = SetDateRemoved(fixedDateRemoved), + mediaId = Some("media-789"), + usageType = Some(PrintUsage), + mediaType = Some("image"), + lastModified = Some(fixedLastModified), + usageStatus = Some("active"), + printUsageMetadata = Some(printMetadata), + digitalUsageMetadata = Some(digitalMetadata), + syndicationUsageMetadata = Some(syndicationMetadata), + frontUsageMetadata = Some(frontMetadata), + downloadUsageMetadata = Some(downloadMetadata), + childUsageMetadata = Some(childMetadata), + dateAdded = Some(fixedDateAdded) + ) + + val expression = record.toExpression + + val names = expression.expressionNames().values().asScala + names should have size 13 + + val expectedNames = List( + "media_id", "usage_type", "media_type", "last_modified", "usage_status", + "print_metadata", "digital_metadata", "syndication_metadata", + "front_metadata", "download_metadata", "child_metadata", "date_added", "date_removed" + ) + expectedNames.foreach { name => names should contain (name) } + + val values = expression.expressionValues().values().asScala + values should have size 13 + + values should contain (AttributeValue.builder().s("media-789").build()) + values should contain (AttributeValue.builder().s(PrintUsage.toString).build()) + values should contain (AttributeValue.builder().s("image").build()) + values should contain (AttributeValue.builder().n(fixedLastModified.getMillis.toString).build()) + values should contain (AttributeValue.builder().s("active").build()) + values should contain (AttributeValue.builder().n(fixedDateAdded.getMillis.toString).build()) + values should contain (AttributeValue.builder().n(fixedDateRemoved.getMillis.toString).build()) + + val attributeMaps = values.collect { case av if av.hasM => av.m().asScala.toMap } + + val actualDigital = attributeMaps.find(_.contains("webUrl")).getOrElse(fail("Missing digital_metadata Map")) + actualDigital("webUrl").s() shouldBe "https://www.theguardian.com/tech" + actualDigital("webTitle").s() shouldBe "Scala adoption grows" + actualDigital("sectionId").s() shouldBe "technology" + actualDigital("composerUrl").s() shouldBe "https://composer.internal/123" + + val actualPrint = attributeMaps.find(_.contains("sectionName")).getOrElse(fail("Missing print_metadata Map")) + actualPrint("sectionName").s() shouldBe "News" + actualPrint("pageNumber").n() shouldBe "5" + actualPrint("layoutId").n() shouldBe "12345" + actualPrint("edition").n() shouldBe "2" + actualPrint("orderedBy").s() shouldBe "Alice" + actualPrint("sectionCode").s() shouldBe "SEC-A" + actualPrint("notes").s() shouldBe "Urgent print" + actualPrint("source").s() shouldBe "Internal" + + val actualSizeMap = actualPrint("size").m().asScala + actualSizeMap("x").n() shouldBe "10" + actualSizeMap("y").n() shouldBe "20" + + val actualSyndication = attributeMaps.find(_.contains("partnerName")).getOrElse(fail("Missing syndication_metadata Map")) + actualSyndication("partnerName").s() shouldBe "Reuters" + actualSyndication("syndicatedBy").s() shouldBe "Bob" + + val actualFront = attributeMaps.find(_.contains("front")).getOrElse(fail("Missing front_metadata Map")) + actualFront("addedBy").s() shouldBe "Charlie" + actualFront("front").s() shouldBe "Main Front" + + val actualDownload = attributeMaps.find(_.contains("downloadedBy")).getOrElse(fail("Missing download_metadata Map")) + actualDownload("downloadedBy").s() shouldBe "David" + + val actualChild = attributeMaps.find(_.contains("childMediaId")).getOrElse(fail("Missing child_metadata Map")) + actualChild("addedBy").s() shouldBe "Eve" + actualChild("childMediaId").s() shouldBe "child-999" + + val expectedExpressionString = + "SET #media_id = :media_id, #usage_type = :usage_type, #media_type = :media_type, " + + "#last_modified = :last_modified, #usage_status = :usage_status, #print_metadata = :print_metadata, " + + "#digital_metadata = :digital_metadata, #syndication_metadata = :syndication_metadata, " + + "#front_metadata = :front_metadata, #download_metadata = :download_metadata, " + + "#child_metadata = :child_metadata, #date_added = :date_added, #date_removed = :date_removed" + + expression.expression() shouldEqual expectedExpressionString + } + } +}