From 67449f97ef7ef5399c6609ce91883c24a0f6a575 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Apr 2026 15:33:07 +0000 Subject: [PATCH 001/373] Bump postcss from 8.4.38 to 8.5.10 in /kahuna Bumps [postcss](https://github.com/postcss/postcss) from 8.4.38 to 8.5.10. - [Release notes](https://github.com/postcss/postcss/releases) - [Changelog](https://github.com/postcss/postcss/blob/main/CHANGELOG.md) - [Commits](https://github.com/postcss/postcss/compare/8.4.38...8.5.10) --- updated-dependencies: - dependency-name: postcss dependency-version: 8.5.10 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- kahuna/package-lock.json | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index f1487a33969..bf93aa11a65 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -12055,9 +12055,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", "dev": true, "funding": [ { @@ -12065,6 +12065,7 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "bin": { "nanoid": "bin/nanoid.cjs" }, @@ -13290,9 +13291,9 @@ } }, "node_modules/postcss": { - "version": "8.4.38", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.38.tgz", - "integrity": "sha512-Wglpdk03BSfXkHoQa3b/oulrotAkwrlLDRSOb9D0bN86FdRyE9lppSp33aHNPgBa0JKCoB+drFLZkQoRRYae5A==", + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.10.tgz", + "integrity": "sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==", "dev": true, "funding": [ { @@ -13308,10 +13309,11 @@ "url": "https://github.com/sponsors/ai" } ], + "license": "MIT", "dependencies": { - "nanoid": "^3.3.7", - "picocolors": "^1.0.0", - "source-map-js": "^1.2.0" + "nanoid": "^3.3.11", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" }, "engines": { "node": "^10 || ^12 || >=14" @@ -14432,10 +14434,11 @@ } }, "node_modules/source-map-js": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", - "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "dev": true, + "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" } From efd77051835249909a97362e026c79187805ef75 Mon Sep 17 00:00:00 2001 From: Andrew Nowak Date: Mon, 15 Jun 2026 16:33:14 +0100 Subject: [PATCH 002/373] updated create-root-collection script --- dev/script/create-root-collection.sh | 58 ++++++++++++++++++++++------ docs/03-apis/02-collections.md | 3 +- 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/dev/script/create-root-collection.sh b/dev/script/create-root-collection.sh index 6ae21d714ef..dbb42e1dc7c 100755 --- a/dev/script/create-root-collection.sh +++ b/dev/script/create-root-collection.sh @@ -5,18 +5,54 @@ set -e DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROOT_DIR=${DIR}/../.. -COLLECTION_NAME=${1:-TEST} +while [[ $# -gt 0 ]] && [[ "$1" == "--"* ]] ; +do + opt="$1"; + shift; + case "$opt" in + "--api-key" ) + api_key="$1"; shift;; + "--domain" ) + domain="$1"; shift;; + "--name" ) + name="$1"; shift;; + * ) + echo >&2 "Invalid option: $@"; exit 1;; + esac +done -# load values from .env into environment variables -# see https://stackoverflow.com/a/30969768/3868241 -set -o allexport -# shellcheck source=../.env -source "$ROOT_DIR/dev/.env" -set +o allexport +if [[ -z "$api_key" && -z "$domain" ]]; then + echo >&2 "API key and domain unset, setting from dotenv file" + # load values from .env into environment variables + # see https://stackoverflow.com/a/30969768/3868241 + set -o allexport + # shellcheck source=../.env + source "$ROOT_DIR/dev/.env" + set +o allexport +fi -echo "creating root collection $COLLECTION_NAME" +if [[ -z "$api_key" || -z "$domain" || -z "$name" ]]; then + echo >&2 "Missing setting(s). -curl -s -X POST -H "X-Gu-Media-Key: $API_KEY" \ +Usage: $0 --api-key --domain --name + + --api-key API key for authenticating grid access + --domain Root domain for grid instance (eg. local.dev-gutools.co.uk) + --name Name of new root collection" + exit 1 +fi + +echo "creating root collection $name" + +curl -s -X POST -H "X-Gu-Media-Key: $api_key" \ -H "Content-Type: application/json" \ - -d '{ "data": "'"$COLLECTION_NAME"'" }' \ - "https://media-collections.$DOMAIN/collections" + -d '{ "data": "'"$name"'" }' \ + "https://media-collections.$domain/collections" + +outcome="$?" + +if [[ "$outcome" == "0" ]]; then + echo "collection created" +else + echo "creation failed..." +fi diff --git a/docs/03-apis/02-collections.md b/docs/03-apis/02-collections.md index 227c1018b89..72e2fd2c462 100644 --- a/docs/03-apis/02-collections.md +++ b/docs/03-apis/02-collections.md @@ -14,10 +14,11 @@ ROOT COLLECTION 2 ``` ## Root collections + Currently, root collections can only be created via the API. That is, Kahuna doesn't have a UI for it. We can use [create-root-collection.sh](../../dev/script/create-root-collection.sh) to create a root collection: ```shell script -./dev/script/create-root-collection.sh COLLECTION_NAME +./dev/script/create-root-collection.sh --api-key [api key] --domain [root domain] --name [collection name] ``` From 98d56696639b64468260ac5ffb939a3a3b16b4b9 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Mon, 15 Jun 2026 17:05:37 +0100 Subject: [PATCH 003/373] Add fill scores tickbox to frontend --- kahuna/public/js/search/index.js | 1 + kahuna/public/js/search/query.html | 8 ++++++++ kahuna/public/js/search/query.js | 24 ++++++++++++++++++---- kahuna/public/js/search/results.js | 1 + kahuna/public/js/services/api/media-api.js | 5 +++-- 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 8897e6006ee..84cb8914627 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -180,6 +180,7 @@ search.config(['$stateProvider', '$urlMatcherFactoryProvider', 'orderBy', 'useAISearch', 'vecWeight', + 'fillScores', 'dateField', 'takenSince', 'takenUntil', diff --git a/kahuna/public/js/search/query.html b/kahuna/public/js/search/query.html index a01d32be66f..9130fb56338 100644 --- a/kahuna/public/js/search/query.html +++ b/kahuna/public/js/search/query.html @@ -7,6 +7,14 @@ Use AI search + + + search dateFilterParams.indexOf(key) === -1). - // Exclude useAISearch, managed separately by its own dedicated watcher - filter(key => key !== 'useAISearch'). + // Exclude useAISearch and fillScores, managed separately by their own dedicated watchers + filter(key => key !== 'useAISearch' && key !== 'fillScores'). forEach(setAndWatchParam); // URL parameters are not decoded when taken out of the params. @@ -458,10 +460,24 @@ query.controller('SearchQueryCtrl', [ $state.go('search.results', { ...ctrl.filter, useAISearch: true, - vecWeight: ctrl.vecWeight + vecWeight: ctrl.vecWeight, + fillScores: ctrl.fillScores ? true : null }); } else { - $state.go('search.results', {...ctrl.filter, useAISearch: null}); + // fillScores is only meaningful with AI search, so clear it when AI search is disabled + ctrl.fillScores = false; + $state.go('search.results', {...ctrl.filter, useAISearch: null, fillScores: null}); + } + }); + $scope.$watch(() => ctrl.fillScores, () => { + // fillScores only applies when AI search is enabled + if (ctrl.useAISearch) { + $state.go('search.results', { + ...ctrl.filter, + useAISearch: true, + vecWeight: ctrl.vecWeight, + fillScores: ctrl.fillScores ? true : null + }); } }); diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 6721bf53ba5..17e43c937b3 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -578,6 +578,7 @@ results.controller('SearchResultsCtrl', [ orderBy: orderBy, useAISearch: $stateParams.useAISearch, vecWeight: $stateParams.vecWeight, + fillScores: $stateParams.fillScores, hasRightsAcquired: $stateParams.hasRightsAcquired, hasCrops: $stateParams.hasCrops, syndicationStatus: $stateParams.syndicationStatus, diff --git a/kahuna/public/js/services/api/media-api.js b/kahuna/public/js/services/api/media-api.js index 4687cb62774..abeaef8176d 100644 --- a/kahuna/public/js/services/api/media-api.js +++ b/kahuna/public/js/services/api/media-api.js @@ -42,7 +42,7 @@ mediaApi.factory('mediaApi', payType, uploadedBy, offset, length, orderBy, takenSince, takenUntil, modifiedSince, modifiedUntil, hasRightsAcquired, hasCrops, - syndicationStatus, countAll, persisted, useAISearch, vecWeight} = {}) { + syndicationStatus, countAll, persisted, useAISearch, vecWeight, fillScores} = {}) { return root.follow('search', { q: query, since: since, @@ -66,7 +66,8 @@ mediaApi.factory('mediaApi', countAll, persisted, useAISearch: maybeStringToBoolean(useAISearch), - vecWeight: vecWeight + vecWeight: vecWeight, + fillScores: maybeStringToBoolean(fillScores) }).get(); } From 9be7d05914aca737f9ccc9476f7b1771fe9b34f3 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Mon, 15 Jun 2026 17:06:07 +0100 Subject: [PATCH 004/373] WIP implementation of missing scores with two rescore queries --- media-api/app/controllers/MediaApi.scala | 5 +- .../app/lib/elasticsearch/ElasticSearch.scala | 79 ++++++++++++++++--- .../elasticsearch/ElasticSearchModel.scala | 4 +- 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 12c99aaeeae..fc39a97bdfe 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -90,7 +90,8 @@ class MediaApi( "countAll", "persisted", "useAISearch", - "vecWeight" + "vecWeight", + "fillScores" ).mkString(",") private val searchLinkHref = s"${config.rootUri}/images{?$searchParamList}" @@ -651,6 +652,7 @@ class MediaApi( } val weight = params.vecWeight.getOrElse(1.0) + val fillScores = params.fillScores.getOrElse(false) // cache.get(key) is atomic: if two requests race on the same key, only one // load fires and both callers receive the same Future. @@ -667,6 +669,7 @@ class MediaApi( k = k, numCandidates = Math.max(k * 2, 100), vecWeight = weight, + fillScores = fillScores, filterOpt = filterOpt ) } yield searchResults diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 0d890ba598a..2be0d4dff5b 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -11,7 +11,8 @@ import com.gu.mediaservice.lib.metrics.FutureSyntax import com.gu.mediaservice.model.{Agencies, Agency, AwaitingReviewForSyndication, Image} import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ -import com.sksamuel.elastic4s.requests.common.Operator.And +import com.sksamuel.elastic4s.requests.common.Operator +import com.sksamuel.elastic4s.requests.common.Operator.{And, Or} import com.sksamuel.elastic4s.requests.get.{GetRequest, GetResponse} import com.sksamuel.elastic4s.requests.script.{Script, ScriptField} import com.sksamuel.elastic4s.requests.searches._ @@ -201,14 +202,14 @@ class ElasticSearch( } } - private def createMultiMatchQuery(query: String, boost: Option[Double] = None): MultiMatchQuery = + private def createMultiMatchQuery(query: String, boost: Option[Double] = None, operator: Operator): MultiMatchQuery = MultiMatchQuery( text = query, fields = matchFields.map(field => FieldWithOptionalBoost(field, None)), `type` = Some(BEST_FIELDS), fuzziness = Some("AUTO"), maxExpansions = Some(50), - operator = Some(And), + operator = Some(operator), prefixLength = Some(1), boost = boost ) @@ -218,7 +219,7 @@ class ElasticSearch( // the scaling factor for the lexical part of the query, so that BM25 and knn scores are both between 0-1 scale // and can be effectively combined in a hybrid query. private def fetchMaxBm25Score(query: String, filterOpt: Option[Query])(implicit ex: ExecutionContext, logMarker: LogMarker): Future[Double] = { - val baseQuery = createMultiMatchQuery(query) + val baseQuery = createMultiMatchQuery(query, operator = And) val filteredQuery = filterOpt.map(filter => boolQuery().must(baseQuery).filter(filter)).getOrElse(baseQuery) val maxScoreRequest = ElasticDsl.search(imagesCurrentAlias) @@ -259,19 +260,69 @@ class ElasticSearch( logger.info(logMarker, s"Scaling factor for BM25 score is $scalingFactor, multi-match boost is $multiMatchBoost") - val multiMatchQuery = createMultiMatchQuery(query, boost = Some(multiMatchBoost)) + val multiMatchQuery = createMultiMatchQuery(query, boost = Some(multiMatchBoost), operator = And) ElasticDsl.search(imagesCurrentAlias) .bool(BoolQuery().should(Seq(multiMatchQuery, knn)).filter(filterOpt)) .size(k) } + // Alternative hybrid search mode, enabled via the `fillScores` query param. + // TODO: implement. For now this is a placeholder that fails fast so the + // mode can be wired up end-to-end before the real implementation lands. + private def fillScoresSearch( + query: String, + queryEmbedding: List[Double], + k: Int, + numCandidates: Int, + vecWeight: Double, + filterOpt: Option[Query] + )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { + val lexicalSearchRequest = ElasticDsl + .search(imagesCurrentAlias) + .query(createMultiMatchQuery(query, operator = Or)) + .size(k) + + val semanticSearchRequest = ElasticDsl + .search(imagesCurrentAlias) + .knn(Knn("embedding.cohereEmbedV4.image", filter = filterOpt) + .queryVector(queryEmbedding) + .k(k) + .numCandidates(numCandidates) + ) + .rescore(Rescore(createMultiMatchQuery(query, operator = Or)) + .window(k) + // We want to replace the knn score with the BM25 score, + // so we can preserve the cosine similarity in a separate field + .originalQueryWeight(0) + .rescoreQueryWeight(1) + ) + + // Assigning to vals here eagerly starts both requests, so they run in + // parallel. The for-comprehension below only sequences the *combination* + // of their results, not their execution. + val lexicalSearchResponse = executeAndLog(withSearchQueryTimeout(lexicalSearchRequest), "lexical") + val semanticSearchResponse = executeAndLog(withSearchQueryTimeout(semanticSearchRequest), "semantic") + + for { + lexical <- lexicalSearchResponse + semantic <- semanticSearchResponse + } yield { + // TODO: merge the BM25-ranked lexical hits with the cosine-similarity + // scores from the semantic (knn + rescore) response into SearchResults. + val lexicalHits = lexical.result.hits.hits + val semanticHits = semantic.result.hits.hits + ??? + } + } + def hybridSearch( query: String, queryEmbedding: List[Float], k: Int, numCandidates: Int, vecWeight: Double, + fillScores: Boolean, filterOpt: Option[Query] )( implicit ex: ExecutionContext, @@ -283,13 +334,17 @@ class ElasticSearch( } else { val queryEmbeddingDouble: List[Double] = queryEmbedding.map(_.toDouble) - for { - maxScore <- fetchMaxBm25Score(query, filterOpt) - searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore, filterOpt) - result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") - } yield { - val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) - SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + if (fillScores) { + fillScoresSearch(query, queryEmbeddingDouble, k, numCandidates, vecWeight, filterOpt) + } else { + for { + maxScore <- fetchMaxBm25Score(query, filterOpt) + searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore, filterOpt) + result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") + } yield { + val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) + SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + } } } } diff --git a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala index e6dd8fed5b0..1883e6758c8 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala @@ -147,7 +147,8 @@ case class SearchParams( printUsageFilters: Option[PrintUsageFilters] = None, shouldFlagGraphicImages: Boolean = false, useAISearch: Option[Boolean] = None, - vecWeight: Option[Double] = None + vecWeight: Option[Double] = None, + fillScores: Option[Boolean] = None ) { lazy val aiQueryParts: AiQueryParts = AiQueryParts.from(structuredQuery) } @@ -242,6 +243,7 @@ object SearchParams { shouldFlagGraphicImages = false, request.getQueryString("useAISearch") flatMap parseBooleanFromQuery, request.getQueryString("vecWeight") flatMap parseBoundedDoubleFromQuery, + request.getQueryString("fillScores") flatMap parseBooleanFromQuery, ) } From 8eb244d41a15f4e4e7655bb1109e93829d770850 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 09:53:58 +0100 Subject: [PATCH 005/373] Fill missing scores with two rescore queries --- .../app/lib/elasticsearch/ElasticSearch.scala | 79 +++++++++++++++++-- 1 file changed, 71 insertions(+), 8 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 2be0d4dff5b..c1d9dcfb99f 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -9,7 +9,7 @@ import com.gu.mediaservice.lib.elasticsearch.{CompletionPreview, ElasticSearchCl import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap} import com.gu.mediaservice.lib.metrics.FutureSyntax import com.gu.mediaservice.model.{Agencies, Agency, AwaitingReviewForSyndication, Image} -import com.sksamuel.elastic4s.ElasticDsl +import com.sksamuel.elastic4s.{ElasticDsl, Hit} import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s.requests.common.Operator import com.sksamuel.elastic4s.requests.common.Operator.{And, Or} @@ -164,6 +164,7 @@ class ElasticSearch( } ) + // TODO: delete because unused? def lookupIds(ids: List[String], offset: Int, length: Int)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { val query = filters.pinnedIds(ids) @@ -267,9 +268,52 @@ class ElasticSearch( .size(k) } + // Computes the exact cosine similarity between the query embedding and each + // hit's image embedding, exposed as a `cosineSimilarity` script field so it + // survives even when the hit's `_score` is something else (e.g. a BM25 score + // after rescoring). Hits without an embedding score 0 so the script doesn't + // throw on documents that lack a dense vector. + // + // NB: the predefined `cosineSimilarity(...)` vector function is ONLY available + // in the `script_score` query context, so calling it from a script field + // throws a painless compile error. We therefore reimplement cosine similarity + // by hand using the doc-value vector accessors (`vectorValue`/`magnitude`), + // which *are* available in the script field context. + // https://www.elastic.co/guide/en/elasticsearch/reference/8.19/query-dsl-script-score-query.html#vector-functions-accessing-the-vector + private def cosineSimilarityScriptField(queryEmbedding: List[Double]): ScriptField = { + val queryMagnitude = math.sqrt(queryEmbedding.map(x => x * x).sum) + ScriptField( + field = "cosineSimilarity", + script = Script( + //language=groovy -- it's actually painless, but that's pretty similar to groovy and this provides syntax highlighting + // The `doc[...].size() == 0` guard is the ES-documented idiom for handling docs without a vector value, + // which otherwise throw: "If a document doesn't have a value for a vector field on which a vector function + // is executed, an error is thrown." + // https://www.elastic.co/guide/en/elasticsearch/reference/8.19/query-dsl-script-score-query.html#vector-functions-missing-values + script = + """ + |if (doc['embedding.cohereEmbedV4.image'].size() == 0) { return 0.0; } + |float[] v = doc['embedding.cohereEmbedV4.image'].vectorValue; + |float vm = doc['embedding.cohereEmbedV4.image'].magnitude; + |if (vm == 0.0 || params.queryMagnitude == 0.0) { return 0.0; } + |double dotProduct = 0.0; + |for (int i = 0; i < v.length; i++) { dotProduct += v[i] * params.queryVector[i]; } + |return dotProduct / (vm * params.queryMagnitude); + |""".stripMargin, + lang = Some("painless") + ).param("queryVector", queryEmbedding).param("queryMagnitude", queryMagnitude) + ) + } + + private def cosineSimilarityFromSearchHit(hit: SearchHit): Double = { + // TODO, blows up if not found. What should it do instead? + // Why a List[Double] not just Double? + // "The fields response always returns an array of values for each field" + // https://www.elastic.co/guide/en/elasticsearch/reference/8.19/search-fields.html#search-fields-response + hit.fields.get("cosineSimilarity").map(_.asInstanceOf[List[Double]]).get.head + } + // Alternative hybrid search mode, enabled via the `fillScores` query param. - // TODO: implement. For now this is a placeholder that fails fast so the - // mode can be wired up end-to-end before the real implementation lands. private def fillScoresSearch( query: String, queryEmbedding: List[Double], @@ -281,6 +325,8 @@ class ElasticSearch( val lexicalSearchRequest = ElasticDsl .search(imagesCurrentAlias) .query(createMultiMatchQuery(query, operator = Or)) + .storedFields("_source") // needs to be explicit when using script fields + .scriptfields(cosineSimilarityScriptField(queryEmbedding)) .size(k) val semanticSearchRequest = ElasticDsl @@ -290,6 +336,8 @@ class ElasticSearch( .k(k) .numCandidates(numCandidates) ) + .storedFields("_source") // needs to be explicit when using script fields + .scriptfields(cosineSimilarityScriptField(queryEmbedding)) .rescore(Rescore(createMultiMatchQuery(query, operator = Or)) .window(k) // We want to replace the knn score with the BM25 score, @@ -308,11 +356,25 @@ class ElasticSearch( lexical <- lexicalSearchResponse semantic <- semanticSearchResponse } yield { - // TODO: merge the BM25-ranked lexical hits with the cosine-similarity - // scores from the semantic (knn + rescore) response into SearchResults. - val lexicalHits = lexical.result.hits.hits - val semanticHits = semantic.result.hits.hits - ??? + val lexicalHits = lexical.result.hits.hits.toList + val maxLexicalScore = lexicalHits.head.score // TODO: assumes we have at least one result. what if none? + val semanticHits = semantic.result.hits.hits.toList + val maxSemanticScore = cosineSimilarityFromSearchHit(semanticHits.head) // TODO: make typesafe? + logger.info(logMarker, s"${lexical.result.hits.total} lexical hits, ${semantic.result.hits.total} semantic hits") + val allHits = lexicalHits ::: semanticHits + // TODO: simpler way of de-duping? + val dedupedHits = allHits.map(hit => hit.id -> hit).toMap.values.toList + logger.info(logMarker, s"${dedupedHits.length} deduped hits") + + val sortedHits = dedupedHits.sortBy { searchHit => + val maxNormedSemanticScore = (cosineSimilarityFromSearchHit(searchHit) + 1) / (maxSemanticScore + 1) + val maxNormedLexicalScore = searchHit.score / maxLexicalScore + + -((vecWeight * maxNormedSemanticScore) + ((1 - vecWeight) * maxNormedLexicalScore)) + } + + val resolvedHits = sortedHits.take(k).flatMap(resolveHit).map(i => (i.instance.id, i)) + SearchResults(hits = resolvedHits, total = resolvedHits.length, extraCounts = None) } } @@ -342,6 +404,7 @@ class ElasticSearch( searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore, filterOpt) result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") } yield { + // TODO: factor out val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) } From 6e96872cc29668d965a6c61028578f5e0d6ff26c Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 11:31:04 +0100 Subject: [PATCH 006/373] Small cleanup --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index c1d9dcfb99f..4dbba0db53a 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -361,19 +361,17 @@ class ElasticSearch( val semanticHits = semantic.result.hits.hits.toList val maxSemanticScore = cosineSimilarityFromSearchHit(semanticHits.head) // TODO: make typesafe? logger.info(logMarker, s"${lexical.result.hits.total} lexical hits, ${semantic.result.hits.total} semantic hits") - val allHits = lexicalHits ::: semanticHits - // TODO: simpler way of de-duping? - val dedupedHits = allHits.map(hit => hit.id -> hit).toMap.values.toList + val dedupedHits = (lexicalHits ::: semanticHits).distinctBy(_.id) logger.info(logMarker, s"${dedupedHits.length} deduped hits") - val sortedHits = dedupedHits.sortBy { searchHit => + val topK = dedupedHits.sortBy { searchHit => val maxNormedSemanticScore = (cosineSimilarityFromSearchHit(searchHit) + 1) / (maxSemanticScore + 1) val maxNormedLexicalScore = searchHit.score / maxLexicalScore -((vecWeight * maxNormedSemanticScore) + ((1 - vecWeight) * maxNormedLexicalScore)) - } + }.take(k) - val resolvedHits = sortedHits.take(k).flatMap(resolveHit).map(i => (i.instance.id, i)) + val resolvedHits = topK.flatMap(resolveHit).map(i => (i.instance.id, i)) SearchResults(hits = resolvedHits, total = resolvedHits.length, extraCounts = None) } } From fa7ccd0ba11e6f17372b035034958f76bcf1024a Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 12:14:08 +0100 Subject: [PATCH 007/373] Add VectorUtils --- .../com/gu/mediaservice/lib/VectorUtils.scala | 15 +++++++ .../gu/mediaservice/lib/VectorUtilsTest.scala | 43 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala create mode 100644 common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala new file mode 100644 index 00000000000..a9f1a822872 --- /dev/null +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -0,0 +1,15 @@ +package com.gu.mediaservice.lib + +object VectorUtils { + def dotProduct(vectorOne: List[Double], vectorTwo: List[Double]): Double = { + vectorOne.zip(vectorTwo).map(component => component._1 * component._2).sum + } + + def magnitude(vector: List[Double]): Double = { + math.sqrt(vector.map(math.pow(_, 2)).sum) + } + + def cosineSimilarity(vectorOne: List[Double], vectorTwo: List[Double]): Double = { + dotProduct(vectorOne, vectorTwo) / (magnitude(vectorOne) * magnitude(vectorTwo)) + } +} diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala new file mode 100644 index 00000000000..ffa284ebe51 --- /dev/null +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -0,0 +1,43 @@ +package com.gu.mediaservice.lib + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + + +class VectorUtilsTest extends AnyFunSpec with Matchers { + it ("should compute the dot product of two vectors") { + VectorUtils.dotProduct(List(1.0, 2.0, 3.0), List(4.0, 5.0, 6.0)) shouldBe 32.0 + } + + it ("should compute a dot product of zero for orthogonal vectors") { + VectorUtils.dotProduct(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 + } + + it ("should return zero for the dot product of empty vectors") { + VectorUtils.dotProduct(List.empty, List.empty) shouldBe 0.0 + } + + it ("should compute the magnitude of a vector") { + VectorUtils.magnitude(List(3.0, 4.0)) shouldBe 5.0 + } + + it ("should return a magnitude of zero for a zero vector") { + VectorUtils.magnitude(List(0.0, 0.0, 0.0)) shouldBe 0.0 + } + + it ("should return a cosine similarity of 1 for identical vectors") { + VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)) shouldBe 1.0 +- 1e-9 + } + + it ("should return a cosine similarity of 1 for parallel vectors") { + VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)) shouldBe 1.0 +- 1e-9 + } + + it ("should return a cosine similarity of 0 for orthogonal vectors") { + VectorUtils.cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 +- 1e-9 + } + + it ("should return a cosine similarity of -1 for opposite vectors") { + VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)) shouldBe -1.0 +- 1e-9 + } +} From ab90510b6121eb0cd9a4fe058a0edf3f005aa8ce Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 12:59:16 +0100 Subject: [PATCH 008/373] Calculate cosine similarity clientside --- .../app/lib/elasticsearch/ElasticSearch.scala | 104 +++++++----------- 1 file changed, 41 insertions(+), 63 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 4dbba0db53a..05baa228226 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -1,7 +1,7 @@ package lib.elasticsearch import org.apache.pekko.actor.Scheduler -import com.gu.mediaservice.lib.ImageFields +import com.gu.mediaservice.lib.{ImageFields, VectorUtils} import com.gu.mediaservice.lib.argo.model.{ExtraCount, ExtraCountConfig, ExtraCounts} import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.lib.auth.Authentication.Principal @@ -268,49 +268,43 @@ class ElasticSearch( .size(k) } - // Computes the exact cosine similarity between the query embedding and each - // hit's image embedding, exposed as a `cosineSimilarity` script field so it - // survives even when the hit's `_score` is something else (e.g. a BM25 score - // after rescoring). Hits without an embedding score 0 so the script doesn't - // throw on documents that lack a dense vector. - // - // NB: the predefined `cosineSimilarity(...)` vector function is ONLY available - // in the `script_score` query context, so calling it from a script field - // throws a painless compile error. We therefore reimplement cosine similarity - // by hand using the doc-value vector accessors (`vectorValue`/`magnitude`), - // which *are* available in the script field context. - // https://www.elastic.co/guide/en/elasticsearch/reference/8.19/query-dsl-script-score-query.html#vector-functions-accessing-the-vector - private def cosineSimilarityScriptField(queryEmbedding: List[Double]): ScriptField = { - val queryMagnitude = math.sqrt(queryEmbedding.map(x => x * x).sum) - ScriptField( - field = "cosineSimilarity", - script = Script( - //language=groovy -- it's actually painless, but that's pretty similar to groovy and this provides syntax highlighting - // The `doc[...].size() == 0` guard is the ES-documented idiom for handling docs without a vector value, - // which otherwise throw: "If a document doesn't have a value for a vector field on which a vector function - // is executed, an error is thrown." - // https://www.elastic.co/guide/en/elasticsearch/reference/8.19/query-dsl-script-score-query.html#vector-functions-missing-values - script = - """ - |if (doc['embedding.cohereEmbedV4.image'].size() == 0) { return 0.0; } - |float[] v = doc['embedding.cohereEmbedV4.image'].vectorValue; - |float vm = doc['embedding.cohereEmbedV4.image'].magnitude; - |if (vm == 0.0 || params.queryMagnitude == 0.0) { return 0.0; } - |double dotProduct = 0.0; - |for (int i = 0; i < v.length; i++) { dotProduct += v[i] * params.queryVector[i]; } - |return dotProduct / (vm * params.queryMagnitude); - |""".stripMargin, - lang = Some("painless") - ).param("queryVector", queryEmbedding).param("queryMagnitude", queryMagnitude) - ) - } + private case class HybridResult(id: String, lexicalScore: Double, semanticScore: Double, image: SourceWrapper[Image]) + private case object HybridResult { + def fromSearchHit(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = { + val lexicalScore = hit.score + resolveHit(hit) map { image => + val semanticScore = (for { + embedding <- image.instance.embedding + cohereEmbedV4 <- embedding.cohereEmbedV4 + } yield { + // Save some computation by assuming normalised vectors + // TODO: double check this assumption + VectorUtils.dotProduct(cohereEmbedV4.image, queryEmbedding) + }).getOrElse(-1.0) + HybridResult(hit.id, lexicalScore = lexicalScore, semanticScore = semanticScore, image = image) + } + } + + def getTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { + // Account for the rare case in which KNN doesn't return the true closest vector, + // and that true closest vector happens to be among the lexical-only results. + // Max lexical score could still be pulled from the top of the lexical results, + // but it reads better to do it in a uniform way for both. + val maxLexicalScore = results.maxBy(_.lexicalScore).lexicalScore + val maxSemanticScore = results.maxBy(_.semanticScore).semanticScore + + val dedupedResults = results.distinctBy(_.id) + logger.info(logMarker, s"${dedupedResults.length} deduped results") + + dedupedResults.sortBy { result => + val maxNormedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) + val maxNormedLexicalScore = result.lexicalScore / maxLexicalScore - private def cosineSimilarityFromSearchHit(hit: SearchHit): Double = { - // TODO, blows up if not found. What should it do instead? - // Why a List[Double] not just Double? - // "The fields response always returns an array of values for each field" - // https://www.elastic.co/guide/en/elasticsearch/reference/8.19/search-fields.html#search-fields-response - hit.fields.get("cosineSimilarity").map(_.asInstanceOf[List[Double]]).get.head + // Negative to get a descending sort order + // Only other way to do this is to .reverse afterwards, apparently + -((vecWeight * maxNormedSemanticScore) + ((1 - vecWeight) * maxNormedLexicalScore)) + }.take(k) + } } // Alternative hybrid search mode, enabled via the `fillScores` query param. @@ -325,8 +319,6 @@ class ElasticSearch( val lexicalSearchRequest = ElasticDsl .search(imagesCurrentAlias) .query(createMultiMatchQuery(query, operator = Or)) - .storedFields("_source") // needs to be explicit when using script fields - .scriptfields(cosineSimilarityScriptField(queryEmbedding)) .size(k) val semanticSearchRequest = ElasticDsl @@ -336,8 +328,6 @@ class ElasticSearch( .k(k) .numCandidates(numCandidates) ) - .storedFields("_source") // needs to be explicit when using script fields - .scriptfields(cosineSimilarityScriptField(queryEmbedding)) .rescore(Rescore(createMultiMatchQuery(query, operator = Or)) .window(k) // We want to replace the knn score with the BM25 score, @@ -356,23 +346,11 @@ class ElasticSearch( lexical <- lexicalSearchResponse semantic <- semanticSearchResponse } yield { - val lexicalHits = lexical.result.hits.hits.toList - val maxLexicalScore = lexicalHits.head.score // TODO: assumes we have at least one result. what if none? - val semanticHits = semantic.result.hits.hits.toList - val maxSemanticScore = cosineSimilarityFromSearchHit(semanticHits.head) // TODO: make typesafe? logger.info(logMarker, s"${lexical.result.hits.total} lexical hits, ${semantic.result.hits.total} semantic hits") - val dedupedHits = (lexicalHits ::: semanticHits).distinctBy(_.id) - logger.info(logMarker, s"${dedupedHits.length} deduped hits") - - val topK = dedupedHits.sortBy { searchHit => - val maxNormedSemanticScore = (cosineSimilarityFromSearchHit(searchHit) + 1) / (maxSemanticScore + 1) - val maxNormedLexicalScore = searchHit.score / maxLexicalScore - - -((vecWeight * maxNormedSemanticScore) + ((1 - vecWeight) * maxNormedLexicalScore)) - }.take(k) - - val resolvedHits = topK.flatMap(resolveHit).map(i => (i.instance.id, i)) - SearchResults(hits = resolvedHits, total = resolvedHits.length, extraCounts = None) + val allHits = lexical.result.hits.hits.toList ::: semantic.result.hits.hits.toList + val results = allHits.flatMap(HybridResult.fromSearchHit(_, queryEmbedding)) + val topK = HybridResult.getTopK(results, vecWeight, k) + SearchResults(hits = topK.map(r => (r.id, r.image)), total = topK.length, extraCounts = None) } } From c9672b7ddc4189ef5aef2816aaaba25697c75d88 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 13:00:34 +0100 Subject: [PATCH 009/373] Remove TODOs --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 05baa228226..571bc83f1f8 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -164,7 +164,6 @@ class ElasticSearch( } ) - // TODO: delete because unused? def lookupIds(ids: List[String], offset: Int, length: Int)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { val query = filters.pinnedIds(ids) @@ -380,7 +379,6 @@ class ElasticSearch( searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore, filterOpt) result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") } yield { - // TODO: factor out val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) } From 22c6c4da4b9d31d33cb686f8abecc00433296348 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 13:03:14 +0100 Subject: [PATCH 010/373] Correct comment --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 571bc83f1f8..67c3e756193 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -330,7 +330,8 @@ class ElasticSearch( .rescore(Rescore(createMultiMatchQuery(query, operator = Or)) .window(k) // We want to replace the knn score with the BM25 score, - // so we can preserve the cosine similarity in a separate field + // because we can calculate cosine similarity clientside, + // but can't do that for BM25. .originalQueryWeight(0) .rescoreQueryWeight(1) ) From 516b04b98688c3eb87e0dd55fcb88587701e70ac Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 13:13:02 +0100 Subject: [PATCH 011/373] Add logging --- .../app/lib/elasticsearch/ElasticSearch.scala | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 67c3e756193..cfaef38e999 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -6,7 +6,7 @@ import com.gu.mediaservice.lib.argo.model.{ExtraCount, ExtraCountConfig, ExtraCo import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.elasticsearch.{CompletionPreview, ElasticSearchClient, ElasticSearchConfig, MigrationStatusProvider, Running} -import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap} +import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap, Stopwatch, combineMarkers} import com.gu.mediaservice.lib.metrics.FutureSyntax import com.gu.mediaservice.model.{Agencies, Agency, AwaitingReviewForSyndication, Image} import com.sksamuel.elastic4s.{ElasticDsl, Hit} @@ -372,8 +372,16 @@ class ElasticSearch( } else { val queryEmbeddingDouble: List[Double] = queryEmbedding.map(_.toDouble) + val stopwatch = Stopwatch.start + if (fillScores) { - fillScoresSearch(query, queryEmbeddingDouble, k, numCandidates, vecWeight, filterOpt) + val searchResults = fillScoresSearch(query, queryEmbeddingDouble, k, numCandidates, vecWeight, filterOpt) + val elapsed = stopwatch.elapsed + logger.info( + combineMarkers(logMarker, elapsed), + s"hybrid search (fill scores) completed in ${elapsed.toMillis} ms" + ) + searchResults } else { for { maxScore <- fetchMaxBm25Score(query, filterOpt) @@ -381,6 +389,11 @@ class ElasticSearch( result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") } yield { val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) + val elapsed = stopwatch.elapsed + logger.info( + combineMarkers(logMarker, elapsed), + s"hybrid search (current) completed in ${elapsed.toMillis} ms" + ) SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) } } From d316266c9371d86499077048c1442bdf76826a41 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 13:36:14 +0100 Subject: [PATCH 012/373] Minor cleanups --- .../app/lib/elasticsearch/ElasticSearch.scala | 49 +++++++++---------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index cfaef38e999..31150b83267 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -269,40 +269,33 @@ class ElasticSearch( private case class HybridResult(id: String, lexicalScore: Double, semanticScore: Double, image: SourceWrapper[Image]) private case object HybridResult { - def fromSearchHit(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = { - val lexicalScore = hit.score - resolveHit(hit) map { image => - val semanticScore = (for { - embedding <- image.instance.embedding - cohereEmbedV4 <- embedding.cohereEmbedV4 - } yield { + def fromSearchHit(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = + resolveHit(hit).map { image => + val semanticScore = image.instance.embedding + .flatMap(_.cohereEmbedV4) // Save some computation by assuming normalised vectors // TODO: double check this assumption - VectorUtils.dotProduct(cohereEmbedV4.image, queryEmbedding) - }).getOrElse(-1.0) - HybridResult(hit.id, lexicalScore = lexicalScore, semanticScore = semanticScore, image = image) + .map(e => VectorUtils.dotProduct(e.image, queryEmbedding)) + .getOrElse(-1.0) + HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } - } def getTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { - // Account for the rare case in which KNN doesn't return the true closest vector, - // and that true closest vector happens to be among the lexical-only results. - // Max lexical score could still be pulled from the top of the lexical results, - // but it reads better to do it in a uniform way for both. - val maxLexicalScore = results.maxBy(_.lexicalScore).lexicalScore - val maxSemanticScore = results.maxBy(_.semanticScore).semanticScore - val dedupedResults = results.distinctBy(_.id) logger.info(logMarker, s"${dedupedResults.length} deduped results") - dedupedResults.sortBy { result => - val maxNormedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) - val maxNormedLexicalScore = result.lexicalScore / maxLexicalScore + // Account for the rare case in which KNN doesn't return the true closest vector, + // and that true closest vector happens to be among the lexical-only results. + val maxLexicalScore = dedupedResults.maxBy(_.lexicalScore).lexicalScore + val maxSemanticScore = dedupedResults.maxBy(_.semanticScore).semanticScore - // Negative to get a descending sort order - // Only other way to do this is to .reverse afterwards, apparently - -((vecWeight * maxNormedSemanticScore) + ((1 - vecWeight) * maxNormedLexicalScore)) - }.take(k) + def combinedScore(result: HybridResult): Double = { + val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) + val normedLexicalScore = result.lexicalScore / maxLexicalScore + (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) + } + + dedupedResults.sortBy(combinedScore)(Ordering[Double].reverse).take(k) } } @@ -315,9 +308,11 @@ class ElasticSearch( vecWeight: Double, filterOpt: Option[Query] )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { + val lexicalQuery = createMultiMatchQuery(query, operator = Or) + val lexicalSearchRequest = ElasticDsl .search(imagesCurrentAlias) - .query(createMultiMatchQuery(query, operator = Or)) + .query(lexicalQuery) .size(k) val semanticSearchRequest = ElasticDsl @@ -327,7 +322,7 @@ class ElasticSearch( .k(k) .numCandidates(numCandidates) ) - .rescore(Rescore(createMultiMatchQuery(query, operator = Or)) + .rescore(Rescore(lexicalQuery) .window(k) // We want to replace the knn score with the BM25 score, // because we can calculate cosine similarity clientside, From fa1884123ae9bef84844e2f8a86fd5a2ccad56f1 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 13:55:53 +0100 Subject: [PATCH 013/373] Add comment about ES-norming of cosine similarity --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 31150b83267..1d8478e47f2 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -275,6 +275,9 @@ class ElasticSearch( .flatMap(_.cohereEmbedV4) // Save some computation by assuming normalised vectors // TODO: double check this assumption + // Note this is true cosine similarity from -1 to 1, + // *not* the ES-normalised score, but when we max-normalise + // later it will end up in the range 0-1. .map(e => VectorUtils.dotProduct(e.image, queryEmbedding)) .getOrElse(-1.0) HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) @@ -290,8 +293,10 @@ class ElasticSearch( val maxSemanticScore = dedupedResults.maxBy(_.semanticScore).semanticScore def combinedScore(result: HybridResult): Double = { - val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) val normedLexicalScore = result.lexicalScore / maxLexicalScore + // This is theoretical min, i.e. -1, to actual max, + // so it effectively does both the max-norming and the ES-score norming. + val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) } From a8f0eb5226ad6bc5ba8e31f494b0cb608b780ea2 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 14:39:33 +0100 Subject: [PATCH 014/373] Update media-api/app/lib/elasticsearch/ElasticSearch.scala Co-authored-by: Alice Thornewill von Essen --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 1d8478e47f2..f3cb3ef05b3 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -294,6 +294,7 @@ class ElasticSearch( def combinedScore(result: HybridResult): Double = { val normedLexicalScore = result.lexicalScore / maxLexicalScore + // normedScore = (score - theoretical_min) / (max - theoretical_min) // This is theoretical min, i.e. -1, to actual max, // so it effectively does both the max-norming and the ES-score norming. val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) From e8d92da4a2df8a385eec4148cf8f5463c4c0ec4d Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 14:47:37 +0100 Subject: [PATCH 015/373] Clarify naming --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 1d8478e47f2..a9c9f8f7659 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -269,7 +269,7 @@ class ElasticSearch( private case class HybridResult(id: String, lexicalScore: Double, semanticScore: Double, image: SourceWrapper[Image]) private case object HybridResult { - def fromSearchHit(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = + def resolveHitAndFillInSemanticScore(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = resolveHit(hit).map { image => val semanticScore = image.instance.embedding .flatMap(_.cohereEmbedV4) @@ -348,8 +348,10 @@ class ElasticSearch( } yield { logger.info(logMarker, s"${lexical.result.hits.total} lexical hits, ${semantic.result.hits.total} semantic hits") val allHits = lexical.result.hits.hits.toList ::: semantic.result.hits.hits.toList - val results = allHits.flatMap(HybridResult.fromSearchHit(_, queryEmbedding)) - val topK = HybridResult.getTopK(results, vecWeight, k) + val resultsWithSemanticScoresFilledIn = allHits.flatMap { hit => + HybridResult.resolveHitAndFillInSemanticScore(hit, queryEmbedding) + } + val topK = HybridResult.getTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) SearchResults(hits = topK.map(r => (r.id, r.image)), total = topK.length, extraCounts = None) } } From 119c60065a3be4e17952a9091891cfd850c78e20 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 14:53:32 +0100 Subject: [PATCH 016/373] More naming clarifications --- .../app/lib/elasticsearch/ElasticSearch.scala | 74 ++++++++++--------- 1 file changed, 39 insertions(+), 35 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 7699fd99626..0d89e55ccb8 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -267,42 +267,46 @@ class ElasticSearch( .size(k) } - private case class HybridResult(id: String, lexicalScore: Double, semanticScore: Double, image: SourceWrapper[Image]) - private case object HybridResult { - def resolveHitAndFillInSemanticScore(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = - resolveHit(hit).map { image => - val semanticScore = image.instance.embedding - .flatMap(_.cohereEmbedV4) - // Save some computation by assuming normalised vectors - // TODO: double check this assumption - // Note this is true cosine similarity from -1 to 1, - // *not* the ES-normalised score, but when we max-normalise - // later it will end up in the range 0-1. - .map(e => VectorUtils.dotProduct(e.image, queryEmbedding)) - .getOrElse(-1.0) - HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) - } + private case class HybridResult( + id: String, + lexicalScore: Double, + semanticScore: Double, + image: SourceWrapper[Image] + ) - def getTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { - val dedupedResults = results.distinctBy(_.id) - logger.info(logMarker, s"${dedupedResults.length} deduped results") - - // Account for the rare case in which KNN doesn't return the true closest vector, - // and that true closest vector happens to be among the lexical-only results. - val maxLexicalScore = dedupedResults.maxBy(_.lexicalScore).lexicalScore - val maxSemanticScore = dedupedResults.maxBy(_.semanticScore).semanticScore - - def combinedScore(result: HybridResult): Double = { - val normedLexicalScore = result.lexicalScore / maxLexicalScore - // normedScore = (score - theoretical_min) / (max - theoretical_min) - // This is theoretical min, i.e. -1, to actual max, - // so it effectively does both the max-norming and the ES-score norming. - val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) - (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) - } + def resolveHitAndFillInSemanticScore(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = + resolveHit(hit).map { image => + val semanticScore = image.instance.embedding + .flatMap(_.cohereEmbedV4) + // Save some computation by assuming normalised vectors + // TODO: double check this assumption + // Note this is true cosine similarity from -1 to 1, + // *not* the ES-normalised score, but when we max-normalise + // later it will end up in the range 0-1. + .map(e => VectorUtils.dotProduct(e.image, queryEmbedding)) + .getOrElse(-1.0) + HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) + } - dedupedResults.sortBy(combinedScore)(Ordering[Double].reverse).take(k) + def combineScoresAndGetTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { + val dedupedResults = results.distinctBy(_.id) + logger.info(logMarker, s"${dedupedResults.length} deduped results") + + // Account for the rare case in which KNN doesn't return the true closest vector, + // and that true closest vector happens to be among the lexical-only results. + val maxLexicalScore = dedupedResults.maxBy(_.lexicalScore).lexicalScore + val maxSemanticScore = dedupedResults.maxBy(_.semanticScore).semanticScore + + def combinedScore(result: HybridResult): Double = { + val normedLexicalScore = result.lexicalScore / maxLexicalScore + // normedScore = (score - theoretical_min) / (max - theoretical_min) + // This is theoretical min, i.e. -1, to actual max, + // so it effectively does both the max-norming and the ES-score norming. + val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) + (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) } + + dedupedResults.sortBy(combinedScore)(Ordering[Double].reverse).take(k) } // Alternative hybrid search mode, enabled via the `fillScores` query param. @@ -350,9 +354,9 @@ class ElasticSearch( logger.info(logMarker, s"${lexical.result.hits.total} lexical hits, ${semantic.result.hits.total} semantic hits") val allHits = lexical.result.hits.hits.toList ::: semantic.result.hits.hits.toList val resultsWithSemanticScoresFilledIn = allHits.flatMap { hit => - HybridResult.resolveHitAndFillInSemanticScore(hit, queryEmbedding) + resolveHitAndFillInSemanticScore(hit, queryEmbedding) } - val topK = HybridResult.getTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) + val topK = combineScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) SearchResults(hits = topK.map(r => (r.id, r.image)), total = topK.length, extraCounts = None) } } From d9bdc8240267b02e88abbb06606e428bfa6ea561 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 14:55:34 +0100 Subject: [PATCH 017/373] Revert "More naming clarifications" This reverts commit 119c60065a3be4e17952a9091891cfd850c78e20. --- .../app/lib/elasticsearch/ElasticSearch.scala | 74 +++++++++---------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 0d89e55ccb8..7699fd99626 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -267,46 +267,42 @@ class ElasticSearch( .size(k) } - private case class HybridResult( - id: String, - lexicalScore: Double, - semanticScore: Double, - image: SourceWrapper[Image] - ) + private case class HybridResult(id: String, lexicalScore: Double, semanticScore: Double, image: SourceWrapper[Image]) + private case object HybridResult { + def resolveHitAndFillInSemanticScore(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = + resolveHit(hit).map { image => + val semanticScore = image.instance.embedding + .flatMap(_.cohereEmbedV4) + // Save some computation by assuming normalised vectors + // TODO: double check this assumption + // Note this is true cosine similarity from -1 to 1, + // *not* the ES-normalised score, but when we max-normalise + // later it will end up in the range 0-1. + .map(e => VectorUtils.dotProduct(e.image, queryEmbedding)) + .getOrElse(-1.0) + HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) + } - def resolveHitAndFillInSemanticScore(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = - resolveHit(hit).map { image => - val semanticScore = image.instance.embedding - .flatMap(_.cohereEmbedV4) - // Save some computation by assuming normalised vectors - // TODO: double check this assumption - // Note this is true cosine similarity from -1 to 1, - // *not* the ES-normalised score, but when we max-normalise - // later it will end up in the range 0-1. - .map(e => VectorUtils.dotProduct(e.image, queryEmbedding)) - .getOrElse(-1.0) - HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) - } + def getTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { + val dedupedResults = results.distinctBy(_.id) + logger.info(logMarker, s"${dedupedResults.length} deduped results") + + // Account for the rare case in which KNN doesn't return the true closest vector, + // and that true closest vector happens to be among the lexical-only results. + val maxLexicalScore = dedupedResults.maxBy(_.lexicalScore).lexicalScore + val maxSemanticScore = dedupedResults.maxBy(_.semanticScore).semanticScore + + def combinedScore(result: HybridResult): Double = { + val normedLexicalScore = result.lexicalScore / maxLexicalScore + // normedScore = (score - theoretical_min) / (max - theoretical_min) + // This is theoretical min, i.e. -1, to actual max, + // so it effectively does both the max-norming and the ES-score norming. + val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) + (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) + } - def combineScoresAndGetTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { - val dedupedResults = results.distinctBy(_.id) - logger.info(logMarker, s"${dedupedResults.length} deduped results") - - // Account for the rare case in which KNN doesn't return the true closest vector, - // and that true closest vector happens to be among the lexical-only results. - val maxLexicalScore = dedupedResults.maxBy(_.lexicalScore).lexicalScore - val maxSemanticScore = dedupedResults.maxBy(_.semanticScore).semanticScore - - def combinedScore(result: HybridResult): Double = { - val normedLexicalScore = result.lexicalScore / maxLexicalScore - // normedScore = (score - theoretical_min) / (max - theoretical_min) - // This is theoretical min, i.e. -1, to actual max, - // so it effectively does both the max-norming and the ES-score norming. - val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) - (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) + dedupedResults.sortBy(combinedScore)(Ordering[Double].reverse).take(k) } - - dedupedResults.sortBy(combinedScore)(Ordering[Double].reverse).take(k) } // Alternative hybrid search mode, enabled via the `fillScores` query param. @@ -354,9 +350,9 @@ class ElasticSearch( logger.info(logMarker, s"${lexical.result.hits.total} lexical hits, ${semantic.result.hits.total} semantic hits") val allHits = lexical.result.hits.hits.toList ::: semantic.result.hits.hits.toList val resultsWithSemanticScoresFilledIn = allHits.flatMap { hit => - resolveHitAndFillInSemanticScore(hit, queryEmbedding) + HybridResult.resolveHitAndFillInSemanticScore(hit, queryEmbedding) } - val topK = combineScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) + val topK = HybridResult.getTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) SearchResults(hits = topK.map(r => (r.id, r.image)), total = topK.length, extraCounts = None) } } From ce45879bb01db718bc8922570503b8996b02d082 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 14:57:42 +0100 Subject: [PATCH 018/373] Naming tweaks --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 7699fd99626..6fdcb5e816d 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -283,7 +283,7 @@ class ElasticSearch( HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } - def getTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { + def combineScoresAndGetTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { val dedupedResults = results.distinctBy(_.id) logger.info(logMarker, s"${dedupedResults.length} deduped results") @@ -314,6 +314,7 @@ class ElasticSearch( vecWeight: Double, filterOpt: Option[Query] )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { + import HybridResult.{resolveHitAndFillInSemanticScore, combineScoresAndGetTopK} val lexicalQuery = createMultiMatchQuery(query, operator = Or) val lexicalSearchRequest = ElasticDsl @@ -350,9 +351,9 @@ class ElasticSearch( logger.info(logMarker, s"${lexical.result.hits.total} lexical hits, ${semantic.result.hits.total} semantic hits") val allHits = lexical.result.hits.hits.toList ::: semantic.result.hits.hits.toList val resultsWithSemanticScoresFilledIn = allHits.flatMap { hit => - HybridResult.resolveHitAndFillInSemanticScore(hit, queryEmbedding) + resolveHitAndFillInSemanticScore(hit, queryEmbedding) } - val topK = HybridResult.getTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) + val topK = combineScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) SearchResults(hits = topK.map(r => (r.id, r.image)), total = topK.length, extraCounts = None) } } From c4097e4718c7ba46e225d3d53b01159ca2aa629c Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 15:00:59 +0100 Subject: [PATCH 019/373] Silly formatting bits --- .../app/lib/elasticsearch/ElasticSearch.scala | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 6fdcb5e816d..032d44837cb 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -267,9 +267,18 @@ class ElasticSearch( .size(k) } - private case class HybridResult(id: String, lexicalScore: Double, semanticScore: Double, image: SourceWrapper[Image]) + private case class HybridResult( + id: String, + lexicalScore: Double, + semanticScore: Double, + image: SourceWrapper[Image] + ) + private case object HybridResult { - def resolveHitAndFillInSemanticScore(hit: SearchHit, queryEmbedding: List[Double]): Option[HybridResult] = + def resolveHitAndFillInSemanticScore( + hit: SearchHit, + queryEmbedding: List[Double] + ): Option[HybridResult] = resolveHit(hit).map { image => val semanticScore = image.instance.embedding .flatMap(_.cohereEmbedV4) @@ -283,7 +292,11 @@ class ElasticSearch( HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } - def combineScoresAndGetTopK(results: List[HybridResult], vecWeight: Double, k: Int)(implicit logMarker: LogMarker): List[HybridResult] = { + def combineScoresAndGetTopK( + results: List[HybridResult], + vecWeight: Double, + k: Int + )(implicit logMarker: LogMarker): List[HybridResult] = { val dedupedResults = results.distinctBy(_.id) logger.info(logMarker, s"${dedupedResults.length} deduped results") From a7115e2992230aa1cab452296b7f70a0b39a9216 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 15:09:41 +0100 Subject: [PATCH 020/373] De-duplicate before doing any extra work --- .../app/lib/elasticsearch/ElasticSearch.scala | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 032d44837cb..115d9cac55c 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -296,14 +296,11 @@ class ElasticSearch( results: List[HybridResult], vecWeight: Double, k: Int - )(implicit logMarker: LogMarker): List[HybridResult] = { - val dedupedResults = results.distinctBy(_.id) - logger.info(logMarker, s"${dedupedResults.length} deduped results") - + ): List[HybridResult] = { // Account for the rare case in which KNN doesn't return the true closest vector, // and that true closest vector happens to be among the lexical-only results. - val maxLexicalScore = dedupedResults.maxBy(_.lexicalScore).lexicalScore - val maxSemanticScore = dedupedResults.maxBy(_.semanticScore).semanticScore + val maxLexicalScore = results.maxBy(_.lexicalScore).lexicalScore + val maxSemanticScore = results.maxBy(_.semanticScore).semanticScore def combinedScore(result: HybridResult): Double = { val normedLexicalScore = result.lexicalScore / maxLexicalScore @@ -314,7 +311,7 @@ class ElasticSearch( (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) } - dedupedResults.sortBy(combinedScore)(Ordering[Double].reverse).take(k) + results.sortBy(combinedScore)(Ordering[Double].reverse).take(k) } } @@ -361,9 +358,15 @@ class ElasticSearch( lexical <- lexicalSearchResponse semantic <- semanticSearchResponse } yield { - logger.info(logMarker, s"${lexical.result.hits.total} lexical hits, ${semantic.result.hits.total} semantic hits") - val allHits = lexical.result.hits.hits.toList ::: semantic.result.hits.hits.toList - val resultsWithSemanticScoresFilledIn = allHits.flatMap { hit => + val lexicalHits = lexical.result.hits.hits.toList + val semanticHits = semantic.result.hits.hits.toList + val allHits = lexicalHits ::: semanticHits + logger.info(logMarker, s"${allHits.length} total hits: ${lexicalHits.length} lexical, ${semanticHits.length} semantic") + + val distinctHits = allHits.distinctBy(_.id) + logger.info(logMarker, s"${distinctHits.length} distinct hits") + + val resultsWithSemanticScoresFilledIn = distinctHits.flatMap { hit => resolveHitAndFillInSemanticScore(hit, queryEmbedding) } val topK = combineScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) From ac5bb489e019015c6c3f6f5e691a1b1d2e656a50 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 15:54:51 +0100 Subject: [PATCH 021/373] Fix cosine similarity to handle division by zero --- .../scala/com/gu/mediaservice/lib/VectorUtils.scala | 4 +++- .../com/gu/mediaservice/lib/VectorUtilsTest.scala | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala index a9f1a822872..81e9ebd64e5 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -10,6 +10,8 @@ object VectorUtils { } def cosineSimilarity(vectorOne: List[Double], vectorTwo: List[Double]): Double = { - dotProduct(vectorOne, vectorTwo) / (magnitude(vectorOne) * magnitude(vectorTwo)) + val magnitudeProduct = magnitude(vectorOne) * magnitude(vectorTwo) + if (magnitudeProduct == 0.0) 0.0 + else dotProduct(vectorOne, vectorTwo) / magnitudeProduct } } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala index ffa284ebe51..28b13fe968f 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -40,4 +40,16 @@ class VectorUtilsTest extends AnyFunSpec with Matchers { it ("should return a cosine similarity of -1 for opposite vectors") { VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)) shouldBe -1.0 +- 1e-9 } + + it ("should return a cosine similarity of 0 when the first vector has zero magnitude") { + VectorUtils.cosineSimilarity(List(0.0, 0.0, 0.0), List(1.0, 2.0, 3.0)) shouldBe 0.0 +- 1e-9 + } + + it ("should return a cosine similarity of 0 when the second vector has zero magnitude") { + VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + } + + it ("should return a cosine similarity of 0 when both vectors have zero magnitude") { + VectorUtils.cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + } } From 7dabe4a020bef7007a664edf81ec11b976678118 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 15:56:53 +0100 Subject: [PATCH 022/373] Use proper cosine similarity --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 115d9cac55c..69a16c58c56 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -282,12 +282,13 @@ class ElasticSearch( resolveHit(hit).map { image => val semanticScore = image.instance.embedding .flatMap(_.cohereEmbedV4) - // Save some computation by assuming normalised vectors - // TODO: double check this assumption + // We can't use the dot product shortcut because image vectors + // are truncated 256-dim versions of a normalised 1536-dim vector, + // meaning they will not have magnitude 1. // Note this is true cosine similarity from -1 to 1, // *not* the ES-normalised score, but when we max-normalise // later it will end up in the range 0-1. - .map(e => VectorUtils.dotProduct(e.image, queryEmbedding)) + .map(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) .getOrElse(-1.0) HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } From 3612a0b4336827c943e8121b7a54fe060a6ec796 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 16:02:58 +0100 Subject: [PATCH 023/373] Log timing after future completes --- .../app/lib/elasticsearch/ElasticSearch.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 69a16c58c56..68b13f94f2b 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -352,8 +352,8 @@ class ElasticSearch( // Assigning to vals here eagerly starts both requests, so they run in // parallel. The for-comprehension below only sequences the *combination* // of their results, not their execution. - val lexicalSearchResponse = executeAndLog(withSearchQueryTimeout(lexicalSearchRequest), "lexical") - val semanticSearchResponse = executeAndLog(withSearchQueryTimeout(semanticSearchRequest), "semantic") + val lexicalSearchResponse = executeAndLog(withSearchQueryTimeout(lexicalSearchRequest), "lexical-fill-scores") + val semanticSearchResponse = executeAndLog(withSearchQueryTimeout(semanticSearchRequest), "semantic-fill-scores") for { lexical <- lexicalSearchResponse @@ -397,11 +397,14 @@ class ElasticSearch( if (fillScores) { val searchResults = fillScoresSearch(query, queryEmbeddingDouble, k, numCandidates, vecWeight, filterOpt) - val elapsed = stopwatch.elapsed - logger.info( - combineMarkers(logMarker, elapsed), - s"hybrid search (fill scores) completed in ${elapsed.toMillis} ms" - ) + searchResults.foreach { _ => + val elapsed = stopwatch.elapsed + logger.info( + combineMarkers(logMarker, elapsed), + s"hybrid search (fill scores) completed in ${elapsed.toMillis} ms" + ) + } + searchResults } else { for { From 26c08d04c8109de3ddfd6c409e0e77e8de987395 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 16 Jun 2026 23:16:54 +0100 Subject: [PATCH 024/373] Add filtering --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 68b13f94f2b..90de7521c04 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -330,7 +330,7 @@ class ElasticSearch( val lexicalSearchRequest = ElasticDsl .search(imagesCurrentAlias) - .query(lexicalQuery) + .query(boolQuery().must(lexicalQuery).filter(filterOpt)) .size(k) val semanticSearchRequest = ElasticDsl From 220d6201c2e0b18f2a02edccf0f5a91212276df4 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 17 Jun 2026 14:00:44 +0100 Subject: [PATCH 025/373] Set size on KNN query --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 90de7521c04..a5129a43701 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -348,6 +348,7 @@ class ElasticSearch( .originalQueryWeight(0) .rescoreQueryWeight(1) ) + .size(k) // Assigning to vals here eagerly starts both requests, so they run in // parallel. The for-comprehension below only sequences the *combination* From 355e3039a71847570664e618cc12bc39cdc4ada8 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 17 Jun 2026 15:01:46 +0100 Subject: [PATCH 026/373] default vecWeight to 0.8 --- media-api/app/controllers/MediaApi.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 12c99aaeeae..9c2ad1369a3 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -650,7 +650,7 @@ class MediaApi( logger.info(markers, s"AI search embedding cache miss query=$semanticQuery") } - val weight = params.vecWeight.getOrElse(1.0) + val weight = params.vecWeight.getOrElse(0.8) // cache.get(key) is atomic: if two requests race on the same key, only one // load fires and both callers receive the same Future. From 0a7e6b8a50163b0bd99377ded452bde1ef8583fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 17 Jun 2026 15:13:32 +0000 Subject: [PATCH 027/373] chore(deps-dev): bump @babel/core from 7.28.0 to 7.29.6 in /kahuna Bumps [@babel/core](https://github.com/babel/babel/tree/HEAD/packages/babel-core) from 7.28.0 to 7.29.6. - [Release notes](https://github.com/babel/babel/releases) - [Changelog](https://github.com/babel/babel/blob/main/CHANGELOG.md) - [Commits](https://github.com/babel/babel/commits/v7.29.6/packages/babel-core) --- updated-dependencies: - dependency-name: "@babel/core" dependency-version: 7.29.6 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- kahuna/package-lock.json | 218 +++++++++++++++++---------------------- kahuna/package.json | 2 +- 2 files changed, 98 insertions(+), 122 deletions(-) diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index f1487a33969..5303b8f8054 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -47,7 +47,7 @@ "uuid": "9.0.0" }, "devDependencies": { - "@babel/core": "7.28.0", + "@babel/core": "7.29.6", "@babel/eslint-parser": "7.16.5", "@babel/preset-env": "7.28.0", "@jest/globals": "30.0.4", @@ -80,32 +80,6 @@ "webpack-merge": "5.8.0" } }, - "node_modules/@ampproject/remapping": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.2.0.tgz", - "integrity": "sha512-qRmjj8nj9qmLTQXXmaR1cck3UXSRMPrbsLJAasZpF+t3riI71BXed5ebIOYwQntykeZuhjsdweEc9BxH5Jc26w==", - "dev": true, - "dependencies": { - "@jridgewell/gen-mapping": "^0.1.0", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@ampproject/remapping/node_modules/@jridgewell/gen-mapping": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.1.1.tgz", - "integrity": "sha512-sQXCasFk+U8lWYEe66WxRDOE9PjVz4vSM51fTu3Hw+ClTpUSQb718772vH3pyS5pShp6lvQM7SxgIDXXXmOX7w==", - "dev": true, - "dependencies": { - "@jridgewell/set-array": "^1.0.0", - "@jridgewell/sourcemap-codec": "^1.4.10" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@asamuzakjp/css-color": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/@asamuzakjp/css-color/-/css-color-3.2.0.tgz", @@ -128,13 +102,13 @@ "license": "ISC" }, "node_modules/@babel/code-frame": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", - "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.27.1", + "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, @@ -143,9 +117,9 @@ } }, "node_modules/@babel/compat-data": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.0.tgz", - "integrity": "sha512-60X7qkglvrap8mn1lh2ebxXdZYtUcpd7gsmy9kLaBJ4i/WdY8PqTSdxyA8qraikqKQK5C1KRBKXqznrVapyNaw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", "dev": true, "license": "MIT", "engines": { @@ -153,22 +127,22 @@ } }, "node_modules/@babel/core": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.0.tgz", - "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.0", - "@babel/helper-compilation-targets": "^7.27.2", - "@babel/helper-module-transforms": "^7.27.3", - "@babel/helpers": "^7.27.6", - "@babel/parser": "^7.28.0", - "@babel/template": "^7.27.2", - "@babel/traverse": "^7.28.0", - "@babel/types": "^7.28.0", + "version": "7.29.6", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.6.tgz", + "integrity": "sha512-QdxmAo/ikZqqRGA8s43ww8lcql6naWRvEz0FFrl6MIlc7Gi6TroXnSdWa5U/kq6fzcpqpHesicQxFZIieZbyIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.0", + "@babel/generator": "^7.29.6", + "@babel/helper-compilation-targets": "^7.28.6", + "@babel/helper-module-transforms": "^7.28.6", + "@babel/helpers": "^7.29.2", + "@babel/parser": "^7.29.3", + "@babel/template": "^7.28.6", + "@babel/traverse": "^7.29.0", + "@babel/types": "^7.29.0", + "@jridgewell/remapping": "^2.3.5", "convert-source-map": "^2.0.0", "debug": "^4.1.0", "gensync": "^1.0.0-beta.2", @@ -242,14 +216,14 @@ } }, "node_modules/@babel/generator": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.0.tgz", - "integrity": "sha512-lJjzvrbEeWrhB4P3QBsH7tey117PjLZnDbLiQEKjQ/fNJTjuq4HSqgFA+UNSwZT8D7dxxbnuSBMsa1lrWzKlQg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", + "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", "dev": true, "license": "MIT", "dependencies": { - "@babel/parser": "^7.28.0", - "@babel/types": "^7.28.0", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7", "@jridgewell/gen-mapping": "^0.3.12", "@jridgewell/trace-mapping": "^0.3.28", "jsesc": "^3.0.2" @@ -272,14 +246,14 @@ } }, "node_modules/@babel/helper-compilation-targets": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", - "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/compat-data": "^7.27.2", - "@babel/helper-validator-option": "^7.27.1", + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", "browserslist": "^4.24.0", "lru-cache": "^5.1.1", "semver": "^6.3.1" @@ -376,9 +350,9 @@ } }, "node_modules/@babel/helper-globals": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", - "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", "dev": true, "license": "MIT", "engines": { @@ -400,29 +374,29 @@ } }, "node_modules/@babel/helper-module-imports": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", - "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", "dev": true, "license": "MIT", "dependencies": { - "@babel/traverse": "^7.27.1", - "@babel/types": "^7.27.1" + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-module-transforms": { - "version": "7.27.3", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.27.3.tgz", - "integrity": "sha512-dSOvYwvyLsWBeIRyOeHXp5vPj5l1I011r52FM1+r1jCERv+aFXYk4whgQccYEGYxK2H3ZAIA8nuPkQ0HaUo3qg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-module-imports": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1", - "@babel/traverse": "^7.27.3" + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -505,9 +479,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -515,9 +489,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.27.1.tgz", - "integrity": "sha512-D2hP9eA+Sqx1kBZgzxZh0y1trbuU+JoDkiEwqhQ36nodYqJwyEIhPSdMNd7lOm/4io72luTPWH20Yda0xOuUow==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", "engines": { @@ -525,9 +499,9 @@ } }, "node_modules/@babel/helper-validator-option": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", - "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", "dev": true, "license": "MIT", "engines": { @@ -550,27 +524,27 @@ } }, "node_modules/@babel/helpers": { - "version": "7.27.6", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.6.tgz", - "integrity": "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/template": "^7.27.2", - "@babel/types": "^7.27.6" + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/parser": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.0.tgz", - "integrity": "sha512-jVZGvOxOuNSsuQuLRTh13nU0AogFlw32w/MT+LV6D3sP5WdbW61E77RnkbaO2dUvmPAYrBDJXGn5gGS6tH4j8g==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.28.0" + "@babel/types": "^7.29.7" }, "bin": { "parser": "bin/babel-parser.js" @@ -1925,33 +1899,33 @@ } }, "node_modules/@babel/template": { - "version": "7.27.2", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", - "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/parser": "^7.27.2", - "@babel/types": "^7.27.1" + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.0.tgz", - "integrity": "sha512-mGe7UK5wWyh0bKRfupsUchrQGqvDbZDbKJw+kcRGSmdHVYrv+ltd0pnpDTVpiTqnaBru9iEvA8pz8W46v0Amwg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", + "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/code-frame": "^7.27.1", - "@babel/generator": "^7.28.0", - "@babel/helper-globals": "^7.28.0", - "@babel/parser": "^7.28.0", - "@babel/template": "^7.27.2", - "@babel/types": "^7.28.0", + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7", "debug": "^4.3.1" }, "engines": { @@ -1959,14 +1933,14 @@ } }, "node_modules/@babel/types": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.0.tgz", - "integrity": "sha512-jYnje+JyZG5YThjHiF28oT4SIZLnYOcSBb6+SDaFIyzDVSkXQmQQYclJ2R+YxcdmK0AX6x1E5OQNtuh3jHDrUg==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.27.1" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" @@ -3276,6 +3250,17 @@ "@jridgewell/trace-mapping": "^0.3.24" } }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", @@ -3285,15 +3270,6 @@ "node": ">=6.0.0" } }, - "node_modules/@jridgewell/set-array": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", - "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@jridgewell/source-map": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@jridgewell/source-map/-/source-map-0.3.2.tgz", diff --git a/kahuna/package.json b/kahuna/package.json index 3f1a820d562..ef05334ea74 100644 --- a/kahuna/package.json +++ b/kahuna/package.json @@ -42,7 +42,7 @@ "uuid": "9.0.0" }, "devDependencies": { - "@babel/core": "7.28.0", + "@babel/core": "7.29.6", "@babel/eslint-parser": "7.16.5", "@babel/preset-env": "7.28.0", "@jest/globals": "30.0.4", From 7683c460edfbd03c1deaa80f207ab36eafee879e Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 17 Jun 2026 16:56:59 +0100 Subject: [PATCH 028/373] Make fill scores the default approach --- kahuna/public/js/search/index.js | 1 - kahuna/public/js/search/query.html | 8 -- kahuna/public/js/search/query.js | 24 +---- kahuna/public/js/search/results.js | 1 - kahuna/public/js/services/api/media-api.js | 5 +- media-api/app/controllers/MediaApi.scala | 5 +- .../app/lib/elasticsearch/ElasticSearch.scala | 100 +++--------------- .../elasticsearch/ElasticSearchModel.scala | 4 +- 8 files changed, 23 insertions(+), 125 deletions(-) diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 84cb8914627..8897e6006ee 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -180,7 +180,6 @@ search.config(['$stateProvider', '$urlMatcherFactoryProvider', 'orderBy', 'useAISearch', 'vecWeight', - 'fillScores', 'dateField', 'takenSince', 'takenUntil', diff --git a/kahuna/public/js/search/query.html b/kahuna/public/js/search/query.html index 9130fb56338..a01d32be66f 100644 --- a/kahuna/public/js/search/query.html +++ b/kahuna/public/js/search/query.html @@ -7,14 +7,6 @@ Use AI search - - - search dateFilterParams.indexOf(key) === -1). - // Exclude useAISearch and fillScores, managed separately by their own dedicated watchers - filter(key => key !== 'useAISearch' && key !== 'fillScores'). + // Exclude useAISearch, managed separately by its own dedicated watcher + filter(key => key !== 'useAISearch'). forEach(setAndWatchParam); // URL parameters are not decoded when taken out of the params. @@ -460,24 +458,10 @@ query.controller('SearchQueryCtrl', [ $state.go('search.results', { ...ctrl.filter, useAISearch: true, - vecWeight: ctrl.vecWeight, - fillScores: ctrl.fillScores ? true : null + vecWeight: ctrl.vecWeight }); } else { - // fillScores is only meaningful with AI search, so clear it when AI search is disabled - ctrl.fillScores = false; - $state.go('search.results', {...ctrl.filter, useAISearch: null, fillScores: null}); - } - }); - $scope.$watch(() => ctrl.fillScores, () => { - // fillScores only applies when AI search is enabled - if (ctrl.useAISearch) { - $state.go('search.results', { - ...ctrl.filter, - useAISearch: true, - vecWeight: ctrl.vecWeight, - fillScores: ctrl.fillScores ? true : null - }); + $state.go('search.results', {...ctrl.filter, useAISearch: null}); } }); diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 17e43c937b3..6721bf53ba5 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -578,7 +578,6 @@ results.controller('SearchResultsCtrl', [ orderBy: orderBy, useAISearch: $stateParams.useAISearch, vecWeight: $stateParams.vecWeight, - fillScores: $stateParams.fillScores, hasRightsAcquired: $stateParams.hasRightsAcquired, hasCrops: $stateParams.hasCrops, syndicationStatus: $stateParams.syndicationStatus, diff --git a/kahuna/public/js/services/api/media-api.js b/kahuna/public/js/services/api/media-api.js index abeaef8176d..4687cb62774 100644 --- a/kahuna/public/js/services/api/media-api.js +++ b/kahuna/public/js/services/api/media-api.js @@ -42,7 +42,7 @@ mediaApi.factory('mediaApi', payType, uploadedBy, offset, length, orderBy, takenSince, takenUntil, modifiedSince, modifiedUntil, hasRightsAcquired, hasCrops, - syndicationStatus, countAll, persisted, useAISearch, vecWeight, fillScores} = {}) { + syndicationStatus, countAll, persisted, useAISearch, vecWeight} = {}) { return root.follow('search', { q: query, since: since, @@ -66,8 +66,7 @@ mediaApi.factory('mediaApi', countAll, persisted, useAISearch: maybeStringToBoolean(useAISearch), - vecWeight: vecWeight, - fillScores: maybeStringToBoolean(fillScores) + vecWeight: vecWeight }).get(); } diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index fc39a97bdfe..12c99aaeeae 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -90,8 +90,7 @@ class MediaApi( "countAll", "persisted", "useAISearch", - "vecWeight", - "fillScores" + "vecWeight" ).mkString(",") private val searchLinkHref = s"${config.rootUri}/images{?$searchParamList}" @@ -652,7 +651,6 @@ class MediaApi( } val weight = params.vecWeight.getOrElse(1.0) - val fillScores = params.fillScores.getOrElse(false) // cache.get(key) is atomic: if two requests race on the same key, only one // load fires and both callers receive the same Future. @@ -669,7 +667,6 @@ class MediaApi( k = k, numCandidates = Math.max(k * 2, 100), vecWeight = weight, - fillScores = fillScores, filterOpt = filterOpt ) } yield searchResults diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index a5129a43701..89c993325d5 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -12,7 +12,7 @@ import com.gu.mediaservice.model.{Agencies, Agency, AwaitingReviewForSyndication import com.sksamuel.elastic4s.{ElasticDsl, Hit} import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s.requests.common.Operator -import com.sksamuel.elastic4s.requests.common.Operator.{And, Or} +import com.sksamuel.elastic4s.requests.common.Operator.Or import com.sksamuel.elastic4s.requests.get.{GetRequest, GetResponse} import com.sksamuel.elastic4s.requests.script.{Script, ScriptField} import com.sksamuel.elastic4s.requests.searches._ @@ -21,7 +21,6 @@ import com.sksamuel.elastic4s.requests.searches.aggs.responses.Aggregations import com.sksamuel.elastic4s.requests.searches.aggs.responses.bucket.{DateHistogram, Terms} import com.sksamuel.elastic4s.requests.searches.queries.Query import com.sksamuel.elastic4s.requests.searches.knn.Knn -import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery import com.sksamuel.elastic4s.requests.searches.queries.matches.MultiMatchQueryBuilderType.BEST_FIELDS import com.sksamuel.elastic4s.requests.searches.queries.matches.{FieldWithOptionalBoost, MultiMatchQuery} import lib.querysyntax.{HierarchyField, Match, Parser, Phrase} @@ -214,59 +213,6 @@ class ElasticSearch( boost = boost ) - // BM25 scores are unbounded [0,inf] and typically much larger in magnitude - // than cosine similarity (knn). So we get the max BM25 score for the query and use that to calculate - // the scaling factor for the lexical part of the query, so that BM25 and knn scores are both between 0-1 scale - // and can be effectively combined in a hybrid query. - private def fetchMaxBm25Score(query: String, filterOpt: Option[Query])(implicit ex: ExecutionContext, logMarker: LogMarker): Future[Double] = { - val baseQuery = createMultiMatchQuery(query, operator = And) - val filteredQuery = filterOpt.map(filter => boolQuery().must(baseQuery).filter(filter)).getOrElse(baseQuery) - - val maxScoreRequest = ElasticDsl.search(imagesCurrentAlias) - .query(filteredQuery) - - executeAndLog(withSearchQueryTimeout(maxScoreRequest), "max BM25 score").map { r => - logger.info(logMarker, s"Max BM25 score for query '$query' is ${r.result.hits.maxScore}") - if (r.result.hits.hits.isEmpty) 1.0 else r.result.hits.maxScore - } - } - - private def makeHybridSearchRequest( - query: String, - queryEmbedding: List[Double], - k: Int, - numCandidates: Int, - vecWeight: Double, - maxScore: Double, - filterOpt: Option[Query] - )(implicit logMarker: LogMarker): SearchRequest = { - val knn = Knn("embedding.cohereEmbedV4.image", filter = filterOpt) - .queryVector(queryEmbedding) - .k(k) - .numCandidates(numCandidates) - .boost(if (vecWeight > 0.0) 1.0 else 0.0) - - val lexicalWeight = 1.0 - vecWeight - - // KNN results are in [0,1], but BM25 scores are unbounded and typically much - // larger than cosine similarity, so we need to apply a scaling factor to the - // BM25 score to bring it to the same range as the cosine similarity. - val scalingFactor = if (maxScore > 0.0) 1.0 / maxScore else 1.0 - - // We want to apply only one boost if we can help it, so we scale the - // multi_match boost to be in line with the max_score and the desired - // lexical_weight/vec_weight balance - val multiMatchBoost = if (vecWeight > 0.0) (lexicalWeight / vecWeight) * scalingFactor else 1.0 - - logger.info(logMarker, s"Scaling factor for BM25 score is $scalingFactor, multi-match boost is $multiMatchBoost") - - val multiMatchQuery = createMultiMatchQuery(query, boost = Some(multiMatchBoost), operator = And) - - ElasticDsl.search(imagesCurrentAlias) - .bool(BoolQuery().should(Seq(multiMatchQuery, knn)).filter(filterOpt)) - .size(k) - } - private case class HybridResult( id: String, lexicalScore: Double, @@ -316,8 +262,9 @@ class ElasticSearch( } } - // Alternative hybrid search mode, enabled via the `fillScores` query param. - private def fillScoresSearch( + // Runs lexical and semantic searches in parallel, fills in the missing scores + // for each result clientside, then combines and re-ranks them. + private def hybridSearchImpl( query: String, queryEmbedding: List[Double], k: Int, @@ -353,8 +300,8 @@ class ElasticSearch( // Assigning to vals here eagerly starts both requests, so they run in // parallel. The for-comprehension below only sequences the *combination* // of their results, not their execution. - val lexicalSearchResponse = executeAndLog(withSearchQueryTimeout(lexicalSearchRequest), "lexical-fill-scores") - val semanticSearchResponse = executeAndLog(withSearchQueryTimeout(semanticSearchRequest), "semantic-fill-scores") + val lexicalSearchResponse = executeAndLog(withSearchQueryTimeout(lexicalSearchRequest), "lexical-hybrid") + val semanticSearchResponse = executeAndLog(withSearchQueryTimeout(semanticSearchRequest), "semantic-hybrid") for { lexical <- lexicalSearchResponse @@ -382,7 +329,6 @@ class ElasticSearch( k: Int, numCandidates: Int, vecWeight: Double, - fillScores: Boolean, filterOpt: Option[Query] )( implicit ex: ExecutionContext, @@ -396,32 +342,16 @@ class ElasticSearch( val stopwatch = Stopwatch.start - if (fillScores) { - val searchResults = fillScoresSearch(query, queryEmbeddingDouble, k, numCandidates, vecWeight, filterOpt) - searchResults.foreach { _ => - val elapsed = stopwatch.elapsed - logger.info( - combineMarkers(logMarker, elapsed), - s"hybrid search (fill scores) completed in ${elapsed.toMillis} ms" - ) - } - - searchResults - } else { - for { - maxScore <- fetchMaxBm25Score(query, filterOpt) - searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore, filterOpt) - result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") - } yield { - val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) - val elapsed = stopwatch.elapsed - logger.info( - combineMarkers(logMarker, elapsed), - s"hybrid search (current) completed in ${elapsed.toMillis} ms" - ) - SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) - } + val searchResults = hybridSearchImpl(query, queryEmbeddingDouble, k, numCandidates, vecWeight, filterOpt) + searchResults.foreach { _ => + val elapsed = stopwatch.elapsed + logger.info( + combineMarkers(logMarker, elapsed), + s"hybrid search completed in ${elapsed.toMillis} ms" + ) } + + searchResults } } diff --git a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala index 1883e6758c8..e6dd8fed5b0 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala @@ -147,8 +147,7 @@ case class SearchParams( printUsageFilters: Option[PrintUsageFilters] = None, shouldFlagGraphicImages: Boolean = false, useAISearch: Option[Boolean] = None, - vecWeight: Option[Double] = None, - fillScores: Option[Boolean] = None + vecWeight: Option[Double] = None ) { lazy val aiQueryParts: AiQueryParts = AiQueryParts.from(structuredQuery) } @@ -243,7 +242,6 @@ object SearchParams { shouldFlagGraphicImages = false, request.getQueryString("useAISearch") flatMap parseBooleanFromQuery, request.getQueryString("vecWeight") flatMap parseBoundedDoubleFromQuery, - request.getQueryString("fillScores") flatMap parseBooleanFromQuery, ) } From d55ae3921b1ad99a03333330d858b58d35c0ddd4 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 17 Jun 2026 17:28:51 +0100 Subject: [PATCH 029/373] Break out HybridResult and add tests --- .../app/lib/elasticsearch/ElasticSearch.scala | 53 +---- .../app/lib/elasticsearch/HybridResult.scala | 55 +++++ .../lib/elasticsearch/HybridResultTest.scala | 191 ++++++++++++++++++ 3 files changed, 248 insertions(+), 51 deletions(-) create mode 100644 media-api/app/lib/elasticsearch/HybridResult.scala create mode 100644 media-api/test/lib/elasticsearch/HybridResultTest.scala diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 89c993325d5..b6c2427f5aa 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -1,7 +1,7 @@ package lib.elasticsearch import org.apache.pekko.actor.Scheduler -import com.gu.mediaservice.lib.{ImageFields, VectorUtils} +import com.gu.mediaservice.lib.ImageFields import com.gu.mediaservice.lib.argo.model.{ExtraCount, ExtraCountConfig, ExtraCounts} import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.lib.auth.Authentication.Principal @@ -213,55 +213,6 @@ class ElasticSearch( boost = boost ) - private case class HybridResult( - id: String, - lexicalScore: Double, - semanticScore: Double, - image: SourceWrapper[Image] - ) - - private case object HybridResult { - def resolveHitAndFillInSemanticScore( - hit: SearchHit, - queryEmbedding: List[Double] - ): Option[HybridResult] = - resolveHit(hit).map { image => - val semanticScore = image.instance.embedding - .flatMap(_.cohereEmbedV4) - // We can't use the dot product shortcut because image vectors - // are truncated 256-dim versions of a normalised 1536-dim vector, - // meaning they will not have magnitude 1. - // Note this is true cosine similarity from -1 to 1, - // *not* the ES-normalised score, but when we max-normalise - // later it will end up in the range 0-1. - .map(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) - .getOrElse(-1.0) - HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) - } - - def combineScoresAndGetTopK( - results: List[HybridResult], - vecWeight: Double, - k: Int - ): List[HybridResult] = { - // Account for the rare case in which KNN doesn't return the true closest vector, - // and that true closest vector happens to be among the lexical-only results. - val maxLexicalScore = results.maxBy(_.lexicalScore).lexicalScore - val maxSemanticScore = results.maxBy(_.semanticScore).semanticScore - - def combinedScore(result: HybridResult): Double = { - val normedLexicalScore = result.lexicalScore / maxLexicalScore - // normedScore = (score - theoretical_min) / (max - theoretical_min) - // This is theoretical min, i.e. -1, to actual max, - // so it effectively does both the max-norming and the ES-score norming. - val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) - (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) - } - - results.sortBy(combinedScore)(Ordering[Double].reverse).take(k) - } - } - // Runs lexical and semantic searches in parallel, fills in the missing scores // for each result clientside, then combines and re-ranks them. private def hybridSearchImpl( @@ -316,7 +267,7 @@ class ElasticSearch( logger.info(logMarker, s"${distinctHits.length} distinct hits") val resultsWithSemanticScoresFilledIn = distinctHits.flatMap { hit => - resolveHitAndFillInSemanticScore(hit, queryEmbedding) + resolveHitAndFillInSemanticScore(hit, queryEmbedding, resolveHit) } val topK = combineScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) SearchResults(hits = topK.map(r => (r.id, r.image)), total = topK.length, extraCounts = None) diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala new file mode 100644 index 00000000000..79f30370d93 --- /dev/null +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -0,0 +1,55 @@ +package lib.elasticsearch + +import com.gu.mediaservice.lib.VectorUtils +import com.gu.mediaservice.model.Image +import com.sksamuel.elastic4s.requests.searches.SearchHit + +case class HybridResult( + id: String, + lexicalScore: Double, + semanticScore: Double, + image: SourceWrapper[Image] +) + +object HybridResult { + def resolveHitAndFillInSemanticScore( + hit: SearchHit, + queryEmbedding: List[Double], + resolveHit: SearchHit => Option[SourceWrapper[Image]] + ): Option[HybridResult] = + resolveHit(hit).map { image => + val semanticScore = image.instance.embedding + .flatMap(_.cohereEmbedV4) + // We can't use the dot product shortcut because image vectors + // are truncated 256-dim versions of a normalised 1536-dim vector, + // meaning they will not have magnitude 1. + // Note this is true cosine similarity from -1 to 1, + // *not* the ES-normalised score, but when we max-normalise + // later it will end up in the range 0-1. + .map(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) + .getOrElse(-1.0) + HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) + } + + def combineScoresAndGetTopK( + results: List[HybridResult], + vecWeight: Double, + k: Int + ): List[HybridResult] = { + // Account for the rare case in which KNN doesn't return the true closest vector, + // and that true closest vector happens to be among the lexical-only results. + val maxLexicalScore = results.maxBy(_.lexicalScore).lexicalScore + val maxSemanticScore = results.maxBy(_.semanticScore).semanticScore + + def combinedScore(result: HybridResult): Double = { + val normedLexicalScore = result.lexicalScore / maxLexicalScore + // normedScore = (score - theoretical_min) / (max - theoretical_min) + // This is theoretical min, i.e. -1, to actual max, + // so it effectively does both the max-norming and the ES-score norming. + val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) + (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) + } + + results.sortBy(combinedScore)(Ordering[Double].reverse).take(k) + } +} diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala new file mode 100644 index 00000000000..10b6bb6d720 --- /dev/null +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -0,0 +1,191 @@ +package lib.elasticsearch + +import com.gu.mediaservice.model.{CohereV4Embedding, Embedding, Handout, Image} +import com.sksamuel.elastic4s.requests.searches.SearchHit +import org.scalactic.Tolerance +import org.scalatest.OptionValues +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import play.api.libs.json.Json + +class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with Tolerance with Fixtures { + + import HybridResult.{combineScoresAndGetTopK, resolveHitAndFillInSemanticScore} + + private val tolerance = 1e-9 + + private def searchHit(id: String, score: Float): SearchHit = SearchHit( + id = id, + index = "test-index", + version = 1L, + seqNo = 0L, + primaryTerm = 0L, + score = score, + parent = None, + shard = None, + node = None, + routing = None, + explanation = None, + sort = None, + _source = Map.empty, + fields = Map.empty, + _highlight = None, + inner_hits = Map.empty, + matchedQueries = None + ) + + private def imageWithEmbedding(id: String, embedding: Option[List[Double]]): Image = + createImage(id, Handout()).copy( + embedding = embedding.map(vec => Embedding(cohereEmbedV4 = Some(CohereV4Embedding(image = vec)))) + ) + + private def sourceWrapperFor(image: Image): SourceWrapper[Image] = + SourceWrapper(Json.obj(), image, fromIndex = "test-index") + + private def resolveTo(image: Image): SearchHit => Option[SourceWrapper[Image]] = + _ => Some(sourceWrapperFor(image)) + + private val resolveToNothing: SearchHit => Option[SourceWrapper[Image]] = _ => None + + private def hybridResult(id: String, lexicalScore: Double, semanticScore: Double): HybridResult = + HybridResult( + id = id, + lexicalScore = lexicalScore, + semanticScore = semanticScore, + image = sourceWrapperFor(imageWithEmbedding(id, embedding = None)) + ) + + describe("resolveHitAndFillInSemanticScore") { + + it("returns None when the hit cannot be resolved to an image") { + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 3.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveToNothing + ) + result should be(None) + } + + it("carries the hit id and uses the hit score as the lexical score") { + val image = imageWithEmbedding("img-1", embedding = Some(List(1.0, 0.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 4.2f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.id should be("img-1") + result.lexicalScore should be(4.2f.toDouble +- tolerance) + } + + it("computes the semantic score as the cosine similarity between the image and query embeddings") { + // Orthogonal vectors -> cosine similarity 0. + val image = imageWithEmbedding("img-1", embedding = Some(List(0.0, 1.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(0.0 +- tolerance) + } + + it("returns a cosine similarity of 1 for identical (parallel) embeddings") { + val image = imageWithEmbedding("img-1", embedding = Some(List(3.0, 4.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(6.0, 8.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(1.0 +- tolerance) + } + + it("returns a cosine similarity of -1 for opposite embeddings") { + val image = imageWithEmbedding("img-1", embedding = Some(List(-1.0, 0.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(-1.0 +- tolerance) + } + + it("falls back to a semantic score of -1 when the image has no embedding") { + val image = imageWithEmbedding("img-1", embedding = None) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(-1.0 +- tolerance) + } + } + + describe("combineScoresAndGetTopK") { + + it("ranks purely by lexical score when vecWeight is 0") { + val results = List( + hybridResult("low-lexical", lexicalScore = 1.0, semanticScore = 1.0), + hybridResult("high-lexical", lexicalScore = 10.0, semanticScore = -1.0), + hybridResult("mid-lexical", lexicalScore = 5.0, semanticScore = 0.0) + ) + + val ranked = combineScoresAndGetTopK(results, vecWeight = 0.0, k = 3) + + ranked.map(_.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) + } + + it("ranks purely by semantic score when vecWeight is 1") { + val results = List( + hybridResult("low-semantic", lexicalScore = 10.0, semanticScore = -1.0), + hybridResult("high-semantic", lexicalScore = 1.0, semanticScore = 1.0), + hybridResult("mid-semantic", lexicalScore = 5.0, semanticScore = 0.0) + ) + + val ranked = combineScoresAndGetTopK(results, vecWeight = 1.0, k = 3) + + ranked.map(_.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) + } + + it("only returns the top k results") { + val results = List( + hybridResult("a", lexicalScore = 1.0, semanticScore = 1.0), + hybridResult("b", lexicalScore = 10.0, semanticScore = 1.0), + hybridResult("c", lexicalScore = 5.0, semanticScore = 1.0) + ) + + val ranked = combineScoresAndGetTopK(results, vecWeight = 0.0, k = 2) + + ranked should have size 2 + ranked.map(_.id) should be(List("b", "c")) + } + + it("blends the max-normalised lexical and semantic scores using vecWeight") { + // maxLexical = 4, maxSemantic = 1. + // For "blend": normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 + // combined = 0.25 * 0.5 + 0.75 * 0.5 = 0.5 + val results = List( + hybridResult("top-lexical", lexicalScore = 4.0, semanticScore = -1.0), + hybridResult("top-semantic", lexicalScore = 0.0, semanticScore = 1.0), + hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0) + ) + + val ranked = combineScoresAndGetTopK(results, vecWeight = 0.25, k = 3) + + val blend = ranked.find(_.id == "blend").value + // Recompute the expected combined score independently of ordering. + val normedLexical = blend.lexicalScore / 4.0 + val normedSemantic = (blend.semanticScore + 1) / (1.0 + 1) + val expectedCombined = 0.25 * normedSemantic + 0.75 * normedLexical + expectedCombined should be(0.5 +- tolerance) + + // top-lexical: normedLexical = 1, normedSemantic = 0 -> 0.75 + // top-semantic: normedLexical = 0, normedSemantic = 1 -> 0.25 + // blend: 0.5 + ranked.map(_.id) should be(List("top-lexical", "blend", "top-semantic")) + } + } +} From 88fedc398b83a2cb223bb2982601a971ef09af7c Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 17 Jun 2026 17:50:18 +0100 Subject: [PATCH 030/373] Add comment --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index b6c2427f5aa..aebc08b9265 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -263,6 +263,8 @@ class ElasticSearch( val allHits = lexicalHits ::: semanticHits logger.info(logMarker, s"${allHits.length} total hits: ${lexicalHits.length} lexical, ${semanticHits.length} semantic") + // This is only valid because the queries ensure that all hits, including semantic, + // have only the lexical score at this point. val distinctHits = allHits.distinctBy(_.id) logger.info(logMarker, s"${distinctHits.length} distinct hits") From 4b2e2db0a1c3826ca2f76412cec7bbcd3657d0e6 Mon Sep 17 00:00:00 2001 From: Andrew Nowak Date: Thu, 18 Jun 2026 11:26:52 +0100 Subject: [PATCH 031/373] upgraded AWS SDK v2 to latest version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 571b0bc9e53..fc00f3d8fa3 100644 --- a/build.sbt +++ b/build.sbt @@ -77,7 +77,7 @@ Global / concurrentRestrictions := Seq( ) val awsSdkVersion = "1.12.797" -val awsSdkV2Version = "2.44.13" +val awsSdkV2Version = "2.46.13" val elastic4sVersion = "8.19.1" val awsKclVersion = "3.4.3" val okHttpVersion = "3.12.1" From eab0c66b499dbdd006ddb3cd86f8e2ea5b1a2369 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Thu, 18 Jun 2026 12:00:39 +0100 Subject: [PATCH 032/373] Break out functions, more tests, more comments --- .../app/lib/elasticsearch/ElasticSearch.scala | 3 + .../app/lib/elasticsearch/HybridResult.scala | 50 +++++--- .../lib/elasticsearch/HybridResultTest.scala | 121 +++++++++++++++++- 3 files changed, 158 insertions(+), 16 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index aebc08b9265..a894120364d 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -215,6 +215,9 @@ class ElasticSearch( // Runs lexical and semantic searches in parallel, fills in the missing scores // for each result clientside, then combines and re-ranks them. + // This approach was inspired by + // "An Analysis of Fusion Functions for Hybrid Retrieval" + // https://arxiv.org/pdf/2210.11934 private def hybridSearchImpl( query: String, queryEmbedding: List[Double], diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index 79f30370d93..d1e55d57781 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -31,25 +31,45 @@ object HybridResult { HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } + // This is the "theoretical min-max" normalisation chosen by + // "An Analysis of Fusion Functions for Hybrid Retrieval" + // https://arxiv.org/pdf/2210.11934 + def normalise(score: Double, max: Double, theoreticalMin: Double): Double = + if (max == theoreticalMin) 0.0 + else (score - theoreticalMin) / (max - theoreticalMin) + + def combinedScore( + result: HybridResult, + maxLexicalScore: Double, + maxSemanticScore: Double, + vecWeight: Double + ): Double = { + val normedLexicalScore = normalise(result.lexicalScore, maxLexicalScore, theoreticalMin = 0.0) + // The semantic theoretical min of -1 means we do both the max-norming + // and the ES-score norming in one step. + val normedSemanticScore = normalise(result.semanticScore, maxSemanticScore, theoreticalMin = -1.0) + (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) + } + def combineScoresAndGetTopK( results: List[HybridResult], vecWeight: Double, k: Int ): List[HybridResult] = { - // Account for the rare case in which KNN doesn't return the true closest vector, - // and that true closest vector happens to be among the lexical-only results. - val maxLexicalScore = results.maxBy(_.lexicalScore).lexicalScore - val maxSemanticScore = results.maxBy(_.semanticScore).semanticScore - - def combinedScore(result: HybridResult): Double = { - val normedLexicalScore = result.lexicalScore / maxLexicalScore - // normedScore = (score - theoretical_min) / (max - theoretical_min) - // This is theoretical min, i.e. -1, to actual max, - // so it effectively does both the max-norming and the ES-score norming. - val normedSemanticScore = (result.semanticScore + 1) / (maxSemanticScore + 1) - (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) - } - - results.sortBy(combinedScore)(Ordering[Double].reverse).take(k) + // Why do this rather than looking at the top result from elasticsearch or hits.maxScore? + // Because for the semantic query, we rescore by BM25, meaning neither maxScore nor the + // top result will actually tell us the max semantic score. + // + // A side benefit is that this approach also accounts for the rare case in which KNN + // doesn't even contain true closest vector, and that true closest vector happens to + // be among the lexical-only results. + (for { + maxLexicalScore <- results.map(_.lexicalScore).maxOption + maxSemanticScore <- results.map(_.semanticScore).maxOption + } yield { + results + .sortBy(combinedScore(_, maxLexicalScore, maxSemanticScore, vecWeight))(Ordering[Double].reverse) + .take(k) + }).getOrElse(List()) } } diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index 10b6bb6d720..9a14ebaff6a 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -10,7 +10,7 @@ import play.api.libs.json.Json class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with Tolerance with Fixtures { - import HybridResult.{combineScoresAndGetTopK, resolveHitAndFillInSemanticScore} + import HybridResult.{combineScoresAndGetTopK, combinedScore, normalise, resolveHitAndFillInSemanticScore} private val tolerance = 1e-9 @@ -124,7 +124,102 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T } } + describe("normalise") { + it("maps a score to its fraction of the way from the theoretical min to the max") { + // Halfway between theoreticalMin 0 and max 10. + normalise(score = 5.0, max = 10.0, theoreticalMin = 0.0) should be(0.5 +- tolerance) + } + + it("maps the max itself to 1") { + normalise(score = 10.0, max = 10.0, theoreticalMin = 0.0) should be(1.0 +- tolerance) + } + + it("maps the theoretical min itself to 0") { + normalise(score = -1.0, max = 1.0, theoreticalMin = -1.0) should be(0.0 +- tolerance) + } + + it("accounts for a non-zero theoretical min when normalising") { + // (0 - -1) / (1 - -1) = 0.5 + normalise(score = 0.0, max = 1.0, theoreticalMin = -1.0) should be(0.5 +- tolerance) + } + + it("returns 0 when the range collapses (max == theoreticalMin)") { + normalise(score = 0.0, max = 0.0, theoreticalMin = 0.0) should be(0.0 +- tolerance) + normalise(score = -1.0, max = -1.0, theoreticalMin = -1.0) should be(0.0 +- tolerance) + } + } + + describe("combinedScore") { + it("blends the normalised lexical and semantic scores using vecWeight") { + // normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 + // combined = 0.25 * 0.5 + 0.75 * 0.5 = 0.5 + val score = combinedScore( + hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0), + maxLexicalScore = 4.0, + maxSemanticScore = 1.0, + vecWeight = 0.25 + ) + score should be(0.5 +- tolerance) + } + + it("uses only the lexical score when vecWeight is 0") { + val score = combinedScore( + hybridResult("lexical-only", lexicalScore = 2.0, semanticScore = 1.0), + maxLexicalScore = 4.0, + maxSemanticScore = 1.0, + vecWeight = 0.0 + ) + score should be(0.5 +- tolerance) + } + + it("uses only the semantic score when vecWeight is 1") { + val score = combinedScore( + hybridResult("semantic-only", lexicalScore = 2.0, semanticScore = 0.0), + maxLexicalScore = 4.0, + maxSemanticScore = 1.0, + vecWeight = 1.0 + ) + // normedSemantic = (0 + 1)/(1 + 1) = 0.5 + score should be(0.5 +- tolerance) + } + + it("contributes 0 from the lexical side when the max lexical score is 0") { + val score = combinedScore( + hybridResult("no-lexical", lexicalScore = 0.0, semanticScore = 1.0), + maxLexicalScore = 0.0, + maxSemanticScore = 1.0, + vecWeight = 0.5 + ) + // normedLexical = 0, normedSemantic = (1 + 1)/(1 + 1) = 1 + // combined = 0.5 * 1 + 0.5 * 0 = 0.5 + score should be(0.5 +- tolerance) + } + + it("contributes 0 from the semantic side when the max semantic score is -1") { + val score = combinedScore( + hybridResult("no-semantic", lexicalScore = 4.0, semanticScore = -1.0), + maxLexicalScore = 4.0, + maxSemanticScore = -1.0, + vecWeight = 0.5 + ) + // normedSemantic = 0, normedLexical = 4/4 = 1 + // combined = 0.5 * 0 + 0.5 * 1 = 0.5 + score should be(0.5 +- tolerance) + } + } + describe("combineScoresAndGetTopK") { + it("returns an empty list if the input is empty") { + combineScoresAndGetTopK(List(), vecWeight = 0.0, k = 3) should be (List()) + } + + it("doesn't break when tha max lexical score is 0") { + val results = List( + hybridResult("low-lexical", lexicalScore = 0.0, semanticScore = 1.0), + hybridResult("high-lexical", lexicalScore = 0.0, semanticScore = -1.0), + hybridResult("mid-lexical", lexicalScore = 0.0, semanticScore = 0.0) + ) + } it("ranks purely by lexical score when vecWeight is 0") { val results = List( @@ -138,6 +233,18 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T ranked.map(_.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) } + it("ranks purely by lexical score when max semantic score is -1.0") { + val results = List( + hybridResult("low-lexical", lexicalScore = 1.0, semanticScore = -1.0), + hybridResult("high-lexical", lexicalScore = 10.0, semanticScore = -1.0), + hybridResult("mid-lexical", lexicalScore = 5.0, semanticScore = -1.0) + ) + + val ranked = combineScoresAndGetTopK(results, vecWeight = 0.5, k = 3) + + ranked.map(_.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) + } + it("ranks purely by semantic score when vecWeight is 1") { val results = List( hybridResult("low-semantic", lexicalScore = 10.0, semanticScore = -1.0), @@ -150,6 +257,18 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T ranked.map(_.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) } + it("ranks purely by semantic score when max lexical score is 0") { + val results = List( + hybridResult("low-semantic", lexicalScore = 0.0, semanticScore = -1.0), + hybridResult("high-semantic", lexicalScore = 0.0, semanticScore = 1.0), + hybridResult("mid-semantic", lexicalScore = 0.0, semanticScore = 0.0) + ) + + val ranked = combineScoresAndGetTopK(results, vecWeight = 0.5, k = 3) + + ranked.map(_.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) + } + it("only returns the top k results") { val results = List( hybridResult("a", lexicalScore = 1.0, semanticScore = 1.0), From 6f23087c5af1b1ef017d9dadceb8b0ffb4421890 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Thu, 18 Jun 2026 12:29:53 +0100 Subject: [PATCH 033/373] Add integration test --- .../lib/elasticsearch/ElasticSearchTest.scala | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 9f333f9cea1..cdc523e470a 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -448,6 +448,119 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast } } + describe("AI / hybrid search") { + implicit val logMarker: LogMarker = MarkerMap() + + // 256-dim vectors to match the `embedding.cohereEmbedV4.image` dense_vector + // mapping. One-hot vectors are orthogonal and unit-magnitude, so cosine + // similarity is 1.0 against themselves and 0.0 against each other - which + // makes the semantic ranking easy to reason about in assertions. + def oneHot(hotIndex: Int): List[Double] = + List.tabulate(256)(i => if (i == hotIndex) 1.0 else 0.0) + + def withEmbedding(image: Image, vector: List[Double]): Image = + image.copy(embedding = Some(Embedding(cohereEmbedV4 = Some(CohereV4Embedding(image = vector))))) + + // The vector we'll "search" with - represents the user's query embedding. + val queryEmbedding: List[Float] = oneHot(0).map(_.toFloat) + + // Matches the text "kitten" but is semantically far from the query vector: + // a pure *lexical* hit (lexicalScore high, semanticScore ~0). + val lexicalMatch = withEmbedding( + createImage("ai-lexical-match", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("a kitten playing in the garden"), keywords = Some(Set("kitten"))) + ), + oneHot(42) + ) + + // Does NOT match the text at all, but IS the nearest neighbour of the query + // vector: a pure *semantic* hit (semanticScore ~1, lexicalScore 0). + val semanticMatch = withEmbedding( + createImage("ai-semantic-match", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("completely unrelated wording"), keywords = Some(Set("unrelated"))) + ), + oneHot(0) + ) + + // Noise: neither a text match nor a close vector match. + val unrelated = withEmbedding( + createImage("ai-unrelated", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("nothing to see here"), keywords = Some(Set("noise"))) + ), + oneHot(99) + ) + + val aiImages = Seq(lexicalMatch, semanticMatch, unrelated) + + // Seed exactly once for the whole describe block: the three tests share the + // same fixtures, and re-indexing the same ids per-test causes version + // conflicts (409s) when the suite's match-all purge runs in afterAll. + lazy val aiImagesIndexed: Unit = { + Await.ready(saveImages(aiImages), 1.minute) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds)) { + totalImages shouldBe (expectedNumberOfImages + aiImages.size) + } + } + + // k = 2 (not, say, 10) is deliberate: with a small k the search is forced + // to discriminate. If we asked for more results than we indexed, kNN would + // return everything and the assertions would pass trivially. + def hybridSearchIds(vecWeight: Double, k: Int = 2): Seq[String] = + Await.result( + ES.hybridSearch( + query = "kitten", + queryEmbedding = queryEmbedding, + k = k, + numCandidates = 10, + vecWeight = vecWeight, + filterOpt = None + ), + fiveSeconds + ).hits.map(_._1) + + it("combines lexical (BM25) and semantic (kNN) search, retrieving images by meaning even when the text doesn't match, and fusing the two rankings") { + aiImagesIndexed + + val ids = hybridSearchIds(vecWeight = 0.5) + + // Semantic recall: the nearest-vector image is returned even though it + // shares no words with the query - only the kNN path could have found it. + ids should contain("ai-semantic-match") + + // Lexical recall still works alongside semantic recall. + ids should contain("ai-lexical-match") + + // The meaningful assertion: the noise image is neither a text match nor a + // near vector, so fusion + top-k ranking drops it. This is what proves the + // search actually discriminates rather than just returning everything. + ids should not contain "ai-unrelated" + } + + it("ranks the semantic match above the lexical match when vecWeight is cranked towards vectors") { + aiImagesIndexed + + // vecWeight ~1 => the fused score is dominated by cosine similarity, so the + // pure-vector match should outrank the pure-text match. + val ids = hybridSearchIds(vecWeight = 0.99) + + ids should contain("ai-semantic-match") + ids should contain("ai-lexical-match") + ids.indexOf("ai-semantic-match") should be < ids.indexOf("ai-lexical-match") + } + + it("ranks the lexical match above the semantic match when vecWeight is cranked towards text") { + aiImagesIndexed + + // vecWeight ~0 => the fused score is dominated by BM25, so the pure-text + // match should outrank the pure-vector match. The mirror image of the above. + val ids = hybridSearchIds(vecWeight = 0.01) + + ids should contain("ai-semantic-match") + ids should contain("ai-lexical-match") + ids.indexOf("ai-lexical-match") should be < ids.indexOf("ai-semantic-match") + } + } + private def saveImages(images: Seq[Image]) = { implicit val logMarker: LogMarker = MarkerMap() From ffc0b9ccb687045f5ccebd75d83a891332180f79 Mon Sep 17 00:00:00 2001 From: Andrew Nowak Date: Thu, 9 Apr 2026 12:08:00 +0100 Subject: [PATCH 034/373] on undeletion, mark image as archived so it cannot be immediately rereaped --- .../com/gu/mediaservice/GridClient.scala | 7 ++++++- .../gr-undelete-image/gr-un-delete-image.js | 1 + media-api/app/controllers/MediaApi.scala | 20 +++++++++++-------- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala index c9058ea2850..73bc0d2a378 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala @@ -8,7 +8,7 @@ import com.gu.mediaservice.model.leases.LeasesByMedia import com.gu.mediaservice.model.usage.Usage import com.typesafe.scalalogging.LazyLogging import play.api.http.HeaderNames -import play.api.libs.json.{JsArray, JsObject, JsValue, Json, Reads} +import play.api.libs.json.{JsArray, JsObject, JsTrue, JsValue, Json, Reads} import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{ExecutionContext, Future} @@ -260,6 +260,11 @@ class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLo authorisedRequest.post(Json.obj("data" -> data)).map { response => validateResponse(response, url)} } + def putArchived(mediaId: String, authFn: WSRequest => WSRequest)(implicit ec: ExecutionContext) = { + val url = new URL(s"${services.metadataBaseUri}/metadata/$mediaId/archived") + val request = authFn(wsClient.url(url.toString)) + request.put(Json.obj("data" -> JsTrue)).map { response => validateResponse(response, url)} + } } class DownstreamApiInBadStateException(message: String, downstreamMessage: String) extends IllegalStateException(message) { diff --git a/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js b/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js index aeff5022cfe..0e25d4be036 100644 --- a/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js +++ b/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js @@ -23,6 +23,7 @@ undeleteImage.controller('grUnDeleteImageCtrl', [ apiPoll(findImage); } + // TODO remove "unusable" banner and recognise that the image is now archived ctrl.unDeleteImage = function (image) { return mediaApi.undelete(image.data.id) .then(() => pollDeleted(image)) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 9c2ad1369a3..7c53dcbcad3 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -404,15 +404,19 @@ class MediaApi( && ImageExtras.userMayUndeleteImage(request.user, image, authorisation) => logger.info(logMarker, s"undeleting image $id") - softDeletedMetadataTable.updateStatus(id, isDeleted = false) - .map { _ => - messageSender.publish( - UpdateMessage( - subject = UnSoftDeleteImage, - id = Some(id) - ) + for { + // Take an "undeletion" as a marker that the file should be kept and not reaped. + // Mark as archived before undeleting, to make sure the reaper doesn't immediately reap it. + _ <- gridClient.putArchived(id, auth.getOnBehalfOfPrincipal(request.user)) + _ <- softDeletedMetadataTable.updateStatus(id, isDeleted = false) + _ = messageSender.publish( + UpdateMessage( + subject = UnSoftDeleteImage, + id = Some(id) ) - }.map { _ => Accepted } + ) + } yield Accepted + case Some(image) if isVisibleToAccessor(request.user, image) => logger.info(logMarker, s"user ${request.user.accessor.identity} was not permitted to undelete image $id") Future.successful(ImageDeleteForbidden) From 9b15ce57646e5de37b7a90ce5772f6a9f85ce0c3 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Thu, 18 Jun 2026 17:04:02 +0100 Subject: [PATCH 035/373] Extra logging, maybe not needed --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index a894120364d..c49ddc84273 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -271,10 +271,17 @@ class ElasticSearch( val distinctHits = allHits.distinctBy(_.id) logger.info(logMarker, s"${distinctHits.length} distinct hits") + val lexicalIds = lexicalHits.map(_.id).toSet + val semanticIds = semanticHits.map(_.id).toSet + val resultsWithSemanticScoresFilledIn = distinctHits.flatMap { hit => resolveHitAndFillInSemanticScore(hit, queryEmbedding, resolveHit) } val topK = combineScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) + topK.foreach { r => + logger.info(logMarker, s"hybrid result ${r.id}: lexicalScore=${r.lexicalScore} semanticScore=${r.semanticScore} " + + s"originallyFromLexical=${lexicalIds.contains(r.id)} originallyFromSemantic=${semanticIds.contains(r.id)}") + } SearchResults(hits = topK.map(r => (r.id, r.image)), total = topK.length, extraCounts = None) } } From 21104999638c7064d73fe8211bcb75fed7f17e93 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Thu, 18 Jun 2026 18:32:39 +0100 Subject: [PATCH 036/373] Short-circuit hybrid search if vecWeight is 0 or 1 --- media-api/app/controllers/MediaApi.scala | 2 +- .../app/lib/elasticsearch/ElasticSearch.scala | 106 ++++++++++++------ .../lib/elasticsearch/ElasticSearchTest.scala | 47 ++++++-- 3 files changed, 105 insertions(+), 50 deletions(-) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 12c99aaeeae..8cc6dea662b 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -625,7 +625,7 @@ class MediaApi( searchResults <- maybeEmbedding match { // If we have an embedding, perform the KNN search. If not, return an empty result set. case Some(embedding) => - elasticSearch.knnSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) + elasticSearch.semanticSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) case None => Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) } diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index c49ddc84273..430f7283c72 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -179,28 +179,6 @@ class ElasticSearch( } } - def knnSearch(queryEmbedding: List[Float], k: Int, numCandidates: Int, filterOpt: Option[Query]) - (implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { - if (!includeDenseVectorMappings) { - logger.warn(logMarker, "knnSearch called but includeDenseVectorMappings=false, returning empty results") - Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) - } else { - val knn = Knn("embedding.cohereEmbedV4.image", filter = filterOpt) - .queryVector(queryEmbedding.map(_.toDouble)) - .k(k) - .numCandidates(numCandidates) - - val searchRequest = ElasticDsl.search(imagesCurrentAlias) - .knn(knn) - .size(k) - - executeAndLog(withSearchQueryTimeout(searchRequest), "knn search").map { r => - val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) - SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) - } - } - } - private def createMultiMatchQuery(query: String, boost: Option[Double] = None, operator: Operator): MultiMatchQuery = MultiMatchQuery( text = query, @@ -213,12 +191,69 @@ class ElasticSearch( boost = boost ) + private def lexicalRequest( + lexicalQuery: MultiMatchQuery, + k: Int, + filterOpt: Option[Query] + ): SearchRequest = + ElasticDsl + .search(imagesCurrentAlias) + .query(boolQuery().must(lexicalQuery).filter(filterOpt)) + .size(k) + + private def lexicalSearch( + query: String, + k: Int, + filterOpt: Option[Query] + )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { + val searchRequest = lexicalRequest(createMultiMatchQuery(query, operator = Or), k, filterOpt) + + executeAndLog(withSearchQueryTimeout(searchRequest), "lexical-only").map { r => + val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) + SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + } + } + + private def semanticRequest( + queryEmbedding: List[Double], + k: Int, numCandidates: Int, + filterOpt: Option[Query] + ): SearchRequest = + ElasticDsl + .search(imagesCurrentAlias) + .knn(Knn("embedding.cohereEmbedV4.image", filter = filterOpt) + .queryVector(queryEmbedding) + .k(k) + .numCandidates(numCandidates) + ) + .size(k) + + def semanticSearch( + queryEmbedding: List[Float], + k: Int, + numCandidates: Int, + filterOpt: Option[Query] + )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { + if (!includeDenseVectorMappings) { + // TODO: could we factor out this check into semanticRequest or elsewhere? + logger.warn(logMarker, "semanticSearch called but includeDenseVectorMappings=false, returning empty results") + Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) + } else { + val searchRequest = semanticRequest(queryEmbedding.map(_.toDouble), k, numCandidates, filterOpt) + + executeAndLog(withSearchQueryTimeout(searchRequest), "semantic search").map { r => + val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) + SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) + } + } + } + // Runs lexical and semantic searches in parallel, fills in the missing scores // for each result clientside, then combines and re-ranks them. // This approach was inspired by // "An Analysis of Fusion Functions for Hybrid Retrieval" // https://arxiv.org/pdf/2210.11934 - private def hybridSearchImpl( + private def fusedLexicalAndSemanticSearch( query: String, queryEmbedding: List[Double], k: Int, @@ -227,20 +262,11 @@ class ElasticSearch( filterOpt: Option[Query] )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { import HybridResult.{resolveHitAndFillInSemanticScore, combineScoresAndGetTopK} - val lexicalQuery = createMultiMatchQuery(query, operator = Or) - val lexicalSearchRequest = ElasticDsl - .search(imagesCurrentAlias) - .query(boolQuery().must(lexicalQuery).filter(filterOpt)) - .size(k) + val lexicalQuery = createMultiMatchQuery(query, operator = Or) + val lexicalSearchRequest = lexicalRequest(lexicalQuery, k, filterOpt) - val semanticSearchRequest = ElasticDsl - .search(imagesCurrentAlias) - .knn(Knn("embedding.cohereEmbedV4.image", filter = filterOpt) - .queryVector(queryEmbedding) - .k(k) - .numCandidates(numCandidates) - ) + val semanticSearchRequest = semanticRequest(queryEmbedding, k, numCandidates, filterOpt) .rescore(Rescore(lexicalQuery) .window(k) // We want to replace the knn score with the BM25 score, @@ -249,7 +275,6 @@ class ElasticSearch( .originalQueryWeight(0) .rescoreQueryWeight(1) ) - .size(k) // Assigning to vals here eagerly starts both requests, so they run in // parallel. The for-comprehension below only sequences the *combination* @@ -305,7 +330,14 @@ class ElasticSearch( val stopwatch = Stopwatch.start - val searchResults = hybridSearchImpl(query, queryEmbeddingDouble, k, numCandidates, vecWeight, filterOpt) + // When the weighting is entirely on one side, short-circuit to that side + // alone rather than running both queries and fusing. + val searchResults = vecWeight match { + case 0.0 => lexicalSearch(query, k, filterOpt) + case 1.0 => semanticSearch(queryEmbedding, k, numCandidates, filterOpt) + case _ => fusedLexicalAndSemanticSearch(query, queryEmbeddingDouble, k, numCandidates, vecWeight, filterOpt) + } + searchResults.foreach { _ => val elapsed = stopwatch.elapsed logger.info( diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index cdc523e470a..66c5a84cc45 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -464,17 +464,17 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast // The vector we'll "search" with - represents the user's query embedding. val queryEmbedding: List[Float] = oneHot(0).map(_.toFloat) - // Matches the text "kitten" but is semantically far from the query vector: - // a pure *lexical* hit (lexicalScore high, semanticScore ~0). - val lexicalMatch = withEmbedding( + // Matches the text "kitten" but has NO embedding at all: a pure *lexical* + // hit. Deliberately un-embedded so it is invisible to the kNN (semantic) + // side of the search - only the BM25 (lexical) side can ever surface it. + val lexicalMatch = createImage("ai-lexical-match", staffPhotographer).copy( metadata = ImageMetadata(title = Some("a kitten playing in the garden"), keywords = Some(Set("kitten"))) - ), - oneHot(42) - ) + ) // Does NOT match the text at all, but IS the nearest neighbour of the query - // vector: a pure *semantic* hit (semanticScore ~1, lexicalScore 0). + // vector: a pure *semantic* hit. The only image in the fixture set with an + // embedding, so it is the sole image the kNN side can ever return. val semanticMatch = withEmbedding( createImage("ai-semantic-match", staffPhotographer).copy( metadata = ImageMetadata(title = Some("completely unrelated wording"), keywords = Some(Set("unrelated"))) @@ -482,13 +482,12 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast oneHot(0) ) - // Noise: neither a text match nor a close vector match. - val unrelated = withEmbedding( + // Noise: neither a text match nor an embedding, so neither side of the + // search can ever surface it. + val unrelated = createImage("ai-unrelated", staffPhotographer).copy( metadata = ImageMetadata(title = Some("nothing to see here"), keywords = Some(Set("noise"))) - ), - oneHot(99) - ) + ) val aiImages = Seq(lexicalMatch, semanticMatch, unrelated) @@ -559,6 +558,30 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast ids should contain("ai-lexical-match") ids.indexOf("ai-lexical-match") should be < ids.indexOf("ai-semantic-match") } + + it("short-circuits to a pure lexical (BM25) search when vecWeight is exactly 0, skipping the semantic side entirely") { + aiImagesIndexed + + // vecWeight == 0 => the semantic side contributes nothing to the fused + // score, so the kNN query should be skipped altogether. The only result + // should be the text match; the semantic-only image must not leak in via + // the kNN path. + val ids = hybridSearchIds(vecWeight = 0) + + ids shouldEqual Seq("ai-lexical-match") + } + + it("short-circuits to a pure semantic (kNN) search when vecWeight is exactly 1, skipping the lexical side entirely") { + aiImagesIndexed + + // vecWeight == 1 => the lexical side contributes nothing to the fused + // score, so the BM25 query should be skipped altogether. The only result + // should be the vector match; the lexical-only image must not leak in via + // the BM25 path. + val ids = hybridSearchIds(vecWeight = 1) + + ids shouldEqual Seq("ai-semantic-match") + } } private def saveImages(images: Seq[Image]) = { From da7badaeab12aa005fbf0a2eaeee1c5a3d170649 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 19 Jun 2026 13:07:39 +0100 Subject: [PATCH 037/373] Add tests that check score filling behaviour for those outside top k of one query --- .../lib/elasticsearch/ElasticSearchTest.scala | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 66c5a84cc45..e6360ff6df0 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -582,6 +582,158 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast ids shouldEqual Seq("ai-semantic-match") } + + // ---------------------------------------------------------------------- + // Score "cross-pollination": every candidate is scored on BOTH dimensions, + // regardless of which side of the search originally surfaced it. + // + // A hit found only by the lexical side still has its true cosine similarity + // filled in client-side; a hit found only by the semantic side still carries + // the BM25 score it earns from the lexical rescore. The two tests below pin + // down each of those behaviours in isolation. + // + // They use their own query word + vector axis (so the fixtures above can't + // interfere) and a few small unit vectors with hand-picked cosine values. + // ---------------------------------------------------------------------- + + // Builds a 256-dim unit vector from (index -> value) components. Used to make + // vectors with specific cosine similarities to a one-hot query axis, e.g. + // unitVector(7 -> 0.8, 8 -> 0.6) has cosine 0.8 with oneHot(7). + def unitVector(components: (Int, Double)*): List[Double] = { + val arr = Array.fill(256)(0.0) + components.foreach { case (i, v) => arr(i) = v } + arr.toList + } + + def hybridSearchIdsFor(query: String, queryEmbedding: List[Float], vecWeight: Double, k: Int): Seq[String] = + Await.result( + ES.hybridSearch( + query = query, + queryEmbedding = queryEmbedding, + k = k, + numCandidates = 10, + vecWeight = vecWeight, + filterOpt = None + ), + fiveSeconds + ).hits.map(_._1) + + // Test 1 fixtures - query axis oneHot(7), query word "axolotl". + // + // A lexical hit that the kNN side never returns. Its embedding sits at cosine + // 0.5 from the query, but 'ai-semantic-hit-near-vector' (cosine 0.8) is nearer, + // so with k = 1 the semantic side only ever returns the latter - leaving this + // image strictly outside the top-k semantic results. + val lexicalHitWithVector = withEmbedding( + createImage("ai-lexical-hit-with-vector", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("a lone axolotl"), keywords = Some(Set("axolotl"))) + ), + unitVector(7 -> 0.5, 8 -> math.sqrt(0.75)) + ) + + // The nearer vector, with no text match: the only image the kNN side returns + // for the query above. + val semanticHitNearVector = withEmbedding( + createImage("ai-semantic-hit-near-vector", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("the blurst of times")) + ), + unitVector(7 -> 0.8, 8 -> 0.6) + ) + + // Test 2 fixtures - query axis oneHot(9), query phrase "narwhal arctic ocean". + // + // A semantic hit that the lexical side never returns. It is one of the two + // nearest vectors AND a (weak) text match for "narwhal", but the two strong + // lexical matches below outscore it on BM25, so with k = 2 it falls strictly + // outside the top-k lexical results. + val semanticHitWithText = withEmbedding( + createImage("ai-semantic-hit-with-text", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("a narwhal")) + ), + unitVector(9 -> 0.9, 10 -> math.sqrt(0.19)) + ) + + // Equally close to the query vector as 'ai-semantic-hit-with-text' (cosine + // 0.9), but with no text match at all - so its lexical score is zero. The + // identical semantic score is what lets us attribute any ranking difference + // purely to the lexical contribution. + val semanticHitNoText = withEmbedding( + createImage("ai-semantic-hit-no-text", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("the blurst of times")) + ), + unitVector(9 -> 0.9, 11 -> math.sqrt(0.19)) + ) + + // Two strong, un-embedded lexical matches that fill the top-k lexical results + // and push 'ai-semantic-hit-with-text' out of them. The first matches all + // three query terms, the second matches two. + val lexicalStrongOne = + createImage("ai-lexical-strong-1", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("narwhal arctic ocean")) + ) + val lexicalStrongTwo = + createImage("ai-lexical-strong-2", staffPhotographer).copy( + metadata = ImageMetadata(title = Some("narwhal arctic")) + ) + + val contributionImages = Seq( + lexicalHitWithVector, + semanticHitNearVector, + semanticHitWithText, + semanticHitNoText, + lexicalStrongOne, + lexicalStrongTwo + ) + + // Seed once, on top of the aiImages above, so the total is deterministic no + // matter which tests run. + lazy val contributionImagesIndexed: Unit = { + aiImagesIndexed + Await.ready(saveImages(contributionImages), 1.minute) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds)) { + totalImages shouldBe (expectedNumberOfImages + aiImages.size + contributionImages.size) + } + } + + it("scores a lexical-only hit by its embedding, even though the kNN side never returned it") { + contributionImagesIndexed + + val query = "axolotl" + val queryVector = oneHot(7).map(_.toFloat) + + // A pure semantic search returns only the nearer vector - proving the + // lexical hit is genuinely outside the top-k semantic results. + hybridSearchIdsFor(query, queryVector, vecWeight = 1.0, k = 1) shouldEqual Seq("ai-semantic-hit-near-vector") + + // At a strongly semantic-leaning weight the lexical-only hit nonetheless + // wins. It can only do so because its true cosine similarity (0.5) was + // filled in client-side: had its semantic score been treated as absent, the + // nearer vector (0.8) would have taken the single slot at this weight. + hybridSearchIdsFor(query, queryVector, vecWeight = 0.8, k = 1) shouldEqual Seq("ai-lexical-hit-with-vector") + } + + it("scores a semantic-only hit by BM25, even though the lexical side never returned it") { + contributionImagesIndexed + + val query = "narwhal arctic ocean" + val queryVector = oneHot(9).map(_.toFloat) + + // A pure lexical search returns the two strong text matches and not + // 'ai-semantic-hit-with-text' - proving it is genuinely outside the top-k + // lexical results. + val pureLexical = hybridSearchIdsFor(query, queryVector, vecWeight = 0.0, k = 2) + pureLexical should contain("ai-lexical-strong-1") + pureLexical should contain("ai-lexical-strong-2") + pureLexical should not contain "ai-semantic-hit-with-text" + + // The two semantic hits are equidistant from the query vector, so their + // semantic scores are identical and both rise above the lexical matches at + // this weight. The ONLY thing that can separate them is the weak BM25 score + // the text-bearing one picked up via the semantic-side rescore - so it must + // rank first. + hybridSearchIdsFor(query, queryVector, vecWeight = 0.6, k = 2) shouldEqual + Seq("ai-semantic-hit-with-text", "ai-semantic-hit-no-text") + } } private def saveImages(images: Seq[Image]) = { From 6debe3cc73176c680b80939b2758c49a0bc564b6 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 19 Jun 2026 13:11:24 +0100 Subject: [PATCH 038/373] Rename combine to fuse --- .../app/lib/elasticsearch/ElasticSearch.scala | 4 +-- .../app/lib/elasticsearch/HybridResult.scala | 6 ++--- .../lib/elasticsearch/HybridResultTest.scala | 26 +++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 430f7283c72..afbaa2f0137 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -261,7 +261,7 @@ class ElasticSearch( vecWeight: Double, filterOpt: Option[Query] )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { - import HybridResult.{resolveHitAndFillInSemanticScore, combineScoresAndGetTopK} + import HybridResult.{resolveHitAndFillInSemanticScore, fuseScoresAndGetTopK} val lexicalQuery = createMultiMatchQuery(query, operator = Or) val lexicalSearchRequest = lexicalRequest(lexicalQuery, k, filterOpt) @@ -302,7 +302,7 @@ class ElasticSearch( val resultsWithSemanticScoresFilledIn = distinctHits.flatMap { hit => resolveHitAndFillInSemanticScore(hit, queryEmbedding, resolveHit) } - val topK = combineScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) + val topK = fuseScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) topK.foreach { r => logger.info(logMarker, s"hybrid result ${r.id}: lexicalScore=${r.lexicalScore} semanticScore=${r.semanticScore} " + s"originallyFromLexical=${lexicalIds.contains(r.id)} originallyFromSemantic=${semanticIds.contains(r.id)}") diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index d1e55d57781..499131c66fe 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -38,7 +38,7 @@ object HybridResult { if (max == theoreticalMin) 0.0 else (score - theoreticalMin) / (max - theoreticalMin) - def combinedScore( + def fuseScores( result: HybridResult, maxLexicalScore: Double, maxSemanticScore: Double, @@ -51,7 +51,7 @@ object HybridResult { (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) } - def combineScoresAndGetTopK( + def fuseScoresAndGetTopK( results: List[HybridResult], vecWeight: Double, k: Int @@ -68,7 +68,7 @@ object HybridResult { maxSemanticScore <- results.map(_.semanticScore).maxOption } yield { results - .sortBy(combinedScore(_, maxLexicalScore, maxSemanticScore, vecWeight))(Ordering[Double].reverse) + .sortBy(fuseScores(_, maxLexicalScore, maxSemanticScore, vecWeight))(Ordering[Double].reverse) .take(k) }).getOrElse(List()) } diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index 9a14ebaff6a..245a7709946 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -10,7 +10,7 @@ import play.api.libs.json.Json class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with Tolerance with Fixtures { - import HybridResult.{combineScoresAndGetTopK, combinedScore, normalise, resolveHitAndFillInSemanticScore} + import HybridResult.{fuseScoresAndGetTopK, fuseScores, normalise, resolveHitAndFillInSemanticScore} private val tolerance = 1e-9 @@ -153,7 +153,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T it("blends the normalised lexical and semantic scores using vecWeight") { // normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 // combined = 0.25 * 0.5 + 0.75 * 0.5 = 0.5 - val score = combinedScore( + val score = fuseScores( hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0), maxLexicalScore = 4.0, maxSemanticScore = 1.0, @@ -163,7 +163,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T } it("uses only the lexical score when vecWeight is 0") { - val score = combinedScore( + val score = fuseScores( hybridResult("lexical-only", lexicalScore = 2.0, semanticScore = 1.0), maxLexicalScore = 4.0, maxSemanticScore = 1.0, @@ -173,7 +173,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T } it("uses only the semantic score when vecWeight is 1") { - val score = combinedScore( + val score = fuseScores( hybridResult("semantic-only", lexicalScore = 2.0, semanticScore = 0.0), maxLexicalScore = 4.0, maxSemanticScore = 1.0, @@ -184,7 +184,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T } it("contributes 0 from the lexical side when the max lexical score is 0") { - val score = combinedScore( + val score = fuseScores( hybridResult("no-lexical", lexicalScore = 0.0, semanticScore = 1.0), maxLexicalScore = 0.0, maxSemanticScore = 1.0, @@ -196,7 +196,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T } it("contributes 0 from the semantic side when the max semantic score is -1") { - val score = combinedScore( + val score = fuseScores( hybridResult("no-semantic", lexicalScore = 4.0, semanticScore = -1.0), maxLexicalScore = 4.0, maxSemanticScore = -1.0, @@ -210,7 +210,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T describe("combineScoresAndGetTopK") { it("returns an empty list if the input is empty") { - combineScoresAndGetTopK(List(), vecWeight = 0.0, k = 3) should be (List()) + fuseScoresAndGetTopK(List(), vecWeight = 0.0, k = 3) should be (List()) } it("doesn't break when tha max lexical score is 0") { @@ -228,7 +228,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T hybridResult("mid-lexical", lexicalScore = 5.0, semanticScore = 0.0) ) - val ranked = combineScoresAndGetTopK(results, vecWeight = 0.0, k = 3) + val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.0, k = 3) ranked.map(_.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) } @@ -240,7 +240,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T hybridResult("mid-lexical", lexicalScore = 5.0, semanticScore = -1.0) ) - val ranked = combineScoresAndGetTopK(results, vecWeight = 0.5, k = 3) + val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.5, k = 3) ranked.map(_.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) } @@ -252,7 +252,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T hybridResult("mid-semantic", lexicalScore = 5.0, semanticScore = 0.0) ) - val ranked = combineScoresAndGetTopK(results, vecWeight = 1.0, k = 3) + val ranked = fuseScoresAndGetTopK(results, vecWeight = 1.0, k = 3) ranked.map(_.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) } @@ -264,7 +264,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T hybridResult("mid-semantic", lexicalScore = 0.0, semanticScore = 0.0) ) - val ranked = combineScoresAndGetTopK(results, vecWeight = 0.5, k = 3) + val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.5, k = 3) ranked.map(_.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) } @@ -276,7 +276,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T hybridResult("c", lexicalScore = 5.0, semanticScore = 1.0) ) - val ranked = combineScoresAndGetTopK(results, vecWeight = 0.0, k = 2) + val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.0, k = 2) ranked should have size 2 ranked.map(_.id) should be(List("b", "c")) @@ -292,7 +292,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0) ) - val ranked = combineScoresAndGetTopK(results, vecWeight = 0.25, k = 3) + val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.25, k = 3) val blend = ranked.find(_.id == "blend").value // Recompute the expected combined score independently of ordering. From f7b6fbafaea8939280f86d165149016b733785f6 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 19 Jun 2026 13:14:23 +0100 Subject: [PATCH 039/373] Put theoretical minima into constants --- .../app/lib/elasticsearch/HybridResult.scala | 11 ++++++++--- .../lib/elasticsearch/HybridResultTest.scala | 16 ++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index 499131c66fe..137bbd75a89 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -12,6 +12,11 @@ case class HybridResult( ) object HybridResult { + // Theoretical minimum cosine similarity (opposite vectors). + val CosineSimilarityTheoreticalMin: Double = -1.0 + // Theoretical minimum BM25 lexical score. + val Bm25TheoreticalMin: Double = 0.0 + def resolveHitAndFillInSemanticScore( hit: SearchHit, queryEmbedding: List[Double], @@ -27,7 +32,7 @@ object HybridResult { // *not* the ES-normalised score, but when we max-normalise // later it will end up in the range 0-1. .map(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) - .getOrElse(-1.0) + .getOrElse(CosineSimilarityTheoreticalMin) HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } @@ -44,10 +49,10 @@ object HybridResult { maxSemanticScore: Double, vecWeight: Double ): Double = { - val normedLexicalScore = normalise(result.lexicalScore, maxLexicalScore, theoreticalMin = 0.0) + val normedLexicalScore = normalise(result.lexicalScore, maxLexicalScore, theoreticalMin = Bm25TheoreticalMin) // The semantic theoretical min of -1 means we do both the max-norming // and the ES-score norming in one step. - val normedSemanticScore = normalise(result.semanticScore, maxSemanticScore, theoreticalMin = -1.0) + val normedSemanticScore = normalise(result.semanticScore, maxSemanticScore, theoreticalMin = CosineSimilarityTheoreticalMin) (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) } diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index 245a7709946..04d98a5e9c5 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -10,7 +10,7 @@ import play.api.libs.json.Json class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with Tolerance with Fixtures { - import HybridResult.{fuseScoresAndGetTopK, fuseScores, normalise, resolveHitAndFillInSemanticScore} + import HybridResult.{fuseScoresAndGetTopK, fuseScores, normalise, resolveHitAndFillInSemanticScore, Bm25TheoreticalMin, CosineSimilarityTheoreticalMin} private val tolerance = 1e-9 @@ -120,32 +120,32 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T resolveHit = resolveTo(image) ).value - result.semanticScore should be(-1.0 +- tolerance) + result.semanticScore should be(CosineSimilarityTheoreticalMin +- tolerance) } } describe("normalise") { it("maps a score to its fraction of the way from the theoretical min to the max") { // Halfway between theoreticalMin 0 and max 10. - normalise(score = 5.0, max = 10.0, theoreticalMin = 0.0) should be(0.5 +- tolerance) + normalise(score = 5.0, max = 10.0, theoreticalMin = Bm25TheoreticalMin) should be(0.5 +- tolerance) } it("maps the max itself to 1") { - normalise(score = 10.0, max = 10.0, theoreticalMin = 0.0) should be(1.0 +- tolerance) + normalise(score = 10.0, max = 10.0, theoreticalMin = Bm25TheoreticalMin) should be(1.0 +- tolerance) } it("maps the theoretical min itself to 0") { - normalise(score = -1.0, max = 1.0, theoreticalMin = -1.0) should be(0.0 +- tolerance) + normalise(score = -1.0, max = 1.0, theoreticalMin = CosineSimilarityTheoreticalMin) should be(0.0 +- tolerance) } it("accounts for a non-zero theoretical min when normalising") { // (0 - -1) / (1 - -1) = 0.5 - normalise(score = 0.0, max = 1.0, theoreticalMin = -1.0) should be(0.5 +- tolerance) + normalise(score = 0.0, max = 1.0, theoreticalMin = CosineSimilarityTheoreticalMin) should be(0.5 +- tolerance) } it("returns 0 when the range collapses (max == theoreticalMin)") { - normalise(score = 0.0, max = 0.0, theoreticalMin = 0.0) should be(0.0 +- tolerance) - normalise(score = -1.0, max = -1.0, theoreticalMin = -1.0) should be(0.0 +- tolerance) + normalise(score = 0.0, max = 0.0, theoreticalMin = Bm25TheoreticalMin) should be(0.0 +- tolerance) + normalise(score = -1.0, max = -1.0, theoreticalMin = CosineSimilarityTheoreticalMin) should be(0.0 +- tolerance) } } From 8d1a9780510f2073a7d357f169998433526cfb52 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Fri, 19 Jun 2026 17:27:34 +0100 Subject: [PATCH 040/373] Sync es CI version with the local and PROD version --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ae66882420..86fc7e5cb16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,10 @@ jobs: pull-requests: write services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.29 + image: docker.elastic.co/elasticsearch/elasticsearch:8.18.3 # Wait for elasticsearch to report healthy before continuing. # see https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml#L28 - options: -e "discovery.type=single-node" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 + options: -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 ports: - 9200:9200 localstack: From fcbefc94510b3e5704b4b5ef5442881c38991d01 Mon Sep 17 00:00:00 2001 From: Andrew Nowak Date: Tue, 2 Jun 2026 17:12:41 +0100 Subject: [PATCH 041/373] only set Photographer usage rights if no other processor has set usage rights first --- .../lib/cleanup/SupplierProcessors.scala | 35 ++++++++++--------- .../lib/cleanup/SupplierProcessorsTest.scala | 12 +++++++ 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala index 87b4f28ccd3..4c897f2f857 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessors.scala @@ -22,9 +22,9 @@ class SupplierProcessors(resources: ImageProcessorResources) ReutersParser, RexParser, RonaldGrantParser, - new PhotographerParser(resources.commonConfiguration.usageRightsConfig), AllstarSportsphotoParser, AllStarParser, + new PhotographerParser(resources.commonConfiguration.usageRightsConfig), UsageRightsToMetadataParser(resources) //This should come after processors that assign usage rights ) @@ -52,21 +52,24 @@ case class UsageRightsToMetadataParser(resources: ImageProcessorResources) exten * Guardian specific logic to correctly identify Guardian and Observer photographers and their contracts */ class PhotographerParser(photographersConfig: UsageRightsConfigProvider) extends ImageProcessor { - def apply(image: Image): Image = { - image.metadata.byline.flatMap { byline => - photographersConfig.getPhotographer(byline, image.metadata.dateTaken.getOrElse(image.uploadTime)).map{ - case p: StaffPhotographer => image.copy( - usageRights = p, - metadata = image.metadata.copy(credit = Some(p.publication), byline = Some(p.photographer)) - ) - case p: ContractPhotographer => image.copy( - usageRights = p, - metadata = image.metadata.copy(credit = p.publication, byline = Some(p.photographer)) - ) - case _ => image - } - } - }.getOrElse(image) + def apply(image: Image): Image = image.usageRights match { + // only attempt to set Photographer Usagerights if no other processor has already set usage rights + case NoRights => + image.metadata.byline.flatMap { byline => + photographersConfig.getPhotographer(byline, image.metadata.dateTaken.getOrElse(image.uploadTime)).map{ + case p: StaffPhotographer => image.copy( + usageRights = p, + metadata = image.metadata.copy(credit = Some(p.publication), byline = Some(p.photographer)) + ) + case p: ContractPhotographer => image.copy( + usageRights = p, + metadata = image.metadata.copy(credit = p.publication, byline = Some(p.photographer)) + ) + case _ => image + } + }.getOrElse(image) + case _ => image + } } object AapParser extends ImageProcessor { diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala index 3cb7da390a6..bb75dca2bd8 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/cleanup/SupplierProcessorsTest.scala @@ -68,6 +68,18 @@ class SupplierProcessorsTest extends AnyFunSpec with Matchers with MetadataHelpe processedImage.usageRights should be(ContractPhotographer("Murdo MacLeod", Option("The Guardian"))) processedImage.metadata.byline should be(Some("Murdo MacLeod")) } + + it("should only assign Photographer rights if agency rights not already assigned") { + val image = createImageFromMetadata( + "byline" -> "Murdo MacLeod", + "credit" -> "PA Wire/PA Images" + ) + val processedImage = applyProcessors(image) + // UsageRights assigned to PA agency rather than ContractPhotographer("Murdo MacLeod") as PA + // runs first, and Staff/ContractPhotographer only assign rights if nothing else has first. + processedImage.usageRights should be(Agency("PA")) + processedImage.metadata.byline should be(Some("Murdo MacLeod")) + } } describe("AAP") { From 0e3d5af26f1dd934b2eb39f00b76f7733908e450 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Mon, 22 Jun 2026 12:18:04 +0100 Subject: [PATCH 042/373] Remove heap size and limit and refine comments on es CI settings --- .github/workflows/ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86fc7e5cb16..90c5be0f3e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,15 @@ jobs: services: elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:8.18.3 - # Wait for elasticsearch to report healthy before continuing. + # These options should mimic those set in ElasticSearchDockerBase.scala, because this is + # the CI version of the TestContainers approach we use for running tests locally. + # `xpack.security.enabled=false` is needed since v8 for tests to use plain + # unauthenticated HTTP, because in v7 => v8 they changed the default to `true`: + # https://www.elastic.co/guide/en/elasticsearch/reference/8.18/release-notes-8.0.0.html#:~:text=Set%20xpack.security.enabled%20to%20true%20for%20all%20licenses + # + # `--health-` settings make it wait for es to report healthy before continuing. # see https://github.com/actions/example-services/blob/master/.github/workflows/postgres-service.yml#L28 - options: -e "discovery.type=single-node" -e "xpack.security.enabled=false" -e "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 + options: -e "discovery.type=single-node" -e "xpack.security.enabled=false" --expose 9200 --health-cmd "curl localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 10 ports: - 9200:9200 localstack: From 665ea22bc0fe5373c0d54091fe02a0a82732ef34 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Mon, 22 Jun 2026 15:08:24 +0100 Subject: [PATCH 043/373] Clean up test --- .../test/lib/elasticsearch/HybridResultTest.scala | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index 04d98a5e9c5..4e945911879 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -208,19 +208,11 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T } } - describe("combineScoresAndGetTopK") { + describe("fuseScoresAndGetTopK") { it("returns an empty list if the input is empty") { fuseScoresAndGetTopK(List(), vecWeight = 0.0, k = 3) should be (List()) } - it("doesn't break when tha max lexical score is 0") { - val results = List( - hybridResult("low-lexical", lexicalScore = 0.0, semanticScore = 1.0), - hybridResult("high-lexical", lexicalScore = 0.0, semanticScore = -1.0), - hybridResult("mid-lexical", lexicalScore = 0.0, semanticScore = 0.0) - ) - } - it("ranks purely by lexical score when vecWeight is 0") { val results = List( hybridResult("low-lexical", lexicalScore = 1.0, semanticScore = 1.0), From b36cc6a1f33d17e1b166e2946d213525979877cd Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Mon, 22 Jun 2026 15:16:55 +0100 Subject: [PATCH 044/373] Use List[Double] everywhere for Cohere v4 embeddings --- .../scala/com/gu/mediaservice/lib/aws/Bedrock.scala | 4 ++-- .../scala/com/gu/mediaservice/lib/aws/Embedder.scala | 2 +- media-api/app/controllers/MediaApi.scala | 4 ++-- media-api/app/lib/elasticsearch/ElasticSearch.scala | 10 ++++------ 4 files changed, 9 insertions(+), 11 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala index bb1a8dfaabb..d5e0321fd83 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Bedrock.scala @@ -76,14 +76,14 @@ class Bedrock(config: CommonConfig) } } - def createTextEmbedding(inputData: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Float]] = { + def createTextEmbedding(inputData: String)(implicit ec: ExecutionContext, logMarker: LogMarker): Future[List[Double]] = { val requestBody = createRequestBody(inputData) val bedrockFuture = Future { sendBedrockEmbeddingRequest(requestBody) } bedrockFuture.map { response => val responseBody = response.body().asUtf8String() val json = Json.parse(responseBody) // Extract the embedding array (first element since it's an array of arrays) - val embedding = (json \ "embeddings" \ "float")(0).as[List[Float]] + val embedding = (json \ "embeddings" \ "float")(0).as[List[Double]] logger.info( logMarker, s"Successfully extracted text embedding. Vector size: ${embedding.size}" diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index f3a5050fed0..f766fec70e2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -18,7 +18,7 @@ object EmbedderMessage { class Embedder(bedrock: Bedrock, sqs: SimpleSqsMessageConsumer)(implicit ec: ExecutionContext) extends GridLogging { - def createQueryEmbedding(query: String)(implicit logMarker: LogMarker): Future[List[Float]] = { + def createQueryEmbedding(query: String)(implicit logMarker: LogMarker): Future[List[Double]] = { logger.info(logMarker, s"Creating text embedding for query: $query") for { embedding <- bedrock.createTextEmbedding(query) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 9c2ad1369a3..1ad9793c36d 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -55,7 +55,7 @@ class MediaApi( // Process-local cache keyed on normalised query text. Stores the Bedrock Future so that // concurrent requests for the same query share a single in-flight Bedrock call, and // subsequent requests within the TTL window skip Bedrock entirely. - private val embeddingCache: AsyncLoadingCache[String, List[Float]] = Scaffeine() + private val embeddingCache: AsyncLoadingCache[String, List[Double]] = Scaffeine() .maximumSize(config.aiSearchEmbeddingCacheMaxSize) .buildAsyncFuture((normQuery: String) => embedder.createQueryEmbedding(normQuery)(MarkerMap()) @@ -621,7 +621,7 @@ class MediaApi( .filter(image => isVisibleToAccessor(request.user, image)) .flatMap(_.embedding) .flatMap(_.cohereEmbedV4) - .map(_.image.map(_.toFloat)) + .map(_.image) searchResults <- maybeEmbedding match { // If we have an embedding, perform the KNN search. If not, return an empty result set. case Some(embedding) => diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 0d890ba598a..9c818aa1c08 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -179,14 +179,14 @@ class ElasticSearch( } } - def knnSearch(queryEmbedding: List[Float], k: Int, numCandidates: Int, filterOpt: Option[Query]) + def knnSearch(queryEmbedding: List[Double], k: Int, numCandidates: Int, filterOpt: Option[Query]) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { if (!includeDenseVectorMappings) { logger.warn(logMarker, "knnSearch called but includeDenseVectorMappings=false, returning empty results") Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) } else { val knn = Knn("embedding.cohereEmbedV4.image", filter = filterOpt) - .queryVector(queryEmbedding.map(_.toDouble)) + .queryVector(queryEmbedding) .k(k) .numCandidates(numCandidates) @@ -268,7 +268,7 @@ class ElasticSearch( def hybridSearch( query: String, - queryEmbedding: List[Float], + queryEmbedding: List[Double], k: Int, numCandidates: Int, vecWeight: Double, @@ -281,11 +281,9 @@ class ElasticSearch( logger.warn(logMarker, "hybridSearch called but includeDenseVectorMappings=false, returning empty results") Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) } else { - val queryEmbeddingDouble: List[Double] = queryEmbedding.map(_.toDouble) - for { maxScore <- fetchMaxBm25Score(query, filterOpt) - searchRequest = makeHybridSearchRequest(query, queryEmbeddingDouble, k, numCandidates, vecWeight, maxScore, filterOpt) + searchRequest = makeHybridSearchRequest(query, queryEmbedding, k, numCandidates, vecWeight, maxScore, filterOpt) result <- executeAndLog(withSearchQueryTimeout(searchRequest), "hybrid search") } yield { val imageHits = result.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) From 8706093727a8074a4be1d5f53cdd4784ba8ace93 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Mon, 22 Jun 2026 18:16:25 +0100 Subject: [PATCH 045/373] Add more vector utils to help with testing --- .../com/gu/mediaservice/lib/VectorUtils.scala | 42 +++++++++ .../gu/mediaservice/lib/VectorUtilsTest.scala | 92 ++++++++++++++++--- 2 files changed, 121 insertions(+), 13 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala index 81e9ebd64e5..7449dae91bf 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -14,4 +14,46 @@ object VectorUtils { if (magnitudeProduct == 0.0) 0.0 else dotProduct(vectorOne, vectorTwo) / magnitudeProduct } + + // The below functions are primarily for constructing artificial vectors for tests + private def oneHotVector(dims: Int, hotIndex: Int): List[Double] = + List.tabulate(dims)(i => if (i == hotIndex) 1.0 else 0.0) + + // The first standard basis vector e0: (1, 0, 0, ...). + def firstBasisVector(dims: Int): List[Double] = oneHotVector(dims, 0) + + private def vectorWithComponents(dims: Int, components: (Int, Double)*): List[Double] = { + val arr = Array.fill(dims)(0.0) + components.foreach { case (i, v) => arr(i) = v } + arr.toList + } + + // Builds a vector whose cosine similarity with the first basis vector equals + // the requested `similarity` (in [-1, 1], where 1 is identical and 0 is orthogonal). + def vectorWithCosineSimilarity(dims: Int, similarity: Double): List[Double] = { + // Assert that similarity is between -1 and 1 + require(similarity >= -1.0 && similarity <= 1.0, "Cosine similarity must be between -1 and 1") + if (similarity == 0.0) { + // A cosine similarity of 0 means orthogonal to the first basis vector. + // That vector is one-hot at index 0, so any vector with a zero + // first component is orthogonal to it; (0, 1) is the simplest choice. + oneHotVector(dims, 1) + } else { + val absVec2D = { + // Let's pretend we're in just 2 dimensions and think about triangles. + // (Diagram would help here!) + // Our queryEmbedding lies flat on the x axis. + // If we want cos 0 to be 1/n, e.g. 1/2, + // then the adjacent side must be 1, and the hypotenuse n. + // We want to find the opposite side + // n^2 = 1^2 + adj^2 + // opposite = sqrt(n^2 - 1) + // The two components of our vector are therefore (1, sqrt(n^2 - 1)) + val n = 1 / Math.abs(similarity) + (1, Math.sqrt(Math.pow(n, 2) - 1)) + } + val vec2D = if (similarity < 0.0) (-absVec2D._1, -absVec2D._2) else absVec2D + vectorWithComponents(dims, 0 -> vec2D._1, 1 -> vec2D._2) + } + } } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala index 28b13fe968f..f3f76314770 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -1,55 +1,121 @@ package com.gu.mediaservice.lib +import org.scalatest.Inspectors import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers +import org.scalacheck.Gen +import VectorUtils.{dotProduct, magnitude, cosineSimilarity, firstBasisVector, vectorWithCosineSimilarity} -class VectorUtilsTest extends AnyFunSpec with Matchers { +class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors { it ("should compute the dot product of two vectors") { - VectorUtils.dotProduct(List(1.0, 2.0, 3.0), List(4.0, 5.0, 6.0)) shouldBe 32.0 + dotProduct(List(1.0, 2.0, 3.0), List(4.0, 5.0, 6.0)) shouldBe 32.0 } it ("should compute a dot product of zero for orthogonal vectors") { - VectorUtils.dotProduct(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 + dotProduct(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 } it ("should return zero for the dot product of empty vectors") { - VectorUtils.dotProduct(List.empty, List.empty) shouldBe 0.0 + dotProduct(List.empty, List.empty) shouldBe 0.0 } it ("should compute the magnitude of a vector") { - VectorUtils.magnitude(List(3.0, 4.0)) shouldBe 5.0 + magnitude(List(3.0, 4.0)) shouldBe 5.0 } it ("should return a magnitude of zero for a zero vector") { - VectorUtils.magnitude(List(0.0, 0.0, 0.0)) shouldBe 0.0 + magnitude(List(0.0, 0.0, 0.0)) shouldBe 0.0 } it ("should return a cosine similarity of 1 for identical vectors") { - VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)) shouldBe 1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)) shouldBe 1.0 +- 1e-9 } it ("should return a cosine similarity of 1 for parallel vectors") { - VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)) shouldBe 1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)) shouldBe 1.0 +- 1e-9 } it ("should return a cosine similarity of 0 for orthogonal vectors") { - VectorUtils.cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 +- 1e-9 + cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 +- 1e-9 } it ("should return a cosine similarity of -1 for opposite vectors") { - VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)) shouldBe -1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)) shouldBe -1.0 +- 1e-9 } it ("should return a cosine similarity of 0 when the first vector has zero magnitude") { - VectorUtils.cosineSimilarity(List(0.0, 0.0, 0.0), List(1.0, 2.0, 3.0)) shouldBe 0.0 +- 1e-9 + cosineSimilarity(List(0.0, 0.0, 0.0), List(1.0, 2.0, 3.0)) shouldBe 0.0 +- 1e-9 } it ("should return a cosine similarity of 0 when the second vector has zero magnitude") { - VectorUtils.cosineSimilarity(List(1.0, 2.0, 3.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 } it ("should return a cosine similarity of 0 when both vectors have zero magnitude") { - VectorUtils.cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + } + + describe("vectorWithCosineSimilarity") { + val dims = 256 + val tolerance = 1e-9 + + it ("should create a vector at the requested cosine similarity across a fine sweep of the valid range") { + val similarities = (-1000 to 1000).map(_ / 1000.0) + forAll(similarities) { s => + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, s)) shouldBe s +- tolerance + } + } + + it ("should create a vector at the requested cosine similarity for randomly generated similarities") { + val similarityGen = Gen.chooseNum(-1.0, 1.0) + val samples = Gen.listOfN(500, similarityGen).sample.getOrElse(Nil) + samples should not be empty + forAll(samples) { s => + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, s)) shouldBe s +- tolerance + } + } + + it ("should work across a range of dimensions") { + forAll(List(2, 3, 8, 16, 128, 256, 1024)) { n => + cosineSimilarity(firstBasisVector(n), vectorWithCosineSimilarity(n, 0.42)) shouldBe 0.42 +- tolerance + } + } + + it ("should return a vector identical to the basis vector for a similarity of 1") { + vectorWithCosineSimilarity(dims, 1.0) shouldBe firstBasisVector(dims) + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, 1.0)) shouldBe 1.0 +- tolerance + } + + it ("should return a vector opposite to the basis vector for a similarity of -1") { + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, -1.0)) shouldBe -1.0 +- tolerance + } + + it ("should return a vector orthogonal to the basis vector for a similarity of 0") { + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, 0.0)) shouldBe 0.0 +- tolerance + } + + it ("should negate the vector when the similarity is negated") { + val positive = vectorWithCosineSimilarity(dims, 0.25) + val negative = vectorWithCosineSimilarity(dims, -0.25) + forAll(positive.zip(negative)) { case (p, n) => n shouldBe (-p) +- tolerance } + } + + it ("should produce only finite components for valid similarities") { + forAll(List(-1.0, -0.5, -1e-3, 0.0, 1e-3, 0.5, 1.0)) { s => + forAll(vectorWithCosineSimilarity(dims, s)) { component => + component.isNaN shouldBe false + component.isInfinite shouldBe false + } + } + } + + it ("should reject a similarity greater than 1") { + an [IllegalArgumentException] should be thrownBy vectorWithCosineSimilarity(dims, 1.0001) + } + + it ("should reject a similarity less than -1") { + an [IllegalArgumentException] should be thrownBy vectorWithCosineSimilarity(dims, -1.0001) + } } } From e9e098aeef166ad2f7c880e7cf77c46e675ba8b3 Mon Sep 17 00:00:00 2001 From: Andrew Nowak Date: Mon, 22 Jun 2026 12:41:36 +0100 Subject: [PATCH 046/373] fix polling for undeletion --- .../js/components/gr-archiver/gr-archiver.js | 26 ++++++++++++--- .../gr-undelete-image/gr-un-delete-image.js | 21 ++++++------ kahuna/public/js/image/controller.js | 7 ++++ kahuna/public/js/search/results.js | 32 +++++++++++-------- 4 files changed, 57 insertions(+), 29 deletions(-) diff --git a/kahuna/public/js/components/gr-archiver/gr-archiver.js b/kahuna/public/js/components/gr-archiver/gr-archiver.js index d2a5b4d28c4..0d676369fb4 100644 --- a/kahuna/public/js/components/gr-archiver/gr-archiver.js +++ b/kahuna/public/js/components/gr-archiver/gr-archiver.js @@ -27,6 +27,7 @@ module.controller('grArchiverCtrl', [ 'humanJoin', 'mediaApi', 'getDeletedState', + 'apiPoll', function ($scope, $window, archiveService, @@ -34,7 +35,8 @@ module.controller('grArchiverCtrl', [ imageLogic, humanJoin, mediaApi, - getDeletedState) { + getDeletedState, + apiPoll) { const ctrl = this; @@ -115,14 +117,30 @@ module.controller('grArchiverCtrl', [ reduce((all, items) => all.union(items)). toArray(); } + + function pollUndeleted(imageId) { + const findImage = () => mediaApi.find(imageId).then( + (i) => i.data?.softDeletedMetadata === undefined && i.data.userMetadata?.data?.archived + ? Promise.resolve() + : Promise.reject() + ); + return apiPoll(findImage); + } function undelete() { ctrl.undeleting = true; const imageId = ctrl.image.data.id; mediaApi.undelete(imageId) - .then( - ctrl.canUndelete = ctrl.isDeleted = false - ).catch(() => { + .then(() => pollUndeleted(imageId)) + .then(() => { + ctrl.canUndelete = ctrl.isDeleted = false; + ctrl.image.softDeletedMetadata = undefined; + if (ctrl.image.userMetadata?.data !== undefined) { + ctrl.image.userMetadata.data.archived = true; + } + $scope.$emit('images-updated', [ctrl.image]); + }).catch((e) => { $window.alert('Failed to undelete image!, please try again.'); + console.error('Image undeletion failed', e); }).finally(() => { ctrl.undeleting = false; }); diff --git a/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js b/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js index 0e25d4be036..d9f326f0d4c 100644 --- a/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js +++ b/kahuna/public/js/components/gr-undelete-image/gr-un-delete-image.js @@ -7,28 +7,27 @@ export const undeleteImage = angular.module('gr.undeleteImage', [ ]); undeleteImage.controller('grUnDeleteImageCtrl', [ - '$rootScope', '$q', '$timeout', 'mediaApi', 'apiPoll', - function ($rootScope, $q, $timeout, mediaApi, apiPoll) { + '$rootScope', '$q', 'mediaApi', 'apiPoll', + function ($rootScope, $q, mediaApi, apiPoll) { var ctrl = this; ctrl.$onInit = () => { - function pollDeleted (image) { + function pollUndeleted (image) { const findImage = () => mediaApi.find(image.data.id).then( - () => $q.reject(), - // resolve when image cannot be found, i.e. image has been deleted. - () => $q.resolve() + (i) => i.data.softDeletedMetadata === undefined && i.data.userMetadata?.data?.archived + ? $q.resolve() + : $q.reject() ); - apiPoll(findImage); + return apiPoll(findImage); } - // TODO remove "unusable" banner and recognise that the image is now archived ctrl.unDeleteImage = function (image) { return mediaApi.undelete(image.data.id) - .then(() => pollDeleted(image)) + .then(() => pollUndeleted(image)) .catch((err) => { - $rootScope.$emit('image-delete-failure', err, image); + $rootScope.$emit('image-undelete-failure', err, image); }); }; @@ -36,7 +35,7 @@ undeleteImage.controller('grUnDeleteImageCtrl', [ // HACK to wait for thrall to process the message so that when we // poll the api, it will be up to date. return $q.all(Array.from(ctrl.images.values()).map(image => ctrl.unDeleteImage(image))) - .then(() => $rootScope.$emit('images-deleted', ctrl.images)); + .then(() => $rootScope.$emit('images-undeleted', ctrl.images)); }; }; } diff --git a/kahuna/public/js/image/controller.js b/kahuna/public/js/image/controller.js index c9a42b0b551..e2a6247a64d 100644 --- a/kahuna/public/js/image/controller.js +++ b/kahuna/public/js/image/controller.js @@ -156,6 +156,13 @@ image.controller('ImageCtrl', [ ctrl.image = image; if (ctrl.image && ctrl.image.data.softDeletedMetadata !== undefined) { ctrl.isDeleted = true; } + + $scope.$watch('ctrl.image.data.softDeletedMetadata', () => { + if (ctrl.image) { + ctrl.isDeleted = ctrl.image?.data.softDeletedMetadata !== undefined; + } + }); + ctrl.optimisedImageUri = optimisedImageUri; ctrl.lowResImageUri = lowResImageUri; diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index 6721bf53ba5..d0665452c9d 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -827,24 +827,27 @@ results.controller('SearchResultsCtrl', [ }); }; - const freeImageDeleteListener = $rootScope.$on('images-deleted', (e, images) => { - images.forEach(image => { - // TODO: should not be needed here, the selection and - // results should listen to these events and update - // itself outside of any controller - ctrl.deselect(image); + const imageDeleteHandler = (_, images) => { + images.forEach(image => { + // TODO: should not be needed here, the selection and + // results should listen to these events and update + // itself outside of any controller + ctrl.deselect(image); - const indexAll = ctrl.imagesAll.findIndex(i => image.data.id === i.data.id); - results.removeAt(indexAll); + const indexAll = ctrl.imagesAll.findIndex(i => image.data.id === i.data.id); + results.removeAt(indexAll); - updateImageArray(ctrl.images, image); - updateImageArray(ctrl.imagesAll, image); + updateImageArray(ctrl.images, image); + updateImageArray(ctrl.imagesAll, image); - updatePositions(image); + updatePositions(image); - ctrl.totalResults--; - }); - }); + ctrl.totalResults--; + }); + }; + + const freeImageDeleteListener = $rootScope.$on('images-deleted', imageDeleteHandler); + const freeImageUndeleteListener = $rootScope.$on('images-undeleted', imageDeleteHandler); // Safer than clearing the timeout in case of race conditions // FIXME: nicer (reactive?) way to do this? @@ -913,6 +916,7 @@ results.controller('SearchResultsCtrl', [ } freeUpdatesListener(); freeImageDeleteListener(); + freeImageUndeleteListener(); scopeGone = true; }); } From e24efb980279c7aea6afe5f1fb9a81a7b757fbf4 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 14:28:32 +0100 Subject: [PATCH 047/373] Rewrite tests myself --- .../lib/elasticsearch/ElasticSearchTest.scala | 288 ------------------ .../test/lib/elasticsearch/Fixtures.scala | 3 +- .../lib/elasticsearch/HybridSearchTest.scala | 166 ++++++++++ 3 files changed, 168 insertions(+), 289 deletions(-) create mode 100644 media-api/test/lib/elasticsearch/HybridSearchTest.scala diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 1e3f7692147..9f333f9cea1 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -448,294 +448,6 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast } } - describe("AI / hybrid search") { - implicit val logMarker: LogMarker = MarkerMap() - - // 256-dim vectors to match the `embedding.cohereEmbedV4.image` dense_vector - // mapping. One-hot vectors are orthogonal and unit-magnitude, so cosine - // similarity is 1.0 against themselves and 0.0 against each other - which - // makes the semantic ranking easy to reason about in assertions. - def oneHot(hotIndex: Int): List[Double] = - List.tabulate(256)(i => if (i == hotIndex) 1.0 else 0.0) - - def withEmbedding(image: Image, vector: List[Double]): Image = - image.copy(embedding = Some(Embedding(cohereEmbedV4 = Some(CohereV4Embedding(image = vector))))) - - // The vector we'll "search" with - represents the user's query embedding. - val queryEmbedding: List[Double] = oneHot(0) - - // Matches the text "kitten" but has NO embedding at all: a pure *lexical* - // hit. Deliberately un-embedded so it is invisible to the kNN (semantic) - // side of the search - only the BM25 (lexical) side can ever surface it. - val lexicalMatch = - createImage("ai-lexical-match", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("a kitten playing in the garden"), keywords = Some(Set("kitten"))) - ) - - // Does NOT match the text at all, but IS the nearest neighbour of the query - // vector: a pure *semantic* hit. The only image in the fixture set with an - // embedding, so it is the sole image the kNN side can ever return. - val semanticMatch = withEmbedding( - createImage("ai-semantic-match", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("completely unrelated wording"), keywords = Some(Set("unrelated"))) - ), - oneHot(0) - ) - - // Noise: neither a text match nor an embedding, so neither side of the - // search can ever surface it. - val unrelated = - createImage("ai-unrelated", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("nothing to see here"), keywords = Some(Set("noise"))) - ) - - val aiImages = Seq(lexicalMatch, semanticMatch, unrelated) - - // Seed exactly once for the whole describe block: the three tests share the - // same fixtures, and re-indexing the same ids per-test causes version - // conflicts (409s) when the suite's match-all purge runs in afterAll. - lazy val aiImagesIndexed: Unit = { - Await.ready(saveImages(aiImages), 1.minute) - eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds)) { - totalImages shouldBe (expectedNumberOfImages + aiImages.size) - } - } - - // k = 2 (not, say, 10) is deliberate: with a small k the search is forced - // to discriminate. If we asked for more results than we indexed, kNN would - // return everything and the assertions would pass trivially. - def hybridSearchIds(vecWeight: Double, k: Int = 2): Seq[String] = - Await.result( - ES.hybridSearch( - query = "kitten", - queryEmbedding = queryEmbedding, - k = k, - numCandidates = 10, - vecWeight = vecWeight, - filterOpt = None - ), - fiveSeconds - ).hits.map(_._1) - - it("combines lexical (BM25) and semantic (kNN) search, retrieving images by meaning even when the text doesn't match, and fusing the two rankings") { - aiImagesIndexed - - val ids = hybridSearchIds(vecWeight = 0.5) - - // Semantic recall: the nearest-vector image is returned even though it - // shares no words with the query - only the kNN path could have found it. - ids should contain("ai-semantic-match") - - // Lexical recall still works alongside semantic recall. - ids should contain("ai-lexical-match") - - // The meaningful assertion: the noise image is neither a text match nor a - // near vector, so fusion + top-k ranking drops it. This is what proves the - // search actually discriminates rather than just returning everything. - ids should not contain "ai-unrelated" - } - - it("ranks the semantic match above the lexical match when vecWeight is cranked towards vectors") { - aiImagesIndexed - - // vecWeight ~1 => the fused score is dominated by cosine similarity, so the - // pure-vector match should outrank the pure-text match. - val ids = hybridSearchIds(vecWeight = 0.99) - - ids should contain("ai-semantic-match") - ids should contain("ai-lexical-match") - ids.indexOf("ai-semantic-match") should be < ids.indexOf("ai-lexical-match") - } - - it("ranks the lexical match above the semantic match when vecWeight is cranked towards text") { - aiImagesIndexed - - // vecWeight ~0 => the fused score is dominated by BM25, so the pure-text - // match should outrank the pure-vector match. The mirror image of the above. - val ids = hybridSearchIds(vecWeight = 0.01) - - ids should contain("ai-semantic-match") - ids should contain("ai-lexical-match") - ids.indexOf("ai-lexical-match") should be < ids.indexOf("ai-semantic-match") - } - - it("short-circuits to a pure lexical (BM25) search when vecWeight is exactly 0, skipping the semantic side entirely") { - aiImagesIndexed - - // vecWeight == 0 => the semantic side contributes nothing to the fused - // score, so the kNN query should be skipped altogether. The only result - // should be the text match; the semantic-only image must not leak in via - // the kNN path. - val ids = hybridSearchIds(vecWeight = 0) - - ids shouldEqual Seq("ai-lexical-match") - } - - it("short-circuits to a pure semantic (kNN) search when vecWeight is exactly 1, skipping the lexical side entirely") { - aiImagesIndexed - - // vecWeight == 1 => the lexical side contributes nothing to the fused - // score, so the BM25 query should be skipped altogether. The only result - // should be the vector match; the lexical-only image must not leak in via - // the BM25 path. - val ids = hybridSearchIds(vecWeight = 1) - - ids shouldEqual Seq("ai-semantic-match") - } - - // ---------------------------------------------------------------------- - // Score "cross-pollination": every candidate is scored on BOTH dimensions, - // regardless of which side of the search originally surfaced it. - // - // A hit found only by the lexical side still has its true cosine similarity - // filled in client-side; a hit found only by the semantic side still carries - // the BM25 score it earns from the lexical rescore. The two tests below pin - // down each of those behaviours in isolation. - // - // They use their own query word + vector axis (so the fixtures above can't - // interfere) and a few small unit vectors with hand-picked cosine values. - // ---------------------------------------------------------------------- - - // Builds a 256-dim unit vector from (index -> value) components. Used to make - // vectors with specific cosine similarities to a one-hot query axis, e.g. - // unitVector(7 -> 0.8, 8 -> 0.6) has cosine 0.8 with oneHot(7). - def unitVector(components: (Int, Double)*): List[Double] = { - val arr = Array.fill(256)(0.0) - components.foreach { case (i, v) => arr(i) = v } - arr.toList - } - - def hybridSearchIdsFor(query: String, queryEmbedding: List[Double], vecWeight: Double, k: Int): Seq[String] = - Await.result( - ES.hybridSearch( - query = query, - queryEmbedding = queryEmbedding, - k = k, - numCandidates = 10, - vecWeight = vecWeight, - filterOpt = None - ), - fiveSeconds - ).hits.map(_._1) - - // Test 1 fixtures - query axis oneHot(7), query word "axolotl". - // - // A lexical hit that the kNN side never returns. Its embedding sits at cosine - // 0.5 from the query, but 'ai-semantic-hit-near-vector' (cosine 0.8) is nearer, - // so with k = 1 the semantic side only ever returns the latter - leaving this - // image strictly outside the top-k semantic results. - val lexicalHitWithVector = withEmbedding( - createImage("ai-lexical-hit-with-vector", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("a lone axolotl"), keywords = Some(Set("axolotl"))) - ), - unitVector(7 -> 0.5, 8 -> math.sqrt(0.75)) - ) - - // The nearer vector, with no text match: the only image the kNN side returns - // for the query above. - val semanticHitNearVector = withEmbedding( - createImage("ai-semantic-hit-near-vector", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("the blurst of times")) - ), - unitVector(7 -> 0.8, 8 -> 0.6) - ) - - // Test 2 fixtures - query axis oneHot(9), query phrase "narwhal arctic ocean". - // - // A semantic hit that the lexical side never returns. It is one of the two - // nearest vectors AND a (weak) text match for "narwhal", but the two strong - // lexical matches below outscore it on BM25, so with k = 2 it falls strictly - // outside the top-k lexical results. - val semanticHitWithText = withEmbedding( - createImage("ai-semantic-hit-with-text", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("a narwhal")) - ), - unitVector(9 -> 0.9, 10 -> math.sqrt(0.19)) - ) - - // Equally close to the query vector as 'ai-semantic-hit-with-text' (cosine - // 0.9), but with no text match at all - so its lexical score is zero. The - // identical semantic score is what lets us attribute any ranking difference - // purely to the lexical contribution. - val semanticHitNoText = withEmbedding( - createImage("ai-semantic-hit-no-text", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("the blurst of times")) - ), - unitVector(9 -> 0.9, 11 -> math.sqrt(0.19)) - ) - - // Two strong, un-embedded lexical matches that fill the top-k lexical results - // and push 'ai-semantic-hit-with-text' out of them. The first matches all - // three query terms, the second matches two. - val lexicalStrongOne = - createImage("ai-lexical-strong-1", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("narwhal arctic ocean")) - ) - val lexicalStrongTwo = - createImage("ai-lexical-strong-2", staffPhotographer).copy( - metadata = ImageMetadata(title = Some("narwhal arctic")) - ) - - val contributionImages = Seq( - lexicalHitWithVector, - semanticHitNearVector, - semanticHitWithText, - semanticHitNoText, - lexicalStrongOne, - lexicalStrongTwo - ) - - // Seed once, on top of the aiImages above, so the total is deterministic no - // matter which tests run. - lazy val contributionImagesIndexed: Unit = { - aiImagesIndexed - Await.ready(saveImages(contributionImages), 1.minute) - eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds)) { - totalImages shouldBe (expectedNumberOfImages + aiImages.size + contributionImages.size) - } - } - - it("scores a lexical-only hit by its embedding, even though the kNN side never returned it") { - contributionImagesIndexed - - val query = "axolotl" - val queryVector = oneHot(7) - - // A pure semantic search returns only the nearer vector - proving the - // lexical hit is genuinely outside the top-k semantic results. - hybridSearchIdsFor(query, queryVector, vecWeight = 1.0, k = 1) shouldEqual Seq("ai-semantic-hit-near-vector") - - // At a strongly semantic-leaning weight the lexical-only hit nonetheless - // wins. It can only do so because its true cosine similarity (0.5) was - // filled in client-side: had its semantic score been treated as absent, the - // nearer vector (0.8) would have taken the single slot at this weight. - hybridSearchIdsFor(query, queryVector, vecWeight = 0.8, k = 1) shouldEqual Seq("ai-lexical-hit-with-vector") - } - - it("scores a semantic-only hit by BM25, even though the lexical side never returned it") { - contributionImagesIndexed - - val query = "narwhal arctic ocean" - val queryVector = oneHot(9) - - // A pure lexical search returns the two strong text matches and not - // 'ai-semantic-hit-with-text' - proving it is genuinely outside the top-k - // lexical results. - val pureLexical = hybridSearchIdsFor(query, queryVector, vecWeight = 0.0, k = 2) - pureLexical should contain("ai-lexical-strong-1") - pureLexical should contain("ai-lexical-strong-2") - pureLexical should not contain "ai-semantic-hit-with-text" - - // The two semantic hits are equidistant from the query vector, so their - // semantic scores are identical and both rise above the lexical matches at - // this weight. The ONLY thing that can separate them is the weak BM25 score - // the text-bearing one picked up via the semantic-side rescore - so it must - // rank first. - hybridSearchIdsFor(query, queryVector, vecWeight = 0.6, k = 2) shouldEqual - Seq("ai-semantic-hit-with-text", "ai-semantic-hit-no-text") - } - } - private def saveImages(images: Seq[Image]) = { implicit val logMarker: LogMarker = MarkerMap() diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala index 431329aeb32..8caca4053d4 100644 --- a/media-api/test/lib/elasticsearch/Fixtures.scala +++ b/media-api/test/lib/elasticsearch/Fixtures.scala @@ -56,6 +56,7 @@ trait Fixtures { usages: List[Usage] = Nil, fileMetadata: Option[FileMetadata] = None, softDeletedMetadata: Option[SoftDeletedMetadata] = None, + vector: Option[List[Double]] = None, ): Image = { Image( id = id, @@ -88,7 +89,7 @@ trait Fixtures { syndicationRights = syndicationRights, leases = leases.getOrElse(LeasesByMedia.build(Nil)), usages = usages, - embedding = None, + embedding = vector.map(v => Embedding(cohereEmbedV4 = Some(CohereV4Embedding(image = v)))), ) } diff --git a/media-api/test/lib/elasticsearch/HybridSearchTest.scala b/media-api/test/lib/elasticsearch/HybridSearchTest.scala new file mode 100644 index 00000000000..076a70e2277 --- /dev/null +++ b/media-api/test/lib/elasticsearch/HybridSearchTest.scala @@ -0,0 +1,166 @@ +package lib.elasticsearch + +import org.apache.pekko.actor.{ActorSystem, Scheduler} +import com.gu.mediaservice.lib.VectorUtils.{firstBasisVector, vectorWithCosineSimilarity} +import com.gu.mediaservice.lib.config.GridConfigResources +import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearchConfig, ElasticSearchExecutions} +import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} +import com.gu.mediaservice.model._ +import com.gu.mediaservice.testlib.ElasticSearchDockerBase +import com.sksamuel.elastic4s.ElasticDsl +import com.sksamuel.elastic4s.ElasticDsl._ +import lib.{MediaApiConfig, MediaApiMetrics} +import org.scalatest.concurrent.{Eventually, ScalaFutures} +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.mockito.MockitoSugar +import play.api.Configuration +import play.api.inject.ApplicationLifecycle +import play.api.libs.json.Json + +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} + +class HybridSearchTest extends AnyFunSpec + with ElasticSearchDockerBase + with Matchers + with ScalaFutures + with Eventually + with ElasticSearchExecutions + with Fixtures + with MockitoSugar { + + private val index = "images" + + private val applicationLifecycle = new ApplicationLifecycle { + override def addStopHook(hook: () => Future[_]): Unit = {} + override def stop(): Future[_] = Future.successful(()) + } + + private val mediaApiConfig = new MediaApiConfig(GridConfigResources( + Configuration.from(USED_CONFIGS_IN_TEST ++ MOCK_CONFIG_KEYS.map(_ -> NOT_USED_IN_TEST).toMap), + null, + applicationLifecycle + )) + private val actorSystem: ActorSystem = ActorSystem() + private val mediaApiMetrics = new MediaApiMetrics(mediaApiConfig, actorSystem, applicationLifecycle) + private val elasticConfig = ElasticSearchConfig( + aliases = ElasticSearchAliases( + current = "Images_Current", + migration = "Images_Migration" + ), + url = esTestUrl, + shards = 1, + replicas = 0 + ) + + private lazy val ES = new ElasticSearch(mediaApiConfig, mediaApiMetrics, elasticConfig, () => List.empty, mock[Scheduler]) + lazy val client = ES.client + + private val oneHundredMilliseconds = Duration(100, MILLISECONDS) + private val fiveSeconds = Duration(5, SECONDS) + + override def beforeAll(): Unit = { + super.beforeAll() + ES.ensureIndexExistsAndAliasAssigned() + } + + describe("AI / hybrid search") { + implicit val logMarker: LogMarker = MarkerMap() + + // The vector we'll "search" with - represents the user's query embedding. + val queryEmbedding: List[Double] = firstBasisVector(256) + + // 256-dim vectors to match the `embedding.cohereEmbedV4.image` dense_vector + // mapping, each constructed to have a known cosine similarity to the query. + val vectorWithSemanticScore: Map[Double, List[Double]] = Map( + 1.0 -> vectorWithCosineSimilarity(256, 1.0), + 0.9 -> vectorWithCosineSimilarity(256, 0.9), + 0.8 -> vectorWithCosineSimilarity(256, 0.8), + 0.7 -> vectorWithCosineSimilarity(256, 0.7), + 0.5 -> vectorWithCosineSimilarity(256, 0.5), + 0.0 -> vectorWithCosineSimilarity(256, 0.0), + -0.5 -> vectorWithCosineSimilarity(256, -0.5), + -0.9 -> vectorWithCosineSimilarity(256, -0.9), + -1.0 -> vectorWithCosineSimilarity(256, -1.0) + ) + + def aiImage(id: String, title: String, vector: List[Double]): Image = { + val base = createImage(id = id, usageRights = Handout(), vector = Some(vector)) + base.copy(metadata = base.metadata.copy(title = Some(title))) + } + + // Replace the images persisted in ES with exactly `imgs`, so each test + // reasons about a small, fully controlled set of documents. + def withOnly(imgs: Seq[Image])(test: => Unit): Unit = { + purgeTestImages + Await.ready(saveImages(imgs), 1.minute) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe imgs.size) + test + } + + it("joe") { + val a = aiImage("a", title = "zero lexical", vector = vectorWithSemanticScore(1.0)) + val b = aiImage("b", title = "zero lexical", vector = vectorWithSemanticScore(0.9)) + val c = aiImage("c", title = "good lexical", vector = vectorWithSemanticScore(0.8)) + val d = aiImage("d", title = "good good lexical", vector = vectorWithSemanticScore(0.7)) + + withOnly(Seq(a, b, c, d)) { + val semanticOnlyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 1.0, filterOpt = None), + fiveSeconds + ) + semanticOnlyResults.hits.map(_._1) shouldBe List("a", "b", "c", "d") + + val lexicalOnlyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 0.0, filterOpt = None), + fiveSeconds + ) + // Because we asked for k = 4 but only got 2 back, this confirms + // the short-circuiting behaviour at vecWeight 0.0, i.e. only the lexical query ran. + lexicalOnlyResults.hits.map(_._1) shouldBe List("d", "c") + + val weightedHeavilyLexicallyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 0.1, filterOpt = None), + fiveSeconds + ) + // We should now get all 4 because vecWeight > 0 + weightedHeavilyLexicallyResults.hits.map(_._1) shouldBe List("d", "c", "a", "b") + + // We've now proven that the lexical top 2 (d, c) are disjunct from the semantic top 2 (a, b) + // Therefore, without score filling, the top 2 hybrid results after theoretical min-max norming + // would have to be the top semantic and the top lexical, i.e. a and d, which both get a score of 1. + // But thanks to score filling, c can make it into the top 2. + val equalWeightingResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 2, numCandidates = 10, vecWeight = 0.5, filterOpt = None), + fiveSeconds + ) + equalWeightingResults.hits.map(_._1) shouldBe List("d", "c") + equalWeightingResults.hits.map(_._2.instance) // is there a way to get the score out here? + } + } + } + + private def saveImages(images: Seq[Image]) = { + implicit val logMarker: LogMarker = MarkerMap() + + Future.sequence(images.map { i => + executeAndLog(indexInto(index) id i.id source Json.stringify(Json.toJson(i)), s"Indexing test image") + }) + } + + private def totalImages: Long = Await.result(ES.client.execute(ElasticDsl.search(ES.imagesCurrentAlias)).map { + _.result.totalHits + }, oneHundredMilliseconds) + + private def purgeTestImages = { + implicit val logMarker: LogMarker = MarkerMap() + + def deleteImages = executeAndLog(deleteByQuery(index, matchAllQuery()), s"Deleting images") + + Await.result(deleteImages, fiveSeconds) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe 0) + } + +} From e22351726b2c306ebebaae02114c515e43a2411c Mon Sep 17 00:00:00 2001 From: Andrew Nowak Date: Mon, 22 Jun 2026 15:25:33 +0100 Subject: [PATCH 048/373] work around the csrf filter for app-to-app requests by passing along the Origin header from the original request As we pass along the authentication cookie used in the original request, we also need to pass along the original Origin header so that the subsequent request belongs takes the same cross-origin status and passes through the CSRF filter --- .../com/gu/mediaservice/GridClient.scala | 6 ++-- image-loader/app/ImageLoaderComponents.scala | 2 +- media-api/app/controllers/MediaApi.scala | 2 +- .../app/controllers/EditsController.scala | 2 +- .../auth/PandaAuthenticationProvider.scala | 30 ++++++++++++------- thrall/app/ThrallComponents.scala | 2 +- 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala index 73bc0d2a378..bf017e31473 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/GridClient.scala @@ -38,8 +38,8 @@ object ClientResponse { case class ClientErrorMessages(errorMessage: String, downstreamErrorMessage: String) object GridClient extends LazyLogging { - def apply(services: Services)(implicit wsClient: WSClient): GridClient = - new GridClient(services) + def apply(services: Services, originUri: String)(implicit wsClient: WSClient): GridClient = + new GridClient(services, originUri) sealed trait Response { def status: Int @@ -96,7 +96,7 @@ object GridClient extends LazyLogging { } -class GridClient(services: Services)(implicit wsClient: WSClient) extends LazyLogging { +class GridClient(services: Services, originDomain: String)(implicit wsClient: WSClient) extends LazyLogging { /* * `requestTimeout` will set the max duration of the request before timing out. You may also want to increase the diff --git a/image-loader/app/ImageLoaderComponents.scala b/image-loader/app/ImageLoaderComponents.scala index db2b9a059e3..be7748d9525 100644 --- a/image-loader/app/ImageLoaderComponents.scala +++ b/image-loader/app/ImageLoaderComponents.scala @@ -21,7 +21,7 @@ class ImageLoaderComponents(context: Context) extends GridComponents(context, ne logger.info(s" $index -> ${processor.description}") } - private val gridClient = GridClient(config.services)(wsClient) + private val gridClient = GridClient(config.services, config.services.loaderBaseUri)(wsClient) val store = new ImageLoaderStore(config) val maybeIngestQueue = config.maybeIngestSqsQueueUrl.map(queueUrl => new SimpleSqsMessageConsumer(queueUrl, config)) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 7c53dcbcad3..af13798fa07 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -50,7 +50,7 @@ class MediaApi( )(implicit val ec: ExecutionContext) extends BaseController with MessageSubjects with ArgoHelpers with ContentDisposition { val services: Services = new Services(config.domainRoot, config.serviceHosts, Set.empty) - val gridClient: GridClient = GridClient(services)(ws) + val gridClient: GridClient = GridClient(services, services.apiBaseUri)(ws) // Process-local cache keyed on normalised query text. Stores the Bedrock Future so that // concurrent requests for the same query share a single in-flight Bedrock call, and diff --git a/metadata-editor/app/controllers/EditsController.scala b/metadata-editor/app/controllers/EditsController.scala index 44be486d4c1..b8585656f52 100644 --- a/metadata-editor/app/controllers/EditsController.scala +++ b/metadata-editor/app/controllers/EditsController.scala @@ -59,7 +59,7 @@ class EditsController( import com.gu.mediaservice.lib.metadata.UsageRightsMetadataMapper.usageRightsToMetadata val services: Services = new Services(config.domainRoot, config.serviceHosts, Set.empty) - val gridClient: GridClient = GridClient(services)(ws) + val gridClient: GridClient = GridClient(services, services.metadataBaseUri)(ws) val metadataBaseUri = config.services.metadataBaseUri private val AuthenticatedAndAuthorised = auth andThen authorisation.CommonActionFilters.authorisedForArchive diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala index 00f45b4690a..22d97cd35e3 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PandaAuthenticationProvider.scala @@ -112,6 +112,22 @@ class PandaAuthenticationProvider( override def flushToken: Option[(RequestHeader, Result) => Result] = Some((rh, _) => processLogout(rh)) val PandaCookieKey: TypedKey[Cookie] = TypedKey[Cookie]("PandaCookie") + val OriginKey: TypedKey[String] = TypedKey[String]("Origin") + + private def withPersistedOrigin(request: Principal): WSRequest => WSRequest = { wsRequest => + request.attributes.get(OriginKey) match { + case Some(origin) => wsRequest.withHttpHeaders(("Origin", origin)) + case None => wsRequest + } + } + + private def withPersistedCookie(request: Principal): Either[String, WSRequest => WSRequest] = { + val cookieName = panDomainSettings.settings.cookieSettings.cookieName + request.attributes.get(PandaCookieKey) match { + case Some(cookie) => Right(wsRequest => wsRequest.addCookies(DefaultWSCookie(cookieName, cookie.value))) + case None => Left(s"Pan domain cookie $cookieName is missing in principal.") + } + } /** * A function that allows downstream API calls to be made using the credentials of the inflight request @@ -119,19 +135,13 @@ class PandaAuthenticationProvider( * @param request The request header of the inflight call * @return A function that adds appropriate data to a WSRequest */ - override def onBehalfOf(request: Principal): Either[String, WSRequest => WSRequest] = { - val cookieName = panDomainSettings.settings.cookieSettings.cookieName - request.attributes.get(PandaCookieKey) match { - case Some(cookie) => Right { wsRequest: WSRequest => - wsRequest.addCookies(DefaultWSCookie(cookieName, cookie.value)) - } - case None => Left(s"Pan domain cookie $cookieName is missing in principal.") - } - } + override def onBehalfOf(request: Principal): Either[String, WSRequest => WSRequest] = + withPersistedCookie(request).map(_.andThen(withPersistedOrigin(request))) private def gridUserFrom(pandaUser: User, request: RequestHeader): UserPrincipal = { val maybePandaCookie: Option[TypedEntry[Cookie]] = request.cookies.get(panDomainSettings.settings.cookieSettings.cookieName).map(TypedEntry[Cookie](PandaCookieKey, _)) - val attributes = TypedMap.empty + (maybePandaCookie.toSeq:_*) + val maybeOrigin = request.headers.get("Origin").map(TypedEntry[String](OriginKey, _)) + val attributes = TypedMap(Seq(maybePandaCookie, maybeOrigin).flatten:_*) UserPrincipal( firstName = pandaUser.firstName, lastName = pandaUser.lastName, diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala index 781ab62bace..cf9c5e75add 100644 --- a/thrall/app/ThrallComponents.scala +++ b/thrall/app/ThrallComponents.scala @@ -31,7 +31,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr es.ensureIndexExistsAndAliasAssigned() val services: Services = new Services(config.domainRoot, config.serviceHosts, Set.empty) - val gridClient: GridClient = GridClient(services)(wsClient) + val gridClient: GridClient = GridClient(services, services.thrallBaseUri)(wsClient) // before firing up anything to consume streams or say we are OK let's do the critical good to go check private val goodToGoCheckResult = Await.ready(GoodToGoCheck.run(es), 30 seconds) From 20ddaf3f6d451346e850b94a4896edd89f3f412e Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 16:40:12 +0100 Subject: [PATCH 049/373] Refine tests and logging --- .../app/lib/elasticsearch/ElasticSearch.scala | 35 ++--- .../app/lib/elasticsearch/HybridResult.scala | 90 +++++++++-- .../lib/elasticsearch/HybridResultTest.scala | 144 ++++++++++++------ 3 files changed, 184 insertions(+), 85 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 5ead6e54eac..c02a8997f47 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -23,6 +23,7 @@ import com.sksamuel.elastic4s.requests.searches.queries.Query import com.sksamuel.elastic4s.requests.searches.knn.Knn import com.sksamuel.elastic4s.requests.searches.queries.matches.MultiMatchQueryBuilderType.BEST_FIELDS import com.sksamuel.elastic4s.requests.searches.queries.matches.{FieldWithOptionalBoost, MultiMatchQuery} +import lib.elasticsearch.ResultSource.{Both, Lexical, Semantic} import lib.querysyntax.{HierarchyField, Match, Parser, Phrase} import lib.{MediaApiConfig, MediaApiMetrics, SupplierUsageSummary} import play.api.libs.json.{JsError, JsObject, JsSuccess, Json} @@ -261,7 +262,7 @@ class ElasticSearch( vecWeight: Double, filterOpt: Option[Query] )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { - import HybridResult.{resolveHitAndFillInSemanticScore, fuseScoresAndGetTopK} + import HybridResult.{resolveHitAndFillInSemanticScore, fuseAndRank} val lexicalQuery = createMultiMatchQuery(query, operator = Or) val lexicalSearchRequest = lexicalRequest(lexicalQuery, k, filterOpt) @@ -288,26 +289,18 @@ class ElasticSearch( } yield { val lexicalHits = lexical.result.hits.hits.toList val semanticHits = semantic.result.hits.hits.toList - val allHits = lexicalHits ::: semanticHits - logger.info(logMarker, s"${allHits.length} total hits: ${lexicalHits.length} lexical, ${semanticHits.length} semantic") - - // This is only valid because the queries ensure that all hits, including semantic, - // have only the lexical score at this point. - val distinctHits = allHits.distinctBy(_.id) - logger.info(logMarker, s"${distinctHits.length} distinct hits") - - val lexicalIds = lexicalHits.map(_.id).toSet - val semanticIds = semanticHits.map(_.id).toSet - - val resultsWithSemanticScoresFilledIn = distinctHits.flatMap { hit => - resolveHitAndFillInSemanticScore(hit, queryEmbedding, resolveHit) - } - val topK = fuseScoresAndGetTopK(resultsWithSemanticScoresFilledIn, vecWeight, k) - topK.foreach { r => - logger.info(logMarker, s"hybrid result ${r.id}: lexicalScore=${r.lexicalScore} semanticScore=${r.semanticScore} " + - s"originallyFromLexical=${lexicalIds.contains(r.id)} originallyFromSemantic=${semanticIds.contains(r.id)}") - } - SearchResults(hits = topK.map(r => (r.id, r.image)), total = topK.length, extraCounts = None) + logger.info(logMarker, s"${lexicalHits.length + semanticHits.length} total hits: ${lexicalHits.length} lexical, ${semanticHits.length} semantic") + + // Resolve each side to images and fill in the client-side semantic score. + // We keep the two sides separate so that fuseAndRank can tag each result + // with where it originally came from (lexical, semantic, or both). + val lexicalResults = lexicalHits.flatMap(resolveHitAndFillInSemanticScore(_, queryEmbedding, resolveHit)) + val semanticResults = semanticHits.flatMap(resolveHitAndFillInSemanticScore(_, queryEmbedding, resolveHit)) + + val ranked = fuseAndRank(lexicalResults, semanticResults, vecWeight, k) + val from = ranked.groupBy(_.source) + logger.info(logMarker, s"${ranked.length} hits (k=$k) after fusing and ranking, ${from(Lexical).length} from lexical, ${from(Semantic)} from semantic, ${from(Both).length} from both") + SearchResults(hits = ranked.map(r => (r.result.id, r.result.image)), total = ranked.length, extraCounts = None) } } diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index 137bbd75a89..63ed7f9d52a 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -11,6 +11,42 @@ case class HybridResult( image: SourceWrapper[Image] ) +// Whether a result was originally returned by the lexical query, the semantic +// query, or both. A shared result tends to rank highly, so this is useful both +// for debugging and for understanding why something appears where it does. +sealed trait ResultSource +object ResultSource { + case object Lexical extends ResultSource + case object Semantic extends ResultSource + case object Both extends ResultSource + + def from(inLexical: Boolean, inSemantic: Boolean): ResultSource = + (inLexical, inSemantic) match { + case (true, true) => Both + case (true, false) => Lexical + // A ranked result always comes from at least one side, so the remaining + // case is "semantic only". + case (false, _) => Semantic + } +} + +// The intermediate scoring detail for a single result: the two normalised +// component scores and the weighted blend of them that we actually rank by. +case class FusedScore( + normedLexicalScore: Double, + normedSemanticScore: Double, + fusedScore: Double +) + +// A fully scored, ranked result. Carries the original scores (via `result`), +// where it came from (`source`), the normalised scores and the fused score +// (both via `score`) so the whole ranking decision is inspectable. +case class RankedResult( + result: HybridResult, + source: ResultSource, + score: FusedScore +) + object HybridResult { // Theoretical minimum cosine similarity (opposite vectors). val CosineSimilarityTheoreticalMin: Double = -1.0 @@ -48,32 +84,56 @@ object HybridResult { maxLexicalScore: Double, maxSemanticScore: Double, vecWeight: Double - ): Double = { + ): FusedScore = { val normedLexicalScore = normalise(result.lexicalScore, maxLexicalScore, theoreticalMin = Bm25TheoreticalMin) // The semantic theoretical min of -1 means we do both the max-norming // and the ES-score norming in one step. val normedSemanticScore = normalise(result.semanticScore, maxSemanticScore, theoreticalMin = CosineSimilarityTheoreticalMin) - (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) + val fusedScore = (vecWeight * normedSemanticScore) + ((1 - vecWeight) * normedLexicalScore) + FusedScore(normedLexicalScore, normedSemanticScore, fusedScore) } - def fuseScoresAndGetTopK( - results: List[HybridResult], + // Combines the lexical and semantic result sets into a single, ranked list. + // + // Each input result already carries both scores (the semantic query is + // rescored to BM25 server-side, and resolveHitAndFillInSemanticScore fills in + // the cosine similarity client-side), so this function only has to: tag each + // result with where it came from, de-duplicate, normalise + fuse the scores, + // then sort and take the top k. + def fuseAndRank( + lexicalResults: List[HybridResult], + semanticResults: List[HybridResult], vecWeight: Double, k: Int - ): List[HybridResult] = { - // Why do this rather than looking at the top result from elasticsearch or hits.maxScore? - // Because for the semantic query, we rescore by BM25, meaning neither maxScore nor the - // top result will actually tell us the max semantic score. + ): List[RankedResult] = { + val lexicalIds = lexicalResults.map(_.id).toSet + val semanticIds = semanticResults.map(_.id).toSet + + // De-duplicating by id is safe because, at this point, every result (lexical + // or semantic) carries only the lexical (BM25) score from Elasticsearch, so + // the two copies of a shared result are identical. + val distinctResults = (lexicalResults ::: semanticResults).distinctBy(_.id) + + // Why compute the maxes ourselves rather than reading them from Elasticsearch's + // top result or hits.maxScore? Because for the semantic query we rescore by BM25, + // so neither maxScore nor the top result reflects the max semantic score. // - // A side benefit is that this approach also accounts for the rare case in which KNN - // doesn't even contain true closest vector, and that true closest vector happens to - // be among the lexical-only results. + // A side benefit is that this also accounts for the rare case in which KNN + // doesn't contain the true closest vector, and that true closest vector + // happens to be among the lexical-only results. (for { - maxLexicalScore <- results.map(_.lexicalScore).maxOption - maxSemanticScore <- results.map(_.semanticScore).maxOption + maxLexicalScore <- distinctResults.map(_.lexicalScore).maxOption + maxSemanticScore <- distinctResults.map(_.semanticScore).maxOption } yield { - results - .sortBy(fuseScores(_, maxLexicalScore, maxSemanticScore, vecWeight))(Ordering[Double].reverse) + distinctResults + .map { result => + RankedResult( + result = result, + source = ResultSource.from(lexicalIds.contains(result.id), semanticIds.contains(result.id)), + score = fuseScores(result, maxLexicalScore, maxSemanticScore, vecWeight) + ) + } + .sortBy(_.score.fusedScore)(Ordering[Double].reverse) .take(k) }).getOrElse(List()) } diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index 4e945911879..cca8339059d 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -10,7 +10,7 @@ import play.api.libs.json.Json class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with Tolerance with Fixtures { - import HybridResult.{fuseScoresAndGetTopK, fuseScores, normalise, resolveHitAndFillInSemanticScore, Bm25TheoreticalMin, CosineSimilarityTheoreticalMin} + import HybridResult.{fuseAndRank, fuseScores, normalise, resolveHitAndFillInSemanticScore, Bm25TheoreticalMin, CosineSimilarityTheoreticalMin} private val tolerance = 1e-9 @@ -149,17 +149,29 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T } } - describe("combinedScore") { + describe("fuseScores") { + it("exposes the normalised lexical and semantic component scores") { + // normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 + val score = fuseScores( + hybridResult("components", lexicalScore = 2.0, semanticScore = 0.0), + maxLexicalScore = 4.0, + maxSemanticScore = 1.0, + vecWeight = 0.25 + ) + score.normedLexicalScore should be(0.5 +- tolerance) + score.normedSemanticScore should be(0.5 +- tolerance) + } + it("blends the normalised lexical and semantic scores using vecWeight") { // normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 - // combined = 0.25 * 0.5 + 0.75 * 0.5 = 0.5 + // fused = 0.25 * 0.5 + 0.75 * 0.5 = 0.5 val score = fuseScores( hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0), maxLexicalScore = 4.0, maxSemanticScore = 1.0, vecWeight = 0.25 ) - score should be(0.5 +- tolerance) + score.fusedScore should be(0.5 +- tolerance) } it("uses only the lexical score when vecWeight is 0") { @@ -169,7 +181,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T maxSemanticScore = 1.0, vecWeight = 0.0 ) - score should be(0.5 +- tolerance) + score.fusedScore should be(0.5 +- tolerance) } it("uses only the semantic score when vecWeight is 1") { @@ -180,10 +192,10 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T vecWeight = 1.0 ) // normedSemantic = (0 + 1)/(1 + 1) = 0.5 - score should be(0.5 +- tolerance) + score.fusedScore should be(0.5 +- tolerance) } - it("contributes 0 from the lexical side when the max lexical score is 0") { + it("falls back to the semantic score for a pure semantic search (no lexical matches, so max lexical score is 0)") { val score = fuseScores( hybridResult("no-lexical", lexicalScore = 0.0, semanticScore = 1.0), maxLexicalScore = 0.0, @@ -191,11 +203,12 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T vecWeight = 0.5 ) // normedLexical = 0, normedSemantic = (1 + 1)/(1 + 1) = 1 - // combined = 0.5 * 1 + 0.5 * 0 = 0.5 - score should be(0.5 +- tolerance) + // fused = 0.5 * 1 + 0.5 * 0 = 0.5 + score.normedLexicalScore should be(0.0 +- tolerance) + score.fusedScore should be(0.5 +- tolerance) } - it("contributes 0 from the semantic side when the max semantic score is -1") { + it("falls back to the lexical score when no result has an embedding (so max semantic score is -1)") { val score = fuseScores( hybridResult("no-semantic", lexicalScore = 4.0, semanticScore = -1.0), maxLexicalScore = 4.0, @@ -203,100 +216,133 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T vecWeight = 0.5 ) // normedSemantic = 0, normedLexical = 4/4 = 1 - // combined = 0.5 * 0 + 0.5 * 1 = 0.5 - score should be(0.5 +- tolerance) + // fused = 0.5 * 0 + 0.5 * 1 = 0.5 + score.normedSemanticScore should be(0.0 +- tolerance) + score.fusedScore should be(0.5 +- tolerance) } } - describe("fuseScoresAndGetTopK") { - it("returns an empty list if the input is empty") { - fuseScoresAndGetTopK(List(), vecWeight = 0.0, k = 3) should be (List()) + describe("fuseAndRank") { + import ResultSource.{Lexical, Semantic, Both} + + it("returns an empty list if both inputs are empty") { + fuseAndRank(lexicalResults = List(), semanticResults = List(), vecWeight = 0.0, k = 3) should be(List()) } it("ranks purely by lexical score when vecWeight is 0") { - val results = List( + val lexical = List( hybridResult("low-lexical", lexicalScore = 1.0, semanticScore = 1.0), hybridResult("high-lexical", lexicalScore = 10.0, semanticScore = -1.0), hybridResult("mid-lexical", lexicalScore = 5.0, semanticScore = 0.0) ) - val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.0, k = 3) + val ranked = fuseAndRank(lexical, semanticResults = List(), vecWeight = 0.0, k = 3) - ranked.map(_.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) + ranked.map(_.result.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) } - it("ranks purely by lexical score when max semantic score is -1.0") { - val results = List( + it("ranks purely by lexical score when no result has an embedding (so max semantic score is -1.0)") { + val lexical = List( hybridResult("low-lexical", lexicalScore = 1.0, semanticScore = -1.0), hybridResult("high-lexical", lexicalScore = 10.0, semanticScore = -1.0), hybridResult("mid-lexical", lexicalScore = 5.0, semanticScore = -1.0) ) - val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.5, k = 3) + val ranked = fuseAndRank(lexical, semanticResults = List(), vecWeight = 0.5, k = 3) - ranked.map(_.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) + ranked.map(_.result.id) should be(List("high-lexical", "mid-lexical", "low-lexical")) } it("ranks purely by semantic score when vecWeight is 1") { - val results = List( + val semantic = List( hybridResult("low-semantic", lexicalScore = 10.0, semanticScore = -1.0), hybridResult("high-semantic", lexicalScore = 1.0, semanticScore = 1.0), hybridResult("mid-semantic", lexicalScore = 5.0, semanticScore = 0.0) ) - val ranked = fuseScoresAndGetTopK(results, vecWeight = 1.0, k = 3) + val ranked = fuseAndRank(lexicalResults = List(), semantic, vecWeight = 1.0, k = 3) - ranked.map(_.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) + ranked.map(_.result.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) } - it("ranks purely by semantic score when max lexical score is 0") { - val results = List( + it("ranks purely by semantic score for a pure semantic search (no lexical matches, so max lexical score is 0)") { + val semantic = List( hybridResult("low-semantic", lexicalScore = 0.0, semanticScore = -1.0), hybridResult("high-semantic", lexicalScore = 0.0, semanticScore = 1.0), hybridResult("mid-semantic", lexicalScore = 0.0, semanticScore = 0.0) ) - val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.5, k = 3) + val ranked = fuseAndRank(lexicalResults = List(), semantic, vecWeight = 0.5, k = 3) - ranked.map(_.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) + ranked.map(_.result.id) should be(List("high-semantic", "mid-semantic", "low-semantic")) } it("only returns the top k results") { - val results = List( + val lexical = List( hybridResult("a", lexicalScore = 1.0, semanticScore = 1.0), hybridResult("b", lexicalScore = 10.0, semanticScore = 1.0), hybridResult("c", lexicalScore = 5.0, semanticScore = 1.0) ) - val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.0, k = 2) + val ranked = fuseAndRank(lexical, semanticResults = List(), vecWeight = 0.0, k = 2) ranked should have size 2 - ranked.map(_.id) should be(List("b", "c")) + ranked.map(_.result.id) should be(List("b", "c")) } - it("blends the max-normalised lexical and semantic scores using vecWeight") { - // maxLexical = 4, maxSemantic = 1. - // For "blend": normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 - // combined = 0.25 * 0.5 + 0.75 * 0.5 = 0.5 - val results = List( + it("de-duplicates a result that appears in both sets, keeping it once") { + val lexical = List(hybridResult("shared", lexicalScore = 4.0, semanticScore = 1.0)) + val semantic = List(hybridResult("shared", lexicalScore = 4.0, semanticScore = 1.0)) + + val ranked = fuseAndRank(lexical, semantic, vecWeight = 0.5, k = 10) + + ranked.map(_.result.id) should be(List("shared")) + } + + it("tags each result with whether it came from lexical, semantic, or both") { + val lexical = List( + hybridResult("lexical-only", lexicalScore = 3.0, semanticScore = -1.0), + hybridResult("shared", lexicalScore = 4.0, semanticScore = 1.0) + ) + val semantic = List( + hybridResult("shared", lexicalScore = 4.0, semanticScore = 1.0), + hybridResult("semantic-only", lexicalScore = 0.0, semanticScore = 0.5) + ) + + val ranked = fuseAndRank(lexical, semantic, vecWeight = 0.5, k = 10) + + ranked.map(r => r.result.id -> r.source).toMap should be(Map( + "lexical-only" -> Lexical, + "semantic-only" -> Semantic, + "shared" -> Both + )) + } + + it("produces a ranked table of original scores, normed scores, source and fused score") { + // maxLexical = 4, maxSemantic = 1, vecWeight = 0.25. + val lexical = List( hybridResult("top-lexical", lexicalScore = 4.0, semanticScore = -1.0), - hybridResult("top-semantic", lexicalScore = 0.0, semanticScore = 1.0), hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0) ) + val semantic = List( + hybridResult("blend", lexicalScore = 2.0, semanticScore = 0.0), + hybridResult("top-semantic", lexicalScore = 0.0, semanticScore = 1.0) + ) - val ranked = fuseScoresAndGetTopK(results, vecWeight = 0.25, k = 3) + val ranked = fuseAndRank(lexical, semantic, vecWeight = 0.25, k = 10) - val blend = ranked.find(_.id == "blend").value - // Recompute the expected combined score independently of ordering. - val normedLexical = blend.lexicalScore / 4.0 - val normedSemantic = (blend.semanticScore + 1) / (1.0 + 1) - val expectedCombined = 0.25 * normedSemantic + 0.75 * normedLexical - expectedCombined should be(0.5 +- tolerance) + // A flat projection of the full ranked output, ordered by fused score descending. + val table = ranked.map { r => + (r.result.id, r.source, r.result.lexicalScore, r.result.semanticScore, + r.score.normedLexicalScore, r.score.normedSemanticScore, r.score.fusedScore) + } - // top-lexical: normedLexical = 1, normedSemantic = 0 -> 0.75 - // top-semantic: normedLexical = 0, normedSemantic = 1 -> 0.25 - // blend: 0.5 - ranked.map(_.id) should be(List("top-lexical", "blend", "top-semantic")) + // id source lexical semantic normedLex normedSem fused + table should be(List( + ("top-lexical", Lexical, 4.0, -1.0, 1.0, 0.0, 0.75), + ("blend", Both, 2.0, 0.0, 0.5, 0.5, 0.50), + ("top-semantic", Semantic, 0.0, 1.0, 0.0, 1.0, 0.25) + )) } } } From 3a8bb29d81f52bba2a0cde6080a6eb0be978e008 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 16:55:50 +0100 Subject: [PATCH 050/373] Clean up test --- .../lib/elasticsearch/HybridSearchTest.scala | 128 +++++++++--------- 1 file changed, 62 insertions(+), 66 deletions(-) diff --git a/media-api/test/lib/elasticsearch/HybridSearchTest.scala b/media-api/test/lib/elasticsearch/HybridSearchTest.scala index 076a70e2277..78f9e59b8f5 100644 --- a/media-api/test/lib/elasticsearch/HybridSearchTest.scala +++ b/media-api/test/lib/elasticsearch/HybridSearchTest.scala @@ -61,84 +61,80 @@ class HybridSearchTest extends AnyFunSpec private val oneHundredMilliseconds = Duration(100, MILLISECONDS) private val fiveSeconds = Duration(5, SECONDS) + // The vector we'll "search" with - represents the user's query embedding. + private val queryEmbedding: List[Double] = firstBasisVector(256) + + private def aiImage(id: String, title: String, vector: List[Double]): Image = { + val base = createImage(id = id, usageRights = Handout(), vector = Some(vector)) + base.copy(metadata = base.metadata.copy(title = Some(title))) + } + + // 256-dim vectors to match the `embedding.cohereEmbedV4.image` dense_vector + // mapping, each constructed to have a known cosine similarity to the queryEmbedding. + private def vectorWithScore(score: Double): List[Double] = { + vectorWithCosineSimilarity(256, score) + } + + // The full set of images shared by every spec below; indexed once in beforeAll. + private val testImages = Seq( + aiImage("a", title = "zero lexical", vector = vectorWithScore(1.0)), + aiImage("b", title = "zero lexical", vector = vectorWithScore(0.9)), + aiImage("c", title = "good lexical", vector = vectorWithScore(0.8)), + aiImage("d", title = "good good lexical", vector = vectorWithScore(0.7)) + ) + override def beforeAll(): Unit = { super.beforeAll() ES.ensureIndexExistsAndAliasAssigned() + + implicit val logMarker: LogMarker = MarkerMap() + purgeTestImages + Await.ready(saveImages(testImages), 1.minute) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe testImages.size) } describe("AI / hybrid search") { implicit val logMarker: LogMarker = MarkerMap() - // The vector we'll "search" with - represents the user's query embedding. - val queryEmbedding: List[Double] = firstBasisVector(256) - - // 256-dim vectors to match the `embedding.cohereEmbedV4.image` dense_vector - // mapping, each constructed to have a known cosine similarity to the query. - val vectorWithSemanticScore: Map[Double, List[Double]] = Map( - 1.0 -> vectorWithCosineSimilarity(256, 1.0), - 0.9 -> vectorWithCosineSimilarity(256, 0.9), - 0.8 -> vectorWithCosineSimilarity(256, 0.8), - 0.7 -> vectorWithCosineSimilarity(256, 0.7), - 0.5 -> vectorWithCosineSimilarity(256, 0.5), - 0.0 -> vectorWithCosineSimilarity(256, 0.0), - -0.5 -> vectorWithCosineSimilarity(256, -0.5), - -0.9 -> vectorWithCosineSimilarity(256, -0.9), - -1.0 -> vectorWithCosineSimilarity(256, -1.0) - ) - - def aiImage(id: String, title: String, vector: List[Double]): Image = { - val base = createImage(id = id, usageRights = Handout(), vector = Some(vector)) - base.copy(metadata = base.metadata.copy(title = Some(title))) + it("orders purely by semantic similarity when vecWeight = 1.0") { + val semanticOnlyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 1.0, filterOpt = None), + fiveSeconds + ) + semanticOnlyResults.hits.map(_._1) shouldBe List("a", "b", "c", "d") + } + + it("runs only the lexical query when vecWeight = 0.0") { + val lexicalOnlyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 0.0, filterOpt = None), + fiveSeconds + ) + // Because we asked for k = 4 but only got 2 back, this confirms + // the short-circuiting behaviour at vecWeight 0.0, i.e. only the lexical query ran. + lexicalOnlyResults.hits.map(_._1) shouldBe List("d", "c") } - // Replace the images persisted in ES with exactly `imgs`, so each test - // reasons about a small, fully controlled set of documents. - def withOnly(imgs: Seq[Image])(test: => Unit): Unit = { - purgeTestImages - Await.ready(saveImages(imgs), 1.minute) - eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe imgs.size) - test + it("includes all results once vecWeight > 0") { + val weightedHeavilyLexicallyResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 0.1, filterOpt = None), + fiveSeconds + ) + // We should now get all 4 because vecWeight > 0 + weightedHeavilyLexicallyResults.hits.map(_._1) shouldBe List("d", "c", "a", "b") } - it("joe") { - val a = aiImage("a", title = "zero lexical", vector = vectorWithSemanticScore(1.0)) - val b = aiImage("b", title = "zero lexical", vector = vectorWithSemanticScore(0.9)) - val c = aiImage("c", title = "good lexical", vector = vectorWithSemanticScore(0.8)) - val d = aiImage("d", title = "good good lexical", vector = vectorWithSemanticScore(0.7)) - - withOnly(Seq(a, b, c, d)) { - val semanticOnlyResults = Await.result( - ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 1.0, filterOpt = None), - fiveSeconds - ) - semanticOnlyResults.hits.map(_._1) shouldBe List("a", "b", "c", "d") - - val lexicalOnlyResults = Await.result( - ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 0.0, filterOpt = None), - fiveSeconds - ) - // Because we asked for k = 4 but only got 2 back, this confirms - // the short-circuiting behaviour at vecWeight 0.0, i.e. only the lexical query ran. - lexicalOnlyResults.hits.map(_._1) shouldBe List("d", "c") - - val weightedHeavilyLexicallyResults = Await.result( - ES.hybridSearch("good", queryEmbedding, k = 4, numCandidates = 10, vecWeight = 0.1, filterOpt = None), - fiveSeconds - ) - // We should now get all 4 because vecWeight > 0 - weightedHeavilyLexicallyResults.hits.map(_._1) shouldBe List("d", "c", "a", "b") - - // We've now proven that the lexical top 2 (d, c) are disjunct from the semantic top 2 (a, b) - // Therefore, without score filling, the top 2 hybrid results after theoretical min-max norming - // would have to be the top semantic and the top lexical, i.e. a and d, which both get a score of 1. - // But thanks to score filling, c can make it into the top 2. - val equalWeightingResults = Await.result( - ES.hybridSearch("good", queryEmbedding, k = 2, numCandidates = 10, vecWeight = 0.5, filterOpt = None), - fiveSeconds - ) - equalWeightingResults.hits.map(_._1) shouldBe List("d", "c") - equalWeightingResults.hits.map(_._2.instance) // is there a way to get the score out here? - } + it("promotes a score-filled result into the top 2 with equal weighting") { + // We've now proven that the lexical top 2 (d, c) are disjunct + // from the semantic top 2 (a, b).Therefore, without score filling, + // the top 2 hybrid results after theoretical min-max norming + // would have to be the top semantic and the top lexical, i.e. a and d, + // which both get a score of 1. But thanks to score filling, + // c can make it into the top 2. + val equalWeightingResults = Await.result( + ES.hybridSearch("good", queryEmbedding, k = 2, numCandidates = 10, vecWeight = 0.5, filterOpt = None), + fiveSeconds + ) + equalWeightingResults.hits.map(_._1) shouldBe List("d", "c") } } From 6003b2a84f30ac7e858d05b117fcb9ea97c6e132 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 17:18:17 +0100 Subject: [PATCH 051/373] Handle empty counts --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index c02a8997f47..e2885d1c42f 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -298,8 +298,8 @@ class ElasticSearch( val semanticResults = semanticHits.flatMap(resolveHitAndFillInSemanticScore(_, queryEmbedding, resolveHit)) val ranked = fuseAndRank(lexicalResults, semanticResults, vecWeight, k) - val from = ranked.groupBy(_.source) - logger.info(logMarker, s"${ranked.length} hits (k=$k) after fusing and ranking, ${from(Lexical).length} from lexical, ${from(Semantic)} from semantic, ${from(Both).length} from both") + val counts = ranked.groupBy(_.source).view.mapValues(_.size).toMap.withDefaultValue(0) + logger.info(logMarker, s"${ranked.length} hits (k=$k) after fusing and ranking, ${counts(Lexical)} from lexical, ${counts(Semantic)} from semantic, ${counts(Both)} from both") SearchResults(hits = ranked.map(r => (r.result.id, r.result.image)), total = ranked.length, extraCounts = None) } } From b7670cb4b8f571bd4e44dc6b99db6d275313aeca Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 17:23:11 +0100 Subject: [PATCH 052/373] Comment cleanup --- .../com/gu/mediaservice/lib/VectorUtils.scala | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala index 7449dae91bf..205efce6be3 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -31,7 +31,6 @@ object VectorUtils { // Builds a vector whose cosine similarity with the first basis vector equals // the requested `similarity` (in [-1, 1], where 1 is identical and 0 is orthogonal). def vectorWithCosineSimilarity(dims: Int, similarity: Double): List[Double] = { - // Assert that similarity is between -1 and 1 require(similarity >= -1.0 && similarity <= 1.0, "Cosine similarity must be between -1 and 1") if (similarity == 0.0) { // A cosine similarity of 0 means orthogonal to the first basis vector. @@ -41,9 +40,18 @@ object VectorUtils { } else { val absVec2D = { // Let's pretend we're in just 2 dimensions and think about triangles. - // (Diagram would help here!) + // + // /| + // / | + // n / | opposite = sqrt(n^2 - 1) + // / | + // /θ___| + // 1 + // (adjacent) + // // Our queryEmbedding lies flat on the x axis. - // If we want cos 0 to be 1/n, e.g. 1/2, + // cos(θ) = adjacent / hypotenuse = 1 / n + // If we want cos θ to be 1/n, e.g. 1/2, // then the adjacent side must be 1, and the hypotenuse n. // We want to find the opposite side // n^2 = 1^2 + adj^2 From 1b25455886e83c8b4ca1c8af88634b896cbf0dc9 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 17:39:31 +0100 Subject: [PATCH 053/373] Comment tweak --- .../src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala index 205efce6be3..66fba4ebcc0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -35,7 +35,7 @@ object VectorUtils { if (similarity == 0.0) { // A cosine similarity of 0 means orthogonal to the first basis vector. // That vector is one-hot at index 0, so any vector with a zero - // first component is orthogonal to it; (0, 1) is the simplest choice. + // first component is orthogonal to it. (0, 1, 0, 0, ...) is the simplest choice. oneHotVector(dims, 1) } else { val absVec2D = { From 298e752e9b1c9eb255811c90fb9efa952d4ee329 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 17:42:08 +0100 Subject: [PATCH 054/373] Better grouping in test --- .../gu/mediaservice/lib/VectorUtilsTest.scala | 72 ++++++++++--------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala index f3f76314770..d27f078825e 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -8,52 +8,58 @@ import VectorUtils.{dotProduct, magnitude, cosineSimilarity, firstBasisVector, v class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors { - it ("should compute the dot product of two vectors") { - dotProduct(List(1.0, 2.0, 3.0), List(4.0, 5.0, 6.0)) shouldBe 32.0 - } + describe("dotProduct") { + it ("should compute the dot product of two vectors") { + dotProduct(List(1.0, 2.0, 3.0), List(4.0, 5.0, 6.0)) shouldBe 32.0 + } - it ("should compute a dot product of zero for orthogonal vectors") { - dotProduct(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 - } + it ("should compute a dot product of zero for orthogonal vectors") { + dotProduct(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 + } - it ("should return zero for the dot product of empty vectors") { - dotProduct(List.empty, List.empty) shouldBe 0.0 + it ("should return zero for the dot product of empty vectors") { + dotProduct(List.empty, List.empty) shouldBe 0.0 + } } - it ("should compute the magnitude of a vector") { - magnitude(List(3.0, 4.0)) shouldBe 5.0 - } + describe("magnitude") { + it ("should compute the magnitude of a vector") { + magnitude(List(3.0, 4.0)) shouldBe 5.0 + } - it ("should return a magnitude of zero for a zero vector") { - magnitude(List(0.0, 0.0, 0.0)) shouldBe 0.0 + it ("should return a magnitude of zero for a zero vector") { + magnitude(List(0.0, 0.0, 0.0)) shouldBe 0.0 + } } - it ("should return a cosine similarity of 1 for identical vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)) shouldBe 1.0 +- 1e-9 - } + describe("cosineSimilarity") { + it ("should return a cosine similarity of 1 for identical vectors") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)) shouldBe 1.0 +- 1e-9 + } - it ("should return a cosine similarity of 1 for parallel vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)) shouldBe 1.0 +- 1e-9 - } + it ("should return a cosine similarity of 1 for parallel vectors") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)) shouldBe 1.0 +- 1e-9 + } - it ("should return a cosine similarity of 0 for orthogonal vectors") { - cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 +- 1e-9 - } + it ("should return a cosine similarity of 0 for orthogonal vectors") { + cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 +- 1e-9 + } - it ("should return a cosine similarity of -1 for opposite vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)) shouldBe -1.0 +- 1e-9 - } + it ("should return a cosine similarity of -1 for opposite vectors") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)) shouldBe -1.0 +- 1e-9 + } - it ("should return a cosine similarity of 0 when the first vector has zero magnitude") { - cosineSimilarity(List(0.0, 0.0, 0.0), List(1.0, 2.0, 3.0)) shouldBe 0.0 +- 1e-9 - } + it ("should return a cosine similarity of 0 when the first vector has zero magnitude") { + cosineSimilarity(List(0.0, 0.0, 0.0), List(1.0, 2.0, 3.0)) shouldBe 0.0 +- 1e-9 + } - it ("should return a cosine similarity of 0 when the second vector has zero magnitude") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 - } + it ("should return a cosine similarity of 0 when the second vector has zero magnitude") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + } - it ("should return a cosine similarity of 0 when both vectors have zero magnitude") { - cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + it ("should return a cosine similarity of 0 when both vectors have zero magnitude") { + cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + } } describe("vectorWithCosineSimilarity") { From 15aff72cdb2b30d338fa4ff3d1c695033aa23123 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 17:48:03 +0100 Subject: [PATCH 055/373] More silly cleanup --- .../app/lib/elasticsearch/HybridResult.scala | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index 63ed7f9d52a..619a60bd843 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -11,9 +11,7 @@ case class HybridResult( image: SourceWrapper[Image] ) -// Whether a result was originally returned by the lexical query, the semantic -// query, or both. A shared result tends to rank highly, so this is useful both -// for debugging and for understanding why something appears where it does. + sealed trait ResultSource object ResultSource { case object Lexical extends ResultSource @@ -24,23 +22,17 @@ object ResultSource { (inLexical, inSemantic) match { case (true, true) => Both case (true, false) => Lexical - // A ranked result always comes from at least one side, so the remaining - // case is "semantic only". - case (false, _) => Semantic + case (false, true) => Semantic + case _ => throw new IllegalArgumentException("Result must be in either lexical or semantic") } } -// The intermediate scoring detail for a single result: the two normalised -// component scores and the weighted blend of them that we actually rank by. case class FusedScore( normedLexicalScore: Double, normedSemanticScore: Double, fusedScore: Double ) -// A fully scored, ranked result. Carries the original scores (via `result`), -// where it came from (`source`), the normalised scores and the fused score -// (both via `score`) so the whole ranking decision is inspectable. case class RankedResult( result: HybridResult, source: ResultSource, @@ -65,15 +57,14 @@ object HybridResult { // are truncated 256-dim versions of a normalised 1536-dim vector, // meaning they will not have magnitude 1. // Note this is true cosine similarity from -1 to 1, - // *not* the ES-normalised score, but when we max-normalise + // *not* the ES-normalised score, but when we normalise // later it will end up in the range 0-1. .map(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) .getOrElse(CosineSimilarityTheoreticalMin) HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } - // This is the "theoretical min-max" normalisation chosen by - // "An Analysis of Fusion Functions for Hybrid Retrieval" + // This is the "theoretical min-max" normalisation chosen in // https://arxiv.org/pdf/2210.11934 def normalise(score: Double, max: Double, theoreticalMin: Double): Double = if (max == theoreticalMin) 0.0 @@ -93,13 +84,6 @@ object HybridResult { FusedScore(normedLexicalScore, normedSemanticScore, fusedScore) } - // Combines the lexical and semantic result sets into a single, ranked list. - // - // Each input result already carries both scores (the semantic query is - // rescored to BM25 server-side, and resolveHitAndFillInSemanticScore fills in - // the cosine similarity client-side), so this function only has to: tag each - // result with where it came from, de-duplicate, normalise + fuse the scores, - // then sort and take the top k. def fuseAndRank( lexicalResults: List[HybridResult], semanticResults: List[HybridResult], From 45accbaf8df91569082b74a4b9b4ca041328e40e Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 17:52:57 +0100 Subject: [PATCH 056/373] Consolidate two tests --- .../lib/elasticsearch/HybridResultTest.scala | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index cca8339059d..06d69e01ac3 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -150,19 +150,7 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T } describe("fuseScores") { - it("exposes the normalised lexical and semantic component scores") { - // normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 - val score = fuseScores( - hybridResult("components", lexicalScore = 2.0, semanticScore = 0.0), - maxLexicalScore = 4.0, - maxSemanticScore = 1.0, - vecWeight = 0.25 - ) - score.normedLexicalScore should be(0.5 +- tolerance) - score.normedSemanticScore should be(0.5 +- tolerance) - } - - it("blends the normalised lexical and semantic scores using vecWeight") { + it("exposes the normalised component scores and blends them using vecWeight") { // normedLexical = 2/4 = 0.5, normedSemantic = (0 + 1)/(1 + 1) = 0.5 // fused = 0.25 * 0.5 + 0.75 * 0.5 = 0.5 val score = fuseScores( @@ -171,6 +159,8 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T maxSemanticScore = 1.0, vecWeight = 0.25 ) + score.normedLexicalScore should be(0.5 +- tolerance) + score.normedSemanticScore should be(0.5 +- tolerance) score.fusedScore should be(0.5 +- tolerance) } From 69468803fecff4e2b9657247db8c5755a7c019b4 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Tue, 23 Jun 2026 18:07:52 +0100 Subject: [PATCH 057/373] Return None when cosine similarity is undefined --- .../com/gu/mediaservice/lib/VectorUtils.scala | 9 +++-- .../gu/mediaservice/lib/VectorUtilsTest.scala | 35 ++++++++++--------- .../app/lib/elasticsearch/HybridResult.scala | 5 ++- .../lib/elasticsearch/HybridResultTest.scala | 11 ++++++ 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala index 66fba4ebcc0..f42edaed209 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -9,10 +9,13 @@ object VectorUtils { math.sqrt(vector.map(math.pow(_, 2)).sum) } - def cosineSimilarity(vectorOne: List[Double], vectorTwo: List[Double]): Double = { + def cosineSimilarity(vectorOne: List[Double], vectorTwo: List[Double]): Option[Double] = { val magnitudeProduct = magnitude(vectorOne) * magnitude(vectorTwo) - if (magnitudeProduct == 0.0) 0.0 - else dotProduct(vectorOne, vectorTwo) / magnitudeProduct + // Cosine similarity is undefined when either vector has zero magnitude + // (the angle to the origin is meaningless), so we return None and let the + // caller choose a domain-appropriate fallback. + if (magnitudeProduct == 0.0) None + else Some(dotProduct(vectorOne, vectorTwo) / magnitudeProduct) } // The below functions are primarily for constructing artificial vectors for tests diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala index d27f078825e..02f7cb1eb8d 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -1,13 +1,14 @@ package com.gu.mediaservice.lib import org.scalatest.Inspectors +import org.scalatest.OptionValues import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers import org.scalacheck.Gen import VectorUtils.{dotProduct, magnitude, cosineSimilarity, firstBasisVector, vectorWithCosineSimilarity} -class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors { +class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with OptionValues { describe("dotProduct") { it ("should compute the dot product of two vectors") { dotProduct(List(1.0, 2.0, 3.0), List(4.0, 5.0, 6.0)) shouldBe 32.0 @@ -34,31 +35,31 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors { describe("cosineSimilarity") { it ("should return a cosine similarity of 1 for identical vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)) shouldBe 1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)).value shouldBe 1.0 +- 1e-9 } it ("should return a cosine similarity of 1 for parallel vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)) shouldBe 1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)).value shouldBe 1.0 +- 1e-9 } it ("should return a cosine similarity of 0 for orthogonal vectors") { - cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)) shouldBe 0.0 +- 1e-9 + cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)).value shouldBe 0.0 +- 1e-9 } it ("should return a cosine similarity of -1 for opposite vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)) shouldBe -1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)).value shouldBe -1.0 +- 1e-9 } - it ("should return a cosine similarity of 0 when the first vector has zero magnitude") { - cosineSimilarity(List(0.0, 0.0, 0.0), List(1.0, 2.0, 3.0)) shouldBe 0.0 +- 1e-9 + it ("should be undefined (None) when the first vector has zero magnitude") { + cosineSimilarity(List(0.0, 0.0, 0.0), List(1.0, 2.0, 3.0)) shouldBe None } - it ("should return a cosine similarity of 0 when the second vector has zero magnitude") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + it ("should be undefined (None) when the second vector has zero magnitude") { + cosineSimilarity(List(1.0, 2.0, 3.0), List(0.0, 0.0, 0.0)) shouldBe None } - it ("should return a cosine similarity of 0 when both vectors have zero magnitude") { - cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe 0.0 +- 1e-9 + it ("should be undefined (None) when both vectors have zero magnitude") { + cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe None } } @@ -69,7 +70,7 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors { it ("should create a vector at the requested cosine similarity across a fine sweep of the valid range") { val similarities = (-1000 to 1000).map(_ / 1000.0) forAll(similarities) { s => - cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, s)) shouldBe s +- tolerance + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, s)).value shouldBe s +- tolerance } } @@ -78,27 +79,27 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors { val samples = Gen.listOfN(500, similarityGen).sample.getOrElse(Nil) samples should not be empty forAll(samples) { s => - cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, s)) shouldBe s +- tolerance + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, s)).value shouldBe s +- tolerance } } it ("should work across a range of dimensions") { forAll(List(2, 3, 8, 16, 128, 256, 1024)) { n => - cosineSimilarity(firstBasisVector(n), vectorWithCosineSimilarity(n, 0.42)) shouldBe 0.42 +- tolerance + cosineSimilarity(firstBasisVector(n), vectorWithCosineSimilarity(n, 0.42)).value shouldBe 0.42 +- tolerance } } it ("should return a vector identical to the basis vector for a similarity of 1") { vectorWithCosineSimilarity(dims, 1.0) shouldBe firstBasisVector(dims) - cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, 1.0)) shouldBe 1.0 +- tolerance + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, 1.0)).value shouldBe 1.0 +- tolerance } it ("should return a vector opposite to the basis vector for a similarity of -1") { - cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, -1.0)) shouldBe -1.0 +- tolerance + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, -1.0)).value shouldBe -1.0 +- tolerance } it ("should return a vector orthogonal to the basis vector for a similarity of 0") { - cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, 0.0)) shouldBe 0.0 +- tolerance + cosineSimilarity(firstBasisVector(dims), vectorWithCosineSimilarity(dims, 0.0)).value shouldBe 0.0 +- tolerance } it ("should negate the vector when the similarity is negated") { diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index 619a60bd843..c2fc00da409 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -59,7 +59,10 @@ object HybridResult { // Note this is true cosine similarity from -1 to 1, // *not* the ES-normalised score, but when we normalise // later it will end up in the range 0-1. - .map(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) + // cosineSimilarity is None for a zero-magnitude (degenerate) embedding, + // which we treat as "no usable semantic signal", the same as a missing + // embedding: both fall through to the theoretical min. + .flatMap(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) .getOrElse(CosineSimilarityTheoreticalMin) HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index 06d69e01ac3..ce7ad33a4b5 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -122,6 +122,17 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T result.semanticScore should be(CosineSimilarityTheoreticalMin +- tolerance) } + + it("falls back to a semantic score of -1 when the image embedding has zero magnitude (cosine similarity is undefined)") { + val image = imageWithEmbedding("img-1", embedding = Some(List(0.0, 0.0))) + val result = resolveHitAndFillInSemanticScore( + searchHit("img-1", score = 1.0f), + queryEmbedding = List(1.0, 0.0), + resolveHit = resolveTo(image) + ).value + + result.semanticScore should be(CosineSimilarityTheoreticalMin +- tolerance) + } } describe("normalise") { From 4d994f3d0aef85b894a6a1f0d47a4c070a56d08a Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 24 Jun 2026 10:20:15 +0100 Subject: [PATCH 058/373] Typo Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- media-api/test/lib/elasticsearch/HybridSearchTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media-api/test/lib/elasticsearch/HybridSearchTest.scala b/media-api/test/lib/elasticsearch/HybridSearchTest.scala index 78f9e59b8f5..61b129c9d42 100644 --- a/media-api/test/lib/elasticsearch/HybridSearchTest.scala +++ b/media-api/test/lib/elasticsearch/HybridSearchTest.scala @@ -125,7 +125,7 @@ class HybridSearchTest extends AnyFunSpec it("promotes a score-filled result into the top 2 with equal weighting") { // We've now proven that the lexical top 2 (d, c) are disjunct - // from the semantic top 2 (a, b).Therefore, without score filling, + // from the semantic top 2 (a, b). Therefore, without score filling, // the top 2 hybrid results after theoretical min-max norming // would have to be the top semantic and the top lexical, i.e. a and d, // which both get a score of 1. But thanks to score filling, From 2a969622e4c1890b3150c580297ddef5684a343e Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 24 Jun 2026 11:47:16 +0100 Subject: [PATCH 059/373] Factor out maybeWithQuery --- .../app/lib/elasticsearch/ElasticSearch.scala | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index e2885d1c42f..b3153ad292a 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -192,6 +192,12 @@ class ElasticSearch( boost = boost ) + private def maybeWithFilter(query: Query, filterOpt: Option[Query]): Query = { + filterOpt.map { f => + boolQuery() must (query) filter f + }.getOrElse(query) + } + private def lexicalRequest( lexicalQuery: MultiMatchQuery, k: Int, @@ -199,7 +205,7 @@ class ElasticSearch( ): SearchRequest = ElasticDsl .search(imagesCurrentAlias) - .query(boolQuery().must(lexicalQuery).filter(filterOpt)) + .query(maybeWithFilter(lexicalQuery, filterOpt)) .size(k) private def lexicalSearch( @@ -346,10 +352,6 @@ class ElasticSearch( val filterOpt: Option[Query] = queryBuilder.buildFilterOpt(params, searchFilters, syndicationFilter) - val withFilter = filterOpt.map { f => - boolQuery() must (query) filter f - }.getOrElse(query) - val sort = params.orderBy match { case Some("dateAddedToCollection") => sorts.dateAddedToCollectionDescending case _ => sorts.createSort(params.orderBy) @@ -380,7 +382,7 @@ class ElasticSearch( Seq.empty } - val searchRequest = prepareSearch(withFilter) + val searchRequest = prepareSearch(maybeWithFilter(query, filterOpt)) .trackTotalHits(trackTotalHits) .runtimeMappings(runtimeMappings) .storedFields("_source") // this needs to be explicit when using script fields From 43b14090e75e7770386677a25e037861f11ca0db Mon Sep 17 00:00:00 2001 From: lindseydew Date: Wed, 24 Jun 2026 11:56:54 +0100 Subject: [PATCH 060/373] Upgrade the identity management service to v2 --- build.sbt | 2 +- usage/app/lib/UsageConfig.scala | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index fc00f3d8fa3..73f642b4ba0 100644 --- a/build.sbt +++ b/build.sbt @@ -93,7 +93,6 @@ lazy val commonLib = project("common-lib").settings( libraryDependencies ++= Seq( "com.gu" %% "editorial-permissions-client" % "4.0.0", "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0", - "com.amazonaws" % "aws-java-sdk-iam" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion, @@ -101,6 +100,7 @@ lazy val commonLib = project("common-lib").settings( "com.amazonaws" % "aws-java-sdk-sts" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion, + "software.amazon.awssdk" % "iam" % awsSdkV2Version, "software.amazon.awssdk" % "s3" % awsSdkV2Version, "nl.gn0s1s" %% "elastic4s-core" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-client-esjava" % elastic4sVersion, diff --git a/usage/app/lib/UsageConfig.scala b/usage/app/lib/UsageConfig.scala index 19f07e6e7e1..ac2af4f4f3e 100644 --- a/usage/app/lib/UsageConfig.scala +++ b/usage/app/lib/UsageConfig.scala @@ -1,9 +1,9 @@ package lib -import com.amazonaws.services.identitymanagement._ import com.gu.mediaservice.lib.config.{CommonConfig, GridConfigResources} import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.lib.net.URI.ensureSecure +import software.amazon.awssdk.services.iam.IamClient import scala.util.Try @@ -37,11 +37,11 @@ class UsageConfig(resources: GridConfigResources) extends CommonConfig(resources val awsRegionName = string("aws.region") - private val iamClient: AmazonIdentityManagement = withAWSCredentials(AmazonIdentityManagementClientBuilder.standard()).build() + private val iamClient: IamClient = withAWSCredentialsV2(IamClient.builder()).build() val postfix: String = if (isDev) { try { - iamClient.getUser.getUser.getUserName + iamClient.getUser.user().userName() } catch { case e:com.amazonaws.SdkClientException => logger.warn("Unable to determine current IAM user, probably because you're using temp credentials. Usage may not be able to determine the live/preview app names", e) From 4c5ef70248a75cb76aa9979d666ec43d836efed3 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Wed, 24 Jun 2026 12:02:04 +0100 Subject: [PATCH 061/373] Remove sts --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index 73f642b4ba0..c70274a2b56 100644 --- a/build.sbt +++ b/build.sbt @@ -97,7 +97,6 @@ lazy val commonLib = project("common-lib").settings( "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-sts" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion, "software.amazon.awssdk" % "iam" % awsSdkV2Version, From 41ef9c397831bdad0ac149ec86de85b8c077a446 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Wed, 24 Jun 2026 14:25:12 +0100 Subject: [PATCH 062/373] Upgrade the sts client --- build.sbt | 4 ++-- usage/app/lib/ContentApis.scala | 29 +++++++++++++++++++---------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/build.sbt b/build.sbt index c70274a2b56..e91c46eae26 100644 --- a/build.sbt +++ b/build.sbt @@ -107,7 +107,7 @@ lazy val commonLib = project("common-lib").settings( "com.gu" %% "thrift-serializer" % "5.0.2", "org.scalaz" %% "scalaz-core" % "7.3.8", "org.im4java" % "im4java" % "1.4.0", - "com.gu" % "kinesis-logback-appender" % "1.4.4", + "com.gu" % "kinesis-logback-appender" % "2.1.3", "net.logstash.logback" % "logstash-logback-encoder" % "5.0", logback, // play-logback; needed when running the scripts "com.typesafe.scala-logging" %% "scala-logging" % "3.9.2", @@ -193,7 +193,7 @@ lazy val thrall = playProject("thrall", 9002) lazy val usage = playProject("usage", 9009).settings( libraryDependencies ++= Seq( "com.gu" %% "content-api-client-default" % "32.0.0", - "com.gu" %% "content-api-client-aws" % "0.7.6", + "com.gu" %% "content-api-client-aws" % "1.0.1", "io.reactivex" %% "rxscala" % "0.27.0", "software.amazon.kinesis" % "amazon-kinesis-client" % awsKclVersion, // explicit dependencies on kinesis and dynamodb to upgrade the versions used by kcl diff --git a/usage/app/lib/ContentApis.scala b/usage/app/lib/ContentApis.scala index 4ce51ce6909..3f5eaa12296 100644 --- a/usage/app/lib/ContentApis.scala +++ b/usage/app/lib/ContentApis.scala @@ -1,8 +1,12 @@ package lib -import com.amazonaws.auth.profile.ProfileCredentialsProvider -import com.amazonaws.auth.{AWSCredentialsProvider, AWSCredentialsProviderChain, STSAssumeRoleSessionCredentialsProvider} -import com.amazonaws.services.securitytoken.{AWSSecurityTokenService, AWSSecurityTokenServiceClientBuilder} +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest +import software.amazon.awssdk.services.sts.StsClient +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider + import com.gu.contentapi.client._ import com.gu.contentapi.client.model.{HttpResponse, ItemQuery} @@ -47,16 +51,21 @@ class PreviewContentApi(protected val config: UsageConfig)(implicit val ex: Sche trait IAMAuthContentApiClient extends ContentApiClient { protected val config: UsageConfig - lazy val sts: AWSSecurityTokenService = AWSSecurityTokenServiceClientBuilder.standard() - .withRegion(config.awsRegionName) + lazy val sts: StsClient = StsClient.builder() + .region(Region.of(config.awsRegionName)) .build() - lazy val capiCredentials: AWSCredentialsProvider = - config.capiPreviewRole.map( - new STSAssumeRoleSessionCredentialsProvider.Builder(_, "capi") - .withStsClient(sts) + private lazy val sessionId: String = "session-" + Math.random() + lazy val capiCredentials: AwsCredentialsProvider = + config.capiPreviewRole.map(arn => { + + val assumeRoleRequest = AssumeRoleRequest.builder().roleArn(arn).roleSessionName(sessionId).build() + + StsAssumeRoleCredentialsProvider.builder() + .refreshRequest(assumeRoleRequest) + .stsClient(sts) .build() - ).getOrElse(new ProfileCredentialsProvider("capi")) // will be used if stream is ever run locally (unusual) + }).getOrElse(ProfileCredentialsProvider.create("capi")) // will be used if stream is ever run locally (unusual) abstract override def get( url: String, From 87ec69f21cb39e3596f6288cb779846f9a6ffd2b Mon Sep 17 00:00:00 2001 From: lindseydew Date: Wed, 24 Jun 2026 15:00:47 +0100 Subject: [PATCH 063/373] Sns + sqs upgrade wip --- build.sbt | 4 +- .../gu/mediaservice/lib/aws/Embedder.scala | 6 +-- .../com/gu/mediaservice/lib/aws/SNS.scala | 9 ++-- .../lib/aws/SimpleSqsMessageConsumer.scala | 48 +++++++++++-------- 4 files changed, 36 insertions(+), 31 deletions(-) diff --git a/build.sbt b/build.sbt index e91c46eae26..d764ccb24f0 100644 --- a/build.sbt +++ b/build.sbt @@ -95,12 +95,12 @@ lazy val commonLib = project("common-lib").settings( "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0", "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-sqs" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-sns" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion, "software.amazon.awssdk" % "iam" % awsSdkV2Version, "software.amazon.awssdk" % "s3" % awsSdkV2Version, + "software.amazon.awssdk" % "sns" % awsSdkV2Version, + "software.amazon.awssdk" % "sqs" % awsSdkV2Version, "nl.gn0s1s" %% "elastic4s-core" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-client-esjava" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-domain" % elastic4sVersion, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala index f766fec70e2..f706b9e2137 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Embedder.scala @@ -1,10 +1,10 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.sqs.model.SendMessageResult import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.{Jpeg, MimeType, Png, Tiff} import play.api.libs.json.{Json, OFormat} import software.amazon.awssdk.services.s3vectors.model.QueryVectorsResponse import software.amazon.awssdk.services.s3vectors.model.{QueryOutputVector, QueryVectorsResponse, VectorData} +import software.amazon.awssdk.services.sqs.model.SendMessageResponse import java.nio.file.{Files, Path} import scala.concurrent.{ExecutionContext, Future} @@ -27,7 +27,7 @@ class Embedder(bedrock: Bedrock, sqs: SimpleSqsMessageConsumer)(implicit ec: Exe def queueImageToEmbed(message: EmbedderMessage)(implicit logMarker: LogMarker) = { val messageBody = Json.stringify(Json.toJson(message)) - val result: SendMessageResult = sqs.sendMessage(messageBody) - logger.info(logMarker, s"Queued image for embedding with message ID: ${result.getMessageId}") + val result: SendMessageResponse = sqs.sendMessage(messageBody) + logger.info(logMarker, s"Queued image for embedding with message ID: ${result.messageId()}") } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SNS.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SNS.scala index 588e1664202..8985d9f5c3c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SNS.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SNS.scala @@ -1,16 +1,15 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.sns.model.PublishRequest -import com.amazonaws.services.sns.{AmazonSNS, AmazonSNSClientBuilder} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import play.api.libs.json.{JsValue, Json} - +import software.amazon.awssdk.services.sns.SnsClient +import software.amazon.awssdk.services.sns.model.PublishRequest class SNS(config: CommonConfig, topicArn: String) extends GridLogging { - lazy val client: AmazonSNS = config.withAWSCredentials(AmazonSNSClientBuilder.standard()).build() + lazy val client = config.withAWSCredentialsV2(SnsClient.builder()).build() def publish(message: JsValue, subject: String): Unit = { - val result = client.publish(new PublishRequest(topicArn, Json.stringify(message), subject)) + val result = client.publish(PublishRequest.builder().topicArn(topicArn).message(Json.stringify(message)).subject(subject).build()) logger.info(s"Published message: $result") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala index 0af9faa80f8..1aa7dce05fe 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala @@ -1,42 +1,48 @@ package com.gu.mediaservice.lib.aws import com.gu.mediaservice.lib.config.CommonConfig -import com.amazonaws.services.sqs.AmazonSQS -import com.amazonaws.services.sqs.AmazonSQSClientBuilder -import com.amazonaws.services.sqs.model.{ChangeMessageVisibilityRequest, DeleteMessageRequest, ReceiveMessageRequest, SendMessageRequest, SendMessageResult, Message => SQSMessage} +import software.amazon.awssdk.services.sqs.SqsClient +import software.amazon.awssdk.services.sqs.model.{ChangeMessageVisibilityRequest, DeleteMessageRequest, GetQueueAttributesRequest, MessageSystemAttributeName, QueueAttributeName, ReceiveMessageRequest, SendMessageRequest, SendMessageResponse, Message => SQSMessage} import scala.jdk.CollectionConverters._ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { - lazy val client: AmazonSQS = config.withAWSCredentials(AmazonSQSClientBuilder.standard()).build() + lazy val client= config.withAWSCredentialsV2(SqsClient.builder()).build() - def getNextMessage(attributeNames: String*): Option[SQSMessage] = + def getNextMessage(attributeNames: MessageSystemAttributeName*): Option[SQSMessage] = client.receiveMessage( - new ReceiveMessageRequest(queueUrl) - .withWaitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html - .withMaxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation - .withMessageSystemAttributeNames(attributeNames: _*) - ).getMessages.asScala.headOption + ReceiveMessageRequest.builder().queueUrl(queueUrl) + .waitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html + .maxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation + .messageSystemAttributeNames(attributeNames: _*) // todo come back to this + .build() + ).messages().asScala.headOption def deleteMessage(message: SQSMessage): Unit = - client.deleteMessage(new DeleteMessageRequest(queueUrl, message.getReceiptHandle)) + client.deleteMessage(new DeleteMessageRequest(queueUrl, message.receiptHandle())) def makeMessageVisible(message: SQSMessage): Unit = - client.changeMessageVisibility(new ChangeMessageVisibilityRequest(queueUrl, message.getReceiptHandle, 0)) + client.changeMessageVisibility(new ChangeMessageVisibilityRequest(queueUrl, message.receiptHandle(), 0)) def getStatus: Map[String, String] = { - client.getQueueAttributes(queueUrl, List( - "ApproximateNumberOfMessagesDelayed", - "ApproximateNumberOfMessages", - "ApproximateNumberOfMessagesNotVisible" - ).asJava).getAttributes.asScala.toMap + client.getQueueAttributes( + GetQueueAttributesRequest.builder(). + queueUrl(queueUrl) + .attributeNames( + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES, + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_DELAYED, + QueueAttributeName.APPROXIMATE_NUMBER_OF_MESSAGES_NOT_VISIBLE + ) + .build() + ).attributes().asScala.toMap.map({ case(k, v) => (k.name(), v)}) } - def sendMessage(messageBody: String): SendMessageResult = { - val sendMessageRequest: SendMessageRequest = new SendMessageRequest() - .withQueueUrl(queueUrl) - .withMessageBody(messageBody) + def sendMessage(messageBody: String): SendMessageResponse = { + val sendMessageRequest: SendMessageRequest = SendMessageRequest.builder() + .queueUrl(queueUrl) + .messageBody(messageBody) + .build() client.sendMessage(sendMessageRequest) } } From 0ac0fa4e4d91737fbf62bcff78ae3130f4cd5f95 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 24 Jun 2026 17:01:04 +0100 Subject: [PATCH 064/373] Improve logging --- media-api/app/controllers/MediaApi.scala | 105 ++++++++++-------- .../app/lib/elasticsearch/ElasticSearch.scala | 12 +- 2 files changed, 66 insertions(+), 51 deletions(-) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 70fc1e2874c..376b1636a0a 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -613,23 +613,32 @@ class MediaApi( } def semanticSearchByImage(imageId: String, k: Int, params: SearchParams): Future[SearchResults] = { - val filterOpt = buildAiFilter(params) - - for { - maybeImage <- elasticSearch.getImageById(imageId) - maybeEmbedding = maybeImage - .filter(image => isVisibleToAccessor(request.user, image)) - .flatMap(_.embedding) - .flatMap(_.cohereEmbedV4) - .map(_.image) - searchResults <- maybeEmbedding match { - // If we have an embedding, perform the KNN search. If not, return an empty result set. - case Some(embedding) => - elasticSearch.semanticSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) - case None => - Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) - } - } yield searchResults + val outerMarker = logMarker + + { + implicit val logMarker: LogMarker = outerMarker ++ Map( + "imageId" -> imageId, + "aiSearchType" -> "image" + ) + + val filterOpt = buildAiFilter(params) + + for { + maybeImage <- elasticSearch.getImageById(imageId) + maybeEmbedding = maybeImage + .filter(image => isVisibleToAccessor(request.user, image)) + .flatMap(_.embedding) + .flatMap(_.cohereEmbedV4) + .map(_.image) + searchResults <- maybeEmbedding match { + // If we have an embedding, perform the KNN search. If not, return an empty result set. + case Some(embedding) => + elasticSearch.semanticSearch(embedding, k = k, numCandidates = Math.max(k * 2, 100), filterOpt = filterOpt) + case None => + Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) + } + } yield searchResults + } } def semanticSearchByText(k: Int, params: SearchParams): Future[SearchResults] = { @@ -640,36 +649,42 @@ class MediaApi( logger.info(logMarker, s"No semantic query found in structured query; returning no AI results") Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) case Some(semanticQuery) => - // Normalise key so that "Dogs" and "dogs " share a cache entry. - val cacheKey = semanticQuery.trim.toLowerCase - val markers = logMarker ++ Map("query" -> semanticQuery) - - if (embeddingCache.getIfPresent(cacheKey).isDefined) { - logger.info(markers, s"AI search embedding cache hit query=$semanticQuery") - } else { - logger.info(markers, s"AI search embedding cache miss query=$semanticQuery") - } - - val weight = params.vecWeight.getOrElse(0.8) - - // cache.get(key) is atomic: if two requests race on the same key, only one - // load fires and both callers receive the same Future. - val embeddingFuture = embeddingCache.get(cacheKey) + val vecWeight = params.vecWeight.getOrElse(0.8) + val outerMarker = logMarker + + { + implicit val logMarker: LogMarker = outerMarker ++ Map( + "query" -> semanticQuery, + "vecWeight" -> vecWeight, + "aiSearchType" -> "hybrid" + ) - val filterOpt = buildAiFilter(params) + // Normalise key so that "Dogs" and "dogs " share a cache entry. + val cacheKey = semanticQuery.trim.toLowerCase + if (embeddingCache.getIfPresent(cacheKey).isDefined) { + logger.info(logMarker, s"AI search embedding cache hit query=$semanticQuery") + } else { + logger.info(logMarker, s"AI search embedding cache miss query=$semanticQuery") + } - logger.info(markers, s"vecWeight for query '$semanticQuery' is $weight") - for { - embedding <- embeddingFuture - searchResults <- elasticSearch.hybridSearch( - query = semanticQuery, - queryEmbedding = embedding, - k = k, - numCandidates = Math.max(k * 2, 100), - vecWeight = weight, - filterOpt = filterOpt - ) - } yield searchResults + // cache.get(key) is atomic: if two requests race on the same key, only one + // load fires and both callers receive the same Future. + val embeddingFuture = embeddingCache.get(cacheKey) + + val filterOpt = buildAiFilter(params) + + for { + embedding <- embeddingFuture + searchResults <- elasticSearch.hybridSearch( + query = semanticQuery, + queryEmbedding = embedding, + k = k, + numCandidates = Math.max(k * 2, 100), + vecWeight = vecWeight, + filterOpt = filterOpt + ) + } yield searchResults + } } } diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index b3153ad292a..e71b8819abe 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -215,7 +215,7 @@ class ElasticSearch( )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { val searchRequest = lexicalRequest(createMultiMatchQuery(query, operator = Or), k, filterOpt) - executeAndLog(withSearchQueryTimeout(searchRequest), "lexical-only").map { r => + executeAndLog(withSearchQueryTimeout(searchRequest), "lexical search").map { r => val imageHits = r.result.hits.hits.map(resolveHit).toSeq.flatten.map(i => (i.instance.id, i)) SearchResults(hits = imageHits, total = imageHits.length, extraCounts = None) } @@ -286,8 +286,8 @@ class ElasticSearch( // Assigning to vals here eagerly starts both requests, so they run in // parallel. The for-comprehension below only sequences the *combination* // of their results, not their execution. - val lexicalSearchResponse = executeAndLog(withSearchQueryTimeout(lexicalSearchRequest), "lexical-hybrid") - val semanticSearchResponse = executeAndLog(withSearchQueryTimeout(semanticSearchRequest), "semantic-hybrid") + val lexicalSearchResponse = executeAndLog(withSearchQueryTimeout(lexicalSearchRequest), "lexical side of hybrid AI search") + val semanticSearchResponse = executeAndLog(withSearchQueryTimeout(semanticSearchRequest), "semantic side of hybrid AI search") for { lexical <- lexicalSearchResponse @@ -295,7 +295,7 @@ class ElasticSearch( } yield { val lexicalHits = lexical.result.hits.hits.toList val semanticHits = semantic.result.hits.hits.toList - logger.info(logMarker, s"${lexicalHits.length + semanticHits.length} total hits: ${lexicalHits.length} lexical, ${semanticHits.length} semantic") + logger.info(logMarker, s"Hybrid AI search returned ${lexicalHits.length + semanticHits.length} initial hits: ${lexicalHits.length} lexical, ${semanticHits.length} semantic") // Resolve each side to images and fill in the client-side semantic score. // We keep the two sides separate so that fuseAndRank can tag each result @@ -305,7 +305,7 @@ class ElasticSearch( val ranked = fuseAndRank(lexicalResults, semanticResults, vecWeight, k) val counts = ranked.groupBy(_.source).view.mapValues(_.size).toMap.withDefaultValue(0) - logger.info(logMarker, s"${ranked.length} hits (k=$k) after fusing and ranking, ${counts(Lexical)} from lexical, ${counts(Semantic)} from semantic, ${counts(Both)} from both") + logger.info(logMarker, s"Hybrid AI search returned ${ranked.length} hits (k=$k) after fusing and ranking, ${counts(Lexical)} from lexical, ${counts(Semantic)} from semantic, ${counts(Both)} from both") SearchResults(hits = ranked.map(r => (r.result.id, r.result.image)), total = ranked.length, extraCounts = None) } } @@ -339,7 +339,7 @@ class ElasticSearch( val elapsed = stopwatch.elapsed logger.info( combineMarkers(logMarker, elapsed), - s"hybrid search completed in ${elapsed.toMillis} ms" + s"Hybrid AI search completed in ${elapsed.toMillis} ms" ) } From 2971f99ea1710c59c4311d0ac75f338ed47c13b4 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Wed, 24 Jun 2026 17:17:41 +0100 Subject: [PATCH 065/373] Upgrade sqs and sns --- .../lib/aws/SimpleSqsMessageConsumer.scala | 15 +++++++++++++-- .../com/gu/mediaservice/lib/aws/SqsHelpers.scala | 8 ++++---- .../lib/aws/SqsViaSnsMessageConsumer.scala | 4 ++-- .../app/controllers/ImageLoaderController.scala | 4 ++-- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala index 1aa7dce05fe..a6dae8f63a6 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala @@ -20,10 +20,21 @@ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { ).messages().asScala.headOption def deleteMessage(message: SQSMessage): Unit = - client.deleteMessage(new DeleteMessageRequest(queueUrl, message.receiptHandle())) + client.deleteMessage( + DeleteMessageRequest.builder() + .queueUrl(queueUrl) + .receiptHandle(message.receiptHandle()) + .build() + ) def makeMessageVisible(message: SQSMessage): Unit = - client.changeMessageVisibility(new ChangeMessageVisibilityRequest(queueUrl, message.receiptHandle(), 0)) + client.changeMessageVisibility( + ChangeMessageVisibilityRequest.builder() + .queueUrl(queueUrl) + .receiptHandle(message.receiptHandle()) + .visibilityTimeout(0) + .build() + ) def getStatus: Map[String, String] = { client.getQueueAttributes( diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsHelpers.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsHelpers.scala index 288c58cd0f6..919753c0876 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsHelpers.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsHelpers.scala @@ -1,6 +1,6 @@ package com.gu.mediaservice.lib.aws -import com.amazonaws.services.sqs.model.{Message => SQSMessage} +import software.amazon.awssdk.services.sqs.model.{MessageSystemAttributeName, Message => SQSMessage} import com.gu.mediaservice.lib.net.URI.{decode => uriDecode} import play.api.libs.json.Json @@ -8,9 +8,9 @@ import scala.util.Try trait SqsHelpers { - val attrApproximateReceiveCount = "ApproximateReceiveCount" + val attrApproximateReceiveCount = MessageSystemAttributeName.APPROXIMATE_RECEIVE_COUNT def getApproximateReceiveCount(message: SQSMessage): Int = - Try(message.getAttributes.get(attrApproximateReceiveCount).toInt).toOption.getOrElse(-1) + Try(message.attributes().get(attrApproximateReceiveCount).toInt).toOption.getOrElse(-1) def extractS3KeyFromSqsMessage(message: SQSMessage): Try[String] = Try { @@ -21,7 +21,7 @@ trait SqsHelpers { // note that we don't bother reading the bucket name currently, but attempt the read elsewhere based on the // configured bucket value, this could be checked here if we're concerned (s3Records.head \ "bucket" \ "name") - (Json.parse(message.getBody) \ "Records" \\ "s3").toList match { + (Json.parse(message.body()) \ "Records" \\ "s3").toList match { case head :: Nil => uriDecode((head \ "object" \ "key").as[String]) case s3Records => throw new Exception(s"Expected 1 record containing 's3' in message body, got ${s3Records.size}") } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsViaSnsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsViaSnsMessageConsumer.scala index ede4d897360..d9398cee525 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsViaSnsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SqsViaSnsMessageConsumer.scala @@ -4,7 +4,7 @@ import java.util.concurrent.Executors import _root_.play.api.libs.functional.syntax._ import _root_.play.api.libs.json._ import org.apache.pekko.actor.ActorSystem -import com.amazonaws.services.sqs.model.{Message => SQSMessage} +import software.amazon.awssdk.services.sqs.model.{Message => SQSMessage} import com.gu.mediaservice.lib.ImageId import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.json.PlayJsonHelpers._ @@ -57,7 +57,7 @@ abstract class SqsViaSnsMessageConsumer(queueUrl: String, config: CommonConfig, f.foreach { _ => deleteMessage(msg) } private def extractSNSMessage(sqsMessage: SQSMessage): Option[SNSMessage] = { - val result = Json.fromJson[SNSMessage](Json.parse(sqsMessage.getBody)) + val result = Json.fromJson[SNSMessage](Json.parse(sqsMessage.body())) logParseErrors(result) result.asOpt } diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index ab44b57a88f..500206ccaa1 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -4,7 +4,7 @@ import org.apache.pekko.Done import org.apache.pekko.stream.Materializer import org.apache.pekko.stream.scaladsl.Source import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.sqs.model.{Message => SQSMessage} +import software.amazon.awssdk.services.sqs.model.{Message => SQSMessage} import com.amazonaws.util.IOUtils import com.drew.imaging.ImageProcessingException import com.gu.mediaservice.GridClient @@ -84,7 +84,7 @@ class ImageLoaderController(auth: Authentication, activeProcessingMessage.set(Some(sqsMessage)) val logMarker: LogMarker = MarkerMap( "requestType" -> "handleMessageFromIngestBucket", - "requestId" -> sqsMessage.getMessageId, + "requestId" -> sqsMessage.messageId(), ) handleMessageFromIngestBucket(sqsMessage)(logMarker) .recover { From fd0db9cc1839241dc13c4f88217583dbe256ae42 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Wed, 24 Jun 2026 17:22:23 +0100 Subject: [PATCH 066/373] Remove comment --- .../com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala index a6dae8f63a6..aeb700643c7 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/SimpleSqsMessageConsumer.scala @@ -15,7 +15,7 @@ class SimpleSqsMessageConsumer (queueUrl: String, config: CommonConfig) { ReceiveMessageRequest.builder().queueUrl(queueUrl) .waitTimeSeconds(20) // Wait for maximum duration (20s) as per doc recommendation: http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-long-polling.html .maxNumberOfMessages(1) // Pull 1 message at a time to avoid starvation - .messageSystemAttributeNames(attributeNames: _*) // todo come back to this + .messageSystemAttributeNames(attributeNames: _*) .build() ).messages().asScala.headOption From 8ed482c088db2cc87e7641d81251e98e71e663b5 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 24 Jun 2026 17:26:36 +0100 Subject: [PATCH 067/373] Cover cosine similarity edge cases and log warning when we cannot calculate a semantic score --- .../com/gu/mediaservice/lib/VectorUtils.scala | 17 +++++++++++------ .../app/lib/elasticsearch/HybridResult.scala | 16 ++++++++++------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala index f42edaed209..50c8701840f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -2,6 +2,7 @@ package com.gu.mediaservice.lib object VectorUtils { def dotProduct(vectorOne: List[Double], vectorTwo: List[Double]): Double = { + require(vectorOne.length == vectorTwo.length, "Vectors must be the same dimensionality") vectorOne.zip(vectorTwo).map(component => component._1 * component._2).sum } @@ -10,12 +11,16 @@ object VectorUtils { } def cosineSimilarity(vectorOne: List[Double], vectorTwo: List[Double]): Option[Double] = { - val magnitudeProduct = magnitude(vectorOne) * magnitude(vectorTwo) - // Cosine similarity is undefined when either vector has zero magnitude - // (the angle to the origin is meaningless), so we return None and let the - // caller choose a domain-appropriate fallback. - if (magnitudeProduct == 0.0) None - else Some(dotProduct(vectorOne, vectorTwo) / magnitudeProduct) + // Cosine similarity is undefined when the vectors differ in dimensionality + // (the dot product is meaningless) or when either has zero magnitude (the + // angle to the origin is meaningless), so we return None in those cases and + // let the caller choose a domain-appropriate fallback. + if (vectorOne.length != vectorTwo.length) None + else { + val magnitudeProduct = magnitude(vectorOne) * magnitude(vectorTwo) + if (magnitudeProduct == 0.0) None + else Some(dotProduct(vectorOne, vectorTwo) / magnitudeProduct) + } } // The below functions are primarily for constructing artificial vectors for tests diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index c2fc00da409..ed872cf81dc 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -1,6 +1,7 @@ package lib.elasticsearch import com.gu.mediaservice.lib.VectorUtils +import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.Image import com.sksamuel.elastic4s.requests.searches.SearchHit @@ -39,7 +40,7 @@ case class RankedResult( score: FusedScore ) -object HybridResult { +object HybridResult extends GridLogging { // Theoretical minimum cosine similarity (opposite vectors). val CosineSimilarityTheoreticalMin: Double = -1.0 // Theoretical minimum BM25 lexical score. @@ -49,7 +50,7 @@ object HybridResult { hit: SearchHit, queryEmbedding: List[Double], resolveHit: SearchHit => Option[SourceWrapper[Image]] - ): Option[HybridResult] = + )(implicit logMarker: LogMarker): Option[HybridResult] = resolveHit(hit).map { image => val semanticScore = image.instance.embedding .flatMap(_.cohereEmbedV4) @@ -59,11 +60,14 @@ object HybridResult { // Note this is true cosine similarity from -1 to 1, // *not* the ES-normalised score, but when we normalise // later it will end up in the range 0-1. - // cosineSimilarity is None for a zero-magnitude (degenerate) embedding, - // which we treat as "no usable semantic signal", the same as a missing - // embedding: both fall through to the theoretical min. .flatMap(e => VectorUtils.cosineSimilarity(e.image, queryEmbedding)) - .getOrElse(CosineSimilarityTheoreticalMin) + // cosineSimilarity is None when there's no usable semantic signal (a + // missing, mismatched, or zero-magnitude embedding), all of which fall + // through to the theoretical min. + .getOrElse { + logger.warn(logMarker, s"Could not calculate semantic score for image ${hit.id}. Falling back to minimum semantic score") + CosineSimilarityTheoreticalMin + } HybridResult(hit.id, lexicalScore = hit.score, semanticScore = semanticScore, image = image) } From 9d2b1fd61e170b453cce0797f282e7f729288ed3 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 24 Jun 2026 17:30:15 +0100 Subject: [PATCH 068/373] Cover off vectorWithCosineSimilarity edge case --- .../src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala index 50c8701840f..3e11f5b8914 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -39,6 +39,7 @@ object VectorUtils { // Builds a vector whose cosine similarity with the first basis vector equals // the requested `similarity` (in [-1, 1], where 1 is identical and 0 is orthogonal). def vectorWithCosineSimilarity(dims: Int, similarity: Double): List[Double] = { + require(dims > 1, "Dimensions must be at least 2") require(similarity >= -1.0 && similarity <= 1.0, "Cosine similarity must be between -1 and 1") if (similarity == 0.0) { // A cosine similarity of 0 means orthogonal to the first basis vector. From 20f8f07c91701fe899da1dff5b4d8710e9cd44f4 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 24 Jun 2026 17:32:28 +0100 Subject: [PATCH 069/373] Add tests for new vector edge cases --- .../com/gu/mediaservice/lib/VectorUtilsTest.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala index 02f7cb1eb8d..e2ad055bbb3 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -21,6 +21,10 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with Opti it ("should return zero for the dot product of empty vectors") { dotProduct(List.empty, List.empty) shouldBe 0.0 } + + it ("should reject vectors of differing dimensionality") { + an [IllegalArgumentException] should be thrownBy dotProduct(List(1.0, 2.0), List(1.0, 2.0, 3.0)) + } } describe("magnitude") { @@ -61,6 +65,10 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with Opti it ("should be undefined (None) when both vectors have zero magnitude") { cosineSimilarity(List(0.0, 0.0, 0.0), List(0.0, 0.0, 0.0)) shouldBe None } + + it ("should be undefined (None) when the vectors differ in dimensionality") { + cosineSimilarity(List(1.0, 2.0), List(1.0, 2.0, 3.0)) shouldBe None + } } describe("vectorWithCosineSimilarity") { @@ -124,5 +132,9 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with Opti it ("should reject a similarity less than -1") { an [IllegalArgumentException] should be thrownBy vectorWithCosineSimilarity(dims, -1.0001) } + + it ("should reject fewer than 2 dimensions") { + an [IllegalArgumentException] should be thrownBy vectorWithCosineSimilarity(1, 0.5) + } } } From 90536d00631204eb8d39d8994b718a811541ad63 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 24 Jun 2026 17:38:05 +0100 Subject: [PATCH 070/373] Add implicit to test --- media-api/test/lib/elasticsearch/HybridResultTest.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/media-api/test/lib/elasticsearch/HybridResultTest.scala b/media-api/test/lib/elasticsearch/HybridResultTest.scala index ce7ad33a4b5..715e5d41db8 100644 --- a/media-api/test/lib/elasticsearch/HybridResultTest.scala +++ b/media-api/test/lib/elasticsearch/HybridResultTest.scala @@ -1,5 +1,6 @@ package lib.elasticsearch +import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.model.{CohereV4Embedding, Embedding, Handout, Image} import com.sksamuel.elastic4s.requests.searches.SearchHit import org.scalactic.Tolerance @@ -14,6 +15,8 @@ class HybridResultTest extends AnyFunSpec with Matchers with OptionValues with T private val tolerance = 1e-9 + implicit val logMarker: LogMarker = MarkerMap() + private def searchHit(id: String, score: Float): SearchHit = SearchHit( id = id, index = "test-index", From e05e37a0fc70ee973cecc9206eb3a5002506ca13 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Thu, 25 Jun 2026 10:16:09 +0100 Subject: [PATCH 071/373] Log timing in andThen instead of foreach Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- media-api/app/lib/elasticsearch/ElasticSearch.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index e71b8819abe..b522ca43b14 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -335,15 +335,13 @@ class ElasticSearch( case _ => fusedLexicalAndSemanticSearch(query, queryEmbedding, k, numCandidates, vecWeight, filterOpt) } - searchResults.foreach { _ => + searchResults.andThen { case _ => val elapsed = stopwatch.elapsed logger.info( combineMarkers(logMarker, elapsed), s"Hybrid AI search completed in ${elapsed.toMillis} ms" ) } - - searchResults } } From 85f1cbbb19b596ea34ddada9ae1f7fe055535c13 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 25 Jun 2026 10:28:03 +0100 Subject: [PATCH 072/373] Deprecate kinesis v1 --- build.sbt | 1 - .../com/gu/mediaservice/lib/aws/Kinesis.scala | 32 +++++++++++-------- .../lib/config/CommonConfig.scala | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/build.sbt b/build.sbt index d764ccb24f0..99baad59441 100644 --- a/build.sbt +++ b/build.sbt @@ -96,7 +96,6 @@ lazy val commonLib = project("common-lib").settings( "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion, "software.amazon.awssdk" % "iam" % awsSdkV2Version, "software.amazon.awssdk" % "s3" % awsSdkV2Version, "software.amazon.awssdk" % "sns" % awsSdkV2Version, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala index 811bfe64007..539d1277d6b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/Kinesis.scala @@ -1,32 +1,36 @@ package com.gu.mediaservice.lib.aws +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider +import software.amazon.awssdk.core.SdkBytes +import software.amazon.awssdk.regions.Region + import java.nio.ByteBuffer import java.util.UUID -import com.amazonaws.services.kinesis.model.PutRecordRequest -import com.amazonaws.services.kinesis.{AmazonKinesis, AmazonKinesisClientBuilder} +import software.amazon.awssdk.services.kinesis.model.PutRecordRequest +import software.amazon.awssdk.services.kinesis.KinesisClient import com.gu.mediaservice.lib.json.JsonByteArrayUtil import com.gu.mediaservice.model.usage.UsageNotice import net.logstash.logback.marker.{LogstashMarker, Markers} import play.api.libs.json.{JodaWrites, Json, Writes} -import com.amazonaws.auth.AWSCredentialsProvider import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import org.joda.time.DateTime +import java.net.URI + case class KinesisSenderConfig( - override val awsRegion: String, - override val awsCredentials: AWSCredentialsProvider, - override val awsLocalEndpoint: Option[String], + override val awsRegionV2: Region, + override val awsCredentialsV2: AwsCredentialsProvider, + override val awsLocalEndpointUri: Option[URI], override val isDev: Boolean, streamName: String -) extends AwsClientV1BuilderUtils +) extends AwsClientV2BuilderUtils class Kinesis(config: KinesisSenderConfig) extends GridLogging{ - private val builder = AmazonKinesisClientBuilder.standard() - private def getKinesisClient: AmazonKinesis = config.withAWSCredentials(builder).build() + private def getKinesisClient = config.withAWSCredentialsV2(KinesisClient.builder()).build() - private lazy val kinesisClient: AmazonKinesis = getKinesisClient + private lazy val kinesisClient: KinesisClient = getKinesisClient def publish[T <: LogMarker](message: T)(implicit messageWrites: Writes[T]): Unit = { val partitionKey = UUID.randomUUID().toString @@ -40,10 +44,10 @@ class Kinesis(config: KinesisSenderConfig) extends GridLogging{ logger.info(markers, "Publishing message to kinesis") val data = ByteBuffer.wrap(payload) - val request = new PutRecordRequest() - .withStreamName(config.streamName) - .withPartitionKey(partitionKey) - .withData(data) + val request = PutRecordRequest.builder() + .streamName(config.streamName) + .partitionKey(partitionKey).data(SdkBytes.fromByteBuffer(data)) + .build() try { val result = kinesisClient.putRecord(request) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 2cdf3edea3f..737dfb150d2 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -219,7 +219,7 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientV1B configuration.getOptional[Map[String, Seq[String]]]("agencyPicks.ingredients") val agencyPicksColour: String = stringDefault("agencyPicks.colour", "#7d0068") - private def getKinesisConfigForStream(streamName: String) = KinesisSenderConfig(awsRegion, awsCredentials, awsLocalEndpoint, isDev, streamName) + private def getKinesisConfigForStream(streamName: String) = KinesisSenderConfig(awsRegionV2, awsCredentialsV2, awsLocalEndpointUri, isDev, streamName) final def getOptionalStringSet(key: String): Option[Set[String]] = Try { configuration.getOptional[Seq[String]](key) From 0b0fcf640b605294d1388d4f66d5ce84f0bb088c Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Thu, 25 Jun 2026 10:56:59 +0100 Subject: [PATCH 073/373] Use tolerance variable --- .../com/gu/mediaservice/lib/VectorUtilsTest.scala | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala index e2ad055bbb3..7a2e42a1863 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -9,6 +9,8 @@ import VectorUtils.{dotProduct, magnitude, cosineSimilarity, firstBasisVector, v class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with OptionValues { + val tolerance = 1e-9 + describe("dotProduct") { it ("should compute the dot product of two vectors") { dotProduct(List(1.0, 2.0, 3.0), List(4.0, 5.0, 6.0)) shouldBe 32.0 @@ -39,19 +41,19 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with Opti describe("cosineSimilarity") { it ("should return a cosine similarity of 1 for identical vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)).value shouldBe 1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(1.0, 2.0, 3.0)).value shouldBe 1.0 +- tolerance } it ("should return a cosine similarity of 1 for parallel vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)).value shouldBe 1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(2.0, 4.0, 6.0)).value shouldBe 1.0 +- tolerance } it ("should return a cosine similarity of 0 for orthogonal vectors") { - cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)).value shouldBe 0.0 +- 1e-9 + cosineSimilarity(List(1.0, 0.0), List(0.0, 1.0)).value shouldBe 0.0 +- tolerance } it ("should return a cosine similarity of -1 for opposite vectors") { - cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)).value shouldBe -1.0 +- 1e-9 + cosineSimilarity(List(1.0, 2.0, 3.0), List(-1.0, -2.0, -3.0)).value shouldBe -1.0 +- tolerance } it ("should be undefined (None) when the first vector has zero magnitude") { @@ -73,7 +75,6 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with Opti describe("vectorWithCosineSimilarity") { val dims = 256 - val tolerance = 1e-9 it ("should create a vector at the requested cosine similarity across a fine sweep of the valid range") { val similarities = (-1000 to 1000).map(_ / 1000.0) From f60e38c2761aadc57963762b459d9b884e4277c4 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Thu, 25 Jun 2026 11:00:02 +0100 Subject: [PATCH 074/373] Add comments about vectorWithCosineSimilarity --- .../src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala | 4 ++++ .../test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala | 2 ++ 2 files changed, 6 insertions(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala index 3e11f5b8914..81ff0bedd7d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/VectorUtils.scala @@ -23,7 +23,9 @@ object VectorUtils { } } + // =============================================================================== // The below functions are primarily for constructing artificial vectors for tests + // =============================================================================== private def oneHotVector(dims: Int, hotIndex: Int): List[Double] = List.tabulate(dims)(i => if (i == hotIndex) 1.0 else 0.0) @@ -38,6 +40,8 @@ object VectorUtils { // Builds a vector whose cosine similarity with the first basis vector equals // the requested `similarity` (in [-1, 1], where 1 is identical and 0 is orthogonal). + // This is specifically for use in tests where we need to create test fixtures with + // specific semantic scores. def vectorWithCosineSimilarity(dims: Int, similarity: Double): List[Double] = { require(dims > 1, "Dimensions must be at least 2") require(similarity >= -1.0 && similarity <= 1.0, "Cosine similarity must be between -1 and 1") diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala index 7a2e42a1863..1a3ce4087ca 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/VectorUtilsTest.scala @@ -73,6 +73,8 @@ class VectorUtilsTest extends AnyFunSpec with Matchers with Inspectors with Opti } } + // This function is specifically for use in tests, + // where we need to create test fixtures with specific semantic scores. describe("vectorWithCosineSimilarity") { val dims = 256 From 6f8e32dde2606e2a4d3bf422b362cc6fc08aed52 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Thu, 25 Jun 2026 11:01:48 +0100 Subject: [PATCH 075/373] Add logging of scores for all hybrid results --- .../app/lib/elasticsearch/ElasticSearch.scala | 8 ++++++- .../app/lib/elasticsearch/HybridResult.scala | 23 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index b522ca43b14..f89a56341fd 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -268,7 +268,7 @@ class ElasticSearch( vecWeight: Double, filterOpt: Option[Query] )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SearchResults] = { - import HybridResult.{resolveHitAndFillInSemanticScore, fuseAndRank} + import HybridResult.{resolveHitAndFillInSemanticScore, fuseAndRank, renderRankedTable} val lexicalQuery = createMultiMatchQuery(query, operator = Or) val lexicalSearchRequest = lexicalRequest(lexicalQuery, k, filterOpt) @@ -306,6 +306,12 @@ class ElasticSearch( val ranked = fuseAndRank(lexicalResults, semanticResults, vecWeight, k) val counts = ranked.groupBy(_.source).view.mapValues(_.size).toMap.withDefaultValue(0) logger.info(logMarker, s"Hybrid AI search returned ${ranked.length} hits (k=$k) after fusing and ranking, ${counts(Lexical)} from lexical, ${counts(Semantic)} from semantic, ${counts(Both)} from both") + // This log will be noisy and we can get rid of it at a later stage if we want, + // but I think it could be indispensable if we see weird results and we + // want to understand why they're there. An alternative would be putting it + // into the search response so we could see it in the network tab and even surface + // in the UI if we wanted, but that would be a bigger change. + logger.info(logMarker, s"Hybrid AI search ranked results:\n${renderRankedTable(ranked)}") SearchResults(hits = ranked.map(r => (r.result.id, r.result.image)), total = ranked.length, extraCounts = None) } } diff --git a/media-api/app/lib/elasticsearch/HybridResult.scala b/media-api/app/lib/elasticsearch/HybridResult.scala index ed872cf81dc..61f3ff0d6f7 100644 --- a/media-api/app/lib/elasticsearch/HybridResult.scala +++ b/media-api/app/lib/elasticsearch/HybridResult.scala @@ -128,4 +128,27 @@ object HybridResult extends GridLogging { .take(k) }).getOrElse(List()) } + + // Renders the full ranked output as an aligned table, ordered as given + // (i.e. by fused score descending), for logging/debugging. + def renderRankedTable(ranked: List[RankedResult]): String = { + val header = Seq("id", "source", "lexical", "semantic", "normedLex", "normedSem", "fused") + val rows = ranked.map { r => + Seq( + r.result.id, + r.source.toString, + f"${r.result.lexicalScore}%.4f", + f"${r.result.semanticScore}%.4f", + f"${r.score.normedLexicalScore}%.4f", + f"${r.score.normedSemanticScore}%.4f", + f"${r.score.fusedScore}%.4f" + ) + } + val widths = header.indices.map { i => + (header +: rows).map(_(i).length).max + } + (header +: rows) + .map(row => row.zip(widths).map { case (cell, w) => cell.padTo(w, ' ') }.mkString(" ")) + .mkString("\n") + } } From 9fe51dbb89806260cd9ff3a6ab52c6fbf282fab1 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 25 Jun 2026 12:19:50 +0100 Subject: [PATCH 076/373] Migrate Syndication to use v2 version of the dynamo lib --- metadata-editor/app/lib/MetadataSqsMessageConsumer.scala | 2 +- metadata-editor/app/lib/Syndication.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metadata-editor/app/lib/MetadataSqsMessageConsumer.scala b/metadata-editor/app/lib/MetadataSqsMessageConsumer.scala index 3dc414102fb..c11fd896d05 100644 --- a/metadata-editor/app/lib/MetadataSqsMessageConsumer.scala +++ b/metadata-editor/app/lib/MetadataSqsMessageConsumer.scala @@ -15,6 +15,6 @@ class MetadataSqsMessageConsumer(config: EditsConfig, metadataEditorMetrics: Met } def processDeletedImage(message: JsValue) = Future { - withImageId(message)(id => store.deleteItem(id)) + withImageId(message)(id => store.deleteItemV2(id)) } } diff --git a/metadata-editor/app/lib/Syndication.scala b/metadata-editor/app/lib/Syndication.scala index d8a6f767fbf..e67583f2477 100644 --- a/metadata-editor/app/lib/Syndication.scala +++ b/metadata-editor/app/lib/Syndication.scala @@ -48,8 +48,8 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { (implicit ec: ExecutionContext): Future[Unit] = publishChangedSyndicationRightsForPhotoshoot[Unit](id, unchangedPhotoshoot = false) { () => for { - edits <- editsStore.removeKey(id, Edits.Photoshoot) - _ <- editsStore.removeKey(id, Edits.PhotoshootTitle) + edits <- editsStore.removeKeyV2(id, Edits.Photoshoot) + _ <- editsStore.removeKeyV2(id, Edits.PhotoshootTitle) _ = publish(id, UpdateImagePhotoshootMetadata)(edits) } yield () } From df56fe084b878647b911a001b3c83b2ab5a9a88a Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 25 Jun 2026 16:10:36 +0100 Subject: [PATCH 077/373] Use batch v2 --- .../gu/mediaservice/lib/aws/DynamoDB.scala | 57 ++++++++++++++++++- metadata-editor/app/lib/Syndication.scala | 2 +- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index 63e763fcaa6..dddf531214b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -15,8 +15,9 @@ import play.api.libs.json._ import software.amazon.awssdk.enhanced.dynamodb._ import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import software.amazon.awssdk.enhanced.dynamodb.model +import software.amazon.awssdk.enhanced.dynamodb.model.{BatchGetItemEnhancedRequest, ReadBatch} import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, ReturnValue => ReturnValueV2} +import software.amazon.awssdk.services.dynamodb.model.{BatchGetItemRequest, UpdateItemRequest, AttributeValue => AttributeValueV2, KeysAndAttributes => KeysAndAttributesV2, ReturnValue => ReturnValueV2} import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} @@ -131,6 +132,60 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti def setAddV2(id: String, key: String, value: List[String])(implicit ex: ExecutionContext): Future[JsObject] = Future { updateV2(id, DynamoDB.addExpr(key, lastModifiedKey), AttributeValueV2.fromSs(value.asJava)) } + + def batchGetV2(ids: List[String], attributeKey: String)(implicit ex: ExecutionContext, rjs: Reads[T]): Future[Map[String, T]] = { + val chunks = + ids.grouped(100).toList.zipWithIndex + + Future + .traverse(chunks) { case (chunk, idx) => + logger.info(s"Fetching records for chunk $idx of ${chunks.size}") + Future { + + val readBatchBuilder = + ReadBatch.builder(classOf[EnhancedDocument]) + .mappedTableResource(table2) + + chunk.foreach { id => + readBatchBuilder.addGetItem( + Key.builder() + .partitionValue(id) + .build() + ) + } + + val results = + dynamo2.batchGetItem( + BatchGetItemEnhancedRequest.builder() + .readBatches(readBatchBuilder.build()) + .build() + ) + + results + .resultsForTable(table2) + .asScala + .toList + .flatMap { doc => + + logger.info(s"Obtained document $doc") + + val json = asJsObject(doc) + + val maybeT = + (json \ attributeKey).asOpt[T] + + logger.info(s"Obtained a T of $maybeT from json $json") + + maybeT.map( + doc.getString(IdKey) -> _ + ) + } + .toMap + } + } + .map(_.foldLeft(Map.empty[String, T])(_ ++ _)) + } + def batchGet(ids: List[String], attributeKey: String) (implicit ex: ExecutionContext, rjs: Reads[T]): Future[Map[String, T]] = { val keyChunkList = ids diff --git a/metadata-editor/app/lib/Syndication.scala b/metadata-editor/app/lib/Syndication.scala index e67583f2477..a3111bab729 100644 --- a/metadata-editor/app/lib/Syndication.scala +++ b/metadata-editor/app/lib/Syndication.scala @@ -135,7 +135,7 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { def getAllImageRightsInPhotoshoot(photoshoot: Photoshoot) (implicit ec: ExecutionContext): Future[Map[String, SyndicationRights]] = for { imageIds <- getImagesInPhotoshoot(photoshoot) - allNonInferredRights <- syndicationStore.batchGet(imageIds, syndicationRightsFieldName) + allNonInferredRights <- syndicationStore.batchGetV2(imageIds, syndicationRightsFieldName) } yield { logger.info(s"Found non-inferred rights for ${allNonInferredRights.size} of ${imageIds.size} images in photoshoot ${photoshoot.title}") val mostRecentInferrableRightsMaybe = getMostRecentInferrableSyndicationRights(allNonInferredRights.values.toList) From 98dc0ab64890b634a5517d01e0616dc6b930862f Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 25 Jun 2026 16:19:01 +0100 Subject: [PATCH 078/373] Upgrade the scan for v2 --- .../gu/mediaservice/lib/aws/DynamoDB.scala | 32 ++++++++++++++++++- metadata-editor/app/lib/Syndication.scala | 2 +- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index dddf531214b..e6fc62f6961 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -17,7 +17,7 @@ import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import software.amazon.awssdk.enhanced.dynamodb.model import software.amazon.awssdk.enhanced.dynamodb.model.{BatchGetItemEnhancedRequest, ReadBatch} import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{BatchGetItemRequest, UpdateItemRequest, AttributeValue => AttributeValueV2, KeysAndAttributes => KeysAndAttributesV2, ReturnValue => ReturnValueV2} +import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, ReturnValue => ReturnValueV2, QueryRequest => QueryRequestV2} import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} @@ -255,6 +255,36 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti items map (a => a.getString("id")) } + def scanForIdV2( + indexName: String, + keyName: String, + key: String + )(implicit ex: ExecutionContext): Future[List[String]] = + Future { + + val response = + client2.query( + QueryRequestV2.builder() + .tableName(tableName) + .indexName(indexName) + .keyConditionExpression(s"$keyName = :key") + .expressionAttributeValues( + Map( + ":key" -> + AttributeValueV2.builder() + .s(key) + .build() + ).asJava + ) + .projectionExpression("id") + .build() + ) + + response.items().asScala.toList.map { item => + item.get("id").s() + } + } + private def updateRequestBuilder(id: String, expression: String) = { UpdateItemRequest.builder() .key(Map(IdKey -> AttributeValueV2.fromS(id)).asJava) diff --git a/metadata-editor/app/lib/Syndication.scala b/metadata-editor/app/lib/Syndication.scala index a3111bab729..1fcbc9f4b13 100644 --- a/metadata-editor/app/lib/Syndication.scala +++ b/metadata-editor/app/lib/Syndication.scala @@ -144,7 +144,7 @@ trait Syndication extends Edit with MessageSubjects with GridLogging { def getImagesInPhotoshoot(photoshoot: Photoshoot) (implicit ec: ExecutionContext): Future[List[String]] = - editsStore.scanForId(config.editsTablePhotoshootIndex, Edits.PhotoshootTitle, photoshoot.title) + editsStore.scanForIdV2(config.editsTablePhotoshootIndex, Edits.PhotoshootTitle, photoshoot.title) .recover { case NoItemFound => Nil } def getChangedRights(before: Map[String, SyndicationRights], after: Map[String, SyndicationRights]): Map[String, Option[SyndicationRights]] = { From f9329666ad59d2a65aa5e5012040ccfd89a0c261 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 25 Jun 2026 16:49:01 +0100 Subject: [PATCH 079/373] Update the tests --- .../scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala index c08c6d3c098..6bf2c57564b 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/aws/SqsHelpersTest.scala @@ -1,7 +1,7 @@ package com.gu.mediaservice.lib.aws import org.scalatest.funsuite.AnyFunSuiteLike -import com.amazonaws.services.sqs.model.{Message => SQSMessage} +import software.amazon.awssdk.services.sqs.model.Message import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper import scala.io.Source @@ -12,8 +12,7 @@ class SqsHelpersTest extends AnyFunSuiteLike with SqsHelpers { test("extractS3KeyFromSqsMessage") { - val message = new SQSMessage() - message.setBody(Source.fromResource("s3SqsMessage.json").mkString) + val message = Message.builder().body(Source.fromResource("s3SqsMessage.json").mkString).build() extractS3KeyFromSqsMessage(message) shouldBe Success("bill.bloggs@example.co.uk/0cd747d665e2c59e86d256e6f45c369664e558bd") } From bcf6b9b29c3ad333162b466180bd3198ac4a425b Mon Sep 17 00:00:00 2001 From: Andrew Nowak Date: Fri, 26 Jun 2026 14:11:20 +0100 Subject: [PATCH 080/373] make double assignment intent clearer --- kahuna/public/js/components/gr-archiver/gr-archiver.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kahuna/public/js/components/gr-archiver/gr-archiver.js b/kahuna/public/js/components/gr-archiver/gr-archiver.js index 0d676369fb4..dd6633b53d3 100644 --- a/kahuna/public/js/components/gr-archiver/gr-archiver.js +++ b/kahuna/public/js/components/gr-archiver/gr-archiver.js @@ -132,7 +132,8 @@ module.controller('grArchiverCtrl', [ mediaApi.undelete(imageId) .then(() => pollUndeleted(imageId)) .then(() => { - ctrl.canUndelete = ctrl.isDeleted = false; + ctrl.canUndelete = false; + ctrl.isDeleted = false; ctrl.image.softDeletedMetadata = undefined; if (ctrl.image.userMetadata?.data !== undefined) { ctrl.image.userMetadata.data.archived = true; From 54630824936ac8ca791ad62a550fa3bb5ceea7ab Mon Sep 17 00:00:00 2001 From: Mateusz Date: Sat, 27 Jun 2026 18:12:45 +0100 Subject: [PATCH 081/373] Enable gzip compression on ES Java REST client --- .../lib/elasticsearch/ElasticSearchClient.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala index 6fbb23682f3..17321133d91 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/ElasticSearchClient.scala @@ -41,8 +41,15 @@ trait ElasticSearchClient extends ElasticSearchExecutions with GridLogging { lazy val client = { logger.info("Connecting to Elastic 8: " + url) - val client = JavaClient(ElasticProperties(url)) - ElasticClient(client) + val hosts = ElasticProperties(url).endpoints.collect { + case ElasticNodeEndpoint(protocol, host, port, _) => + new org.apache.http.HttpHost(host, port, protocol) + } + val restClient = org.elasticsearch.client.RestClient + .builder(hosts: _*) + .setCompressionEnabled(true) // request gzip from ES; reduces response body ~5-6× + .build() + ElasticClient(JavaClient.fromRestClient(restClient)) } //TODO: this function should fail and cause healthcheck fails From 4ba78c76cdc81a87083906feebb748b2756d9c5e Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Mon, 29 Jun 2026 12:16:52 +0100 Subject: [PATCH 082/373] Add GNM-owned, agency picks and filtered matches to AI search --- kahuna/public/js/search/results.html | 3 +- kahuna/public/js/search/results.js | 5 ++ media-api/app/controllers/MediaApi.scala | 2 +- .../app/lib/elasticsearch/ElasticSearch.scala | 83 ++++++++++++++----- 4 files changed, 69 insertions(+), 24 deletions(-) diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index 2c61e3c77bb..069a889fd90 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -22,7 +22,8 @@
- {{ctrl.totalResults | toLocaleString}} matches + Best {{ctrl.aiResultsShown | toLocaleString}} of {{ctrl.totalResults | toLocaleString}} matches + {{ctrl.totalResults | toLocaleString}} matches
-
+ + Your filters have narrowed your pool of images down from {{ctrl.filterPoolCounts.totalPool | toLocaleString}} + to {{ctrl.filterPoolCounts.filteredPool | toLocaleString}}. + Please add a search query to search over these. + +
+ +
No matches, sorry! Try altering your search. Please enter a query. You can include filters, but must include either some text or a similar:image-id term. diff --git a/kahuna/public/js/search/results.js b/kahuna/public/js/search/results.js index a8c5dc3b3f4..ab352f3787e 100644 --- a/kahuna/public/js/search/results.js +++ b/kahuna/public/js/search/results.js @@ -233,6 +233,12 @@ results.controller('SearchResultsCtrl', [ // FIXME: https://github.com/argo-rest/theseus has forced us to co-opt the actions field for this ctrl.tickerCounts = images.$response?.$$state?.value?.actions?.tickerCounts; + // Present only when an AI search had filters but no text/similar-image query to + // rank by. When set, we show a prompt explaining how the filters have narrowed + // the pool of images and inviting the user to add a query. + ctrl.filterPoolCounts = images.$response?.$$state?.value?.actions?.filterPoolCounts; + ctrl.filtersOnlyAiSearch = !!ctrl.filterPoolCounts; + ctrl.hasQuery = !!$stateParams.query; ctrl.initialSearchUri = images.uri; ctrl.embeddableUrl = window.location.href; diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 5bd706cd613..7d2c728651b 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -602,9 +602,42 @@ class MediaApi( elasticSearch.searchFilters.filterAndFilter(chipFilterOpt, requestFilterOpt) } + // The pool of images before the user's chip filters are applied (i.e. only the + // filters implied by the request itself, e.g. syndication/permissions). Used to + // explain how the chip filters have narrowed things down. + def buildBasePoolFilter(params: SearchParams): Option[Query] = + elasticSearch.queryBuilder.buildFilterOpt( + params, + elasticSearch.searchFilters, + elasticSearch.syndicationFilter + ) + def emptyAiSearchResponse = Future.successful(respondCollection(List.empty[EmbeddedEntity[JsValue]], Some(0), Some(0), None, List())) + // Response for an AI search that only has filters (no text/similar-image query to + // rank by). Rather than erroring, we return an empty result set annotated with how + // the filters have narrowed the pool of images, so the client can prompt the user + // to add a query. + def filtersOnlyAiSearchResponse(params: SearchParams): Future[Result] = { + val basePoolFilter = buildBasePoolFilter(params) + val filteredPoolFilter = buildAiFilter(params) + + for { + totalPool <- elasticSearch.countMatchingFilter(basePoolFilter) + filteredPool <- elasticSearch.countMatchingFilter(filteredPoolFilter) + } yield respondCollection( + data = List.empty[EmbeddedEntity[JsValue]], + offset = Some(0), + total = Some(0), + maybeExtraCounts = Some(ExtraCounts( + tickerCounts = Map.empty, + filterPoolCounts = Some(FilterPoolCounts(totalPool = totalPool, filteredPool = filteredPool)) + )), + links = Nil + ) + } + def aiSearchResponseFromResults(searchResults: SearchResults): Result = { val imageEntities = searchResults.hits map (hitToImageEntity _).tupled respondCollection( @@ -695,8 +728,13 @@ class MediaApi( def performAiSearchAndRespond(params: SearchParams): Future[Result] = { params.aiQueryParts.validationError match { case Left(errorMessage) => - logger.info(logMarker, s"Invalid AI search query: $errorMessage") - Future.successful(respondError(UnprocessableEntity, "invalid-ai-search", errorMessage)) + if (params.aiQueryParts.hasFilterConditions) { + logger.info(logMarker, s"AI search with only filters and no query to rank by; returning filter pool counts: ${params.aiQueryParts.filterConditions}") + filtersOnlyAiSearchResponse(params) + } else { + logger.info(logMarker, s"Invalid AI search query: $errorMessage") + Future.successful(respondError(UnprocessableEntity, "invalid-ai-search", errorMessage)) + } case scala.util.Right(_) => val k = config.aiSearchResultLimit val searchResultsFuture = parseAiSearchMode(params) match { diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 6bd3737a040..6d57a73cb32 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -362,7 +362,7 @@ class ElasticSearch( // How many images match the active filters in total, so we can show // "Best k of N matches". - private def countMatchingFilter(filterOpt: Option[Query])(implicit ex: ExecutionContext, logMarker: LogMarker): Future[Long] = { + def countMatchingFilter(filterOpt: Option[Query])(implicit ex: ExecutionContext, logMarker: LogMarker): Future[Long] = { val countRequest = ElasticDsl.count(imagesCurrentAlias).query(filterOpt.getOrElse(matchAllQuery())) executeAndLog(countRequest, "hybrid AI search filter count").map(_.result.count) diff --git a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala index e6dd8fed5b0..71e5d08f321 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearchModel.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearchModel.scala @@ -28,6 +28,8 @@ case class AiQueryParts( // a text query (text KNN) or a similar image (image KNN) to be valid. def hasKnn: Boolean = similarImageId.nonEmpty || semanticQuery.nonEmpty + def hasFilterConditions: Boolean = filterConditions.nonEmpty + // Returns Left(error message) if the AI query cannot be satisfied, otherwise Right(()). def validationError: Either[String, Unit] = { if (hasSimilarAndSemanticText) @@ -35,12 +37,12 @@ case class AiQueryParts( "AI search can't combine a similar image search with a text query because the two rankings can't be merged. " + "Please use either a text query or a similar image, not both." ) - else if (!hasKnn) - Left( - "AI search needs something to rank by. " + - "Add a text query or a similar image alongside your filters." - ) - else + else if (!hasKnn) { + Left( + "AI search needs something to rank by. " + + "Add a text query or a similar image alongside your filters." + ) + } else Right(()) } } From 3590ccec810f87ee194b078cfbe5eb70f1b8fe88 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 2 Jul 2026 17:27:02 +0100 Subject: [PATCH 091/373] improve wording --- kahuna/public/js/search/results.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index 7c62cbe1031..287497d5b6c 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -222,9 +222,7 @@
- Your filters have narrowed your pool of images down from {{ctrl.filterPoolCounts.totalPool | toLocaleString}} - to {{ctrl.filterPoolCounts.filteredPool | toLocaleString}}. - Please add a search query to search over these. + {{ctrl.filterPoolCounts.filteredPool | toLocaleString}} images match your filters. Add a query to see the best 200.
From e438c7e218985238cde7036d75115d34919c70f0 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 6 Jul 2026 16:38:08 +0100 Subject: [PATCH 092/373] Update the querybyusageid --- .../lib/usage/ItemToMediaUsage.scala | 56 ++++++++++--------- usage/app/model/UsageTable.scala | 32 ++++++++--- 2 files changed, 55 insertions(+), 33 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala index eb3194bfe5b..af1e95c2c8b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala @@ -5,42 +5,46 @@ import com.amazonaws.services.dynamodbv2.document.Item import com.gu.mediaservice.model.usage._ import org.joda.time.DateTime import org.joda.time.format.ISODateTimeFormat +import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import scala.jdk.CollectionConverters._ import scala.util.Try object ItemToMediaUsage { - def transform(item: Item): MediaUsage = { + def transform(doc: EnhancedDocument): MediaUsage = { MediaUsage( - UsageId(item.getString("usage_id")), - item.getString("grouping"), - item.getString("media_id"), - UsageType(item.getString("usage_type")), - item.getString("media_type"), - UsageStatus(item.getString("usage_status")), - Option(item.getMap[Any]("print_metadata")) - .map(_.asScala.toMap).flatMap(buildPrint), - Option(item.getMap[Any]("digital_metadata")) - .map(_.asScala.toMap).flatMap(buildDigital), - Option(item.getMap[Any]("syndication_metadata")) - .map(_.asScala.toMap).flatMap(buildSyndication), - Option(item.getMap[Any]("front_metadata")) - .map(_.asScala.toMap).flatMap(buildFront), - Option(item.getMap[Any]("download_metadata")) - .map(_.asScala.toMap).flatMap(buildDownload), - Option(item.getMap[Any]("child_metadata")) - .map(_.asScala.toMap).flatMap(buildChild), - new DateTime(item.getLong("last_modified")), - Try { - item.getLong("date_added") - }.toOption.map(new DateTime(_)), - Try { - item.getLong("date_removed") - }.toOption.map(new DateTime(_)) + UsageId(doc.getString("usage_id")), + doc.getString("grouping"), + doc.getString("media_id"), + UsageType(doc.getString("usage_type")), + doc.getString("media_type"), + UsageStatus(doc.getString("usage_status")), + Option(doc.getMapOfUnknownType("print_metadata")) + .map(_.asScala.toMap) + .flatMap(buildPrint), + Option(doc.getMapOfUnknownType("digital_metadata")) + .map(_.asScala.toMap) + .flatMap(buildDigital), + Option(doc.getMapOfUnknownType("syndication_metadata")) + .map(_.asScala.toMap) + .flatMap(buildSyndication), + Option(doc.getMapOfUnknownType("front_metadata")) + .map(_.asScala.toMap) + .flatMap(buildFront), + Option(doc.getMapOfUnknownType("download_metadata")) + .map(_.asScala.toMap) + .flatMap(buildDownload), + Option(doc.getMapOfUnknownType("child_metadata")) + .map(_.asScala.toMap) + .flatMap(buildChild), + new DateTime(doc.getNumber("last_modified").longValue()), + Try(doc.getNumber("date_added").longValue()).toOption.map(new DateTime(_)), + Try(doc.getNumber("date_removed").longValue()).toOption.map(new DateTime(_)) ) } + private def buildFront(metadataMap: Map[String, Any]): Option[FrontUsageMetadata] = { Try { FrontUsageMetadata( diff --git a/usage/app/model/UsageTable.scala b/usage/app/model/UsageTable.scala index 356c175b544..b11b2c5c92a 100644 --- a/usage/app/model/UsageTable.scala +++ b/usage/app/model/UsageTable.scala @@ -10,25 +10,43 @@ import com.gu.mediaservice.model.usage.{MediaUsage, PendingUsageStatus, Publishe import lib.{BadInputException, UsageConfig, WithLogMarker} import play.api.libs.json._ import rx.lang.scala.Observable +import software.amazon.awssdk.enhanced.dynamodb.model.QueryConditional +import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, Key, TableMetadata, TableSchema} +import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import scala.jdk.CollectionConverters._ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future +import scala.jdk.CollectionConverters.IterableHasAsScala -class UsageTable(config: UsageConfig) extends DynamoDB(config, config.usageRecordTable) with GridLogging { +class UsageTable(config: UsageConfig) extends GridLogging { val hashKeyName = "grouping" val rangeKeyName = "usage_id" val imageIndexName = "media_id" + lazy val client2: DynamoDbClient = config.withAWSCredentialsV2(DynamoDbClient.builder()).build() + lazy val dynamo2: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client2).build() + lazy val tableSchema = TableSchema.documentSchemaBuilder() + .addIndexPartitionKey(TableMetadata.primaryIndexName(), hashKeyName, AttributeValueType.S) + .addIndexSortKey(TableMetadata.primaryIndexName(), rangeKeyName, AttributeValueType.S) + .addIndexPartitionKey( + "CustomIndex", + imageIndexName, + AttributeValueType.S + ) + .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) + .build() + lazy val table = dynamo2.table(config.usageRecordTable, tableSchema) + def queryByUsageId(id: String): Future[Option[MediaUsage]] = Future { UsageTableFullKey.build(id).flatMap((tableFullKey: UsageTableFullKey) => { - val keyAttribute: KeyAttribute = new KeyAttribute(hashKeyName, tableFullKey.hashKey) - val rangeKeyCondition: RangeKeyCondition = new RangeKeyCondition(rangeKeyName).eq(tableFullKey.rangeKey) - - val queryResult = table.query(keyAttribute, rangeKeyCondition) - queryResult.asScala.map(ItemToMediaUsage.transform).headOption + val key = Key.builder() + .partitionValue(tableFullKey.hashKey) + .sortValue(tableFullKey.rangeKey) + .build() + val queryResult = table.query(QueryConditional.keyEqualTo(key)) + queryResult.items().asScala.map(ItemToMediaUsage.transform).headOption }) } From 63ab0711037878fa6a1a32277b04a9ffb7837328 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 6 Jul 2026 16:53:26 +0100 Subject: [PATCH 093/373] Add the query by image id --- usage/app/model/UsageTable.scala | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/usage/app/model/UsageTable.scala b/usage/app/model/UsageTable.scala index b11b2c5c92a..3b1eb98364e 100644 --- a/usage/app/model/UsageTable.scala +++ b/usage/app/model/UsageTable.scala @@ -16,7 +16,7 @@ import software.amazon.awssdk.services.dynamodb.DynamoDbClient import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -import scala.jdk.CollectionConverters.IterableHasAsScala +import scala.jdk.CollectionConverters.{IterableHasAsScala, IteratorHasAsScala} class UsageTable(config: UsageConfig) extends GridLogging { @@ -30,7 +30,7 @@ class UsageTable(config: UsageConfig) extends GridLogging { .addIndexPartitionKey(TableMetadata.primaryIndexName(), hashKeyName, AttributeValueType.S) .addIndexSortKey(TableMetadata.primaryIndexName(), rangeKeyName, AttributeValueType.S) .addIndexPartitionKey( - "CustomIndex", + "MediaIdIndex", imageIndexName, AttributeValueType.S ) @@ -56,11 +56,18 @@ class UsageTable(config: UsageConfig) extends GridLogging { throw new BadInputException("Empty string received for image id") logger.info(logMarkerWithId, s"Querying usages table for $id") - val imageIndex = table.getIndex(imageIndexName) - val keyAttribute = new KeyAttribute(imageIndexName, id) - val queryResult = imageIndex.query(keyAttribute) - val unsortedUsages = queryResult.asScala.map(ItemToMediaUsage.transform).toList + val key = Key.builder() + .partitionValue(id) + .build() + + val queryResult = table.index("MediaIdIndex").query(QueryConditional.keyEqualTo(key)) + + val unsortedUsages = queryResult.iterator() + .asScala + .flatMap(_.items().asScala) + .map(ItemToMediaUsage.transform) + .toList logger.info(logMarkerWithId, s"Query of usages table for $id found ${unsortedUsages.size} results") From 1360bdd443c96258e78b0d338e31d55643178ee4 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 6 Jul 2026 18:46:10 +0100 Subject: [PATCH 094/373] Add an update builder --- .../gu/mediaservice/lib/aws/DynamoDB.scala | 10 ++ usage/app/model/UsageRecord.scala | 93 ++++++++++++++++++- usage/app/model/UsageTable.scala | 73 ++++++++------- 3 files changed, 142 insertions(+), 34 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index e6fc62f6961..1b188e91b03 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -342,6 +342,7 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti def asJsObject(outcome: UpdateItemOutcome): JsObject = Option(outcome.getItem) map asJsObject getOrElse Json.obj() + // FIXME: Dynamo accepts `null`, but not `""`. This is a well documented issue // around the community. This guard keeps the introduction of `null` fairly // fenced in this Dynamo play area. `null` is continual and big annoyance with AWS libs. @@ -447,4 +448,13 @@ object DynamoDB { def generateExpression(baseExpression: String, lastModifiedKey: Option[String]) = { lastModifiedKey.fold(baseExpression)(lastModifiedKey => s"$baseExpression SET $lastModifiedKey = :$lastModifiedKey") } + + + def avToAny(av: AttributeValueV2): Any = + if (av.s() != null) av.s() + else if (av.n() != null) av.n() + else if (av.bool() != null) av.bool() + else if (av.m() != null) av.m().asScala.view.mapValues(avToAny).toMap + else if (av.l() != null) av.l().asScala.map(avToAny) + else null } diff --git a/usage/app/model/UsageRecord.scala b/usage/app/model/UsageRecord.scala index 69e286ea78e..b89919ff619 100644 --- a/usage/app/model/UsageRecord.scala +++ b/usage/app/model/UsageRecord.scala @@ -1,17 +1,20 @@ package model +import software.amazon.awssdk.services.dynamodb.model.AttributeValue +import scala.jdk.CollectionConverters._ import com.amazonaws.services.dynamodbv2.xspec.{ExpressionSpecBuilder, UpdateItemExpressionSpec} import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder.{M, N, S} import com.gu.mediaservice.model.usage._ -import scala.jdk.CollectionConverters._ import org.joda.time.DateTime +import software.amazon.awssdk.enhanced.dynamodb.Expression sealed trait DateRemovedOperation case object ClearDateRemoved extends DateRemovedOperation case object LeaveDateRemovedUntouched extends DateRemovedOperation case class SetDateRemoved(dateRemoved: DateTime) extends DateRemovedOperation + case class UsageRecord( hashKey: String, rangeKey: String, @@ -52,6 +55,94 @@ case class UsageRecord( ).flatten.foreach(specBuilder.addUpdate) specBuilder.buildForUpdate } + + def toAttributeValueMap(m: Map[String, Any]): java.util.Map[String, AttributeValue] = + m.map { case (k, v) => + k -> toAttr(v) + }.asJava + def toAttr(v: Any): AttributeValue = v match { + case s: String => AttributeValue.builder().s(s).build() + case n: Long => AttributeValue.builder().n(n.toString).build() + case n: Int => AttributeValue.builder().n(n.toString).build() + case b: Boolean => AttributeValue.builder().bool(b).build() + case null => AttributeValue.builder().nul(true).build() + case other => throw new IllegalArgumentException(s"Unsupported type: $other") + } + def toExpression: Expression = { + val setOps = scala.collection.mutable.ListBuffer.empty[(String, AttributeValue)] + val removeOps = scala.collection.mutable.ListBuffer.empty[String] + val names = scala.collection.mutable.Map.empty[String, String] + + def setS(field: String, value: String): Unit = { + val name = s"#$field" + val valueKey = s":$field" + names += name -> field + setOps += valueKey -> AttributeValue.builder().s(value).build() + } + + def setN(field: String, value: Long): Unit = { + val name = s"#$field" + val valueKey = s":$field" + names += name -> field + setOps += valueKey -> AttributeValue.builder().n(value.toString).build() + } + + def setM(field: String, value: java.util.Map[String, AttributeValue]): Unit = { + val name = s"#$field" + val valueKey = s":$field" + names += name -> field + setOps += valueKey -> AttributeValue.builder().m(value).build() + } + + mediaId.filter(_.nonEmpty).foreach(setS("media_id", _)) + + usageType.foreach(t => setS("usage_type", t.toString)) + + mediaType.filter(_.nonEmpty).foreach(setS("media_type", _)) + + lastModified.foreach(dt => setN("last_modified", dt.getMillis)) + + usageStatus.filter(_.nonEmpty).foreach(setS("usage_status", _)) + + printUsageMetadata.map(p => p.toMap).foreach(p => setM("print_metadata", toAttributeValueMap(p))) + digitalUsageMetadata.foreach(m => setM("digital_metadata", toAttributeValueMap(m.toMap))) + syndicationUsageMetadata.foreach(m => setM("syndication_metadata", toAttributeValueMap(m.toMap))) + frontUsageMetadata.foreach(m => setM("front_metadata", toAttributeValueMap(m.toMap))) + downloadUsageMetadata.foreach(m => setM("download_metadata", toAttributeValueMap(m.toMap))) + childUsageMetadata.foreach(m => setM("child_metadata", toAttributeValueMap(m.toMap))) + + dateAdded.foreach(dt => setN("date_added", dt.getMillis)) + + dateRemovedOperation match { + case ClearDateRemoved => + removeOps += "date_removed" + + case LeaveDateRemovedUntouched => + () + + case SetDateRemoved(dt) => + setN("date_removed", dt.getMillis) + } + + val setExpr = + if (setOps.nonEmpty) + "SET " + setOps.map { case (v, _) => s"#${v.drop(1)} = $v" }.mkString(", ") + else "" + + val removeExpr = + if (removeOps.nonEmpty) + "REMOVE " + removeOps.map(f => s"#$f").mkString(", ") + else "" + + val expression = + Seq(setExpr, removeExpr).filter(_.nonEmpty).mkString(" ") + + Expression.builder() + .expression(expression) + .expressionValues(setOps.toMap.asJava) + .expressionNames(names.asJava) + .build() + } } object UsageRecord { diff --git a/usage/app/model/UsageTable.scala b/usage/app/model/UsageTable.scala index 3b1eb98364e..350a28106ee 100644 --- a/usage/app/model/UsageTable.scala +++ b/usage/app/model/UsageTable.scala @@ -1,22 +1,21 @@ package model -import com.amazonaws.services.dynamodbv2.document.spec.{DeleteItemSpec, QuerySpec, UpdateItemSpec} -import com.amazonaws.services.dynamodbv2.document.{DeleteItemOutcome, KeyAttribute, RangeKeyCondition} -import com.amazonaws.services.dynamodbv2.model.ReturnValue -import com.gu.mediaservice.lib.aws.DynamoDB +import com.gu.mediaservice.lib.aws.DynamoDB.avToAny import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.lib.usage.ItemToMediaUsage import com.gu.mediaservice.model.usage.{MediaUsage, PendingUsageStatus, PublishedUsageStatus, UsageTableFullKey} import lib.{BadInputException, UsageConfig, WithLogMarker} import play.api.libs.json._ import rx.lang.scala.Observable -import software.amazon.awssdk.enhanced.dynamodb.model.QueryConditional +import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument +import software.amazon.awssdk.enhanced.dynamodb.model.{DeleteItemEnhancedRequest, QueryConditional, QueryEnhancedRequest, UpdateItemEnhancedRequest} import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverterProvider, AttributeValueType, DynamoDbEnhancedClient, Key, TableMetadata, TableSchema} import software.amazon.awssdk.services.dynamodb.DynamoDbClient +import software.amazon.awssdk.services.dynamodb.model.{AttributeValue, ReturnValue, UpdateItemRequest} import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -import scala.jdk.CollectionConverters.{IterableHasAsScala, IteratorHasAsScala} +import scala.jdk.CollectionConverters.{IterableHasAsScala, IteratorHasAsScala, MapHasAsJava, MapHasAsScala} class UsageTable(config: UsageConfig) extends GridLogging { @@ -24,6 +23,7 @@ class UsageTable(config: UsageConfig) extends GridLogging { val rangeKeyName = "usage_id" val imageIndexName = "media_id" + lazy val client2: DynamoDbClient = config.withAWSCredentialsV2(DynamoDbClient.builder()).build() lazy val dynamo2: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client2).build() lazy val tableSchema = TableSchema.documentSchemaBuilder() @@ -111,13 +111,17 @@ class UsageTable(config: UsageConfig) extends GridLogging { val grouping = usageGroup.grouping logger.info(logMarker, s"Querying table for $grouping") - val queryResult = table.query( - new QuerySpec() - .withConsistentRead(true) - .withHashKey(new KeyAttribute("grouping", grouping)) - ) - - val usages = queryResult.asScala + val key = Key.builder() + .partitionValue(grouping) + .build() + val request = + QueryEnhancedRequest.builder() + .queryConditional(QueryConditional.keyEqualTo(key)) + .consistentRead(true) + .build() + val queryResult = table.query(request) + + val usages = queryResult.items().asScala .map(ItemToMediaUsage.transform) .toSet @@ -136,36 +140,39 @@ class UsageTable(config: UsageConfig) extends GridLogging { def markAsRemoved(mediaUsage: MediaUsage)(implicit logMarker: LogMarker): Observable[JsObject] = upsertFromRecord(UsageRecord.buildMarkAsRemovedRecord(mediaUsage)) - def deleteRecord(mediaUsage: MediaUsage)(implicit logMarker: LogMarker): DeleteItemOutcome = { + def deleteRecord(mediaUsage: MediaUsage)(implicit logMarker: LogMarker): EnhancedDocument = { logger.info(logMarker, s"deleting usage ${mediaUsage.usageId} for media id ${mediaUsage.mediaId}") - val deleteSpec = new DeleteItemSpec() - .withPrimaryKey( - hashKeyName, mediaUsage.grouping, - rangeKeyName, mediaUsage.usageId.toString - ) + val key = Key.builder() + .partitionValue(mediaUsage.grouping) + .sortValue(mediaUsage.usageId.toString) + .build() - table.deleteItem(deleteSpec) + table.deleteItem(DeleteItemEnhancedRequest.builder().key(key).build()) } def upsertFromRecord(record: UsageRecord)(implicit logMarker: LogMarker): Observable[JsObject] = Observable.from(Future { + val key = Map( + hashKeyName -> AttributeValue.builder().s(record.hashKey).build(), + rangeKeyName -> AttributeValue.builder().s(record.rangeKey).build() + ).asJava + + val request = UpdateItemRequest.builder() + .tableName(table.tableName()) + .key(key) + .updateExpression(record.toExpression.expression()) + .expressionAttributeNames(record.toExpression.expressionNames()) + .expressionAttributeValues(record.toExpression.expressionValues()) + .returnValues(ReturnValue.ALL_NEW) + .build() - val updateSpec = new UpdateItemSpec() - .withPrimaryKey( - hashKeyName, - record.hashKey, - rangeKeyName, - record.rangeKey - ) - .withExpressionSpec(record.toXSpec) - .withReturnValues(ReturnValue.ALL_NEW) - - table.updateItem(updateSpec) - + client2.updateItem(request) }) .onErrorResumeNext(e => { logger.error(logMarker, s"Dynamo update fail for $record!", e) Observable.error(e) }) - .map(asJsObject) + .map(response => { + Json.toJson(response.attributes().asScala.view.mapValues(avToAny).toMap).as[JsObject] + }) } From b5f25af31965fd22a63521df9a8cb16a75add926 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 7 Jul 2026 09:47:52 +0100 Subject: [PATCH 095/373] Bump @guardian/cql to 1.8.5 --- kahuna/package-lock.json | 8 ++++---- kahuna/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index e2b34f156a6..97ce16b3d12 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "@babel/polyfill": "7.8.7", - "@guardian/cql": "1.8.4", + "@guardian/cql": "1.8.5", "@guardian/user-telemetry-client": "1.1.0", "@sentry/browser": "6.11.0", "@sentry/integrations": "6.11.0", @@ -2243,9 +2243,9 @@ "license": "MIT" }, "node_modules/@guardian/cql": { - "version": "1.8.4", - "resolved": "https://registry.npmjs.org/@guardian/cql/-/cql-1.8.4.tgz", - "integrity": "sha512-cDhPd63XcAs6Xj4Rerstjbo9uO0aJL8DEyrZ6FX2tmlpm0WY6vVzQ6aKmXpej9hw9wka8jHE02ZvoHY0yfhbdQ==", + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@guardian/cql/-/cql-1.8.5.tgz", + "integrity": "sha512-VU8UPeFeoMUpEc4wMHkXo65nEXGWmxYu5CofKXNrOQvtGSwTE3mYYdwZT9IG22ILpeWhlMF4Wzmo1EnFfwffxA==", "dependencies": { "@floating-ui/dom": "^1.6.11", "preact": "^10.25.4", diff --git a/kahuna/package.json b/kahuna/package.json index ef05334ea74..9897b7cde67 100644 --- a/kahuna/package.json +++ b/kahuna/package.json @@ -1,7 +1,7 @@ { "dependencies": { "@babel/polyfill": "7.8.7", - "@guardian/cql": "1.8.4", + "@guardian/cql": "1.8.5", "@guardian/user-telemetry-client": "1.1.0", "@sentry/browser": "6.11.0", "@sentry/integrations": "6.11.0", From 8fb195ed38a81b14e6fe79c1329f6d85d15f42a3 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 7 Jul 2026 13:46:15 +0100 Subject: [PATCH 096/373] Add the update method --- .../gu/mediaservice/lib/aws/DynamoDB.scala | 34 +++++++++---------- usage/app/model/UsageTable.scala | 8 +++-- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index 1b188e91b03..c99370c08a4 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -7,7 +7,7 @@ import com.amazonaws.services.dynamodbv2.document.utils.ValueMap import com.amazonaws.services.dynamodbv2.document.{DynamoDB => AwsDynamoDB, _} import com.amazonaws.services.dynamodbv2.model.{AttributeValue, DeleteItemRequest, KeysAndAttributes, ReturnValue} import com.amazonaws.services.dynamodbv2.{AmazonDynamoDBAsync, AmazonDynamoDBAsyncClientBuilder} -import com.gu.mediaservice.lib.aws.DynamoDB.{deleteExpr, setExpr} +import com.gu.mediaservice.lib.aws.DynamoDB.{deleteExpr, jsonWithNullAsEmptyString, setExpr} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import org.joda.time.DateTime @@ -17,7 +17,7 @@ import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import software.amazon.awssdk.enhanced.dynamodb.model import software.amazon.awssdk.enhanced.dynamodb.model.{BatchGetItemEnhancedRequest, ReadBatch} import software.amazon.awssdk.services.dynamodb.DynamoDbClient -import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, ReturnValue => ReturnValueV2, QueryRequest => QueryRequestV2} +import software.amazon.awssdk.services.dynamodb.model.{UpdateItemRequest, AttributeValue => AttributeValueV2, QueryRequest => QueryRequestV2, ReturnValue => ReturnValueV2} import scala.annotation.tailrec import scala.concurrent.{ExecutionContext, Future} @@ -343,21 +343,6 @@ class DynamoDB[T](config: CommonConfig, tableName: String, lastModifiedKey: Opti Option(outcome.getItem) map asJsObject getOrElse Json.obj() - // FIXME: Dynamo accepts `null`, but not `""`. This is a well documented issue - // around the community. This guard keeps the introduction of `null` fairly - // fenced in this Dynamo play area. `null` is continual and big annoyance with AWS libs. - // see: https://forums.aws.amazon.com/message.jspa?messageID=389032 - // see: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html - def mapJsValue(jsValue: JsValue)(f: JsValue => JsValue): JsValue = jsValue match { - case JsObject(items) => JsObject(items.map{ case (k, v) => k -> mapJsValue(v)(f) }) - case JsArray(items) => JsArray(items.map(f)) - case value => f(value) - } - - def jsonWithNullAsEmptyString(jsValue: JsValue): JsValue = mapJsValue(jsValue) { - case JsNull => JsString("") - case value => value - } } @@ -449,6 +434,21 @@ object DynamoDB { lastModifiedKey.fold(baseExpression)(lastModifiedKey => s"$baseExpression SET $lastModifiedKey = :$lastModifiedKey") } + // FIXME: Dynamo accepts `null`, but not `""`. This is a well documented issue + // around the community. This guard keeps the introduction of `null` fairly + // fenced in this Dynamo play area. `null` is continual and big annoyance with AWS libs. + // see: https://forums.aws.amazon.com/message.jspa?messageID=389032 + // see: http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html + def mapJsValue(jsValue: JsValue)(f: JsValue => JsValue): JsValue = jsValue match { + case JsObject(items) => JsObject(items.map{ case (k, v) => k -> mapJsValue(v)(f) }) + case JsArray(items) => JsArray(items.map(f)) + case value => f(value) + } + def jsonWithNullAsEmptyString(jsValue: JsValue): JsValue = mapJsValue(jsValue) { + case JsNull => JsString("") + case value => value + } + def avToAny(av: AttributeValueV2): Any = if (av.s() != null) av.s() diff --git a/usage/app/model/UsageTable.scala b/usage/app/model/UsageTable.scala index 350a28106ee..84292ce87f3 100644 --- a/usage/app/model/UsageTable.scala +++ b/usage/app/model/UsageTable.scala @@ -1,6 +1,6 @@ package model -import com.gu.mediaservice.lib.aws.DynamoDB.avToAny +import com.gu.mediaservice.lib.aws.DynamoDB.{avToAny, jsonWithNullAsEmptyString} import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.lib.usage.ItemToMediaUsage import com.gu.mediaservice.model.usage.{MediaUsage, PendingUsageStatus, PublishedUsageStatus, UsageTableFullKey} @@ -167,12 +167,14 @@ class UsageTable(config: UsageConfig) extends GridLogging { .build() client2.updateItem(request) + }) .onErrorResumeNext(e => { logger.error(logMarker, s"Dynamo update fail for $record!", e) Observable.error(e) }) - .map(response => { - Json.toJson(response.attributes().asScala.view.mapValues(avToAny).toMap).as[JsObject] + .map(updateResponse => { + val doc = EnhancedDocument.fromAttributeValueMap(updateResponse.attributes()) + jsonWithNullAsEmptyString(Json.parse(doc.toJson)).as[JsObject] }) } From 05f2dab5762784abd31d5e2755031213e38fa3d9 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 7 Jul 2026 14:03:00 +0100 Subject: [PATCH 097/373] Remove unused dynamo method --- .../scala/com/gu/mediaservice/lib/aws/DynamoDB.scala | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala index c99370c08a4..a5cfec1693b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/DynamoDB.scala @@ -448,13 +448,5 @@ object DynamoDB { case JsNull => JsString("") case value => value } - - - def avToAny(av: AttributeValueV2): Any = - if (av.s() != null) av.s() - else if (av.n() != null) av.n() - else if (av.bool() != null) av.bool() - else if (av.m() != null) av.m().asScala.view.mapValues(avToAny).toMap - else if (av.l() != null) av.l().asScala.map(avToAny) - else null + } From 3c56bd083b817e32262e60688b092c2ec61d5ba9 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 7 Jul 2026 14:26:07 +0100 Subject: [PATCH 098/373] Rename from v2 --- usage/app/model/UsageTable.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/usage/app/model/UsageTable.scala b/usage/app/model/UsageTable.scala index 84292ce87f3..b2a124fc1e9 100644 --- a/usage/app/model/UsageTable.scala +++ b/usage/app/model/UsageTable.scala @@ -24,8 +24,8 @@ class UsageTable(config: UsageConfig) extends GridLogging { val imageIndexName = "media_id" - lazy val client2: DynamoDbClient = config.withAWSCredentialsV2(DynamoDbClient.builder()).build() - lazy val dynamo2: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client2).build() + lazy val client: DynamoDbClient = config.withAWSCredentialsV2(DynamoDbClient.builder()).build() + lazy val dynamo: DynamoDbEnhancedClient = DynamoDbEnhancedClient.builder().dynamoDbClient(client).build() lazy val tableSchema = TableSchema.documentSchemaBuilder() .addIndexPartitionKey(TableMetadata.primaryIndexName(), hashKeyName, AttributeValueType.S) .addIndexSortKey(TableMetadata.primaryIndexName(), rangeKeyName, AttributeValueType.S) @@ -36,7 +36,7 @@ class UsageTable(config: UsageConfig) extends GridLogging { ) .attributeConverterProviders(AttributeConverterProvider.defaultProvider()) .build() - lazy val table = dynamo2.table(config.usageRecordTable, tableSchema) + lazy val table = dynamo.table(config.usageRecordTable, tableSchema) def queryByUsageId(id: String): Future[Option[MediaUsage]] = Future { UsageTableFullKey.build(id).flatMap((tableFullKey: UsageTableFullKey) => { @@ -166,7 +166,7 @@ class UsageTable(config: UsageConfig) extends GridLogging { .returnValues(ReturnValue.ALL_NEW) .build() - client2.updateItem(request) + client.updateItem(request) }) .onErrorResumeNext(e => { From 184beca554f58f407f03c23c0f8a1f9315887eba Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 7 Jul 2026 15:02:06 +0100 Subject: [PATCH 099/373] 1.8.5? ancient history. 1.8.6, now we're talkin --- kahuna/package-lock.json | 8 ++++---- kahuna/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index 97ce16b3d12..4d5a59a5129 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "@babel/polyfill": "7.8.7", - "@guardian/cql": "1.8.5", + "@guardian/cql": "1.8.6", "@guardian/user-telemetry-client": "1.1.0", "@sentry/browser": "6.11.0", "@sentry/integrations": "6.11.0", @@ -2243,9 +2243,9 @@ "license": "MIT" }, "node_modules/@guardian/cql": { - "version": "1.8.5", - "resolved": "https://registry.npmjs.org/@guardian/cql/-/cql-1.8.5.tgz", - "integrity": "sha512-VU8UPeFeoMUpEc4wMHkXo65nEXGWmxYu5CofKXNrOQvtGSwTE3mYYdwZT9IG22ILpeWhlMF4Wzmo1EnFfwffxA==", + "version": "1.8.6", + "resolved": "https://registry.npmjs.org/@guardian/cql/-/cql-1.8.6.tgz", + "integrity": "sha512-R8UoHaSiUVCOfgbFOCBxfPUQmUNoasd/Rj0o1KF7ikO0g+amsX3Dro2mabuzeuszMjAT4kMD809kuS+w+igvXg==", "dependencies": { "@floating-ui/dom": "^1.6.11", "preact": "^10.25.4", diff --git a/kahuna/package.json b/kahuna/package.json index 9897b7cde67..0fc91cd538a 100644 --- a/kahuna/package.json +++ b/kahuna/package.json @@ -1,7 +1,7 @@ { "dependencies": { "@babel/polyfill": "7.8.7", - "@guardian/cql": "1.8.5", + "@guardian/cql": "1.8.6", "@guardian/user-telemetry-client": "1.1.0", "@sentry/browser": "6.11.0", "@sentry/integrations": "6.11.0", From 4a9f49bd67ef767599983e4dcdbb26c0874e96b5 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 7 Jul 2026 16:50:42 +0100 Subject: [PATCH 100/373] Add tests for the expression builder --- usage/app/model/UsageRecord.scala | 4 +- usage/app/model/UsageTable.scala | 2 +- usage/test/model/UsageRecordTest.scala | 203 +++++++++++++++++++++++++ 3 files changed, 206 insertions(+), 3 deletions(-) create mode 100644 usage/test/model/UsageRecordTest.scala diff --git a/usage/app/model/UsageRecord.scala b/usage/app/model/UsageRecord.scala index b89919ff619..e5cb9d527d7 100644 --- a/usage/app/model/UsageRecord.scala +++ b/usage/app/model/UsageRecord.scala @@ -1,11 +1,11 @@ package model import software.amazon.awssdk.services.dynamodb.model.AttributeValue + import scala.jdk.CollectionConverters._ -import com.amazonaws.services.dynamodbv2.xspec.{ExpressionSpecBuilder, UpdateItemExpressionSpec} +import com.amazonaws.services.dynamodbv2.xspec.{ExpressionSpecBuilder, UpdateAction, UpdateItemExpressionSpec} import com.amazonaws.services.dynamodbv2.xspec.ExpressionSpecBuilder.{M, N, S} import com.gu.mediaservice.model.usage._ - import org.joda.time.DateTime import software.amazon.awssdk.enhanced.dynamodb.Expression diff --git a/usage/app/model/UsageTable.scala b/usage/app/model/UsageTable.scala index b2a124fc1e9..b075eb2dd20 100644 --- a/usage/app/model/UsageTable.scala +++ b/usage/app/model/UsageTable.scala @@ -1,6 +1,6 @@ package model -import com.gu.mediaservice.lib.aws.DynamoDB.{avToAny, jsonWithNullAsEmptyString} +import com.gu.mediaservice.lib.aws.DynamoDB.jsonWithNullAsEmptyString import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.lib.usage.ItemToMediaUsage import com.gu.mediaservice.model.usage.{MediaUsage, PendingUsageStatus, PublishedUsageStatus, UsageTableFullKey} diff --git a/usage/test/model/UsageRecordTest.scala b/usage/test/model/UsageRecordTest.scala new file mode 100644 index 00000000000..cefd06f0a6f --- /dev/null +++ b/usage/test/model/UsageRecordTest.scala @@ -0,0 +1,203 @@ +package model + +import com.gu.mediaservice.model.usage.{DigitalUsageMetadata, PrintUsage, PrintUsageMetadata} +import org.joda.time.DateTime +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +import java.net.URI +import scala.jdk.CollectionConverters.{CollectionHasAsScala, MapHasAsScala} +class UsageRecordTest extends AnyFunSpec with Matchers { + + def getMapFromValues(rawValues: Iterable[AnyRef]) = { + rawValues.collectFirst { + case javaMap: java.util.Map[_, _] => + javaMap.asInstanceOf[java.util.Map[String, Any]].asScala.toMap + }.getOrElse(fail("Could not find the digital_metadata map in the value map")) + } + + describe("updateExpressions") { + it("should handle an empty record with default/None values correctly") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched + ) + val spec = record.toXSpec + val names = Option(spec.getNameMap).map(_.values().asScala) + names shouldBe None + + val values = Option(spec.getValueMap).map(_.values().asScala) + + values shouldBe None + + val updateExpression = Option(spec.getUpdateExpression) + updateExpression shouldBe Some("") + } + + it("should include standard String and UsageType updates when present") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + mediaId = Some("media-789"), + usageType = Some(PrintUsage), + mediaType = Some("image"), + usageStatus = Some("active") + ) + + val spec = record.toXSpec + + val names = spec.getNameMap.values().asScala + + names.size shouldBe 4 + names should contain ("media_id") + names should contain ("usage_type") + names should contain ("media_type") + names should contain ("usage_status") + + val values = spec.getValueMap.values().asScala + + values.size shouldBe 4 + values should contain ("media-789") + values should contain (PrintUsage.toString) + values should contain ("image") + values should contain ("active") + + spec.getUpdateExpression shouldEqual "SET #0 = :0, #1 = :1, #2 = :2, #3 = :3" + + } + } + + describe("dateRemovedOperation") { + it("should generate a set block when SetDateRemoved is specified") { + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = SetDateRemoved(DateTime.now()) + ) + + val spec = record.toXSpec + val names = spec.getNameMap.values().asScala + names.size shouldBe 1 + names should contain ("date_removed") + + spec.getUpdateExpression shouldEqual "SET #0 = :0" + } + + it("should generate a REMOVE block when ClearDateRemoved is specified") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = ClearDateRemoved + ) + + val spec = record.toXSpec + + val names = spec.getNameMap.values().asScala + names.size shouldBe 1 + names should contain ("date_removed") + spec.getUpdateExpression shouldEqual "REMOVE #0" + } + } + + describe("PrintUsageMetadata") { + it("should handle PrintUsageMetadata") { + val targetDate = DateTime.parse("2026-07-07T12:00:00Z") + val metadata = PrintUsageMetadata( + sectionName = "News", + issueDate = targetDate, + pageNumber = 5, + storyName = "Scala Breakthrough", + publicationCode = "PUB1", + publicationName = "The Tech Daily", + sectionCode = "SEC-A", + layoutId = Some(12345L), + edition = Some(2), + size = None, + orderedBy = Some("Alice"), + notes = Some("Urgent print"), + source = Some("Internal") + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + printUsageMetadata = Some(metadata) + ) + val spec = record.toXSpec + val rawValues = spec.getValueMap.values().asScala + + val printMetadataMap: Map[String, Any] = getMapFromValues(rawValues) + + printMetadataMap("sectionName") shouldBe "News" + printMetadataMap("issueDate") shouldBe "2026-07-07T12:00:00.000Z" + printMetadataMap("pageNumber") shouldBe 5 + printMetadataMap("storyName") shouldBe "Scala Breakthrough" + printMetadataMap("publicationCode") shouldBe "PUB1" + printMetadataMap("publicationName") shouldBe "The Tech Daily" + printMetadataMap("sectionCode") shouldBe "SEC-A" + printMetadataMap("layoutId") shouldBe 12345 + printMetadataMap("edition") shouldBe 2 + printMetadataMap("orderedBy") shouldBe "Alice" + printMetadataMap("notes") shouldBe "Urgent print" + printMetadataMap("source") shouldBe "Internal" + + printMetadataMap should have size 12 + } + + describe("UsageRecord Update Expressions - digitalUsageMetadata") { + + it("should correctly compile DigitalUsageMetadata with all optional fields present") { + val metadata = DigitalUsageMetadata( + webUrl = new URI("https://www.theguardian.com/tech"), + webTitle = "Scala 3 adoption grows", + sectionId = "technology", + composerUrl = Some(new URI("https://composer.internal/123")) + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + digitalUsageMetadata = Some(metadata) + ) + + val spec = record.toXSpec + val rawValues = spec.getValueMap.values().asScala + val digitalMetadataMap = getMapFromValues(rawValues) + + digitalMetadataMap shouldBe Map( + "webUrl" -> "https://www.theguardian.com/tech", + "webTitle" -> "Scala 3 adoption grows", + "sectionId" -> "technology", + "composerUrl" -> "https://composer.internal/123" + ) + } + + it("should fall back to placeholder when webTitle is empty and omit composerUrl when None") { + val metadata = DigitalUsageMetadata( + webUrl = new URI("https://www.theguardian.com/media"), + webTitle = "", + sectionId = "media", + composerUrl = None + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + digitalUsageMetadata = Some(metadata) + ) + + val spec = record.toXSpec + val rawValues = spec.getValueMap.values().asScala + val digitalMetadataMap = getMapFromValues(rawValues) + digitalMetadataMap("webTitle") shouldBe "No title given" + digitalMetadataMap.contains("composerUrl") shouldBe false + } + } + } +} From 4857a491d77a84bd6fb6854f63f38fd4d75fed6f Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 7 Jul 2026 17:49:22 +0100 Subject: [PATCH 101/373] Add tests for the toExpression --- usage/test/model/UsageRecordTest.scala | 258 ++++++++++++++++++++++--- 1 file changed, 234 insertions(+), 24 deletions(-) diff --git a/usage/test/model/UsageRecordTest.scala b/usage/test/model/UsageRecordTest.scala index cefd06f0a6f..df24334f4e2 100644 --- a/usage/test/model/UsageRecordTest.scala +++ b/usage/test/model/UsageRecordTest.scala @@ -1,6 +1,7 @@ package model import com.gu.mediaservice.model.usage.{DigitalUsageMetadata, PrintUsage, PrintUsageMetadata} +import software.amazon.awssdk.services.dynamodb.model.AttributeValue import org.joda.time.DateTime import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers @@ -13,10 +14,10 @@ class UsageRecordTest extends AnyFunSpec with Matchers { rawValues.collectFirst { case javaMap: java.util.Map[_, _] => javaMap.asInstanceOf[java.util.Map[String, Any]].asScala.toMap - }.getOrElse(fail("Could not find the digital_metadata map in the value map")) + }.getOrElse(fail("Could not find the value map")) } - describe("updateExpressions") { + describe("toXSpec") { it("should handle an empty record with default/None values correctly") { val record = UsageRecord( hashKey = "hash-123", @@ -100,6 +101,23 @@ class UsageRecordTest extends AnyFunSpec with Matchers { names should contain ("date_removed") spec.getUpdateExpression shouldEqual "REMOVE #0" } + + it("should handle a mix of remove and set updates when the relevant fields are set") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = ClearDateRemoved, + mediaId = Some("media-789") + ) + + val spec = record.toXSpec + + val names = spec.getNameMap.values().asScala + names.size shouldBe 2 + names should contain ("date_removed") + names should contain ("media_id") + spec.getUpdateExpression shouldEqual "SET #0 = :0 REMOVE #1" + } } describe("PrintUsageMetadata") { @@ -132,23 +150,214 @@ class UsageRecordTest extends AnyFunSpec with Matchers { val printMetadataMap: Map[String, Any] = getMapFromValues(rawValues) - printMetadataMap("sectionName") shouldBe "News" - printMetadataMap("issueDate") shouldBe "2026-07-07T12:00:00.000Z" - printMetadataMap("pageNumber") shouldBe 5 - printMetadataMap("storyName") shouldBe "Scala Breakthrough" + printMetadataMap("sectionName") shouldBe "News" + printMetadataMap("issueDate") shouldBe "2026-07-07T12:00:00.000Z" + printMetadataMap("pageNumber") shouldBe 5 + printMetadataMap("storyName") shouldBe "Scala Breakthrough" printMetadataMap("publicationCode") shouldBe "PUB1" printMetadataMap("publicationName") shouldBe "The Tech Daily" - printMetadataMap("sectionCode") shouldBe "SEC-A" - printMetadataMap("layoutId") shouldBe 12345 - printMetadataMap("edition") shouldBe 2 - printMetadataMap("orderedBy") shouldBe "Alice" - printMetadataMap("notes") shouldBe "Urgent print" - printMetadataMap("source") shouldBe "Internal" + printMetadataMap("sectionCode") shouldBe "SEC-A" + printMetadataMap("layoutId") shouldBe 12345 + printMetadataMap("edition") shouldBe 2 + printMetadataMap("orderedBy") shouldBe "Alice" + printMetadataMap("notes") shouldBe "Urgent print" + printMetadataMap("source") shouldBe "Internal" printMetadataMap should have size 12 } + } + + describe("UsageRecord Update Expressions - digitalUsageMetadata") { + + it("should correctly compile DigitalUsageMetadata with all optional fields present") { + val metadata = DigitalUsageMetadata( + webUrl = new URI("https://www.theguardian.com/tech"), + webTitle = "Scala 3 adoption grows", + sectionId = "technology", + composerUrl = Some(new URI("https://composer.internal/123")) + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + digitalUsageMetadata = Some(metadata) + ) + + val spec = record.toXSpec + val rawValues = spec.getValueMap.values().asScala + val digitalMetadataMap = getMapFromValues(rawValues) + + digitalMetadataMap shouldBe Map( + "webUrl" -> "https://www.theguardian.com/tech", + "webTitle" -> "Scala 3 adoption grows", + "sectionId" -> "technology", + "composerUrl" -> "https://composer.internal/123" + ) + } + + it("should fall back to placeholder when webTitle is empty and omit composerUrl when None") { + val metadata = DigitalUsageMetadata( + webUrl = new URI("https://www.theguardian.com/media"), + webTitle = "", + sectionId = "media", + composerUrl = None + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + digitalUsageMetadata = Some(metadata) + ) + + val spec = record.toXSpec + val rawValues = spec.getValueMap.values().asScala + val digitalMetadataMap = getMapFromValues(rawValues) + digitalMetadataMap("webTitle") shouldBe "No title given" + digitalMetadataMap.contains("composerUrl") shouldBe false + } + } + describe("toExpression") { + + it("should handle an empty record with default/None values correctly") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched + ) + val expression = record.toExpression + val names = expression.expressionNames().values().asScala + names shouldBe empty + + val values = expression.expressionValues().values().asScala + + values shouldBe empty + expression.expression() shouldBe "" + } + + it("should include standard String and UsageType updates when present") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + mediaId = Some("media-789"), + usageType = Some(PrintUsage), + mediaType = Some("image"), + usageStatus = Some("active") + ) + + val expression = record.toExpression + + val names = expression.expressionNames().values().asScala + + names.size shouldBe 4 + names should contain ("media_id") + names should contain ("usage_type") + names should contain ("media_type") + names should contain ("usage_status") + + val values = expression.expressionValues().values().asScala + values.size shouldBe 4 + values should contain (AttributeValue.builder.s("media-789").build()) + values should contain (AttributeValue.builder.s(PrintUsage.toString).build()) + values should contain (AttributeValue.builder.s("image").build()) + values should contain (AttributeValue.builder.s("active").build()) + + val setExp = "SET #media_id = :media_id, #usage_type = :usage_type," + + " #media_type = :media_type, #usage_status = :usage_status" + + expression.expression() shouldEqual setExp + } + describe("dateRemovedOperation") { + it("should generate a set block when SetDateRemoved is specified") { + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = SetDateRemoved(DateTime.now()) + ) - describe("UsageRecord Update Expressions - digitalUsageMetadata") { + val expression = record.toExpression + val names = expression.expressionNames().values().asScala + names.size shouldBe 1 + names should contain ("date_removed") + + expression.expression() shouldEqual "SET #date_removed = :date_removed" + } + it("should generate a REMOVE block when ClearDateRemoved is specified") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = ClearDateRemoved + ) + + val expression = record.toExpression + expression.expression() shouldEqual "REMOVE #date_removed" + } + it("should handle a mix of remove and set updates when the relevant fields are set") { + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = ClearDateRemoved, + mediaId = Some("media-789") + ) + + val expression = record.toExpression + + val names = expression.expressionNames().values().asScala + names.size shouldBe 1 + names should contain ("media_id") + expression.expression() shouldEqual "SET #media_id = :media_id REMOVE #date_removed" + } + } + describe("PrintUsageMetadata") { + it("should handle PrintUsageMetadata") { + val targetDate = DateTime.parse("2026-07-07T12:00:00Z") + val metadata = PrintUsageMetadata( + sectionName = "News", + issueDate = targetDate, + pageNumber = 5, + storyName = "Scala Breakthrough", + publicationCode = "PUB1", + publicationName = "The Tech Daily", + sectionCode = "SEC-A", + layoutId = Some(12345L), + edition = Some(2), + size = None, + orderedBy = Some("Alice"), + notes = Some("Urgent print"), + source = Some("Internal") + ) + + val record = UsageRecord( + hashKey = "hash-123", + rangeKey = "range-456", + dateRemovedOperation = LeaveDateRemovedUntouched, + printUsageMetadata = Some(metadata) + ) + val expression = record.toExpression + val rawValues = expression.expressionValues().values().asScala + + val printMetadataMap = rawValues.head.m().asScala + + printMetadataMap("sectionName") shouldBe AttributeValue.builder().s("News").build() + printMetadataMap("issueDate") shouldBe AttributeValue.builder().s("2026-07-07T12:00:00.000Z").build() + printMetadataMap("pageNumber") shouldBe AttributeValue.builder().n("5").build() + printMetadataMap("storyName") shouldBe AttributeValue.builder().s("Scala Breakthrough").build() + printMetadataMap("publicationCode") shouldBe AttributeValue.builder().s("PUB1").build() + printMetadataMap("publicationName") shouldBe AttributeValue.builder().s("The Tech Daily").build() + printMetadataMap("sectionCode") shouldBe AttributeValue.builder().s("SEC-A").build() + printMetadataMap("layoutId") shouldBe AttributeValue.builder().n("12345").build() + printMetadataMap("edition") shouldBe AttributeValue.builder().n("2").build() + printMetadataMap("orderedBy") shouldBe AttributeValue.builder().s("Alice").build() + printMetadataMap("notes") shouldBe AttributeValue.builder().s("Urgent print").build() + printMetadataMap("source") shouldBe AttributeValue.builder().s("Internal").build() + + printMetadataMap should have size 12 + } + } + describe("digitalUsageMetadata") { it("should correctly compile DigitalUsageMetadata with all optional fields present") { val metadata = DigitalUsageMetadata( @@ -165,15 +374,15 @@ class UsageRecordTest extends AnyFunSpec with Matchers { digitalUsageMetadata = Some(metadata) ) - val spec = record.toXSpec - val rawValues = spec.getValueMap.values().asScala - val digitalMetadataMap = getMapFromValues(rawValues) + val expression = record.toExpression + val rawValues = expression.expressionValues().values().asScala + val digitalMetadataMap = rawValues.head.m().asScala digitalMetadataMap shouldBe Map( - "webUrl" -> "https://www.theguardian.com/tech", - "webTitle" -> "Scala 3 adoption grows", - "sectionId" -> "technology", - "composerUrl" -> "https://composer.internal/123" + "webUrl" -> AttributeValue.builder().s("https://www.theguardian.com/tech").build(), + "webTitle" -> AttributeValue.builder().s("Scala 3 adoption grows").build(), + "sectionId" -> AttributeValue.builder().s("technology").build(), + "composerUrl" -> AttributeValue.builder().s("https://composer.internal/123").build() ) } @@ -192,10 +401,11 @@ class UsageRecordTest extends AnyFunSpec with Matchers { digitalUsageMetadata = Some(metadata) ) - val spec = record.toXSpec - val rawValues = spec.getValueMap.values().asScala - val digitalMetadataMap = getMapFromValues(rawValues) - digitalMetadataMap("webTitle") shouldBe "No title given" + val expression = record.toExpression + val rawValues = expression.expressionValues().values().asScala + + val digitalMetadataMap = rawValues.head.m().asScala + digitalMetadataMap("webTitle") shouldBe AttributeValue.builder().s("No title given").build() digitalMetadataMap.contains("composerUrl") shouldBe false } } From f4195a4334303e6eaab267028f38c523ce6e980b Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Wed, 8 Jul 2026 12:16:57 +0100 Subject: [PATCH 102/373] Add node version to .tool-versions for mise --- .tool-versions | 1 + 1 file changed, 1 insertion(+) diff --git a/.tool-versions b/.tool-versions index b60a0e5cfc6..135d79b17d6 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1,2 @@ java corretto-11.0.24.8.1 +nodejs 22.12.0 From eeae41b0e94b1339556b4688fe8fe76b2b94ce09 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 8 Jul 2026 13:39:05 +0100 Subject: [PATCH 103/373] unify messaging --- kahuna/public/js/search/results.html | 5 ++--- media-api/app/controllers/MediaApi.scala | 14 +++++++------- .../app/lib/elasticsearch/ElasticSearch.scala | 11 +++++++++++ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index 287497d5b6c..fd0e8381e2d 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -222,14 +222,13 @@
- {{ctrl.filterPoolCounts.filteredPool | toLocaleString}} images match your filters. Add a query to see the best 200. + {{ctrl.filterPoolCounts.filteredPool | toLocaleString}} images match your filters. Enter a query to rank them and surface the best 200 matches.
- No matches, sorry! Try altering your search. - Please enter a query. You can include filters, but must include either some text or a similar:image-id term. + No matches, sorry! Try altering your search.
diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 52f9dbbf12a..170bd10b7ea 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -727,13 +727,13 @@ class MediaApi( def performAiSearchAndRespond(params: SearchParams): Future[Result] = { params.aiQueryParts.validationError match { - case Left(errorMessage) => + case Left(_) => if (params.aiQueryParts.hasFilterConditions) { logger.info(logMarker, s"AI search with only filters and no query to rank by; returning filter pool counts: ${params.aiQueryParts.filterConditions}") filtersOnlyAiSearchResponse(params) } else { - logger.info(logMarker, s"Invalid AI search query: $errorMessage") - Future.successful(respondError(UnprocessableEntity, "invalid-ai-search", errorMessage)) + logger.info(logMarker, s"AI search with no query to rank by and no filters; returning filter pool counts over the whole pool") + filtersOnlyAiSearchResponse(params) } case scala.util.Right(_) => val k = Math.min(params.length, config.aiSearchResultLimit) @@ -753,11 +753,11 @@ class MediaApi( emptyAiSearchResponse case Some(q) if !q.isBlank => performAiSearchAndRespond(_searchParams) - // Empty queries do not make sense for AI search as we can - // only rank results once we have a meaningful vector to compare with. - // So return 0 results if the query was empty. + // No query text to rank by, so we can't return ranked results. Instead + // return the size of the pool we'd be searching over, so the client can + // prompt the user to add a query. case _ => - emptyAiSearchResponse + filtersOnlyAiSearchResponse(_searchParams) } } else { val searchParams = if (canViewDeletedImages) { diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 03e85f3b1e1..38566c9dbd4 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -374,6 +374,17 @@ class ElasticSearch( } } + // How many images match the given filter (or the whole index if no filter), + // used to tell the user the size of the pool an AI search would rank over. + def countMatchingFilter(filterOpt: Option[Query])(implicit ex: ExecutionContext, logMarker: LogMarker): Future[Long] = { + val searchRequest = ElasticDsl.search(imagesCurrentAlias) + .query(filterOpt.getOrElse(matchAllQuery())) + .trackTotalHits(true) + .size(0) + + executeAndLog(withSearchQueryTimeout(searchRequest), "AI search filter pool count").map(_.result.totalHits) + } + def search(params: SearchParams)(implicit ex: ExecutionContext, request: AuthenticatedRequest[AnyContent, Principal], logMarker: LogMarker = MarkerMap()): Future[SearchResults] = { val query: Query = queryBuilder.makeQuery(params.structuredQuery) From 230c63bdbe92088f86ec378cd30ed5329f9bced7 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 8 Jul 2026 13:56:10 +0100 Subject: [PATCH 104/373] display filteredpool as matches --- kahuna/public/js/search/results.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index fd0e8381e2d..4e1a93f8a57 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -23,7 +23,7 @@
Best {{ctrl.aiResultsShown | toLocaleString}} of {{ctrl.totalResults | toLocaleString}} matches - {{ctrl.totalResults | toLocaleString}} matches + {{ctrl.filterPoolCounts.filteredPool | toLocaleString}} matches From 03a71f74c1801ca11b471e382c8c4f39cc4e571c Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 20 Jul 2026 14:39:03 +0100 Subject: [PATCH 130/373] Add test for complex fields with map string --- .../lib/usage/ItemToMediaUsage.scala | 19 ++++++------ .../lib/usage/ItemToMediaUsageTest.scala | 29 ++++++++++++++++--- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala index ab143a58ac0..d1ef4acf1b0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala @@ -5,6 +5,7 @@ import com.amazonaws.services.dynamodbv2.document.Item import com.gu.mediaservice.model.usage._ import org.joda.time.DateTime import org.joda.time.format.ISODateTimeFormat +import software.amazon.awssdk.enhanced.dynamodb.EnhancedType import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument import scala.jdk.CollectionConverters._ @@ -28,9 +29,9 @@ object ItemToMediaUsage { .map(_.asScala.toMap).flatMap(buildSyndication), Option(item.getMap[Any]("front_metadata")) .map(_.asScala.toMap).flatMap(buildFront), - Option(item.getMap[Any]("download_metadata")) + Option(item.getMap[String]("download_metadata")) .map(_.asScala.toMap).flatMap(buildDownload), - Option(item.getMap[Any]("child_metadata")) + Option(item.getMap[String]("child_metadata")) .map(_.asScala.toMap).flatMap(buildChild), new DateTime(item.getLong("last_modified")), Try { @@ -61,10 +62,10 @@ object ItemToMediaUsage { Option(doc.getMapOfUnknownType("front_metadata")) .map(_.asScala.toMap) .flatMap(buildFront), - Option(doc.getMapOfUnknownType("download_metadata")) + Option(doc.getMap("download_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) .map(_.asScala.toMap) .flatMap(buildDownload), - Option(doc.getMapOfUnknownType("child_metadata")) + Option(doc.getMap("child_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) .map(_.asScala.toMap) .flatMap(buildChild), new DateTime(doc.getNumber("last_modified").longValue()), @@ -127,19 +128,19 @@ object ItemToMediaUsage { }.toOption } - private def buildDownload(metadataMap: Map[String, Any]): Option[DownloadUsageMetadata] = { + private def buildDownload(metadataMap: Map[String, String]): Option[DownloadUsageMetadata] = { Try { DownloadUsageMetadata( - metadataMap("downloadedBy").asInstanceOf[String] + metadataMap("downloadedBy") ) }.toOption } - private def buildChild(metadataMap: Map[String, Any]): Option[ChildUsageMetadata] = { + private def buildChild(metadataMap: Map[String, String]): Option[ChildUsageMetadata] = { Try { ChildUsageMetadata( - metadataMap("addedBy").asInstanceOf[String], - metadataMap("childMediaId").asInstanceOf[String], + metadataMap("addedBy"), + metadataMap("childMediaId"), ) }.toOption } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala index 3dd4b0eb825..74969375ed3 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala @@ -1,6 +1,7 @@ package com.gu.mediaservice.lib.usage import com.amazonaws.services.dynamodbv2.document.Item +import software.amazon.awssdk.enhanced.dynamodb.EnhancedType import com.gu.mediaservice.model.usage.{ChildUsageMetadata, DigitalUsageMetadata, DownloadUsageMetadata, FrontUsageMetadata, MediaUsage, PrintImageSize, PrintUsageMetadata, SyndicationUsageMetadata, UsageId, UsageStatus, UsageType} import org.joda.time.DateTime import org.scalatest.funsuite.AnyFunSuiteLike @@ -54,11 +55,11 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { "front" -> "uk-news" ).asJava - val downloadMetadata = Map[String, Any]( + val downloadMetadata = Map[String, String]( "downloadedBy" -> "designer123" ).asJava - val childMetadata = Map[String, Any]( + val childMetadata = Map[String, String]( "addedBy" -> "parent_editor", "childMediaId" -> "child-media-999" ).asJava @@ -143,7 +144,6 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { val mediaUsage = ItemToMediaUsage.transform(enchancedDoc) - mediaUsage shouldEqual MediaUsage( UsageId("usage-123"), "group-abc", @@ -161,7 +161,28 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { Some(new DateTime(dateAddedMillis)), Some(new DateTime(dateRemovedMillis)) ) - + } + + test("testTransform dynamo v2 with complex fields") { + val enchancedDoc = EnhancedDocument.builder() + .attributeConverterProviders(DefaultAttributeConverterProvider.create()) + .putString("usage_id", "usage-123") + .putString("grouping","group-abc") + .putString("media_id", "media-789") + .putString("usage_type", "print") + .putString("media_type", "image") + .putString("usage_status", "pending") + .putNumber("last_modified", lastModifiedMillis) + .putNumber("date_added", dateAddedMillis) + .putNumber("date_removed", dateRemovedMillis) + .putMap("download_metadata", downloadMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .putMap("child_metadata", childMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .build() + + val mediaUsage = ItemToMediaUsage.transform(enchancedDoc) + + mediaUsage.downloadUsageMetadata shouldEqual Some(DownloadUsageMetadata("designer123")) + mediaUsage.childUsageMetadata shouldEqual Some(ChildUsageMetadata("parent_editor", "child-media-999")) } } From 2287adbcebdfe0dda02cc8f8cb9417c0618880fe Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 20 Jul 2026 16:37:50 +0100 Subject: [PATCH 131/373] Update the attribute value map --- .../lib/usage/ItemToMediaUsage.scala | 31 +++++++++- .../lib/usage/ItemToMediaUsageTest.scala | 57 ++++++++++++++++++- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala index d1ef4acf1b0..864b0d4ec6d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala @@ -7,6 +7,7 @@ import org.joda.time.DateTime import org.joda.time.format.ISODateTimeFormat import software.amazon.awssdk.enhanced.dynamodb.EnhancedType import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument +import software.amazon.awssdk.services.dynamodb.model.AttributeValue import scala.jdk.CollectionConverters._ import scala.util.Try @@ -50,9 +51,13 @@ object ItemToMediaUsage { UsageType(doc.getString("usage_type")), doc.getString("media_type"), UsageStatus(doc.getString("usage_status")), - Option(doc.getMapOfUnknownType("print_metadata")) + Option(doc.getMap( + "print_metadata", + EnhancedType.of(classOf[String]), + EnhancedType.of(classOf[AttributeValue]) + )) .map(_.asScala.toMap) - .flatMap(buildPrint), + .flatMap(buildPrintFromAttr), Option(doc.getMapOfUnknownType("digital_metadata")) .map(_.asScala.toMap) .flatMap(buildDigital), @@ -128,6 +133,28 @@ object ItemToMediaUsage { }.toOption } + private def buildPrintFromAttr(metadataMap: Map[String, AttributeValue]): Option[PrintUsageMetadata] = { + Try { + PrintUsageMetadata( + sectionName = metadataMap.apply("sectionName").s(), + issueDate = metadataMap.get("issueDate").map(_.s()) + .map(ISODateTimeFormat.dateTimeParser().parseDateTime).get, + pageNumber = metadataMap.apply("pageNumber").n().toInt, + storyName = metadataMap.apply("storyName").s(), + publicationCode = metadataMap.apply("publicationCode").s(), + publicationName = metadataMap.apply("publicationName").s(), + layoutId = metadataMap.get("layoutId").map(_.n()).map(BigDecimal(_)).map(_.intValue), + edition = metadataMap.get("edition").map(_.n()).map(BigDecimal(_)).map(_.intValue), + size = metadataMap.get("size").map(_.m()) + .map(m => PrintImageSize(m.get("x").n().toInt, m.get("y").n().toInt)), + orderedBy = metadataMap.get("orderedBy").map(_.s()), + sectionCode = metadataMap.apply("sectionCode").s(), + notes = metadataMap.get("notes").map(_.s()), + source = metadataMap.get("source").map(_.s()) + ) + }.toOption + } + private def buildDownload(metadataMap: Map[String, String]): Option[DownloadUsageMetadata] = { Try { DownloadUsageMetadata( diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala index 74969375ed3..c85edfa5e46 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala @@ -1,18 +1,31 @@ package com.gu.mediaservice.lib.usage import com.amazonaws.services.dynamodbv2.document.Item -import software.amazon.awssdk.enhanced.dynamodb.EnhancedType +import software.amazon.awssdk.enhanced.dynamodb.{AttributeConverter, AttributeConverterProvider, AttributeValueType, DefaultAttributeConverterProvider, EnhancedType} import com.gu.mediaservice.model.usage.{ChildUsageMetadata, DigitalUsageMetadata, DownloadUsageMetadata, FrontUsageMetadata, MediaUsage, PrintImageSize, PrintUsageMetadata, SyndicationUsageMetadata, UsageId, UsageStatus, UsageType} import org.joda.time.DateTime import org.scalatest.funsuite.AnyFunSuiteLike import org.scalatest.matchers.should.Matchers.convertToAnyShouldWrapper -import software.amazon.awssdk.enhanced.dynamodb.DefaultAttributeConverterProvider import software.amazon.awssdk.enhanced.dynamodb.document.EnhancedDocument +import software.amazon.awssdk.services.dynamodb.model.AttributeValue import scala.jdk.CollectionConverters.MapHasAsJava import java.math.BigDecimal import java.net.URI +object AttributeValueConverterProvider extends AttributeConverterProvider { + private val converter = new AttributeConverter[AttributeValue] { + override def transformFrom(input: AttributeValue): AttributeValue = input + override def transformTo(input: AttributeValue): AttributeValue = input + override def attributeValueType(): AttributeValueType = AttributeValueType.M + override def `type`(): EnhancedType[AttributeValue] = EnhancedType.of(classOf[AttributeValue]) + } + + override def converterFor[T](enhancedType: EnhancedType[T]): AttributeConverter[T] = { + if (enhancedType.rawClass() == classOf[AttributeValue]) converter.asInstanceOf[AttributeConverter[T]] + else null + } +} class ItemToMediaUsageTest extends AnyFunSuiteLike { val lastModifiedMillis = 1700000000000L @@ -38,6 +51,25 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { "source" -> "Desk" ).asJava + val printMetadataAttr = Map[String, AttributeValue]( + "sectionName" -> AttributeValue.builder().s("News").build(), + "issueDate" -> AttributeValue.builder().s("2026-07-20T10:00:00.000Z").build(), + "pageNumber" -> AttributeValue.builder().n(new BigDecimal(5).toString).build(), + "storyName" -> AttributeValue.builder().s("Lead Story").build(), + "publicationCode" -> AttributeValue.builder().s("GND").build(), + "publicationName" -> AttributeValue.builder().s("The Guardian").build(), + "layoutId" -> AttributeValue.builder().n(new BigDecimal(12).toString).build(), + "edition" -> AttributeValue.builder().n(new BigDecimal(1).toString).build(), + "size" -> AttributeValue.builder().m(Map( + "x" -> AttributeValue.builder().n(new BigDecimal(100).toString).build(), + "y" -> AttributeValue.builder().n(new BigDecimal(200).toString).build() + ).asJava).build(), + "orderedBy" -> AttributeValue.builder.s("editor1").build(), + "sectionCode" -> AttributeValue.builder.s("GND-NEWS").build(), + "notes" -> AttributeValue.builder.s("Top priority").build(), + "source" -> AttributeValue.builder.s("Desk").build() + ).asJava + val digitalMetadata = Map[String, Any]( "webUrl" -> "https://www.theguardian.com/world/2026/jul/20/article", "webTitle" -> "Breaking News", @@ -165,7 +197,10 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { test("testTransform dynamo v2 with complex fields") { val enchancedDoc = EnhancedDocument.builder() - .attributeConverterProviders(DefaultAttributeConverterProvider.create()) + .attributeConverterProviders( + AttributeValueConverterProvider, + DefaultAttributeConverterProvider.create() + ) .putString("usage_id", "usage-123") .putString("grouping","group-abc") .putString("media_id", "media-789") @@ -175,6 +210,7 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { .putNumber("last_modified", lastModifiedMillis) .putNumber("date_added", dateAddedMillis) .putNumber("date_removed", dateRemovedMillis) + .putMap("print_metadata", printMetadataAttr, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[AttributeValue])) .putMap("download_metadata", downloadMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) .putMap("child_metadata", childMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) .build() @@ -183,6 +219,21 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { mediaUsage.downloadUsageMetadata shouldEqual Some(DownloadUsageMetadata("designer123")) mediaUsage.childUsageMetadata shouldEqual Some(ChildUsageMetadata("parent_editor", "child-media-999")) + mediaUsage.printUsageMetadata shouldEqual Some(PrintUsageMetadata( + sectionName = "News", + issueDate = new DateTime("2026-07-20T10:00:00.000Z"), + pageNumber = 5, + storyName = "Lead Story", + publicationCode = "GND", + publicationName = "The Guardian", + layoutId = Some(12), + edition = Some(1), + size = Some(PrintImageSize(100, 200)), + orderedBy = Some("editor1"), + sectionCode = "GND-NEWS", + notes = Some("Top priority"), + source = Some("Desk") + )) } } From 5cd0a81307e23bcc13042397ecd5b23f9cacd47b Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 20 Jul 2026 16:49:44 +0100 Subject: [PATCH 132/373] Add tests for the extra fields --- .../lib/usage/ItemToMediaUsage.scala | 22 +++++++++---------- .../lib/usage/ItemToMediaUsageTest.scala | 22 ++++++++++++++++--- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala index 864b0d4ec6d..83261566b4e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsage.scala @@ -24,11 +24,11 @@ object ItemToMediaUsage { UsageStatus(item.getString("usage_status")), Option(item.getMap[Any]("print_metadata")) .map(_.asScala.toMap).flatMap(buildPrint), - Option(item.getMap[Any]("digital_metadata")) + Option(item.getMap[String]("digital_metadata")) .map(_.asScala.toMap).flatMap(buildDigital), - Option(item.getMap[Any]("syndication_metadata")) + Option(item.getMap[String]("syndication_metadata")) .map(_.asScala.toMap).flatMap(buildSyndication), - Option(item.getMap[Any]("front_metadata")) + Option(item.getMap[String]("front_metadata")) .map(_.asScala.toMap).flatMap(buildFront), Option(item.getMap[String]("download_metadata")) .map(_.asScala.toMap).flatMap(buildDownload), @@ -58,13 +58,13 @@ object ItemToMediaUsage { )) .map(_.asScala.toMap) .flatMap(buildPrintFromAttr), - Option(doc.getMapOfUnknownType("digital_metadata")) + Option(doc.getMap("digital_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) .map(_.asScala.toMap) .flatMap(buildDigital), - Option(doc.getMapOfUnknownType("syndication_metadata")) + Option(doc.getMap("syndication_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) .map(_.asScala.toMap) .flatMap(buildSyndication), - Option(doc.getMapOfUnknownType("front_metadata")) + Option(doc.getMap("front_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) .map(_.asScala.toMap) .flatMap(buildFront), Option(doc.getMap("download_metadata", EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String]))) @@ -98,13 +98,13 @@ object ItemToMediaUsage { }.toOption } - private def buildDigital(metadataMap: Map[String, Any]): Option[DigitalUsageMetadata] = { + private def buildDigital(metadataMap: Map[String, String]): Option[DigitalUsageMetadata] = { Try { DigitalUsageMetadata( - URI.create(metadataMap("webUrl").asInstanceOf[String]), - metadataMap("webTitle").asInstanceOf[String], - metadataMap("sectionId").asInstanceOf[String], - metadataMap.get("composerUrl").map(x => URI.create(x.asInstanceOf[String])) + URI.create(metadataMap("webUrl")), + metadataMap("webTitle"), + metadataMap("sectionId"), + metadataMap.get("composerUrl").map(x => URI.create(x)) ) }.toOption } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala index c85edfa5e46..df89dbfec97 100644 --- a/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala +++ b/common-lib/src/test/scala/com/gu/mediaservice/lib/usage/ItemToMediaUsageTest.scala @@ -70,19 +70,19 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { "source" -> AttributeValue.builder.s("Desk").build() ).asJava - val digitalMetadata = Map[String, Any]( + val digitalMetadata = Map[String, String]( "webUrl" -> "https://www.theguardian.com/world/2026/jul/20/article", "webTitle" -> "Breaking News", "sectionId" -> "world", "composerUrl" -> "https://composer.gutools.co.uk/content/123" ).asJava - val syndicationMetadata = Map[String, Any]( + val syndicationMetadata = Map[String, String]( "partnerName" -> "Reuters", "syndicatedBy" -> "syndication_user" ).asJava - val frontMetadata = Map[String, Any]( + val frontMetadata = Map[String, String]( "addedBy" -> "front_editor", "front" -> "uk-news" ).asJava @@ -210,6 +210,9 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { .putNumber("last_modified", lastModifiedMillis) .putNumber("date_added", dateAddedMillis) .putNumber("date_removed", dateRemovedMillis) + .putMap("digital_metadata", digitalMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .putMap("syndication_metadata", syndicationMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) + .putMap("front_metadata", frontMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) .putMap("print_metadata", printMetadataAttr, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[AttributeValue])) .putMap("download_metadata", downloadMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) .putMap("child_metadata", childMetadata, EnhancedType.of(classOf[String]), EnhancedType.of(classOf[String])) @@ -234,6 +237,19 @@ class ItemToMediaUsageTest extends AnyFunSuiteLike { notes = Some("Top priority"), source = Some("Desk") )) + mediaUsage.syndicationUsageMetadata shouldEqual Some( + SyndicationUsageMetadata("Reuters", Some("syndication_user")) + ) + mediaUsage.frontUsageMetadata shouldEqual Some(FrontUsageMetadata("front_editor", "uk-news")) + mediaUsage.digitalUsageMetadata shouldEqual Some( + DigitalUsageMetadata( + URI.create("https://www.theguardian.com/world/2026/jul/20/article"), + "Breaking News", + "world", + Some(URI.create("https://composer.gutools.co.uk/content/123")) + ) + ) + } } From 8f87e2a887af8accf80d5ab16ec7af0be3639ca8 Mon Sep 17 00:00:00 2001 From: hk-15 <168672047+hk-15@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:15:42 +0100 Subject: [PATCH 133/373] Add background to advanced search info button on hover --- kahuna/public/stylesheets/main.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index d92bc3dcc3f..4037944cd58 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -1195,10 +1195,13 @@ textarea.ng-invalid { .search__advanced-toggle { margin-left: 10px; + padding: 0 5px 3px; } .search__advanced-toggle:hover { color: white; + background-color: #666666; + border-radius: 2px; } .search__advanced-toggle:focus { From dd917224ce55edbe5f568e8edb4198c988af703c Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Tue, 21 Jul 2026 10:42:11 +0100 Subject: [PATCH 134/373] Add loading state for crop view --- kahuna/public/js/crop/controller.js | 19 +++++++++++++++---- kahuna/public/js/crop/index.js | 4 +--- kahuna/public/js/crop/view.html | 5 ++++- .../js/directives/ui-crop-box/ui-crop-box.js | 8 +++++++- kahuna/public/stylesheets/main.css | 11 +++++++++++ 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/kahuna/public/js/crop/controller.js b/kahuna/public/js/crop/controller.js index 69aaacbeb22..8f43516385b 100644 --- a/kahuna/public/js/crop/controller.js +++ b/kahuna/public/js/crop/controller.js @@ -5,12 +5,14 @@ import {radioList} from '../components/gr-radio-list/gr-radio-list'; import {cropUtil} from "../util/crop"; import {cropOptions} from "../util/constants/cropOptions"; import {storage as storageModule} from "../util/storage"; +import '../imgops/service'; const crop = angular.module('kahuna.crop.controller', [ 'gr.keyboardShortcut', radioList.name, cropUtil.name, - storageModule.name + storageModule.name, + 'kahuna.imgops' ]); crop.controller('ImageCropCtrl', [ @@ -21,7 +23,7 @@ crop.controller('ImageCropCtrl', [ 'mediaApi', 'mediaCropper', 'image', - 'optimisedImageUri', + 'imgops', 'keyboardShortcut', 'defaultCrop', 'cropSettings', @@ -37,7 +39,7 @@ crop.controller('ImageCropCtrl', [ mediaApi, mediaCropper, image, - optimisedImageUri, + imgops, keyboardShortcut, defaultCrop, cropSettings, @@ -84,7 +86,16 @@ crop.controller('ImageCropCtrl', [ ctrl.cropType = storageCropType || storageDefaultCropType || defaultCrop.key; ctrl.image = image; - ctrl.optimisedImageUri = optimisedImageUri; + ctrl.optimisedImageUri = null; + ctrl.imageLoading = true; + + imgops.getFullScreenUri(image).then(uri => { + ctrl.optimisedImageUri = uri; + }); + + ctrl.onCropperReady = () => { + ctrl.imageLoading = false; + }; ctrl.cropping = false; diff --git a/kahuna/public/js/crop/index.js b/kahuna/public/js/crop/index.js index ac11db7c785..216e094a1f9 100644 --- a/kahuna/public/js/crop/index.js +++ b/kahuna/public/js/crop/index.js @@ -44,9 +44,7 @@ crop.config(['$stateProvider', return $q.reject(error); } }); - }], - optimisedImageUri: ['image', 'imgops', - (image, imgops) => imgops.getFullScreenUri(image)] + }] } }); diff --git a/kahuna/public/js/crop/view.html b/kahuna/public/js/crop/view.html index 89aaa3e6775..ec131fac5e4 100644 --- a/kahuna/public/js/crop/view.html +++ b/kahuna/public/js/crop/view.html @@ -125,6 +125,9 @@ 'circular-mask': ctrl.shouldShowCircularGuideline && ctrl.shouldUseCircularMask }">
+
+ Loading image… +
scope.onReady()); } function update(c) { diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index 4037944cd58..5a79b9daf91 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -2025,6 +2025,7 @@ FIXME: what to do with touch devices } .easel__canvas { + position: relative; height: calc(100vh - 50px); /* viewport - top-bar */ vertical-align: middle; overflow: hidden; @@ -2034,6 +2035,16 @@ FIXME: what to do with touch devices margin: 10px; } +.easel__loading { + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + color: #999; + font-size: 1.6rem; + white-space: nowrap; +} + .easel__image-container { /*style rules also affect the cropper preview*/ display: inline-block; From 68c42405d8fe7586d73d2a630766165530442967 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 21 Jul 2026 15:47:50 +0100 Subject: [PATCH 135/373] Add the kinesis client to common lib --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 99baad59441..dc17d6d6b26 100644 --- a/build.sbt +++ b/build.sbt @@ -100,6 +100,7 @@ lazy val commonLib = project("common-lib").settings( "software.amazon.awssdk" % "s3" % awsSdkV2Version, "software.amazon.awssdk" % "sns" % awsSdkV2Version, "software.amazon.awssdk" % "sqs" % awsSdkV2Version, + "software.amazon.awssdk" % "kinesis" % awsSdkV2Version, "nl.gn0s1s" %% "elastic4s-core" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-client-esjava" % elastic4sVersion, "nl.gn0s1s" %% "elastic4s-domain" % elastic4sVersion, From 7f26d71e8e9e15fb964bda69a6d2562ce2124fff Mon Sep 17 00:00:00 2001 From: lindseydew Date: Tue, 21 Jul 2026 16:13:47 +0100 Subject: [PATCH 136/373] Remove the ec2 lib --- build.sbt | 1 - .../gu/mediaservice/lib/discovery/EC2.scala | 38 ------------------- 2 files changed, 39 deletions(-) delete mode 100644 common-lib/src/main/scala/com/gu/mediaservice/lib/discovery/EC2.scala diff --git a/build.sbt b/build.sbt index 9e5aa3850ad..16faa7feddc 100644 --- a/build.sbt +++ b/build.sbt @@ -94,7 +94,6 @@ lazy val commonLib = project("common-lib").settings( "com.gu" %% "editorial-permissions-client" % "4.0.0", "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0", "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion, - "com.amazonaws" % "aws-java-sdk-ec2" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-kinesis" % awsSdkVersion, "software.amazon.awssdk" % "iam" % awsSdkV2Version, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/discovery/EC2.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/discovery/EC2.scala deleted file mode 100644 index 7ae37625a2b..00000000000 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/discovery/EC2.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.gu.mediaservice.lib.discovery - -import com.amazonaws.services.ec2.AmazonEC2 -import com.amazonaws.services.ec2.model.{DescribeInstancesRequest, Filter, InstanceStateName} -import com.gu.mediaservice.lib.logging.GridLogging - -import scala.jdk.CollectionConverters._ -import scala.util.Random - -object EC2 extends GridLogging { - - @annotation.tailrec - def findElasticsearchHostByTags(client: AmazonEC2, tags: Map[String, Seq[String]]): String = { - val instances = client.describeInstances(new DescribeInstancesRequest().withFilters( - new Filter("instance-state-name", List(InstanceStateName.Running.toString).asJava) +: - tagFilters(tags): _* - )) - - val hosts = instances.getReservations.asScala - .flatMap(_.getInstances.asScala) - .map(_.getPublicDnsName) - logger.info(s"Available Elasticsearch hosts in EC2: [${hosts.mkString(", ")}]") - - Random.shuffle(hosts).headOption match { - case None => - logger.warn("Could not find an Elasticsearch host. Trying again...") - Thread.sleep(1000) - findElasticsearchHostByTags(client, tags) - case Some(host) => - logger.info(s"Using Elasticsearch host $host") - host - } - } - - def tagFilters(tags: Map[String, Seq[String]]): List[Filter] = - for ((key, values) <- tags.toList) yield new Filter(s"tag:$key", values.asJava) - -} From 7bd19cc71ade51d42849c0b3f994cd451432e464 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 07:23:08 +0000 Subject: [PATCH 137/373] chore(deps): bump immutable from 3.8.3 to 4.3.9 in /kahuna Bumps [immutable](https://github.com/immutable-js/immutable-js) from 3.8.3 to 4.3.9. - [Release notes](https://github.com/immutable-js/immutable-js/releases) - [Changelog](https://github.com/immutable-js/immutable-js/blob/main/CHANGELOG.md) - [Commits](https://github.com/immutable-js/immutable-js/compare/v3.8.3...v4.3.9) --- updated-dependencies: - dependency-name: immutable dependency-version: 4.3.9 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- kahuna/package-lock.json | 13 +++++-------- kahuna/package.json | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index 4d5a59a5129..d5b54597ed6 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -23,7 +23,7 @@ "any-promise-angular": "0.1.1", "cropperjs": "1.5.12", "filehash": "1.0.0", - "immutable": "3.8.3", + "immutable": "4.3.9", "javascript-detect-element-resize": "0.5.3", "jszip": "3.8.0", "moment": "2.29.4", @@ -8718,13 +8718,10 @@ "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" }, "node_modules/immutable": { - "version": "3.8.3", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.3.tgz", - "integrity": "sha512-AUY/VyX0E5XlibOmWt10uabJzam1zlYjwiEgQSDc5+UIkFNaF9WM0JxXKaNMGf+F/ffUF+7kRKXM9A7C0xXqMg==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } + "version": "4.3.9", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.9.tgz", + "integrity": "sha512-ObHy4YN7ycwZOUCLI1/6svfyAFu7vL8RhAvVu/bh/RZW9EPlOyDaQ9jDQWCtdqzaXUjgXZCW1migtHE7YI7UGQ==", + "license": "MIT" }, "node_modules/import-fresh": { "version": "3.3.1", diff --git a/kahuna/package.json b/kahuna/package.json index 0fc91cd538a..8aa0376247f 100644 --- a/kahuna/package.json +++ b/kahuna/package.json @@ -18,7 +18,7 @@ "any-promise-angular": "0.1.1", "cropperjs": "1.5.12", "filehash": "1.0.0", - "immutable": "3.8.3", + "immutable": "4.3.9", "javascript-detect-element-resize": "0.5.3", "jszip": "3.8.0", "moment": "2.29.4", From 990e1cda730ac8f922806278784306b390e9bb65 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 07:23:10 +0000 Subject: [PATCH 138/373] chore(deps): bump sharp from 0.34.5 to 0.35.0 in /image-embedder-lambda Bumps [sharp](https://github.com/lovell/sharp) from 0.34.5 to 0.35.0. - [Release notes](https://github.com/lovell/sharp/releases) - [Commits](https://github.com/lovell/sharp/compare/v0.34.5...v0.35.0) --- updated-dependencies: - dependency-name: sharp dependency-version: 0.35.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- image-embedder-lambda/package-lock.json | 361 +++++++++++++++--------- image-embedder-lambda/package.json | 2 +- 2 files changed, 223 insertions(+), 140 deletions(-) diff --git a/image-embedder-lambda/package-lock.json b/image-embedder-lambda/package-lock.json index 8bdb0a1d16f..743f28d4ca8 100644 --- a/image-embedder-lambda/package-lock.json +++ b/image-embedder-lambda/package-lock.json @@ -15,7 +15,7 @@ "@aws-sdk/client-s3vectors": "^3.1030.0", "@aws-sdk/client-sqs": "^3.1030.0", "@aws-sdk/config": "^3.1030.0", - "sharp": "^0.34.5", + "sharp": "^0.35.0", "source-map-support": "^0.5.21" }, "devDependencies": { @@ -1928,9 +1928,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", - "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "version": "1.11.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.2.tgz", + "integrity": "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA==", "license": "MIT", "optional": true, "dependencies": { @@ -2401,18 +2401,18 @@ } }, "node_modules/@img/colour": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.0.0.tgz", - "integrity": "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", + "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", "license": "MIT", "engines": { "node": ">=18" } }, "node_modules/@img/sharp-darwin-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", - "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.0.tgz", + "integrity": "sha512-ZgaYEwaj+lx/5n4W8GmZ2IYz0PQHjN5eqRcfijWGB+2Aq7ZInZGa0qJyAn6DEtyLuWHRSrmWOqT9q3qqTBvmUQ==", "cpu": [ "arm64" ], @@ -2422,19 +2422,19 @@ "darwin" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.2.4" + "@img/sharp-libvips-darwin-arm64": "1.3.0" } }, "node_modules/@img/sharp-darwin-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", - "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.0.tgz", + "integrity": "sha512-c1z9LFpKB0slQW3RchwBE8iSVzGp70TNjUUO9k4BZwwW4HH7JBGHeIy4b+kk4n/kcBASb9evKCE3/7Slmslgiw==", "cpu": [ "x64" ], @@ -2444,19 +2444,38 @@ "darwin" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.2.4" + "@img/sharp-libvips-darwin-x64": "1.3.0" + } + }, + "node_modules/@img/sharp-freebsd-wasm32": { + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.0.tgz", + "integrity": "sha512-Li2KTev0H90kEtnJHkI9xQojXt1AqWmFBMXiPw5kqd1jQgP7gi5HVK/qC5Rmh/59NuAwUuPzzPITmX22NomYYQ==", + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "dependencies": { + "@img/sharp-wasm32": "0.35.0" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", - "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.0.tgz", + "integrity": "sha512-EKbmBKtyTH+GPFDRw2TgK2oV6hyxxlJVIar4hoTYSNmIwipgMFdxPQqR392GmfdsPGWga0mCFN1cCKjRb9cljw==", "cpu": [ "arm64" ], @@ -2470,9 +2489,9 @@ } }, "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", - "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.0.tgz", + "integrity": "sha512-Pl2OmOvrJ42adUllESxBsG54PfXLo1OYg9i3c5/5Ln/qJ0gZuTM9YMhQJPIbXqwidLRc/c2zuHt4RsrymmNv7A==", "cpu": [ "x64" ], @@ -2486,12 +2505,15 @@ } }, "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", - "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.0.tgz", + "integrity": "sha512-A8UpHoUDW4DwnXoV6+q3C1s7QLRAHtPDEjWuNZjwHMyoCNZnm0GeNN8ls9f/bsEYTRQRW96C/n34XJQHJ2fT7A==", "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2502,12 +2524,15 @@ } }, "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", - "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.0.tgz", + "integrity": "sha512-C0SqjoFKnszqa44EQ7xoaT48nnO0lOyXEULfXMWi8krrjOPGYkeK30Okzla6ATbBYsyZ0ySinK0FVkpv3DwzfQ==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2518,12 +2543,15 @@ } }, "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", - "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.0.tgz", + "integrity": "sha512-WOpkVxAjFd369iaIzEgNRreFD+gWdUMIGD5zplhNKNeqS6mm5dac3q2AFyCBmzYoAdouzZvRBgxy4z8QHZb4/A==", "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2534,12 +2562,15 @@ } }, "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", - "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.0.tgz", + "integrity": "sha512-DRWw0mOHusrCCuw2rqP87oLg6PGlkomVDFqw2hIwsSfwWpu4k3XLcBPaKKl6ct/GtL/cwNkgwjV/tc0Mqht3VA==", "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2550,12 +2581,15 @@ } }, "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", - "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.0.tgz", + "integrity": "sha512-9APy+nFWhHS+kzLgWZfLcyrUd7YqnAQVa4BPOo4xkoHpdoktOAPG4cEr9+Jpl0TtqfVmcMJimNL5qNTyyOHZNA==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2566,12 +2600,15 @@ } }, "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", - "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.0.tgz", + "integrity": "sha512-y9RNUYDe2A1UAdhLyfeOodGRszQdaEoe4nfOpp/sNVPl2CWIcUyFaDoCh4vPLPxu19803j2naLqZup2WxDXCLA==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2582,12 +2619,15 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", - "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.0.tgz", + "integrity": "sha512-cC1wkC0Mlucd0KSiGrLkJnB/ZqPvZCntc/Lk7ZnYO5ZSbF2euNek4Xvxafojq+wN1q/W0eprdpUIjUr/EV2PBg==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2598,12 +2638,15 @@ } }, "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", - "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.0.tgz", + "integrity": "sha512-LiYMhUZicB1QG//+RvmYZpXJO8fYRENfp+MZUCnG9aw+AKvGAy9gPaCnuwsPcBFs8EV66M0NNxj9VHcNklE8zw==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -2614,204 +2657,244 @@ } }, "node_modules/@img/sharp-linux-arm": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", - "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.0.tgz", + "integrity": "sha512-VVlpEWwizEFIOom0zdoeKuO5nuTswzVE5uHcBNvHzmeHUpNFajY3HFfbQ+zIH4E2kVaZ/yVxmsShW56TtEy4uA==", "cpu": [ "arm" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.2.4" + "@img/sharp-libvips-linux-arm": "1.3.0" } }, "node_modules/@img/sharp-linux-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", - "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.0.tgz", + "integrity": "sha512-4+4XHLNT5wDT0roYlHTEmH9lDKt0acf9Tv+3hM3iceOirkxrR404/3WjAYZ9F9CkHrxeRcGLJXbi4vluMZ9O+A==", "cpu": [ "arm64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.2.4" + "@img/sharp-libvips-linux-arm64": "1.3.0" } }, "node_modules/@img/sharp-linux-ppc64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", - "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.0.tgz", + "integrity": "sha512-N3hzbEpUTJC8pWpPVJvgzGxM+so/MAXc8O2s/53B0LL9ZGpfXpME7Wizkc5d/8fRBlBtkDjzoZGDCqqNDHqLEw==", "cpu": [ "ppc64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.2.4" + "@img/sharp-libvips-linux-ppc64": "1.3.0" } }, "node_modules/@img/sharp-linux-riscv64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", - "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.0.tgz", + "integrity": "sha512-l6vmKVPnbS0RhVMbyxP5meAARsbhCnBN4fy31qz0+3a6Rv4jEqfzDrT89y6ZPkCi0AJGnwp2En528yXo401Hpw==", "cpu": [ "riscv64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.2.4" + "@img/sharp-libvips-linux-riscv64": "1.3.0" } }, "node_modules/@img/sharp-linux-s390x": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", - "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.0.tgz", + "integrity": "sha512-MYlMiPFiv/EKPAHnp3yNZ9AAWFsxga9c5Bkc6wkar6bqzHLlkGVJHRm0u1ei+VXnZxp3Mz9MG9ZIsI8vSOf3sQ==", "cpu": [ "s390x" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.2.4" + "@img/sharp-libvips-linux-s390x": "1.3.0" } }, "node_modules/@img/sharp-linux-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", - "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.0.tgz", + "integrity": "sha512-TYaItB5oj1ioXjhyn2xrR208vf+YuIIcHptQWRRaBmFhvIvL9D72DXN8w75xup0KXA8UdEAhQ9Qb2S49FD/9Cw==", "cpu": [ "x64" ], + "libc": [ + "glibc" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.2.4" + "@img/sharp-libvips-linux-x64": "1.3.0" } }, "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", - "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.0.tgz", + "integrity": "sha512-DSTb6ijQzqe6DdAaOBVqJ/SYf1vO8EW5bK6X6LRXufEBebf2722VCdvBUtZ3rtV0x2ApfPNDy/p7LrrjaWjiyQ==", "cpu": [ "arm64" ], + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + "@img/sharp-libvips-linuxmusl-arm64": "1.3.0" } }, "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", - "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.0.tgz", + "integrity": "sha512-K7ykQ+26Rt6+4BTU80AuGgTPIYX86UxiAKT4rcXX/WNTo7k1ZxpKz+TguHnwVpCqQK3B5PK0vZ0ZBe6nz/ib1w==", "cpu": [ "x64" ], + "libc": [ + "musl" + ], "license": "Apache-2.0", "optional": true, "os": [ "linux" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + "@img/sharp-libvips-linuxmusl-x64": "1.3.0" } }, "node_modules/@img/sharp-wasm32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", - "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.0.tgz", + "integrity": "sha512-9woLIFORERCr+6cWu87dQ22J34EExkhc73U1kZW0c+RclQqWetoodByp4dWZ/hN8/KVmTRAx2HOnUwib8AwZdA==", + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.11.0" + }, + "engines": { + "node": ">=20.9.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-webcontainers-wasm32": { + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.0.tgz", + "integrity": "sha512-t+kie1TOyaDM6Dho+f+y0VqIUNhYQaKCUahuZVi0E0frgdiaOaPsDxDW3wfKacUdaNBCnK/ZDBMg33ydvHj8uA==", "cpu": [ "wasm32" ], - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "license": "Apache-2.0", "optional": true, "dependencies": { - "@emnapi/runtime": "^1.7.0" + "@img/sharp-wasm32": "0.35.0" }, "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-arm64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", - "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.0.tgz", + "integrity": "sha512-M5eKxug0dabbaWgFKvPa3odNs2OpaP+81NASfGKkt4GcYXpNhSu7CaeYxWkLNV6vHmUp4hnCxnxrUyhUJhXbKA==", "cpu": [ "arm64" ], @@ -2821,16 +2904,16 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-ia32": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", - "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.0.tgz", + "integrity": "sha512-z0+pZ03QCDvdVN0Ez9IX/yjWC19ikMlXrmdYMwYNLTh2BLPx3hXWPvyqWfquZ0BTO9O6GVOjIVoTcyyacMnWlQ==", "cpu": [ "ia32" ], @@ -2840,16 +2923,16 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": "^20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" } }, "node_modules/@img/sharp-win32-x64": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", - "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.0.tgz", + "integrity": "sha512-feNnlz5ZHKr0MY1LPHvZQyJeBkbo4ctsn0D8FvA53VTw5TC63rfEL2UrWbkSBR19htSE7Mw78xYVwdJqoMWVHw==", "cpu": [ "x64" ], @@ -2859,7 +2942,7 @@ "win32" ], "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" @@ -10322,53 +10405,53 @@ } }, "node_modules/sharp": { - "version": "0.34.5", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", - "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", - "hasInstallScript": true, + "version": "0.35.0", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.0.tgz", + "integrity": "sha512-BqvG5XbwPZ4NV0DK90d86leEECMsoa8bO0nqnKWlBDYxri4GJ7c4EDInaF6q20lTh/mATmnDIKWJFfXnoVfH5g==", "license": "Apache-2.0", "dependencies": { - "@img/colour": "^1.0.0", + "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", - "semver": "^7.7.3" + "semver": "^7.8.4" }, "engines": { - "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + "node": ">=20.9.0" }, "funding": { "url": "https://opencollective.com/libvips" }, "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.34.5", - "@img/sharp-darwin-x64": "0.34.5", - "@img/sharp-libvips-darwin-arm64": "1.2.4", - "@img/sharp-libvips-darwin-x64": "1.2.4", - "@img/sharp-libvips-linux-arm": "1.2.4", - "@img/sharp-libvips-linux-arm64": "1.2.4", - "@img/sharp-libvips-linux-ppc64": "1.2.4", - "@img/sharp-libvips-linux-riscv64": "1.2.4", - "@img/sharp-libvips-linux-s390x": "1.2.4", - "@img/sharp-libvips-linux-x64": "1.2.4", - "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", - "@img/sharp-libvips-linuxmusl-x64": "1.2.4", - "@img/sharp-linux-arm": "0.34.5", - "@img/sharp-linux-arm64": "0.34.5", - "@img/sharp-linux-ppc64": "0.34.5", - "@img/sharp-linux-riscv64": "0.34.5", - "@img/sharp-linux-s390x": "0.34.5", - "@img/sharp-linux-x64": "0.34.5", - "@img/sharp-linuxmusl-arm64": "0.34.5", - "@img/sharp-linuxmusl-x64": "0.34.5", - "@img/sharp-wasm32": "0.34.5", - "@img/sharp-win32-arm64": "0.34.5", - "@img/sharp-win32-ia32": "0.34.5", - "@img/sharp-win32-x64": "0.34.5" + "@img/sharp-darwin-arm64": "0.35.0", + "@img/sharp-darwin-x64": "0.35.0", + "@img/sharp-freebsd-wasm32": "0.35.0", + "@img/sharp-libvips-darwin-arm64": "1.3.0", + "@img/sharp-libvips-darwin-x64": "1.3.0", + "@img/sharp-libvips-linux-arm": "1.3.0", + "@img/sharp-libvips-linux-arm64": "1.3.0", + "@img/sharp-libvips-linux-ppc64": "1.3.0", + "@img/sharp-libvips-linux-riscv64": "1.3.0", + "@img/sharp-libvips-linux-s390x": "1.3.0", + "@img/sharp-libvips-linux-x64": "1.3.0", + "@img/sharp-libvips-linuxmusl-arm64": "1.3.0", + "@img/sharp-libvips-linuxmusl-x64": "1.3.0", + "@img/sharp-linux-arm": "0.35.0", + "@img/sharp-linux-arm64": "0.35.0", + "@img/sharp-linux-ppc64": "0.35.0", + "@img/sharp-linux-riscv64": "0.35.0", + "@img/sharp-linux-s390x": "0.35.0", + "@img/sharp-linux-x64": "0.35.0", + "@img/sharp-linuxmusl-arm64": "0.35.0", + "@img/sharp-linuxmusl-x64": "0.35.0", + "@img/sharp-webcontainers-wasm32": "0.35.0", + "@img/sharp-win32-arm64": "0.35.0", + "@img/sharp-win32-ia32": "0.35.0", + "@img/sharp-win32-x64": "0.35.0" } }, "node_modules/sharp/node_modules/semver": { - "version": "7.7.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", - "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "license": "ISC", "bin": { "semver": "bin/semver.js" diff --git a/image-embedder-lambda/package.json b/image-embedder-lambda/package.json index 35364bf9a72..b23b6e9daeb 100644 --- a/image-embedder-lambda/package.json +++ b/image-embedder-lambda/package.json @@ -39,7 +39,7 @@ "@aws-sdk/client-s3vectors": "^3.1030.0", "@aws-sdk/client-sqs": "^3.1030.0", "@aws-sdk/config": "^3.1030.0", - "sharp": "^0.34.5", + "sharp": "^0.35.0", "source-map-support": "^0.5.21" }, "prettier": "@guardian/prettier" From 603b8f3065d54a43603aff356db0898e2868f160 Mon Sep 17 00:00:00 2001 From: Joseph Smith Date: Wed, 22 Jul 2026 12:18:02 +0100 Subject: [PATCH 139/373] Do not display incorrect 'too many results' after AI search results --- kahuna/public/js/search/results.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/js/search/results.html b/kahuna/public/js/search/results.html index bd4951e21af..1d4e707d3b8 100644 --- a/kahuna/public/js/search/results.html +++ b/kahuna/public/js/search/results.html @@ -216,7 +216,7 @@ -
Too many results to display
Please refine your search to limit the number of results
From c07a723b4ed50900a3f45427562eb412b8b34275 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 23 Jul 2026 11:30:37 +0100 Subject: [PATCH 140/373] update vecweight to 0.85 --- media-api/app/controllers/MediaApi.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index eee30db15db..f7b07995b5e 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -678,7 +678,7 @@ class MediaApi( logger.info(logMarker, s"No semantic query found in structured query; returning no AI results") Future.successful(SearchResults(Nil, total = 0, extraCounts = None)) case Some(semanticQuery) => - val vecWeight = params.vecWeight.getOrElse(0.8) + val vecWeight = params.vecWeight.getOrElse(0.85) val outerMarker = logMarker { From e56cf34de35ac8ae0e2980cc2d0b92e95eaab1d9 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 23 Jul 2026 13:45:23 +0100 Subject: [PATCH 141/373] Revert "chore(deps): bump immutable from 3.8.3 to 4.3.9 in /kahuna" --- kahuna/package-lock.json | 13 ++++++++----- kahuna/package.json | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/kahuna/package-lock.json b/kahuna/package-lock.json index d5b54597ed6..4d5a59a5129 100644 --- a/kahuna/package-lock.json +++ b/kahuna/package-lock.json @@ -23,7 +23,7 @@ "any-promise-angular": "0.1.1", "cropperjs": "1.5.12", "filehash": "1.0.0", - "immutable": "4.3.9", + "immutable": "3.8.3", "javascript-detect-element-resize": "0.5.3", "jszip": "3.8.0", "moment": "2.29.4", @@ -8718,10 +8718,13 @@ "integrity": "sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=" }, "node_modules/immutable": { - "version": "4.3.9", - "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.3.9.tgz", - "integrity": "sha512-ObHy4YN7ycwZOUCLI1/6svfyAFu7vL8RhAvVu/bh/RZW9EPlOyDaQ9jDQWCtdqzaXUjgXZCW1migtHE7YI7UGQ==", - "license": "MIT" + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/immutable/-/immutable-3.8.3.tgz", + "integrity": "sha512-AUY/VyX0E5XlibOmWt10uabJzam1zlYjwiEgQSDc5+UIkFNaF9WM0JxXKaNMGf+F/ffUF+7kRKXM9A7C0xXqMg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } }, "node_modules/import-fresh": { "version": "3.3.1", diff --git a/kahuna/package.json b/kahuna/package.json index 8aa0376247f..0fc91cd538a 100644 --- a/kahuna/package.json +++ b/kahuna/package.json @@ -18,7 +18,7 @@ "any-promise-angular": "0.1.1", "cropperjs": "1.5.12", "filehash": "1.0.0", - "immutable": "4.3.9", + "immutable": "3.8.3", "javascript-detect-element-resize": "0.5.3", "jszip": "3.8.0", "moment": "2.29.4", From 4ef57275978487fc89b25ff70347f0c04762daaa Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 23 Jul 2026 18:16:16 +0100 Subject: [PATCH 142/373] Port Media Api to use s3 get object v2 --- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 13 ++++++++++--- media-api/app/controllers/MediaApi.scala | 12 ++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index da4e1eec97d..60d8f7f7cc0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -8,8 +8,10 @@ import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ import org.joda.time.{DateTime, Duration} +import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.{GetObjectResponse, GetObjectRequest => GetObjectRequestV2} import java.io.File import java.net.URI @@ -68,6 +70,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with type UserMetadata = Map[String, String] lazy val client: AmazonS3 = S3Ops.buildS3Client(config) + lazy val clientV2: S3Client = S3Ops.buildS3ClientV2(config) def signUrl(bucket: Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = { // get path and remove leading `/` @@ -80,11 +83,15 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) client.generatePresignedUrl(request).toExternalForm } - - def getObject(bucket: Bucket, url: URI): model.S3Object = { + + def getObjectV2(bucket: Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { // get path and remove leading `/` val key: Key = url.getPath.drop(1) - client.getObject(new GetObjectRequest(bucket, key)) + clientV2.getObject(GetObjectRequestV2.builder().key(key).bucket(bucket).build()) + } + + def getObjectV2(bucket: Bucket, key: String): ResponseInputStream[GetObjectResponse] = { + clientV2.getObject(GetObjectRequestV2.builder().key(key).bucket(bucket).build()) } def getObjectAsString(bucket: Bucket, key: String): Option[String] = { diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index f7b07995b5e..9d63ea76ad2 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -310,8 +310,12 @@ class MediaApi( val maybeResult = for { export <- source.exports.find(_.id.contains(exportId)) asset <- export.assets.find(_.dimensions.exists(_.width == width)) - s3Object <- Try(s3Client.getObject(config.imgPublishingBucket, asset.file)).toOption - file = StreamConverters.fromInputStream(() => s3Object.getObjectContent) + s3Res = Try(s3Client.getObjectV2(config.imgPublishingBucket, asset.file)) + _ = s3Res.failed.foreach { ex => + logger.error("Failed to fetch S3 object", ex) + } + s3Object <- s3Res.toOption + file = StreamConverters.fromInputStream(() => s3Object) entity = HttpEntity.Streamed(file, asset.size, asset.mimeType.map(_.name)) result = Result(ResponseHeader(OK), entity).withHeaders("Content-Disposition" -> getContentDisposition(source, export, asset, config.shortenDownloadFilename)) } yield { @@ -437,8 +441,8 @@ class MediaApi( val apiKey = request.user.accessor logger.info(logMarker, s"Download original image: $id from user: ${Authentication.getIdentity(request.user)}") mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OriginalDownloadType) - val s3Object = s3Client.getObject(config.imageBucket, image.source.file) - val file = StreamConverters.fromInputStream(() => s3Object.getObjectContent) + val s3Object = s3Client.getObjectV2(config.imageBucket, image.source.file) + val file = StreamConverters.fromInputStream(() => s3Object) val entity = HttpEntity.Streamed(file, image.source.size, image.source.mimeType.map(_.name)) if(config.recordDownloadAsUsage) { From 4df902e392a653b003a86513d2e2eae067a5ad14 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 23 Jul 2026 18:25:54 +0100 Subject: [PATCH 143/373] Update the reading from s3 --- .../com/gu/mediaservice/lib/BaseStore.scala | 17 ++++++++++------- .../com/gu/mediaservice/lib/auth/KeyStore.scala | 5 ++++- .../scala/com/gu/mediaservice/lib/aws/S3.scala | 15 +++++++++++++-- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala index 0aeb698cccb..118c30fa1d6 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/BaseStore.scala @@ -5,6 +5,7 @@ import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.GridLogging import org.joda.time.DateTime +import software.amazon.awssdk.services.s3.model.{GetObjectRequest, ListObjectsV2Request} import java.util.concurrent.atomic.AtomicReference import java.io.InputStream @@ -22,18 +23,20 @@ abstract class BaseStore[TStoreKey, TStoreVal](bucket: String, config: CommonCon protected val store: AtomicReference[Map[TStoreKey, TStoreVal]] = new AtomicReference(Map.empty) protected val lastUpdated: AtomicReference[DateTime] = new AtomicReference(DateTime.now()) - protected def getS3Object(key: String): Option[String] = s3.getObjectAsString(bucket, key) + protected def getS3Object(key: String): Option[String] = s3.getObjectAsStringV2(bucket, key) protected def getLatestS3Stream: Option[InputStream] = { - val objects = s3.client - .listObjects(bucket).getObjectSummaries.asScala - .filterNot(_.getKey == "AMAZON_SES_SETUP_NOTIFICATION") + val objects = s3.clientV2.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) + .contents().asScala.toList + .filterNot(_.key() == "AMAZON_SES_SETUP_NOTIFICATION") if (objects.nonEmpty) { - val obj = objects.maxBy(_.getLastModified) - logger.info(s"Latest key ${obj.getKey} in bucket $bucket") + val obj = objects.maxBy(_.lastModified()) + logger.info(s"Latest key ${obj.key} in bucket $bucket") - val stream = s3.client.getObject(bucket, obj.getKey).getObjectContent + val stream = s3.clientV2.getObject( + GetObjectRequest.builder().key(obj.key()).bucket(bucket).build() + ) Some(stream) } else { logger.error(s"Bucket $bucket is empty") diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index d3c90bc739e..855f4bf7e9b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -2,6 +2,7 @@ package com.gu.mediaservice.lib.auth import com.gu.mediaservice.lib.BaseStore import com.gu.mediaservice.lib.config.CommonConfig +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request import scala.jdk.CollectionConverters._ import scala.concurrent.ExecutionContext @@ -18,7 +19,9 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte } private def fetchAll: Map[String, ApiAccessor] = { - val keys = s3.client.listObjects(bucket).getObjectSummaries.asScala.map(_.getKey) + val contents = s3.clientV2.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).build()) + .contents().asScala.toList + val keys = contents.map(_.key()) keys.flatMap(k => getS3Object(k).map(k -> ApiAccessor(_))).toMap } } diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 60d8f7f7cc0..b696f2d364e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -11,10 +11,11 @@ import org.joda.time.{DateTime, Duration} import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.{GetObjectResponse, GetObjectRequest => GetObjectRequestV2} +import software.amazon.awssdk.services.s3.model.{GetObjectResponse, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2} import java.io.File import java.net.URI +import java.nio.charset.StandardCharsets import scala.jdk.CollectionConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -83,7 +84,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) client.generatePresignedUrl(request).toExternalForm } - + def getObjectV2(bucket: Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { // get path and remove leading `/` val key: Key = url.getPath.drop(1) @@ -108,6 +109,16 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with stream.close() } } + def getObjectAsStringV2(bucket: Bucket, key: String): Option[String] = { + try { + val stream = clientV2.getObject(GetObjectRequestV2.builder().key(key).bucket(bucket).build()); + Some(new String(stream.readAllBytes(), StandardCharsets.UTF_8)) + } catch { + case e: NoSuchKeyException => + logger.warn(s"Cannot find key: $key in bucket: $bucket") + None + } + } def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = From ab88dddca832fd852c65f14696027214cabf5ecf Mon Sep 17 00:00:00 2001 From: lindseydew Date: Thu, 23 Jul 2026 18:45:28 +0100 Subject: [PATCH 144/373] Port to use store v2 --- .../gu/mediaservice/lib/S3ImageStorage.scala | 4 +- .../com/gu/mediaservice/lib/aws/S3.scala | 55 ++++++++++++------- image-loader/app/lib/ImageLoaderStore.scala | 2 +- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 7e164f876bb..59b396dd2aa 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -19,9 +19,9 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage (implicit logMarker: LogMarker) = { logger.info(logMarker, s"bucket: $bucket, id: $id, meta: $meta") val eventualObject = if (overwrite) { - store(bucket, id, file, mimeType, meta, cacheSetting) + storeV2(bucket, id, file, mimeType, meta, cacheSetting) } else { - storeIfNotPresent(bucket, id, file, mimeType, meta, cacheSetting) + storeIfNotPresentV2(bucket, id, file, mimeType, meta, cacheSetting) } eventualObject.onComplete(o => logger.info(logMarker, s"storeImage completed $o")) eventualObject diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index b696f2d364e..21cf11c67f9 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -7,11 +7,12 @@ import com.amazonaws.{AmazonServiceException, ClientConfiguration} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ -import org.joda.time.{DateTime, Duration} +import org.joda.time.{DateTime, DateTimeZone, Duration} import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.{GetObjectResponse, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2} +import software.amazon.awssdk.services.s3.model.{GetObjectResponse, HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2, PutObjectRequest => PutObjectRequestV2} import java.io.File import java.net.URI @@ -61,6 +62,16 @@ object S3Metadata { ) ) } + def apply(meta: HeadObjectResponse): S3Metadata = { + S3Metadata( + meta.metadata().asScala.toMap, + S3ObjectMetadata( + contentType = Option(meta.contentType()).filterNot(_.toLowerCase == "application/octet-stream").map(MimeType.apply), + cacheControl = Option(meta.cacheControl()), + lastModified = Option(meta.lastModified()).map(l => new DateTime(l.toEpochMilli).withZone(DateTimeZone.UTC)) + ) + ) + } } case class S3ObjectMetadata(contentType: Option[MimeType], cacheControl: Option[String], lastModified: Option[DateTime]) @@ -120,43 +131,47 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def store(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) + def storeV2(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = Future { - val metadata = new ObjectMetadata - mimeType.foreach(m => metadata.setContentType(m.name)) - cacheControl.foreach(metadata.setCacheControl) - metadata.setUserMetadata(meta.asJava) val fileMarkers = Map( "bucket" -> bucket, - "fileName" -> id, - "mimeType" -> mimeType.getOrElse("none"), ) val markers = logMarker ++ fileMarkers - val req = new PutObjectRequest(bucket, id, file).withMetadata(metadata) + val reqBuilder = PutObjectRequestV2.builder().key(id).bucket(bucket) + cacheControl.foreach(c => reqBuilder.cacheControl(c)) + mimeType.foreach(m => reqBuilder.contentType(m.name)) + reqBuilder.metadata(meta.asJava) + val req = reqBuilder.build() + Stopwatch(s"S3 client.putObject ($req)"){ - client.putObject(req) + clientV2.putObject(req, RequestBody.fromFile(file)) // once we've completed the PUT read back to ensure that we are returning reality - val metadata = client.getObjectMetadata(bucket, id) - S3Object(bucket, id, metadata.getContentLength, S3Metadata(metadata)) + val metadata = clientV2.headObject( + HeadObjectRequest.builder().key(id).bucket(bucket).build() + ) + + S3Object(bucket, id, metadata.contentLength(), S3Metadata(metadata)) }(markers) } - def storeIfNotPresent(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) + def storeIfNotPresentV2(bucket: Bucket, id: Key, file: File, mimeType: Option[MimeType], meta: UserMetadata = Map.empty, cacheControl: Option[String] = None) (implicit ex: ExecutionContext, logMarker: LogMarker): Future[S3Object] = { - Future{ - Some(client.getObjectMetadata(bucket, id)) + Future { + Some(clientV2.headObject( + HeadObjectRequest.builder().key(id).bucket(bucket).build() + )) }.recover { // translate this exception into the object not existing - case as3e:AmazonS3Exception if as3e.getStatusCode == 404 => None + case _: NoSuchKeyException => None }.flatMap { - case Some(objectMetadata) => + case Some(metadata) => logger.info(logMarker, s"Skipping storing of S3 file $id as key is already present in bucket $bucket") - Future.successful(S3Object(bucket, id, objectMetadata.getContentLength, S3Metadata(objectMetadata))) + Future.successful(S3Object(bucket, id, metadata.contentLength(), S3Metadata(metadata))) case None => - store(bucket, id, file, mimeType, meta, cacheControl) + storeV2(bucket, id, file, mimeType, meta, cacheControl) } } diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index fefb71177c7..661bfb39898 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -32,7 +32,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati } def queueS3Object(uploader: String, filename: String, s3Meta: Map[String, String], file: File)(implicit logMarker: LogMarker) = { - store( + storeV2( config.maybeIngestBucket.get, s"$uploader/$filename", file, From c1a14904003c6aad1f369bea40642feac2b0ac81 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 10:20:01 +0100 Subject: [PATCH 145/373] Migrate the list method --- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 13 ++++++++++++- cropper/app/lib/CropStore.scala | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 21cf11c67f9..183be81d5c0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -12,7 +12,7 @@ import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.{GetObjectResponse, HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2, PutObjectRequest => PutObjectRequestV2} +import software.amazon.awssdk.services.s3.model.{GetObjectResponse, HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2, PutObjectRequest => PutObjectRequestV2, ListObjectsV2Request} import java.io.File import java.net.URI @@ -187,6 +187,17 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } + def listV2(bucket: Bucket, prefixDir: String) + (implicit ex: ExecutionContext): Future[List[S3Object]] = + Future { + val req = ListObjectsV2Request.builder().bucket(bucket).prefix(s"$prefixDir/").build() + val listing = clientV2.listObjectsV2(req) + val s3Objects = listing.contents().asScala.toList + s3Objects.map(s3Object => { + S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + }) + } + def getMetadata(bucket: Bucket, key: Key): S3Metadata = { val meta = client.getObjectMetadata(bucket, key) S3Metadata(meta) diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala index a81b141793c..537766b915b 100644 --- a/cropper/app/lib/CropStore.scala +++ b/cropper/app/lib/CropStore.scala @@ -27,7 +27,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS } def listCrops(id: String): Future[List[Crop]] = { - list(config.imgPublishingBucket, id).map { crops => + listV2(config.imgPublishingBucket, id).map { crops => crops.foldLeft(Map[String, Crop]()) { case (map, (s3Object)) => { val filename::containingFolder::_ = s3Object.uri.getPath.split("/").reverse.toList From f03a00b3a6d704c47b4d5e17fbcb0bc5c00a3886 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 10:28:18 +0100 Subject: [PATCH 146/373] Port list, getMetadata, syncFindKey to v2 --- .../com/gu/mediaservice/lib/auth/KeyStore.scala | 2 +- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index 855f4bf7e9b..3c15d8c113f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -12,7 +12,7 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte def lookupIdentity(key: String): Option[ApiAccessor] = store.get().get(key) - def findKey(prefix: String): Option[String] = s3.syncFindKey(bucket, prefix) + def findKey(prefix: String): Option[String] = s3.syncFindKeyV2(bucket, prefix) def update(): Unit = { store.set(fetchAll) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 183be81d5c0..ee09af0e652 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -194,7 +194,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val listing = clientV2.listObjectsV2(req) val s3Objects = listing.contents().asScala.toList s3Objects.map(s3Object => { - S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadataV2(bucket, s3Object.key())) }) } @@ -203,6 +203,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with S3Metadata(meta) } + def getMetadataV2(bucket: Bucket, key: Key): S3Metadata = { + val meta = clientV2.headObject(HeadObjectRequest.builder().key(key).bucket(bucket).build()) + S3Metadata(meta) + } + def getUserMetadata(bucket: Bucket, key: Key): Map[Bucket, Bucket] = client.getObjectMetadata(bucket, key).getUserMetadata.asScala.toMap @@ -212,6 +217,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val summaries = listing.getObjectSummaries.asScala summaries.headOption.map(_.getKey) } + def syncFindKeyV2(bucket: Bucket, prefixName: String): Option[Key] = { + val req = ListObjectsV2Request.builder().bucket(bucket).prefix(s"$prefixName-").build() + val objects = clientV2.listObjectsV2(req).contents().asScala.toList + objects.headOption.map(_.key()) + } } From f226b1674dc44faadf8e36dbbb0f0b03fe7963ff Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 10:20:01 +0100 Subject: [PATCH 147/373] Migrate the list method --- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 13 ++++++++++++- cropper/app/lib/CropStore.scala | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 21cf11c67f9..183be81d5c0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -12,7 +12,7 @@ import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.{GetObjectResponse, HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2, PutObjectRequest => PutObjectRequestV2} +import software.amazon.awssdk.services.s3.model.{GetObjectResponse, HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2, PutObjectRequest => PutObjectRequestV2, ListObjectsV2Request} import java.io.File import java.net.URI @@ -187,6 +187,17 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } + def listV2(bucket: Bucket, prefixDir: String) + (implicit ex: ExecutionContext): Future[List[S3Object]] = + Future { + val req = ListObjectsV2Request.builder().bucket(bucket).prefix(s"$prefixDir/").build() + val listing = clientV2.listObjectsV2(req) + val s3Objects = listing.contents().asScala.toList + s3Objects.map(s3Object => { + S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + }) + } + def getMetadata(bucket: Bucket, key: Key): S3Metadata = { val meta = client.getObjectMetadata(bucket, key) S3Metadata(meta) diff --git a/cropper/app/lib/CropStore.scala b/cropper/app/lib/CropStore.scala index a81b141793c..537766b915b 100644 --- a/cropper/app/lib/CropStore.scala +++ b/cropper/app/lib/CropStore.scala @@ -27,7 +27,7 @@ class CropStore(config: CropperConfig) extends S3ImageStorage(config) with CropS } def listCrops(id: String): Future[List[Crop]] = { - list(config.imgPublishingBucket, id).map { crops => + listV2(config.imgPublishingBucket, id).map { crops => crops.foldLeft(Map[String, Crop]()) { case (map, (s3Object)) => { val filename::containingFolder::_ = s3Object.uri.getPath.split("/").reverse.toList From a8b7c6d6bf766277546376838ab67e1fe5832b06 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 10:28:18 +0100 Subject: [PATCH 148/373] Port list, getMetadata, syncFindKey to v2 --- .../com/gu/mediaservice/lib/auth/KeyStore.scala | 2 +- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala index 855f4bf7e9b..3c15d8c113f 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/auth/KeyStore.scala @@ -12,7 +12,7 @@ class KeyStore(bucket: String, config: CommonConfig)(implicit ec: ExecutionConte def lookupIdentity(key: String): Option[ApiAccessor] = store.get().get(key) - def findKey(prefix: String): Option[String] = s3.syncFindKey(bucket, prefix) + def findKey(prefix: String): Option[String] = s3.syncFindKeyV2(bucket, prefix) def update(): Unit = { store.set(fetchAll) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 183be81d5c0..ee09af0e652 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -194,7 +194,7 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val listing = clientV2.listObjectsV2(req) val s3Objects = listing.contents().asScala.toList s3Objects.map(s3Object => { - S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadata(bucket, s3Object.key())) + S3Object(bucket, s3Object.key(), size = s3Object.size(), metadata = getMetadataV2(bucket, s3Object.key())) }) } @@ -203,6 +203,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with S3Metadata(meta) } + def getMetadataV2(bucket: Bucket, key: Key): S3Metadata = { + val meta = clientV2.headObject(HeadObjectRequest.builder().key(key).bucket(bucket).build()) + S3Metadata(meta) + } + def getUserMetadata(bucket: Bucket, key: Key): Map[Bucket, Bucket] = client.getObjectMetadata(bucket, key).getUserMetadata.asScala.toMap @@ -212,6 +217,11 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val summaries = listing.getObjectSummaries.asScala summaries.headOption.map(_.getKey) } + def syncFindKeyV2(bucket: Bucket, prefixName: String): Option[Key] = { + val req = ListObjectsV2Request.builder().bucket(bucket).prefix(s"$prefixName-").build() + val objects = clientV2.listObjectsV2(req).contents().asScala.toList + objects.headOption.map(_.key()) + } } From d23ef4862a01170bc150abb82941dc93e2487268 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 11:27:16 +0100 Subject: [PATCH 149/373] Port bulk delete --- .../lib/ImageIngestOperations.scala | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index 0fdd57bb676..eb401fb07a1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -1,6 +1,5 @@ package com.gu.mediaservice.lib -import com.amazonaws.services.s3.model.{DeleteObjectsRequest, MultiObjectDeleteException} import java.io.File import com.gu.mediaservice.lib.config.CommonConfig @@ -8,6 +7,7 @@ import com.gu.mediaservice.lib.aws.S3Object import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.{MimeType, Png} import org.joda.time.DateTime +import software.amazon.awssdk.services.s3.model.{Delete, DeleteObjectsRequest, ObjectIdentifier} import scala.concurrent.Future import scala.jdk.CollectionConverters._ @@ -45,33 +45,33 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config storeImage(imageBucket, optimisedPngKeyFromId(storableImage.id), storableImage.file, Some(storableImage.mimeType), overwrite = true) - private def bulkDelete(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match { + + private def bulkDeleteV2(bucket: String, keys: List[String]): Future[Map[String, Boolean]] = keys match { case Nil => Future.successful(Map.empty) case _ => Future { - try { - client.deleteObjects( - new DeleteObjectsRequest(bucket).withKeys(keys: _*) - ) - keys.map { key => - key -> true - }.toMap - } catch { - case partialFailure: MultiObjectDeleteException => - logger.warn(s"Partial failure when deleting images from $bucket: ${partialFailure.getMessage} ${partialFailure.getErrors}") - val errorKeys = partialFailure.getErrors.asScala.map(_.getKey).toSet - keys.map { key => - key -> !errorKeys.contains(key) - }.toMap - } + val objects = keys.map { key => + ObjectIdentifier.builder() + .key(key) + .build() + }.asJava + val response = clientV2.deleteObjects( + DeleteObjectsRequest.builder().bucket(bucket) + .delete(Delete.builder().objects(objects).build()) + .build() + ) + val errorKeys = response.errors().asScala.toList.map(_.key()) + keys.map { key => + key -> !errorKeys.contains(key) + }.toMap } } def deleteOriginal(id: String)(implicit logMarker: LogMarker): Future[Unit] = if(isVersionedS3) deleteVersionedImage(imageBucket, fileKeyFromId(id)) else deleteImage(imageBucket, fileKeyFromId(id)) - def deleteOriginals(ids: Set[String]) = bulkDelete(imageBucket, ids.map(fileKeyFromId).toList) + def deleteOriginals(ids: Set[String]) = bulkDeleteV2(imageBucket, ids.map(fileKeyFromId).toList) def deleteThumbnail(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(thumbnailBucket, fileKeyFromId(id)) - def deleteThumbnails(ids: Set[String]) = bulkDelete(thumbnailBucket, ids.map(fileKeyFromId).toList) + def deleteThumbnails(ids: Set[String]) = bulkDeleteV2(thumbnailBucket, ids.map(fileKeyFromId).toList) def deletePNG(id: String)(implicit logMarker: LogMarker): Future[Unit] = deleteImage(imageBucket, optimisedPngKeyFromId(id)) - def deletePNGs(ids: Set[String]) = bulkDelete(imageBucket, ids.map(optimisedPngKeyFromId).toList) + def deletePNGs(ids: Set[String]) = bulkDeleteV2(imageBucket, ids.map(optimisedPngKeyFromId).toList) def doesOriginalExist(id: String): Boolean = client.doesObjectExist(imageBucket, fileKeyFromId(id)) From 5d2c452d15f9d436fe25f88ad996fbc20c1311e5 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 11:35:19 +0100 Subject: [PATCH 150/373] Port doesObjectExist --- .../lib/ImageIngestOperations.scala | 18 +++++++++++++++++- .../controllers/ImageLoaderController.scala | 2 +- thrall/app/controllers/ThrallController.scala | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala index eb401fb07a1..eaa8e54a726 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/ImageIngestOperations.scala @@ -7,7 +7,8 @@ import com.gu.mediaservice.lib.aws.S3Object import com.gu.mediaservice.lib.logging.LogMarker import com.gu.mediaservice.model.{MimeType, Png} import org.joda.time.DateTime -import software.amazon.awssdk.services.s3.model.{Delete, DeleteObjectsRequest, ObjectIdentifier} +import software.amazon.awssdk.core.exception.SdkClientException +import software.amazon.awssdk.services.s3.model.{Delete, DeleteObjectsRequest, HeadObjectRequest, NoSuchKeyException, ObjectIdentifier} import scala.concurrent.Future import scala.jdk.CollectionConverters._ @@ -75,6 +76,21 @@ class ImageIngestOperations(imageBucket: String, thumbnailBucket: String, config def doesOriginalExist(id: String): Boolean = client.doesObjectExist(imageBucket, fileKeyFromId(id)) + + def doesOriginalExistV2(id: String): Boolean = + try { + clientV2.headObject(HeadObjectRequest.builder().bucket(imageBucket).key(fileKeyFromId(id)).build()) + logger.info(s"found ${fileKeyFromId(id)} ") + true + } catch { + case _: NoSuchKeyException => + logger.info(s"did not find ${fileKeyFromId(id)} ") + false + case ex: SdkClientException => + logger.error(s"error calling head on s3 bucket", ex) + throw ex + } + } sealed trait ImageWrapper { diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index 500206ccaa1..3ae99414780 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -577,7 +577,7 @@ class ImageLoaderController(auth: Authentication, Future { config.maybeImageReplicaBucket match { - case _ if store.doesOriginalExist(imageId) => + case _ if store.doesOriginalExistV2(imageId) => Future.successful(Conflict("Image already exists in main bucket")) case None => Future.successful(NotImplemented("No replica bucket configured")) diff --git a/thrall/app/controllers/ThrallController.scala b/thrall/app/controllers/ThrallController.scala index f0ecea7fd6c..2dfbf5a3eeb 100644 --- a/thrall/app/controllers/ThrallController.scala +++ b/thrall/app/controllers/ThrallController.scala @@ -68,7 +68,7 @@ class ThrallController( def upsertProjectPage(imageId: Option[String]) = withLoginRedirectAsync { implicit request => imageId match { - case Some(id) if store.doesOriginalExist(id) => + case Some(id) if store.doesOriginalExistV2(id) => gridClient.getProjectionDiff(id, auth.innerServiceCall).map { case None => NotFound("couldn't generate projection for that image!!") case Some(diff) => Ok(views.html.previewUpsertProject(id, Json.prettyPrint(diff))) From 830fd70afd4e4ee05c800f9616f2aa8033517e83 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 12:18:53 +0100 Subject: [PATCH 151/373] Upgrade the restore s3client --- .../controllers/ImageLoaderController.scala | 32 ++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index 3ae99414780..7628839ed6e 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -3,7 +3,6 @@ package controllers import org.apache.pekko.Done import org.apache.pekko.stream.Materializer import org.apache.pekko.stream.scaladsl.Source -import com.amazonaws.services.s3.AmazonS3 import software.amazon.awssdk.services.sqs.model.{Message => SQSMessage} import com.amazonaws.util.IOUtils import com.drew.imaging.ImageProcessingException @@ -26,18 +25,23 @@ import lib.storage.{ImageLoaderStore, S3FileDoesNotExistException} import lib._ import model.upload.UploadRequest import model.{Projector, QuarantineUploader, S3FileExtractedMetadata, S3IngestObject, StatusType, UploadStatus, UploadStatusRecord, UploadStatusUri, Uploader} +import org.joda.time.{DateTime, DateTimeZone} import play.api.data.Form import play.api.data.Forms._ import play.api.inject.ApplicationLifecycle import play.api.libs.json.Json import play.api.mvc._ +import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.cloudwatch.model.Dimension +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.{GetObjectRequest, HeadObjectRequest, NoSuchKeyException} import java.io.{File, FileOutputStream} import java.net.URI import java.time.Instant import java.util.concurrent.atomic.AtomicReference import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters.MapHasAsScala import scala.util.control.NonFatal import scala.util.{Failure, Success, Try} @@ -558,7 +562,15 @@ class ImageLoaderController(auth: Authentication, } } - lazy val replicaS3: AmazonS3 = S3Ops.buildS3Client(config, maybeRegionOverride = Some("us-west-1")) + lazy val replicaS3: S3Client = S3Ops.buildS3ClientV2(config, maybeRegionOverride = Some(Region.US_WEST_1)) + def doesObjectExist(bucket: String, key: String) = { + try { + replicaS3.headObject(HeadObjectRequest.builder().bucket(bucket).key(key).build()) + true + } catch { + case _: NoSuchKeyException => false + } + } private case class RestoreFromReplicaForm(imageId: String) def restoreFromReplica: Action[AnyContent] = AuthenticatedAndAuthorised.async { implicit request => @@ -581,20 +593,24 @@ class ImageLoaderController(auth: Authentication, Future.successful(Conflict("Image already exists in main bucket")) case None => Future.successful(NotImplemented("No replica bucket configured")) - case Some(replicaBucket) if replicaS3.doesObjectExist(replicaBucket, fileKeyFromId(imageId)) => + case Some(replicaBucket) if doesObjectExist(replicaBucket, fileKeyFromId(imageId)) => val s3Key = fileKeyFromId(imageId) logger.info(logMarker, s"Restoring image $imageId from replica bucket $replicaBucket (key: $s3Key)") - val replicaObject = replicaS3.getObject(replicaBucket, s3Key) - val metadata = S3FileExtractedMetadata(replicaObject.getObjectMetadata) - val stream = replicaObject.getObjectContent + val replicaObject = replicaS3.getObject( + GetObjectRequest.builder().bucket(replicaBucket).key(s3Key).build() + ) + val lastModified = replicaObject.response().lastModified() + val metaMap = replicaObject.response().metadata().asScala.toMap + val metadata = S3FileExtractedMetadata.apply(new DateTime(lastModified.toEpochMilli).withZone(DateTimeZone.UTC), metaMap) val tempFile = createTempFile(s"restoringReplica-$imageId") val fos = new FileOutputStream(tempFile) try { - IOUtils.copy(stream, fos) + replicaObject.transferTo(fos) } finally { - stream.close() + replicaObject.close() + fos.close() } val future = uploader.restoreFile( From 22d986ef53abc13f5a878eaa03480993404e77b1 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 12:51:14 +0100 Subject: [PATCH 152/373] Port image loader store --- .../controllers/ImageLoaderController.scala | 5 +- image-loader/app/lib/ImageLoaderStore.scala | 55 ++++++++++++------- image-loader/app/model/S3IngestObject.scala | 15 +++-- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/image-loader/app/controllers/ImageLoaderController.scala b/image-loader/app/controllers/ImageLoaderController.scala index 7628839ed6e..65fd6ee1636 100644 --- a/image-loader/app/controllers/ImageLoaderController.scala +++ b/image-loader/app/controllers/ImageLoaderController.scala @@ -38,7 +38,7 @@ import software.amazon.awssdk.services.s3.model.{GetObjectRequest, HeadObjectReq import java.io.{File, FileOutputStream} import java.net.URI -import java.time.Instant +import java.time.{Duration, Instant} import java.util.concurrent.atomic.AtomicReference import scala.concurrent.{ExecutionContext, Future} import scala.jdk.CollectionConverters.MapHasAsScala @@ -241,6 +241,7 @@ class ImageLoaderController(auth: Authentication, def getPreSignedUploadUrlsAndTrack: Action[AnyContent] = AuthenticatedAndAuthorised.async { request => val expiration = DateTimeUtils.now().plusHours(1) + val signatureDuration = Duration.ofHours(1) val mediaIdToFilenameMap = request.body.asJson.get.as[Map[String, String]] @@ -250,7 +251,7 @@ class ImageLoaderController(auth: Authentication, mediaIdToFilenameMap.map{case (mediaId, filename) => - val preSignedUrl = store.generatePreSignedUploadUrl(filename, expiration, uploadedBy, mediaId) + val preSignedUrl = store.generatePreSignedUploadUrl(filename, signatureDuration, uploadedBy, mediaId) uploadStatusTable.setStatus(UploadStatusRecord( id = mediaId, diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 661bfb39898..78dcafcc236 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -1,14 +1,17 @@ package lib.storage -import com.amazonaws.HttpMethod -import com.amazonaws.services.s3.model.{AmazonS3Exception, GeneratePresignedUrlRequest, S3Object} +import software.amazon.awssdk.services.s3.model.{CopyObjectRequest, DeleteObjectRequest, GetObjectRequest, GetObjectResponse, PutObjectRequest, S3Exception} + +import java.time.Duration +import scala.jdk.CollectionConverters.MapHasAsJava +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.services.s3.presigner.S3Presigner +import software.amazon.awssdk.services.s3.presigner.model.PutObjectPresignRequest import lib.ImageLoaderConfig import com.gu.mediaservice.lib import com.gu.mediaservice.lib.logging.LogMarker import java.io.File -import java.time.ZonedDateTime -import java.util.Date class S3FileDoesNotExistException extends Exception() @@ -18,15 +21,16 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati try { doWork } catch { - case e: AmazonS3Exception if e.getStatusCode == 404 || e.getStatusCode == 403 => + case e: S3Exception if e.statusCode() == 404 || e.statusCode() == 403 => loggingIfNotFound throw new S3FileDoesNotExistException case other: Throwable => throw other } } - def getS3Object(key: String)(implicit logMarker: LogMarker): S3Object = handleNotFound(key) { - client.getObject(config.maybeIngestBucket.get, key) + def getS3Object(key: String)(implicit logMarker: LogMarker): ResponseInputStream[GetObjectResponse] = handleNotFound(key) { + clientV2.getObject( + GetObjectRequest.builder().bucket(config.maybeIngestBucket.get).key(key).build()) } { logger.error(logMarker, s"Attempted to read $key from ingest bucket, but it does not exist.") } @@ -41,29 +45,42 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati ) } - def generatePreSignedUploadUrl(filename: String, expiration: ZonedDateTime, uploadedBy: String, mediaId: String): String = { - val request = new GeneratePresignedUrlRequest( - config.maybeBucketForUIUploads.get, // bucket - s"$uploadedBy/$filename", // key - ) - .withMethod(HttpMethod.PUT) - .withExpiration(Date.from(expiration.toInstant)); + def generatePreSignedUploadUrl(filename: String, duration: Duration, uploadedBy: String, mediaId: String): String = { + val presigner = S3Presigner.create() - // sent by the client in manager.js - request.putCustomRequestHeader("x-amz-meta-media-id", mediaId) + val putObjectRequest = PutObjectRequest.builder() + .bucket(config.maybeBucketForUIUploads.get).key(s"$uploadedBy/$filename").metadata(Map( + "media-id" -> mediaId).asJava) + .build() + val putObjectPresignRequest = + PutObjectPresignRequest.builder() + .putObjectRequest(putObjectRequest) + .signatureDuration(duration) - client.generatePresignedUrl(request).toString + .build(); + + val req = presigner.presignPutObject(putObjectPresignRequest) + req.url().toExternalForm } def moveObjectToFailedBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key){ - client.copyObject(config.maybeIngestBucket.get, key, config.maybeFailBucket.get, key) + clientV2.copyObject( + CopyObjectRequest.builder() + .sourceBucket(config.maybeIngestBucket.get) + .sourceKey(key) + .destinationBucket(config.maybeFailBucket.get) + .destinationKey(key) + .build() + ) deleteObjectFromIngestBucket(key) } { logger.warn(logMarker, s"Attempted to copy $key from ingest bucket to fail bucket, but it does not exist.") } def deleteObjectFromIngestBucket(key: String)(implicit logMarker: LogMarker) = handleNotFound(key) { - client.deleteObject(config.maybeIngestBucket.get,key) + clientV2.deleteObject( + DeleteObjectRequest.builder().bucket(config.maybeIngestBucket.get).key(key).build()) + () } { logger.warn(logMarker, s"Attempted to delete $key from ingest bucket, but it does not exist.") } diff --git a/image-loader/app/model/S3IngestObject.scala b/image-loader/app/model/S3IngestObject.scala index 7621ac2067e..c6179d83635 100644 --- a/image-loader/app/model/S3IngestObject.scala +++ b/image-loader/app/model/S3IngestObject.scala @@ -4,6 +4,7 @@ import com.gu.mediaservice.lib.ImageStorageProps import com.gu.mediaservice.lib.logging.LogMarker import lib.storage.ImageLoaderStore +import java.util.Date import scala.jdk.CollectionConverters._ case class S3IngestObject ( @@ -20,21 +21,19 @@ case class S3IngestObject ( object S3IngestObject { def apply (key: String, store: ImageLoaderStore)(implicit logMarker: LogMarker): S3IngestObject = { - val keyParts = key.split("/") val s3Object = store.getS3Object(key) - val metadata = s3Object.getObjectMetadata - + val metadata = s3Object.response().metadata() S3IngestObject( key, uploadedBy = keyParts.head, filename = keyParts.last, - maybeMediaIdFromUiUpload = metadata.getUserMetadata.asScala.get("media-id"), // set by the client in upload in manager.js - uploadTime = metadata.getLastModified, - contentLength = metadata.getContentLength, - getInputStream = () => s3Object.getObjectContent, - identifiers = metadata.getUserMetadata.asScala.collect{ + maybeMediaIdFromUiUpload = metadata.asScala.toMap.get("media-id"), // set by the client in upload in manager.js + uploadTime = new Date(s3Object.response().lastModified().toEpochMilli), + contentLength = s3Object.response().contentLength(), + getInputStream = () => s3Object, + identifiers = metadata.asScala.collect{ case (key, value) if key.startsWith(ImageStorageProps.identifierMetadataKeyPrefix) => key.stripPrefix(ImageStorageProps.identifierMetadataKeyPrefix) -> value }.toMap From e228981d7d9157545da526aca92d6332c4d9707e Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 14:28:35 +0100 Subject: [PATCH 153/373] Upgrade the rest of the parts --- build.sbt | 4 +- .../com/gu/mediaservice/lib/aws/S3.scala | 10 +++++ image-loader/app/model/Projector.scala | 42 +++++++++---------- image-loader/app/model/Uploader.scala | 20 +++++---- .../test/scala/model/ProjectorTest.scala | 31 +++++--------- .../PermissionsAuthorisationProvider.scala | 4 +- thrall/app/ThrallComponents.scala | 2 +- thrall/app/lib/SyncChecker.scala | 24 +++++------ 8 files changed, 69 insertions(+), 68 deletions(-) diff --git a/build.sbt b/build.sbt index 16faa7feddc..e4cc712dd4a 100644 --- a/build.sbt +++ b/build.sbt @@ -91,7 +91,7 @@ val maybeBBCLib: Option[sbt.ProjectReference] = if(bbcBuildProcess) Some(bbcProj lazy val commonLib = project("common-lib").settings( libraryDependencies ++= Seq( - "com.gu" %% "editorial-permissions-client" % "4.0.0", + "com.gu" %% "editorial-permissions-client" % "7.0.0", "com.gu" %% "pan-domain-auth-play_3-0" % "19.0.0", "com.amazonaws" % "aws-java-sdk-s3" % awsSdkVersion, "com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion, @@ -125,7 +125,7 @@ lazy val commonLib = project("common-lib").settings( "software.amazon.awssdk" % "bedrockruntime" % awsSdkV2Version, "software.amazon.awssdk" % "s3vectors" % awsSdkV2Version, ws, - "org.testcontainers" % "elasticsearch" % "1.21.4" % Test + "org.testcontainers" % "elasticsearch" % "1.21.4" % Test, ), dependencyOverrides += "ch.qos.logback" % "logback-classic" % "1.2.13" % Test ) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index ee09af0e652..be887785ea0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -222,6 +222,16 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with val objects = clientV2.listObjectsV2(req).contents().asScala.toList objects.headOption.map(_.key()) } + def doesObjectExistV2(bucket: Bucket, key: String) = { + try { + clientV2.headObject( + HeadObjectRequest.builder().key(key).bucket(bucket).build() + ) + true + } catch { + case _: NoSuchKeyException => false + } + } } diff --git a/image-loader/app/model/Projector.scala b/image-loader/app/model/Projector.scala index 20b222382a1..9d6de6f2a31 100644 --- a/image-loader/app/model/Projector.scala +++ b/image-loader/app/model/Projector.scala @@ -1,13 +1,14 @@ package model +import software.amazon.awssdk.core.ResponseInputStream +import software.amazon.awssdk.services.s3.model.GetObjectResponse + import java.io.{File, FileOutputStream} -import com.amazonaws.services.s3.AmazonS3 import com.gu.mediaservice.{GridClient, ImageDataMerger} import com.gu.mediaservice.lib.auth.Authentication -import com.amazonaws.services.s3.model.{GetObjectRequest, ObjectMetadata, S3Object => AwsS3Object} import com.gu.mediaservice.lib.ImageIngestOperations.{fileKeyFromId, optimisedPngKeyFromId} import com.gu.mediaservice.lib.{ImageIngestOperations, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Ops} +import com.gu.mediaservice.lib.aws.{Embedder, S3, S3Object} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} @@ -19,9 +20,8 @@ import model.upload.UploadRequest import org.apache.commons.io.IOUtils import org.joda.time.{DateTime, DateTimeZone} import play.api.libs.ws.WSRequest -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse -import java.nio.file.Path +import java.time.Instant import scala.jdk.CollectionConverters._ import scala.concurrent.duration.Duration import scala.concurrent.{Await, ExecutionContext, Future} @@ -31,7 +31,7 @@ object Projector { import Uploader.toImageUploadOpsCfg def apply(config: ImageLoaderConfig, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, maybeEmbedder: Option[Embedder])(implicit ec: ExecutionContext): Projector - = new Projector(toImageUploadOpsCfg(config), S3Ops.buildS3Client(config), imageOps, processor, auth, maybeEmbedder) + = new Projector(toImageUploadOpsCfg(config), new S3(config), imageOps, processor, auth, maybeEmbedder) } case class S3FileExtractedMetadata( @@ -42,12 +42,12 @@ case class S3FileExtractedMetadata( ) object S3FileExtractedMetadata { - def apply(s3ObjectMetadata: ObjectMetadata): S3FileExtractedMetadata = { - val lastModified = new DateTime(s3ObjectMetadata.getLastModified) - val userMetadata = s3ObjectMetadata.getUserMetadata.asScala.toMap + + def apply(s3Object: S3Object): S3FileExtractedMetadata = { + val lastModified = s3Object.metadata.objectMetadata.lastModified.getOrElse(new DateTime) + val userMetadata = s3Object.metadata.userMetadata apply(lastModified, userMetadata) } - def apply(lastModified: DateTime, userMetadata: Map[String, String]): S3FileExtractedMetadata = { val fileUserMetadata = userMetadata.map { case (key, value) => // Fix up the contents of the metadata. @@ -81,7 +81,7 @@ object S3FileExtractedMetadata { } class Projector(config: ImageUploadOpsCfg, - s3: AmazonS3, + s3: S3, imageOps: ImageOperations, processor: ImageProcessor, auth: Authentication, @@ -95,17 +95,17 @@ class Projector(config: ImageUploadOpsCfg, import ImageIngestOperations.fileKeyFromId val s3Key = fileKeyFromId(imageId) - if (!s3.doesObjectExist(config.originalFileBucket, s3Key)) + if (!s3.doesObjectExistV2(config.originalFileBucket, s3Key)) throw new NoSuchImageExistsInS3(config.originalFileBucket, s3Key) val s3Source = Stopwatch(s"object exists, getting s3 object at s3://${config.originalFileBucket}/$s3Key to perform Image projection"){ - s3.getObject(config.originalFileBucket, s3Key) + s3.getObjectV2(config.originalFileBucket, s3Key) }(logMarker) try { val digestedFile = getSrcFileDigestForProjection(s3Source, imageId, tempFile) - val extractedS3Meta = S3FileExtractedMetadata(s3Source.getObjectMetadata) - + val lastModifiedInstant = s3Source.response().lastModified() + val extractedS3Meta = S3FileExtractedMetadata(new DateTime(lastModifiedInstant.toEpochMilli).withZone(DateTimeZone.UTC), s3Source.response().metadata().asScala.toMap) val finalImageFuture = projectImage(digestedFile, extractedS3Meta, gridClient, onBehalfOfFn) val finalImage = Await.result(finalImageFuture, Duration.Inf) @@ -116,10 +116,10 @@ class Projector(config: ImageUploadOpsCfg, } } - private def getSrcFileDigestForProjection(s3Src: AwsS3Object, imageId: String, tempFile: File) = { + private def getSrcFileDigestForProjection(s3Src: ResponseInputStream[GetObjectResponse], imageId: String, tempFile: File) = { val fos = new FileOutputStream(tempFile) try { - IOUtils.copy(s3Src.getObjectContent, fos) + IOUtils.copy(s3Src, fos) DigestedFile(tempFile, imageId) } finally { fos.close() @@ -159,7 +159,7 @@ class Projector(config: ImageUploadOpsCfg, class ImageUploadProjectionOps(config: ImageUploadOpsCfg, imageOps: ImageOperations, processor: ImageProcessor, - s3: AmazonS3, + s3: S3, maybeEmbedder: Option[Embedder], ) extends GridLogging { @@ -210,16 +210,16 @@ class ImageUploadProjectionOps(config: ImageUploadOpsCfg, bucket: String, key: String, outFile: File )(implicit ec: ExecutionContext, logMarker: LogMarker): Future[Option[(File, MimeType)]] = { logger.info(logMarker, s"Trying fetch existing image from S3 bucket - $bucket at key $key") - val doesFileExist = Future { s3.doesObjectExist(bucket, key) } recover { case _ => false } + val doesFileExist = Future { s3.doesObjectExistV2(bucket, key) } recover { case _ => false } doesFileExist.flatMap { case false => logger.warn(logMarker, s"image did not exist in bucket $bucket at key $key") Future.successful(None) // falls back to creating from original file case true => - val obj = s3.getObject(new GetObjectRequest(bucket, key)) + val obj = s3.getObjectV2(bucket, key) val fos = new FileOutputStream(outFile) try { - IOUtils.copy(obj.getObjectContent, fos) + IOUtils.copy(obj, fos) } finally { fos.close() obj.close() diff --git a/image-loader/app/model/Uploader.scala b/image-loader/app/model/Uploader.scala index aaa811e52c6..390a7484740 100644 --- a/image-loader/app/model/Uploader.scala +++ b/image-loader/app/model/Uploader.scala @@ -9,7 +9,7 @@ import java.nio.file.{Files, Path} import com.gu.mediaservice.lib.argo.ArgoHelpers import com.gu.mediaservice.lib.auth.Authentication import com.gu.mediaservice.lib.{BrowserViewableImage, ImageStorageProps, StorableOptimisedImage, StorableOriginalImage, StorableThumbImage} -import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, S3Vectors, UpdateMessage} +import com.gu.mediaservice.lib.aws.{Embedder, EmbedderMessage, S3Object, UpdateMessage} import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.formatting._ import com.gu.mediaservice.lib.imaging.ImageOperations @@ -25,12 +25,13 @@ import lib.storage.ImageLoaderStore import model.Uploader.{fromUploadRequestShared, toImageUploadOpsCfg} import model.upload.{OptimiseOps, OptimiseWithPngQuant, UploadRequest} import org.joda.time.DateTime -import play.api.libs.json.{JsObject, Json} +import play.api.libs.json.Json import play.api.libs.ws.WSRequest -import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse +import software.amazon.awssdk.services.s3.model.CopyObjectRequest import scala.collection.compat._ import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters.MapHasAsScala case class ImageUpload(uploadRequest: UploadRequest, image: Image) @@ -320,7 +321,8 @@ object Uploader extends GridLogging { } def patchUploadRequestWithS3Metadata(request: UploadRequest, s3Object: S3Object): UploadRequest = { - val metadata = S3FileExtractedMetadata(s3Object.metadata.objectMetadata.lastModified.getOrElse(new DateTime), s3Object.metadata.userMetadata) + val lastModified = s3Object.metadata.objectMetadata.lastModified + val metadata = S3FileExtractedMetadata(lastModified.getOrElse(new DateTime), s3Object.metadata.userMetadata) request.copy( uploadTime = metadata.uploadTime, uploadedBy = metadata.uploadedBy, @@ -470,9 +472,13 @@ class Uploader( val mediaId = imageUpload.image.id logger.info(logMarker, s"Copying $mediaId to lower environment queue bucket $lowerEnvironmentQueueBucket") try { - store.client.copyObject( - config.imageBucket, fileKeyFromId(mediaId), - lowerEnvironmentQueueBucket, s"${uploadRequest.uploadedBy}/${uploadRequest.uploadInfo.filename.getOrElse(mediaId)}" + store.clientV2.copyObject( + CopyObjectRequest.builder() + .sourceBucket(config.imageBucket) + .sourceKey(fileKeyFromId(mediaId)) + .destinationBucket(lowerEnvironmentQueueBucket) + .destinationKey(s"${uploadRequest.uploadedBy}/${uploadRequest.uploadInfo.filename.getOrElse(mediaId)}") + .build() ) } catch { case e: Throwable => diff --git a/image-loader/test/scala/model/ProjectorTest.scala b/image-loader/test/scala/model/ProjectorTest.scala index eccec5b65f0..a65669cfa7a 100644 --- a/image-loader/test/scala/model/ProjectorTest.scala +++ b/image-loader/test/scala/model/ProjectorTest.scala @@ -7,6 +7,7 @@ import com.amazonaws.services.s3.AmazonS3 import com.amazonaws.services.s3.model.ObjectMetadata import com.gu.mediaservice.GridClient import com.gu.mediaservice.lib.auth.Authentication +import com.gu.mediaservice.lib.aws.S3 import com.gu.mediaservice.lib.cleanup.ImageProcessor import com.gu.mediaservice.lib.imaging.ImageOperations import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} @@ -22,6 +23,7 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.time.{Millis, Span} import org.scalatestplus.mockito.MockitoSugar import play.api.libs.json.{JsArray, JsString} +import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3vectors.model.PutVectorsResponse import test.lib.ResourceHelpers @@ -44,7 +46,7 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc private val maybeEmbedder = None - private val s3 = mock[AmazonS3] + private val s3 = mock[S3] private val auth = mock[Authentication] private val projector = new Projector(config, s3, imageOperations, ImageProcessor.identity, auth, maybeEmbedder) @@ -234,16 +236,12 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc "S3FileExtractedMetadata" - { "should extract URL encoded metadata" in { - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( + val result = S3FileExtractedMetadata(new DateTime(1613388118000L), Map( "file-name" -> "This%20photo%20was%20taken%20in%20%C5%81%C3%B3d%C5%BA.jpg", "uploaded-by" -> "s%C3%A9b.cevey%40theguardian.co.uk", "upload-time" -> "2021-02-01T12%3A52%3A34%2B09%3A00", "identifier!picdarurn" -> "12*543%5E25" - ).asJava) - - val result = S3FileExtractedMetadata(s3Metadata) + )) result.uploadFileName shouldBe Some("This photo was taken in Łódź.jpg") result.uploadedBy shouldBe "séb.cevey@theguardian.co.uk" result.uploadTime.toString shouldBe "2021-02-01T03:52:34.000Z" @@ -252,16 +250,13 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc } "should remap headers with underscores to dashes" in { - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( + + val result = S3FileExtractedMetadata(new DateTime(1613388118000L), Map( "file_name" -> "filename.jpg", "uploaded_by" -> "user", "upload_time" -> "2021-02-01T12%3A52%3A34%2B09%3A00", "identifier!picdarurn" -> "12*543" - ).asJava) - - val result = S3FileExtractedMetadata(s3Metadata) + )) result.uploadFileName shouldBe Some("filename.jpg") result.uploadedBy shouldBe "user" result.uploadTime.toString shouldBe "2021-02-01T03:52:34.000Z" @@ -270,16 +265,10 @@ class ProjectorTest extends AnyFreeSpec with Matchers with ScalaFutures with Moc } "should correctly read in non URL encoded values" in { - // we have plenty of values in S3 that are not URL encoded - // and we must be able to read them correctly - val s3Metadata = new ObjectMetadata() - s3Metadata.setLastModified(new Date(1613388118000L)) - s3Metadata.setUserMetadata(Map( + val result = S3FileExtractedMetadata(new DateTime(1613388118000L), Map( "uploaded_by" -> "user", "upload_time" -> "2019-12-11T01:12:10.427Z", - ).asJava) - - val result = S3FileExtractedMetadata(s3Metadata) + )) result.uploadedBy shouldBe "user" result.uploadTime.toString shouldBe "2019-12-11T01:12:10.427Z" } diff --git a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala index c374d5b6112..9fd64dd32c5 100644 --- a/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala +++ b/rest-lib/src/main/scala/com/gu/mediaservice/lib/guardian/auth/PermissionsAuthorisationProvider.scala @@ -20,12 +20,12 @@ class PermissionsAuthorisationProvider(configuration: Configuration, resources: private val permissions: PermissionsProvider = config.awsLocalEndpoint match { case Some(_) if config.isDev && config.useLocalAuth => - val provider = new S3PermissionsProvider(permissionsBucket, "permissions.json", 1.minute, PermissionsS3(S3Ops.buildS3Client(config))) + val provider = new S3PermissionsProvider(permissionsBucket, "permissions.json", 1.minute, PermissionsS3(S3Ops.buildS3ClientV2(config))) provider.start() provider case _ => val permissionsStage = if(config.isProd) { "PROD" } else { "CODE" } - PermissionsProvider(PermissionsConfig(permissionsStage, config.awsRegion, config.awsCredentials, permissionsBucket)) + PermissionsProvider(PermissionsConfig(permissionsStage, config.awsRegion, config.awsCredentialsV2, permissionsBucket)) } override def initialise(): Unit = { diff --git a/thrall/app/ThrallComponents.scala b/thrall/app/ThrallComponents.scala index cf9c5e75add..e0893484d6d 100644 --- a/thrall/app/ThrallComponents.scala +++ b/thrall/app/ThrallComponents.scala @@ -73,7 +73,7 @@ class ThrallComponents(context: Context) extends GridComponents(context, new Thr val streamRunning: Future[Done] = thrallStreamProcessor.run() - val s3 = S3Ops.buildS3Client(config) + val s3 = S3Ops.buildS3ClientV2(config) val s3Vectors = new S3Vectors(config) val syncChecker = new SyncChecker( s3, diff --git a/thrall/app/lib/SyncChecker.scala b/thrall/app/lib/SyncChecker.scala index 959057ce581..112a04325f2 100644 --- a/thrall/app/lib/SyncChecker.scala +++ b/thrall/app/lib/SyncChecker.scala @@ -4,11 +4,11 @@ import org.apache.pekko.Done import org.apache.pekko.actor.ActorSystem import org.apache.pekko.stream.{Materializer, RestartSettings} import org.apache.pekko.stream.scaladsl.{RestartSource, Source} -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.model.ListObjectsV2Request import com.gu.mediaservice.lib.elasticsearch.InProgress import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap} import lib.elasticsearch.ElasticSearch +import software.amazon.awssdk.services.s3.S3Client +import software.amazon.awssdk.services.s3.model.ListObjectsV2Request import scala.jdk.CollectionConverters._ import scala.concurrent.duration.DurationInt @@ -20,7 +20,7 @@ case class Prefix(prefix: String) extends SyncCheckJob case object Other extends SyncCheckJob class SyncChecker( - s3: AmazonS3, + s3: S3Client, es: ElasticSearch, imageBucketName: String, actorSystem: ActorSystem @@ -36,15 +36,15 @@ class SyncChecker( z <- expectedAlphabet } yield Prefix(s"$x$y$z")) :+ Other - private def paginatedListObjects(request: ListObjectsV2Request, maybeContinuationToken: Option[String] = None): Future[Seq[String]] = { + private def paginatedListObjects(request: ListObjectsV2Request.Builder, maybeContinuationToken: Option[String] = None): Future[Seq[String]] = { Future { val fullRequest = maybeContinuationToken match { - case Some(token) => request.withContinuationToken(token) + case Some(token) => request.continuationToken(token) case None => request } - val result = s3.listObjectsV2(fullRequest) - val keys = result.getObjectSummaries.asScala.toSeq.map(_.getKey.split("/").last) - val nextContinuationToken = Option(result.getNextContinuationToken) + val result = s3.listObjectsV2(fullRequest.build()) + val keys = result.contents().asScala.toSeq.map(_.key().split("/").last) + val nextContinuationToken = Option(result.nextContinuationToken()) keys -> nextContinuationToken } flatMap { case (keys, None) => Future.successful(keys) @@ -54,15 +54,11 @@ class SyncChecker( } private def listFilesWithPrefix(prefix: String): Future[Seq[String]] = { - val request = new ListObjectsV2Request() - .withBucketName(imageBucketName) - .withPrefix(prefix) + val request = ListObjectsV2Request.builder().bucket(imageBucketName).prefix(prefix) paginatedListObjects(request) } private def listFilesAtRoot(): Future[Seq[String]] = { - val request = new ListObjectsV2Request() - .withBucketName(imageBucketName) - .withDelimiter("/") + val request = ListObjectsV2Request.builder().bucket(imageBucketName).delimiter("/") paginatedListObjects(request) } From 50812e39d424adc938dde19c6210a82dd4801477 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 15:02:06 +0100 Subject: [PATCH 154/373] Remove the presigner to outside of the class --- image-loader/app/lib/ImageLoaderStore.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 78dcafcc236..925cc2649b1 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -44,9 +44,8 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati meta = s3Meta, ) } - + val presigner = S3Presigner.create() def generatePreSignedUploadUrl(filename: String, duration: Duration, uploadedBy: String, mediaId: String): String = { - val presigner = S3Presigner.create() val putObjectRequest = PutObjectRequest.builder() .bucket(config.maybeBucketForUIUploads.get).key(s"$uploadedBy/$filename").metadata(Map( @@ -56,7 +55,6 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati PutObjectPresignRequest.builder() .putObjectRequest(putObjectRequest) .signatureDuration(duration) - .build(); val req = presigner.presignPutObject(putObjectPresignRequest) From 1b32ab01ead0c92454f34fb8c21aa717623ea193 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 15:21:26 +0100 Subject: [PATCH 155/373] Update the signurl --- .../com/gu/mediaservice/lib/aws/S3.scala | 42 ++++++++++++++----- media-api/app/controllers/MediaApi.scala | 2 +- media-api/app/lib/ImageResponse.scala | 6 +-- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index be887785ea0..1ab26edbfac 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -7,16 +7,19 @@ import com.amazonaws.{AmazonServiceException, ClientConfiguration} import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, Stopwatch} import com.gu.mediaservice.model._ -import org.joda.time.{DateTime, DateTimeZone, Duration} +import org.joda.time.{DateTime, DateTimeZone} import software.amazon.awssdk.core.ResponseInputStream import software.amazon.awssdk.core.sync.RequestBody import software.amazon.awssdk.regions.Region import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.services.s3.model.{GetObjectResponse, HeadObjectRequest, HeadObjectResponse, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2, PutObjectRequest => PutObjectRequestV2, ListObjectsV2Request} +import software.amazon.awssdk.services.s3.model.{GetObjectResponse, HeadObjectRequest, HeadObjectResponse, ListObjectsV2Request, NoSuchKeyException, GetObjectRequest => GetObjectRequestV2, PutObjectRequest => PutObjectRequestV2} +import software.amazon.awssdk.services.s3.presigner.S3Presigner +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest import java.io.File import java.net.URI import java.nio.charset.StandardCharsets +import java.time.Duration import scala.jdk.CollectionConverters._ import scala.concurrent.{ExecutionContext, Future} @@ -83,19 +86,38 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with lazy val client: AmazonS3 = S3Ops.buildS3Client(config) lazy val clientV2: S3Client = S3Ops.buildS3ClientV2(config) - - def signUrl(bucket: Bucket, url: URI, image: Image, expiration: DateTime = cachableExpiration(), imageType: ImageFileType = Source): String = { - // get path and remove leading `/` - val key: Key = url.getPath.drop(1) + val presigner = S3Presigner.create() + def signUrlV2( + bucket: Bucket, + url: URI, + image: Image, + expiration: DateTime = cachableExpiration(), + imageType: ImageFileType = Source + ): String = { + // Fix key extraction (use stripPrefix to avoid corrupting relative paths) + val key: Key = url.getPath.stripPrefix("/") val contentDisposition = getContentDisposition(image, imageType, config.shortenDownloadFilename) - val headers = new ResponseHeaderOverrides().withContentDisposition(contentDisposition) + val nowMillis = System.currentTimeMillis() + val targetExpirationMillis = expiration.getMillis + val remainingSeconds = Math.max(1, (targetExpirationMillis - nowMillis) / 1000) - val request = new GeneratePresignedUrlRequest(bucket, key).withExpiration(expiration.toDate).withResponseHeaders(headers) - client.generatePresignedUrl(request).toExternalForm - } + val getObjectRequest = GetObjectRequestV2.builder() + .bucket(bucket) + .key(key) + .responseContentDisposition(contentDisposition) + .build() + + val getObjectPresignRequest = GetObjectPresignRequest.builder() + .getObjectRequest(getObjectRequest) + .signatureDuration(Duration.ofSeconds(remainingSeconds)) + .build() + val req = presigner.presignGetObject(getObjectPresignRequest) + req.url().toExternalForm + } + def getObjectV2(bucket: Bucket, url: URI): ResponseInputStream[GetObjectResponse]= { // get path and remove leading `/` val key: Key = url.getPath.drop(1) diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 9d63ea76ad2..873626badf2 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -494,7 +494,7 @@ class MediaApi( mediaApiMetrics.incrementImageDownload(apiKey, mediaApiMetrics.OptimisedDownloadType) val sourceImageUri = - new URI(s3Client.signUrl(config.imageBucket, image.optimisedPng.getOrElse(image.source).file, image, imageType = image.optimisedPng match { + new URI(s3Client.signUrlV2(config.imageBucket, image.optimisedPng.getOrElse(image.source).file, image, imageType = image.optimisedPng match { case Some(_) => OptimisedPng case _ => Source })) diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala index fb50aa33511..541b2ddcb51 100644 --- a/media-api/app/lib/ImageResponse.scala +++ b/media-api/app/lib/ImageResponse.scala @@ -78,11 +78,11 @@ class ImageResponse(config: MediaApiConfig, s3Client: S3, usageQuota: UsageQuota val fileUri = image.source.file - val imageUrl = s3Client.signUrl(config.imageBucket, fileUri, image, imageType = Source) + val imageUrl = s3Client.signUrlV2(config.imageBucket, fileUri, image, imageType = Source) val pngUrl: Option[String] = pngFileUri - .map(s3Client.signUrl(config.imageBucket, _, image, imageType = OptimisedPng)) + .map(s3Client.signUrlV2(config.imageBucket, _, image, imageType = OptimisedPng)) - def s3SignedThumbUrl = s3Client.signUrl(config.thumbBucket, fileUri, image, imageType = Thumbnail) + def s3SignedThumbUrl = s3Client.signUrlV2(config.thumbBucket, fileUri, image, imageType = Thumbnail) val thumbUrl = config.cloudFrontDomainThumbBucket .map(domain => s"https://$domain${fileUri.getPath}") From aef000e1a14331ddf23294b41dfcad4d938dfe59 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 15:36:49 +0100 Subject: [PATCH 156/373] Update S3ImageStorage --- .../gu/mediaservice/lib/S3ImageStorage.scala | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala index 59b396dd2aa..79910293d92 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/S3ImageStorage.scala @@ -5,6 +5,7 @@ import com.gu.mediaservice.lib.config.CommonConfig import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker} import com.gu.mediaservice.model.MimeType import org.slf4j.LoggerFactory +import software.amazon.awssdk.services.s3.model.{DeleteObjectRequest, HeadObjectRequest, ListObjectsV2Request} import java.io.File import scala.jdk.CollectionConverters._ @@ -28,19 +29,23 @@ class S3ImageStorage(config: CommonConfig) extends S3(config) with ImageStorage } def deleteImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - client.deleteObject(bucket, id) - logger.info(logMarker, s"Deleted image $id from bucket $bucket") + clientV2.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(id).build()) } def deleteVersionedImage(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val objectVersion = client.getObjectMetadata(bucket, id).getVersionId - client.deleteVersion(bucket, id, objectVersion) + val objectVersion = clientV2.headObject(HeadObjectRequest.builder().bucket(bucket).key(id).build()).versionId() + clientV2.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(id).versionId(objectVersion).build()) logger.info(logMarker, s"Deleted image $id from bucket $bucket (version: $objectVersion)") } def deleteFolder(bucket: String, id: String)(implicit logMarker: LogMarker) = Future { - val files = client.listObjects(bucket, id).getObjectSummaries.asScala - files.foreach(file => client.deleteObject(bucket, file.getKey)) + val files = clientV2.listObjectsV2( + ListObjectsV2Request.builder().bucket(bucket).prefix(id).build() + ).contents().asScala.toList + files.foreach(file => clientV2.deleteObject( + DeleteObjectRequest.builder().bucket(bucket).key(file.key()).build() + )) logger.info(logMarker, s"Deleting images in folder $id from bucket $bucket") } From 14284cc966f394cc09ec87f3701133f360933f18 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 16:02:20 +0100 Subject: [PATCH 157/373] Upgrade the rest of reaper --- .../com/gu/mediaservice/lib/aws/S3.scala | 11 ++++++ thrall/app/controllers/ReaperController.scala | 39 ++++++++++++------- 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index ee09af0e652..f185ff5eb2d 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -223,6 +223,17 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with objects.headOption.map(_.key()) } + def doesObjectExistV2(bucket: Bucket, key: String) = { + try { + clientV2.headObject( + HeadObjectRequest.builder().key(key).bucket(bucket).build() + ) + true + } catch { + case _: NoSuchKeyException => false + } + } + } object S3Ops { diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala index 42e454d025a..65d69350bfb 100644 --- a/thrall/app/controllers/ReaperController.scala +++ b/thrall/app/controllers/ReaperController.scala @@ -16,6 +16,8 @@ import org.joda.time.{DateTime, DateTimeZone} import play.api.libs.json.{JsValue, Json} import play.api.mvc.{Action, AnyContent, ControllerComponents} import scalaz.NonEmptyList +import software.amazon.awssdk.core.sync.RequestBody +import software.amazon.awssdk.services.s3.model.{ListObjectsV2Request, PutObjectRequest, DeleteObjectRequest} import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} @@ -66,7 +68,7 @@ class ReaperController( interval = INTERVAL, ){ () => try { - if (store.client.doesObjectExist(reaperBucket, CONTROL_FILE_NAME)) { + if (store.doesObjectExistV2(reaperBucket, CONTROL_FILE_NAME)) { logger.info("Reaper is paused") es.countTotalSoftReapable(isReapable).map(metrics.softReapable.increment(Nil, _)) es.countTotalHardReapable(isReapable, config.hardReapImagesAge).map(metrics.hardReapable.increment(Nil, _)) @@ -109,7 +111,9 @@ class ReaperController( case Some(reaperBucket) => doBatchDelete.map { json => val now = DateTime.now(DateTimeZone.UTC) val key = s"$deleteType/${s3DirNameFromDate(now)}/$deleteType-${now.toString()}.json" - store.client.putObject(reaperBucket, key, json.toString()) + store.clientV2.putObject( + PutObjectRequest.builder().bucket(reaperBucket).key(key).build(), + RequestBody.fromString(json.toString())) json } } @@ -186,18 +190,24 @@ class ReaperController( case (None, _) => NotImplemented("'s3.reaper.bucket' not configured in thrall.conf") case (_, None) => NotImplemented("'reaper.countPerRun' not configured in thrall.conf") case (Some(reaperBucket), Some(countOfImagesToReap)) => - val isPaused = store.client.doesObjectExist(reaperBucket, CONTROL_FILE_NAME) + val isPaused = store.doesObjectExistV2(reaperBucket, CONTROL_FILE_NAME) val recentRecords = List(now, now.minusDays(1), now.minusDays(2)).flatMap { day => val s3DirName = s3DirNameFromDate(day) - store.client.listObjects(reaperBucket, s"soft/$s3DirName/").getObjectSummaries.asScala.toList ++ - store.client.listObjects(reaperBucket, s"hard/$s3DirName/").getObjectSummaries.asScala.toList - } + val softDeletes = store.clientV2.listObjectsV2( + ListObjectsV2Request.builder().bucket(reaperBucket).prefix(s"soft/$s3DirName/").build() + ).contents().asScala.toList + + val hardDeletes = store.clientV2.listObjectsV2( + ListObjectsV2Request.builder().bucket(reaperBucket).prefix(s"hard/$s3DirName/").build() + ).contents().asScala.toList + softDeletes ++ hardDeletes + } val recentRecordKeys = recentRecords - .filter(_.getLastModified after now.minusHours(48).toDate) - .sortBy(_.getLastModified) + .filter(_.lastModified() isAfter now.minusHours(48).toDate.toInstant) + .sortBy(_.lastModified()) .reverse - .map(_.getKey) + .map(_.key()) Ok(views.html.reaper(isPaused, INTERVAL.toString(), countOfImagesToReap, recentRecordKeys)) }} @@ -205,22 +215,23 @@ class ReaperController( def reaperRecord(key: String) = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - Ok( - store.client.getObjectAsString(reaperBucket, key) - ).as(JSON) + store.getObjectAsString(reaperBucket, key) match { + case Some(res) => Ok(res).as(JSON) + case None => NotFound + } }} def pauseReaper = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - store.client.putObject(reaperBucket, CONTROL_FILE_NAME, "") + store.clientV2.putObject(PutObjectRequest.builder().bucket(reaperBucket).key(CONTROL_FILE_NAME).build(), RequestBody.fromString("")) Redirect(routes.ReaperController.index) }} def resumeReaper = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - store.client.deleteObject(reaperBucket, CONTROL_FILE_NAME) + store.clientV2.deleteObject(DeleteObjectRequest.builder().bucket(reaperBucket).key(CONTROL_FILE_NAME).build()) Redirect(routes.ReaperController.index) }} From 569d0b12f82c37f8b60161a0b194d80bd5add764 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 16:20:18 +0100 Subject: [PATCH 158/373] Update the reaperRecord --- thrall/app/controllers/ReaperController.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thrall/app/controllers/ReaperController.scala b/thrall/app/controllers/ReaperController.scala index 65d69350bfb..2a5ac188666 100644 --- a/thrall/app/controllers/ReaperController.scala +++ b/thrall/app/controllers/ReaperController.scala @@ -215,7 +215,7 @@ class ReaperController( def reaperRecord(key: String) = auth { config.maybeReaperBucket match { case None => NotImplemented("Reaper bucket not configured") case Some(reaperBucket) => - store.getObjectAsString(reaperBucket, key) match { + store.getObjectAsStringV2(reaperBucket, key) match { case Some(res) => Ok(res).as(JSON) case None => NotFound } From 9b1e31815387634caff65f5f062b65c47ca9b0aa Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 16:25:45 +0100 Subject: [PATCH 159/373] Remove the method --- .../main/scala/com/gu/mediaservice/lib/aws/S3.scala | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index b89f9b4c5b8..be887785ea0 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -233,17 +233,6 @@ class S3(config: CommonConfig) extends GridLogging with ContentDisposition with } } - def doesObjectExistV2(bucket: Bucket, key: String) = { - try { - clientV2.headObject( - HeadObjectRequest.builder().key(key).bucket(bucket).build() - ) - true - } catch { - case _: NoSuchKeyException => false - } - } - } object S3Ops { From 37ffe3ced472c4c2d79584d827202d027b86fbc3 Mon Sep 17 00:00:00 2001 From: lindseydew Date: Mon, 27 Jul 2026 16:27:38 +0100 Subject: [PATCH 160/373] Supply an override val --- image-loader/app/lib/ImageLoaderStore.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image-loader/app/lib/ImageLoaderStore.scala b/image-loader/app/lib/ImageLoaderStore.scala index 925cc2649b1..a3cc0d06aab 100644 --- a/image-loader/app/lib/ImageLoaderStore.scala +++ b/image-loader/app/lib/ImageLoaderStore.scala @@ -44,7 +44,7 @@ class ImageLoaderStore(config: ImageLoaderConfig) extends lib.ImageIngestOperati meta = s3Meta, ) } - val presigner = S3Presigner.create() + override val presigner = S3Presigner.create() def generatePreSignedUploadUrl(filename: String, duration: Duration, uploadedBy: String, mediaId: String): String = { val putObjectRequest = PutObjectRequest.builder() From 547bc1e57350cfa4e6a4b4ea1eddbf2af00ace7d Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Tue, 28 Jul 2026 11:28:12 +0100 Subject: [PATCH 161/373] Rename "No Rights" to "Unknown Rights" --- .../src/main/scala/com/gu/mediaservice/model/UsageRights.scala | 2 +- .../js/components/gr-image-metadata/gr-image-metadata.html | 2 +- .../gr-usagerights-summary/gr-usagerights-summary.tsx | 2 +- kahuna/public/js/edits/image-editor.html | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala index 2db517e51d9..05c0c2f133b 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/UsageRights.scala @@ -158,7 +158,7 @@ case object NoRights val category = "" val defaultCost = None val restrictions = None - def name(commonConfig: CommonConfig) = "No Rights" + def name(commonConfig: CommonConfig) = "Unknown Rights" def description(commonConfig: CommonConfig) = "Images which we do not currently have the rights to use." diff --git a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html index a2d891cc287..b30edde7c3a 100644 --- a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html +++ b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html @@ -28,7 +28,7 @@
- {{ctrl.usageCategory || 'None'}} + {{ctrl.usageCategory || 'Unknown'}}
= ({ props }) => { return (
-
{ category || 'None'}
+
{ category || 'Unknown'}
diff --git a/kahuna/public/js/edits/image-editor.html b/kahuna/public/js/edits/image-editor.html index 45658896e27..55cbe453030 100644 --- a/kahuna/public/js/edits/image-editor.html +++ b/kahuna/public/js/edits/image-editor.html @@ -147,7 +147,7 @@

Image Rights

- {{ctrl.usageRightsCategory || 'None'}} + {{ctrl.usageRightsCategory || 'Unknown'}}
-
You are not authorised to upload images. Send an email to {{ctrl.systemName}} team if you think this is incorrect.
+
You are not authorised to upload images. Send an email to support if you think this is incorrect.
From 679435b2cd3d500480227c52b4932417c39fa27f Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Fri, 14 Aug 2026 12:24:59 +0100 Subject: [PATCH 202/373] Tweak top-bar spacing --- .../components/gr-sort-control/base-sort-control.tsx | 2 +- .../js/components/gr-sort-control/gr-sort-control.css | 2 +- .../js/components/gu-date-range/gu-date-range.css | 2 +- kahuna/public/stylesheets/main.css | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx index 8881a026351..b102de908ba 100644 --- a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx @@ -13,7 +13,7 @@ const PANEL_IDENTIFIER = "info"; const SCROLL_IDENTIFIER = "scroll"; const downArrowIcon = () => - + ; diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css index 78b23df5a28..f7eab4465da 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.css +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.css @@ -149,7 +149,7 @@ } } -@media screen and (max-width: 1380px) { +@media screen and (max-width: 1400px) { .sort-selection-title { display: none; } diff --git a/kahuna/public/js/components/gu-date-range/gu-date-range.css b/kahuna/public/js/components/gu-date-range/gu-date-range.css index 6ad383d8949..c106bb3a388 100644 --- a/kahuna/public/js/components/gu-date-range/gu-date-range.css +++ b/kahuna/public/js/components/gu-date-range/gu-date-range.css @@ -18,7 +18,7 @@ .gu-date-range__display__icon__toggle { display: inline-block; - margin-left: 5px; + margin-left: 3px; } .gu-date-range__icon { diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index 4fb0ee0bfd9..a7a54f31d7d 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -736,7 +736,7 @@ textarea.ng-invalid { display: inline-flex; flex-direction: row; align-items: center; - border-left: 2px solid #565656; + border-left: 1px solid #565656; height: 50px; } @@ -745,7 +745,7 @@ textarea.ng-invalid { } .top-bar-item:last-child { - border-right: 2px solid #565656; + border-right: 1px solid #565656; } .top-bar-item__form { @@ -953,7 +953,7 @@ textarea.ng-invalid { border: 1px solid #999; box-sizing: border-box; - min-width: 300px; + min-width: 270px; flex-grow: 1; } @@ -980,7 +980,7 @@ textarea.ng-invalid { display: flex; flex-direction: row; justify-content: flex-start; - margin-left: 25px; + margin-left: 10px; align-items: center; /* vertical align all filters */ } @@ -1050,7 +1050,7 @@ textarea.ng-invalid { } } -@media screen and (max-width: 1380px) { +@media screen and (max-width: 1400px) { .search__modifier-toggle { display: flex; flex-direction: row; From 3866ff8062b018b7256819fa6b2a61ea4f1914da Mon Sep 17 00:00:00 2001 From: AlexAscherson Date: Tue, 18 Aug 2026 18:08:42 +0100 Subject: [PATCH 203/373] Add workaround info for common auth error --- docs/02-running/01-running-locally.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/02-running/01-running-locally.md b/docs/02-running/01-running-locally.md index 99593002620..f0d4718f9b6 100644 --- a/docs/02-running/01-running-locally.md +++ b/docs/02-running/01-running-locally.md @@ -59,6 +59,13 @@ If you haven't used the `--with-local-auth` flag, you'll begin a Google authenti If you have used the `--no-auth` flag you'll be signed in automatically as John Doe (with no oauth flow) and will have ALL permissions. +# Auth Failed Errors +Sometimes you will see a red notification relating to a failure to auth. Visiting: + +https://media.test.dev-gutools.co.uk/search + +and refreshing the page will resolve this. + ## Cerebro To inspect the elasticsearch database, cerebro is included as part of the grids local stack. From aa5c72f8fcf8ae2b0bc5d59905eeabadeb43f0bf Mon Sep 17 00:00:00 2001 From: AlexAscherson Date: Wed, 19 Aug 2026 10:42:03 +0100 Subject: [PATCH 204/373] Update docs/02-running/01-running-locally.md Co-authored-by: Holly Knox <168672047+hk-15@users.noreply.github.com> --- docs/02-running/01-running-locally.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/02-running/01-running-locally.md b/docs/02-running/01-running-locally.md index f0d4718f9b6..25e707d5a56 100644 --- a/docs/02-running/01-running-locally.md +++ b/docs/02-running/01-running-locally.md @@ -59,7 +59,7 @@ If you haven't used the `--with-local-auth` flag, you'll begin a Google authenti If you have used the `--no-auth` flag you'll be signed in automatically as John Doe (with no oauth flow) and will have ALL permissions. -# Auth Failed Errors +### Auth Failed Errors Sometimes you will see a red notification relating to a failure to auth. Visiting: https://media.test.dev-gutools.co.uk/search From 152949807f61ddd6d757394d8061a824511a008e Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Thu, 13 Aug 2026 15:44:41 +0100 Subject: [PATCH 205/373] Add an endpoint for showing the usages and supplier of image ids --- .../app/controllers/UsageController.scala | 30 ++++++- media-api/app/lib/ImageUsagesBySupplier.scala | 11 +++ .../app/lib/elasticsearch/ElasticSearch.scala | 66 +++++++++++++- media-api/conf/routes | 3 +- .../lib/elasticsearch/ElasticSearchTest.scala | 88 ++++++++++++++++++- .../elasticsearch/ElasticSearchTestBase.scala | 37 ++++++++ .../test/lib/elasticsearch/Fixtures.scala | 2 +- 7 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 media-api/app/lib/ImageUsagesBySupplier.scala diff --git a/media-api/app/controllers/UsageController.scala b/media-api/app/controllers/UsageController.scala index 4db59c979be..863e0185306 100644 --- a/media-api/app/controllers/UsageController.scala +++ b/media-api/app/controllers/UsageController.scala @@ -6,8 +6,11 @@ import com.gu.mediaservice.lib.auth.Authentication.Principal import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.lib.play.RequestLoggingFilter import com.gu.mediaservice.model.Agencies +import com.gu.mediaservice.model.usage.Usage import lib._ -import lib.elasticsearch.ElasticSearch +import lib.elasticsearch.{ElasticSearch, InvalidUriParams, SearchParams} +import lib.elasticsearch.SearchParams.parseIntFromQuery +import lib.querysyntax.Parser import play.api.mvc.Security.AuthenticatedRequest import play.api.mvc._ @@ -90,4 +93,29 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc respondError(InternalServerError, "unknown-error", e.toString) } } + + def imageUsagesBySupplier(id: String) = auth.async { implicit request => + implicit val logMarker: LogMarker = MarkerMap( + "requestType" -> "images-by-supplier", + "requestId" -> RequestLoggingFilter.getRequestId(request), + "supplierId" -> id, + ) ++ RequestLoggingFilter.loggablePrincipal(request.user) + + val structuredQuery = request.getQueryString("q").map(Parser.run).getOrElse(List.empty) + val searchParams = SearchParams(request) + + SearchParams.validate(searchParams) match { + case Left(errors) => + Future.successful(respondError(BadRequest, InvalidUriParams.errorKey, errors.map(_.message).mkString(", "))) + case Right(_) => + elasticSearch.imageUsagesBySupplier(id, structuredQuery, searchParams.offset, searchParams.length) + .map { result => + respondCollection(result.images, Some(searchParams.offset.toLong), Some(result.total)) + } + .recover { + case e: IllegalArgumentException => respondError(BadRequest, InvalidUriParams.errorKey, e.getMessage) + case e => respondError(InternalServerError, "unknown-error", e.toString) + } + } + } } diff --git a/media-api/app/lib/ImageUsagesBySupplier.scala b/media-api/app/lib/ImageUsagesBySupplier.scala new file mode 100644 index 00000000000..4a785397cb6 --- /dev/null +++ b/media-api/app/lib/ImageUsagesBySupplier.scala @@ -0,0 +1,11 @@ +package lib + +import com.gu.mediaservice.model.usage.Usage +import play.api.libs.json.{Json, OFormat} + +case class ImageUsagesBySupplier(id: String, supplier: String, usages: List[Usage]) +object ImageUsagesBySupplier { + implicit val jsonFormat: OFormat[ImageUsagesBySupplier] = Json.format[ImageUsagesBySupplier] +} + +case class ImageUsagesBySupplierResult(images: List[ImageUsagesBySupplier], total: Long) diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 929a50c23fe..375073cf492 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -2,6 +2,7 @@ package lib.elasticsearch import org.apache.pekko.actor.Scheduler import com.gu.mediaservice.lib.ImageFields +import com.gu.mediaservice.lib.formatting.printDateTime import com.gu.mediaservice.lib.argo.model.{ExtraCount, ExtraCountConfig, ExtraCounts} import com.gu.mediaservice.lib.elasticsearch.filters import com.gu.mediaservice.lib.auth.Authentication.Principal @@ -9,6 +10,7 @@ import com.gu.mediaservice.lib.elasticsearch.{CompletionPreview, ElasticSearchCl import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap, Stopwatch, combineMarkers} import com.gu.mediaservice.lib.metrics.FutureSyntax import com.gu.mediaservice.model.{Agencies, Agency, AwaitingReviewForSyndication, Image} +import com.gu.mediaservice.model.usage.{DigitalUsage, PrintUsage, PublishedUsageStatus, RemovedUsageStatus, Usage, UnknownUsageStatus, UsageStatus, UsageType} import com.sksamuel.elastic4s.{ElasticDsl, Hit} import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s.requests.common.Operator @@ -24,8 +26,8 @@ import com.sksamuel.elastic4s.requests.searches.knn.Knn import com.sksamuel.elastic4s.requests.searches.queries.matches.MultiMatchQueryBuilderType.BEST_FIELDS import com.sksamuel.elastic4s.requests.searches.queries.matches.{FieldWithOptionalBoost, MultiMatchQuery} import lib.elasticsearch.ResultSource.{Both, Lexical, Semantic} -import lib.querysyntax.{HierarchyField, Match, Parser, Phrase} -import lib.{MediaApiConfig, MediaApiMetrics, SupplierUsageSummary} +import lib.querysyntax.{Condition, DateRange, HierarchyField, Match, Nested, Parser, Phrase, SingleField} +import lib.{MediaApiConfig, MediaApiMetrics, SupplierUsageSummary, ImageUsagesBySupplier, ImageUsagesBySupplierResult} import play.api.libs.json.{JsError, JsObject, JsSuccess, Json} import play.api.mvc.AnyContent import play.api.mvc.Security.AuthenticatedRequest @@ -483,6 +485,66 @@ class ElasticSearch( } } + def imageUsagesBySupplier( + id: String, + structuredQuery: List[Condition] = List.empty, + offset: Int = 0, + length: Int = 10 + )(implicit ex: ExecutionContext, logMarker: LogMarker): Future[ImageUsagesBySupplierResult] = { + if (offset + length > 10000) + return Future.failed(new IllegalArgumentException(s"offset + length cannot exceed 10000 (Elasticsearch result window limit)")) + + val supplier = Agencies.get(id) + val supplierName = supplier.supplier + + val qualifyingStatuses = Set[UsageStatus](PublishedUsageStatus, UnknownUsageStatus, RemovedUsageStatus) + val qualifyingPlatforms = Set[UsageType](PrintUsage, DigitalUsage) + + val haveQualifyingStatus = termsQuery("usages.status", qualifyingStatuses.map(_.toString)) + val haveQualifyingPlatform = termsQuery("usages.platform", qualifyingPlatforms.map(_.toString)) + + // e.g. usages@added:2026-07-01 - each date bound is inclusive, + // and since they all apply within the same nested "usages" entry, multiple bounds combine into a range. + val dateAddedRanges = structuredQuery.collect { + case Nested(SingleField("usages"), SingleField("dateAdded"), DateRange(start, end)) => (start, end) + } + val maybeDateAddedRange = dateAddedRanges match { + case Nil => None + case ranges => + val from = ranges.map(_._1).maxBy(_.getMillis) + val to = ranges.map(_._2).minBy(_.getMillis) + Some((from, to)) + } + + val qualifyingUsageClauses = List(haveQualifyingStatus, haveQualifyingPlatform) ++ + maybeDateAddedRange.map { case (from, to) => rangeQuery("usages.dateAdded").gte(printDateTime(from)).lte(printDateTime(to)) } + val haveQualifyingUsage = nestedQuery("usages", boolQuery().must(qualifyingUsageClauses)) + + val beSupplier = termQuery("usageRights.supplier", supplierName) + + val query = boolQuery().must(matchAllQuery()).filter(boolQuery().must(beSupplier, haveQualifyingUsage)) + + val search = prepareSearch(query).trackTotalHits(true).from(offset).size(length) + + def isQualifyingUsage(usage: Usage): Boolean = + qualifyingStatuses.contains(usage.status) && + qualifyingPlatforms.contains(usage.platform) && + maybeDateAddedRange.forall { case (from, to) => + usage.dateAdded.exists(dateAdded => !dateAdded.isBefore(from) && !dateAdded.isAfter(to)) + } + + executeAndLog(search, s"$id image usages by supplier search").map { r => + import r.result + logSearchQueryIfTimedOut(search, result) + val images = result.hits.hits.toList + .flatMap(resolveHit) + .map(sourceWrapperImage => sourceWrapperImage.instance) + .map(image => ImageUsagesBySupplier(image.id, supplierName, image.usages.filter(isQualifyingUsage))) + .distinctBy(_.id) + ImageUsagesBySupplierResult(images, result.totalHits) + } + } + def dateHistogramAggregate(params: AggregateSearchParams)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[AggregateSearchResults] = { def fromDateHistogramAggregation(name: String, aggregations: Aggregations): Seq[BucketResult] = aggregations.result[DateHistogram](name). diff --git a/media-api/conf/routes b/media-api/conf/routes index 2b22b7b6692..909bf21ac78 100644 --- a/media-api/conf/routes +++ b/media-api/conf/routes @@ -28,7 +28,6 @@ DELETE /images/:id controllers. DELETE /images/:id/hard-delete controllers.MediaApi.hardDeleteImage(id: String) PUT /images/:id/undelete controllers.MediaApi.unSoftDeleteImage(id: String) GET /images controllers.MediaApi.imageSearch() - # completion GET /suggest/metadata/credit controllers.SuggestionController.suggestMetadataCredit(q: Option[String], size: Option[Int]) GET /suggest/metadata/photoshoot controllers.SuggestionController.suggestPhotoshoot(q: Option[String], size: Option[Int]) @@ -36,9 +35,9 @@ GET /suggest/metadata/photoshoot controllers. # usage quotas GET /usage/suppliers controllers.UsageController.bySupplier GET /usage/suppliers/:id controllers.UsageController.forSupplier(id: String) +GET /usage/suppliers/:id/images controllers.UsageController.imageUsagesBySupplier(id: String) GET /usage/quotas controllers.UsageController.quotas GET /usage/quotas/:id controllers.UsageController.quotaForImage(id: String) - # configuration GET /configuration/crop-variations controllers.ConfigurationController.cropVariations diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 9f333f9cea1..38d7aaa1118 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -8,7 +8,7 @@ import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearc import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.DenySyndicationLease -import com.gu.mediaservice.model.usage.PublishedUsageStatus +import com.gu.mediaservice.model.usage.{PublishedUsageStatus, RemovedUsageStatus, UnknownUsageStatus} import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ import lib.querysyntax._ @@ -152,12 +152,12 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast implicit val logMarker: LogMarker = MarkerMap() val publishedAgencyImages = images.filter(i => i.usageRights.isInstanceOf[Agency] && i.usages.exists(_.status == PublishedUsageStatus)) - publishedAgencyImages.size shouldBe 2 + publishedAgencyImages.size shouldBe 7 // Reporting date range is implemented as round down to last full day val withinReportedDateRange = publishedAgencyImages.filter(i => i.usages. exists(u => u.dateAdded.exists(_.isBefore(DateTime.now.withTimeAtStartOfDay())))) - withinReportedDateRange.size shouldBe 1 + withinReportedDateRange.size shouldBe 6 val results = Await.result(ES.usageForSupplier("ACME", 5), fiveSeconds) @@ -165,6 +165,88 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast } } + describe("images usages by supplier") { + it("only returns images with a qualifying usage status/platform for the given supplier, with the full usages list and supplier populated per item") { + implicit val logMarker: LogMarker = MarkerMap() + + val expectedIds = Set( + "usages-by-supplier-qualified-published-digital", + "usages-by-supplier-qualified-unknown-print", + "usages-by-supplier-qualified-removed-digital", + "usages-by-supplier-multi-usage", + "usages-by-supplier-out-of-date-range" + ) + + val result = Await.result(ES.imageUsagesBySupplier("test-wire", length = 100), fiveSeconds) + + result.total shouldBe expectedIds.size + result.images.map(_.id).toSet shouldBe expectedIds + result.images.foreach(_.supplier shouldBe "test-wire") + + // wrong supplier, non-qualifying status and non-qualifying platform are all excluded + result.images.map(_.id) should not contain "usages-by-supplier-wrong-supplier" + result.images.map(_.id) should not contain "usages-by-supplier-non-qualifying-status" + result.images.map(_.id) should not contain "usages-by-supplier-non-qualifying-platform" + + // distinctBy(_.id) collapses multiple qualifying usages into one result. Only the qualifying + // usages should be returned - the non-qualifying (pending status) usage on this image is excluded. + val multiUsageResult = result.images.find(_.id == "usages-by-supplier-multi-usage").get + multiUsageResult.usages should have size 2 + multiUsageResult.usages.map(_.status) should contain theSameElementsAs List(PublishedUsageStatus, RemovedUsageStatus) + } + + it("only returns usages within the requested date range on a matching image, excluding out-of-range usages on the same image") { + implicit val logMarker: LogMarker = MarkerMap() + + val dateRangeQuery = List(Nested( + SingleField("usages"), + SingleField("dateAdded"), + DateRange(DateTime.parse("2020-06-20"), DateTime.parse("2020-06-30")) + )) + + // the multi-usage image has one qualifying usage in range (2020-06-25, removed/print) and + // one qualifying usage out of range (2020-06-01, published/digital) - only the in-range one should be returned. + val result = Await.result(ES.imageUsagesBySupplier("test-wire", dateRangeQuery, length = 100), fiveSeconds) + + val multiUsageResult = result.images.find(_.id == "usages-by-supplier-multi-usage").get + multiUsageResult.usages should have size 1 + multiUsageResult.usages.head.status shouldBe RemovedUsageStatus + } + + it("applies an inclusive date range filter on usages.dateAdded and paginates the results") { + implicit val logMarker: LogMarker = MarkerMap() + + val dateRangeQuery = List(Nested( + SingleField("usages"), + SingleField("dateAdded"), + DateRange(DateTime.parse("2020-06-01"), DateTime.parse("2020-06-30")) + )) + + val filteredResult = Await.result(ES.imageUsagesBySupplier("test-wire", dateRangeQuery, length = 100), fiveSeconds) + filteredResult.images.map(_.id).toSet shouldBe Set( + "usages-by-supplier-qualified-published-digital", + "usages-by-supplier-qualified-unknown-print", + "usages-by-supplier-qualified-removed-digital", + "usages-by-supplier-multi-usage" + ) + + // pagination: paging through with a small length shouldn't drop or duplicate results + val pageSize = 2 + val pages = (0 until 3).map { page => + Await.result(ES.imageUsagesBySupplier("test-wire", offset = page * pageSize, length = pageSize), fiveSeconds) + } + pages.map(_.images.size) shouldBe Seq(2, 2, 1) + pages.foreach(_.total shouldBe 5) + pages.flatMap(_.images.map(_.id)).toSet shouldBe Set( + "usages-by-supplier-qualified-published-digital", + "usages-by-supplier-qualified-unknown-print", + "usages-by-supplier-qualified-removed-digital", + "usages-by-supplier-multi-usage", + "usages-by-supplier-out-of-date-range" + ) + } + } + describe("aggregations") { it("can load date aggregations") { implicit val logMarker: LogMarker = MarkerMap() diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala index 466c604a659..8211dbc43bd 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala @@ -1,6 +1,7 @@ package lib.elasticsearch import com.gu.mediaservice.model._ +import com.gu.mediaservice.model.usage._ import com.gu.mediaservice.testlib.ElasticSearchDockerBase import org.joda.time.DateTime import org.scalatest.concurrent.PatienceConfiguration.{Interval, Timeout} @@ -144,6 +145,17 @@ trait ElasticSearchTestBase extends AnyFunSpec with ElasticSearchDockerBase with usages = List(createDigitalUsage(date = DateTime.now)) ), + // Agency image with an "unknown" status usage just now - unknown usages should still + // count towards the supplier usage summary, alongside "published" usages. + createImageForSyndication( + id = "test-image-agency-unknown-status", + rightsAcquired = false, + None, + None, + usageRights = agency, + usages = List(createUsage(ComposerUsageReference, DigitalUsage, UnknownUsageStatus, date = DateTime.now.minusDays(1))) + ), + // Screen grab with rights acquired, not eligible for syndication review createImageForSyndication( id = "test-image-11", @@ -187,5 +199,30 @@ trait ElasticSearchTestBase extends AnyFunSpec with ElasticSearchDockerBase with // ) // ) // ) + + // Fixtures for ES.imageUsagesBySupplier: a dedicated supplier ("test-wire" isn't in Agencies.all, + // so Agencies.get("test-wire").supplier falls back to == "test-wire"). + createImage("usages-by-supplier-qualified-published-digital", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-06-15")))), + createImage("usages-by-supplier-qualified-unknown-print", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, PrintUsage, UnknownUsageStatus, DateTime.parse("2020-06-16")))), + createImage("usages-by-supplier-qualified-removed-digital", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, RemovedUsageStatus, DateTime.parse("2020-06-17")))), + createImage("usages-by-supplier-non-qualifying-status", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PendingUsageStatus, DateTime.parse("2020-06-18")))), + createImage("usages-by-supplier-non-qualifying-platform", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, SyndicationUsage, PublishedUsageStatus, DateTime.parse("2020-06-19")))), + createImage("usages-by-supplier-wrong-supplier", Agency("other-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-06-20")))), + // Has two qualifying usages plus one non-qualifying usage (wrong status) - checks distinctBy(_.id) + // collapses to a single result, and that only the qualifying usages are returned (not the non-qualifying one). + createImage("usages-by-supplier-multi-usage", Agency("test-wire"), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-06-01")), + createUsage(ComposerUsageReference, PrintUsage, RemovedUsageStatus, DateTime.parse("2020-06-25")), + createUsage(ComposerUsageReference, DigitalUsage, PendingUsageStatus, DateTime.parse("2020-06-10")) + )), + createImage("usages-by-supplier-out-of-date-range", Agency("test-wire"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-08-01")))) ) } diff --git a/media-api/test/lib/elasticsearch/Fixtures.scala b/media-api/test/lib/elasticsearch/Fixtures.scala index 8caca4053d4..57fe3c005de 100644 --- a/media-api/test/lib/elasticsearch/Fixtures.scala +++ b/media-api/test/lib/elasticsearch/Fixtures.scala @@ -153,7 +153,7 @@ trait Fixtures { out } - private def createUsage(t: UsageReferenceType, usageType: UsageType, status: Status, date: DateTime): Usage = { + def createUsage(t: UsageReferenceType, usageType: UsageType, status: Status, date: DateTime): Usage = { Usage( UUID.randomUUID().toString, List(UsageReference(t)), From 075412eb8ca0bf8155e9843f9a4b20c7d5081675 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Wed, 12 Aug 2026 14:31:34 +0100 Subject: [PATCH 206/373] add searchKey so we can filter by users and their distinct queries --- kahuna/public/js/services/telemetry.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kahuna/public/js/services/telemetry.ts b/kahuna/public/js/services/telemetry.ts index b6531608ec3..bd1e0723087 100644 --- a/kahuna/public/js/services/telemetry.ts +++ b/kahuna/public/js/services/telemetry.ts @@ -51,6 +51,7 @@ const sendFilterTelemetryEvent = (key: string, value: string, searchUuid: string export const sendTelemetryForQuery = (query: string, nonFree?: boolean | string, uploadedByMe?: boolean, useAISearch?: boolean ) => { const structuredQuery = structureQuery(query || ""); + const searchUuid = v4(); // nonFree is unfortunately either a boolean, stringified boolean, or undefined const freeToUseOnly = (!(nonFree === 'true' || nonFree === true)); @@ -74,11 +75,16 @@ export const sendTelemetryForQuery = (query: string, nonFree?: boolean | string, return `GRID_${type.toUpperCase()}`; }; + const searchKey = (type: string) => { + return type === 'text' ? { searchKey: `${getBrowserId()}::${queryComponent.value}` } : undefined; + }; + // In case search is empty, as with a search containing only filters sendTelemetryEvent(formattedType(type), { ...queryComponent, searchUuid: searchUuid, - useAISearch: useAISearch ?? false + useAISearch: useAISearch ?? false, + ...searchKey(type) }, 1); }); }; From 35c3b52af830a4ec3a8a16d63c219185c3ebc6e8 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 13 Aug 2026 16:46:18 +0100 Subject: [PATCH 207/373] also send telemetry when useAISearch is toggled --- kahuna/public/js/search/query.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index fd20f7d2c8f..4c31cc24ea2 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -453,6 +453,19 @@ query.controller('SearchQueryCtrl', [ $scope.$watch(() => ctrl.useAISearch, () => { // Note: $watch expressions execute at least once during initialization, so this is executed on page refresh. // This is the behaviour we want so that the URL is updated based on the AI search toggle + + const { nonFree, uploadedByMe } = ctrl.filter; + let nonFreeCheck = nonFree; + if (ctrl.usePermissionsFilter && nonFreeCheck === undefined) { + const defaultShowPaid = storage.getJs("defaultIsNonFree", true); + nonFreeCheck = defaultShowPaid; + } else if (!ctrl.usePermissionsFilter && (nonFreeCheck === 'false' || nonFreeCheck === false)) { + nonFreeCheck = undefined; + } + ctrl.filter.nonFree = nonFreeCheck; + + sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, uploadedByMe, ctrl.useAISearch); + if (ctrl.useAISearch) { $state.go('search.results', { ...ctrl.filter, From b9c0bbc612285db6c542b0db8646f849c64096c9 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 20 Aug 2026 16:38:21 +0100 Subject: [PATCH 208/373] refactor emitQueryTelemetry --- kahuna/public/js/search/query.js | 49 +++++++++++++++++--------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 4c31cc24ea2..43e111413b3 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -244,6 +244,22 @@ query.controller('SearchQueryCtrl', [ } } + function resolveNonFree() { + let nonFreeCheck = ctrl.filter.nonFree; + if (ctrl.usePermissionsFilter && nonFreeCheck === undefined) { + nonFreeCheck = storage.getJs("defaultIsNonFree", true); + } else if (!ctrl.usePermissionsFilter && (nonFreeCheck === 'false' || nonFreeCheck === false)) { + nonFreeCheck = undefined; + } + ctrl.filter.nonFree = nonFreeCheck; + return nonFreeCheck; + } + + function emitQueryTelemetry() { + const nonFreeCheck = resolveNonFree(); + sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, ctrl.filter.uploadedByMe, ctrl.useAISearch); + } + // eslint-disable-next-line complexity function watchSearchChange(newFilter, sender) { let showPaid = newFilter.nonFree ? newFilter.nonFree : false; @@ -277,17 +293,8 @@ query.controller('SearchQueryCtrl', [ manageDefaultNonFree(newFilter); manageOrgOwnedSetting(newFilter); - const { nonFree, uploadedByMe } = ctrl.filter; - let nonFreeCheck = nonFree; - if (ctrl.usePermissionsFilter && nonFreeCheck === undefined) { - const defaultShowPaid = storage.getJs("defaultIsNonFree", true); - nonFreeCheck = defaultShowPaid; - } else if (!ctrl.usePermissionsFilter && (nonFreeCheck === 'false' || nonFreeCheck === false)) { - nonFreeCheck = undefined; - } - ctrl.filter.nonFree = nonFreeCheck; - - sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, uploadedByMe, ctrl.useAISearch); + emitQueryTelemetry() + if (ctrl.collectionSearch && !curCollectionSearch) { storage.setJs("orderBy", CollectionSortOption.value); ctrl.ordering["orderBy"] = CollectionSortOption.value; @@ -452,20 +459,16 @@ query.controller('SearchQueryCtrl', [ })); $scope.$watch(() => ctrl.useAISearch, () => { // Note: $watch expressions execute at least once during initialization, so this is executed on page refresh. - // This is the behaviour we want so that the URL is updated based on the AI search toggle - - const { nonFree, uploadedByMe } = ctrl.filter; - let nonFreeCheck = nonFree; - if (ctrl.usePermissionsFilter && nonFreeCheck === undefined) { - const defaultShowPaid = storage.getJs("defaultIsNonFree", true); - nonFreeCheck = defaultShowPaid; - } else if (!ctrl.usePermissionsFilter && (nonFreeCheck === 'false' || nonFreeCheck === false)) { - nonFreeCheck = undefined; + // This is the behaviour we want so that the URL is updated based on the AI search toggle. + // We only emit telemetry on an actual toggle though - the load-time search event is emitted + // exactly once via the getSession() -> watchSearchChange path, so emitting here on init would double-count. + if (aiSearchInitialised) { + emitQueryTelemetry(); + } else { + resolveNonFree(); } - ctrl.filter.nonFree = nonFreeCheck; + aiSearchInitialised = true; - sendTelemetryForQuery(ctrl.filter.query, nonFreeCheck, uploadedByMe, ctrl.useAISearch); - if (ctrl.useAISearch) { $state.go('search.results', { ...ctrl.filter, From ba545439474bc535b1ca6ac091449632cd33dc90 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 20 Aug 2026 16:55:02 +0100 Subject: [PATCH 209/373] oops initialise aiSearchEnabled --- kahuna/public/js/search/query.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 43e111413b3..cd87954b88e 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -457,6 +457,8 @@ query.controller('SearchQueryCtrl', [ $scope.$watch(() => ctrl.ordering.orderBy, onValChange(newVal => { $state.go('search.results', {...ctrl.filter, orderBy: newVal}); })); + + let aiSearchInitialised = false; $scope.$watch(() => ctrl.useAISearch, () => { // Note: $watch expressions execute at least once during initialization, so this is executed on page refresh. // This is the behaviour we want so that the URL is updated based on the AI search toggle. From d830c6e330d7d17afc1d456578ca1accf1f339c5 Mon Sep 17 00:00:00 2001 From: Ellen Muller Date: Thu, 20 Aug 2026 17:00:39 +0100 Subject: [PATCH 210/373] linting --- kahuna/public/js/search/query.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index cd87954b88e..c0666a0fe25 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -293,8 +293,8 @@ query.controller('SearchQueryCtrl', [ manageDefaultNonFree(newFilter); manageOrgOwnedSetting(newFilter); - emitQueryTelemetry() - + emitQueryTelemetry(); + if (ctrl.collectionSearch && !curCollectionSearch) { storage.setJs("orderBy", CollectionSortOption.value); ctrl.ordering["orderBy"] = CollectionSortOption.value; From f4d638ea9deb3c28629e7633849e07fd8af7505e Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Thu, 20 Aug 2026 17:48:06 +0100 Subject: [PATCH 211/373] Include images with usageRights.suppliers field when querying for usages --- media-api/app/lib/ImageUsagesBySupplier.scala | 10 +++++++ .../app/lib/elasticsearch/ElasticSearch.scala | 10 ++++--- .../lib/elasticsearch/ElasticSearchTest.scala | 26 ++++++++++++++----- .../elasticsearch/ElasticSearchTestBase.scala | 7 +++-- 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/media-api/app/lib/ImageUsagesBySupplier.scala b/media-api/app/lib/ImageUsagesBySupplier.scala index 4a785397cb6..f3370643d55 100644 --- a/media-api/app/lib/ImageUsagesBySupplier.scala +++ b/media-api/app/lib/ImageUsagesBySupplier.scala @@ -1,10 +1,20 @@ package lib +import com.gu.mediaservice.model.{Agency, CommissionedAgency, Composite, UsageRights} import com.gu.mediaservice.model.usage.Usage import play.api.libs.json.{Json, OFormat} case class ImageUsagesBySupplier(id: String, supplier: String, usages: List[Usage]) object ImageUsagesBySupplier { + def apply(id: String, usageRights: UsageRights, usages: List[Usage]): ImageUsagesBySupplier = { + val supplier = usageRights match { + case c: Composite => c.suppliers + case a: Agency => a.supplier + case ca: CommissionedAgency => ca.supplier + case u => "" + } + ImageUsagesBySupplier(id, supplier, usages) + } implicit val jsonFormat: OFormat[ImageUsagesBySupplier] = Json.format[ImageUsagesBySupplier] } diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 375073cf492..6795673ebcc 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -512,7 +512,8 @@ class ElasticSearch( case Nil => None case ranges => val from = ranges.map(_._1).maxBy(_.getMillis) - val to = ranges.map(_._2).minBy(_.getMillis) + // ` rangeQuery("usages.dateAdded").gte(printDateTime(from)).lte(printDateTime(to)) } val haveQualifyingUsage = nestedQuery("usages", boolQuery().must(qualifyingUsageClauses)) - val beSupplier = termQuery("usageRights.supplier", supplierName) + val beSupplier = boolQuery().should( + termQuery("usageRights.supplier", supplierName), + matchQuery("usageRights.suppliers", supplierName) + ).minimumShouldMatch(1) val query = boolQuery().must(matchAllQuery()).filter(boolQuery().must(beSupplier, haveQualifyingUsage)) @@ -539,7 +543,7 @@ class ElasticSearch( val images = result.hits.hits.toList .flatMap(resolveHit) .map(sourceWrapperImage => sourceWrapperImage.instance) - .map(image => ImageUsagesBySupplier(image.id, supplierName, image.usages.filter(isQualifyingUsage))) + .map(image => ImageUsagesBySupplier(image.id, image.usageRights, image.usages.filter(isQualifyingUsage))) .distinctBy(_.id) ImageUsagesBySupplierResult(images, result.totalHits) } diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 38d7aaa1118..8c1bca9968c 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -174,14 +174,15 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast "usages-by-supplier-qualified-unknown-print", "usages-by-supplier-qualified-removed-digital", "usages-by-supplier-multi-usage", - "usages-by-supplier-out-of-date-range" + "usages-by-supplier-out-of-date-range", + "usages-by-supplier-composite" ) val result = Await.result(ES.imageUsagesBySupplier("test-wire", length = 100), fiveSeconds) result.total shouldBe expectedIds.size result.images.map(_.id).toSet shouldBe expectedIds - result.images.foreach(_.supplier shouldBe "test-wire") + result.images.filterNot(_.id == "usages-by-supplier-composite").foreach(_.supplier shouldBe "test-wire") // wrong supplier, non-qualifying status and non-qualifying platform are all excluded result.images.map(_.id) should not contain "usages-by-supplier-wrong-supplier" @@ -227,7 +228,8 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast "usages-by-supplier-qualified-published-digital", "usages-by-supplier-qualified-unknown-print", "usages-by-supplier-qualified-removed-digital", - "usages-by-supplier-multi-usage" + "usages-by-supplier-multi-usage", + "usages-by-supplier-composite" ) // pagination: paging through with a small length shouldn't drop or duplicate results @@ -235,16 +237,28 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast val pages = (0 until 3).map { page => Await.result(ES.imageUsagesBySupplier("test-wire", offset = page * pageSize, length = pageSize), fiveSeconds) } - pages.map(_.images.size) shouldBe Seq(2, 2, 1) - pages.foreach(_.total shouldBe 5) + pages.map(_.images.size) shouldBe Seq(2, 2, 2) + pages.foreach(_.total shouldBe 6) pages.flatMap(_.images.map(_.id)).toSet shouldBe Set( "usages-by-supplier-qualified-published-digital", "usages-by-supplier-qualified-unknown-print", "usages-by-supplier-qualified-removed-digital", "usages-by-supplier-multi-usage", - "usages-by-supplier-out-of-date-range" + "usages-by-supplier-out-of-date-range", + "usages-by-supplier-composite" ) } + + it("includes composite images whose suppliers field contains the given supplier name") { + implicit val logMarker: LogMarker = MarkerMap() + + val result = Await.result(ES.imageUsagesBySupplier("test-wire", length = 100), fiveSeconds) + + val compositeResult = result.images.find(_.id == "usages-by-supplier-composite") + compositeResult shouldBe defined + // supplier field is populated from usageRights.suppliers for composite images + compositeResult.get.supplier shouldBe "test-wire, other-supplier" + } } describe("aggregations") { diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala index 8211dbc43bd..3e43a63f071 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTestBase.scala @@ -207,7 +207,7 @@ trait ElasticSearchTestBase extends AnyFunSpec with ElasticSearchDockerBase with createImage("usages-by-supplier-qualified-unknown-print", Agency("test-wire"), usages = List(createUsage(ComposerUsageReference, PrintUsage, UnknownUsageStatus, DateTime.parse("2020-06-16")))), createImage("usages-by-supplier-qualified-removed-digital", Agency("test-wire"), - usages = List(createUsage(ComposerUsageReference, DigitalUsage, RemovedUsageStatus, DateTime.parse("2020-06-17")))), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, RemovedUsageStatus, DateTime.parse("2020-06-30T23:59:59")))), createImage("usages-by-supplier-non-qualifying-status", Agency("test-wire"), usages = List(createUsage(ComposerUsageReference, DigitalUsage, PendingUsageStatus, DateTime.parse("2020-06-18")))), createImage("usages-by-supplier-non-qualifying-platform", Agency("test-wire"), @@ -223,6 +223,9 @@ trait ElasticSearchTestBase extends AnyFunSpec with ElasticSearchDockerBase with createUsage(ComposerUsageReference, DigitalUsage, PendingUsageStatus, DateTime.parse("2020-06-10")) )), createImage("usages-by-supplier-out-of-date-range", Agency("test-wire"), - usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-08-01")))) + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-07-01")))), + // Composite image whose `suppliers` field contains "test-wire" alongside another supplier + createImage("usages-by-supplier-composite", Composite("test-wire, other-supplier"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.parse("2020-06-15")))) ) } From c831f8bf4cf7c27b64341e4e1e0d1434ef1f1594 Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Fri, 21 Aug 2026 16:18:56 +0100 Subject: [PATCH 212/373] use multiple aggs --- .../app/controllers/UsageController.scala | 28 ++- media-api/app/lib/UsageStore.scala | 45 ++--- .../app/lib/elasticsearch/ElasticSearch.scala | 98 +++++++++-- media-api/conf/routes | 1 + media-api/test/lib/UsageStoreTest.scala | 2 +- .../lib/elasticsearch/ElasticSearchTest.scala | 163 +++++++++++++++++- 6 files changed, 296 insertions(+), 41 deletions(-) diff --git a/media-api/app/controllers/UsageController.scala b/media-api/app/controllers/UsageController.scala index 863e0185306..61afccc559b 100644 --- a/media-api/app/controllers/UsageController.scala +++ b/media-api/app/controllers/UsageController.scala @@ -21,8 +21,6 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc override val controllerComponents: ControllerComponents)(implicit val ec: ExecutionContext) extends BaseController with ArgoHelpers { - val numberOfDayInPeriod = 30 - def bySupplier = auth.async { implicit request => implicit val logMarker: LogMarker = MarkerMap( "requestType" -> "usage-by-supplier", @@ -30,9 +28,9 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc ) ++ RequestLoggingFilter.loggablePrincipal(request.user) Future.sequence( - Agencies.all.keys.map(elasticSearch.usageForSupplier(_, numberOfDayInPeriod))) + Agencies.all.keys.map(elasticSearch.usageForSupplier(_, UsageStore.countPeriodInDays))) .map(_.toList) - .map((s: List[SupplierUsageSummary]) => respond(s)) + .map((s: List[SupplierQuotaCount]) => respond(s)) .recover { case e => respondError(InternalServerError, "unknown-error", e.toString) } @@ -45,15 +43,29 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc "imageId" -> id, ) ++ RequestLoggingFilter.loggablePrincipal(request.user) - elasticSearch.usageForSupplier(id, numberOfDayInPeriod) - .map((s: SupplierUsageSummary) => respond(s)) + elasticSearch.usageForSupplier(id, UsageStore.countPeriodInDays) + .map((s: SupplierQuotaCount) => respond(s)) .recover { case e => respondError(InternalServerError, "unknown-error", e.toString) } } - def usageStatusForImage(id: String)(implicit logMarker: LogMarker): Future[UsageStatus] = for { + def quotaCountForSupplier(id: String) = auth.async { implicit request => + implicit val logMarker: LogMarker = MarkerMap( + "requestType" -> "quota-count-for-supplier", + "requestId" -> RequestLoggingFilter.getRequestId(request), + "supplierId" -> id, + ) ++ RequestLoggingFilter.loggablePrincipal(request.user) + + elasticSearch.quotaCountBySupplier(id, UsageStore.countPeriodInDays) + .map((s: SupplierQuotaCount) => respond(s)) + .recover { + case e => respondError(InternalServerError, "unknown-error", e.toString) + } + } + + def usageStatusForImage(id: String)(implicit logMarker: LogMarker): Future[SupplierUsageStatus] = for { imageOption <- elasticSearch.getImageById(id) image <- Future { imageOption.get } @@ -72,7 +84,7 @@ class UsageController(auth: Authentication, config: MediaApiConfig, elasticSearc ) ++ RequestLoggingFilter.loggablePrincipal(request.user) usageStatusForImage(id) - .map((u: UsageStatus) => respond(u)) + .map((u: SupplierUsageStatus) => respond(u)) .recover { case e: ImageNotFound => respondError(NotFound, "image-not-found", e.toString) case e => respondError(InternalServerError, "unknown-error", e.toString) diff --git a/media-api/app/lib/UsageStore.scala b/media-api/app/lib/UsageStore.scala index ea999fb2f93..d4cb1e5546d 100644 --- a/media-api/app/lib/UsageStore.scala +++ b/media-api/app/lib/UsageStore.scala @@ -6,6 +6,7 @@ import java.util.Properties import com.gu.mediaservice.lib.BaseStore import com.gu.mediaservice.lib.logging.GridLogging import com.gu.mediaservice.model.{Agencies, Agency, UsageRights} +import com.gu.mediaservice.model.usage.{DigitalUsage, PrintUsage, PublishedUsageStatus, RemovedUsageStatus, UnknownUsageStatus, Usage, UsageStatus, UsageType} import javax.mail.Session import javax.mail.internet.{MimeBodyPart, MimeMultipart} import org.apache.commons.mail.util.MimeMessageUtils @@ -30,30 +31,30 @@ object SupplierUsageQuota { )(SupplierUsageQuota.apply _) } -case class SupplierUsageSummary(agency: Agency, count: Long) -object SupplierUsageSummary { - implicit val customReads: Reads[SupplierUsageSummary] = ( +case class SupplierQuotaCount(agency: Agency, count: Long) +object SupplierQuotaCount { + implicit val customReads: Reads[SupplierQuotaCount] = ( (__ \ "Supplier").read[String].map(Agency(_)) ~ (__ \ "Usage").read[Long] - )(SupplierUsageSummary.apply _) + )(SupplierQuotaCount.apply _) - implicit val writes: Writes[SupplierUsageSummary] = ( + implicit val writes: Writes[SupplierQuotaCount] = ( (__ \ "agency").write[String].contramap((a: Agency) => a.supplier) ~ (__ \ "count").write[Long] - )(unlift(SupplierUsageSummary.unapply)) + )(unlift(SupplierQuotaCount.unapply)) } -case class UsageStatus( +case class SupplierUsageStatus( exceeded: Boolean, fractionOfQuota: Float, - usage: SupplierUsageSummary, + usage: SupplierQuotaCount, quota: Option[SupplierUsageQuota] ) -object UsageStatus { - implicit val writes: Writes[UsageStatus] = Json.writes[UsageStatus] +object SupplierUsageStatus { + implicit val writes: Writes[SupplierUsageStatus] = Json.writes[SupplierUsageStatus] } -case class StoreAccess(store: Map[String, UsageStatus], lastUpdated: DateTime) +case class StoreAccess(store: Map[String, SupplierUsageStatus], lastUpdated: DateTime) object StoreAccess { import play.api.libs.json.JodaWrites._ @@ -61,6 +62,10 @@ object StoreAccess { } object UsageStore extends GridLogging { + val countQualifyingStatuses: Set[UsageStatus] = Set(PublishedUsageStatus, UnknownUsageStatus, RemovedUsageStatus) + val countQualifyingPlatforms: Set[UsageType] = Set(PrintUsage, DigitalUsage) + val countPeriodInDays: Int = 30 + def extractEmail(stream: InputStream): List[String] = { val s = Session.getDefaultInstance(new Properties()) val message = MimeMessageUtils.createMimeMessage(s, stream) @@ -88,7 +93,7 @@ object UsageStore extends GridLogging { } } - def csvParser(list: List[String]): List[SupplierUsageSummary] = { + def csvParser(list: List[String]): List[SupplierQuotaCount] = { def stripQuotes(s: String): String = s .stripSuffix("\"") .stripPrefix("\"") @@ -108,7 +113,7 @@ object UsageStore extends GridLogging { case Some("Cpro Name" :: "Id" :: Nil) => lines.tail.map { case supplier :: count :: Nil => - SupplierUsageSummary(Agency(supplier), count.toInt) + SupplierQuotaCount(Agency(supplier), count.toInt) case _ => logger.error("CSV body error. Expected 2 columns") @@ -132,10 +137,10 @@ class UsageStore( bucket: String, config: MediaApiConfig, quotaStore: QuotaStore -)(implicit val ec: ExecutionContext) extends BaseStore[String, UsageStatus](bucket, config) with GridLogging { +)(implicit val ec: ExecutionContext) extends BaseStore[String, SupplierUsageStatus](bucket, config) with GridLogging { import UsageStore._ - def getUsageStatusForUsageRights(usageRights: UsageRights): Future[UsageStatus] = { + def getUsageStatusForUsageRights(usageRights: UsageRights): Future[SupplierUsageStatus] = { usageRights match { case agency: Agency => Future.successful(store.get().getOrElse(agency.supplier, { throw NoUsageQuota() })) case _ => Future.failed(new Exception("Image is not supplied by Agency")) @@ -154,7 +159,7 @@ class UsageStore( store.set(fetchUsage) } - private def fetchUsage: Map[String, UsageStatus] = { + private def fetchUsage: Map[String, SupplierUsageStatus] = { logger.info("Updating usage store") val maybeLines: Option[List[String]] = getLatestS3Stream.map(extractEmail) @@ -163,9 +168,9 @@ class UsageStore( case None => Map.empty case Some(lines) => logger.info(s"Last usage file has ${lines.length} lines") - val summary: List[SupplierUsageSummary] = csvParser(lines) + val summary: List[SupplierQuotaCount] = csvParser(lines) - def copyAgency(supplier: SupplierUsageSummary, id: String) = Agencies.all.get(id) + def copyAgency(supplier: SupplierQuotaCount, id: String) = Agencies.all.get(id) .map(a => supplier.copy(agency = a)) .getOrElse(supplier) @@ -184,12 +189,12 @@ class UsageStore( .groupBy(_.agency.supplier) .view .mapValues(_.head) - .mapValues((summary: SupplierUsageSummary) => { + .mapValues((summary: SupplierQuotaCount) => { val quota = summary.agency.id.flatMap(id => supplierQuota.get(id)) val exceeded = quota.exists(q => summary.count > q.count) val fractionOfQuota: Float = quota.map(q => summary.count.toFloat / q.count).getOrElse(0F) - UsageStatus( + SupplierUsageStatus( exceeded, fractionOfQuota, summary, diff --git a/media-api/app/lib/elasticsearch/ElasticSearch.scala b/media-api/app/lib/elasticsearch/ElasticSearch.scala index 6795673ebcc..35a514edc65 100644 --- a/media-api/app/lib/elasticsearch/ElasticSearch.scala +++ b/media-api/app/lib/elasticsearch/ElasticSearch.scala @@ -10,7 +10,7 @@ import com.gu.mediaservice.lib.elasticsearch.{CompletionPreview, ElasticSearchCl import com.gu.mediaservice.lib.logging.{GridLogging, LogMarker, MarkerMap, Stopwatch, combineMarkers} import com.gu.mediaservice.lib.metrics.FutureSyntax import com.gu.mediaservice.model.{Agencies, Agency, AwaitingReviewForSyndication, Image} -import com.gu.mediaservice.model.usage.{DigitalUsage, PrintUsage, PublishedUsageStatus, RemovedUsageStatus, Usage, UnknownUsageStatus, UsageStatus, UsageType} +import com.gu.mediaservice.model.usage.{ComposerUsageReference, DigitalUsage, FrontUsageReference, InDesignUsageReference, PrintUsage, PublishedUsageStatus, RemovedUsageStatus, Usage, UnknownUsageStatus, UsageStatus, UsageType} import com.sksamuel.elastic4s.{ElasticDsl, Hit} import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s.requests.common.Operator @@ -27,7 +27,7 @@ import com.sksamuel.elastic4s.requests.searches.queries.matches.MultiMatchQueryB import com.sksamuel.elastic4s.requests.searches.queries.matches.{FieldWithOptionalBoost, MultiMatchQuery} import lib.elasticsearch.ResultSource.{Both, Lexical, Semantic} import lib.querysyntax.{Condition, DateRange, HierarchyField, Match, Nested, Parser, Phrase, SingleField} -import lib.{MediaApiConfig, MediaApiMetrics, SupplierUsageSummary, ImageUsagesBySupplier, ImageUsagesBySupplierResult} +import lib.{MediaApiConfig, MediaApiMetrics, SupplierQuotaCount, ImageUsagesBySupplier, ImageUsagesBySupplierResult, UsageStore} import play.api.libs.json.{JsError, JsObject, JsSuccess, Json} import play.api.mvc.AnyContent import play.api.mvc.Security.AuthenticatedRequest @@ -460,7 +460,7 @@ class ElasticSearch( } ) - def usageForSupplier(id: String, numDays: Int)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SupplierUsageSummary] = { + def usageForSupplier(id: String, numDays: Int)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SupplierQuotaCount] = { val supplier = Agencies.get(id) val supplierName = supplier.supplier @@ -481,7 +481,86 @@ class ElasticSearch( executeAndLog(search, s"$id usage search").map { r => import r.result logSearchQueryIfTimedOut(search, result) - SupplierUsageSummary(supplier, result.hits.total.value) + SupplierQuotaCount(supplier, result.hits.total.value) + } + } + + def quotaCountBySupplier(id: String, numDays: Int)(implicit ex: ExecutionContext, logMarker: LogMarker): Future[SupplierQuotaCount] = { + val supplier = Agencies.get(id) + val supplierName = supplier.supplier + + val haveQualifyingStatus = termsQuery("usages.status", UsageStore.countQualifyingStatuses.map(_.toString)) + // lt("now+1d/d") instead of lte("now") so the query is day-rounded and fully request-cacheable + val beInLastPeriod = rangeQuery("usages.dateAdded").gt(s"now-${numDays}d/d").lt("now+1d/d") + val haveQualifyingPlatform = termsQuery("usages.platform", UsageStore.countQualifyingPlatforms.map(_.toString)) + val haveQualifyingUsage = nestedQuery("usages", boolQuery().must(haveQualifyingStatus, haveQualifyingPlatform, beInLastPeriod)) + + val beSupplier = boolQuery().should( + termQuery("usageRights.supplier", supplierName), + matchQuery("usageRights.suppliers", supplierName) + ).minimumShouldMatch(1) + val query = boolQuery().must(matchAllQuery()).filter(boolQuery().must(beSupplier, haveQualifyingUsage)) + + + // Usage-level filters for counting inside the nested aggregation context + val composerUsageFilter = boolQuery().must( + haveQualifyingStatus, beInLastPeriod, + termQuery("usages.platform", DigitalUsage.toString), + termQuery("usages.references.type", ComposerUsageReference.toString) + ) + val frontsUsageFilter = boolQuery().must( + haveQualifyingStatus, beInLastPeriod, + termQuery("usages.platform", DigitalUsage.toString), + termQuery("usages.references.type", FrontUsageReference.toString) + ) + val printUsageFilter = boolQuery().must( + haveQualifyingStatus, beInLastPeriod, + termQuery("usages.platform", PrintUsage.toString) + ) + // Document-level filters: classify images by which quota bucket they fall into. + val hasQualifyingComposer = nestedQuery("usages", composerUsageFilter) + val hasQualifyingFronts = nestedQuery("usages", frontsUsageFilter) + val hasQualifyingPrint = nestedQuery("usages", printUsageFilter) + + val imagesWithComposer = hasQualifyingComposer + val imagesWithFrontsButNoComposer = boolQuery().must(hasQualifyingFronts).withNot(hasQualifyingComposer) + val imagesWithPrintButNoComposer = boolQuery().must(hasQualifyingPrint).withNot(hasQualifyingComposer) + + // Quota counting logic per image: + // - Each Composer usage counts as 1; if any exist, Fronts and Print on the same image are not counted additionally. + // - Multiple Fronts count as 1 in total. + // - Multiple Print usages each count separately. + // Three mutually exclusive aggregations: + // 1. Images with composer: sum of composer usage counts + // 2. Images with fronts, no composer: count of images (each image contributes 1) + // 3. Images with print, no composer: sum of print usage counts + val composerQuotaAgg = filterAgg("has_composer", imagesWithComposer) + .subAggregations( + nestedAggregation("usages_agg", "usages") + .subAggregations( + filterAgg("qualifying", composerUsageFilter) + ) + ) + val frontsQuotaAgg = filterAgg("fronts_no_composer", imagesWithFrontsButNoComposer) + val printQuotaAgg = filterAgg("print_no_composer", imagesWithPrintButNoComposer) + .subAggregations( + nestedAggregation("usages_agg", "usages") + .subAggregations( + filterAgg("qualifying", printUsageFilter) + ) + ) + + val search = prepareSearch(query).size(0).aggs(composerQuotaAgg, frontsQuotaAgg, printQuotaAgg) + + executeAndLog(search, s"$id quota count by supplier search").map { r => + import r.result + logSearchQueryIfTimedOut(search, result) + val aggs = result.aggregations + val composerQuota = aggs.filter("has_composer").nested("usages_agg").filter("qualifying").docCount.toInt + val frontsQuota = aggs.filter("fronts_no_composer").docCount.toInt + val printQuota = aggs.filter("print_no_composer").nested("usages_agg").filter("qualifying").docCount.toInt + logger.info(s"Quota count for supplier $supplierName in last $numDays days: composer=$composerQuota, fronts=$frontsQuota, print=$printQuota") + SupplierQuotaCount(supplier, composerQuota + frontsQuota + printQuota) } } @@ -497,11 +576,8 @@ class ElasticSearch( val supplier = Agencies.get(id) val supplierName = supplier.supplier - val qualifyingStatuses = Set[UsageStatus](PublishedUsageStatus, UnknownUsageStatus, RemovedUsageStatus) - val qualifyingPlatforms = Set[UsageType](PrintUsage, DigitalUsage) - - val haveQualifyingStatus = termsQuery("usages.status", qualifyingStatuses.map(_.toString)) - val haveQualifyingPlatform = termsQuery("usages.platform", qualifyingPlatforms.map(_.toString)) + val haveQualifyingStatus = termsQuery("usages.status", UsageStore.countQualifyingStatuses.map(_.toString)) + val haveQualifyingPlatform = termsQuery("usages.platform", UsageStore.countQualifyingPlatforms.map(_.toString)) // e.g. usages@added:2026-07-01 - each date bound is inclusive, // and since they all apply within the same nested "usages" entry, multiple bounds combine into a range. @@ -531,8 +607,8 @@ class ElasticSearch( val search = prepareSearch(query).trackTotalHits(true).from(offset).size(length) def isQualifyingUsage(usage: Usage): Boolean = - qualifyingStatuses.contains(usage.status) && - qualifyingPlatforms.contains(usage.platform) && + UsageStore.countQualifyingStatuses.contains(usage.status) && + UsageStore.countQualifyingPlatforms.contains(usage.platform) && maybeDateAddedRange.forall { case (from, to) => usage.dateAdded.exists(dateAdded => !dateAdded.isBefore(from) && !dateAdded.isAfter(to)) } diff --git a/media-api/conf/routes b/media-api/conf/routes index 909bf21ac78..30f5196932f 100644 --- a/media-api/conf/routes +++ b/media-api/conf/routes @@ -36,6 +36,7 @@ GET /suggest/metadata/photoshoot controllers. GET /usage/suppliers controllers.UsageController.bySupplier GET /usage/suppliers/:id controllers.UsageController.forSupplier(id: String) GET /usage/suppliers/:id/images controllers.UsageController.imageUsagesBySupplier(id: String) +GET /usage/suppliers/:id/quotaCount controllers.UsageController.quotaCountForSupplier(id: String) GET /usage/quotas controllers.UsageController.quotas GET /usage/quotas/:id controllers.UsageController.quotaForImage(id: String) # configuration diff --git a/media-api/test/lib/UsageStoreTest.scala b/media-api/test/lib/UsageStoreTest.scala index 018f49a3fd8..e464a5dcc7f 100644 --- a/media-api/test/lib/UsageStoreTest.scala +++ b/media-api/test/lib/UsageStoreTest.scala @@ -16,7 +16,7 @@ class UsageStoreTest extends AnyFunSpec with Matchers { val list = UsageStore.csvParser(lines) - list.head should be (SupplierUsageSummary(Agency("Australian Associated Press Pty Limited (Stacey Shipton)"), 397)) + list.head should be (SupplierQuotaCount(Agency("Australian Associated Press Pty Limited (Stacey Shipton)"), 397)) } it("should parse non-ASCII RCS usage emails") { diff --git a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala index 8c1bca9968c..8380ae468f7 100644 --- a/media-api/test/lib/elasticsearch/ElasticSearchTest.scala +++ b/media-api/test/lib/elasticsearch/ElasticSearchTest.scala @@ -8,7 +8,7 @@ import com.gu.mediaservice.lib.elasticsearch.{ElasticSearchAliases, ElasticSearc import com.gu.mediaservice.lib.logging.{LogMarker, MarkerMap} import com.gu.mediaservice.model._ import com.gu.mediaservice.model.leases.DenySyndicationLease -import com.gu.mediaservice.model.usage.{PublishedUsageStatus, RemovedUsageStatus, UnknownUsageStatus} +import com.gu.mediaservice.model.usage.{PendingUsageStatus, PublishedUsageStatus, RemovedUsageStatus, SyndicationUsage, UnknownUsageStatus, ComposerUsageReference, DigitalUsage, FrontUsageReference, InDesignUsageReference, PrintUsage} import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.ElasticDsl._ import lib.querysyntax._ @@ -165,6 +165,167 @@ class ElasticSearchTest extends ElasticSearchTestBase with Eventually with Elast } } + // Saves images for a single quota test and deletes them by ID afterwards, leaving the + // shared image set (loaded in beforeAll) untouched. + private def withQuotaImages[A](images: Seq[Image])(test: => A): A = { + implicit val logMarker: LogMarker = MarkerMap() + Await.ready(saveImages(images), 1.minute) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe expectedNumberOfImages + images.size) + try test finally { + val deletes = images.map(i => executeAndLog(deleteById(index, i.id), s"Deleting quota test image ${i.id}")) + Await.ready(Future.sequence(deletes), fiveSeconds) + eventually(timeout(fiveSeconds), interval(oneHundredMilliseconds))(totalImages shouldBe expectedNumberOfImages) + } + } + + describe("quotaCountBySupplier") { + // "quota-agency" is not in Agencies.all so Agencies.get("quota-agency").supplier falls back to "quota-agency" + val supplier = "quota-agency" + val inRange = DateTime.now.minusDays(15) + val numDays = 30 + + it("counts a single composer usage as 1") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composer-1", Agency(supplier), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("counts multiple composer usages on the same image individually") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composer-2", Agency(supplier), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(ComposerUsageReference, DigitalUsage, UnknownUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 2 + } + } + + it("counts multiple fronts usages on the same image as 1") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-fronts-multi", Agency(supplier), + usages = List( + createUsage(FrontUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(FrontUsageReference, DigitalUsage, RemovedUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("counts each qualifying print usage individually") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-print-multi", Agency(supplier), + usages = List( + createUsage(InDesignUsageReference, PrintUsage, PublishedUsageStatus, inRange), + createUsage(InDesignUsageReference, PrintUsage, RemovedUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 2 + } + } + + it("composer precedence: only counts composer usages when an image has both composer and fronts usages") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composer-and-fronts", Agency(supplier), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(FrontUsageReference, DigitalUsage, PublishedUsageStatus, inRange) + ))) + withQuotaImages(images) { + // fronts usage must not be counted — if it were, result would be 2 + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("composer precedence: only counts composer usages when an image has both composer and print usages") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composer-and-print", Agency(supplier), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(InDesignUsageReference, PrintUsage, PublishedUsageStatus, inRange) + ))) + withQuotaImages(images) { + // print usage must not be counted — if it were, result would be 2 + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("counts both fronts (as 1) and print (per usage) when there are no composer usages") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-fronts-and-print", Agency(supplier), + usages = List( + createUsage(FrontUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(InDesignUsageReference, PrintUsage, PublishedUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 2 + } + } + + it("returns 0 when all usages are outside the date range") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-out-of-range", Agency(supplier), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, DateTime.now.minusDays(31))))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 0 + } + } + + it("excludes usages with a non-qualifying status") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-pending-status", Agency(supplier), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PendingUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 0 + } + } + + it("excludes usages with a non-qualifying platform") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-syndication-platform", Agency(supplier), + usages = List(createUsage(ComposerUsageReference, SyndicationUsage, PublishedUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 0 + } + } + + it("excludes images belonging to a different supplier") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-wrong-supplier", Agency("completely-different-agency"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 0 + } + } + + it("includes Composite images matched via the usageRights.suppliers field") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-composite", Composite(s"$supplier, other-supplier"), + usages = List(createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange)))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 1 + } + } + + it("qualifies all three counting statuses: published, unknown, and removed") { + implicit val logMarker: LogMarker = MarkerMap() + val images = Seq(createImage("qc-all-statuses", Agency(supplier), + usages = List( + createUsage(ComposerUsageReference, DigitalUsage, PublishedUsageStatus, inRange), + createUsage(ComposerUsageReference, DigitalUsage, UnknownUsageStatus, inRange), + createUsage(ComposerUsageReference, DigitalUsage, RemovedUsageStatus, inRange) + ))) + withQuotaImages(images) { + Await.result(ES.quotaCountBySupplier(supplier, numDays), fiveSeconds).count shouldBe 3 + } + } + } + describe("images usages by supplier") { it("only returns images with a qualifying usage status/platform for the given supplier, with the full usages list and supplier populated per item") { implicit val logMarker: LogMarker = MarkerMap() From 4b3ebfb1c272bdd394d1c3e7433227f5ed6bf025 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 16 Jul 2026 15:15:34 +0100 Subject: [PATCH 213/373] Add playwright --- .github/workflows/playwright.yml | 27 ++++++++++ .gitignore | 7 +++ e2e-tests/example.spec.ts | 18 +++++++ package-lock.json | 93 +++++++++++++++++++++++++++++++- package.json | 25 +++++++++ playwright.config.ts | 79 +++++++++++++++++++++++++++ 6 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/playwright.yml create mode 100644 e2e-tests/example.spec.ts create mode 100644 package.json create mode 100644 playwright.config.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 00000000000..3eb13143c3d --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,27 @@ +name: Playwright Tests +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install dependencies + run: npm ci + - name: Install Playwright Browsers + run: npx playwright install --with-deps + - name: Run Playwright tests + run: npx playwright test + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index c8d01fbf7a9..8c9373efc9a 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,10 @@ image-embedder.zip # Integration test data (cached S3 downloads and outputs) image-embedder-lambda/__tests__/embedder/integration/test-data/ image-embedder-lambda/__tests__/embedder/test-data + +# Playwright +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ +/playwright/.auth/ diff --git a/e2e-tests/example.spec.ts b/e2e-tests/example.spec.ts new file mode 100644 index 00000000000..54a906a4e84 --- /dev/null +++ b/e2e-tests/example.spec.ts @@ -0,0 +1,18 @@ +import { test, expect } from '@playwright/test'; + +test('has title', async ({ page }) => { + await page.goto('https://playwright.dev/'); + + // Expect a title "to contain" a substring. + await expect(page).toHaveTitle(/Playwright/); +}); + +test('get started link', async ({ page }) => { + await page.goto('https://playwright.dev/'); + + // Click the get started link. + await page.getByRole('link', { name: 'Get started' }).click(); + + // Expects page to have a heading with the name of Installation. + await expect(page.getByRole('heading', { name: 'Installation' })).toBeVisible(); +}); diff --git a/package-lock.json b/package-lock.json index fb49cefeab8..b7e0e01fe10 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,97 @@ { "name": "grid", + "version": "1.0.0", "lockfileVersion": 3, "requires": true, - "packages": {} + "packages": { + "": { + "name": "grid", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.61.1", + "@types/node": "^26.1.1" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/node": { + "version": "26.1.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", + "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + } + } } diff --git a/package.json b/package.json new file mode 100644 index 00000000000..1e50600cca5 --- /dev/null +++ b/package.json @@ -0,0 +1,25 @@ +{ + "name": "grid", + "version": "1.0.0", + "description": "Grid ==== [![License](https://img.shields.io/github/license/guardian/grid.svg)](https://github.com/guardian/grid/blob/master/LICENSE) [![Join the chat at https://gitter.im/guardian/grid](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/guardian/grid?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)", + "main": "index.js", + "directories": { + "doc": "docs" + }, + "scripts": {}, + "repository": { + "type": "git", + "url": "git+https://github.com/guardian/grid.git" + }, + "keywords": [], + "author": "", + "license": "ISC", + "bugs": { + "url": "https://github.com/guardian/grid/issues" + }, + "homepage": "https://github.com/guardian/grid#readme", + "devDependencies": { + "@playwright/test": "^1.61.1", + "@types/node": "^26.1.1" + } +} diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 00000000000..badddb2563c --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,79 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// import dotenv from 'dotenv'; +// import path from 'path'; +// dotenv.config({ path: path.resolve(__dirname, '.env') }); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './e2e-tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('')`. */ + // baseURL: 'http://localhost:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://localhost:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); From 2c850170eb4b7d703bd4c092860bfa4e2e6dfe61 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 17 Jul 2026 14:28:32 +0100 Subject: [PATCH 214/373] Add testcontainers configuration to playwright, with Kahuna booting correctly --- .github/workflows/playwright.yml | 10 +- .gitignore | 13 +- e2e-tests/example.spec.ts | 18 - e2e-tests/global-setup.ts | 130 ++ e2e-tests/global-teardown.ts | 45 + e2e-tests/images/Dockerfile | 108 + e2e-tests/images/Dockerfile.dockerignore | 10 + e2e-tests/images/README.md | 62 + e2e-tests/images/entrypoint.sh | 64 + e2e-tests/package-lock.json | 2653 ++++++++++++++++++++++ e2e-tests/package.json | 23 + e2e-tests/plans/001-testcontainers.md | 117 + e2e-tests/playwright.config.ts | 83 + e2e-tests/testcontainers/config.ts | 91 + e2e-tests/testcontainers/constants.ts | 42 + e2e-tests/testcontainers/provision.ts | 93 + e2e-tests/testcontainers/state.ts | 28 + e2e-tests/tests/example.spec.ts | 18 + e2e-tests/tests/fixtures.ts | 27 + 19 files changed, 3611 insertions(+), 24 deletions(-) delete mode 100644 e2e-tests/example.spec.ts create mode 100644 e2e-tests/global-setup.ts create mode 100644 e2e-tests/global-teardown.ts create mode 100644 e2e-tests/images/Dockerfile create mode 100644 e2e-tests/images/Dockerfile.dockerignore create mode 100644 e2e-tests/images/README.md create mode 100644 e2e-tests/images/entrypoint.sh create mode 100644 e2e-tests/package-lock.json create mode 100644 e2e-tests/package.json create mode 100644 e2e-tests/plans/001-testcontainers.md create mode 100644 e2e-tests/playwright.config.ts create mode 100644 e2e-tests/testcontainers/config.ts create mode 100644 e2e-tests/testcontainers/constants.ts create mode 100644 e2e-tests/testcontainers/provision.ts create mode 100644 e2e-tests/testcontainers/state.ts create mode 100644 e2e-tests/tests/example.spec.ts create mode 100644 e2e-tests/tests/fixtures.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 3eb13143c3d..a5abfd5d1fb 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -8,6 +8,9 @@ jobs: test: timeout-minutes: 60 runs-on: ubuntu-latest + defaults: + run: + working-directory: e2e-tests steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -17,11 +20,16 @@ jobs: run: npm ci - name: Install Playwright Browsers run: npx playwright install --with-deps + # The Testcontainers setup expects a pre-built `grid-all` image. Build it from + # the repository root (the build context must be the repo root). + - name: Build grid-all image + working-directory: . + run: DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile -t grid-all . - name: Run Playwright tests run: npx playwright test - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: name: playwright-report - path: playwright-report/ + path: e2e-tests/playwright-report/ retention-days: 30 diff --git a/.gitignore b/.gitignore index 8c9373efc9a..c582278d2dd 100644 --- a/.gitignore +++ b/.gitignore @@ -36,8 +36,11 @@ image-embedder-lambda/__tests__/embedder/integration/test-data/ image-embedder-lambda/__tests__/embedder/test-data # Playwright -/test-results/ -/playwright-report/ -/blob-report/ -/playwright/.cache/ -/playwright/.auth/ +/e2e-tests/test-results/ +/e2e-tests/playwright-report/ +/e2e-tests/blob-report/ +/e2e-tests/playwright/.cache/ +/e2e-tests/playwright/.auth/ + +# Testcontainers global-setup artifacts +/e2e-tests/.grid-urls.json diff --git a/e2e-tests/example.spec.ts b/e2e-tests/example.spec.ts deleted file mode 100644 index 54a906a4e84..00000000000 --- a/e2e-tests/example.spec.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { test, expect } from '@playwright/test'; - -test('has title', async ({ page }) => { - await page.goto('https://playwright.dev/'); - - // Expect a title "to contain" a substring. - await expect(page).toHaveTitle(/Playwright/); -}); - -test('get started link', async ({ page }) => { - await page.goto('https://playwright.dev/'); - - // Click the get started link. - await page.getByRole('link', { name: 'Get started' }).click(); - - // Expects page to have a heading with the name of Installation. - await expect(page.getByRole('heading', { name: 'Installation' })).toBeVisible(); -}); diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts new file mode 100644 index 00000000000..64fa071d8f8 --- /dev/null +++ b/e2e-tests/global-setup.ts @@ -0,0 +1,130 @@ +/** + * Playwright global setup. + * + * Boots the full local Grid stack with Testcontainers: + * 1. a shared network, + * 2. Elasticsearch + LocalStack (infrastructure), + * 3. the CloudFormation core stack + seeded buckets (provisioning), + * 4. generated per-service config (reusing dev/script/generate-config), + * 5. the pre-built `grid-all` image running all eight services. + * + * The Kahuna base URL is exposed to tests via `GRID_BASE_URL`, and the started + * containers are stashed for `global-teardown.ts`. + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { LocalstackContainer } from '@testcontainers/localstack'; +import { GenericContainer, Network, Wait } from 'testcontainers'; +import type { StartedTestContainer } from 'testcontainers'; +import { + ELASTICSEARCH_ALIAS, + ELASTICSEARCH_IMAGE, + GRID_IMAGE, + KAHUNA_PORT, + LOCALSTACK_ALIAS, + LOCALSTACK_IMAGE, + LOCALSTACK_PORT, + MEDIA_API_PORT, + REGION, + URLS_FILE, +} from './testcontainers/constants'; +import { generateServiceConfig } from './testcontainers/config'; +import { provisionCoreStack } from './testcontainers/provision'; +import { setEnvironment } from './testcontainers/state'; + +const LOCALSTACK_SERVICES = 'cloudformation,cloudwatch,dynamodb,kinesis,s3,sns,sqs,iam'; + +async function globalSetup(): Promise { + const started: StartedTestContainer[] = []; + const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); + + const network = await new Network().start(); + + // --- Infrastructure: Elasticsearch + LocalStack -------------------------- + const elasticsearch = await new GenericContainer(ELASTICSEARCH_IMAGE) + .withNetwork(network) + .withNetworkAliases(ELASTICSEARCH_ALIAS) + .withEnvironment({ + 'discovery.type': 'single-node', + 'xpack.security.enabled': 'false', + ES_JAVA_OPTS: '-Xms1024m -Xmx1024m', + }) + .withExposedPorts(9200) + .withWaitStrategy( + Wait.forHttp('/_cluster/health', 9200).forStatusCodeMatching((code) => code < 300), + ) + .withStartupTimeout(180_000) + .start(); + started.push(elasticsearch); + + const localstack = await new LocalstackContainer(LOCALSTACK_IMAGE) + .withNetwork(network) + .withNetworkAliases(LOCALSTACK_ALIAS) + .withEnvironment({ + SERVICES: LOCALSTACK_SERVICES, + DEFAULT_REGION: REGION, + KINESIS_ERROR_PROBABILITY: '0.0', + // Make resource URLs (SQS queues, etc.) resolve via the network alias so the + // app container can reach them, and keep queue URLs path-style. + LOCALSTACK_HOST: `${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`, + SQS_ENDPOINT_STRATEGY: 'path', + }) + .withStartupTimeout(120_000) + .start(); + started.push(localstack); + + // --- Provisioning + config generation ------------------------------------ + const coreStackProps = await provisionCoreStack(localstack.getConnectionUri()); + + const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); + generateServiceConfig(configDir, coreStackProps); + + // --- Application: the pre-built grid-all image --------------------------- + // All eight services run inside this single container and talk to each other over + // its localhost, so only the ports the tests target are exposed (dynamically, to + // avoid clashing with any locally-running Grid). + let gridBuilder = new GenericContainer(GRID_IMAGE) + .withNetwork(network) + .withExposedPorts(KAHUNA_PORT, MEDIA_API_PORT) + .withBindMounts([ + // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. + { source: configDir, target: '/root/.grid', mode: 'ro' }, + { source: configDir, target: '/etc/grid', mode: 'ro' }, + ]) + .withEnvironment({ + AWS_ACCESS_KEY_ID: 'test', + AWS_SECRET_ACCESS_KEY: 'test', + AWS_REGION: REGION, + AWS_DEFAULT_REGION: REGION, + AWS_CBOR_DISABLE: 'true', + // Staged Play apps run in prod mode, so an application secret must be provided. + // entrypoint.sh appends GRID_JAVA_OPTS to every service. Must be >= 256 bits. + GRID_JAVA_OPTS: + '-Dplay.http.secret.key=testcontainers-e2e-application-secret-0123456789', + }) + .withWaitStrategy(Wait.forHttp('/management/healthcheck', KAHUNA_PORT).forStatusCode(200)) + .withStartupTimeout(startupTimeoutMs); + + if (process.env.GRID_DEBUG) { + const logStream = fs.createWriteStream(path.join(os.tmpdir(), 'grid-boot.log')); + gridBuilder = gridBuilder.withLogConsumer((stream) => { + stream.on('data', (line) => logStream.write(line)); + stream.on('err', (line) => logStream.write(line)); + }); + } + + const grid = await gridBuilder.start(); + started.push(grid); + + const host = grid.getHost(); + const baseUrl = `http://${host}:${grid.getMappedPort(KAHUNA_PORT)}`; + const mediaApiUrl = `http://${host}:${grid.getMappedPort(MEDIA_API_PORT)}`; + + process.env.GRID_BASE_URL = baseUrl; + fs.writeFileSync(URLS_FILE, JSON.stringify({ kahuna: baseUrl, mediaApi: mediaApiUrl })); + + setEnvironment({ network, containers: started, configDir, baseUrl }); +} + +export default globalSetup; diff --git a/e2e-tests/global-teardown.ts b/e2e-tests/global-teardown.ts new file mode 100644 index 00000000000..2dc7ee08048 --- /dev/null +++ b/e2e-tests/global-teardown.ts @@ -0,0 +1,45 @@ +/** + * Playwright global teardown. Stops the containers started in `global-setup.ts` + * (in reverse order), removes the shared network, and deletes the generated config. + */ +import * as fs from 'fs'; +import { URLS_FILE } from './testcontainers/constants'; +import { getEnvironment } from './testcontainers/state'; + +async function globalTeardown(): Promise { + const environment = getEnvironment(); + if (!environment) { + return; + } + + const { containers, network, configDir } = environment; + + for (const container of [...containers].reverse()) { + try { + await container.stop(); + } catch (error) { + // Best effort: keep tearing down the rest of the stack. + console.warn(`Failed to stop container: ${(error as Error).message}`); + } + } + + try { + await network.stop(); + } catch (error) { + console.warn(`Failed to stop network: ${(error as Error).message}`); + } + + try { + fs.rmSync(configDir, { recursive: true, force: true }); + } catch (error) { + console.warn(`Failed to remove config dir: ${(error as Error).message}`); + } + + try { + fs.rmSync(URLS_FILE, { force: true }); + } catch (error) { + console.warn(`Failed to remove urls file: ${(error as Error).message}`); + } +} + +export default globalTeardown; diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile new file mode 100644 index 00000000000..77fd6989929 --- /dev/null +++ b/e2e-tests/images/Dockerfile @@ -0,0 +1,108 @@ +# syntax=docker/dockerfile:1 + +# Single-container image that builds and runs eight Grid Play services +# (auth, collections, cropper, kahuna, leases, media-api, metadata-editor, +# thrall) together with Kahuna's compiled frontend. +# +# Build from the repository root: +# docker build -f images/Dockerfile -t grid-all . +# +# Runtime configuration is NOT baked into the image. Mount per-environment +# config (and a stage marker) into /etc/grid at run time, e.g.: +# docker run --rm -v "$PWD/my-config:/etc/grid" grid-all +# See images/README.md for details. + +# --------------------------------------------------------------------------- +# Stage 1: build the Kahuna frontend (webpack production bundle) +# --------------------------------------------------------------------------- +FROM node:22.12.0-bookworm AS frontend + +WORKDIR /build/kahuna + +# Install dependencies first for better layer caching. +COPY kahuna/package.json kahuna/package-lock.json ./ +RUN npm ci + +# Build the production bundle into kahuna/public/dist. +COPY kahuna/ ./ +RUN npm run dist + +# --------------------------------------------------------------------------- +# Stage 2: compile and stage the Play services with sbt +# --------------------------------------------------------------------------- +FROM eclipse-temurin:11-jdk-jammy AS backend + +ARG SBT_VERSION=1.10.3 + +# Install sbt from the official release tarball. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends curl ca-certificates; \ + curl -fLo /tmp/sbt.tgz "https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz"; \ + tar -xzf /tmp/sbt.tgz -C /opt; \ + ln -s /opt/sbt/bin/sbt /usr/local/bin/sbt; \ + rm -f /tmp/sbt.tgz; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +# Copy the full repository (build context is the repo root). A .dockerignore +# keeps target/, node_modules/ and logs/ out of the context. +COPY --exclude=e2e-tests/images . . + +# Drop in the pre-built frontend so `kahuna/stage` packages the bundled assets. +COPY --from=frontend /build/kahuna/public/dist ./kahuna/public/dist + +# Stage every requested service in a single sbt invocation to share compilation. +RUN sbt \ + "auth/stage" \ + "collections/stage" \ + "cropper/stage" \ + "kahuna/stage" \ + "leases/stage" \ + "media-api/stage" \ + "metadata-editor/stage" \ + "thrall/stage" + +# --------------------------------------------------------------------------- +# Stage 3: slim runtime image with native image tooling and all services +# --------------------------------------------------------------------------- +FROM eclipse-temurin:11-jre-jammy AS runtime + +# Native tools used by the image processing services (see repo Brewfile). +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + graphicsmagick \ + imagemagick \ + pngquant \ + libimage-exiftool-perl \ + libgd3; \ + rm -rf /var/lib/apt/lists/* + +# Staged apps expect their config at /usr/share//conf/application.conf +# and write gc logs to /var/log// (see build.sbt javaOptions). +COPY --from=backend /build/auth/target/universal/stage /usr/share/auth +COPY --from=backend /build/collections/target/universal/stage /usr/share/collections +COPY --from=backend /build/cropper/target/universal/stage /usr/share/cropper +COPY --from=backend /build/kahuna/target/universal/stage /usr/share/kahuna +COPY --from=backend /build/leases/target/universal/stage /usr/share/leases +COPY --from=backend /build/media-api/target/universal/stage /usr/share/media-api +COPY --from=backend /build/metadata-editor/target/universal/stage /usr/share/metadata-editor +COPY --from=backend /build/thrall/target/universal/stage /usr/share/thrall + +RUN set -eux; \ + for svc in auth collections cropper kahuna leases media-api metadata-editor thrall; do \ + mkdir -p "/var/log/$svc"; \ + done + +COPY e2e-tests/images/entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +# The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. +ENV AWS_CBOR_DISABLE=true + +# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases +EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/e2e-tests/images/Dockerfile.dockerignore b/e2e-tests/images/Dockerfile.dockerignore new file mode 100644 index 00000000000..f28462df9be --- /dev/null +++ b/e2e-tests/images/Dockerfile.dockerignore @@ -0,0 +1,10 @@ +# BuildKit uses this file (named .dockerignore) to trim the build +# context for images/Dockerfile. Keeps build artefacts and caches out of the +# context sent to the daemon. +**/target +**/node_modules +**/logs +**/.git +.git +**/.DS_Store +dev/.localstack diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md new file mode 100644 index 00000000000..9b5c576858e --- /dev/null +++ b/e2e-tests/images/README.md @@ -0,0 +1,62 @@ +# Grid all-in-one container + +`images/Dockerfile` builds a single container that runs eight Grid Play +services plus Kahuna's compiled frontend: + +| Service | Port | +| ----------------- | ---- | +| `media-api` | 9001 | +| `thrall` | 9002 | +| `kahuna` (+ UI) | 9005 | +| `cropper` | 9006 | +| `metadata-editor` | 9007 | +| `collections` | 9010 | +| `auth` | 9011 | +| `leases` | 9012 | + +## Build + +Build from the **repository root** (the build context must be the repo root): + +```bash +DOCKER_BUILDKIT=1 docker build -f images/Dockerfile -t grid-all . +``` + +The build has three stages: + +1. **frontend** — `node:22.12.0` builds the Kahuna webpack bundle + (`npm ci && npm run dist`). +2. **backend** — `eclipse-temurin:11-jdk` installs sbt and stages all eight + services (`sbt /stage`), packaging the frontend assets into Kahuna. +3. **runtime** — `eclipse-temurin:11-jre` with the native image tools + (`graphicsmagick`, `imagemagick`, `pngquant`, `exiftool`, `libgd3`) and the + staged apps under `/usr/share/`. + +## Runtime configuration + +Configuration is **not** baked into the image. At runtime each service loads +`/etc/grid/common.conf` and `/etc/grid/.conf` (and reads the stage +from `/etc/grid/stage`; defaults to `DEV`). Mount your environment's config in: + +```bash +docker run --rm \ + -v "$PWD/my-config:/etc/grid:ro" \ + -p 9001:9001 -p 9002:9002 -p 9005:9005 -p 9006:9006 \ + -p 9007:9007 -p 9010:9010 -p 9011:9011 -p 9012:9012 \ + grid-all +``` + +The services still need their backing infrastructure (Elasticsearch, S3/ +DynamoDB/Kinesis/SNS/SQS, the image resizer, auth provider) to be reachable — +see the root `docker-compose.yml` for the local dev topology. + +## Selecting services + +Run a subset via `GRID_SERVICES` (space-separated), and append JVM options with +`GRID_JAVA_OPTS`: + +```bash +docker run --rm -e GRID_SERVICES="media-api kahuna auth" grid-all +``` + +If any running service exits, the container stops. diff --git a/e2e-tests/images/entrypoint.sh b/e2e-tests/images/entrypoint.sh new file mode 100644 index 00000000000..9779d152002 --- /dev/null +++ b/e2e-tests/images/entrypoint.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# Supervises the Grid Play services inside a single container. +# +# Each staged application defaults to port 9000 in production, so the correct +# port is passed explicitly per service via -Dhttp.port. Services run in the +# background; if any one exits the container stops, and SIGTERM/SIGINT are +# forwarded for a clean shutdown. +# +# Set GRID_SERVICES to a space-separated subset to run fewer services, e.g. +# docker run -e GRID_SERVICES="media-api kahuna" grid-all +# Extra JVM options can be appended via GRID_JAVA_OPTS. + +set -euo pipefail + +# service -> http port +declare -A PORTS=( + [media-api]=9001 + [thrall]=9002 + [kahuna]=9005 + [cropper]=9006 + [metadata-editor]=9007 + [collections]=9010 + [auth]=9011 + [leases]=9012 +) + +DEFAULT_SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall" +SERVICES="${GRID_SERVICES:-$DEFAULT_SERVICES}" +EXTRA_JAVA_OPTS="${GRID_JAVA_OPTS:-}" + +pids=() + +shutdown() { + echo "Shutting down Grid services..." + for pid in "${pids[@]}"; do + kill -TERM "$pid" 2>/dev/null || true + done + wait + exit 0 +} + +trap shutdown TERM INT + +for svc in $SERVICES; do + port="${PORTS[$svc]:-}" + if [[ -z "$port" ]]; then + echo "Unknown service '$svc' (no port mapping); skipping." >&2 + continue + fi + + bin="/usr/share/$svc/bin/$svc" + if [[ ! -x "$bin" ]]; then + echo "Executable for service '$svc' not found at $bin; skipping." >&2 + continue + fi + + echo "Starting $svc on port $port" + # shellcheck disable=SC2086 + "$bin" -Dhttp.port="$port" $EXTRA_JAVA_OPTS & + pids+=("$!") +done + +sleep 100000 diff --git a/e2e-tests/package-lock.json b/e2e-tests/package-lock.json new file mode 100644 index 00000000000..5693a9a0736 --- /dev/null +++ b/e2e-tests/package-lock.json @@ -0,0 +1,2653 @@ +{ + "name": "e2e-tests", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "e2e-tests", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@aws-sdk/client-cloudformation": "^3.658.0", + "@aws-sdk/client-s3": "^3.658.0", + "@playwright/test": "^1.61.1", + "@testcontainers/localstack": "^10.13.2", + "@types/node": "^26.1.1", + "json5": "^2.2.3", + "testcontainers": "^10.13.2" + } + }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.18", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.18.tgz", + "integrity": "sha512-IImkbEyXdV6/uaF5r6Wkk+8718mQw1ll83j0a4a30R3JM/rHVFdWAiT4jtJpFjJiIwM/oJ6SxIxr0z2TaQUGqw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-cloudformation": { + "version": "3.1089.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-cloudformation/-/client-cloudformation-3.1089.0.tgz", + "integrity": "sha512-56kyaFKOnU0rE1DPTxIsmegpMb9z8uJJJpKjg+/6vynPbnGVHOAoxBxbVjCHhSLuozezVEM/k0T9g3fRzxy2tA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-node": "^3.972.70", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1089.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1089.0.tgz", + "integrity": "sha512-Sc+JOtNeP+v+XdP9Rb4Hh+uhiNO2hh/CZQbKYooNakhOIgK6/K0cjoDCEGeFmkG0vf2DopTmSctzKlcKwy1BPw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.18", + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-node": "^3.972.70", + "@aws-sdk/middleware-sdk-s3": "^3.972.64", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.975.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.3.tgz", + "integrity": "sha512-7ur3kCKuvPLqlsZ2XlvnNBVQ7KkpSu6Y6dOTwSPHLrFpTEfZM8isLBJc4cgv96WB7GifeVM436mpycwxBd2vEA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@aws-sdk/xml-builder": "^3.972.36", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.29.4", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.59.tgz", + "integrity": "sha512-Ny5e4Mfh3QPmiAc0AiUe+cbTXDlxkU3Rc+EpWOfyWeWEy6yp7Fa1KmfNeCc+1a8by9zQ9gtohmiQUkMPScF3ng==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.61", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.61.tgz", + "integrity": "sha512-8jAjgStl5Ytq4+HF3X/9f+EmRinaRbGRRtQGktlPfBRVx73H+R1y48vIeXerQtYGFaUqkEp3fT6jP854rVO2yQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.4.tgz", + "integrity": "sha512-e6ZvVsj90aRALf1kHP+J4iqC1496ZpVgqI/+u0LJ5HL7q7ATauGy4gdDvRCP13L1pN/fMiZLah162PGIYkbUVQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-env": "^3.972.59", + "@aws-sdk/credential-provider-http": "^3.972.61", + "@aws-sdk/credential-provider-login": "^3.972.66", + "@aws-sdk/credential-provider-process": "^3.972.59", + "@aws-sdk/credential-provider-sso": "^3.973.3", + "@aws-sdk/credential-provider-web-identity": "^3.972.65", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/credential-provider-imds": "^4.4.9", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.66", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.66.tgz", + "integrity": "sha512-g2fsqm87r/nKthLZ0VkkDBElkGg0PvSa8d97HQ6EilMbJTZ6hxa8FxkSZyJfgPfFdZn0TTmkOffQmTSUcAHIng==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.70.tgz", + "integrity": "sha512-3xzvkGdykBunxqh8WudmUpSyLWvIhfI6aBQo1b5rb3mDO5mNLadK+0hiI0qBQBMVynJbfLO+Ajy9dztMwy9O8w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.59", + "@aws-sdk/credential-provider-http": "^3.972.61", + "@aws-sdk/credential-provider-ini": "^3.973.4", + "@aws-sdk/credential-provider-process": "^3.972.59", + "@aws-sdk/credential-provider-sso": "^3.973.3", + "@aws-sdk/credential-provider-web-identity": "^3.972.65", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/credential-provider-imds": "^4.4.9", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.59.tgz", + "integrity": "sha512-DlZF2/MhLlatDdlrIy3CUCpfdbLrKx+3SMjVo+WyHnPpwzkc/M3vwAHw4OVJf7DMvO+4vfRqSCMc/E9I1auN0g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.3.tgz", + "integrity": "sha512-hmdDHoy2G5Es2e8IgelNMYUuSQI6uCIAKZMJ2u2PdKDhxvbk1uWD/g4+R7R5c/tJfKEB1+KjjWiaoCr/S+ZTiQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/token-providers": "3.1088.0", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.65", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.65.tgz", + "integrity": "sha512-gHQb/Kt0chjk/JQDa/GJDqmAvEuVn8n7z10wK2h0LFM9TUDRkohgOO4aEF+s2sBLM0br7Cl5W6P7phgjrrJvLQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.64", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.64.tgz", + "integrity": "sha512-RBi43anhDBUv+HCfxCOXwGOE7GmT4n7ChV04Mwr22RhXTNcamW/iWnJlOotDPCZSrJ4dEvhZSiWWQMwLX+ZhFA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.33", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.33.tgz", + "integrity": "sha512-dVZOroI/r3/ENvqNGgjMPul+jjlz9GddfVusgTXlVjfZj5isibOxecLkGQbRPp8XOuX+RAfjXLFgPkD1JS5xrw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.41.tgz", + "integrity": "sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1088.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1088.0.tgz", + "integrity": "sha512-4ObatWt2qpJg5FBk4LOOKrTQYzaqeewAtdO3r9ZO8lH9YqLtpTzLyIdy0mJ+nVdfYOnqISkKNfmzP22bNDhwyw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.2", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.2.tgz", + "integrity": "sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.36", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.36.tgz", + "integrity": "sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@balena/dockerignore": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@balena/dockerignore/-/dockerignore-1.0.2.tgz", + "integrity": "sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.4.tgz", + "integrity": "sha512-k9Dj3DV/itK9D06Y8f190Qgop7/Ui+D0njFV3LHMPwPT75DpXLQohE9Wmz0QElrJnzsjB7KPWiKJbOl7IPDArQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/grpc-js/node_modules/@grpc/proto-loader": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.1.tgz", + "integrity": "sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.7.15", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.15.tgz", + "integrity": "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.2.5", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "dev": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.5.tgz", + "integrity": "sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.1.tgz", + "integrity": "sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.1.tgz", + "integrity": "sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.2.tgz", + "integrity": "sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@smithy/core": { + "version": "3.29.5", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.5.tgz", + "integrity": "sha512-i0dk2t5B+CwV/dcJdUHILYkOQF5lof8f44dFCfDWToGCxjT9YQ+CgHqTAvJxzc3+zqQwm2QtVoJ5IqiNar/CnQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.10.tgz", + "integrity": "sha512-MJenAe4OKRZUo1LdYYFDCsSHxaHvInIU/z52GsheO9vl1/VSySVCr0zkyKD6TFiGkSUaWGxvKZ/70OvgUZR5HQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.6.7", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.7.tgz", + "integrity": "sha512-3zpg8yqqyXzoK2TsRDdkqVOj2RDBFfLXwCczOZ5c7TWB4eiaebfSCsbMjDPYB3PJ9ihV62QaeadZ+wLadZtNGA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.9.7", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.7.tgz", + "integrity": "sha512-wCU8HCLjAtAVqxxe0j2xff9LcEPw3yjBbg5IdQDIYFnxnPxbxcSLc7rgex7kqm9L/WYOnJEgaWQlfDkZleozMA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.6.6", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.6.tgz", + "integrity": "sha512-efP6DN3UTFrzIsGO42/xcabv8jU7+9nwEdphFUH7yL0k010ERyAWaO41KFQIDLcFZLZ8xzIQr4wplFxNzslSGQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", + "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@testcontainers/localstack": { + "version": "10.28.0", + "resolved": "https://registry.npmjs.org/@testcontainers/localstack/-/localstack-10.28.0.tgz", + "integrity": "sha512-3izFtLEklezU5gmqSYtT0xKsLvdXV2AusNRXeO7hMWTb9JypoztT+Nf0dza22qczqpRi6mDKgqrgla5YoIIjJg==", + "dev": true, + "license": "MIT", + "dependencies": { + "testcontainers": "^10.28.0" + } + }, + "node_modules/@types/docker-modem": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/docker-modem/-/docker-modem-3.0.6.tgz", + "integrity": "sha512-yKpAGEuKRSS8wwx0joknWxsmLha78wNMe9R2S3UNsVOkZded8UqOrV8KoeDXoXsjndxwyF3eIhyClGbO1SEhEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/dockerode": { + "version": "3.3.47", + "resolved": "https://registry.npmjs.org/@types/dockerode/-/dockerode-3.3.47.tgz", + "integrity": "sha512-ShM1mz7rCjdssXt7Xz0u1/R2BJC7piWa3SJpUBiVjCf2A3XNn4cP6pUVaD8bLanpPVVn4IKzJuw3dOvkJ8IbYw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/docker-modem": "*", + "@types/node": "*", + "@types/ssh2": "*" + } + }, + "node_modules/@types/node": { + "version": "26.1.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", + "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@types/ssh2": { + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-1.15.5.tgz", + "integrity": "sha512-N1ASjp/nXH3ovBHddRJpli4ozpk6UdDYIX4RJWFa9L1YKnzdhTlVmiGHm4DZnj/jLbqZpes4aeR30EFGQtvhQQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18" + } + }, + "node_modules/@types/ssh2-streams": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/ssh2-streams/-/ssh2-streams-0.1.13.tgz", + "integrity": "sha512-faHyY3brO9oLEA0QlcO8N2wT7R0+1sHWZvQ+y3rMLwdY1ZyS1z0W3t65j9PqT4HmQ6ALzNe7RZlNuCNE0wBSWA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/ssh2/node_modules/@types/node": { + "version": "18.19.130", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.130.tgz", + "integrity": "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/ssh2/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/asn1": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", + "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true, + "license": "MIT" + }, + "node_modules/async-lock": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/async-lock/-/async-lock-1.4.1.tgz", + "integrity": "sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/b4a": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.8.1.tgz", + "integrity": "sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/bare-events": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.9.1.tgz", + "integrity": "sha512-Z0oHEHAFDZkffN8Qc39zNZjQlMDkPJRyyyZieU1VH7u8c5S+qHZ2S8ixdKIAxEjfHO7FJxXmJWgteOghVanIsg==", + "dev": true, + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.7.4", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.7.4.tgz", + "integrity": "sha512-y1kC+ffIx/tPLdTE693uNjHfzTfr+ravR5tvWlMXe25nELbkqV400S71qHDwbkAQ1FVEZobB1NFRzFbCCcyBCQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.1.1.tgz", + "integrity": "sha512-JprUlveX3QjApC1cTpsUOiscADftCGVWkzitbHsRqv84hzYwYHw2mbluddsq5TvI8mH/8Ov1f4BiMAdcB0oYnQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/bare-stream": { + "version": "2.13.3", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.13.3.tgz", + "integrity": "sha512-Kc+brLqvEqGkjyfiwJmImAOqLZL7OsoLKuavx+hJjgVV3nLTOjloJyPMFxjUPerGGHrNH0fLU06jjykMLWrERQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.8.1", + "streamx": "^2.25.0", + "teex": "^1.0.1" + }, + "peerDependencies": { + "bare-abort-controller": "*", + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + }, + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.4.5.tgz", + "integrity": "sha512-K+y9xF1tN+CdPu4qWwr0QiK1Al07eFPGYK5M2pDXcmHdMdgC/tT/bpmMe1hrmRHaidKLkXrC+cRNYf3XVDUhSQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/bl/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.2.tgz", + "integrity": "sha512-w5JZcKgdhDOgOwm8H+KgbosopHMuGcl6qbulwjtz3SM7I7P3yW1eAjzMPLrIE+NQ9vjgANKHWeMHnrT0OXW1oA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/buildcheck": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/byline": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", + "integrity": "sha512-s6webAy+R4SR8XVuJWt2V2rGvhnrhxN+9S15GNuTK3wKPOXFF6RNc+8ug2XhH+2s4f+uudG4kUVYmYOQWL2g0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, + "license": "ISC" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cliui/node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cpu-features": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/cpu-features/-/cpu-features-0.0.10.tgz", + "integrity": "sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "dependencies": { + "buildcheck": "~0.0.6", + "nan": "^2.19.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "dev": true, + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/docker-compose": { + "version": "0.24.8", + "resolved": "https://registry.npmjs.org/docker-compose/-/docker-compose-0.24.8.tgz", + "integrity": "sha512-plizRs/Vf15H+GCVxq2EUvyPK7ei9b/cVesHvjnX4xaXjM9spHe2Ytq0BitndFgvTJ3E3NljPNUEl7BAN43iZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "yaml": "^2.2.2" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/docker-modem": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.1.1", + "readable-stream": "^3.5.0", + "split-ca": "^1.0.1", + "ssh2": "^1.15.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/docker-modem/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode": { + "version": "4.0.12", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.12.tgz", + "integrity": "sha512-/bCZd6KlGcjZO8Buqmi/vXuqEGVEZ0PNjx/biBNqJD3MhK9DmdiAuKxqfNhflgDESDIiBz3qF+0e55+CpnrUcw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@grpc/grpc-js": "^1.11.1", + "@grpc/proto-loader": "^0.7.13", + "docker-modem": "^5.0.7", + "protobufjs": "^7.3.2", + "tar-fs": "^2.1.4", + "uuid": "^10.0.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/dockerode/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/dockerode/node_modules/tar-fs": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", + "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/dockerode/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true, + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/foreground-child": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", + "integrity": "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==", + "dev": true, + "license": "ISC", + "dependencies": { + "cross-spawn": "^7.0.6", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-port": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.2.0.tgz", + "integrity": "sha512-afP4W205ONCuMoPBqcR6PSXnzX35KTcJygfJfcp+QY+uwm3p20p1YczWXhlICIzGMCxYBQcySEcOgsJcrkyobg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "10.5.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.5.0.tgz", + "integrity": "sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/jackspeak": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.3.tgz", + "integrity": "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/minimatch": { + "version": "9.0.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", + "integrity": "sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.2" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", + "dev": true, + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true, + "license": "MIT", + "bin": { + "mkdirp": "bin/cmd.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nan": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.28.0.tgz", + "integrity": "sha512-fTsDz99OTq2sVePhGdp4qQhggZFtKr64ZNVyVajRKtMOkJxYekplBh577PiJB12v/D3s2E5cGtOI45LWp6rnLQ==", + "dev": true, + "license": "MIT", + "optional": true + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/package-json-from-dist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", + "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/proper-lockfile/node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/properties-reader": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/properties-reader/-/properties-reader-2.3.0.tgz", + "integrity": "sha512-z597WicA7nDZxK12kZqHr2TcvwNU1GCfA5UwfDY/HDp3hXPoPlb5rlEx9bwGTiJnc0OqbBTkU975jDToth8Gxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mkdirp": "^1.0.4" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "type": "github", + "url": "https://github.com/steveukx/properties?sponsor=1" + } + }, + "node_modules/protobufjs": { + "version": "7.6.5", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.6.5.tgz", + "integrity": "sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==", + "dev": true, + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.5", + "@protobufjs/eventemitter": "^1.1.1", + "@protobufjs/fetch": "^1.1.1", + "@protobufjs/float": "^1.0.2", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.1", + "@types/node": ">=13.7.0", + "long": "^5.3.2" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/readable-stream": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.7.0.tgz", + "integrity": "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg==", + "dev": true, + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.9", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.9.tgz", + "integrity": "sha512-7o1wEA2RyMP7Iu7GNba9vc0RWWGACJOCZBJX2GJWip0ikV+wcOsgVuY9uE8CPiyQhkGFSlhuSkZPavN7u1c2Fw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/split-ca": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/ssh-remote-port-forward": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ssh-remote-port-forward/-/ssh-remote-port-forward-1.0.4.tgz", + "integrity": "sha512-x0LV1eVDwjf1gmG7TTnfqIzf+3VPRz7vrNIjX6oYLbeCrf/PeVY6hkT68Mg+q02qXxQhrLjB0jfgvhevoCRmLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/ssh2": "^0.5.48", + "ssh2": "^1.4.0" + } + }, + "node_modules/ssh-remote-port-forward/node_modules/@types/ssh2": { + "version": "0.5.52", + "resolved": "https://registry.npmjs.org/@types/ssh2/-/ssh2-0.5.52.tgz", + "integrity": "sha512-lbLLlXxdCZOSJMCInKH2+9V/77ET2J6NPQHpFI0kda61Dd1KglJs+fPQBchizmzYSOJBgdTajhPqBO1xxLywvg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/ssh2-streams": "*" + } + }, + "node_modules/ssh2": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "asn1": "^0.2.6", + "bcrypt-pbkdf": "^1.0.2" + }, + "engines": { + "node": ">=10.16.0" + }, + "optionalDependencies": { + "cpu-features": "~0.0.10", + "nan": "^2.23.0" + } + }, + "node_modules/streamx": { + "version": "2.28.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.28.0.tgz", + "integrity": "sha512-1Yowhzjf0ivGMrTIkY9hav5TxobO9qIVqUE41fiCGMGgc3CLlf4MY+9AHmZqBWgDTue0fY9zWjYFVyf6Diuobw==", + "dev": true, + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.3.tgz", + "integrity": "sha512-/hU4AXnIdZu+Gvl1pk0oI5f5HxWsCJRtY2aFaJdk9VvyL48DWU6iU5WAIPG+wIi1YvWA6eTJvIviP/tMAZZNwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.2.0.tgz", + "integrity": "sha512-ojzvCvVaNp6aOTFmG7jaRD0meowIAuPc3cMMhSgKiVWws1GyHbGd/xvnyuRKcKlMpt3qvxx6r0hreCNITP9hIg==", + "dev": true, + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "bare-fs": "^4.5.5", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/teex": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/teex/-/teex-1.0.1.tgz", + "integrity": "sha512-eYE6iEI62Ni1H8oIa7KlDU6uQBtqr4Eajni3wX7rpfXD8ysFx8z0+dri+KWEPWpBsxXfxu58x/0jvTVT1ekOSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "streamx": "^2.12.5" + } + }, + "node_modules/testcontainers": { + "version": "10.28.0", + "resolved": "https://registry.npmjs.org/testcontainers/-/testcontainers-10.28.0.tgz", + "integrity": "sha512-1fKrRRCsgAQNkarjHCMKzBKXSJFmzNTiTbhb5E/j5hflRXChEtHvkefjaHlgkNUjfw92/Dq8LTgwQn6RDBFbMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@balena/dockerignore": "^1.0.2", + "@types/dockerode": "^3.3.35", + "archiver": "^7.0.1", + "async-lock": "^1.4.1", + "byline": "^5.0.0", + "debug": "^4.3.5", + "docker-compose": "^0.24.8", + "dockerode": "^4.0.5", + "get-port": "^7.1.0", + "proper-lockfile": "^4.1.2", + "properties-reader": "^2.3.0", + "ssh-remote-port-forward": "^1.0.4", + "tar-fs": "^3.0.7", + "tmp": "^0.2.3", + "undici": "^5.29.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.7.tgz", + "integrity": "sha512-vlLytXkeP4xvEq2otHeJfSQIRyWxo/oZGEbXrtEEF9Hnmrdly59sUbzZ/QgyWuLYHctCHxFF4tRQZNQ9k60ExQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tmp": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", + "integrity": "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.14" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", + "dev": true, + "license": "Unlicense" + }, + "node_modules/undici": { + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, + "license": "MIT" + }, + "node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/yargs": { + "version": "17.7.3", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.3.tgz", + "integrity": "sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "dev": true, + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + } + } +} diff --git a/e2e-tests/package.json b/e2e-tests/package.json new file mode 100644 index 00000000000..a8d3109dac0 --- /dev/null +++ b/e2e-tests/package.json @@ -0,0 +1,23 @@ +{ + "name": "e2e-tests", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "playwright test", + "test:headed": "playwright test --headed", + "test:report": "playwright show-report" + }, + "keywords": [], + "author": "", + "license": "ISC", + "devDependencies": { + "@aws-sdk/client-cloudformation": "^3.658.0", + "@aws-sdk/client-s3": "^3.658.0", + "@playwright/test": "^1.61.1", + "@testcontainers/localstack": "^10.13.2", + "@types/node": "^26.1.1", + "json5": "^2.2.3", + "testcontainers": "^10.13.2" + } +} diff --git a/e2e-tests/plans/001-testcontainers.md b/e2e-tests/plans/001-testcontainers.md new file mode 100644 index 00000000000..ed3891c3e6b --- /dev/null +++ b/e2e-tests/plans/001-testcontainers.md @@ -0,0 +1,117 @@ +# Testcontainers setup for e2e-tests + +Add a Testcontainers-driven setup to the Playwright `e2e-tests/` project that boots the +pre-built **grid-all** image (from `e2e-tests/images/Dockerfile`) alongside +**Elasticsearch** and **LocalStack**, fully provisions the backing AWS resources in +LocalStack, generates real service config by **reusing the existing +`service-config.js`**, and runs all 8 Grid services — wired through Playwright's +`globalSetup`/`globalTeardown`. + +## Decisions +- **Scope:** grid-all container + Elasticsearch + LocalStack. Skip imgops & oidc for now. +- **Image source:** assume the `grid-all` image is already built; do not build in tests. +- **Config:** full provisioning — apply the CloudFormation core stack, seed buckets, and + generate per-service config from stack outputs; mount at `/etc/grid`. +- **Config generation:** reuse `dev/script/generate-config/service-config.js` + (`getCoreConfigs`) rather than reimplementing. +- **Services:** start all 8 services (the `entrypoint.sh` default `GRID_SERVICES`). +- **Lifecycle:** Playwright `globalSetup` / `globalTeardown`; expose `baseURL` via env. + +## Key facts +- grid-all runs 8 Play services. Ports: media-api 9001, thrall 9002, kahuna 9005, + cropper 9006, metadata-editor 9007, collections 9010, auth 9011, leases 9012. + Kahuna (9005) is the UI entrypoint -> `baseURL` target. +- Config not baked in: each service loads `/etc/grid/common.conf` + + `/etc/grid/.conf`, stage from `/etc/grid/stage` (default DEV). `entrypoint.sh` + respects `GRID_SERVICES`/`GRID_JAVA_OPTS`. `ENV AWS_CBOR_DISABLE=true` is set. +- Backing infra (docker-compose.yml): elasticsearch 8.18.3 (9200); localstack 4.5.0 + (4566; `SERVICES=cloudformation,cloudwatch,dynamodb,kinesis,s3,sns,sqs,iam`, eu-west-1). + +## Provisioning details (from setup.sh / generate-config) +- CFN core stack `dev/cloudformation/grid-dev-core.yml` creates: Kinesis streams + (`media-service-DEV-thrall`, low-priority), SQS `IngestSqsQueue`, S3 buckets + (Ingest/Fail, Image, Thumb, Reaper, Quarantine, Key, ImageOrigin, Config, Collections, + UsageMail), DynamoDB tables (Syndication, Edits, SoftDeletedMetadata, etc). +- Seed: upload API key (`"DEV Key"`) to KeyBucket; `dev/config/photographers.json`, + `rcs-quota.json`, `usage_rights.json` to ConfigBucket; `usages.eml` to UsageMailBucket. +- `generate-config.js` reads CFN stack resources via the AWS SDK and feeds + `service-config.js` (`getCoreConfigs`) to write per-service `.conf` + `common.conf`. + Defaults come from `config.json5` (es cluster `media-service`, shards 1, replicas 0). +- **Reuse strategy:** `require` `service-config.js` from `e2e-tests`. Do NOT run + `generate-config.js` directly (it writes to `~/.grid` and assumes AWS SDK v2 + `.env`). + Instead build the config object ourselves from live CFN stack resources + (LogicalResourceId -> PhysicalResourceId map) + `config.json5` defaults + `DOMAIN`, + call `ServiceConfig.getCoreConfigs`, then post-process. +- **Critical adaptation:** on a Testcontainers network the grid-all container must reach + ES/LocalStack via network **aliases** (`http://elasticsearch:9200`, + `http://localstack:4566`), not `localhost` — post-process `es6.url` / + `aws.local.endpoint` accordingly. Run in **NO_AUTH** mode + (`LocalAuthenticationProvider`/`LocalAuthorisationProvider`) to skip pan-domain/oidc. + +## Steps + +### Phase 1 — Dependencies +1. Add dev deps to `e2e-tests/package.json`: `testcontainers`, + `@testcontainers/localstack`, `@aws-sdk/client-cloudformation`, `@aws-sdk/client-s3`, + plus `json5` (to load `config.json5`). ES via `GenericContainer` or + `@testcontainers/elasticsearch`. Add a `test` script. + +### Phase 2 — Infra containers (`global-setup.ts`) +2. Create a shared Testcontainers `Network`. +3. Start Elasticsearch (`docker.elastic.co/.../elasticsearch:8.18.3`, single-node, + security disabled) with alias `elasticsearch`. +4. Start LocalStack (`localstack:4.5.0`, the required `SERVICES`, region `eu-west-1`) + with alias `localstack`. + +### Phase 3 — Provisioning (`global-setup.ts`) +5. Apply `grid-dev-core.yml` via the CloudFormation client against LocalStack; wait for + `CREATE_COMPLETE`; read stack resources into a logical->physical name map. +6. Seed buckets: API key -> KeyBucket; config JSON -> ConfigBucket; `usages.eml` -> + UsageMailBucket (reuse files in `dev/config/`). +7. Generate service config by **reusing `service-config.js`**: `require` `ServiceConfig`, + build a config object from the live CFN stack resources + `config.json5` defaults + + `DOMAIN`, call `getCoreConfigs`, then post-process `es6.url` / `aws.local.endpoint` + to the network aliases. Write results to `e2e-tests/.tmp-config`. Set `NO_AUTH=true` + so `common.conf` uses the Local auth/authorisation providers. + +### Phase 4 — App container (`global-setup.ts`) +8. Start grid-all (pre-built tag) on the network with default `GRID_SERVICES` (all 8), + AWS creds/endpoint env + `AWS_CBOR_DISABLE`, bind-mount `.tmp-config` -> `/etc/grid`, + expose 9001/9002/9005/9006/9007/9010/9011/9012. +9. Wait strategies: ES healthy, then kahuna HTTP 200 on 9005 (generous timeout). Export + `baseURL` (mapped 9005 host port) via `process.env` + a temp file for teardown. + +### Phase 5 — Teardown & Playwright wiring +10. `global-teardown.ts`: stop grid-all/ES/LocalStack, remove the network, clean `.tmp-config`. +11. `playwright.config.ts`: set `globalSetup`/`globalTeardown`, `use.baseURL` from env, + keep browser projects, bump the global/test timeout for slow boot. +12. Update `example.spec.ts` to hit the local `baseURL` (kahuna) instead of `playwright.dev`. +13. Update `.github/workflows/playwright.yml`: ensure Docker is available and the + `grid-all` image is present before tests (build step out of scope — assumed pre-built). + +## Relevant files +- `e2e-tests/global-setup.ts` (new) — network, infra, provisioning, config gen, app container +- `e2e-tests/global-teardown.ts` (new) — teardown +- `e2e-tests/.tmp-config/` (generated) — mounted at `/etc/grid` +- `e2e-tests/playwright.config.ts` — `globalSetup`/`globalTeardown`, `baseURL`, timeout +- `e2e-tests/package.json` — deps + scripts +- `e2e-tests/images/entrypoint.sh` — reference for `GRID_SERVICES`/ports +- `dev/cloudformation/grid-dev-core.yml` — resources to provision +- `dev/script/setup.sh` — reference for provisioning + seeding flow +- `dev/script/generate-config/service-config.js` — **reused** for config generation +- `dev/script/generate-config/config.json5` — default config values consumed by reuse +- `dev/config/*` — seed data (photographers, quotas, usage rights, usages.eml) + +## Verification +1. `cd e2e-tests && npm install` succeeds with new deps. +2. `npx playwright test` starts ES + LocalStack, provisions the CFN stack, seeds buckets, + generates config via `service-config.js`, boots grid-all, kahuna responds on 9005, + and the smoke test passes. +3. Teardown removes all containers/network and `.tmp-config` (verify with `docker ps`). +4. Re-run to confirm no leaked containers/networks between runs. + +## Notes +- Reusing `service-config.js` keeps parity with dev. Watch for: it's CommonJS, relies on + `config.json5` (needs `json5`) and `DOMAIN`/env inputs; the `aws.local.endpoint` and + `es6.url` values are Guardian-domain/localhost-oriented and must be post-processed to + the Testcontainers network aliases. diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts new file mode 100644 index 00000000000..50ec63526b4 --- /dev/null +++ b/e2e-tests/playwright.config.ts @@ -0,0 +1,83 @@ +import { defineConfig, devices } from '@playwright/test'; +import { KAHUNA_PORT } from './testcontainers/constants'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// import dotenv from 'dotenv'; +// import path from 'path'; +// dotenv.config({ path: path.resolve(__dirname, '.env') }); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Boot the Grid stack with Testcontainers before tests, and tear it down after. */ + globalSetup: './global-setup.ts', + globalTeardown: './global-teardown.ts', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL (Kahuna) to use in actions like `await page.goto('/')`. Set by global-setup. */ + baseURL: process.env.GRID_BASE_URL ?? `http://localhost:${KAHUNA_PORT}`, + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + + { + name: 'firefox', + use: { ...devices['Desktop Firefox'] }, + }, + + { + name: 'webkit', + use: { ...devices['Desktop Safari'] }, + }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://localhost:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts new file mode 100644 index 00000000000..48a52825a27 --- /dev/null +++ b/e2e-tests/testcontainers/config.ts @@ -0,0 +1,91 @@ +/** + * Reuses the existing dev config generator (`dev/script/generate-config/service-config.js`) + * to produce per-service Grid config from live CloudFormation stack resources, then + * rewrites the LocalStack/Elasticsearch endpoints so that they resolve via the + * Testcontainers network aliases instead of the Guardian dev domains / localhost. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import JSON5 from 'json5'; +import { + DOMAIN, + EMAIL_DOMAIN, + ELASTICSEARCH_ALIAS, + LOCALSTACK_ALIAS, + LOCALSTACK_PORT, + REGION, + REPO_ROOT, +} from './constants'; + +const GENERATE_CONFIG_DIR = path.join(REPO_ROOT, 'dev', 'script', 'generate-config'); + +// The services packaged into the grid-all image (see e2e-tests/images/entrypoint.sh). +const GRID_SERVICES = [ + 'auth', + 'collections', + 'cropper', + 'kahuna', + 'leases', + 'media-api', + 'metadata-editor', + 'thrall', +]; + +type StackProps = Record; + +/** + * Rewrite the Guardian dev endpoints baked in by `service-config.js` so that the + * app container reaches the infrastructure containers over the shared network. + */ +function rewriteEndpoints(conf: string): string { + const localstackUrl = `http://${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`; + return conf + .split(`https://${LOCALSTACK_ALIAS}.media.${DOMAIN}`) + .join(localstackUrl) + .split('http://localhost:4576') + .join(localstackUrl) + .split('http://localhost:4566') + .join(localstackUrl); +} + +/** + * Generate all service config files into `configDir`, reusing `service-config.js`. + */ +export function generateServiceConfig(configDir: string, coreStackProps: StackProps): void { + // NO_AUTH makes `getCommonConfig` emit the Local authentication/authorisation + // providers, so we don't need pan-domain / OIDC infrastructure. + process.env.NO_AUTH = 'true'; + + // eslint-disable-next-line @typescript-eslint/no-var-requires + const ServiceConfig = require(path.join(GENERATE_CONFIG_DIR, 'service-config.js')); + const defaultConfig = JSON5.parse( + fs.readFileSync(path.join(GENERATE_CONFIG_DIR, 'config.json5'), 'utf8'), + ); + + const config = { + ...defaultConfig, + DOMAIN, + EMAIL_DOMAIN, + AWS_DEFAULT_REGION: REGION, + coreStackProps, + es6: { + ...defaultConfig.es6, + url: `http://${ELASTICSEARCH_ALIAS}:9200`, + }, + }; + + const serviceConfigs: Record = ServiceConfig.getCoreConfigs(config); + + fs.mkdirSync(configDir, { recursive: true }); + + // Mark the stage as DEV so services load `~/.grid/.conf` (see GridConfigLoader). + fs.writeFileSync(path.join(configDir, 'stage'), 'DEV'); + + for (const service of GRID_SERVICES) { + const conf = serviceConfigs[service]; + if (!conf) { + throw new Error(`service-config.js did not produce config for '${service}'`); + } + fs.writeFileSync(path.join(configDir, `${service}.conf`), rewriteEndpoints(conf)); + } +} diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts new file mode 100644 index 00000000000..f992d485b17 --- /dev/null +++ b/e2e-tests/testcontainers/constants.ts @@ -0,0 +1,42 @@ +import * as path from 'path'; + +/** Repository root (two levels up from this file: e2e-tests/testcontainers/ -> repo). */ +export const REPO_ROOT = path.resolve(__dirname, '..', '..'); + +/** Values mirror dev/.env — only used to shape generated config, not real infra. */ +export const DOMAIN = process.env.GRID_DOMAIN ?? 'local.dev-gutools.co.uk'; +export const EMAIL_DOMAIN = 'guardian.co.uk'; +export const REGION = 'eu-west-1'; + +export const CORE_STACK_NAME = 'grid-dev-core'; + +/** Pre-built application image (see e2e-tests/images/Dockerfile). Assumed to exist. */ +export const GRID_IMAGE = process.env.GRID_IMAGE ?? 'grid-all'; +export const ELASTICSEARCH_IMAGE = 'docker.elastic.co/elasticsearch/elasticsearch:8.18.3'; +export const LOCALSTACK_IMAGE = 'localstack/localstack:4.5.0'; + +/** Network aliases the app container uses to reach the infrastructure containers. */ +export const ELASTICSEARCH_ALIAS = 'elasticsearch'; +export const LOCALSTACK_ALIAS = 'localstack'; +export const LOCALSTACK_PORT = 4566; + +/** service -> http port, from e2e-tests/images/entrypoint.sh. */ +export const SERVICE_PORTS: Record = { + 'media-api': 9001, + thrall: 9002, + kahuna: 9005, + cropper: 9006, + 'metadata-editor': 9007, + collections: 9010, + auth: 9011, + leases: 9012, +}; + +export const KAHUNA_PORT = SERVICE_PORTS.kahuna; +export const MEDIA_API_PORT = SERVICE_PORTS['media-api']; + +/** File (repo-relative to e2e-tests) where global-setup records the resolved service URLs. */ +export const URLS_FILE = path.join(__dirname, '..', '.grid-urls.json'); + +/** The API key value uploaded to the KeyBucket (dev/.env API_KEY). */ +export const API_KEY = 'dev-'; diff --git a/e2e-tests/testcontainers/provision.ts b/e2e-tests/testcontainers/provision.ts new file mode 100644 index 00000000000..54a4e1aed6b --- /dev/null +++ b/e2e-tests/testcontainers/provision.ts @@ -0,0 +1,93 @@ +/** + * Provisions the Grid core infrastructure inside LocalStack: applies the CloudFormation + * core stack, waits for completion, reads the created resource names, and seeds the + * buckets with the config files the services expect (mirroring dev/script/setup.sh). + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { + CloudFormationClient, + CreateStackCommand, + DescribeStackResourcesCommand, + waitUntilStackCreateComplete, +} from '@aws-sdk/client-cloudformation'; +import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { API_KEY, CORE_STACK_NAME, REGION, REPO_ROOT } from './constants'; + +const CREDENTIALS = { accessKeyId: 'test', secretAccessKey: 'test' }; + +type StackProps = Record; + +function clients(endpoint: string) { + const cfn = new CloudFormationClient({ endpoint, region: REGION, credentials: CREDENTIALS }); + const s3 = new S3Client({ + endpoint, + region: REGION, + credentials: CREDENTIALS, + forcePathStyle: true, + }); + return { cfn, s3 }; +} + +async function createCoreStack(cfn: CloudFormationClient): Promise { + const templateBody = fs.readFileSync( + path.join(REPO_ROOT, 'dev', 'cloudformation', 'grid-dev-core.yml'), + 'utf8', + ); + + await cfn.send(new CreateStackCommand({ StackName: CORE_STACK_NAME, TemplateBody: templateBody })); + await waitUntilStackCreateComplete( + { client: cfn, maxWaitTime: 180 }, + { StackName: CORE_STACK_NAME }, + ); + + const { StackResources = [] } = await cfn.send( + new DescribeStackResourcesCommand({ StackName: CORE_STACK_NAME }), + ); + + return Object.fromEntries( + StackResources.filter((r) => r.LogicalResourceId && r.PhysicalResourceId).map((r) => [ + r.LogicalResourceId as string, + r.PhysicalResourceId as string, + ]), + ); +} + +async function putObject( + s3: S3Client, + bucket: string, + key: string, + body: Buffer | string, +): Promise { + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })); +} + +async function seedBuckets(s3: S3Client, props: StackProps): Promise { + const devConfig = path.join(REPO_ROOT, 'dev', 'config'); + + // API key used by the machine authentication provider. + await putObject(s3, props.KeyBucket, API_KEY, 'DEV Key'); + + // Static config consumed by the services. + for (const file of ['photographers.json', 'rcs-quota.json', 'usage_rights.json']) { + await putObject(s3, props.ConfigBucket, file, fs.readFileSync(path.join(devConfig, file))); + } + + await putObject( + s3, + props.UsageMailBucket, + 'usages.eml', + fs.readFileSync(path.join(devConfig, 'usages.eml')), + ); +} + +/** + * Apply the core stack and seed its buckets. Returns the LogicalResourceId -> + * PhysicalResourceId map used to generate service config. + */ +export async function provisionCoreStack(localstackEndpoint: string): Promise { + const { cfn, s3 } = clients(localstackEndpoint); + const props = await createCoreStack(cfn); + await seedBuckets(s3, props); + return props; +} diff --git a/e2e-tests/testcontainers/state.ts b/e2e-tests/testcontainers/state.ts new file mode 100644 index 00000000000..86406d78716 --- /dev/null +++ b/e2e-tests/testcontainers/state.ts @@ -0,0 +1,28 @@ +/** + * Shared, process-scoped registry for the Testcontainers instances started by + * `global-setup.ts`. Playwright runs global setup and global teardown in the same + * Node process, so a module-level singleton is a reliable way to hand references + * from setup to teardown. + */ +import type { StartedTestContainer } from 'testcontainers'; +import type { StartedNetwork } from 'testcontainers'; + +export interface GridEnvironment { + network: StartedNetwork; + /** Every started container, in start order. Stopped in reverse on teardown. */ + containers: StartedTestContainer[]; + /** Absolute path to the generated config directory mounted into the app container. */ + configDir: string; + /** Base URL of the Kahuna UI, e.g. http://localhost:9005 */ + baseUrl: string; +} + +let environment: GridEnvironment | undefined; + +export function setEnvironment(env: GridEnvironment): void { + environment = env; +} + +export function getEnvironment(): GridEnvironment | undefined { + return environment; +} diff --git a/e2e-tests/tests/example.spec.ts b/e2e-tests/tests/example.spec.ts new file mode 100644 index 00000000000..27e8edddfb0 --- /dev/null +++ b/e2e-tests/tests/example.spec.ts @@ -0,0 +1,18 @@ +import { test, expect } from './fixtures'; + +/** + * Smoke tests against the Grid stack booted by Testcontainers in global-setup. + * `baseURL` points at the Kahuna service (see fixtures.ts / playwright.config.ts). + */ + +test('kahuna healthcheck responds OK', async ({ request }) => { + const response = await request.get('/management/healthcheck'); + expect(response.ok()).toBeTruthy(); +}); + +test('kahuna serves the application', async ({ request }) => { + // The root path may redirect to auth; any non-server-error response proves Kahuna + // is serving requests against the provisioned infrastructure. + const response = await request.get('/', { maxRedirects: 0 }); + expect(response.status()).toBeLessThan(500); +}); diff --git a/e2e-tests/tests/fixtures.ts b/e2e-tests/tests/fixtures.ts new file mode 100644 index 00000000000..41288ad6f7c --- /dev/null +++ b/e2e-tests/tests/fixtures.ts @@ -0,0 +1,27 @@ +import { test as base, expect } from '@playwright/test'; +import * as fs from 'fs'; +import { URLS_FILE } from '../testcontainers/constants'; + +interface GridUrls { + kahuna: string; + mediaApi: string; +} + +function readGridUrls(): GridUrls { + return JSON.parse(fs.readFileSync(URLS_FILE, 'utf8')) as GridUrls; +} + +/** + * Test fixture that exposes the Grid service URLs resolved by `global-setup.ts` + * (dynamic host ports) and points `baseURL` at Kahuna. + */ +export const test = base.extend<{ gridUrls: GridUrls }>({ + gridUrls: async ({}, use) => { + await use(readGridUrls()); + }, + baseURL: async ({}, use) => { + await use(readGridUrls().kahuna); + }, +}); + +export { expect }; From f99875ec99643cbaeb8c62d046c01c4b4af46397 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 17 Jul 2026 15:29:09 +0100 Subject: [PATCH 215/373] Add permissions fixture --- .../fixtures/permissions/permissions.json | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 e2e-tests/fixtures/permissions/permissions.json diff --git a/e2e-tests/fixtures/permissions/permissions.json b/e2e-tests/fixtures/permissions/permissions.json new file mode 100644 index 00000000000..8eaec7cfa97 --- /dev/null +++ b/e2e-tests/fixtures/permissions/permissions.json @@ -0,0 +1,50 @@ +[ + { + "permission": { + "name": "grid_access", + "app": "grid", + "defaultValue": true + }, + "overrides": [] + }, + { + "permission": { + "name": "edit_metadata", + "app": "grid", + "defaultValue": true + }, + "overrides": [] + }, + { + "permission": { + "name": "delete_image", + "app": "grid", + "defaultValue": true + }, + "overrides": [] + }, + { + "permission": { + "name": "delete_crops_or_usages", + "app": "grid", + "defaultValue": true + }, + "overrides": [] + }, + { + "permission": { + "name": "delete_crops", + "app": "grid", + "defaultValue": true + }, + "overrides": [] + }, + { + "permission": { + "name": "show_paid", + "app": "grid", + "defaultValue": true + }, + "overrides": [] + } +] From 6d2425d1616177ac35ce5505e617f7c790da2452 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 17 Jul 2026 16:18:43 +0100 Subject: [PATCH 216/373] Add permissions bucket on startup, and populate with permissions fixture --- dev/script/generate-config/service-config.js | 18 ++++++++++-- .../features/kahuna.feature.spec.js | 29 +++++++++++++++++++ .../features/media-api.feature.spec.js | 24 +++++++++++++++ e2e-tests/testcontainers/config.ts | 7 +++-- e2e-tests/testcontainers/constants.ts | 7 +++++ e2e-tests/testcontainers/provision.ts | 28 ++++++++++++++++-- 6 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 e2e-tests/.features-gen/features/kahuna.feature.spec.js create mode 100644 e2e-tests/.features-gen/features/media-api.feature.spec.js diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js index bfbdff1035a..6b73c78cf14 100644 --- a/dev/script/generate-config/service-config.js +++ b/dev/script/generate-config/service-config.js @@ -10,11 +10,23 @@ function getCorsAllowedOriginString(config) { } function getCommonConfig(config) { + // `NO_AUTH` is a backwards-compatible shorthand that disables both authentication and + // authorisation. The two can also be controlled independently via `NO_AUTHENTICATION` + // and `NO_AUTHORISATION` (e.g. the e2e tests use local authentication but exercise the + // real S3-backed authorisation provider). const isNoAuth = process.env.NO_AUTH === "true"; // AI search depends on Bedrock/vector-ES infrastructure that isn't wired up for every org, // so it defaults to false (see CommonConfig.scala) and is only turned on here for Guardian's // own local dev environment. Shared by media-api and kahuna via the common config key. const aiSearchEnabled = process.env.BUILD_ORG ? false : true; + const isNoAuthentication = isNoAuth || process.env.NO_AUTHENTICATION === "true"; + const isNoAuthorisation = isNoAuth || process.env.NO_AUTHORISATION === "true"; + + // When the real authorisation provider is in use and a permissions bucket has been + // provisioned locally (localstack), point the provider at it and enable the local-auth + // code path so it reads `permissions.json` from that bucket instead of Guardian infra. + const useLocalPermissions = !isNoAuthorisation && Boolean(config.coreStackProps.PermissionsBucket); + return `domain.root="${config.DOMAIN}" |authentication.providers.machine.config.authKeyStoreBucket="${config.coreStackProps.KeyBucket}" |aws.local.endpoint="https://localstack.media.${config.DOMAIN}" @@ -27,8 +39,10 @@ function getCommonConfig(config) { |filters.shouldDisplayOrgOwnedCountAndFilterCheckbox=true |ai.search.enabled=${aiSearchEnabled} |dynamo.table.softDelete.metadata="SoftDeletedMetadataTable" - ${isNoAuth ? '|authentication.providers.user="com.gu.mediaservice.lib.auth.provider.LocalAuthenticationProvider"' : ''} - ${isNoAuth ? '|authorisation.provider="com.gu.mediaservice.lib.auth.provider.LocalAuthorisationProvider"' : ''} + ${isNoAuthentication ? '|authentication.providers.user="com.gu.mediaservice.lib.auth.provider.LocalAuthenticationProvider"' : ''} + ${isNoAuthorisation ? '|authorisation.provider="com.gu.mediaservice.lib.auth.provider.LocalAuthorisationProvider"' : ''} + ${useLocalPermissions ? '|auth.useLocal=true' : ''} + ${useLocalPermissions ? `|permissions.bucket="${config.coreStackProps.PermissionsBucket}"` : ''} |sqs.ingest.queue.url="${config.coreStackProps.IngestSqsQueue.replace("http://localhost:4576", `https://localstack.media.${config.DOMAIN}`)}" |s3.ingest.bucket="${config.coreStackProps.IngestQueueBucket}" |s3.fail.bucket="${config.coreStackProps.IngestQueueFailBucket}" diff --git a/e2e-tests/.features-gen/features/kahuna.feature.spec.js b/e2e-tests/.features-gen/features/kahuna.feature.spec.js new file mode 100644 index 00000000000..c560c8d25e2 --- /dev/null +++ b/e2e-tests/.features-gen/features/kahuna.feature.spec.js @@ -0,0 +1,29 @@ +// Generated from: features/kahuna.feature +import { test } from "../../steps/fixtures.ts"; + +test.describe('Kahuna service availability', () => { + + test('Healthcheck responds OK', async ({ When, Then, request, testContext }) => { + await When('I request the Kahuna healthcheck endpoint', null, { request, testContext }); + await Then('the response is successful', null, { testContext }); + }); + + test('Kahuna serves the application', async ({ When, Then, request, testContext }) => { + await When('I request the Kahuna root path without following redirects', null, { request, testContext }); + await Then('the response status is below 500', null, { testContext }); + }); + +}); + +// == technical section == + +test.use({ + $test: [({}, use) => use(test), { scope: 'test', box: true }], + $uri: [({}, use) => use('features/kahuna.feature'), { scope: 'test', box: true }], + $bddFileData: [({}, use) => use(bddFileData), { scope: "test", box: true }], +}); + +const bddFileData = [ // bdd-data-start + {"pwTestLine":6,"pickleLine":6,"tags":[],"steps":[{"pwStepLine":7,"gherkinStepLine":7,"keywordType":"Action","textWithKeyword":"When I request the Kahuna healthcheck endpoint","stepMatchArguments":[]},{"pwStepLine":8,"gherkinStepLine":8,"keywordType":"Outcome","textWithKeyword":"Then the response is successful","stepMatchArguments":[]}]}, + {"pwTestLine":11,"pickleLine":10,"tags":[],"steps":[{"pwStepLine":12,"gherkinStepLine":11,"keywordType":"Action","textWithKeyword":"When I request the Kahuna root path without following redirects","stepMatchArguments":[]},{"pwStepLine":13,"gherkinStepLine":12,"keywordType":"Outcome","textWithKeyword":"Then the response status is below 500","stepMatchArguments":[{"group":{"start":29,"value":"500"},"parameterTypeName":"int"}]}]}, +]; // bdd-data-end \ No newline at end of file diff --git a/e2e-tests/.features-gen/features/media-api.feature.spec.js b/e2e-tests/.features-gen/features/media-api.feature.spec.js new file mode 100644 index 00000000000..398c869f67d --- /dev/null +++ b/e2e-tests/.features-gen/features/media-api.feature.spec.js @@ -0,0 +1,24 @@ +// Generated from: features/media-api.feature +import { test } from "../../steps/fixtures.ts"; + +test.describe('Frontend loads media-api', () => { + + test('The SPA discovers and requests media-api from the grid-all container', async ({ When, Then, And, page, testContext }) => { + await When('I open the Grid application', null, { page, testContext }); + await Then('the page exposes a media-api URI link', null, { page }); + await And('the media-api is served successfully', null, { testContext }); + }); + +}); + +// == technical section == + +test.use({ + $test: [({}, use) => use(test), { scope: 'test', box: true }], + $uri: [({}, use) => use('features/media-api.feature'), { scope: 'test', box: true }], + $bddFileData: [({}, use) => use(bddFileData), { scope: "test", box: true }], +}); + +const bddFileData = [ // bdd-data-start + {"pwTestLine":6,"pickleLine":7,"tags":[],"steps":[{"pwStepLine":7,"gherkinStepLine":8,"keywordType":"Action","textWithKeyword":"When I open the Grid application","stepMatchArguments":[]},{"pwStepLine":8,"gherkinStepLine":9,"keywordType":"Outcome","textWithKeyword":"Then the page exposes a media-api URI link","stepMatchArguments":[]},{"pwStepLine":9,"gherkinStepLine":10,"keywordType":"Outcome","textWithKeyword":"And the media-api is served successfully","stepMatchArguments":[]}]}, +]; // bdd-data-end \ No newline at end of file diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 48a52825a27..11cb72101cc 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -52,9 +52,10 @@ function rewriteEndpoints(conf: string): string { * Generate all service config files into `configDir`, reusing `service-config.js`. */ export function generateServiceConfig(configDir: string, coreStackProps: StackProps): void { - // NO_AUTH makes `getCommonConfig` emit the Local authentication/authorisation - // providers, so we don't need pan-domain / OIDC infrastructure. - process.env.NO_AUTH = 'true'; + // NO_AUTHENTICATION makes `getCommonConfig` emit the Local authentication provider, so we + // don't need pan-domain / OIDC infrastructure. Authorisation is left as the real + // (S3-backed) provider, which reads `permissions.json` from the provisioned bucket. + process.env.NO_AUTHENTICATION = 'true'; // eslint-disable-next-line @typescript-eslint/no-var-requires const ServiceConfig = require(path.join(GENERATE_CONFIG_DIR, 'service-config.js')); diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index f992d485b17..f9db585566d 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -10,6 +10,13 @@ export const REGION = 'eu-west-1'; export const CORE_STACK_NAME = 'grid-dev-core'; +/** + * Bucket holding `permissions.json` for the real authorisation provider. Not part of the + * core CloudFormation stack, so it is created directly during provisioning. The name + * matches the code default in `PermissionsAuthorisationProvider`. + */ +export const PERMISSIONS_BUCKET = 'permissions-cache'; + /** Pre-built application image (see e2e-tests/images/Dockerfile). Assumed to exist. */ export const GRID_IMAGE = process.env.GRID_IMAGE ?? 'grid-all'; export const ELASTICSEARCH_IMAGE = 'docker.elastic.co/elasticsearch/elasticsearch:8.18.3'; diff --git a/e2e-tests/testcontainers/provision.ts b/e2e-tests/testcontainers/provision.ts index 54a4e1aed6b..b51bf05000d 100644 --- a/e2e-tests/testcontainers/provision.ts +++ b/e2e-tests/testcontainers/provision.ts @@ -11,8 +11,8 @@ import { DescribeStackResourcesCommand, waitUntilStackCreateComplete, } from '@aws-sdk/client-cloudformation'; -import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { API_KEY, CORE_STACK_NAME, REGION, REPO_ROOT } from './constants'; +import { CreateBucketCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { API_KEY, CORE_STACK_NAME, PERMISSIONS_BUCKET, REGION, REPO_ROOT } from './constants'; const CREDENTIALS = { accessKeyId: 'test', secretAccessKey: 'test' }; @@ -81,6 +81,29 @@ async function seedBuckets(s3: S3Client, props: StackProps): Promise { ); } +/** + * Create the permissions bucket (not part of the core stack) and seed it with the + * permissions fixture so the real authorisation provider can read `permissions.json`. + * Returns the bucket name so it can be added to the stack props map. + */ +async function provisionPermissionsBucket(s3: S3Client): Promise { + await s3.send( + new CreateBucketCommand({ + Bucket: PERMISSIONS_BUCKET, + CreateBucketConfiguration: { LocationConstraint: REGION }, + }), + ); + + await putObject( + s3, + PERMISSIONS_BUCKET, + 'permissions.json', + fs.readFileSync(path.join(REPO_ROOT, 'e2e-tests', 'fixtures', 'permissions', 'permissions.json')), + ); + + return PERMISSIONS_BUCKET; +} + /** * Apply the core stack and seed its buckets. Returns the LogicalResourceId -> * PhysicalResourceId map used to generate service config. @@ -89,5 +112,6 @@ export async function provisionCoreStack(localstackEndpoint: string): Promise Date: Wed, 22 Jul 2026 10:28:14 +0100 Subject: [PATCH 217/373] Add user permissions to permissions.json fixture working around an issue where the permissions code would read the permissions as empty --- .../fixtures/permissions/permissions.json | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/e2e-tests/fixtures/permissions/permissions.json b/e2e-tests/fixtures/permissions/permissions.json index 8eaec7cfa97..b38d0e5d3f9 100644 --- a/e2e-tests/fixtures/permissions/permissions.json +++ b/e2e-tests/fixtures/permissions/permissions.json @@ -5,7 +5,11 @@ "app": "grid", "defaultValue": true }, - "overrides": [] + "overrides": [{ + "userId": "composer.application@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] }, { "permission": { @@ -13,7 +17,11 @@ "app": "grid", "defaultValue": true }, - "overrides": [] + "overrides": [{ + "userId": "composer.application@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] }, { "permission": { @@ -21,7 +29,11 @@ "app": "grid", "defaultValue": true }, - "overrides": [] + "overrides": [{ + "userId": "composer.application@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] }, { "permission": { @@ -29,7 +41,11 @@ "app": "grid", "defaultValue": true }, - "overrides": [] + "overrides": [{ + "userId": "composer.application@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] }, { "permission": { @@ -37,7 +53,11 @@ "app": "grid", "defaultValue": true }, - "overrides": [] + "overrides": [{ + "userId": "composer.application@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] }, { "permission": { @@ -45,6 +65,10 @@ "app": "grid", "defaultValue": true }, - "overrides": [] + "overrides": [{ + "userId": "composer.application@guardian.co.uk", + "active": true, + "isCasualNotOnShift": false + }] } ] From b2c276a27cb821c19f0dc3dd965194a06cb06ac6 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 22 Jul 2026 10:28:25 +0100 Subject: [PATCH 218/373] Correct path in Dockerfile --- e2e-tests/images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 77fd6989929..0aa0bcded15 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -5,7 +5,7 @@ # thrall) together with Kahuna's compiled frontend. # # Build from the repository root: -# docker build -f images/Dockerfile -t grid-all . +# docker build -f e2e-tests/images/Dockerfile -t grid-all . # # Runtime configuration is NOT baked into the image. Mount per-environment # config (and a stage marker) into /etc/grid at run time, e.g.: From 467815d88b7eff6f767c0a405d431d3c4e63ea73 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 22 Jul 2026 16:31:58 +0100 Subject: [PATCH 219/373] Fix ports to enable us to resolve with dev-nginx locally, and via a reverse proxy in CI --- e2e-tests/global-setup.ts | 15 +++++++++--- e2e-tests/playwright.config.ts | 42 ++++----------------------------- e2e-tests/tests/example.spec.ts | 22 ++++++++++++++++- e2e-tests/tests/fixtures.ts | 19 ++++++++++++--- 4 files changed, 54 insertions(+), 44 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 64fa071d8f8..0986000b41e 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -27,6 +27,7 @@ import { LOCALSTACK_PORT, MEDIA_API_PORT, REGION, + SERVICE_PORTS, URLS_FILE, } from './testcontainers/constants'; import { generateServiceConfig } from './testcontainers/config'; @@ -61,6 +62,9 @@ async function globalSetup(): Promise { const localstack = await new LocalstackContainer(LOCALSTACK_IMAGE) .withNetwork(network) .withNetworkAliases(LOCALSTACK_ALIAS) + // Pin to the fixed host port dev-nginx expects for the S3 vanity domains + // (images.media / public.media / localstack.media -> 4566). + .withExposedPorts({ container: LOCALSTACK_PORT, host: LOCALSTACK_PORT }) .withEnvironment({ SERVICES: LOCALSTACK_SERVICES, DEFAULT_REGION: REGION, @@ -82,11 +86,16 @@ async function globalSetup(): Promise { // --- Application: the pre-built grid-all image --------------------------- // All eight services run inside this single container and talk to each other over - // its localhost, so only the ports the tests target are exposed (dynamically, to - // avoid clashing with any locally-running Grid). + // its localhost. Each is published on the *fixed* host port its dev-nginx mapping + // expects (dev/nginx-mappings.yml), so the developer's dev-nginx routes the + // https://*.media. domains straight into this container. This trades the + // previous dynamic-port isolation for compatibility with the existing nginx setup: + // it cannot run alongside a locally-running Grid that already owns these ports. let gridBuilder = new GenericContainer(GRID_IMAGE) .withNetwork(network) - .withExposedPorts(KAHUNA_PORT, MEDIA_API_PORT) + .withExposedPorts( + ...Object.values(SERVICE_PORTS).map((port) => ({ container: port, host: port })), + ) .withBindMounts([ // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. { source: configDir, target: '/root/.grid', mode: 'ro' }, diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index 50ec63526b4..a736a18a0c7 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -32,6 +32,11 @@ export default defineConfig({ /* Base URL (Kahuna) to use in actions like `await page.goto('/')`. Set by global-setup. */ baseURL: process.env.GRID_BASE_URL ?? `http://localhost:${KAHUNA_PORT}`, + /* The Grid service domains are served over https by the developer's dev-nginx, which + terminates TLS with a locally-generated (self-signed) certificate. Ignore cert + validation so browser tests can load the https://*.media. origins. */ + ignoreHTTPSErrors: true, + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ trace: 'on-first-retry', }, @@ -42,42 +47,5 @@ export default defineConfig({ name: 'chromium', use: { ...devices['Desktop Chrome'] }, }, - - { - name: 'firefox', - use: { ...devices['Desktop Firefox'] }, - }, - - { - name: 'webkit', - use: { ...devices['Desktop Safari'] }, - }, - - /* Test against mobile viewports. */ - // { - // name: 'Mobile Chrome', - // use: { ...devices['Pixel 5'] }, - // }, - // { - // name: 'Mobile Safari', - // use: { ...devices['iPhone 12'] }, - // }, - - /* Test against branded browsers. */ - // { - // name: 'Microsoft Edge', - // use: { ...devices['Desktop Edge'], channel: 'msedge' }, - // }, - // { - // name: 'Google Chrome', - // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, - // }, ], - - /* Run your local dev server before starting the tests */ - // webServer: { - // command: 'npm run start', - // url: 'http://localhost:3000', - // reuseExistingServer: !process.env.CI, - // }, }); diff --git a/e2e-tests/tests/example.spec.ts b/e2e-tests/tests/example.spec.ts index 27e8edddfb0..3966b095907 100644 --- a/e2e-tests/tests/example.spec.ts +++ b/e2e-tests/tests/example.spec.ts @@ -1,4 +1,4 @@ -import { test, expect } from './fixtures'; +import { test, expect, KAHUNA_APP_URL } from './fixtures'; /** * Smoke tests against the Grid stack booted by Testcontainers in global-setup. @@ -16,3 +16,23 @@ test('kahuna serves the application', async ({ request }) => { const response = await request.get('/', { maxRedirects: 0 }); expect(response.status()).toBeLessThan(500); }); + +test('frontend loads media-api from the grid-all container', async ({ page }) => { + // The SPA reads the media-api URL from and immediately + // requests the media-api root to discover its hypermedia links. dev-nginx routes + // https://api.media. to the media-api fixed port on the grid-all container, + // so this request should be served (200) by the container rather than failing. + const mediaApiResponse = page.waitForResponse( + (response) => + response.url().startsWith('https://api.media.') && response.request().method() === 'GET', + { timeout: 60_000 }, + ); + + await page.goto(KAHUNA_APP_URL); + + const mediaApiLink = await page.getAttribute('link[rel="media-api-uri"]', 'href'); + expect(mediaApiLink).toContain('api.media.'); + + const response = await mediaApiResponse; + expect(response.status()).toBe(200); +}); diff --git a/e2e-tests/tests/fixtures.ts b/e2e-tests/tests/fixtures.ts index 41288ad6f7c..12c22296cdc 100644 --- a/e2e-tests/tests/fixtures.ts +++ b/e2e-tests/tests/fixtures.ts @@ -1,19 +1,32 @@ import { test as base, expect } from '@playwright/test'; import * as fs from 'fs'; -import { URLS_FILE } from '../testcontainers/constants'; +import { DOMAIN, URLS_FILE } from '../testcontainers/constants'; interface GridUrls { + /** Kahuna base URL on its fixed host port (for API-level `request` tests). */ kahuna: string; + /** media-api base URL on its fixed host port (for API-level `request` tests). */ mediaApi: string; } +/** + * URL the browser loads the Kahuna SPA from. It is the real `https://media.` + * origin, served by the developer's dev-nginx, which routes it (and the other + * `https://*.media.` service domains the SPA follows via hypermedia links) to + * the grid-all container's fixed ports (see global-setup.ts / dev/nginx-mappings.yml). + * dev-nginx must be running with the media-service mappings for browser tests to work. + */ +export const KAHUNA_APP_URL = `https://media.${DOMAIN}`; + function readGridUrls(): GridUrls { return JSON.parse(fs.readFileSync(URLS_FILE, 'utf8')) as GridUrls; } /** - * Test fixture that exposes the Grid service URLs resolved by `global-setup.ts` - * (dynamic host ports) and points `baseURL` at Kahuna. + * Test fixture that exposes the Grid service URLs resolved by `global-setup.ts` and + * points `baseURL` at Kahuna's fixed host port for API-level `request` tests. Browser + * tests should navigate to `KAHUNA_APP_URL` so the page origin is a real Grid domain + * (required for the services' CORS origins to match) and dev-nginx does the routing. */ export const test = base.extend<{ gridUrls: GridUrls }>({ gridUrls: async ({}, use) => { From 0662ce7e16a3040b51c500f46a58288f1b2fd090 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 22 Jul 2026 16:37:04 +0100 Subject: [PATCH 220/373] Add a reverse proxy, similar to dev-nginx's setup, that ensures we resolve the service endpoints to local ports in CI --- e2e-tests/global-setup.ts | 67 +++++++++++++++++++++++++++ e2e-tests/testcontainers/constants.ts | 4 ++ 2 files changed, 71 insertions(+) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 0986000b41e..aba3a22c0db 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -18,14 +18,17 @@ import { LocalstackContainer } from '@testcontainers/localstack'; import { GenericContainer, Network, Wait } from 'testcontainers'; import type { StartedTestContainer } from 'testcontainers'; import { + DOMAIN, ELASTICSEARCH_ALIAS, ELASTICSEARCH_IMAGE, + GRID_ALIAS, GRID_IMAGE, KAHUNA_PORT, LOCALSTACK_ALIAS, LOCALSTACK_IMAGE, LOCALSTACK_PORT, MEDIA_API_PORT, + PROXY_IMAGE, REGION, SERVICE_PORTS, URLS_FILE, @@ -36,6 +39,51 @@ import { setEnvironment } from './testcontainers/state'; const LOCALSTACK_SERVICES = 'cloudformation,cloudwatch,dynamodb,kinesis,s3,sns,sqs,iam'; +/** + * Build a Caddyfile that reproduces the dev-nginx subdomain routing: each Grid service + * domain (https://.media.) reverse-proxies to the grid-all container on + * its in-container port, and the S3 vanity domains proxy to localstack (prepending the + * real bucket to the path). `tls internal` serves a self-signed cert per site; Playwright + * runs with `ignoreHTTPSErrors`, so the internal CA does not need to be trusted. + */ +function buildCaddyfile(coreStackProps: Record): string { + const appServices: Record = { + [`media.${DOMAIN}`]: SERVICE_PORTS.kahuna, + [`api.media.${DOMAIN}`]: SERVICE_PORTS['media-api'], + [`cropper.media.${DOMAIN}`]: SERVICE_PORTS.cropper, + [`thrall.media.${DOMAIN}`]: SERVICE_PORTS.thrall, + [`media-metadata.${DOMAIN}`]: SERVICE_PORTS['metadata-editor'], + [`media-collections.${DOMAIN}`]: SERVICE_PORTS.collections, + [`media-leases.${DOMAIN}`]: SERVICE_PORTS.leases, + [`media-auth.${DOMAIN}`]: SERVICE_PORTS.auth, + }; + + // S3 vanity domains that omit the bucket -> localstack, with the bucket prepended. + const imageBuckets: Record = { + [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket, + [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket, + }; + + const blocks: string[] = []; + + for (const [siteHost, port] of Object.entries(appServices)) { + blocks.push(`${siteHost} {\n\ttls internal\n\treverse_proxy ${GRID_ALIAS}:${port}\n}`); + } + + for (const [siteHost, bucket] of Object.entries(imageBuckets)) { + blocks.push( + `${siteHost} {\n\ttls internal\n\trewrite * /${bucket}{uri}\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, + ); + } + + // Thumbnails / direct S3 access already include the bucket in the path. + blocks.push( + `localstack.media.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, + ); + + return `${blocks.join('\n\n')}\n`; +} + async function globalSetup(): Promise { const started: StartedTestContainer[] = []; const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); @@ -93,6 +141,7 @@ async function globalSetup(): Promise { // it cannot run alongside a locally-running Grid that already owns these ports. let gridBuilder = new GenericContainer(GRID_IMAGE) .withNetwork(network) + .withNetworkAliases(GRID_ALIAS) .withExposedPorts( ...Object.values(SERVICE_PORTS).map((port) => ({ container: port, host: port })), ) @@ -126,6 +175,24 @@ async function globalSetup(): Promise { const grid = await gridBuilder.start(); started.push(grid); + // --- CI routing: bundled reverse proxy ----------------------------------- + // Locally, the browser reaches the https://*.media. domains via the developer's + // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) + // start a Caddy proxy that replays the same subdomain routing and terminates TLS with a + // self-signed cert, published on the standard https port the domains resolve to. + if (process.env.CI) { + const proxy = await new GenericContainer(PROXY_IMAGE) + .withNetwork(network) + .withExposedPorts({ container: 443, host: 443 }) + .withCopyContentToContainer([ + { content: buildCaddyfile(coreStackProps), target: '/etc/caddy/Caddyfile' }, + ]) + .withWaitStrategy(Wait.forListeningPorts()) + .withStartupTimeout(60_000) + .start(); + started.push(proxy); + } + const host = grid.getHost(); const baseUrl = `http://${host}:${grid.getMappedPort(KAHUNA_PORT)}`; const mediaApiUrl = `http://${host}:${grid.getMappedPort(MEDIA_API_PORT)}`; diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index f9db585566d..39d8d422305 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -21,11 +21,15 @@ export const PERMISSIONS_BUCKET = 'permissions-cache'; export const GRID_IMAGE = process.env.GRID_IMAGE ?? 'grid-all'; export const ELASTICSEARCH_IMAGE = 'docker.elastic.co/elasticsearch/elasticsearch:8.18.3'; export const LOCALSTACK_IMAGE = 'localstack/localstack:4.5.0'; +/** Reverse proxy used in CI to stand in for the developer's dev-nginx (see global-setup). */ +export const PROXY_IMAGE = 'caddy:2.8-alpine'; /** Network aliases the app container uses to reach the infrastructure containers. */ export const ELASTICSEARCH_ALIAS = 'elasticsearch'; export const LOCALSTACK_ALIAS = 'localstack'; export const LOCALSTACK_PORT = 4566; +/** Network alias the CI reverse proxy uses to reach the grid-all app container. */ +export const GRID_ALIAS = 'grid-all'; /** service -> http port, from e2e-tests/images/entrypoint.sh. */ export const SERVICE_PORTS: Record = { From 8a5c03c716fcef46e70df63cb8f4c5332eedfd1a Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 22 Jul 2026 16:37:29 +0100 Subject: [PATCH 221/373] Amend Playwright workflow --- .github/workflows/playwright.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index a5abfd5d1fb..88454b490ed 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -19,12 +19,16 @@ jobs: - name: Install dependencies run: npm ci - name: Install Playwright Browsers - run: npx playwright install --with-deps + # The suite runs Chromium only (see playwright.config.ts). + run: npx playwright install --with-deps chromium # The Testcontainers setup expects a pre-built `grid-all` image. Build it from # the repository root (the build context must be the repo root). - name: Build grid-all image working-directory: . run: DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile -t grid-all . + # GitHub sets CI=true, so global-setup starts the bundled Caddy reverse proxy on + # :443 to route the https://*.media. domains to the grid-all container + # (standing in for the developer's dev-nginx). No extra setup is required. - name: Run Playwright tests run: npx playwright test - uses: actions/upload-artifact@v4 From 56a129e4168afdd770401eee42472b6a028c2265 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 23 Jul 2026 10:18:25 +0100 Subject: [PATCH 222/373] Seed elasticsearch with images on container start --- e2e-tests/fixtures/elasticsearch/images.json | 2810 +++++++++++++++++ e2e-tests/global-setup.ts | 7 + .../testcontainers/seed-elasticsearch.ts | 111 + 3 files changed, 2928 insertions(+) create mode 100644 e2e-tests/fixtures/elasticsearch/images.json create mode 100644 e2e-tests/testcontainers/seed-elasticsearch.ts diff --git a/e2e-tests/fixtures/elasticsearch/images.json b/e2e-tests/fixtures/elasticsearch/images.json new file mode 100644 index 00000000000..242ed82c53f --- /dev/null +++ b/e2e-tests/fixtures/elasticsearch/images.json @@ -0,0 +1,2810 @@ +{ + "status": 200, + "body": { + "took": 7, + "timed_out": false, + "_shards": { + "total": 5, + "successful": 5, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 10000, + "relation": "gte" + }, + "max_score": 1, + "hits": [ + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "f6bfd2f0e54c232fd642237dfe773bb14275b3b3", + "_score": 1, + "_source": { + "id": "f6bfd2f0e54c232fd642237dfe773bb14275b3b3", + "uploadTime": "2020-03-04T10:41:37.204Z", + "uploadedBy": "stingray", + "lastModified": "2020-03-04T10:55:06.059Z", + "identifiers": {}, + "uploadInfo": { + "filename": "Swans_Nesting_004.JPG" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/f/6/b/f/d/2/f6bfd2f0e54c232fd642237dfe773bb14275b3b3", + "size": 587773, + "mimeType": "image/jpeg", + "dimensions": { + "width": 3156, + "height": 1728 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/f/6/b/f/d/2/f6bfd2f0e54c232fd642237dfe773bb14275b3b3", + "size": 14830, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 140 + } + }, + "fileMetadata": { + "iptc": { + "Copyright Notice": "© Richard Austin", + "Coded Character Set": "UTF-8", + "Application Record Version": "4", + "Caption/Abstract": "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \rThis week will see the staff putting out bundles of reeds for the swans to build their nests.\rThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\rThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \rAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \rThe colony can number over 600 swans with around 150 nesting pairs. \rWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\r© Richard Austin\rTel: 07831 566005", + "Credit": "Richard Austin", + "Source": "Richard Austin", + "Keywords": "richardaustinimages.com", + "By-line": "Richard Austin", + "Caption Writer/Editor": "Richard Austin", + "Date Created": "2020:03:04" + }, + "exif": { + "Image Description": "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \nThis week will see the staff putting out bundles of reeds for the swans to build their nests.\nThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\nThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \nAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \nThe colony can number over 600 swans with around 150 nesting pairs. \nWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\n© Richard Austin\nTel: 07831 566005", + "X Resolution": "350 dots per inch", + "Software": "Digital Photo Professional", + "Photometric Interpretation": "RGB", + "Make": "Canon", + "Image Width": "5184 pixels", + "YCbCr Positioning": "Center of pixel array", + "Samples Per Pixel": "3 samples/pixel", + "Copyright": "© Richard Austin", + "Date/Time": "2020:03:03 19:10:38", + "Model": "Canon EOS-1D X", + "Orientation": "Top, left side (Horizontal / normal)", + "Resolution Unit": "Inch", + "Bits Per Sample": "8 8 8 bits/component/pixel", + "Y Resolution": "350 dots per inch", + "Artist": "Richard Austin", + "Image Height": "3456 pixels" + }, + "exifSub": { + "Exif Version": "2.30", + "Exposure Mode": "Auto exposure", + "ISO Speed Ratings": "400", + "Lens Specification": "24-70mm", + "Body Serial Number": "063012002171", + "Custom Rendered": "Normal process", + "Exif Image Height": "1728 pixels", + "Lens Serial Number": "5903000717", + "Flash": "Flash did not fire", + "Focal Length": "61 mm", + "Date/Time Original": "2020:03:03 14:54:13", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/789 sec", + "Exif Image Width": "3156 pixels", + "Focal Plane Y Resolution": "49/172800 inches", + "Sub-Sec Time Original": "82", + "Exposure Time": "1/800 sec", + "Metering Mode": "Multi-segment", + "Exposure Program": "Shutter priority", + "Exposure Bias Value": "2/3 EV", + "F-Number": "f/10.0", + "Color Space": "sRGB", + "FlashPix Version": "1.00", + "Components Configuration": "YCbCr", + "Lens Model": "EF24-70mm f/4L IS USM", + "Date/Time Digitized": "2016:07:28 15:57:54", + "Recommended Exposure Index": "400", + "Date/Time Original Composite": "2020-03-03T14:54:13.820Z", + "Aperture Value": "f/9.9", + "Sensitivity Type": "Recommended Exposure Index", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "Inches", + "Sub-Sec Time Digitized": "76", + "Focal Plane X Resolution": "731/2592000 inches", + "Max Aperture Value": "f/4.0" + }, + "xmp": { + "dc:format": "image/jpeg", + "dc:description": [ + "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \nThis week will see the staff putting out bundles of reeds for the swans to build their nests.\nThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\nThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \nAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \nThe colony can number over 600 swans with around 150 nesting pairs. \nWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\n© Richard Austin\nTel: 07831 566005", + [ + "{'xml:lang':'x-default'}" + ] + ], + "xmp:MetadataDate": "2020-03-04T10:04:59.000Z", + "aux:LensSerialNumber": "5903000717", + "aux:ImageNumber": "0", + "dc:subject": [ + "richardaustinimages.com" + ], + "photoshop:CaptionWriter": "Richard Austin", + "aux:FlashCompensation": "0/1", + "aux:Lens": "EF24-70mm f/4L IS USM", + "xmp:ModifyDate": "2020-03-03T19:10:38.000Z", + "xmpMM:OriginalDocumentID": "B5CA1C6D9E598ABFD1488E67E2EC3F78", + "xmp:Rating": "0", + "photoshop:DateCreated": "2020-03-04T00:00:00.000Z", + "photoshop:Credit": "Richard Austin", + "xmp:CreateDate": "2016-07-28T15:57:54.760Z", + "aux:LensID": "504", + "aux:ApproximateFocusDistance": "595/100", + "dc:rights": [ + "© Richard Austin", + [ + "{'xml:lang':'x-default'}" + ] + ], + "xmp:CreatorTool": "Digital Photo Professional", + "aux:LensInfo": "24/1 70/1 0/0 0/0", + "xmpMM:DocumentID": "B5CA1C6D9E598ABFD1488E67E2EC3F78", + "xmpRights:Marked": "True", + "dc:creator": [ + "Richard Austin" + ], + "Iptc4xmpCore:CreatorContactInfo": [ + "{'Iptc4xmpCore:CiUrlWork':'richardaustinimages.com'}", + "{'Iptc4xmpCore:CiEmailWork':'r_austin@btconnect.com'}", + "{'Iptc4xmpCore:CiTelWork':'07831 566005'}", + "{'Iptc4xmpCore:CiAdrCtry':'UK'}", + "{'Iptc4xmpCore:CiAdrPcode':'DT7 3PW'}" + ], + "aux:Firmware": "2.0.3", + "aux:SerialNumber": "063012002171", + "xmpMM:History": [ + [ + "{'stEvt:softwareAgent':'Adobe Photoshop CC 2015.5 (Macintosh)'}", + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-03-03T19:10:38Z'}", + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:e8c994fa-f18f-4333-815c-aa852b3beced'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 9.6'}", + "{'stEvt:instanceID':'xmp.iid:e9307528-6f0e-451c-830a-0580bd9c73a2'}", + "{'stEvt:changed':'/metadata'}", + "{'stEvt:when':'2020-03-03T21:20:25Z'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-03-04T10:04:59Z'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 9.6.1 (Macintosh)'}", + "{'stEvt:changed':'/metadata'}", + "{'stEvt:instanceID':'xmp.iid:f013e587-41b4-4496-a7c8-5be3c1495b5c'}" + ] + ], + "photoshop:Source": "Richard Austin", + "xmpMM:InstanceID": "xmp.iid:f013e587-41b4-4496-a7c8-5be3c1495b5c" + }, + "icc": { + "Chromatic Adaptation": "sf32 (0x73663332): 44 bytes", + "Profile Size": "3140", + "Media White Point": "(0.9642, 1, 0.8249)", + "Device model": "Z009", + "Red Colorant": "(0.436, 0.2224, 0.0139)", + "Blue TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Version": "2.4.0", + "Tag Count": "14", + "Profile Description": "sRGB v1.31 (Canon)", + "Green Colorant": "(0.3851, 0.7169, 0.0971)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "CANO", + "Device Mfg Description": "Canon Inc.", + "Green TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Red TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Blue Colorant": "(0.1431, 0.0606, 0.7139)", + "Color space": "RGB", + "Primary Platform": "Microsoft Corporation", + "Profile Connection Space": "XYZ", + "CMM Type": "UCCM", + "Class": "Display Device", + "Profile Date/Time": "2003:04:04 00:00:00", + "Device Model Description": "sRGB v1.31 (Canon)", + "Profile Copyright": "Copyright (c) 2003, Canon Inc. All rights reserved.", + "Technology": "CRT" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "photometricInterpretation": "RGB", + "bitsPerSample": "8" + } + }, + "metadata": { + "dateTaken": "2020-03-03T14:54:13.820Z", + "description": "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \nThis week will see the staff putting out bundles of reeds for the swans to build their nests.\nThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\nThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \nAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \nThe colony can number over 600 swans with around 150 nesting pairs. \nWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\n© Richard Austin\nTel: 07831 566005", + "credit": "Richard Austin", + "byline": "Richard Austin", + "copyright": "© Richard Austin", + "source": "Richard Austin", + "keywords": [ + "richardaustinimages.com" + ], + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2020-03-03T14:54:13.820Z", + "description": "Its that time of year again, early spring and the birds and wildlife in general are getting into the reproduction mood especially the swans at the Abbotsbury Swannery in Dorset. \nThis week will see the staff putting out bundles of reeds for the swans to build their nests.\nThe swan numbers are around 700 but will thin out a little as some swans nest elsewhere away from the colony.\nThe swans are looking in first class condition and the male swans (Cob) are becoming territorial leading to violent bouts of fighting for the females (Pen) and space around their chosen nesting site. \nAbbotsbury Swannery is the only managed colony of nesting mute swans in the world. It is situated near the village of Abbotsbury in Dorset, where the Fleet Lagoon is protected from the weather of Lyme Bay by Chesil Beach (The longest Peeble beach in the world). \nThe colony can number over 600 swans with around 150 nesting pairs. \nWritten records of the swannery's existence go back to 1393 but it probably existed well before that, and is believed to have been set up by Benedictine Monks in the eleventh century.\n© Richard Austin\nTel: 07831 566005", + "credit": "Richard Austin", + "byline": "Richard Austin", + "copyright": "© Richard Austin", + "source": "Richard Austin", + "keywords": [ + "richardaustinimages.com" + ], + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [ + { + "path": [ + "Daily Edit", + "Daily edit home" + ], + "pathId": "daily edit/daily edit home", + "description": "Daily edit home", + "actionData": { + "author": "paul.bellsham@guardian.co.uk", + "date": "2020-03-04T10:55:06.059+00:00" + } + } + ], + "userMetadataLastModified": "2020-03-04T10:55:06.059Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "aa22008a915bf22184136ac94a3553977a8921bc", + "_score": 1, + "_source": { + "id": "aa22008a915bf22184136ac94a3553977a8921bc", + "uploadTime": "2019-01-24T16:21:51.712Z", + "uploadedBy": "karin.andreasson@guardian.co.uk", + "lastModified": "2019-01-25T11:23:46.597Z", + "identifiers": {}, + "uploadInfo": { + "filename": "GettyImages-143581630.jpg" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/a/a/2/2/0/0/aa22008a915bf22184136ac94a3553977a8921bc", + "size": 5698095, + "mimeType": "image/jpeg", + "dimensions": { + "width": 5200, + "height": 3454 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/a/a/2/2/0/0/aa22008a915bf22184136ac94a3553977a8921bc", + "size": 10107, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 170 + } + }, + "fileMetadata": { + "iptc": { + "By-line Title": "Contributor", + "Country/Primary Location Name": "United Kingdom", + "Country/Primary Location Code": "GBR", + "Copyright Notice": "Gary Tack Moral Rights Asserted", + "Credit": "Getty Images", + "Source": "Oxford Scientific RM", + "City": "Preston", + "Keywords": "Blackbird,Songbird,Animal,", + "By-line": "Gary Tack", + "Special Instructions": "Not Released (NR)", + "Headline": "Blackbird in winter English garden", + "Province/State": "Lancashire", + "Object Name": "143581630", + "Date Created": "2012:02:15" + }, + "exif": { + "Date/Time Digitized": "2012-02-15T00:00:00.000Z", + "Copyright": "Gary Tack Moral Rights Asserted", + "Orientation": "Top, left side (Horizontal / normal)" + }, + "exifSub": {}, + "xmp": { + "GettyImagesGIFT:ImageRank": "3", + "photoshop:CopyrightFlag": "true", + "photoshop:Headline": "Blackbird in winter English garden", + "photoshop:Instructions": "Not Released (NR)", + "photoshop:AuthorsPosition": "Contributor", + "dc:subject": [ + "Blackbird", + "Songbird", + "Animal" + ], + "photoshop:URL": "http://www.gettyimages.com", + "photoshop:City": "Preston", + "photoshop:DateCreated": "2012-02-15T00:00:00.000Z", + "photoshop:Credit": "Getty Images", + "photoshop:State": "Lancashire", + "plus:LicsensorURL": "http://www.gettyimages.com", + "dc:Rights": "Gary Tack Moral Rights Asserted", + "dc:creator": [ + "Gary Tack" + ], + "dc:title": [ + "143581630", + [ + "{'xml:lang':'x-default'}" + ] + ], + "GettyImagesGIFT:Dlref": "WKdcV+rIBAMIAwr1jFqQgg==", + "Iptc4xmpCore:CountryCode": "GBR", + "photoshop:Country": "United Kingdom", + "photoshop:Source": "Oxford Scientific RM", + "GettyImagesGIFT:AssetID": "143581630" + }, + "icc": { + "Profile Size": "560", + "Media White Point": "(0.9505, 1, 1.0891)", + "Red Colorant": "(0.6097, 0.3111, 0.0195)", + "Blue TRC": "0.0085908", + "Version": "2.1.0", + "Tag Count": "10", + "Profile Description": "Adobe RGB (1998)", + "Green Colorant": "(0.2053, 0.6257, 0.0609)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "none", + "Green TRC": "0.0085908", + "Red TRC": "0.0085908", + "Blue Colorant": "(0.1492, 0.0632, 0.7446)", + "Color space": "RGB", + "Primary Platform": "Apple Computer, Inc.", + "Profile Connection Space": "XYZ", + "CMM Type": "ADBE", + "Rendering Intent": "Media-Relative Colorimetric", + "Class": "Display Device", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1999:06:03 00:00:00", + "Profile Copyright": "Copyright 1999 Adobe Systems Incorporated" + }, + "getty": { + "Image Rank": "3", + "Asset ID": "143581630" + }, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "GettyImages-143581630" + }, + "lastModified": "2019-01-25T11:23:46.597Z" + }, + "metadata": { + "dateTaken": "2012-02-15T00:00:00.000Z", + "description": "GettyImages-143581630", + "credit": "Getty Images", + "byline": "Gary Tack", + "bylineTitle": "Contributor", + "title": "Blackbird in winter English garden", + "copyright": "Gary Tack Moral Rights Asserted", + "suppliersReference": "143581630", + "source": "Oxford Scientific RM", + "specialInstructions": "Not Released (NR)", + "keywords": [ + "Blackbird", + "Songbird", + "Animal" + ], + "city": "Preston", + "state": "Lancashire", + "country": "United Kingdom", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2012-02-15T00:00:00.000Z", + "credit": "Getty Images", + "byline": "Gary Tack", + "bylineTitle": "Contributor", + "title": "Blackbird in winter English garden", + "copyright": "Gary Tack Moral Rights Asserted", + "suppliersReference": "143581630", + "source": "Oxford Scientific RM", + "specialInstructions": "Not Released (NR)", + "keywords": [ + "Blackbird", + "Songbird", + "Animal" + ], + "city": "Preston", + "state": "Lancashire", + "country": "United Kingdom", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": { + "category": "agency", + "supplier": "Getty Images", + "suppliersCollection": "Oxford Scientific RM" + }, + "originalUsageRights": { + "category": "agency", + "supplier": "Getty Images", + "suppliersCollection": "Oxford Scientific RM" + }, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2019-01-25T11:23:46.597Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "02aa01b0ad57aa2794a5765dab3010d53387e677", + "_score": 1, + "_source": { + "id": "02aa01b0ad57aa2794a5765dab3010d53387e677", + "uploadTime": "2020-03-04T07:23:14.490Z", + "uploadedBy": "stingray", + "lastModified": "2020-03-04T10:47:49.175Z", + "identifiers": {}, + "uploadInfo": { + "filename": "PHT_China_Medicinal_herb_Plant_5.JPG" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/0/2/a/a/0/1/02aa01b0ad57aa2794a5765dab3010d53387e677", + "size": 2609472, + "mimeType": "image/jpeg", + "dimensions": { + "width": 3000, + "height": 1902 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/0/2/a/a/0/1/02aa01b0ad57aa2794a5765dab3010d53387e677", + "size": 22684, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 162 + } + }, + "fileMetadata": { + "iptc": { + "Country/Primary Location Name": "CHINA", + "Category": "MAC", + "Country/Primary Location Code": "CHN", + "Copyright Notice": "Avalon.red. All rights reserved.", + "Supplemental Category(s)": "Macro economics", + "Application Record Version": "2", + "Caption/Abstract": "(200304) -- MIANXIAN, March 4, 2020 (Xinhua) -- Aerial photo taken on March 3, 2020 shows villagers planting Ophiopogon japonicus, a traditional Chinese medicinal (TCM) herb, in the fields of Chunfeng Village in Mianxian County, northwest China's Shaanxi Province.\r\n Since 2018, the Chunfeng Village of Mianxian County has tried planting Ophiopogon japonicus, a TCM herb newly introduced into the village, and made a success. The village then established professional planting and processing cooperatives to extend its production chain of the TCM herb. \r\n Up till now, a 400 mu (about 26.7 hectares) standardized Ophiopogon japonicus breeding and planting base has been set up here with an annual output value of more than 7.2 million yuan (about 1 million U.S. dollars), creating jobs for local villagers and at the same time helping them increase their incomes. (Xinhua/Tao Ming)", + "Credit": "Avalon.red", + "Source": "Avalon.red", + "City": "MIANXIAN", + "Keywords": "cn;PHT_WIRE;Sent to UK Press on 04.03.2020 at 07:21:40", + "By-line": "Tao Ming/Avalon.red", + "Urgency": "53", + "Special Instructions": "Supplied by AVALON - Fee Payable Upon Reproduction - For queries contact Avalon - sales@avalon.red London: +44 (0) 20 7421 6000 Los Angeles: +1 (310) 822 0419", + "Headline": "China Medicinal herb Planting", + "Date Created": "2020:03:03" + }, + "exif": {}, + "exifSub": {}, + "xmp": {}, + "icc": { + "Profile Size": "3144", + "Media White Point": "(0.9505, 1, 1.0891)", + "Device model": "sRGB", + "Red Colorant": "(0.4361, 0.2225, 0.0139)", + "Blue TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Viewing Conditions Description": "Reference Viewing Condition in IEC61966-2.1", + "Luminance": "(76.0365, 80, 87.1246)", + "Viewing Conditions": "view (0x76696577): 36 bytes", + "Version": "2.1.0", + "Tag Count": "17", + "Profile Description": "sRGB IEC61966-2.1", + "Green Colorant": "(0.3851, 0.7169, 0.0971)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "IEC", + "Device Mfg Description": "IEC http://www.iec.ch", + "Green TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Red TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Blue Colorant": "(0.1431, 0.0606, 0.7141)", + "Color space": "RGB", + "Primary Platform": "Microsoft Corporation", + "Profile Connection Space": "XYZ", + "CMM Type": "Lino", + "Class": "Display Device", + "Measurement": "1931 2° Observer, Backing (0, 0, 0), Geometry Unknown, Flare 1%, Illuminant D65", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1998:02:09 06:49:00", + "Device Model Description": "IEC 61966-2.1 Default RGB colour space - sRGB", + "Profile Copyright": "Copyright (c) 1998 Hewlett-Packard Company", + "Technology": "CRT" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "metadata": { + "description": "(200304) -- MIANXIAN, March 4, 2020 (Xinhua) -- Aerial photo taken on March 3, 2020 shows villagers planting Ophiopogon japonicus, a traditional Chinese medicinal (TCM) herb, in the fields of Chunfeng Village in Mianxian County, northwest China's Shaanxi Province.\r\n Since 2018, the Chunfeng Village of Mianxian County has tried planting Ophiopogon japonicus, a TCM herb newly introduced into the village, and made a success. The village then established professional planting and processing cooperatives to extend its production chain of the TCM herb. \r\n Up till now, a 400 mu (about 26.7 hectares) standardized Ophiopogon japonicus breeding and planting base has been set up here with an annual output value of more than 7.2 million yuan (about 1 million U.S. dollars), creating jobs for local villagers and at the same time helping them increase their incomes. (Xinhua/Tao Ming)", + "credit": "Avalon.red", + "byline": "Tao Ming", + "title": "China Medicinal herb Planting", + "copyright": "Avalon.red. All rights reserved.", + "source": "Avalon.red", + "specialInstructions": "Supplied by AVALON - Fee Payable Upon Reproduction - For queries contact Avalon - sales@avalon.red London: +44 (0) 20 7421 6000 Los Angeles: +1 (310) 822 0419", + "keywords": [ + "cn", + "PHT_WIRE", + "Sent to UK Press on 04.03.2020 at 07:21:40" + ], + "city": "Mianxian", + "country": "China", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "description": "(200304) -- MIANXIAN, March 4, 2020 (Xinhua) -- Aerial photo taken on March 3, 2020 shows villagers planting Ophiopogon japonicus, a traditional Chinese medicinal (TCM) herb, in the fields of Chunfeng Village in Mianxian County, northwest China's Shaanxi Province.\r\n Since 2018, the Chunfeng Village of Mianxian County has tried planting Ophiopogon japonicus, a TCM herb newly introduced into the village, and made a success. The village then established professional planting and processing cooperatives to extend its production chain of the TCM herb. \r\n Up till now, a 400 mu (about 26.7 hectares) standardized Ophiopogon japonicus breeding and planting base has been set up here with an annual output value of more than 7.2 million yuan (about 1 million U.S. dollars), creating jobs for local villagers and at the same time helping them increase their incomes. (Xinhua/Tao Ming)", + "credit": "Avalon.red", + "byline": "Tao Ming", + "title": "China Medicinal herb Planting", + "copyright": "Avalon.red. All rights reserved.", + "source": "Avalon.red", + "specialInstructions": "Supplied by AVALON - Fee Payable Upon Reproduction - For queries contact Avalon - sales@avalon.red London: +44 (0) 20 7421 6000 Los Angeles: +1 (310) 822 0419", + "keywords": [ + "cn", + "PHT_WIRE", + "Sent to UK Press on 04.03.2020 at 07:21:40" + ], + "city": "Mianxian", + "country": "China", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [ + { + "path": [ + "Daily Edit", + "Daily edit foreign" + ], + "pathId": "daily edit/daily edit foreign", + "description": "Daily edit foreign", + "actionData": { + "author": "paul.bellsham@guardian.co.uk", + "date": "2020-03-04T10:47:49.175+00:00" + } + } + ], + "userMetadataLastModified": "2020-03-04T10:47:49.175Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "dafbac97097c9438bd3cbad8d7a04d2eaadee25e", + "_score": 1, + "_source": { + "id": "dafbac97097c9438bd3cbad8d7a04d2eaadee25e", + "uploadTime": "2015-06-17T12:59:31.000Z", + "uploadedBy": "Picdar Export", + "lastModified": "2015-06-17T23:00:00.000Z", + "identifiers": { + "picdarurn": "GD*53549404" + }, + "uploadInfo": {}, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/d/a/f/b/a/c/dafbac97097c9438bd3cbad8d7a04d2eaadee25e", + "size": 808161, + "mimeType": "image/jpeg", + "dimensions": { + "width": 1661, + "height": 2403 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/d/a/f/b/a/c/dafbac97097c9438bd3cbad8d7a04d2eaadee25e", + "size": 13890, + "mimeType": "image/jpeg", + "dimensions": { + "width": 177, + "height": 256 + } + }, + "fileMetadata": { + "iptc": {}, + "exif": { + "Subject Distance Range": "Macro", + "Rating": "0", + "Exposure Mode": "Auto exposure", + "Custom Rendered": "Normal process", + "Make": "EASTMAN KODAK COMPANY", + "White Balance Mode": "Manual white balance", + "YCbCr Positioning": "Center of pixel array", + "Gain Control": "Low gain down", + "Date/Time": "2011:01:07 12:03:59", + "Model": "KODAK EASYSHARE Z915 DIGITAL CAMERA", + "Sharpness": "None", + "Orientation": "Top, left side (Horizontal / normal)", + "Digital Zoom Ratio": "Digital zoom not used", + "Resolution Unit": "Inch", + "Focal Length 35": "35 mm", + "Contrast": "None", + "Scene Capture Type": "Standard", + "Saturation": "None" + }, + "exifSub": { + "Exif Version": "2.21", + "ISO Speed Ratings": "400", + "Exif Image Height": "2736 pixels", + "Flash": "Flash did not fire", + "Related Sound File": "RelatedSound", + "White Balance": "Daylight", + "Focal Length": "6.2 mm", + "Date/Time Original": "2011:01:07 12:04:00", + "Shutter Speed Value": "1/20 sec", + "Exif Image Width": "3648 pixels", + "Sensing Method": "One-chip color area sensor", + "Scene Type": "Directly photographed image", + "Sub-Sec Time Original": "59", + "Brightness Value": "1.0", + "Makernote": "[40960 values]", + "Exposure Time": "0.05 sec", + "Metering Mode": "Center weighted average", + "Exposure Program": "Program normal", + "Exposure Bias Value": "0 EV", + "F-Number": "f/3.5", + "Color Space": "sRGB", + "FlashPix Version": "1.00", + "Exposure Index": "400", + "Components Configuration": "YCbCr", + "Date/Time Digitized": "2011:01:07 12:04:00", + "Date/Time Original Composite": "2011-01-07T12:04:00.590Z", + "Aperture Value": "f/3.4", + "File Source": "Digital Still Camera (DSC)", + "Max Aperture Value": "f/3.5" + }, + "xmp": {}, + "icc": {}, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "Daphne Du Maurier - Rebecca" + } + }, + "metadata": { + "dateTaken": "2011-01-07T12:04:00.590Z", + "description": "Daphne Du Maurier - Rebecca", + "keywords": [], + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2011-01-07T12:04:00.590Z", + "keywords": [], + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [ + { + "id": "print/b01ff63186ac8b771d8d589227c5a3c9_140a9429c07583239faf0c7e4b1e256d", + "references": [ + { + "type": "indesign", + "name": "2015-06-17, The Guardian, g2f, Page 2" + } + ], + "platform": "print", + "media": "image", + "status": "published", + "dateAdded": "2015-06-17T23:00:00.000Z", + "lastModified": "2015-06-17T23:00:00.000Z", + "printUsageMetadata": { + "sectionName": "g2f", + "issueDate": "2015-06-17T23:00:00.000Z", + "pageNumber": 2, + "storyName": "scMWkatemo_02_g2f_0618_002", + "publicationCode": "gdn", + "publicationName": "The Guardian", + "edition": 1, + "sectionCode": "g2f", + "notes": "Andrew Stocks", + "source": "picdar" + } + } + ], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2015-06-17T23:00:00.000Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "ad61e5bee9ceb7e47350d1380ee6c6d7500ebe52", + "_score": 1, + "_source": { + "id": "ad61e5bee9ceb7e47350d1380ee6c6d7500ebe52", + "uploadTime": "2015-07-28T15:10:22.000Z", + "uploadedBy": "DEV Ingest Script", + "lastModified": "2014-01-07T05:01:26.000Z", + "identifiers": { + "picdarurn": "GD*40548955" + }, + "uploadInfo": {}, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/a/d/6/1/e/5/ad61e5bee9ceb7e47350d1380ee6c6d7500ebe52", + "size": 1653855, + "mimeType": "image/jpeg", + "dimensions": { + "width": 4928, + "height": 3280 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/a/d/6/1/e/5/ad61e5bee9ceb7e47350d1380ee6c6d7500ebe52", + "size": 41245, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 170 + } + }, + "fileMetadata": { + "iptc": { + "Country/Primary Location Name": "England", + "Copyright Notice": "Lucy Young", + "Application Record Version": "3", + "Caption/Abstract": "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges Ð including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\rBy Lucy Young \r07799118984\rlucyyounguk@gmail.com\r\rwww.lucyyoungphotos.co.uk", + "Credit": "Lucy Young", + "City": "London", + "Time Created": "12:43:54+0000", + "By-line": "Lucy Young", + "Special Instructions": "From: Lucy Young ", + "Headline": "A Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel.", + "Date Time Created Composite": "2014-01-02T12:43:54.000Z", + "Date Created": "2014:01:02" + }, + "exif": { + "X Resolution": "300 dots per inch", + "Software": "Adobe Photoshop CS6 (Macintosh)", + "Photometric Interpretation": "RGB", + "Make": "NIKON CORPORATION", + "Image Width": "4928 pixels", + "Samples Per Pixel": "3 samples/pixel", + "Copyright": "Lucy Young", + "Date/Time": "2014:01:02 20:18:33", + "Model": "NIKON D4", + "Orientation": "Top, left side (Horizontal / normal)", + "Resolution Unit": "Inch", + "Bits Per Sample": "8 8 8 bits/component/pixel", + "Y Resolution": "300 dots per inch", + "Artist": "Lucy Young", + "Image Height": "3280 pixels" + }, + "exifSub": { + "CFA Pattern": "[Red,Green][Green,Blue]", + "Exif Version": "2.21", + "Subject Distance Range": "Unknown", + "Exposure Mode": "Auto exposure", + "ISO Speed Ratings": "400", + "Lens Specification": "14-24mm f/2.8", + "Body Serial Number": "2055843", + "Custom Rendered": "Normal process", + "Exif Image Height": "3280 pixels", + "Flash": "Flash did not fire", + "White Balance": "Unknown", + "Focal Length": "14 mm", + "Date/Time Original": "2014:01:02 12:43:54", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/499 sec", + "Exif Image Width": "4928 pixels", + "Gain Control": "Low gain up", + "Sensing Method": "One-chip color area sensor", + "Scene Type": "Directly photographed image", + "Focal Plane Y Resolution": "32768/4485575 unknown (4)", + "Sub-Sec Time Original": "10", + "Exposure Time": "1/500 sec", + "Sharpness": "None", + "Metering Mode": "Multi-segment", + "Exposure Program": "Shutter priority", + "Digital Zoom Ratio": "1", + "Exposure Bias Value": "0 EV", + "F-Number": "f/9.0", + "Color Space": "Undefined", + "Sub-Sec Time": "10", + "Lens Model": "14.0-24.0 mm f/2.8", + "Focal Length 35": "14 mm", + "Date/Time Digitized": "2014:01:02 12:43:54", + "Date/Time Original Composite": "2014-01-02T12:43:54.100Z", + "Aperture Value": "f/9.0", + "Sensitivity Type": "Recommended Exposure Index", + "Contrast": "None", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "Unknown (4)", + "File Source": "Digital Still Camera (DSC)", + "Sub-Sec Time Digitized": "10", + "Focal Plane X Resolution": "32768/4485575 unknown (4)", + "Saturation": "None", + "Max Aperture Value": "f/2.8" + }, + "xmp": { + "crs:ParametricDarks": "0", + "dc:format": "image/jpeg", + "crs:DefringeGreenHueLo": "40", + "crs:SaturationAdjustmentPurple": "0", + "crs:HasSettings": "True", + "crs:GreenHue": "0", + "photoshop:ColorMode": "3", + "dc:description": [ + "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges – including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\nBy Lucy Young \n07799118984\nlucyyounguk@gmail.com\n\nwww.lucyyoungphotos.co.uk", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:SaturationAdjustmentAqua": "0", + "crs:UprightVersion": "134217728", + "crs:SharpenDetail": "25", + "crs:GrainAmount": "0", + "xmp:MetadataDate": "2014-01-02T20:18:33.000Z", + "crs:CameraProfile": "Adobe Standard", + "crs:WhiteBalance": "As Shot", + "crs:ColorNoiseReductionDetail": "50", + "crs:Blacks2012": "0", + "crs:SaturationAdjustmentBlue": "0", + "xmpMM:DerivedFrom": [ + "{'stRef:instanceID':'xmp.iid:F63F7CEC112068118083C60981F28C85'}", + "{'stRef:documentID':'xmp.did:bfa36716-faee-4eb2-9e22-02910f56286a'}", + "{'stRef:originalDocumentID':'4CC35D0B77263D9E2D74FAAEEEB7E101'}" + ], + "crs:UprightPreview": "False", + "aux:ImageNumber": "19408", + "crs:SplitToningShadowSaturation": "0", + "crs:SplitToningShadowHue": "0", + "photoshop:Headline": "A Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel.", + "crs:ParametricHighlightSplit": "75", + "crs:PerspectiveRotate": "0.0", + "crs:PerspectiveUpright": "0", + "photoshop:ICCProfile": "Adobe RGB (1998)", + "crs:DefringePurpleHueLo": "30", + "crs:ToneCurveName2012": "Linear", + "crs:ToneCurvePV2012": [ + "0, 0", + "255, 255" + ], + "crs:HueAdjustmentPurple": "0", + "crs:VignetteAmount": "0", + "crs:Exposure2012": "-0.25", + "crs:ParametricShadowSplit": "25", + "crs:LuminanceAdjustmentAqua": "0", + "crs:LuminanceAdjustmentPurple": "0", + "photomechanic:PMVersion": "PM5", + "crs:SplitToningBalance": "0", + "crs:SaturationAdjustmentMagenta": "0", + "crs:Sharpness": "25", + "crs:HueAdjustmentAqua": "0", + "crs:BlueHue": "0", + "crs:DefringeGreenHueHi": "60", + "crs:SaturationAdjustmentOrange": "0", + "crs:HueAdjustmentYellow": "0", + "crs:SplitToningHighlightHue": "0", + "crs:PerspectiveScale": "100", + "crs:HueAdjustmentGreen": "0", + "photomechanic:Prefs": "0:3:0:019408", + "crs:Temperature": "5050", + "crs:ConvertToGrayscale": "False", + "crs:RawFileName": "DSC_9706.NEF", + "crs:ParametricLights": "0", + "aux:Lens": "14.0-24.0 mm f/2.8", + "crs:UprightFocalMode": "0", + "photomechanic:Tagged": "False", + "crs:ToneCurvePV2012Green": [ + "0, 0", + "255, 255" + ], + "crs:ParametricHighlights": "0", + "crs:SharpenEdgeMasking": "0", + "crs:SaturationAdjustmentRed": "0", + "crs:Whites2012": "0", + "crs:Clarity2012": "+14", + "crs:UprightFocalLength35mm": "35", + "xmp:ModifyDate": "2014-01-02T20:18:33.000Z", + "crs:UprightTransformCount": "0", + "crs:UprightCenterNormX": "0.5", + "photoshop:City": "London", + "xmpMM:OriginalDocumentID": "4CC35D0B77263D9E2D74FAAEEEB7E101", + "xmp:Rating": "0", + "crs:ToneCurvePV2012Blue": [ + "0, 0", + "255, 255" + ], + "crs:LuminanceAdjustmentRed": "0", + "crs:DefringeGreenAmount": "0", + "crs:PerspectiveVertical": "0", + "crs:HueAdjustmentBlue": "0", + "crs:LensManualDistortionAmount": "0", + "crs:AutoLateralCA": "0", + "crs:LuminanceSmoothing": "0", + "crs:CameraProfileDigest": "939B4E1427152B8FC65E6BBE46CF1813", + "crs:Contrast2012": "0", + "photoshop:DateCreated": "2014-01-02T12:43:54.000Z", + "photoshop:Credit": "Lucy Young", + "crs:SaturationAdjustmentYellow": "0", + "xmp:CreateDate": "2014-01-02T12:43:54.000Z", + "crs:LensProfileSetup": "LensDefaults", + "crs:Shadows2012": "0", + "crs:UprightCenterMode": "0", + "crs:LensProfileEnable": "0", + "aux:LensID": "146", + "aux:ApproximateFocusDistance": "4294967295/1", + "crs:Tint": "-1", + "crs:ParametricMidtoneSplit": "50", + "crs:PerspectiveHorizontal": "0", + "crs:ShadowTint": "0", + "crs:UprightCenterNormY": "0.5", + "dc:rights": [ + "Lucy Young", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:HueAdjustmentMagenta": "0", + "xmp:CreatorTool": "Adobe Photoshop CS6 (Macintosh)", + "aux:LensInfo": "140/10 240/10 28/10 28/10", + "crs:LuminanceAdjustmentMagenta": "0", + "photoshop:LegacyIPTCDigest": "F9B58E32E365BE2D442491E7BA5AEE2F", + "photomechanic:ColorClass": "3", + "crs:BlueSaturation": "0", + "crs:LuminanceAdjustmentYellow": "0", + "crs:AlreadyApplied": "True", + "crs:DefringePurpleHueHi": "70", + "crs:PostCropVignetteAmount": "0", + "xmp:Label": "Superior", + "xmpMM:DocumentID": "xmp.did:bfa36716-faee-4eb2-9e22-02910f56286a", + "crs:PerspectiveAspect": "0", + "crs:LuminanceAdjustmentGreen": "0", + "crs:RedSaturation": "0", + "xmpRights:Marked": "True", + "crs:SaturationAdjustmentGreen": "0", + "dc:creator": [ + "Lucy Young" + ], + "crs:LuminanceAdjustmentOrange": "0", + "crs:Version": "8.1", + "crs:ToneCurvePV2012Red": [ + "0, 0", + "255, 255" + ], + "Iptc4xmpCore:CreatorContactInfo": [ + "{'Iptc4xmpCore:CiUrlWork':'www.lucyyoungphotos.co.uk'}", + "{'Iptc4xmpCore:CiEmailWork':'lucyyounguk@gmail.com'}", + "{'Iptc4xmpCore:CiAdrCtry':'England'}", + "{'Iptc4xmpCore:CiTelWork':'+44 (0)7799118984'}" + ], + "crs:ParametricShadows": "0", + "crs:HueAdjustmentRed": "0", + "crs:GreenSaturation": "0", + "crs:RedHue": "0", + "crs:SplitToningHighlightSaturation": "0", + "aux:SerialNumber": "2055843", + "crs:DefringePurpleAmount": "0", + "crs:Saturation": "+17", + "crs:HueAdjustmentOrange": "0", + "crs:LuminanceAdjustmentBlue": "0", + "xmpMM:History": [ + [ + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 8.1 (Macintosh)'}", + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2014-01-02T15:37:39Z'}", + "{'stEvt:changed':'/metadata'}", + "{'stEvt:instanceID':'xmp.iid:8f376438-551e-40c7-8867-759ff1c1e0c8'}" + ], + [ + "{'stEvt:parameters':'converted from image/x-nikon-nef to image/tiff'}", + "{'stEvt:action':'derived'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2014-01-02T15:39:41Z'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 8.1 (Macintosh)'}", + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:bfa36716-faee-4eb2-9e22-02910f56286a'}" + ], + [ + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:F63F7CEC112068118083C60981F28C85'}", + "{'stEvt:when':'2014-01-02T15:41:22Z'}", + "{'stEvt:softwareAgent':'Adobe Photoshop CS6 (Macintosh)'}", + "{'stEvt:action':'saved'}" + ], + [ + "{'stEvt:parameters':'from image/tiff to image/jpeg'}", + "{'stEvt:action':'converted'}" + ], + [ + "{'stEvt:action':'derived'}", + "{'stEvt:parameters':'converted from image/tiff to image/jpeg'}" + ], + [ + "{'stEvt:instanceID':'xmp.iid:F73F7CEC112068118083C60981F28C85'}", + "{'stEvt:action':'saved'}", + "{'stEvt:changed':'/'}", + "{'stEvt:softwareAgent':'Adobe Photoshop CS6 (Macintosh)'}", + "{'stEvt:when':'2014-01-02T15:41:22Z'}" + ], + [ + "{'stEvt:when':'2014-01-02T20:18:33Z'}", + "{'stEvt:instanceID':'xmp.iid:1D7ECFD70A2068118C1496C090CBA674'}", + "{'stEvt:changed':'/'}", + "{'stEvt:softwareAgent':'Adobe Photoshop CS6 (Macintosh)'}", + "{'stEvt:action':'saved'}" + ] + ], + "crs:Highlights2012": "0", + "photoshop:Country": "England", + "crs:ProcessVersion": "6.7", + "crs:Vibrance": "+14", + "crs:SharpenRadius": "+1.0", + "xmpMM:InstanceID": "xmp.iid:1D7ECFD70A2068118C1496C090CBA674", + "crs:ColorNoiseReduction": "25" + }, + "icc": { + "Profile Size": "560", + "Media White Point": "(0.9505, 1, 1.0891)", + "Red Colorant": "(0.6097, 0.3111, 0.0195)", + "Blue TRC": "0.0085908", + "Version": "2.1.0", + "Tag Count": "10", + "Profile Description": "Adobe RGB (1998)", + "Green Colorant": "(0.2053, 0.6257, 0.0609)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "none", + "Green TRC": "0.0085908", + "Red TRC": "0.0085908", + "Blue Colorant": "(0.1492, 0.0632, 0.7446)", + "Color space": "RGB", + "Primary Platform": "Apple Computer, Inc.", + "Profile Connection Space": "XYZ", + "CMM Type": "ADBE", + "Class": "Display Device", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1999:06:03 00:00:00", + "Profile Copyright": "Copyright 1999 Adobe Systems Incorporated" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "photometricInterpretation": "RGB", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of 11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\nBy Lucy Young \n07799118984\nlucyyounguk@gmail.com\nwww.lucyyoungphotos.co.uk", + "title": "A Sunseeker 115 Sport Super Yacht, which has a starting price of 11.5 million exc tax. arrives for The London Boat Show at ExCel." + } + }, + "metadata": { + "dateTaken": "2014-01-02T12:43:54.100Z", + "description": "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of 11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\nBy Lucy Young \n07799118984\nlucyyounguk@gmail.com\nwww.lucyyoungphotos.co.uk", + "credit": "Lucy Young", + "byline": "Lucy Young", + "title": "A Sunseeker 115 Sport Super Yacht, which has a starting price of 11.5 million exc tax. arrives for The London Boat Show at ExCel.", + "copyright": "Lucy Young", + "specialInstructions": "From: Lucy Young ", + "keywords": [], + "city": "London", + "country": "England", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2014-01-02T12:43:54.100Z", + "description": "Rear sun deck on board a Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel. The eight-cabin vessel has sleeping quarters for 15 people and boasts nine bathrooms, including one with a Jacuzzi. There are five bars and seven fridges – including four specialist wine coolers and three ice makers. The Sunseeker is the costliest vessel on display at the London Boat Show, which opens tomorrow and runs until January 12.\nBy Lucy Young \n07799118984\nlucyyounguk@gmail.com\n\nwww.lucyyoungphotos.co.uk", + "credit": "Lucy Young", + "byline": "Lucy Young", + "title": "A Sunseeker 115 Sport Super Yacht, which has a starting price of £11.5 million exc tax. arrives for The London Boat Show at ExCel.", + "copyright": "Lucy Young", + "specialInstructions": "From: Lucy Young ", + "keywords": [], + "city": "London", + "country": "England", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [ + { + "id": "print/90dab991a7b8d6dc95066ea91694c9e0_26d733c73fae0793858104574cc9df69", + "references": [ + { + "type": "indesign", + "name": "2014-01-07, The Guardian, frt, Page 3" + } + ], + "platform": "print", + "media": "image", + "status": "published", + "dateAdded": "2014-01-07T05:01:26.000Z", + "lastModified": "2014-01-07T05:01:26.000Z", + "printUsageMetadata": { + "sectionName": "frt", + "issueDate": "2014-01-07T00:00:00.000Z", + "pageNumber": 3, + "storyName": "yachts_04_frt_0107_003", + "publicationCode": "gdn", + "publicationName": "The Guardian", + "edition": 1, + "sectionCode": "frt", + "notes": "Martin Nicholls", + "source": "picdar" + } + } + ], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2014-01-07T05:01:26.000Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "1a5b08c19cd3a68ac496c46bdd761f8dc05313aa", + "_score": 1, + "_source": { + "id": "1a5b08c19cd3a68ac496c46bdd761f8dc05313aa", + "uploadTime": "2020-01-23T15:10:16.577Z", + "uploadedBy": "getty", + "lastModified": "2020-06-06T14:14:16.513Z", + "identifiers": {}, + "uploadInfo": { + "filename": "1195421274.jpg" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/1/a/5/b/0/8/1a5b08c19cd3a68ac496c46bdd761f8dc05313aa", + "size": 5413118, + "mimeType": "image/jpeg", + "dimensions": { + "width": 5472, + "height": 3648 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/1/a/5/b/0/8/1a5b08c19cd3a68ac496c46bdd761f8dc05313aa", + "size": 15062, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 171 + } + }, + "fileMetadata": { + "iptc": { + "By-line Title": "Contributor", + "Country/Primary Location Name": "Russia", + "Category": "I", + "Country/Primary Location Code": "RUS", + "Supplemental Category(s)": "FIN", + "Coded Character Set": "UTF-8", + "Application Record Version": "4", + "Caption/Abstract": "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)", + "Enveloped Record Version": "4", + "Credit": "Donat Sorokin/TASS", + "Source": "TASS", + "City": "Yekaterinburg", + "Keywords": "medicine;coronavirus;drug;healthcare;corona virus;wuhan virus;china virus;urfu;ural federal university;treatment;remedy", + "By-line": "Donat Sorokin", + "Urgency": "51", + "Special Instructions": "0", + "Headline": "Ural Federal University researchers propose using Triazavirin against China coronavirus", + "Object Name": "1195421274", + "Caption Writer/Editor": "DF", + "Original Transmission Reference": "1195421274", + "Date Created": "20200123T00:00:00Z" + }, + "exif": { + "Image Description": "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)" + }, + "exifSub": {}, + "xmp": { + "GettyImagesGIFT:ImageRank": "3", + "GettyImagesGIFT:OriginalFilename": "ts0cb425.jpg", + "dc:description": [ + "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)", + [ + "{'xml:lang':'x-default'}" + ] + ], + "photoshop:Headline": "Ural Federal University researchers propose using Triazavirin against China coronavirus", + "photoshop:TransmissionReference": "0", + "photoshop:Instructions": "0", + "photoshop:AuthorsPosition": "Contributor", + "dc:subject": [ + "medicine", + "coronavirus", + "drug", + "healthcare", + "corona virus", + "wuhan virus", + "china virus", + "urfu", + "ural federal university", + "treatment", + "remedy" + ], + "photoshop:CaptionWriter": "DF", + "photoshop:SupplementalCategories": [ + "FIN" + ], + "plus:ImageSupplierImageId": "ts0cb425", + "photoshop:City": "Yekaterinburg", + "GettyImagesGIFT:ExclusiveCoverage": "False", + "photoshop:DateCreated": "2020-01-23T00:00:00.000Z", + "photoshop:Credit": "Donat Sorokin/TASS", + "plus:LicsensorURL": "http://www.gettyimages.com", + "GettyImagesGIFT:OriginalCreateDateTime": "1970-01-01T00:00:02.000Z", + "dc:creator": [ + "Donat Sorokin" + ], + "GettyImagesGIFT:AssetId": "1195421274", + "dc:title": [ + "1195421274", + [ + "{'xml:lang':'x-default'}" + ] + ], + "Iptc4xmpCore:CountryCode": "RUS", + "GettyImagesGIFT:CallForImage": "False", + "photoshop:Country": "Russia", + "photoshop:Source": "TASS", + "photoshop:Category": "I" + }, + "icc": {}, + "getty": { + "Call For Image": "False", + "Image Rank": "3", + "Asset ID": "1195421274", + "Original Filename": "ts0cb425.jpg", + "Exclusive Coverage": "False", + "Original Create Date Time": "1970-01-01T00:00:02.000Z" + }, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "metadata": { + "dateTaken": "2020-01-23T00:00:00.000Z", + "description": "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)", + "credit": "TASS", + "byline": "Donat Sorokin", + "bylineTitle": "Contributor", + "title": "Ural Federal University researchers propose using Triazavirin against China coronavirus", + "suppliersReference": "1195421274", + "source": "TASS", + "specialInstructions": "0", + "keywords": [ + "ural federal university", + "treatment", + "remedy", + "coronavirus", + "china virus", + "medicine", + "corona virus", + "healthcare", + "urfu", + "drug", + "wuhan virus" + ], + "city": "Yekaterinburg", + "country": "Russia", + "subjects": [ + "finance", + "news" + ], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2020-01-23T00:00:00.000Z", + "description": "YEKATERINABURG, RUSSIA - JANUARY 23, 2020: The image shows blister packs of capsules of the antiviral medicine Triazavirin in an organic synthesis lab in the Centre for Pharmaceutical and Chemical Technologies at the Ural Federal University (UrFU); UrFU researchers have proposed using Triazavirin against the new strain of coronavirus in China given the fact that Triazavirin is effective against RNA viruses, to which the new virus belongs. Donat Sorokin/TASS (Photo by Donat Sorokin\\TASS via Getty Images)", + "credit": "TASS", + "byline": "Donat Sorokin", + "bylineTitle": "Contributor", + "title": "Ural Federal University researchers propose using Triazavirin against China coronavirus", + "suppliersReference": "1195421274", + "source": "TASS", + "specialInstructions": "0", + "keywords": [ + "ural federal university", + "treatment", + "remedy", + "coronavirus", + "china virus", + "medicine", + "corona virus", + "healthcare", + "urfu", + "drug", + "wuhan virus" + ], + "city": "Yekaterinburg", + "country": "Russia", + "subjects": [ + "finance", + "news" + ], + "peopleInImage": [] + }, + "usageRights": { + "category": "agency", + "supplier": "Getty Images", + "suppliersCollection": "TASS" + }, + "originalUsageRights": { + "category": "agency", + "supplier": "Getty Images", + "suppliersCollection": "TASS" + }, + "exports": [], + "usages": [ + { + "id": "print/1ac0407e35d79a4b71bf9cb05d7da3e8_106ebc81cf0cde3c657607647b34fc93", + "references": [ + { + "type": "indesign", + "name": "2020-06-07, Observer, UK News (Observer), Page 13" + } + ], + "platform": "print", + "media": "image", + "status": "pending", + "dateAdded": "2020-06-06T14:14:16.513Z", + "lastModified": "2020-06-06T14:14:16.513Z", + "printUsageMetadata": { + "sectionName": "UK News (Observer)", + "issueDate": "2020-06-07T00:00:00.000Z", + "pageNumber": 13, + "storyName": "HOMRMCovid7", + "publicationCode": "obs", + "publicationName": "Observer", + "layoutId": 15284077, + "edition": 1, + "size": { + "x": 240, + "y": 240 + }, + "orderedBy": "Liz Boulter", + "sectionCode": "2hn" + } + } + ], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2020-06-06T14:14:16.513Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "ea763a169dd6574c6ac1ccaf3443d363e011533f", + "_score": 1, + "_source": { + "id": "ea763a169dd6574c6ac1ccaf3443d363e011533f", + "uploadTime": "2015-07-26T14:50:49.000Z", + "uploadedBy": "DEV Ingest Script", + "lastModified": "2006-03-31T23:00:00.000Z", + "identifiers": { + "picdarurn": "PD*9095009" + }, + "uploadInfo": {}, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/e/a/7/6/3/a/ea763a169dd6574c6ac1ccaf3443d363e011533f", + "size": 495188, + "mimeType": "image/jpeg", + "dimensions": { + "width": 2482, + "height": 1793 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/e/a/7/6/3/a/ea763a169dd6574c6ac1ccaf3443d363e011533f", + "size": 27755, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 185 + } + }, + "fileMetadata": { + "iptc": { + "Country/Primary Location Name": "United Kingdom", + "Category": "S", + "Supplemental Category(s)": "Rugby Union", + "Application Record Version": "2", + "Caption/Abstract": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\rNewcastle Falcons' Toby Flood prepears to take a penalty\rMandatory Credit: Action Images / Lee Smith\rLivepic", + "Credit": "Action Images", + "Source": "Action Images", + "City": "Newcastle", + "Keywords": "2006", + "By-line": "Lee Smith", + "Special Instructions": "01", + "Caption Writer/Editor": "tp", + "Original Transmission Reference": "GCLY-6LBGXV", + "Date Created": "2006:03:31" + }, + "exif": { + "Image Description": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\rNewcastle Falcons' Toby Flood prepears to take a penalty\rMandatory Credit: Action Images / Lee Smith\rLivepic", + "Primary Chromaticities": "64/100 33/100 21/100 71/100 15/100 6/100", + "X Resolution": "72 dots per inch", + "Software": "Adobe Photoshop 7.0", + "Make": "Canon", + "YCbCr Coefficients": "299/1000 587/1000 114/1000", + "YCbCr Positioning": "Datum point", + "Date/Time": "2006:03:31 20:44:35", + "Model": "Canon EOS-1D Mark II", + "Orientation": "Top, left side (Horizontal / normal)", + "White Point": "313/1000 329/1000", + "Resolution Unit": "Inch", + "Y Resolution": "72 dots per inch", + "Artist": "Lee Smith" + }, + "exifSub": { + "Exif Version": "2.21", + "Exposure Mode": "Auto exposure", + "ISO Speed Ratings": "3200", + "Custom Rendered": "Normal process", + "Exif Image Height": "1793 pixels", + "Flash": "Flash did not fire", + "Focal Length": "300 mm", + "Date/Time Original": "2006:03:31 19:04:17", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/511 sec", + "Exif Image Width": "2482 pixels", + "Focal Plane Y Resolution": "189/585500 inches", + "Exposure Time": "1/500 sec", + "Metering Mode": "Multi-segment", + "Exposure Program": "Shutter priority", + "Gamma": "2.2", + "Exposure Bias Value": "1/3 EV", + "F-Number": "f/2.8", + "Color Space": "Undefined", + "FlashPix Version": "1.00", + "Components Configuration": "YCbCr", + "Date/Time Digitized": "2006:03:31 19:04:17", + "Date/Time Original Composite": "2006-03-31T19:04:17.000Z", + "Aperture Value": "f/2.8", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "Inches", + "Focal Plane X Resolution": "71/220000 inches" + }, + "xmp": { + "dc:description": [ + "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\rNewcastle Falcons' Toby Flood prepears to take a penalty\rMandatory Credit: Action Images / Lee Smith\rLivepic", + [ + "{'xml:lang':'x-default'}" + ] + ], + "photoshop:TransmissionReference": "GCLY-6LBGXV", + "photoshop:Instructions": "01", + "dc:subject": [ + "2006" + ], + "photoshop:CaptionWriter": "tp", + "photoshop:SupplementalCategories": [ + "Rugby Union" + ], + "photoshop:City": "Newcastle", + "photoshop:DateCreated": "2006-03-31T00:00:00.000Z", + "photoshop:Credit": "Action Images", + "xmpMM:DocumentID": "adobe:docid:photoshop:bb6e8fa1-c0e8-11da-8611-8b5221e90d2f", + "xmpRights:Marked": "True", + "dc:creator": [ + "Lee Smith" + ], + "photoshop:Country": "United Kingdom", + "photoshop:Source": "Action Images", + "photoshop:Category": "S", + "xmpMM:InstanceID": "uuid:e3938d28-c0ed-11da-81f9-cc201dca07e4" + }, + "icc": {}, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\nNewcastle Falcons' Toby Flood prepears to take a penalty\nMandatory Credit: Action Images / Lee Smith\nLivepic" + }, + "usageRights": { + "category": "agency", + "supplier": "Corbis" + } + }, + "metadata": { + "dateTaken": "2006-03-31T19:04:17.000Z", + "description": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\nNewcastle Falcons' Toby Flood prepears to take a penalty\nMandatory Credit: Action Images / Lee Smith\nLivepic", + "credit": "Action Images/Reuters", + "byline": "Lee Smith", + "suppliersReference": "GCLY-6LBGXV", + "source": "Action Images", + "specialInstructions": "01", + "keywords": [ + "2006" + ], + "city": "Newcastle", + "country": "United Kingdom", + "subjects": [ + "sport" + ], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2006-03-31T19:04:17.000Z", + "description": "Rugby Union - Newcastle Falcons v Connacht - European Challenge Cup Quarter Final - Kingston Park - 31/3/06\rNewcastle Falcons' Toby Flood prepears to take a penalty\rMandatory Credit: Action Images / Lee Smith\rLivepic", + "credit": "Action Images/Reuters", + "byline": "Lee Smith", + "suppliersReference": "GCLY-6LBGXV", + "source": "Action Images", + "specialInstructions": "01", + "keywords": [ + "2006" + ], + "city": "Newcastle", + "country": "United Kingdom", + "subjects": [ + "sport" + ], + "peopleInImage": [] + }, + "usageRights": { + "category": "agency", + "supplier": "Corbis" + }, + "originalUsageRights": { + "category": "agency", + "supplier": "Action Images" + }, + "exports": [], + "usages": [ + { + "id": "print/e02f7633cbd760e61a9f1c4cbef78699_d6848c99a477f095de6dab5660465049", + "references": [ + { + "type": "indesign", + "name": "2006-03-31, The Guardian, Guardian Sport, Page 10" + } + ], + "platform": "print", + "media": "image", + "status": "published", + "dateAdded": "2006-03-31T23:00:00.000Z", + "lastModified": "2006-03-31T23:00:00.000Z", + "printUsageMetadata": { + "sectionName": "Guardian Sport", + "issueDate": "2006-03-31T23:00:00.000Z", + "pageNumber": 10, + "storyName": "flood01", + "publicationCode": "gdn", + "publicationName": "The Guardian", + "sectionCode": "Guardian Sport", + "notes": "45mm wide", + "source": "picdar" + } + } + ], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2006-03-31T23:00:00.000Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "82c4d17a78148cc7ab4006d5a1d8ddbf9810da72", + "_score": 1, + "_source": { + "id": "82c4d17a78148cc7ab4006d5a1d8ddbf9810da72", + "uploadTime": "2020-06-12T14:51:12.323Z", + "uploadedBy": "stingray", + "lastModified": "2020-06-12T15:04:11.817Z", + "identifiers": {}, + "uploadInfo": { + "filename": "pxl_churchill_mandela_statues_westminster_001.JPG" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/8/2/c/4/d/1/82c4d17a78148cc7ab4006d5a1d8ddbf9810da72", + "size": 6411306, + "mimeType": "image/jpeg", + "dimensions": { + "width": 6720, + "height": 4480 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/8/2/c/4/d/1/82c4d17a78148cc7ab4006d5a1d8ddbf9810da72", + "size": 20696, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 171 + } + }, + "fileMetadata": { + "iptc": { + "Country/Primary Location Name": "United Kingdom", + "Country/Primary Location Code": "GBR", + "Copyright Notice": "Gavin Rodgers/ Pixel8000", + "Application Record Version": "3", + "Caption/Abstract": "Pic shows: What would Mandela do?\n\nNelson Mandela statue appears to be either removing of placing the box on Winston Churchill’s statue in Parliament Square today\n\nafter it was actually covered Sadiq Khan ordered the council to cover the memorial due to Black Lives Matter protests\n\n\n\n\npicture by Gavin Rodgers/ Pixel8000", + "Credit": "gavin rodgers/pixel8000", + "City": "London", + "Digital Date Created": "2020:06:12", + "Keywords": "Sadiq Khan, statue, Nelson Mandela, Westminster, Parliament Square, London, protests, race relations", + "Time Created": "14:01:30+0100", + "By-line": "gavin rodgers/pixel8000", + "Urgency": "49", + "Date Time Created Composite": "2020-06-12T13:01:30.000Z", + "Date Created": "2020:06:12" + }, + "exif": { + "X Resolution": "300 dots per inch", + "Software": "PhotoShelter http://www.photoshelter.com", + "Make": "Canon", + "Date/Time": "2020:06:12 15:39:59", + "Model": "Canon EOS 5D Mark IV", + "Orientation": "Top, left side (Horizontal / normal)", + "Resolution Unit": "Inch", + "Y Resolution": "300 dots per inch" + }, + "exifSub": { + "Exif Version": "2.30", + "Exposure Mode": "Auto exposure", + "ISO Speed Ratings": "640", + "Lens Specification": "100-400mm", + "Body Serial Number": "018021000814", + "Custom Rendered": "Normal process", + "Exif Image Height": "4480 pixels", + "Lens Serial Number": "9720000826", + "Flash": "Flash did not fire", + "Focal Length": "110 mm", + "Date/Time Original": "2020:06:12 14:01:30", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/499 sec", + "Exif Image Width": "6720 pixels", + "Focal Plane Y Resolution": "32768/61166933 cm", + "Sub-Sec Time Original": "16", + "Exposure Time": "1/500 sec", + "Metering Mode": "Multi-segment", + "Exposure Program": "Shutter priority", + "Exposure Bias Value": "0 EV", + "F-Number": "f/5.0", + "Color Space": "Undefined", + "Sub-Sec Time": "16", + "Lens Model": "EF100-400mm f/4.5-5.6L IS II USM", + "Date/Time Digitized": "2020:06:12 14:01:30", + "Recommended Exposure Index": "640", + "Date/Time Original Composite": "2020-06-12T14:01:30.160Z", + "Aperture Value": "f/5.0", + "Sensitivity Type": "Recommended Exposure Index", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "cm", + "Sub-Sec Time Digitized": "16", + "Focal Plane X Resolution": "32768/61166933 cm", + "Max Aperture Value": "f/4.6" + }, + "xmp": { + "crs:ParametricDarks": "0", + "dc:format": "image/jpeg", + "crs:DefringeGreenHueLo": "40", + "crs:SaturationAdjustmentPurple": "0", + "crs:HasSettings": "True", + "crs:GreenHue": "0", + "photoshop:ColorMode": "3", + "dc:description": [ + "Pic shows: What would Mandela do?\n\nNelson Mandela statue appears to be either removing of placing the box on Winston Churchill’s statue in Parliament Square today\n\nafter it was actually covered Sadiq Khan ordered the council to cover the memorial due to Black Lives Matter protests\n\n\n\n\npicture by Gavin Rodgers/ Pixel8000", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:SaturationAdjustmentAqua": "0", + "crs:UprightVersion": "151388160", + "crs:SharpenDetail": "25", + "crs:GrainAmount": "0", + "xmp:MetadataDate": "2020-06-12T14:39:59.000Z", + "crs:CameraProfile": "Adobe Standard", + "aux:LensSerialNumber": "9720000826", + "crs:WhiteBalance": "As Shot", + "crs:ToneCurveRed": [ + "0, 0", + "255, 255" + ], + "crs:ColorNoiseReductionDetail": "50", + "crs:Blacks2012": "-33", + "crs:SaturationAdjustmentBlue": "0", + "xmpMM:DerivedFrom": [ + "{'stRef:instanceID':'xmp.iid:5eb5ca6f-073b-4823-a9e6-f689444584d7'}", + "{'stRef:documentID':'xmp.did:c2407944-aa6a-45df-9cd6-e408cba2e9e9'}", + "{'stRef:originalDocumentID':'E83BCB4532750F9B267DF41F7CC37811'}" + ], + "crs:UprightPreview": "False", + "crs:ToneCurveName": "Medium Contrast", + "aux:ImageNumber": "6280", + "dc:date": [ + "2020-06-12T14:01:30.000Z" + ], + "crs:SplitToningShadowSaturation": "0", + "crs:SplitToningShadowHue": "0", + "crs:ParametricHighlightSplit": "75", + "crs:PerspectiveRotate": "0.0", + "crs:PerspectiveUpright": "0", + "photoshop:ICCProfile": "Adobe RGB (1998)", + "crs:PerspectiveX": "0.00", + "crs:DefringePurpleHueLo": "30", + "crs:ToneCurveName2012": "Linear", + "crs:ToneCurvePV2012": [ + "0, 0", + "255, 255" + ], + "crs:HueAdjustmentPurple": "0", + "crs:VignetteAmount": "0", + "crs:Exposure2012": "+0.25", + "crs:ParametricShadowSplit": "25", + "crs:LuminanceAdjustmentAqua": "0", + "crs:LuminanceAdjustmentPurple": "0", + "photomechanic:PMVersion": "PM6", + "crs:SplitToningBalance": "0", + "crs:SaturationAdjustmentMagenta": "0", + "crs:Sharpness": "40", + "crs:HueAdjustmentAqua": "0", + "dc:subject": [ + "Sadiq Khan, statue, Nelson Mandela, Westminster, Parliament Square, London, protests, race relations" + ], + "crs:ColorNoiseReductionSmoothness": "50", + "crs:BlueHue": "0", + "crs:DefringeGreenHueHi": "60", + "crs:SaturationAdjustmentOrange": "0", + "crs:HueAdjustmentYellow": "0", + "crs:SplitToningHighlightHue": "0", + "crs:PerspectiveScale": "100", + "crs:HueAdjustmentGreen": "0", + "aux:FlashCompensation": "0/1", + "photomechanic:Prefs": "0:1:0:006280", + "crs:Temperature": "5200", + "crs:ConvertToGrayscale": "False", + "crs:ParametricLights": "0", + "aux:Lens": "EF100-400mm f/4.5-5.6L IS II USM", + "crs:UprightFocalMode": "0", + "photomechanic:Tagged": "False", + "crs:ToneCurvePV2012Green": [ + "0, 0", + "255, 255" + ], + "crs:ParametricHighlights": "0", + "crs:OverrideLookVignette": "False", + "crs:SharpenEdgeMasking": "0", + "crs:ToneCurve": [ + "0, 0", + "32, 22", + "64, 56", + "128, 128", + "192, 196", + "255, 255" + ], + "crs:SaturationAdjustmentRed": "0", + "crs:Whites2012": "+46", + "crs:Clarity2012": "+14", + "crs:UprightFocalLength35mm": "35", + "xmp:ModifyDate": "2020-06-12T14:39:59.000Z", + "crs:UprightTransformCount": "6", + "crs:UprightCenterNormX": "0.5", + "photoshop:City": "London", + "xmpMM:OriginalDocumentID": "E83BCB4532750F9B267DF41F7CC37811", + "xmp:Rating": "0", + "crs:AutoToneDigestNoSat": "06984843E46C78312DAF35D2B764C928", + "crs:ToneCurvePV2012Blue": [ + "0, 0", + "255, 255" + ], + "crs:Look": [ + "{'crs:Parameters':['{'crs:ToneCurvePV2012Blue':['0, 0','255, 255']}','{'crs:ToneCurvePV2012Red':['0, 0','255, 255']}','{'crs:ToneCurvePV2012':['0, 0','22, 16','40, 35','127, 127','224, 230','240, 246','255, 255']}','{'crs:ToneCurvePV2012Green':['0, 0','255, 255']}']}", + "{'crs:Group':['Profiles']}", + "{'crs:Parameters':['{'crs:LookTable':'E1095149FDB39D7A057BAB208837E2E1'}','{'crs:CameraProfile':'Adobe Standard'}','{'crs:ConvertToGrayscale':'False'}','{'crs:Version':'12.2.1'}','{'crs:ProcessVersion':'11.0'}']}", + "{'crs:SupportsMonochrome':'false'}", + "{'crs:UUID':'B952C231111CD8E0ECCF14B86BAA7077'}", + "{'crs:Amount':'1.000000'}", + "{'crs:SupportsOutputReferred':'false'}", + "{'crs:SupportsAmount':'false'}", + "{'crs:Copyright':'© 2018 Adobe Systems, Inc.'}", + "{'crs:Name':'Adobe Color'}", + "{'crs:Group':[['{'xml:lang':'x-default'}']]}" + ], + "crs:LuminanceAdjustmentRed": "0", + "crs:DefringeGreenAmount": "0", + "crs:PerspectiveVertical": "0", + "crs:HueAdjustmentBlue": "0", + "crs:PerspectiveY": "0.00", + "crs:LensManualDistortionAmount": "0", + "crs:AutoLateralCA": "0", + "crs:LuminanceSmoothing": "0", + "crs:CameraProfileDigest": "661433344C8532AFA5A1E9091401E43C", + "crs:Contrast2012": "+17", + "photoshop:DateCreated": "2020-06-12T14:01:30.000Z", + "photoshop:Credit": "gavin rodgers/pixel8000", + "crs:SaturationAdjustmentYellow": "0", + "xmp:CreateDate": "2020-06-12T14:01:30.000Z", + "crs:LensProfileSetup": "LensDefaults", + "crs:Shadows2012": "+45", + "crs:UprightCenterMode": "0", + "crs:LensProfileEnable": "0", + "aux:LensID": "747", + "aux:ApproximateFocusDistance": "192/10", + "crs:Tint": "+6", + "crs:ParametricMidtoneSplit": "50", + "crs:PerspectiveHorizontal": "0", + "crs:ShadowTint": "0", + "crs:UprightCenterNormY": "0.5", + "dc:rights": [ + "Gavin Rodgers/ Pixel8000", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:HueAdjustmentMagenta": "0", + "crs:AutoToneDigest": "47AB1A2F4B28A8F46A86BB6CA1E72290", + "aux:LensInfo": "100/1 400/1 0/0 0/0", + "crs:ToneCurveBlue": [ + "0, 0", + "255, 255" + ], + "crs:LuminanceAdjustmentMagenta": "0", + "crs:ToneCurveGreen": [ + "0, 0", + "255, 255" + ], + "photomechanic:ColorClass": "1", + "crs:BlueSaturation": "0", + "crs:LuminanceAdjustmentYellow": "0", + "crs:AlreadyApplied": "True", + "crs:DefringePurpleHueHi": "70", + "crs:PostCropVignetteAmount": "0", + "xmp:Label": "Winner", + "xmpMM:DocumentID": "adobe:docid:photoshop:e34c6534-8d64-9842-81e1-3a10bd6f8fae", + "crs:PerspectiveAspect": "0", + "crs:LuminanceAdjustmentGreen": "0", + "crs:Dehaze": "+4", + "crs:RedSaturation": "0", + "xmpRights:Marked": "True", + "crs:SaturationAdjustmentGreen": "0", + "exifEX:LensModel": "EF100-400mm f/4.5-5.6L IS II USM", + "photoshop:Urgency": "1", + "dc:creator": [ + "gavin rodgers/pixel8000" + ], + "crs:LuminanceAdjustmentOrange": "0", + "crs:Version": "12.2.1", + "crs:ToneCurvePV2012Red": [ + "0, 0", + "255, 255" + ], + "crs:Texture": "0", + "crs:ToneMapStrength": "0", + "crs:UprightFourSegmentsCount": "0", + "crs:ParametricShadows": "0", + "crs:HueAdjustmentRed": "0", + "aux:Firmware": "1.0.1", + "crs:GreenSaturation": "0", + "crs:RedHue": "0", + "crs:SplitToningHighlightSaturation": "0", + "aux:SerialNumber": "018021000814", + "crs:DefringePurpleAmount": "0", + "crs:Saturation": "+2", + "crs:HueAdjustmentOrange": "0", + "crs:LuminanceAdjustmentBlue": "0", + "xmpMM:History": [ + [ + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 12.2.1 (Macintosh)'}", + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-06-12T15:38:35+01:00'}", + "{'stEvt:changed':'/metadata'}", + "{'stEvt:instanceID':'xmp.iid:940c3657-87c0-44a2-81e0-0a01312ff71e'}" + ], + [ + "{'stEvt:parameters':'converted from image/x-canon-cr2 to image/tiff'}", + "{'stEvt:action':'derived'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-06-12T15:38:36+01:00'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Camera Raw 12.2.1 (Macintosh)'}", + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:c2407944-aa6a-45df-9cd6-e408cba2e9e9'}" + ], + [ + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:5eb5ca6f-073b-4823-a9e6-f689444584d7'}", + "{'stEvt:when':'2020-06-12T15:39:59+01:00'}", + "{'stEvt:softwareAgent':'Adobe Photoshop 21.1 (Macintosh)'}", + "{'stEvt:action':'saved'}" + ], + [ + "{'stEvt:parameters':'from image/tiff to image/jpeg'}", + "{'stEvt:action':'converted'}" + ], + [ + "{'stEvt:action':'derived'}", + "{'stEvt:parameters':'converted from image/tiff to image/jpeg'}" + ], + [ + "{'stEvt:instanceID':'xmp.iid:7789b156-9aaf-4034-b0d3-9ba1356bb1d1'}", + "{'stEvt:action':'saved'}", + "{'stEvt:changed':'/'}", + "{'stEvt:softwareAgent':'Adobe Photoshop 21.1 (Macintosh)'}", + "{'stEvt:when':'2020-06-12T15:39:59+01:00'}" + ] + ], + "Iptc4xmpCore:CountryCode": "GBR", + "crs:Highlights2012": "-64", + "photoshop:Country": "United Kingdom", + "crs:ProcessVersion": "11.0", + "crs:Vibrance": "+15", + "crs:SharpenRadius": "+1.0", + "xmpMM:InstanceID": "xmp.iid:7789b156-9aaf-4034-b0d3-9ba1356bb1d1", + "crs:ColorNoiseReduction": "25" + }, + "icc": { + "Profile Size": "560", + "Media White Point": "(0.9505, 1, 1.0891)", + "Red Colorant": "(0.6097, 0.3111, 0.0195)", + "Blue TRC": "0.0085908", + "Version": "2.1.0", + "Tag Count": "10", + "Profile Description": "Adobe RGB (1998)", + "Green Colorant": "(0.2053, 0.6257, 0.0609)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "none", + "Green TRC": "0.0085908", + "Red TRC": "0.0085908", + "Blue Colorant": "(0.1492, 0.0632, 0.7446)", + "Color space": "RGB", + "Primary Platform": "Apple Computer, Inc.", + "Profile Connection Space": "XYZ", + "CMM Type": "ADBE", + "Class": "Display Device", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1999:06:03 00:00:00", + "Profile Copyright": "Copyright 1999 Adobe Systems Incorporated" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "metadata": { + "dateTaken": "2020-06-12T14:01:30.160Z", + "description": "Pic shows: What would Mandela do?\n\nNelson Mandela statue appears to be either removing of placing the box on Winston Churchill’s statue in Parliament Square today\n\nafter it was actually covered Sadiq Khan ordered the council to cover the memorial due to Black Lives Matter protests\n\n\n\n\npicture by Gavin Rodgers/ Pixel8000", + "credit": "pixel8000", + "byline": "Gavin Rodgers", + "copyright": "Gavin Rodgers/ Pixel8000", + "keywords": [ + "Sadiq Khan, statue, Nelson Mandela, Westminster, Parliament Square, London, protests, race relations" + ], + "city": "London", + "country": "United Kingdom", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2020-06-12T14:01:30.160Z", + "description": "Pic shows: What would Mandela do?\n\nNelson Mandela statue appears to be either removing of placing the box on Winston Churchill’s statue in Parliament Square today\n\nafter it was actually covered Sadiq Khan ordered the council to cover the memorial due to Black Lives Matter protests\n\n\n\n\npicture by Gavin Rodgers/ Pixel8000", + "credit": "pixel8000", + "byline": "Gavin Rodgers", + "copyright": "Gavin Rodgers/ Pixel8000", + "keywords": [ + "Sadiq Khan, statue, Nelson Mandela, Westminster, Parliament Square, London, protests, race relations" + ], + "city": "London", + "country": "United Kingdom", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [ + { + "path": [ + "Daily Edit", + "Daily edit home" + ], + "pathId": "daily edit/daily edit home", + "description": "Daily edit home", + "actionData": { + "author": "fiona.shields@guardian.co.uk", + "date": "2020-06-12T15:04:11.817+00:00" + } + } + ], + "userMetadataLastModified": "2020-06-12T15:04:11.817Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "119960415d35d0f29d86b1513266296535b4d636", + "_score": 1, + "_source": { + "id": "119960415d35d0f29d86b1513266296535b4d636", + "uploadTime": "2020-02-24T09:02:54.358Z", + "uploadedBy": "guy.lane@guardian.co.uk", + "lastModified": "2020-02-24T09:08:18.715Z", + "identifiers": {}, + "uploadInfo": { + "filename": "_9DS9316.jpg" + }, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/1/1/9/9/6/0/119960415d35d0f29d86b1513266296535b4d636", + "size": 8349532, + "mimeType": "image/jpeg", + "dimensions": { + "width": 4788, + "height": 3196 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/1/1/9/9/6/0/119960415d35d0f29d86b1513266296535b4d636", + "size": 9913, + "mimeType": "image/jpeg", + "dimensions": { + "width": 256, + "height": 171 + } + }, + "fileMetadata": { + "iptc": { + "By-line Title": "photographer", + "Country/Primary Location Name": "spain", + "Country/Primary Location Code": "ESP", + "Copyright Notice": "Didier Bizet", + "Coded Character Set": "UTF-8", + "Application Record Version": "4", + "Caption/Abstract": "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + "Credit": "Didier Bizet", + "Source": "hans lucas", + "City": "Valencia", + "Digital Date Created": "2019:04:27", + "Keywords": "reborn;reborn doll;reborn dolls;dolls;baby;bebes;nouveau ne;new born;doll;poupee;phenomene;phenomenon;realism;hyper-realistic;reborn artist;mother;parents;mère;parents;fabrication;reborn workshop;silicon;reborn silicon", + "Time Created": "19:46:03+0000", + "By-line": "Didier Bizet", + "Object Name": "baby boom", + "Date Time Created Composite": "2019-04-27T19:46:03.000Z", + "Caption Writer/Editor": "photographer", + "Original Transmission Reference": "babyboom_bizet", + "Date Created": "2019:04:27" + }, + "exif": { + "Image Description": "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + "X Resolution": "300 dots per inch", + "Software": "Adobe Photoshop CC 2019 (Macintosh)", + "Photometric Interpretation": "RGB", + "Make": "NIKON CORPORATION", + "Image Width": "4788 pixels", + "Samples Per Pixel": "3 samples/pixel", + "Copyright": "Didier Bizet", + "Date/Time": "2020:01:02 17:46:06", + "Model": "NIKON D750", + "Orientation": "Top, left side (Horizontal / normal)", + "Resolution Unit": "Inch", + "Bits Per Sample": "8 8 8 bits/component/pixel", + "Y Resolution": "300 dots per inch", + "Artist": "Didier Bizet", + "Image Height": "3196 pixels" + }, + "exifSub": { + "CFA Pattern": "[Red,Green][Green,Blue]", + "Exif Version": "2.31", + "Subject Distance Range": "Unknown", + "Exposure Mode": "Manual exposure", + "ISO Speed Ratings": "2000", + "Lens Specification": "35mm f/1.8", + "Body Serial Number": "6085734", + "Custom Rendered": "Normal process", + "Exif Image Height": "3196 pixels", + "Flash": "Flash did not fire", + "Time Zone": "+02:00", + "White Balance": "Unknown", + "Focal Length": "35 mm", + "Date/Time Original": "2019:04:27 19:46:03", + "White Balance Mode": "Auto white balance", + "Shutter Speed Value": "1/199 sec", + "Exif Image Width": "4788 pixels", + "Gain Control": "Low gain down", + "Sensing Method": "One-chip color area sensor", + "Scene Type": "Directly photographed image", + "Focal Plane Y Resolution": "32768/54886891 cm", + "Sub-Sec Time Original": "32", + "Exposure Time": "1/200 sec", + "Sharpness": "None", + "Metering Mode": "Center weighted average", + "Exposure Program": "Manual control", + "Digital Zoom Ratio": "1", + "Exposure Bias Value": "0 EV", + "F-Number": "f/4.0", + "Color Space": "Undefined", + "Lens Model": "35.0 mm f/1.8", + "Focal Length 35": "35 mm", + "Date/Time Digitized": "2019:04:27 19:46:03", + "Date/Time Original Composite": "2019-04-27T19:46:03.320Z", + "Aperture Value": "f/4.0", + "Sensitivity Type": "Recommended Exposure Index", + "Contrast": "None", + "Scene Capture Type": "Standard", + "Focal Plane Resolution Unit": "cm", + "File Source": "Digital Still Camera (DSC)", + "Sub-Sec Time Digitized": "32", + "Focal Plane X Resolution": "32768/54886891 cm", + "Saturation": "None", + "Max Aperture Value": "f/1.7" + }, + "xmp": { + "crs:ParametricDarks": "0", + "dc:format": "image/jpeg", + "crs:DefringeGreenHueLo": "40", + "crs:SaturationAdjustmentPurple": "0", + "crs:CropConstrainToWarp": "0", + "crs:HasSettings": "True", + "crs:GreenHue": "0", + "photoshop:ColorMode": "3", + "dc:description": [ + "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:SaturationAdjustmentAqua": "0", + "crs:UprightVersion": "151388160", + "crs:SharpenDetail": "25", + "crs:GrainAmount": "0", + "xmp:MetadataDate": "2020-01-02T16:46:06.000Z", + "crs:CameraProfile": "Adobe Standard", + "crs:WhiteBalance": "Custom", + "crs:ToneCurveRed": [ + "0, 0", + "255, 255" + ], + "crs:ColorNoiseReductionDetail": "50", + "crs:Blacks2012": "+17", + "crs:SaturationAdjustmentBlue": "0", + "xmpMM:DerivedFrom": [ + "{'stRef:documentID':'8ACD7691E7D0DB5C6CB40855D7469AAE'}", + "{'stRef:originalDocumentID':'8ACD7691E7D0DB5C6CB40855D7469AAE'}" + ], + "crs:UprightPreview": "False", + "crs:ToneCurveName": "Linear", + "aux:ImageNumber": "46087", + "crs:SplitToningShadowSaturation": "0", + "crs:SplitToningShadowHue": "0", + "photoshop:TransmissionReference": "babyboom_bizet", + "crs:ParametricHighlightSplit": "75", + "crs:PerspectiveRotate": "0.0", + "crs:PerspectiveUpright": "0", + "crs:LuminanceNoiseReductionContrast": "0", + "crs:PerspectiveX": "0.00", + "crs:DefringePurpleHueLo": "30", + "crs:ToneCurveName2012": "Linear", + "crs:ToneCurvePV2012": [ + "0, 0", + "255, 255" + ], + "crs:HueAdjustmentPurple": "0", + "crs:VignetteAmount": "0", + "crs:Exposure2012": "0.00", + "photoshop:AuthorsPosition": "photographer", + "crs:ParametricShadowSplit": "25", + "crs:LuminanceAdjustmentAqua": "0", + "crs:CropAngle": "0", + "crs:LuminanceAdjustmentPurple": "0", + "crs:SplitToningBalance": "0", + "crs:SaturationAdjustmentMagenta": "0", + "crs:Sharpness": "40", + "crs:HueAdjustmentAqua": "0", + "dc:subject": [ + "reborn", + "reborn doll", + "reborn dolls", + "dolls", + "baby", + "bebes", + "nouveau ne", + "new born", + "doll", + "poupee", + "phenomene", + "phenomenon", + "realism", + "hyper-realistic", + "reborn artist", + "mother", + "parents", + "mère", + "parents", + "fabrication", + "reborn workshop", + "silicon", + "reborn silicon" + ], + "photoshop:CaptionWriter": "photographer", + "crs:ColorNoiseReductionSmoothness": "50", + "crs:BlueHue": "0", + "crs:HasCrop": "True", + "crs:DefringeGreenHueHi": "60", + "crs:SaturationAdjustmentOrange": "0", + "crs:HueAdjustmentYellow": "0", + "crs:SplitToningHighlightHue": "0", + "crs:PerspectiveScale": "100", + "crs:HueAdjustmentGreen": "0", + "crs:Temperature": "3301", + "crs:ConvertToGrayscale": "False", + "crs:RawFileName": "_9DS9316.NEF", + "crs:ParametricLights": "0", + "aux:Lens": "35.0 mm f/1.8", + "crs:UprightFocalMode": "0", + "crs:ToneCurvePV2012Green": [ + "0, 0", + "255, 255" + ], + "crs:LuminanceNoiseReductionDetail": "50", + "crs:ParametricHighlights": "0", + "crs:OverrideLookVignette": "False", + "crs:SharpenEdgeMasking": "32", + "crs:ToneCurve": [ + "0, 0", + "255, 255" + ], + "crs:SaturationAdjustmentRed": "0", + "crs:Whites2012": "0", + "crs:Clarity2012": "0", + "crs:UprightFocalLength35mm": "35", + "xmp:ModifyDate": "2020-01-02T16:46:06.000Z", + "crs:UprightTransformCount": "6", + "crs:UprightCenterNormX": "0.5", + "xmpRights:UsageTerms": [ + "All Rights reserved", + [ + "{'xml:lang':'x-default'}" + ] + ], + "photoshop:City": "Valencia", + "xmpMM:OriginalDocumentID": "8ACD7691E7D0DB5C6CB40855D7469AAE", + "xmp:Rating": "5", + "crs:ToneCurvePV2012Blue": [ + "0, 0", + "255, 255" + ], + "crs:Look": [ + "{'crs:Parameters':['{'crs:ToneCurvePV2012Blue':['0, 0','255, 255']}','{'crs:ToneCurvePV2012Red':['0, 0','255, 255']}','{'crs:ToneCurvePV2012':['0, 0','22, 16','40, 35','127, 127','224, 230','240, 246','255, 255']}','{'crs:ToneCurvePV2012Green':['0, 0','255, 255']}']}", + "{'crs:Group':['Profiles']}", + "{'crs:Parameters':['{'crs:LookTable':'E1095149FDB39D7A057BAB208837E2E1'}','{'crs:CameraProfile':'Adobe Standard'}','{'crs:ConvertToGrayscale':'False'}','{'crs:Version':'11.3'}','{'crs:ProcessVersion':'11.0'}']}", + "{'crs:SupportsMonochrome':'false'}", + "{'crs:UUID':'B952C231111CD8E0ECCF14B86BAA7077'}", + "{'crs:Amount':'1.000000'}", + "{'crs:SupportsOutputReferred':'false'}", + "{'crs:Name':'Adobe Color'}", + "{'crs:SupportsAmount':'false'}", + "{'crs:Group':[['{'xml:lang':'x-default'}']]}" + ], + "crs:LuminanceAdjustmentRed": "0", + "crs:DefringeGreenAmount": "0", + "crs:PerspectiveVertical": "0", + "crs:HueAdjustmentBlue": "0", + "crs:PerspectiveY": "0.00", + "crs:LensManualDistortionAmount": "0", + "crs:AutoLateralCA": "0", + "crs:LuminanceSmoothing": "17", + "crs:CameraProfileDigest": "5F02E85A544BE2CCC1DFAA73D311C57C", + "crs:Contrast2012": "0", + "photoshop:DateCreated": "2019-04-27T19:46:03.032Z", + "photoshop:Credit": "Didier Bizet", + "crs:SaturationAdjustmentYellow": "0", + "xmp:CreateDate": "2019-04-27T19:46:03.000Z", + "crs:LensProfileSetup": "LensDefaults", + "crs:Shadows2012": "+19", + "crs:UprightCenterMode": "0", + "crs:CropRight": "0.884268", + "crs:LensProfileEnable": "0", + "aux:LensID": "165", + "aux:ApproximateFocusDistance": "79/100", + "crs:Tint": "+11", + "crs:ParametricMidtoneSplit": "50", + "xmpRights:WebStatement": "hanslucas.com", + "crs:PerspectiveHorizontal": "0", + "crs:ShadowTint": "0", + "crs:UprightCenterNormY": "0.5", + "dc:rights": [ + "Didier Bizet", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:HueAdjustmentMagenta": "0", + "xmp:CreatorTool": "Adobe Photoshop CC 2019 (Macintosh)", + "aux:LensInfo": "350/10 350/10 18/10 18/10", + "crs:ToneCurveBlue": [ + "0, 0", + "255, 255" + ], + "crs:LuminanceAdjustmentMagenta": "0", + "photoshop:LegacyIPTCDigest": "CFA151707B37962499516E28485C8B15", + "crs:ToneCurveGreen": [ + "0, 0", + "255, 255" + ], + "crs:BlueSaturation": "0", + "crs:LuminanceAdjustmentYellow": "0", + "crs:AlreadyApplied": "True", + "crs:DefringePurpleHueHi": "70", + "crs:PostCropVignetteAmount": "0", + "xmpMM:DocumentID": "xmp.did:d080270a-0a88-4df3-b158-8c47f73340ef", + "crs:PerspectiveAspect": "0", + "crs:LuminanceAdjustmentGreen": "0", + "crs:Dehaze": "0", + "crs:RedSaturation": "0", + "xmpRights:Marked": "True", + "crs:SaturationAdjustmentGreen": "0", + "dc:creator": [ + "Didier Bizet" + ], + "crs:LuminanceAdjustmentOrange": "0", + "crs:Version": "11.3", + "crs:ToneCurvePV2012Red": [ + "0, 0", + "255, 255" + ], + "crs:Texture": "0", + "Iptc4xmpCore:CreatorContactInfo": [ + "{'Iptc4xmpCore:CiUrlWork':'http://hanslucas.com/dbizet/photo ; http://didierbizet.com'}", + "{'Iptc4xmpCore:CiEmailWork':'dbizet@orange.fr'}", + "{'Iptc4xmpCore:CiTelWork':'0033675952033'}", + "{'Iptc4xmpCore:CiAdrCity':'Paris'}", + "{'Iptc4xmpCore:CiAdrPcode':'75019'}", + "{'Iptc4xmpCore:CiAdrCtry':'France'}", + "{'Iptc4xmpCore:CiAdrExtadr':'30 rue clavel'}" + ], + "crs:ToneMapStrength": "0", + "crs:UprightFourSegmentsCount": "0", + "crs:CropTop": "0.177282", + "crs:ParametricShadows": "0", + "crs:HueAdjustmentRed": "0", + "crs:GreenSaturation": "0", + "crs:CropLeft": "0.088464", + "crs:RedHue": "0", + "dc:title": [ + "baby boom", + [ + "{'xml:lang':'x-default'}" + ] + ], + "crs:SplitToningHighlightSaturation": "0", + "aux:SerialNumber": "6085734", + "crs:DefringePurpleAmount": "0", + "crs:Saturation": "0", + "crs:HueAdjustmentOrange": "0", + "crs:LuminanceAdjustmentBlue": "0", + "xmpMM:PreservedFileName": "_9DS9316.NEF", + "xmpMM:History": [ + [ + "{'stEvt:parameters':'converted from image/x-nikon-nef to image/jpeg, saved to new location'}", + "{'stEvt:action':'derived'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:softwareAgent':'Adobe Photoshop Lightroom Classic 8.3 (Macintosh)'}", + "{'stEvt:instanceID':'xmp.iid:d080270a-0a88-4df3-b158-8c47f73340ef'}", + "{'stEvt:changed':'/'}", + "{'stEvt:when':'2019-06-14T13:15:17+02:00'}" + ], + [ + "{'stEvt:action':'saved'}", + "{'stEvt:when':'2020-01-02T17:46:06+01:00'}", + "{'stEvt:softwareAgent':'Adobe Photoshop CC 2019 (Macintosh)'}", + "{'stEvt:changed':'/'}", + "{'stEvt:instanceID':'xmp.iid:15644f3f-1e61-4033-a456-8b43605ed09b'}" + ] + ], + "Iptc4xmpCore:CountryCode": "ESP", + "crs:Highlights2012": "0", + "crs:CropBottom": "0.973086", + "photoshop:Country": "spain", + "crs:ProcessVersion": "11.0", + "photoshop:Source": "hans lucas", + "crs:Vibrance": "0", + "crs:SharpenRadius": "+1.0", + "xmpMM:InstanceID": "xmp.iid:15644f3f-1e61-4033-a456-8b43605ed09b", + "crs:ColorNoiseReduction": "25" + }, + "icc": {}, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "photometricInterpretation": "RGB", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [ + "reborn" + ], + "usageRights": { + "category": "chargeable", + "restrictions": "PHOTO ESSAY ONLY" + }, + "lastModified": "2020-02-24T09:08:18.715Z" + }, + "metadata": { + "dateTaken": "2019-04-27T19:46:03.320Z", + "description": "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + "credit": "Didier Bizet", + "byline": "Didier Bizet", + "bylineTitle": "photographer", + "copyright": "Didier Bizet", + "suppliersReference": "babyboom_bizet", + "source": "hans lucas", + "keywords": [ + "mother", + "fabrication", + "reborn dolls", + "phenomenon", + "doll", + "poupee", + "dolls", + "parents", + "hyper-realistic", + "reborn workshop", + "mère", + "realism", + "phenomene", + "reborn artist", + "reborn", + "new born", + "silicon", + "bebes", + "nouveau ne", + "baby", + "reborn doll", + "reborn silicon" + ], + "city": "Valencia", + "country": "Spain", + "subjects": [], + "peopleInImage": [] + }, + "originalMetadata": { + "dateTaken": "2019-04-27T19:46:03.320Z", + "description": "Azaraya, one of Alicia Jiménez’s reborns. It is a model of the famous kit designer Bonnie Braun that Alicia has redesigned. Kits are purchased at the factory or directly through some kit creators. A kit sold includes a head, a bust, legs and arms that reborn artists assemble and paint. Valencia April 2019.\rAzaraya, un des reborns d’Alicia Jiménez. C’est un modèle de la célèbre créatrice de kits Bonnie Braun qu’Alicia a reborné. Les kits sont achetés en usine ou directement via certains créateurs de kits. Un kit vendu rassemble une tête, un buste, des jambes et bras que les reborn artists assemblent et peignent. Valencia avril 2019.", + "credit": "Didier Bizet", + "byline": "Didier Bizet", + "bylineTitle": "photographer", + "copyright": "Didier Bizet", + "suppliersReference": "babyboom_bizet", + "source": "hans lucas", + "keywords": [ + "mother", + "fabrication", + "reborn dolls", + "phenomenon", + "doll", + "poupee", + "dolls", + "parents", + "hyper-realistic", + "reborn workshop", + "mère", + "realism", + "phenomene", + "reborn artist", + "reborn", + "new born", + "silicon", + "bebes", + "nouveau ne", + "baby", + "reborn doll", + "reborn silicon" + ], + "city": "Valencia", + "country": "Spain", + "subjects": [], + "peopleInImage": [] + }, + "usageRights": { + "category": "chargeable", + "restrictions": "PHOTO ESSAY ONLY" + }, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [], + "userMetadataLastModified": "2020-02-24T09:08:18.715Z" + } + }, + { + "_index": "images_2024-04-03_15-45-31_e37e3ac", + "_id": "1e8491b56759fdf85c28dd0b02c593ba468ba3b9", + "_score": 1, + "_source": { + "id": "1e8491b56759fdf85c28dd0b02c593ba468ba3b9", + "uploadTime": "2015-07-27T17:37:11.000Z", + "uploadedBy": "DEV Ingest Script", + "identifiers": { + "picdarurn": "PD*1693653" + }, + "uploadInfo": {}, + "source": { + "file": "http://media-service-prod-imagebucket-1luk2yux3owkh.s3.amazonaws.com/1/e/8/4/9/1/1e8491b56759fdf85c28dd0b02c593ba468ba3b9", + "size": 301115, + "mimeType": "image/jpeg", + "dimensions": { + "width": 1534, + "height": 2160 + } + }, + "thumbnail": { + "file": "http://media-service-prod-thumbbucket-1pyxt0kyxzho.s3.amazonaws.com/1/e/8/4/9/1/1e8491b56759fdf85c28dd0b02c593ba468ba3b9", + "size": 17159, + "mimeType": "image/jpeg", + "dimensions": { + "width": 182, + "height": 256 + } + }, + "fileMetadata": { + "iptc": { + "By-line Title": "Atlas Photography", + "Country/Primary Location Name": "United States", + "Category": "ACE", + "Country/Primary Location Code": "USA", + "Copyright Notice": "Atlas Photography", + "Supplemental Category(s)": "ENT", + "Application Record Version": "2", + "Caption/Abstract": "Casey Chaos, lead singert with the group Amen on the Nautica Stage, in Cleveland Ohio, USA.\r\n\r\n\"Amen's more pissed off than we ever were.\" So says former Sex Pistol Steve Jones of L.A. punk band Amen. The group took shape in the mid-'90s when frontman Casey Chaos hooked up with guitarist Paul Fig and began experimenting with their raw, visceral sound and in-your-face lyrics. Later adding Larkin on drums, Tumor on bass and ex-Snot guitarist Sonny Mayo on guitar, the lineup of Amen was complete. \r\n\r\nThe band signed to Ross Robinson's I Am Records, an imprint of Roadrunner, to record their self -- titled debut. \"Amen\", produced by Robinson, was released in 1999, followed by Amen's sophomore effort, We Have Come For Your Parents, the following year. \r\n\r\nDate Taken: 19/08/2001\r\nPicture Credit: Amy L Weiser/Atlas Photography\r\n\r\nAtlas Photography\r\n20 The Limes\r\nBurniston\r\nNorth Yorkshire\r\nYO13 OEE - England\r\n\r\nTel: +44(0)1723 870659\r\nEmail: atlasphotography@ukonline.co.uk", + "Credit": "Amy L Weiser/Atlas Photography", + "Source": "Atlas Photography", + "City": "Cleveland", + "Keywords": "controversial;vertical;upright;thrash;stage;singing;rock;portrait;playing;performing;outrageous;music;metal;loud;live;heavy", + "By-line": "Atlas Photography", + "Urgency": "48", + "Headline": "Casey Chaos of the group Amen", + "Province/State": "Ohio", + "Object Name": "Casey Chaos of the group Amen", + "Caption Writer/Editor": "SN", + "Date Created": "2001:08:19" + }, + "exif": {}, + "exifSub": {}, + "xmp": {}, + "icc": { + "Profile Size": "3144", + "Media White Point": "(0.9505, 1, 1.0891)", + "Device model": "sRGB", + "Red Colorant": "(0.4361, 0.2225, 0.0139)", + "Blue TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Viewing Conditions Description": "Reference Viewing Condition in IEC61966-2.1", + "Luminance": "(76.0365, 80, 87.1246)", + "Viewing Conditions": "view (0x76696577): 36 bytes", + "Version": "2.1.0", + "Tag Count": "17", + "Profile Description": "sRGB IEC61966-2.1", + "Green Colorant": "(0.3851, 0.7169, 0.0971)", + "XYZ values": "0.964 1 0.825", + "Signature": "acsp", + "Device manufacturer": "IEC", + "Device Mfg Description": "IEC http://www.iec.ch", + "Green TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Red TRC": "REDACTED (value longer than 5000 characters, please refer to the metadata stored in the file itself)", + "Blue Colorant": "(0.1431, 0.0606, 0.7141)", + "Color space": "RGB", + "Primary Platform": "Microsoft Corporation", + "Profile Connection Space": "XYZ", + "CMM Type": "Lino", + "Class": "Display Device", + "Measurement": "1931 2° Observer, Backing (0, 0, 0), Geometry Unknown, Flare 1%, Illuminant D65", + "Media Black Point": "(0, 0, 0)", + "Profile Date/Time": "1998:02:09 06:49:00", + "Device Model Description": "IEC 61966-2.1 Default RGB colour space - sRGB", + "Profile Copyright": "Copyright (c) 1998 Hewlett-Packard Company", + "Technology": "CRT" + }, + "getty": {}, + "colourModel": "RGB", + "colourModelInformation": { + "hasAlpha": "false", + "colorType": "TrueColor", + "bitsPerSample": "8" + } + }, + "userMetadata": { + "archived": false, + "labels": [], + "metadata": { + "description": "Casey Chaos, lead singert with the group Amen on the Nautica Stage, in Cleveland Ohio, USA.\n \n \"Amen's more pissed off than we ever were.\" So says former Sex Pistol Steve Jones of L.A. punk band Amen. The group took shape in the mid-'90s when frontman Casey Chaos hooked up with guitarist Paul Fig and began experimenting with their raw, visceral sound and in-your-face lyrics. Later adding Larkin on drums, Tumor on bass and ex-Snot guitarist Sonny Mayo on guitar, the lineup of Amen was complete. \n \n The band signed to Ross Robinson's I Am Records, an imprint of Roadrunner, to record their self -- titled debut. \"Amen\", produced by Robinson, was released in 1999, followed by Amen's sophomore effort, We Have Come For Your Parents, the following year. \n \n Date Taken: 19/08/2001\n Picture Credit: Amy L Weiser/Atlas Photography\n \n Atlas Photography\n 20 The Limes\n Burniston\n North Yorkshire\n YO13 OEE - England\n \n Tel: +44(0)1723 870659\n Email: atlasphotography@ukonline.co.uk\n " + } + }, + "metadata": { + "description": "Casey Chaos, lead singert with the group Amen on the Nautica Stage, in Cleveland Ohio, USA.\n \n \"Amen's more pissed off than we ever were.\" So says former Sex Pistol Steve Jones of L.A. punk band Amen. The group took shape in the mid-'90s when frontman Casey Chaos hooked up with guitarist Paul Fig and began experimenting with their raw, visceral sound and in-your-face lyrics. Later adding Larkin on drums, Tumor on bass and ex-Snot guitarist Sonny Mayo on guitar, the lineup of Amen was complete. \n \n The band signed to Ross Robinson's I Am Records, an imprint of Roadrunner, to record their self -- titled debut. \"Amen\", produced by Robinson, was released in 1999, followed by Amen's sophomore effort, We Have Come For Your Parents, the following year. \n \n Date Taken: 19/08/2001\n Picture Credit: Amy L Weiser/Atlas Photography\n \n Atlas Photography\n 20 The Limes\n Burniston\n North Yorkshire\n YO13 OEE - England\n \n Tel: +44(0)1723 870659\n Email: atlasphotography@ukonline.co.uk\n ", + "credit": "Amy L Weiser", + "byline": "Atlas Photography", + "bylineTitle": "Atlas Photography", + "title": "Casey Chaos of the group Amen", + "copyright": "Atlas Photography", + "suppliersReference": "Casey Chaos of the group Amen", + "source": "Atlas Photography", + "keywords": [ + "rock", + "stage", + "loud", + "music", + "portrait", + "performing", + "vertical", + "heavy", + "live", + "outrageous", + "playing", + "thrash", + "metal", + "controversial", + "singing", + "upright" + ], + "city": "Cleveland", + "state": "Ohio", + "country": "United States", + "subjects": [ + "arts" + ], + "peopleInImage": [] + }, + "originalMetadata": { + "description": "Casey Chaos, lead singert with the group Amen on the Nautica Stage, in Cleveland Ohio, USA.\r\n\r\n\"Amen's more pissed off than we ever were.\" So says former Sex Pistol Steve Jones of L.A. punk band Amen. The group took shape in the mid-'90s when frontman Casey Chaos hooked up with guitarist Paul Fig and began experimenting with their raw, visceral sound and in-your-face lyrics. Later adding Larkin on drums, Tumor on bass and ex-Snot guitarist Sonny Mayo on guitar, the lineup of Amen was complete. \r\n\r\nThe band signed to Ross Robinson's I Am Records, an imprint of Roadrunner, to record their self -- titled debut. \"Amen\", produced by Robinson, was released in 1999, followed by Amen's sophomore effort, We Have Come For Your Parents, the following year. \r\n\r\nDate Taken: 19/08/2001\r\nPicture Credit: Amy L Weiser/Atlas Photography\r\n\r\nAtlas Photography\r\n20 The Limes\r\nBurniston\r\nNorth Yorkshire\r\nYO13 OEE - England\r\n\r\nTel: +44(0)1723 870659\r\nEmail: atlasphotography@ukonline.co.uk", + "credit": "Amy L Weiser", + "byline": "Atlas Photography", + "bylineTitle": "Atlas Photography", + "title": "Casey Chaos of the group Amen", + "copyright": "Atlas Photography", + "suppliersReference": "Casey Chaos of the group Amen", + "source": "Atlas Photography", + "keywords": [ + "rock", + "stage", + "loud", + "music", + "portrait", + "performing", + "vertical", + "heavy", + "live", + "outrageous", + "playing", + "thrash", + "metal", + "controversial", + "singing", + "upright" + ], + "city": "Cleveland", + "state": "Ohio", + "country": "United States", + "subjects": [ + "arts" + ], + "peopleInImage": [] + }, + "usageRights": {}, + "originalUsageRights": {}, + "exports": [], + "usages": [], + "leases": { + "leases": [], + "lastModified": null + }, + "collections": [] + } + } + ] + } + } +} diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index aba3a22c0db..e820a35b329 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -35,6 +35,7 @@ import { } from './testcontainers/constants'; import { generateServiceConfig } from './testcontainers/config'; import { provisionCoreStack } from './testcontainers/provision'; +import { seedElasticsearch } from './testcontainers/seed-elasticsearch'; import { setEnvironment } from './testcontainers/state'; const LOCALSTACK_SERVICES = 'cloudformation,cloudwatch,dynamodb,kinesis,s3,sns,sqs,iam'; @@ -175,6 +176,12 @@ async function globalSetup(): Promise { const grid = await gridBuilder.start(); started.push(grid); + // --- Seed Elasticsearch with image fixtures ------------------------------ + // The app creates the `images` index + `Images_Current` alias on startup; seed once the + // stack is healthy so searches during the tests return the fixture documents. + const esBaseUrl = `http://${elasticsearch.getHost()}:${elasticsearch.getMappedPort(9200)}`; + await seedElasticsearch(esBaseUrl); + // --- CI routing: bundled reverse proxy ----------------------------------- // Locally, the browser reaches the https://*.media. domains via the developer's // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) diff --git a/e2e-tests/testcontainers/seed-elasticsearch.ts b/e2e-tests/testcontainers/seed-elasticsearch.ts new file mode 100644 index 00000000000..7bf02112546 --- /dev/null +++ b/e2e-tests/testcontainers/seed-elasticsearch.ts @@ -0,0 +1,111 @@ +/** + * Seeds Elasticsearch with image fixtures once the Grid stack is up. + * + * The Grid app (media-api/thrall) creates the `images` index with the correct mappings + * and assigns the `Images_Current` alias on startup. This module waits for that alias to + * exist, then bulk-inserts the documents from `fixtures/elasticsearch/images.json` into it + * so searches performed during the tests return them. + * + * The fixture is a raw Elasticsearch search response, so documents live at + * `hits.hits[]` as `{ _id, _source }`. Each document is (re)indexed under its `_id` — the + * stale `_index` recorded in the fixture is ignored in favour of the live alias. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { REPO_ROOT } from './constants'; + +/** Alias the Grid app assigns to the live images index (dev service-config `es.index.aliases.current`). */ +const IMAGES_ALIAS = 'Images_Current'; + +const FIXTURE_PATH = path.join( + REPO_ROOT, + 'e2e-tests', + 'fixtures', + 'elasticsearch', + 'images.json', +); + +interface EsHit { + _id: string; + _source: Record; +} + +interface EsSearchFixture { + body?: { + hits?: { + hits?: EsHit[]; + }; + }; +} + +const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); + +/** + * Poll the alias until the Grid app has created the index and assigned it, so the bulk + * insert below has a valid write target. Throws if it never appears within the timeout. + */ +async function waitForAlias(esBaseUrl: string, timeoutMs = 60_000): Promise { + const deadline = Date.now() + timeoutMs; + let lastStatus = 'no response'; + while (Date.now() < deadline) { + try { + const response = await fetch(`${esBaseUrl}/${IMAGES_ALIAS}`); + if (response.ok) { + return; + } + lastStatus = `HTTP ${response.status}`; + } catch (error) { + lastStatus = error instanceof Error ? error.message : String(error); + } + await sleep(1000); + } + throw new Error( + `Elasticsearch alias '${IMAGES_ALIAS}' was not assigned within ${timeoutMs}ms (last: ${lastStatus})`, + ); +} + +/** + * Load the image fixtures into Elasticsearch. No-op if the fixture has no documents. + */ +export async function seedElasticsearch(esBaseUrl: string): Promise { + const fixture = JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) as EsSearchFixture; + const hits = fixture.body?.hits?.hits ?? []; + + if (hits.length === 0) { + console.warn('No image fixtures found to seed into Elasticsearch'); + return; + } + + await waitForAlias(esBaseUrl); + + // NDJSON bulk body: an index action line (targeting the document's `_id`) followed by + // the source document, repeated for every hit. + const body = hits + .flatMap((hit) => [JSON.stringify({ index: { _id: hit._id } }), JSON.stringify(hit._source)]) + .join('\n') + // The bulk API requires a trailing newline. + .concat('\n'); + + // `refresh=wait_for` makes the documents searchable before this call resolves. + const response = await fetch(`${esBaseUrl}/${IMAGES_ALIAS}/_bulk?refresh=wait_for`, { + method: 'POST', + headers: { 'Content-Type': 'application/x-ndjson' }, + body, + }); + + if (!response.ok) { + throw new Error(`Elasticsearch bulk seed failed: HTTP ${response.status} ${await response.text()}`); + } + + const result = (await response.json()) as { + errors: boolean; + items: Array<{ index?: { error?: unknown } }>; + }; + + if (result.errors) { + const firstError = result.items.find((item) => item.index?.error)?.index?.error; + throw new Error(`Elasticsearch bulk seed reported errors: ${JSON.stringify(firstError)}`); + } + + console.log(`Seeded ${hits.length} image fixture(s) into Elasticsearch alias '${IMAGES_ALIAS}'`); +} From 66be13e4c6dfe0f7b531c45ef3f2fd1e7826258f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 23 Jul 2026 10:18:58 +0100 Subject: [PATCH 223/373] Add a separate Dockerfile for local development --- e2e-tests/images/Dockerfile.dev | 98 +++++++++++++++++ e2e-tests/images/Dockerfile.dev.dockerignore | 10 ++ e2e-tests/images/README.md | 67 +++++++++++- e2e-tests/images/entrypoint.dev.sh | 104 +++++++++++++++++++ 4 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 e2e-tests/images/Dockerfile.dev create mode 100644 e2e-tests/images/Dockerfile.dev.dockerignore create mode 100644 e2e-tests/images/entrypoint.dev.sh diff --git a/e2e-tests/images/Dockerfile.dev b/e2e-tests/images/Dockerfile.dev new file mode 100644 index 00000000000..281e2781759 --- /dev/null +++ b/e2e-tests/images/Dockerfile.dev @@ -0,0 +1,98 @@ +# syntax=docker/dockerfile:1 + +# Development image for the Grid Play services. +# +# Unlike the CI image (images/Dockerfile), this image does NOT stage compiled +# artefacts. Instead it runs the selected services under `sbt /run` +# (Play dev mode), which recompiles changed Scala sources on the next request. +# Kahuna's webpack bundle is rebuilt continuously via `npm run watch`. +# +# The repository is expected to be bind-mounted over /build at run time so host +# edits are reflected live. Build from the repository root: +# DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile.dev -t grid-dev . +# +# Run with the repo and your DEV config mounted: +# docker run --rm \ +# -v "$PWD:/build" \ +# -v "$HOME/.grid:/root/.grid:ro" \ +# -p 9001:9001 -p 9005:9005 -p 9011:9011 \ +# grid-dev +# +# See images/README.md for details. + +# --------------------------------------------------------------------------- +# Single stage: JDK + sbt + Node + native image tooling, with warmed caches. +# --------------------------------------------------------------------------- +FROM eclipse-temurin:11-jdk-jammy + +ARG SBT_VERSION=1.10.3 +ARG NODE_VERSION=22.12.0 +# Provided automatically by BuildKit (e.g. amd64, arm64). +ARG TARGETARCH + +# Base tooling: curl/xz for downloads plus the native tools the image +# processing services need (mirrors images/Dockerfile's runtime stage). +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + xz-utils \ + graphicsmagick \ + imagemagick \ + pngquant \ + libimage-exiftool-perl \ + libgd3; \ + rm -rf /var/lib/apt/lists/* + +# Install sbt from the official release tarball. +RUN set -eux; \ + curl -fLo /tmp/sbt.tgz "https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz"; \ + tar -xzf /tmp/sbt.tgz -C /opt; \ + ln -s /opt/sbt/bin/sbt /usr/local/bin/sbt; \ + rm -f /tmp/sbt.tgz + +# Install Node (pinned to match kahuna's required version) from the official +# tarball so `npm run watch` matches local development. +RUN set -eux; \ + case "${TARGETARCH:-amd64}" in \ + amd64) node_arch=x64 ;; \ + arm64) node_arch=arm64 ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + curl -fLo /tmp/node.tar.xz "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz"; \ + tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner; \ + rm -f /tmp/node.tar.xz; \ + node --version; \ + npm --version + +WORKDIR /build + +# Warm the sbt dependency cache (~/.ivy2, ~/.sbt, ~/.cache/coursier). These live +# under $HOME, so a runtime bind-mount over /build does not shadow them: the +# first runtime compile only needs to compile sources, not re-resolve deps. +# Copying just the build definition keeps this layer cached across source edits. +COPY build.sbt ./ +COPY project/ ./project/ +RUN sbt update + +# Warm Kahuna's node_modules. The dockerignore keeps node_modules out of the +# build context, so this install is not clobbered by the COPY below. +COPY kahuna/package.json kahuna/package-lock.json ./kahuna/ +RUN cd kahuna && npm ci + +# Copy the rest of the source so the image is runnable standalone; a runtime +# bind-mount of the repo over /build overrides this for live editing. +COPY . . + +COPY e2e-tests/images/entrypoint.dev.sh /usr/local/bin/entrypoint.dev.sh +RUN chmod +x /usr/local/bin/entrypoint.dev.sh + +# The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. +ENV AWS_CBOR_DISABLE=true + +# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases +# plus the JDWP debug port used when GRID_DEBUG is set. +EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 5005 + +ENTRYPOINT ["/usr/local/bin/entrypoint.dev.sh"] diff --git a/e2e-tests/images/Dockerfile.dev.dockerignore b/e2e-tests/images/Dockerfile.dev.dockerignore new file mode 100644 index 00000000000..1a1a7091b5b --- /dev/null +++ b/e2e-tests/images/Dockerfile.dev.dockerignore @@ -0,0 +1,10 @@ +# BuildKit uses this file (named .dockerignore) to trim the build +# context for images/Dockerfile.dev. Keeps build artefacts and caches out of the +# context sent to the daemon; they are regenerated at build/run time. +**/target +**/node_modules +**/logs +**/.git +.git +**/.DS_Store +dev/.localstack diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index 9b5c576858e..697ae38d11d 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,7 +1,15 @@ # Grid all-in-one container -`images/Dockerfile` builds a single container that runs eight Grid Play -services plus Kahuna's compiled frontend: +This directory provides **two** single-container images that run eight Grid Play +services plus Kahuna's frontend: + +- **CI image** — `Dockerfile`: stages pre-compiled artefacts and runs them in a + production-style JRE. Used by the e2e-tests testcontainers harness. +- **Local-dev image** — `Dockerfile.dev`: runs the services under `sbt /run` + (Play dev mode) with the repo bind-mounted, so source changes recompile live. + See [Development image (live reload)](#development-image-live-reload) below. + +Both expose the same service ports: | Service | Port | | ----------------- | ---- | @@ -60,3 +68,58 @@ docker run --rm -e GRID_SERVICES="media-api kahuna auth" grid-all ``` If any running service exits, the container stops. + +## Development image (live reload) + +`Dockerfile.dev` is for local development: instead of staging compiled +artefacts it runs the services under `sbt /run` (Play dev mode), so changed +Scala sources are recompiled on the next request. When `kahuna` is selected its +webpack bundle is rebuilt continuously via `npm run watch`. + +Build from the **repository root**: + +```bash +DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile.dev -t grid-dev . +``` + +The build warms the sbt dependency cache (`sbt update`) and Kahuna's +`node_modules` (`npm ci`) so the first run only has to compile sources. + +Run with the repo bind-mounted over `/build` so host edits are picked up live, +and your DEV config mounted at `/root/.grid`: + +```bash +docker run --rm \ + -v "$PWD:/build" \ + -v "$HOME/.grid:/root/.grid:ro" \ + -p 9001:9001 -p 9005:9005 -p 9011:9011 \ + grid-dev +``` + +Because the services run in Play **dev mode**, no `play.http.secret.key` is +required (it is auto-generated), unlike the CI image's production-style run. +Edit a `.scala` file on the host and hit an endpoint to trigger a recompile; +edit a Kahuna asset and the webpack watcher rebuilds `public/dist`. + +### Options + +All configurable via environment variables: + +| Variable | Default | Purpose | +| ----------------------- | -------------------- | -------------------------------------------------------------- | +| `GRID_SERVICES` | `auth media-api kahuna` | Space-separated services to run. | +| `GRID_EXTRA_CONFIG_DIR` | _(unset)_ | Dir of `.conf` overrides (sets `-DextraConfigDir`). | +| `GRID_JAVA_OPTS` | _(unset)_ | Extra JVM options forwarded to the run JVM (as `-J...`). | +| `GRID_DEBUG` | _(unset)_ | If set, opens a JDWP debug server on port `5005` (also `EXPOSE`d). | + +```bash +docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" \ + -e GRID_SERVICES="media-api kahuna" -e GRID_DEBUG=1 \ + -p 9001:9001 -p 9005:9005 -p 5005:5005 \ + grid-dev +``` + +The services still need their backing infrastructure reachable (see the root +`docker-compose.yml`). The image also works against the e2e-tests testcontainers +harness: mount the generated config the same way the CI image does (to +`/root/.grid` and/or `/etc/grid`) and join the same network. diff --git a/e2e-tests/images/entrypoint.dev.sh b/e2e-tests/images/entrypoint.dev.sh new file mode 100644 index 00000000000..0d1c6e63b55 --- /dev/null +++ b/e2e-tests/images/entrypoint.dev.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# +# Development entrypoint for the Grid all-in-one image. +# +# Runs the selected Play services under `sbt /run` (Play dev mode), which +# recompiles changed Scala sources on the next request. When kahuna is selected, +# its webpack bundle is rebuilt continuously via `npm run watch`. +# +# The repository is expected to be bind-mounted at /build so host edits are +# picked up live, e.g.: +# docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" grid-dev +# +# Environment: +# GRID_SERVICES space-separated services to run (default below) +# GRID_JAVA_OPTS extra JVM options, forwarded to the run JVM +# GRID_EXTRA_CONFIG_DIR dir of .conf overrides (sets -DextraConfigDir) +# GRID_DEBUG if non-empty, opens a JDWP debug server on port 5005 + +set -euo pipefail + +REPO=/build +cd "$REPO" + +DEFAULT_SERVICES="auth media-api kahuna" +SERVICES="${GRID_SERVICES:-$DEFAULT_SERVICES}" + +# service -> http port. Informational only: `sbt /run` binds the port from +# playDefaultPort in build.sbt, so these must stay in sync with it. +declare -A PORTS=( + [media-api]=9001 + [thrall]=9002 + [kahuna]=9005 + [cropper]=9006 + [metadata-editor]=9007 + [collections]=9010 + [auth]=9011 + [leases]=9012 +) + +# --- Kahuna frontend watcher ------------------------------------------------ +watch_pid="" +if [[ " $SERVICES " == *" kahuna "* ]]; then + echo "Starting Kahuna webpack watcher..." + ( + cd "$REPO/kahuna" + # node_modules may be shadowed by the host bind-mount; install if absent. + if [[ ! -d node_modules ]]; then + npm install + fi + npm run watch + ) & + watch_pid=$! +fi + +# --- Assemble the sbt run command ------------------------------------------- +run_tasks="" +for svc in $SERVICES; do + if [[ -z "${PORTS[$svc]:-}" ]]; then + echo "Unknown service '$svc' (no port mapping); skipping." >&2 + continue + fi + echo "Will run $svc on port ${PORTS[$svc]}" + run_tasks="$run_tasks ${svc}/run" +done + +if [[ -z "$run_tasks" ]]; then + echo "No valid services selected in GRID_SERVICES='$SERVICES'." >&2 + exit 1 +fi + +# `all` runs the per-service `run` tasks in parallel from a single sbt session. +SBT_COMMAND="all${run_tasks}" + +# --- sbt / JVM options ------------------------------------------------------ +SBT_OPTS="${SBT_OPTS:-}" +if [[ -n "${GRID_EXTRA_CONFIG_DIR:-}" ]]; then + SBT_OPTS="$SBT_OPTS -J-DextraConfigDir=${GRID_EXTRA_CONFIG_DIR}" +fi +if [[ -n "${GRID_DEBUG:-}" ]]; then + SBT_OPTS="$SBT_OPTS -jvm-debug 5005" +fi +if [[ -n "${GRID_JAVA_OPTS:-}" ]]; then + for opt in $GRID_JAVA_OPTS; do + SBT_OPTS="$SBT_OPTS -J${opt}" + done +fi + +shutdown() { + echo "Shutting down dev services..." + [[ -n "$watch_pid" ]] && kill -TERM "$watch_pid" 2>/dev/null || true + [[ -n "${sbt_pid:-}" ]] && kill -TERM "$sbt_pid" 2>/dev/null || true + wait 2>/dev/null || true + exit 0 +} +trap shutdown TERM INT + +echo "Running: sbt $SBT_OPTS \"$SBT_COMMAND\"" + +# Play's dev-mode `run` blocks reading stdin and stops on EOF/Enter. Keep stdin +# open (via a never-ending source) so the services stay up in a non-interactive +# container; shutdown is driven by the SIGTERM/SIGINT trap above. +tail -f /dev/null | sbt $SBT_OPTS "$SBT_COMMAND" & +sbt_pid=$! +wait "$sbt_pid" From 72d7b02c07268a708271f3318c66ea7af1a156c5 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 23 Jul 2026 10:19:55 +0100 Subject: [PATCH 224/373] Translate spec files to bdd features and steps --- .gitignore | 1 + e2e-tests/features/kahuna.feature | 12 + e2e-tests/features/media-api.feature | 10 + e2e-tests/package-lock.json | 413 +++++++++++++++++++++++++ e2e-tests/package.json | 5 +- e2e-tests/playwright.config.ts | 10 +- e2e-tests/{tests => steps}/fixtures.ts | 21 +- e2e-tests/steps/kahuna.steps.ts | 19 ++ e2e-tests/steps/media-api.steps.ts | 24 ++ e2e-tests/tests/example.spec.ts | 38 --- 10 files changed, 509 insertions(+), 44 deletions(-) create mode 100644 e2e-tests/features/kahuna.feature create mode 100644 e2e-tests/features/media-api.feature rename e2e-tests/{tests => steps}/fixtures.ts (67%) create mode 100644 e2e-tests/steps/kahuna.steps.ts create mode 100644 e2e-tests/steps/media-api.steps.ts delete mode 100644 e2e-tests/tests/example.spec.ts diff --git a/.gitignore b/.gitignore index c582278d2dd..e28f06d7bb7 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ image-embedder-lambda/__tests__/embedder/integration/test-data/ image-embedder-lambda/__tests__/embedder/test-data # Playwright +/e2e-tests/.features-gen/ /e2e-tests/test-results/ /e2e-tests/playwright-report/ /e2e-tests/blob-report/ diff --git a/e2e-tests/features/kahuna.feature b/e2e-tests/features/kahuna.feature new file mode 100644 index 00000000000..d99ec3ece8a --- /dev/null +++ b/e2e-tests/features/kahuna.feature @@ -0,0 +1,12 @@ +Feature: Kahuna service availability + + Smoke checks that the Kahuna service, booted by Testcontainers in global-setup, + is serving requests against the provisioned infrastructure. + + Scenario: Healthcheck responds OK + When I request the Kahuna healthcheck endpoint + Then the response is successful + + Scenario: Kahuna serves the application + When I request the Kahuna root path without following redirects + Then the response status is below 500 diff --git a/e2e-tests/features/media-api.feature b/e2e-tests/features/media-api.feature new file mode 100644 index 00000000000..7e0948f3c54 --- /dev/null +++ b/e2e-tests/features/media-api.feature @@ -0,0 +1,10 @@ +Feature: Frontend loads media-api + + The Kahuna SPA reads the media-api URL from a element and + immediately requests the media-api root to discover its hypermedia links. dev-nginx + routes https://api.media. to the media-api port on the grid-all container. + + Scenario: The SPA discovers and requests media-api from the grid-all container + When I open the Grid application + Then the page exposes a media-api URI link + And the media-api is served successfully diff --git a/e2e-tests/package-lock.json b/e2e-tests/package-lock.json index 5693a9a0736..de39c578fd9 100644 --- a/e2e-tests/package-lock.json +++ b/e2e-tests/package-lock.json @@ -15,6 +15,7 @@ "@testcontainers/localstack": "^10.13.2", "@types/node": "^26.1.1", "json5": "^2.2.3", + "playwright-bdd": "^9.2.0", "testcontainers": "^10.13.2" } }, @@ -371,6 +372,139 @@ "dev": true, "license": "Apache-2.0" }, + "node_modules/@colors/colors": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", + "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", + "dev": true, + "license": "MIT", + "optional": true, + "engines": { + "node": ">=0.1.90" + } + }, + "node_modules/@cucumber/ci-environment": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/ci-environment/-/ci-environment-13.0.0.tgz", + "integrity": "sha512-cs+3NzfNkGbcmHPddjEv4TKFiBpZRQ6WJEEufB9mw+ExS22V/4R/zpDSEG+fsJ/iSNCd6A2sATdY8PFOyY3YnA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@cucumber/cucumber-expressions": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/cucumber-expressions/-/cucumber-expressions-19.0.0.tgz", + "integrity": "sha512-4FKoOQh2Uf6F6/Ln+1OxuK8LkTg6PyAqekhf2Ix8zqV2M54sH+m7XNJNLhOFOAW/t9nxzRbw2CcvXbCLjcvHZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "regexp-match-indices": "1.0.2" + } + }, + "node_modules/@cucumber/gherkin": { + "version": "39.1.0", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-39.1.0.tgz", + "integrity": "sha512-pqmSO2bUWxJm3TbNrKXlDaHjL6c77+ez9kWmfCd9oRPeTRPEVH3spZvpAqdXYWOZYSNYwWFCAAeZ4RGpkauNoQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/messages": ">=31.0.0 <33" + } + }, + "node_modules/@cucumber/gherkin-utils": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin-utils/-/gherkin-utils-11.0.0.tgz", + "integrity": "sha512-LJ+s4+TepHTgdKWDR4zbPyT7rQjmYIcukTwNbwNwgqr6i8Gjcmzf6NmtbYDA19m1ZFg6kWbFsmHnj37ZuX+kZA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/gherkin": "^38.0.0", + "@cucumber/messages": "^32.0.0", + "@teppeis/multimaps": "3.0.0", + "commander": "14.0.2", + "source-map-support": "^0.5.21" + }, + "bin": { + "gherkin-utils": "bin/gherkin-utils" + } + }, + "node_modules/@cucumber/gherkin-utils/node_modules/@cucumber/gherkin": { + "version": "38.0.0", + "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-38.0.0.tgz", + "integrity": "sha512-duEXK+KDfQUzu3vsSzXjkxQ2tirF5PRsc1Xrts6THKHJO6mjw4RjM8RV+vliuDasmhhrmdLcOcM7d9nurNTJKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/messages": ">=31.0.0 <33" + } + }, + "node_modules/@cucumber/gherkin-utils/node_modules/commander": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", + "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/@cucumber/html-formatter": { + "version": "23.1.0", + "resolved": "https://registry.npmjs.org/@cucumber/html-formatter/-/html-formatter-23.1.0.tgz", + "integrity": "sha512-DcCSFoGs6jbwzXPgX1CwgJKEE+ZMcIEzq/0Memg0o24maNn9NJizBFHmoFWG4iv/OxHza+mvc+56cTHetfHndw==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@cucumber/messages": ">=18" + } + }, + "node_modules/@cucumber/junit-xml-formatter": { + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/@cucumber/junit-xml-formatter/-/junit-xml-formatter-0.13.3.tgz", + "integrity": "sha512-w9ujOxiuKDtU6fLzJz+wp4Sgp5Xu6ba7ls00LHJccVmQU0Ba7zs+AHnv3iIgPjKZAQe1w8x93dr8Gaubh7Vqkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/query": "^15.0.1", + "@teppeis/multimaps": "^3.0.0", + "luxon": "^3.5.0", + "xmlbuilder": "^15.1.1" + }, + "peerDependencies": { + "@cucumber/messages": "*" + } + }, + "node_modules/@cucumber/messages": { + "version": "32.3.1", + "resolved": "https://registry.npmjs.org/@cucumber/messages/-/messages-32.3.1.tgz", + "integrity": "sha512-yNQq1KoXRYaEKrWMFmpUQX7TdeQuU9jeGgJAZ3dArTsC/T4NpJ6DnqaJIIgwPnz/wtQIQTNX7/h0rOuF5xY4qQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "class-transformer": "0.5.1", + "reflect-metadata": "0.2.2" + } + }, + "node_modules/@cucumber/query": { + "version": "15.0.1", + "resolved": "https://registry.npmjs.org/@cucumber/query/-/query-15.0.1.tgz", + "integrity": "sha512-FMfT3orJblRsOxvU2doECBvQmauizYlj+5JsM8atAKKPbnQTj7v2/OrnuykvQpfZNBf19DYbRq1e832vllRP/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@teppeis/multimaps": "3.0.0", + "lodash.sortby": "^4.7.0" + }, + "peerDependencies": { + "@cucumber/messages": "*" + } + }, + "node_modules/@cucumber/tag-expressions": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/@cucumber/tag-expressions/-/tag-expressions-9.1.0.tgz", + "integrity": "sha512-bvHjcRFZ+J1TqIa9eFNO1wGHqwx4V9ZKV3hYgkuK/VahHx73uiP4rKV3JVrvWSMrwrFvJG6C8aEwnCWSvbyFdQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@fastify/busboy": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", @@ -642,6 +776,16 @@ "node": ">=18.0.0" } }, + "node_modules/@teppeis/multimaps": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@teppeis/multimaps/-/multimaps-3.0.0.tgz", + "integrity": "sha512-ID7fosbc50TbT0MK0EG12O+gAP3W3Aa/Pz4DaTtQtEvlc9Odaqi0de+xuZ7Li2GtK4HzEX7IuRWS/JmZLksR3Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/@testcontainers/localstack": { "version": "10.28.0", "resolved": "https://registry.npmjs.org/@testcontainers/localstack/-/localstack-10.28.0.tgz", @@ -1065,6 +1209,13 @@ "node": ">=8.0.0" } }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, "node_modules/buildcheck": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", @@ -1092,6 +1243,74 @@ "dev": true, "license": "ISC" }, + "node_modules/class-transformer": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz", + "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-table3": { + "version": "0.6.5", + "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.5.tgz", + "integrity": "sha512-+W/5efTR7y5HRD7gACw9yQjqMVvEMLBHmboM/kPWam+H+Hmyrgjh6YncVKK122YZkXrLudzTuAukUw9FnMf7IQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "string-width": "^4.2.0" + }, + "engines": { + "node": "10.* || >= 12.*" + }, + "optionalDependencies": { + "@colors/colors": "1.5.0" + } + }, + "node_modules/cli-table3/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-table3/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/cli-table3/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-table3/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -1206,6 +1425,16 @@ "dev": true, "license": "MIT" }, + "node_modules/commander": { + "version": "13.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz", + "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/compress-commons": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", @@ -1484,6 +1713,24 @@ "dev": true, "license": "MIT" }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/foreground-child": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", @@ -1729,6 +1976,13 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==", + "dev": true, + "license": "MIT" + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -1743,6 +1997,43 @@ "dev": true, "license": "ISC" }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/mime-db": { + "version": "1.54.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.54.0.tgz", + "integrity": "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-3.0.2.tgz", + "integrity": "sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": "^1.54.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/minimatch": { "version": "9.0.9", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", @@ -1858,6 +2149,19 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/picomatch": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/playwright": { "version": "1.61.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", @@ -1877,6 +2181,40 @@ "fsevents": "2.3.2" } }, + "node_modules/playwright-bdd": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/playwright-bdd/-/playwright-bdd-9.2.0.tgz", + "integrity": "sha512-1tBTmo4DpOhLsc+A6PB4isWO3DHKb4BQ3Tzw5+ze/PmgBW9W2m9c+nc0TPy2nByWJtw0gKSfMoexyBR+y82+pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cucumber/ci-environment": "^13.0.0", + "@cucumber/cucumber-expressions": "19.0.0", + "@cucumber/gherkin": "^39.1.0", + "@cucumber/gherkin-utils": "^11.0.0", + "@cucumber/html-formatter": "^23.1.0", + "@cucumber/junit-xml-formatter": "^0.13.3", + "@cucumber/messages": "^32.3.1", + "@cucumber/query": "^15.0.1", + "@cucumber/tag-expressions": "^9.1.0", + "cli-table3": "0.6.5", + "commander": "^13.1.0", + "mime-types": "^3.0.2", + "tinyglobby": "0.2.17" + }, + "bin": { + "bddgen": "dist/cli/index.js" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/vitalets" + }, + "peerDependencies": { + "@playwright/test": ">=1.44" + } + }, "node_modules/playwright-core": { "version": "1.61.1", "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", @@ -2018,6 +2356,33 @@ "node": ">=10" } }, + "node_modules/reflect-metadata": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz", + "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/regexp-match-indices": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regexp-match-indices/-/regexp-match-indices-1.0.2.tgz", + "integrity": "sha512-DwZuAkt8NF5mKwGGER1EGh2PRqyvhRhhLviH+R8y8dIuaQROlUfXjt4s9ZTXstIsSkptf06BSvwcEmmfheJJWQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "regexp-tree": "^0.1.11" + } + }, + "node_modules/regexp-tree": { + "version": "0.1.27", + "resolved": "https://registry.npmjs.org/regexp-tree/-/regexp-tree-0.1.27.tgz", + "integrity": "sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==", + "dev": true, + "license": "MIT", + "bin": { + "regexp-tree": "bin/regexp-tree" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -2102,6 +2467,27 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", @@ -2347,6 +2733,23 @@ "b4a": "^1.6.4" } }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, "node_modules/tmp": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.7.tgz", @@ -2534,6 +2937,16 @@ "dev": true, "license": "ISC" }, + "node_modules/xmlbuilder": { + "version": "15.1.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-15.1.1.tgz", + "integrity": "sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/e2e-tests/package.json b/e2e-tests/package.json index a8d3109dac0..383dfebe8a7 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -4,8 +4,8 @@ "description": "", "main": "index.js", "scripts": { - "test": "playwright test", - "test:headed": "playwright test --headed", + "test": "bddgen && playwright test", + "test:headed": "bddgen && playwright test --headed", "test:report": "playwright show-report" }, "keywords": [], @@ -18,6 +18,7 @@ "@testcontainers/localstack": "^10.13.2", "@types/node": "^26.1.1", "json5": "^2.2.3", + "playwright-bdd": "^9.2.0", "testcontainers": "^10.13.2" } } diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index a736a18a0c7..d3579ebaeb1 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -1,6 +1,14 @@ import { defineConfig, devices } from '@playwright/test'; +import { defineBddConfig } from 'playwright-bdd'; import { KAHUNA_PORT } from './testcontainers/constants'; +/* Generate Playwright test files from the Gherkin feature files and step definitions. + Run via `bddgen` (see package.json scripts) before `playwright test`. */ +const testDir = defineBddConfig({ + features: './features/**/*.feature', + steps: './steps/**/*.ts', +}); + /** * Read environment variables from file. * https://github.com/motdotla/dotenv @@ -13,7 +21,7 @@ import { KAHUNA_PORT } from './testcontainers/constants'; * See https://playwright.dev/docs/test-configuration. */ export default defineConfig({ - testDir: './tests', + testDir, /* Run tests in files in parallel */ fullyParallel: true, /* Fail the build on CI if you accidentally left test.only in the source code. */ diff --git a/e2e-tests/tests/fixtures.ts b/e2e-tests/steps/fixtures.ts similarity index 67% rename from e2e-tests/tests/fixtures.ts rename to e2e-tests/steps/fixtures.ts index 12c22296cdc..ca6d986f83f 100644 --- a/e2e-tests/tests/fixtures.ts +++ b/e2e-tests/steps/fixtures.ts @@ -1,4 +1,6 @@ -import { test as base, expect } from '@playwright/test'; +import { expect } from '@playwright/test'; +import type { APIResponse, Response } from '@playwright/test'; +import { test as base, createBdd } from 'playwright-bdd'; import * as fs from 'fs'; import { DOMAIN, URLS_FILE } from '../testcontainers/constants'; @@ -22,19 +24,32 @@ function readGridUrls(): GridUrls { return JSON.parse(fs.readFileSync(URLS_FILE, 'utf8')) as GridUrls; } +/** + * Mutable per-scenario state shared between steps (e.g. the response captured in a + * `When` step and asserted on in a `Then` step). + */ +interface TestContext { + response?: APIResponse; + mediaApiResponse?: Promise; +} + /** * Test fixture that exposes the Grid service URLs resolved by `global-setup.ts` and * points `baseURL` at Kahuna's fixed host port for API-level `request` tests. Browser - * tests should navigate to `KAHUNA_APP_URL` so the page origin is a real Grid domain + * steps should navigate to `KAHUNA_APP_URL` so the page origin is a real Grid domain * (required for the services' CORS origins to match) and dev-nginx does the routing. */ -export const test = base.extend<{ gridUrls: GridUrls }>({ +export const test = base.extend<{ gridUrls: GridUrls; testContext: TestContext }>({ gridUrls: async ({}, use) => { await use(readGridUrls()); }, baseURL: async ({}, use) => { await use(readGridUrls().kahuna); }, + testContext: async ({}, use) => { + await use({}); + }, }); +export const { Given, When, Then, Before, After } = createBdd(test); export { expect }; diff --git a/e2e-tests/steps/kahuna.steps.ts b/e2e-tests/steps/kahuna.steps.ts new file mode 100644 index 00000000000..c81f964c8e9 --- /dev/null +++ b/e2e-tests/steps/kahuna.steps.ts @@ -0,0 +1,19 @@ +import { When, Then, expect } from './fixtures'; + +When('I request the Kahuna healthcheck endpoint', async ({ request, testContext }) => { + testContext.response = await request.get('/management/healthcheck'); +}); + +Then('the response is successful', async ({ testContext }) => { + expect(testContext.response?.ok()).toBeTruthy(); +}); + +When('I request the Kahuna root path without following redirects', async ({ request, testContext }) => { + // The root path may redirect to auth; any non-server-error response proves Kahuna + // is serving requests against the provisioned infrastructure. + testContext.response = await request.get('/', { maxRedirects: 0 }); +}); + +Then('the response status is below {int}', async ({ testContext }, status: number) => { + expect(testContext.response?.status()).toBeLessThan(status); +}); diff --git a/e2e-tests/steps/media-api.steps.ts b/e2e-tests/steps/media-api.steps.ts new file mode 100644 index 00000000000..cbc15b3d53d --- /dev/null +++ b/e2e-tests/steps/media-api.steps.ts @@ -0,0 +1,24 @@ +import { When, Then, expect, KAHUNA_APP_URL } from './fixtures'; + +When('I open the Grid application', async ({ page, testContext }) => { + // Register the media-api listener before navigating so the request the SPA fires on + // load is captured. dev-nginx routes https://api.media. to the media-api port + // on the grid-all container, so this request should be served (200) by the container. + testContext.mediaApiResponse = page.waitForResponse( + (response) => + response.url().startsWith('https://api.media.') && response.request().method() === 'GET', + { timeout: 60_000 }, + ); + + await page.goto(KAHUNA_APP_URL); +}); + +Then('the page exposes a media-api URI link', async ({ page }) => { + const mediaApiLink = await page.getAttribute('link[rel="media-api-uri"]', 'href'); + expect(mediaApiLink).toContain('api.media.'); +}); + +Then('the media-api is served successfully', async ({ testContext }) => { + const response = await testContext.mediaApiResponse!; + expect(response.status()).toBe(200); +}); diff --git a/e2e-tests/tests/example.spec.ts b/e2e-tests/tests/example.spec.ts deleted file mode 100644 index 3966b095907..00000000000 --- a/e2e-tests/tests/example.spec.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { test, expect, KAHUNA_APP_URL } from './fixtures'; - -/** - * Smoke tests against the Grid stack booted by Testcontainers in global-setup. - * `baseURL` points at the Kahuna service (see fixtures.ts / playwright.config.ts). - */ - -test('kahuna healthcheck responds OK', async ({ request }) => { - const response = await request.get('/management/healthcheck'); - expect(response.ok()).toBeTruthy(); -}); - -test('kahuna serves the application', async ({ request }) => { - // The root path may redirect to auth; any non-server-error response proves Kahuna - // is serving requests against the provisioned infrastructure. - const response = await request.get('/', { maxRedirects: 0 }); - expect(response.status()).toBeLessThan(500); -}); - -test('frontend loads media-api from the grid-all container', async ({ page }) => { - // The SPA reads the media-api URL from and immediately - // requests the media-api root to discover its hypermedia links. dev-nginx routes - // https://api.media. to the media-api fixed port on the grid-all container, - // so this request should be served (200) by the container rather than failing. - const mediaApiResponse = page.waitForResponse( - (response) => - response.url().startsWith('https://api.media.') && response.request().method() === 'GET', - { timeout: 60_000 }, - ); - - await page.goto(KAHUNA_APP_URL); - - const mediaApiLink = await page.getAttribute('link[rel="media-api-uri"]', 'href'); - expect(mediaApiLink).toContain('api.media.'); - - const response = await mediaApiResponse; - expect(response.status()).toBe(200); -}); From bc1310ec4bea8c535f1cf57726903c992fa4e190 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 23 Jul 2026 11:12:07 +0100 Subject: [PATCH 225/373] Clean up files and add documentation after review --- .github/workflows/playwright.yml | 2 +- dev/script/generate-config/service-config.js | 18 ++-- e2e-tests/README.md | 60 +++++++++++++ e2e-tests/global-teardown.ts | 38 ++++---- e2e-tests/images/entrypoint.sh | 12 ++- e2e-tests/plans/001-testcontainers.md | 14 +-- e2e-tests/testcontainers/config.ts | 31 +++---- e2e-tests/testcontainers/provision.ts | 4 +- package-lock.json | 93 +------------------- package.json | 25 ------ playwright.config.ts | 79 ----------------- 11 files changed, 122 insertions(+), 254 deletions(-) create mode 100644 e2e-tests/README.md delete mode 100644 package.json delete mode 100644 playwright.config.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 88454b490ed..75fd4c317ec 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -30,7 +30,7 @@ jobs: # :443 to route the https://*.media. domains to the grid-all container # (standing in for the developer's dev-nginx). No extra setup is required. - name: Run Playwright tests - run: npx playwright test + run: npm test - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js index 6b73c78cf14..d937984c866 100644 --- a/dev/script/generate-config/service-config.js +++ b/dev/script/generate-config/service-config.js @@ -10,17 +10,21 @@ function getCorsAllowedOriginString(config) { } function getCommonConfig(config) { - // `NO_AUTH` is a backwards-compatible shorthand that disables both authentication and - // authorisation. The two can also be controlled independently via `NO_AUTHENTICATION` - // and `NO_AUTHORISATION` (e.g. the e2e tests use local authentication but exercise the - // real S3-backed authorisation provider). - const isNoAuth = process.env.NO_AUTH === "true"; // AI search depends on Bedrock/vector-ES infrastructure that isn't wired up for every org, // so it defaults to false (see CommonConfig.scala) and is only turned on here for Guardian's // own local dev environment. Shared by media-api and kahuna via the common config key. const aiSearchEnabled = process.env.BUILD_ORG ? false : true; - const isNoAuthentication = isNoAuth || process.env.NO_AUTHENTICATION === "true"; - const isNoAuthorisation = isNoAuth || process.env.NO_AUTHORISATION === "true"; + + // `NO_AUTH` is shorthand that disables both authentication and authorisation. + // The two can also be controlled independently via `NO_AUTHENTICATION` and + // `NO_AUTHORISATION`. + // + // Each flag can be supplied on the `config` object or via the equivalent + // environment variable (used by the dev `setup.sh` flow). + const flag = (name) => config[name] === true || process.env[name] === "true"; + const isNoAuth = flag("NO_AUTH"); + const isNoAuthentication = isNoAuth || flag("NO_AUTHENTICATION"); + const isNoAuthorisation = isNoAuth || flag("NO_AUTHORISATION"); // When the real authorisation provider is in use and a permissions bucket has been // provisioned locally (localstack), point the provider at it and enable the local-auth diff --git a/e2e-tests/README.md b/e2e-tests/README.md new file mode 100644 index 00000000000..11ecef04cb1 --- /dev/null +++ b/e2e-tests/README.md @@ -0,0 +1,60 @@ +# End-to-end tests + +Playwright + [`playwright-bdd`](https://vitalets.github.io/playwright-bdd/) e2e tests that +run against a full Grid stack. Scenarios live in [`features/`](features) (Gherkin) with the +step definitions in [`steps/`](steps). + +## How to run tests + +Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses +[Testcontainers](https://testcontainers.com/) to stand up everything the tests need: + +1. a shared Docker network; +2. **Elasticsearch** and **LocalStack** (the backing infrastructure); +3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded + buckets, provisioned into LocalStack; +4. per-service config, generated by `dev/script/generate-config/service-config.js`; +5. the **`grid-all`** image — a single container running all eight Play services. + +Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is +exposed to the tests. `globalTeardown` stops everything and cleans up. + +## Two ways to run + +### 1. CI / against the built stack (production mode) + +This is what [`.github/workflows/playwright.yml`](../.github/workflows/playwright.yml) does, +and how you reproduce a CI failure locally: + +```bash +# From the repo root: build the production image the harness runs. +DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile -t grid-all . + +cd e2e-tests +npm ci +npx playwright install --with-deps chromium +npm test +``` + +`grid-all` ([`images/Dockerfile`](images/Dockerfile)) stages the services with `sbt stage` +and runs them under a production JRE, so it exercises the same code paths as a deployed +Grid. There is no dev-nginx in CI, so when `CI=true` `global-setup` also starts a bundled +Caddy reverse proxy on `:443` that replays dev-nginx's subdomain routing for the +`https://*.media.` browser origins. + +### 2. Local iteration (live reload) + +For iterating on failing tests you can run the services from source with live reload using +the dev image ([`images/Dockerfile.dev`](images/Dockerfile.dev)), which runs +`sbt /run` (recompiling on change) and rebuilds Kahuna's webpack bundle via +`npm run watch`. See [`images/README.md`](images/README.md) for how to build and run it. + +Useful loop-level commands: + +```bash +npm run test:headed # run with a visible browser +npm run test:report # open the last HTML report +``` + +Traces are captured `on-first-retry` (see [`playwright.config.ts`](playwright.config.ts)), +so a failed test on CI leaves a trace you can open with `npx playwright show-trace`. diff --git a/e2e-tests/global-teardown.ts b/e2e-tests/global-teardown.ts index 2dc7ee08048..bbb6495382d 100644 --- a/e2e-tests/global-teardown.ts +++ b/e2e-tests/global-teardown.ts @@ -6,6 +6,15 @@ import * as fs from 'fs'; import { URLS_FILE } from './testcontainers/constants'; import { getEnvironment } from './testcontainers/state'; +/** Run a best-effort teardown step, warning (not throwing) so the rest still runs. */ +async function warnOnException(label: string, fn: () => unknown): Promise { + try { + await fn(); + } catch (error) { + console.warn(`${label}: ${(error as Error).message}`); + } +} + async function globalTeardown(): Promise { const environment = getEnvironment(); if (!environment) { @@ -15,31 +24,14 @@ async function globalTeardown(): Promise { const { containers, network, configDir } = environment; for (const container of [...containers].reverse()) { - try { - await container.stop(); - } catch (error) { - // Best effort: keep tearing down the rest of the stack. - console.warn(`Failed to stop container: ${(error as Error).message}`); - } - } - - try { - await network.stop(); - } catch (error) { - console.warn(`Failed to stop network: ${(error as Error).message}`); - } - - try { - fs.rmSync(configDir, { recursive: true, force: true }); - } catch (error) { - console.warn(`Failed to remove config dir: ${(error as Error).message}`); + await warnOnException('Failed to stop container', () => container.stop()); } - try { - fs.rmSync(URLS_FILE, { force: true }); - } catch (error) { - console.warn(`Failed to remove urls file: ${(error as Error).message}`); - } + await warnOnException('Failed to stop network', () => network.stop()); + await warnOnException('Failed to remove config dir', () => + fs.rmSync(configDir, { recursive: true, force: true }), + ); + await warnOnException('Failed to remove urls file', () => fs.rmSync(URLS_FILE, { force: true })); } export default globalTeardown; diff --git a/e2e-tests/images/entrypoint.sh b/e2e-tests/images/entrypoint.sh index 9779d152002..aac2e18b8df 100644 --- a/e2e-tests/images/entrypoint.sh +++ b/e2e-tests/images/entrypoint.sh @@ -61,4 +61,14 @@ for svc in $SERVICES; do pids+=("$!") done -sleep 100000 +# Block until the first service exits. During a healthy run the services stay up, +# so this only returns if one crashed; when that happens, tear the rest down and +# exit non-zero so the failure surfaces instead of leaving a half-running stack +# that still looks alive. +wait -n || true +echo "A Grid service exited unexpectedly; shutting down the rest." >&2 +for pid in "${pids[@]}"; do + kill -TERM "$pid" 2>/dev/null || true +done +wait 2>/dev/null || true +exit 1 diff --git a/e2e-tests/plans/001-testcontainers.md b/e2e-tests/plans/001-testcontainers.md index ed3891c3e6b..1dbdccea487 100644 --- a/e2e-tests/plans/001-testcontainers.md +++ b/e2e-tests/plans/001-testcontainers.md @@ -79,15 +79,17 @@ LocalStack, generates real service config by **reusing the existing AWS creds/endpoint env + `AWS_CBOR_DISABLE`, bind-mount `.tmp-config` -> `/etc/grid`, expose 9001/9002/9005/9006/9007/9010/9011/9012. 9. Wait strategies: ES healthy, then kahuna HTTP 200 on 9005 (generous timeout). Export - `baseURL` (mapped 9005 host port) via `process.env` + a temp file for teardown. + `baseURL` (fixed 9005 host port) via `process.env` + a temp file for teardown. ### Phase 5 — Teardown & Playwright wiring 10. `global-teardown.ts`: stop grid-all/ES/LocalStack, remove the network, clean `.tmp-config`. 11. `playwright.config.ts`: set `globalSetup`/`globalTeardown`, `use.baseURL` from env, keep browser projects, bump the global/test timeout for slow boot. -12. Update `example.spec.ts` to hit the local `baseURL` (kahuna) instead of `playwright.dev`. +12. Author the smoke scenarios as Gherkin `.feature` files with `playwright-bdd` steps + that hit the local `baseURL` (kahuna) instead of `playwright.dev`. 13. Update `.github/workflows/playwright.yml`: ensure Docker is available and the - `grid-all` image is present before tests (build step out of scope — assumed pre-built). + `grid-all` image is built before tests; run via `npm test` so `bddgen` generates the + test files first. Under `CI=true` a bundled Caddy proxy stands in for dev-nginx. ## Relevant files - `e2e-tests/global-setup.ts` (new) — network, infra, provisioning, config gen, app container @@ -104,9 +106,9 @@ LocalStack, generates real service config by **reusing the existing ## Verification 1. `cd e2e-tests && npm install` succeeds with new deps. -2. `npx playwright test` starts ES + LocalStack, provisions the CFN stack, seeds buckets, - generates config via `service-config.js`, boots grid-all, kahuna responds on 9005, - and the smoke test passes. +2. `npm test` starts ES + LocalStack, provisions the CFN + stack, seeds buckets, generates config via `service-config.js`, boots grid-all, kahuna + responds on 9005, and the smoke test passes. 3. Teardown removes all containers/network and `.tmp-config` (verify with `docker ps`). 4. Re-run to confirm no leaked containers/networks between runs. diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 11cb72101cc..83785e54f01 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -15,21 +15,13 @@ import { LOCALSTACK_PORT, REGION, REPO_ROOT, + SERVICE_PORTS, } from './constants'; const GENERATE_CONFIG_DIR = path.join(REPO_ROOT, 'dev', 'script', 'generate-config'); // The services packaged into the grid-all image (see e2e-tests/images/entrypoint.sh). -const GRID_SERVICES = [ - 'auth', - 'collections', - 'cropper', - 'kahuna', - 'leases', - 'media-api', - 'metadata-editor', - 'thrall', -]; +const GRID_SERVICES = Object.keys(SERVICE_PORTS); type StackProps = Record; @@ -39,12 +31,16 @@ type StackProps = Record; */ function rewriteEndpoints(conf: string): string { const localstackUrl = `http://${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`; + // Endpoints baked in by `service-config.js` that must be redirected to the container. + const guardianLocalstackUrl = `https://${LOCALSTACK_ALIAS}.media.${DOMAIN}`; + const legacyLocalstackUrl = 'http://localhost:4576'; + const localLocalstackUrl = `http://localhost:${LOCALSTACK_PORT}`; return conf - .split(`https://${LOCALSTACK_ALIAS}.media.${DOMAIN}`) + .split(guardianLocalstackUrl) .join(localstackUrl) - .split('http://localhost:4576') + .split(legacyLocalstackUrl) .join(localstackUrl) - .split('http://localhost:4566') + .split(localLocalstackUrl) .join(localstackUrl); } @@ -52,11 +48,6 @@ function rewriteEndpoints(conf: string): string { * Generate all service config files into `configDir`, reusing `service-config.js`. */ export function generateServiceConfig(configDir: string, coreStackProps: StackProps): void { - // NO_AUTHENTICATION makes `getCommonConfig` emit the Local authentication provider, so we - // don't need pan-domain / OIDC infrastructure. Authorisation is left as the real - // (S3-backed) provider, which reads `permissions.json` from the provisioned bucket. - process.env.NO_AUTHENTICATION = 'true'; - // eslint-disable-next-line @typescript-eslint/no-var-requires const ServiceConfig = require(path.join(GENERATE_CONFIG_DIR, 'service-config.js')); const defaultConfig = JSON5.parse( @@ -68,6 +59,10 @@ export function generateServiceConfig(configDir: string, coreStackProps: StackPr DOMAIN, EMAIL_DOMAIN, AWS_DEFAULT_REGION: REGION, + // NO_AUTHENTICATION makes `getCommonConfig` emit the Local authentication provider, so + // we don't need pan-domain / OIDC infrastructure. Authorisation is left as the real + // (S3-backed) provider, which reads `permissions.json` from the provisioned bucket. + NO_AUTHENTICATION: true, coreStackProps, es6: { ...defaultConfig.es6, diff --git a/e2e-tests/testcontainers/provision.ts b/e2e-tests/testcontainers/provision.ts index b51bf05000d..f5fa99443a6 100644 --- a/e2e-tests/testcontainers/provision.ts +++ b/e2e-tests/testcontainers/provision.ts @@ -12,7 +12,7 @@ import { waitUntilStackCreateComplete, } from '@aws-sdk/client-cloudformation'; import { CreateBucketCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { API_KEY, CORE_STACK_NAME, PERMISSIONS_BUCKET, REGION, REPO_ROOT } from './constants'; +import { API_KEY as API_KEY_PATH, CORE_STACK_NAME, PERMISSIONS_BUCKET, REGION, REPO_ROOT } from './constants'; const CREDENTIALS = { accessKeyId: 'test', secretAccessKey: 'test' }; @@ -66,7 +66,7 @@ async function seedBuckets(s3: S3Client, props: StackProps): Promise { const devConfig = path.join(REPO_ROOT, 'dev', 'config'); // API key used by the machine authentication provider. - await putObject(s3, props.KeyBucket, API_KEY, 'DEV Key'); + await putObject(s3, props.KeyBucket, API_KEY_PATH, 'DEV Key'); // Static config consumed by the services. for (const file of ['photographers.json', 'rcs-quota.json', 'usage_rights.json']) { diff --git a/package-lock.json b/package-lock.json index b7e0e01fe10..fb49cefeab8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,97 +1,6 @@ { "name": "grid", - "version": "1.0.0", "lockfileVersion": 3, "requires": true, - "packages": { - "": { - "name": "grid", - "version": "1.0.0", - "license": "ISC", - "devDependencies": { - "@playwright/test": "^1.61.1", - "@types/node": "^26.1.1" - } - }, - "node_modules/@playwright/test": { - "version": "1.61.1", - "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", - "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "playwright": "1.61.1" - }, - "bin": { - "playwright": "cli.js" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@types/node": { - "version": "26.1.1", - "resolved": "https://registry.npmjs.org/@types/node/-/node-26.1.1.tgz", - "integrity": "sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~8.3.0" - } - }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/playwright": { - "version": "1.61.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", - "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "playwright-core": "1.61.1" - }, - "bin": { - "playwright": "cli.js" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "fsevents": "2.3.2" - } - }, - "node_modules/playwright-core": { - "version": "1.61.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", - "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "playwright-core": "cli.js" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/undici-types": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", - "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", - "dev": true, - "license": "MIT" - } - } + "packages": {} } diff --git a/package.json b/package.json deleted file mode 100644 index 1e50600cca5..00000000000 --- a/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "grid", - "version": "1.0.0", - "description": "Grid ==== [![License](https://img.shields.io/github/license/guardian/grid.svg)](https://github.com/guardian/grid/blob/master/LICENSE) [![Join the chat at https://gitter.im/guardian/grid](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/guardian/grid?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)", - "main": "index.js", - "directories": { - "doc": "docs" - }, - "scripts": {}, - "repository": { - "type": "git", - "url": "git+https://github.com/guardian/grid.git" - }, - "keywords": [], - "author": "", - "license": "ISC", - "bugs": { - "url": "https://github.com/guardian/grid/issues" - }, - "homepage": "https://github.com/guardian/grid#readme", - "devDependencies": { - "@playwright/test": "^1.61.1", - "@types/node": "^26.1.1" - } -} diff --git a/playwright.config.ts b/playwright.config.ts deleted file mode 100644 index badddb2563c..00000000000 --- a/playwright.config.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { defineConfig, devices } from '@playwright/test'; - -/** - * Read environment variables from file. - * https://github.com/motdotla/dotenv - */ -// import dotenv from 'dotenv'; -// import path from 'path'; -// dotenv.config({ path: path.resolve(__dirname, '.env') }); - -/** - * See https://playwright.dev/docs/test-configuration. - */ -export default defineConfig({ - testDir: './e2e-tests', - /* Run tests in files in parallel */ - fullyParallel: true, - /* Fail the build on CI if you accidentally left test.only in the source code. */ - forbidOnly: !!process.env.CI, - /* Retry on CI only */ - retries: process.env.CI ? 2 : 0, - /* Opt out of parallel tests on CI. */ - workers: process.env.CI ? 1 : undefined, - /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', - /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ - use: { - /* Base URL to use in actions like `await page.goto('')`. */ - // baseURL: 'http://localhost:3000', - - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', - }, - - /* Configure projects for major browsers */ - projects: [ - { - name: 'chromium', - use: { ...devices['Desktop Chrome'] }, - }, - - { - name: 'firefox', - use: { ...devices['Desktop Firefox'] }, - }, - - { - name: 'webkit', - use: { ...devices['Desktop Safari'] }, - }, - - /* Test against mobile viewports. */ - // { - // name: 'Mobile Chrome', - // use: { ...devices['Pixel 5'] }, - // }, - // { - // name: 'Mobile Safari', - // use: { ...devices['iPhone 12'] }, - // }, - - /* Test against branded browsers. */ - // { - // name: 'Microsoft Edge', - // use: { ...devices['Desktop Edge'], channel: 'msedge' }, - // }, - // { - // name: 'Google Chrome', - // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, - // }, - ], - - /* Run your local dev server before starting the tests */ - // webServer: { - // command: 'npm run start', - // url: 'http://localhost:3000', - // reuseExistingServer: !process.env.CI, - // }, -}); From 2535fa4fb04280359d0f88a1fa1787ccb99923da Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 23 Jul 2026 13:36:43 +0100 Subject: [PATCH 226/373] Potential fix for pull request finding 'CodeQL / Workflow does not contain permissions' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/playwright.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 75fd4c317ec..dde57f5e338 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -4,6 +4,8 @@ on: branches: [ main, master ] pull_request: branches: [ main, master ] +permissions: + contents: read jobs: test: timeout-minutes: 60 From cb322bea03d137c11de9bde17dd35c53534a6736 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 11:14:06 +0100 Subject: [PATCH 227/373] Amend permissions user --- e2e-tests/fixtures/permissions/permissions.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/e2e-tests/fixtures/permissions/permissions.json b/e2e-tests/fixtures/permissions/permissions.json index b38d0e5d3f9..b6176cd6c8e 100644 --- a/e2e-tests/fixtures/permissions/permissions.json +++ b/e2e-tests/fixtures/permissions/permissions.json @@ -6,7 +6,7 @@ "defaultValue": true }, "overrides": [{ - "userId": "composer.application@guardian.co.uk", + "userId": "e2e.user@guardian.co.uk", "active": true, "isCasualNotOnShift": false }] @@ -18,7 +18,7 @@ "defaultValue": true }, "overrides": [{ - "userId": "composer.application@guardian.co.uk", + "userId": "e2e.user@guardian.co.uk", "active": true, "isCasualNotOnShift": false }] @@ -30,7 +30,7 @@ "defaultValue": true }, "overrides": [{ - "userId": "composer.application@guardian.co.uk", + "userId": "e2e.user@guardian.co.uk", "active": true, "isCasualNotOnShift": false }] @@ -42,7 +42,7 @@ "defaultValue": true }, "overrides": [{ - "userId": "composer.application@guardian.co.uk", + "userId": "e2e.user@guardian.co.uk", "active": true, "isCasualNotOnShift": false }] @@ -54,7 +54,7 @@ "defaultValue": true }, "overrides": [{ - "userId": "composer.application@guardian.co.uk", + "userId": "e2e.user@guardian.co.uk", "active": true, "isCasualNotOnShift": false }] @@ -66,7 +66,7 @@ "defaultValue": true }, "overrides": [{ - "userId": "composer.application@guardian.co.uk", + "userId": "e2e.user@guardian.co.uk", "active": true, "isCasualNotOnShift": false }] From 5c02f41f4830db3a34497bf9dee7ddb65c45b866 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 11:35:59 +0100 Subject: [PATCH 228/373] Consolidate the two dockerfiles into a single dockerfile, separating the local env from CI with --target --- .github/workflows/playwright.yml | 2 +- e2e-tests/README.md | 4 +- e2e-tests/images/Dockerfile | 124 +++++++++++++++---- e2e-tests/images/Dockerfile.dev | 98 --------------- e2e-tests/images/Dockerfile.dev.dockerignore | 10 -- e2e-tests/images/README.md | 47 ++++--- 6 files changed, 131 insertions(+), 154 deletions(-) delete mode 100644 e2e-tests/images/Dockerfile.dev delete mode 100644 e2e-tests/images/Dockerfile.dev.dockerignore diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index dde57f5e338..e4d4549d21c 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -27,7 +27,7 @@ jobs: # the repository root (the build context must be the repo root). - name: Build grid-all image working-directory: . - run: DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile -t grid-all . + run: DOCKER_BUILDKIT=1 docker build --target ci -f e2e-tests/images/Dockerfile -t grid-all . # GitHub sets CI=true, so global-setup starts the bundled Caddy reverse proxy on # :443 to route the https://*.media. domains to the grid-all container # (standing in for the developer's dev-nginx). No extra setup is required. diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 11ecef04cb1..39d47ddd183 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -28,7 +28,7 @@ and how you reproduce a CI failure locally: ```bash # From the repo root: build the production image the harness runs. -DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile -t grid-all . +DOCKER_BUILDKIT=1 docker build --target runtime -f e2e-tests/images/Dockerfile -t grid-all . cd e2e-tests npm ci @@ -45,7 +45,7 @@ Caddy reverse proxy on `:443` that replays dev-nginx's subdomain routing for the ### 2. Local iteration (live reload) For iterating on failing tests you can run the services from source with live reload using -the dev image ([`images/Dockerfile.dev`](images/Dockerfile.dev)), which runs +the dev image (`docker build --target dev` from [`images/Dockerfile`](images/Dockerfile)), which runs `sbt /run` (recompiling on change) and rebuilds Kahuna's webpack bundle via `npm run watch`. See [`images/README.md`](images/README.md) for how to build and run it. diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 0aa0bcded15..4691d512c00 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -2,20 +2,36 @@ # Single-container image that builds and runs eight Grid Play services # (auth, collections, cropper, kahuna, leases, media-api, metadata-editor, -# thrall) together with Kahuna's compiled frontend. +# thrall) together with Kahuna's frontend. # -# Build from the repository root: -# docker build -f e2e-tests/images/Dockerfile -t grid-all . +# This one Dockerfile produces two images, selected with `--target`: # -# Runtime configuration is NOT baked into the image. Mount per-environment +# * ci (grid-all) — CI/production style. Stages pre-compiled artefacts +# and runs them under a slim JRE. +# docker build --target ci -f e2e-tests/images/Dockerfile -t grid-all . +# +# * dev (grid-dev) — local development. Runs the services under +# `sbt /run` (Play dev mode) with the repo +# bind-mounted over /build so host edits recompile live. +# docker build --target dev -f e2e-tests/images/Dockerfile -t grid-dev . +# +# `--target` is REQUIRED: a plain `docker build` hits the `default` guard stage +# below and fails with a reminder. See images/README.md for details. +# +# Runtime configuration is NOT baked into the ci image. Mount per-environment # config (and a stage marker) into /etc/grid at run time, e.g.: # docker run --rm -v "$PWD/my-config:/etc/grid" grid-all -# See images/README.md for details. -# --------------------------------------------------------------------------- -# Stage 1: build the Kahuna frontend (webpack production bundle) -# --------------------------------------------------------------------------- -FROM node:22.12.0-bookworm AS frontend +# Shared build args (single source of truth for both images). +ARG SBT_VERSION=1.10.3 +ARG NODE_VERSION=22.12.0 +# Native tools used by the image processing services (see repo Brewfile). Kept +# as one arg so the ci and dev stages install the same set. +ARG NATIVE_PKGS="graphicsmagick imagemagick pngquant libimage-exiftool-perl libgd3" + +# Stage: build the Kahuna frontend (webpack production bundle). Uses the official +# Node image because this throwaway build stage only needs Node + npm. +FROM node:${NODE_VERSION}-bookworm AS frontend WORKDIR /build/kahuna @@ -27,12 +43,10 @@ RUN npm ci COPY kahuna/ ./ RUN npm run dist -# --------------------------------------------------------------------------- -# Stage 2: compile and stage the Play services with sbt -# --------------------------------------------------------------------------- -FROM eclipse-temurin:11-jdk-jammy AS backend +# Stage: shared JDK + sbt toolchain used by both `backend` (prod) and `dev`. +FROM eclipse-temurin:11-jdk-jammy AS toolchain -ARG SBT_VERSION=1.10.3 +ARG SBT_VERSION # Install sbt from the official release tarball. RUN set -eux; \ @@ -46,6 +60,9 @@ RUN set -eux; \ WORKDIR /build +# Stage: compile and stage the Play services with sbt (prod path). +FROM toolchain AS backend + # Copy the full repository (build context is the repo root). A .dockerignore # keeps target/, node_modules/ and logs/ out of the context. COPY --exclude=e2e-tests/images . . @@ -64,20 +81,15 @@ RUN sbt \ "metadata-editor/stage" \ "thrall/stage" -# --------------------------------------------------------------------------- -# Stage 3: slim runtime image with native image tooling and all services -# --------------------------------------------------------------------------- -FROM eclipse-temurin:11-jre-jammy AS runtime +# Stage: slim ci image with native image tooling and all services (grid-all). +FROM eclipse-temurin:11-jre-jammy AS ci + +ARG NATIVE_PKGS # Native tools used by the image processing services (see repo Brewfile). RUN set -eux; \ apt-get update; \ - apt-get install -y --no-install-recommends \ - graphicsmagick \ - imagemagick \ - pngquant \ - libimage-exiftool-perl \ - libgd3; \ + apt-get install -y --no-install-recommends ${NATIVE_PKGS}; \ rm -rf /var/lib/apt/lists/* # Staged apps expect their config at /usr/share//conf/application.conf @@ -106,3 +118,67 @@ ENV AWS_CBOR_DISABLE=true EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] + +# Stage: development image (grid-dev). JDK + sbt + Node + native tooling in a +# single stage with warmed caches, running services under `sbt /run`. +FROM toolchain AS dev + +ARG NODE_VERSION +ARG NATIVE_PKGS +# Provided automatically by BuildKit (e.g. amd64, arm64). +ARG TARGETARCH + +# xz-utils is needed to unpack the Node tarball; the native tools mirror the +# ci stage. +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends xz-utils ${NATIVE_PKGS}; \ + rm -rf /var/lib/apt/lists/* + +# Install Node (pinned to match kahuna's required version) from the official +# tarball so `npm run watch` matches local development. +RUN set -eux; \ + case "${TARGETARCH:-amd64}" in \ + amd64) node_arch=x64 ;; \ + arm64) node_arch=arm64 ;; \ + *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ + esac; \ + curl -fLo /tmp/node.tar.xz "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz"; \ + tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner; \ + rm -f /tmp/node.tar.xz; \ + node --version; \ + npm --version + +# Warm the sbt dependency cache (~/.ivy2, ~/.sbt, ~/.cache/coursier). These live +# under $HOME, so a runtime bind-mount over /build does not shadow them: the +# first runtime compile only needs to compile sources, not re-resolve deps. +# Copying just the build definition keeps this layer cached across source edits. +COPY build.sbt ./ +COPY project/ ./project/ +RUN sbt update + +# Warm Kahuna's node_modules. The dockerignore keeps node_modules out of the +# build context, so this install is not clobbered by the COPY below. +COPY kahuna/package.json kahuna/package-lock.json ./kahuna/ +RUN cd kahuna && npm ci + +# Copy the rest of the source so the image is runnable standalone; a runtime +# bind-mount of the repo over /build overrides this for live editing. +COPY . . + +COPY e2e-tests/images/entrypoint.dev.sh /usr/local/bin/entrypoint.dev.sh +RUN chmod +x /usr/local/bin/entrypoint.dev.sh + +# The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. +ENV AWS_CBOR_DISABLE=true + +# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases +# plus the JDWP debug port used when GRID_DEBUG is set. +EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 5005 + +ENTRYPOINT ["/usr/local/bin/entrypoint.dev.sh"] + +# Stage: default guard. This is the last stage, so a plain `docker build` (no +# --target) builds it and fails, forcing an explicit `--target ci|dev`. +FROM eclipse-temurin:11-jre-jammy AS default +RUN echo 'ERROR: --target is required. Pass --target ci (grid-all) or --target dev (grid-dev) when building the image' >&2; exit 1 diff --git a/e2e-tests/images/Dockerfile.dev b/e2e-tests/images/Dockerfile.dev deleted file mode 100644 index 281e2781759..00000000000 --- a/e2e-tests/images/Dockerfile.dev +++ /dev/null @@ -1,98 +0,0 @@ -# syntax=docker/dockerfile:1 - -# Development image for the Grid Play services. -# -# Unlike the CI image (images/Dockerfile), this image does NOT stage compiled -# artefacts. Instead it runs the selected services under `sbt /run` -# (Play dev mode), which recompiles changed Scala sources on the next request. -# Kahuna's webpack bundle is rebuilt continuously via `npm run watch`. -# -# The repository is expected to be bind-mounted over /build at run time so host -# edits are reflected live. Build from the repository root: -# DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile.dev -t grid-dev . -# -# Run with the repo and your DEV config mounted: -# docker run --rm \ -# -v "$PWD:/build" \ -# -v "$HOME/.grid:/root/.grid:ro" \ -# -p 9001:9001 -p 9005:9005 -p 9011:9011 \ -# grid-dev -# -# See images/README.md for details. - -# --------------------------------------------------------------------------- -# Single stage: JDK + sbt + Node + native image tooling, with warmed caches. -# --------------------------------------------------------------------------- -FROM eclipse-temurin:11-jdk-jammy - -ARG SBT_VERSION=1.10.3 -ARG NODE_VERSION=22.12.0 -# Provided automatically by BuildKit (e.g. amd64, arm64). -ARG TARGETARCH - -# Base tooling: curl/xz for downloads plus the native tools the image -# processing services need (mirrors images/Dockerfile's runtime stage). -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - curl \ - ca-certificates \ - xz-utils \ - graphicsmagick \ - imagemagick \ - pngquant \ - libimage-exiftool-perl \ - libgd3; \ - rm -rf /var/lib/apt/lists/* - -# Install sbt from the official release tarball. -RUN set -eux; \ - curl -fLo /tmp/sbt.tgz "https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz"; \ - tar -xzf /tmp/sbt.tgz -C /opt; \ - ln -s /opt/sbt/bin/sbt /usr/local/bin/sbt; \ - rm -f /tmp/sbt.tgz - -# Install Node (pinned to match kahuna's required version) from the official -# tarball so `npm run watch` matches local development. -RUN set -eux; \ - case "${TARGETARCH:-amd64}" in \ - amd64) node_arch=x64 ;; \ - arm64) node_arch=arm64 ;; \ - *) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \ - esac; \ - curl -fLo /tmp/node.tar.xz "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz"; \ - tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner; \ - rm -f /tmp/node.tar.xz; \ - node --version; \ - npm --version - -WORKDIR /build - -# Warm the sbt dependency cache (~/.ivy2, ~/.sbt, ~/.cache/coursier). These live -# under $HOME, so a runtime bind-mount over /build does not shadow them: the -# first runtime compile only needs to compile sources, not re-resolve deps. -# Copying just the build definition keeps this layer cached across source edits. -COPY build.sbt ./ -COPY project/ ./project/ -RUN sbt update - -# Warm Kahuna's node_modules. The dockerignore keeps node_modules out of the -# build context, so this install is not clobbered by the COPY below. -COPY kahuna/package.json kahuna/package-lock.json ./kahuna/ -RUN cd kahuna && npm ci - -# Copy the rest of the source so the image is runnable standalone; a runtime -# bind-mount of the repo over /build overrides this for live editing. -COPY . . - -COPY e2e-tests/images/entrypoint.dev.sh /usr/local/bin/entrypoint.dev.sh -RUN chmod +x /usr/local/bin/entrypoint.dev.sh - -# The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. -ENV AWS_CBOR_DISABLE=true - -# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases -# plus the JDWP debug port used when GRID_DEBUG is set. -EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 5005 - -ENTRYPOINT ["/usr/local/bin/entrypoint.dev.sh"] diff --git a/e2e-tests/images/Dockerfile.dev.dockerignore b/e2e-tests/images/Dockerfile.dev.dockerignore deleted file mode 100644 index 1a1a7091b5b..00000000000 --- a/e2e-tests/images/Dockerfile.dev.dockerignore +++ /dev/null @@ -1,10 +0,0 @@ -# BuildKit uses this file (named .dockerignore) to trim the build -# context for images/Dockerfile.dev. Keeps build artefacts and caches out of the -# context sent to the daemon; they are regenerated at build/run time. -**/target -**/node_modules -**/logs -**/.git -.git -**/.DS_Store -dev/.localstack diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index 697ae38d11d..a19de5873f8 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,13 +1,17 @@ # Grid all-in-one container -This directory provides **two** single-container images that run eight Grid Play -services plus Kahuna's frontend: +A single `Dockerfile` produces two single-container images that run eight +Grid Play services plus Kahuna's frontend, selected with `--target`: -- **CI image** — `Dockerfile`: stages pre-compiled artefacts and runs them in a - production-style JRE. Used by the e2e-tests testcontainers harness. -- **Local-dev image** — `Dockerfile.dev`: runs the services under `sbt /run` - (Play dev mode) with the repo bind-mounted, so source changes recompile live. - See [Development image (live reload)](#development-image-live-reload) below. +- **CI image** (`--target ci`, tagged `grid-all`): stages pre-compiled + artefacts and runs them in a production-style JRE. Used by the e2e-tests + testcontainers harness. +- **Local-dev image** (`--target dev`, tagged `grid-dev`): runs the services + under `sbt /run` (Play dev mode) with the repo bind-mounted, so source + changes recompile live. See + [Development image (live reload)](#development-image-live-reload) below. + +`--target` is required — a plain `docker build` fails with a reminder. Both expose the same service ports: @@ -27,20 +31,22 @@ Both expose the same service ports: Build from the **repository root** (the build context must be the repo root): ```bash -DOCKER_BUILDKIT=1 docker build -f images/Dockerfile -t grid-all . +DOCKER_BUILDKIT=1 docker build --target ci -f images/Dockerfile -t grid-all . ``` -The build has three stages: +The `grid-all` build uses these stages: -1. **frontend** — `node:22.12.0` builds the Kahuna webpack bundle - (`npm ci && npm run dist`). -2. **backend** — `eclipse-temurin:11-jdk` installs sbt and stages all eight - services (`sbt /stage`), packaging the frontend assets into Kahuna. -3. **runtime** — `eclipse-temurin:11-jre` with the native image tools +1. **frontend** — the official `node:22.12.0` image builds the Kahuna webpack + bundle (`npm ci && npm run dist`). +2. **toolchain** — `eclipse-temurin:11-jdk` installs sbt (shared with the dev + image). +3. **backend** — from `toolchain`, stages all eight services + (`sbt /stage`), packaging the frontend assets into Kahuna. +4. **ci** — `eclipse-temurin:11-jre` with the native image tools (`graphicsmagick`, `imagemagick`, `pngquant`, `exiftool`, `libgd3`) and the staged apps under `/usr/share/`. -## Runtime configuration +## CI configuration Configuration is **not** baked into the image. At runtime each service loads `/etc/grid/common.conf` and `/etc/grid/.conf` (and reads the stage @@ -71,7 +77,7 @@ If any running service exits, the container stops. ## Development image (live reload) -`Dockerfile.dev` is for local development: instead of staging compiled +The `--target dev` build is for local development: instead of staging compiled artefacts it runs the services under `sbt /run` (Play dev mode), so changed Scala sources are recompiled on the next request. When `kahuna` is selected its webpack bundle is rebuilt continuously via `npm run watch`. @@ -79,11 +85,14 @@ webpack bundle is rebuilt continuously via `npm run watch`. Build from the **repository root**: ```bash -DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile.dev -t grid-dev . +DOCKER_BUILDKIT=1 docker build --target dev -f e2e-tests/images/Dockerfile -t grid-dev . ``` -The build warms the sbt dependency cache (`sbt update`) and Kahuna's -`node_modules` (`npm ci`) so the first run only has to compile sources. +The `dev` stage shares the `toolchain` (JDK + sbt) base with the CI build and +adds Node (installed from the official tarball to match kahuna's pinned +version) plus the native image tools. It warms the sbt dependency cache +(`sbt update`) and Kahuna's `node_modules` (`npm ci`) so the first run only has +to compile sources. Run with the repo bind-mounted over `/build` so host edits are picked up live, and your DEV config mounted at `/root/.grid`: From a47ed16395e405c87c50bd1ef0148f1039d56a4f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 11:44:39 +0100 Subject: [PATCH 229/373] grid-all, grid-dev -> grid-e2e-ci, grid-e2e-dev --- .github/workflows/playwright.yml | 10 +-- e2e-tests/README.md | 6 +- e2e-tests/features/media-api.feature | 4 +- e2e-tests/global-setup.ts | 6 +- e2e-tests/images/Dockerfile | 20 ++--- e2e-tests/images/README.md | 18 ++-- e2e-tests/images/entrypoint.sh | 2 +- e2e-tests/plans/001-testcontainers.md | 119 -------------------------- e2e-tests/steps/fixtures.ts | 9 +- e2e-tests/steps/media-api.steps.ts | 3 - e2e-tests/testcontainers/config.ts | 3 +- e2e-tests/testcontainers/constants.ts | 12 ++- 12 files changed, 37 insertions(+), 175 deletions(-) delete mode 100644 e2e-tests/plans/001-testcontainers.md diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index e4d4549d21c..0cce7c3ecae 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -21,16 +21,10 @@ jobs: - name: Install dependencies run: npm ci - name: Install Playwright Browsers - # The suite runs Chromium only (see playwright.config.ts). run: npx playwright install --with-deps chromium - # The Testcontainers setup expects a pre-built `grid-all` image. Build it from - # the repository root (the build context must be the repo root). - - name: Build grid-all image + - name: Build CI image working-directory: . - run: DOCKER_BUILDKIT=1 docker build --target ci -f e2e-tests/images/Dockerfile -t grid-all . - # GitHub sets CI=true, so global-setup starts the bundled Caddy reverse proxy on - # :443 to route the https://*.media. domains to the grid-all container - # (standing in for the developer's dev-nginx). No extra setup is required. + run: DOCKER_BUILDKIT=1 docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci . - name: Run Playwright tests run: npm test - uses: actions/upload-artifact@v4 diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 39d47ddd183..0c96f824316 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -14,7 +14,7 @@ Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses 3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded buckets, provisioned into LocalStack; 4. per-service config, generated by `dev/script/generate-config/service-config.js`; -5. the **`grid-all`** image — a single container running all eight Play services. +5. the **`grid-e2e-ci`** image — a single container running all eight Play services. Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is exposed to the tests. `globalTeardown` stops everything and cleans up. @@ -28,7 +28,7 @@ and how you reproduce a CI failure locally: ```bash # From the repo root: build the production image the harness runs. -DOCKER_BUILDKIT=1 docker build --target runtime -f e2e-tests/images/Dockerfile -t grid-all . +DOCKER_BUILDKIT=1 docker build --target runtime -f e2e-tests/images/Dockerfile -t grid-e2e-ci . cd e2e-tests npm ci @@ -36,7 +36,7 @@ npx playwright install --with-deps chromium npm test ``` -`grid-all` ([`images/Dockerfile`](images/Dockerfile)) stages the services with `sbt stage` +`grid-e2e-ci` ([`images/Dockerfile`](images/Dockerfile)) stages the services with `sbt stage` and runs them under a production JRE, so it exercises the same code paths as a deployed Grid. There is no dev-nginx in CI, so when `CI=true` `global-setup` also starts a bundled Caddy reverse proxy on `:443` that replays dev-nginx's subdomain routing for the diff --git a/e2e-tests/features/media-api.feature b/e2e-tests/features/media-api.feature index 7e0948f3c54..a1b681ff104 100644 --- a/e2e-tests/features/media-api.feature +++ b/e2e-tests/features/media-api.feature @@ -2,9 +2,9 @@ Feature: Frontend loads media-api The Kahuna SPA reads the media-api URL from a element and immediately requests the media-api root to discover its hypermedia links. dev-nginx - routes https://api.media. to the media-api port on the grid-all container. + routes https://api.media. to the media-api port on the grid-e2e-ci container. - Scenario: The SPA discovers and requests media-api from the grid-all container + Scenario: The SPA discovers and requests media-api from the grid-e2e-ci container When I open the Grid application Then the page exposes a media-api URI link And the media-api is served successfully diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index e820a35b329..e7bcf7c2b33 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -6,7 +6,7 @@ * 2. Elasticsearch + LocalStack (infrastructure), * 3. the CloudFormation core stack + seeded buckets (provisioning), * 4. generated per-service config (reusing dev/script/generate-config), - * 5. the pre-built `grid-all` image running all eight services. + * 5. the pre-built `grid-e2e-ci` image running all eight services. * * The Kahuna base URL is exposed to tests via `GRID_BASE_URL`, and the started * containers are stashed for `global-teardown.ts`. @@ -42,7 +42,7 @@ const LOCALSTACK_SERVICES = 'cloudformation,cloudwatch,dynamodb,kinesis,s3,sns,s /** * Build a Caddyfile that reproduces the dev-nginx subdomain routing: each Grid service - * domain (https://.media.) reverse-proxies to the grid-all container on + * domain (https://.media.) reverse-proxies to the grid-e2e-ci container on * its in-container port, and the S3 vanity domains proxy to localstack (prepending the * real bucket to the path). `tls internal` serves a self-signed cert per site; Playwright * runs with `ignoreHTTPSErrors`, so the internal CA does not need to be trusted. @@ -133,7 +133,7 @@ async function globalSetup(): Promise { const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); generateServiceConfig(configDir, coreStackProps); - // --- Application: the pre-built grid-all image --------------------------- + // --- Application: the pre-built grid-e2e-ci image --------------------------- // All eight services run inside this single container and talk to each other over // its localhost. Each is published on the *fixed* host port its dev-nginx mapping // expects (dev/nginx-mappings.yml), so the developer's dev-nginx routes the diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 4691d512c00..dbcb2ff0512 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -6,21 +6,21 @@ # # This one Dockerfile produces two images, selected with `--target`: # -# * ci (grid-all) — CI/production style. Stages pre-compiled artefacts +# * ci (grid-e2e-ci) — CI/production style. Stages pre-compiled artefacts # and runs them under a slim JRE. -# docker build --target ci -f e2e-tests/images/Dockerfile -t grid-all . +# docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci . # -# * dev (grid-dev) — local development. Runs the services under +# * dev (grid-e2e-dev) — local development. Runs the services under # `sbt /run` (Play dev mode) with the repo # bind-mounted over /build so host edits recompile live. -# docker build --target dev -f e2e-tests/images/Dockerfile -t grid-dev . +# docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . # -# `--target` is REQUIRED: a plain `docker build` hits the `default` guard stage +# `--target` is required: a plain `docker build` hits the `default` guard stage # below and fails with a reminder. See images/README.md for details. # -# Runtime configuration is NOT baked into the ci image. Mount per-environment +# Runtime configuration is not baked into the ci image. Mount per-environment # config (and a stage marker) into /etc/grid at run time, e.g.: -# docker run --rm -v "$PWD/my-config:/etc/grid" grid-all +# docker run --rm -v "$PWD/my-config:/etc/grid" grid-e2e-ci # Shared build args (single source of truth for both images). ARG SBT_VERSION=1.10.3 @@ -81,7 +81,7 @@ RUN sbt \ "metadata-editor/stage" \ "thrall/stage" -# Stage: slim ci image with native image tooling and all services (grid-all). +# Stage: slim ci image with native image tooling and all services (grid-e2e-ci). FROM eclipse-temurin:11-jre-jammy AS ci ARG NATIVE_PKGS @@ -119,7 +119,7 @@ EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] -# Stage: development image (grid-dev). JDK + sbt + Node + native tooling in a +# Stage: development image (grid-e2e-dev). JDK + sbt + Node + native tooling in a # single stage with warmed caches, running services under `sbt /run`. FROM toolchain AS dev @@ -181,4 +181,4 @@ ENTRYPOINT ["/usr/local/bin/entrypoint.dev.sh"] # Stage: default guard. This is the last stage, so a plain `docker build` (no # --target) builds it and fails, forcing an explicit `--target ci|dev`. FROM eclipse-temurin:11-jre-jammy AS default -RUN echo 'ERROR: --target is required. Pass --target ci (grid-all) or --target dev (grid-dev) when building the image' >&2; exit 1 +RUN echo 'ERROR: --target is required. Pass --target ci (grid-e2e-ci) or --target dev (grid-e2e-dev) when building the image' >&2; exit 1 diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index a19de5873f8..b4ab16a7b43 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -3,10 +3,10 @@ A single `Dockerfile` produces two single-container images that run eight Grid Play services plus Kahuna's frontend, selected with `--target`: -- **CI image** (`--target ci`, tagged `grid-all`): stages pre-compiled +- **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. -- **Local-dev image** (`--target dev`, tagged `grid-dev`): runs the services +- **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. @@ -31,10 +31,10 @@ Both expose the same service ports: Build from the **repository root** (the build context must be the repo root): ```bash -DOCKER_BUILDKIT=1 docker build --target ci -f images/Dockerfile -t grid-all . +DOCKER_BUILDKIT=1 docker build --target ci -f images/Dockerfile -t grid-e2e-ci . ``` -The `grid-all` build uses these stages: +The `grid-e2e-ci` build uses these stages: 1. **frontend** — the official `node:22.12.0` image builds the Kahuna webpack bundle (`npm ci && npm run dist`). @@ -57,7 +57,7 @@ docker run --rm \ -v "$PWD/my-config:/etc/grid:ro" \ -p 9001:9001 -p 9002:9002 -p 9005:9005 -p 9006:9006 \ -p 9007:9007 -p 9010:9010 -p 9011:9011 -p 9012:9012 \ - grid-all + grid-e2e-ci ``` The services still need their backing infrastructure (Elasticsearch, S3/ @@ -70,7 +70,7 @@ Run a subset via `GRID_SERVICES` (space-separated), and append JVM options with `GRID_JAVA_OPTS`: ```bash -docker run --rm -e GRID_SERVICES="media-api kahuna auth" grid-all +docker run --rm -e GRID_SERVICES="media-api kahuna auth" grid-e2e-ci ``` If any running service exits, the container stops. @@ -85,7 +85,7 @@ webpack bundle is rebuilt continuously via `npm run watch`. Build from the **repository root**: ```bash -DOCKER_BUILDKIT=1 docker build --target dev -f e2e-tests/images/Dockerfile -t grid-dev . +DOCKER_BUILDKIT=1 docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . ``` The `dev` stage shares the `toolchain` (JDK + sbt) base with the CI build and @@ -102,7 +102,7 @@ docker run --rm \ -v "$PWD:/build" \ -v "$HOME/.grid:/root/.grid:ro" \ -p 9001:9001 -p 9005:9005 -p 9011:9011 \ - grid-dev + grid-e2e-dev ``` Because the services run in Play **dev mode**, no `play.http.secret.key` is @@ -125,7 +125,7 @@ All configurable via environment variables: docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" \ -e GRID_SERVICES="media-api kahuna" -e GRID_DEBUG=1 \ -p 9001:9001 -p 9005:9005 -p 5005:5005 \ - grid-dev + grid-e2e-dev ``` The services still need their backing infrastructure reachable (see the root diff --git a/e2e-tests/images/entrypoint.sh b/e2e-tests/images/entrypoint.sh index aac2e18b8df..6a30d7ddb92 100644 --- a/e2e-tests/images/entrypoint.sh +++ b/e2e-tests/images/entrypoint.sh @@ -8,7 +8,7 @@ # forwarded for a clean shutdown. # # Set GRID_SERVICES to a space-separated subset to run fewer services, e.g. -# docker run -e GRID_SERVICES="media-api kahuna" grid-all +# docker run -e GRID_SERVICES="media-api kahuna" grid-e2e-ci # Extra JVM options can be appended via GRID_JAVA_OPTS. set -euo pipefail diff --git a/e2e-tests/plans/001-testcontainers.md b/e2e-tests/plans/001-testcontainers.md deleted file mode 100644 index 1dbdccea487..00000000000 --- a/e2e-tests/plans/001-testcontainers.md +++ /dev/null @@ -1,119 +0,0 @@ -# Testcontainers setup for e2e-tests - -Add a Testcontainers-driven setup to the Playwright `e2e-tests/` project that boots the -pre-built **grid-all** image (from `e2e-tests/images/Dockerfile`) alongside -**Elasticsearch** and **LocalStack**, fully provisions the backing AWS resources in -LocalStack, generates real service config by **reusing the existing -`service-config.js`**, and runs all 8 Grid services — wired through Playwright's -`globalSetup`/`globalTeardown`. - -## Decisions -- **Scope:** grid-all container + Elasticsearch + LocalStack. Skip imgops & oidc for now. -- **Image source:** assume the `grid-all` image is already built; do not build in tests. -- **Config:** full provisioning — apply the CloudFormation core stack, seed buckets, and - generate per-service config from stack outputs; mount at `/etc/grid`. -- **Config generation:** reuse `dev/script/generate-config/service-config.js` - (`getCoreConfigs`) rather than reimplementing. -- **Services:** start all 8 services (the `entrypoint.sh` default `GRID_SERVICES`). -- **Lifecycle:** Playwright `globalSetup` / `globalTeardown`; expose `baseURL` via env. - -## Key facts -- grid-all runs 8 Play services. Ports: media-api 9001, thrall 9002, kahuna 9005, - cropper 9006, metadata-editor 9007, collections 9010, auth 9011, leases 9012. - Kahuna (9005) is the UI entrypoint -> `baseURL` target. -- Config not baked in: each service loads `/etc/grid/common.conf` + - `/etc/grid/.conf`, stage from `/etc/grid/stage` (default DEV). `entrypoint.sh` - respects `GRID_SERVICES`/`GRID_JAVA_OPTS`. `ENV AWS_CBOR_DISABLE=true` is set. -- Backing infra (docker-compose.yml): elasticsearch 8.18.3 (9200); localstack 4.5.0 - (4566; `SERVICES=cloudformation,cloudwatch,dynamodb,kinesis,s3,sns,sqs,iam`, eu-west-1). - -## Provisioning details (from setup.sh / generate-config) -- CFN core stack `dev/cloudformation/grid-dev-core.yml` creates: Kinesis streams - (`media-service-DEV-thrall`, low-priority), SQS `IngestSqsQueue`, S3 buckets - (Ingest/Fail, Image, Thumb, Reaper, Quarantine, Key, ImageOrigin, Config, Collections, - UsageMail), DynamoDB tables (Syndication, Edits, SoftDeletedMetadata, etc). -- Seed: upload API key (`"DEV Key"`) to KeyBucket; `dev/config/photographers.json`, - `rcs-quota.json`, `usage_rights.json` to ConfigBucket; `usages.eml` to UsageMailBucket. -- `generate-config.js` reads CFN stack resources via the AWS SDK and feeds - `service-config.js` (`getCoreConfigs`) to write per-service `.conf` + `common.conf`. - Defaults come from `config.json5` (es cluster `media-service`, shards 1, replicas 0). -- **Reuse strategy:** `require` `service-config.js` from `e2e-tests`. Do NOT run - `generate-config.js` directly (it writes to `~/.grid` and assumes AWS SDK v2 + `.env`). - Instead build the config object ourselves from live CFN stack resources - (LogicalResourceId -> PhysicalResourceId map) + `config.json5` defaults + `DOMAIN`, - call `ServiceConfig.getCoreConfigs`, then post-process. -- **Critical adaptation:** on a Testcontainers network the grid-all container must reach - ES/LocalStack via network **aliases** (`http://elasticsearch:9200`, - `http://localstack:4566`), not `localhost` — post-process `es6.url` / - `aws.local.endpoint` accordingly. Run in **NO_AUTH** mode - (`LocalAuthenticationProvider`/`LocalAuthorisationProvider`) to skip pan-domain/oidc. - -## Steps - -### Phase 1 — Dependencies -1. Add dev deps to `e2e-tests/package.json`: `testcontainers`, - `@testcontainers/localstack`, `@aws-sdk/client-cloudformation`, `@aws-sdk/client-s3`, - plus `json5` (to load `config.json5`). ES via `GenericContainer` or - `@testcontainers/elasticsearch`. Add a `test` script. - -### Phase 2 — Infra containers (`global-setup.ts`) -2. Create a shared Testcontainers `Network`. -3. Start Elasticsearch (`docker.elastic.co/.../elasticsearch:8.18.3`, single-node, - security disabled) with alias `elasticsearch`. -4. Start LocalStack (`localstack:4.5.0`, the required `SERVICES`, region `eu-west-1`) - with alias `localstack`. - -### Phase 3 — Provisioning (`global-setup.ts`) -5. Apply `grid-dev-core.yml` via the CloudFormation client against LocalStack; wait for - `CREATE_COMPLETE`; read stack resources into a logical->physical name map. -6. Seed buckets: API key -> KeyBucket; config JSON -> ConfigBucket; `usages.eml` -> - UsageMailBucket (reuse files in `dev/config/`). -7. Generate service config by **reusing `service-config.js`**: `require` `ServiceConfig`, - build a config object from the live CFN stack resources + `config.json5` defaults + - `DOMAIN`, call `getCoreConfigs`, then post-process `es6.url` / `aws.local.endpoint` - to the network aliases. Write results to `e2e-tests/.tmp-config`. Set `NO_AUTH=true` - so `common.conf` uses the Local auth/authorisation providers. - -### Phase 4 — App container (`global-setup.ts`) -8. Start grid-all (pre-built tag) on the network with default `GRID_SERVICES` (all 8), - AWS creds/endpoint env + `AWS_CBOR_DISABLE`, bind-mount `.tmp-config` -> `/etc/grid`, - expose 9001/9002/9005/9006/9007/9010/9011/9012. -9. Wait strategies: ES healthy, then kahuna HTTP 200 on 9005 (generous timeout). Export - `baseURL` (fixed 9005 host port) via `process.env` + a temp file for teardown. - -### Phase 5 — Teardown & Playwright wiring -10. `global-teardown.ts`: stop grid-all/ES/LocalStack, remove the network, clean `.tmp-config`. -11. `playwright.config.ts`: set `globalSetup`/`globalTeardown`, `use.baseURL` from env, - keep browser projects, bump the global/test timeout for slow boot. -12. Author the smoke scenarios as Gherkin `.feature` files with `playwright-bdd` steps - that hit the local `baseURL` (kahuna) instead of `playwright.dev`. -13. Update `.github/workflows/playwright.yml`: ensure Docker is available and the - `grid-all` image is built before tests; run via `npm test` so `bddgen` generates the - test files first. Under `CI=true` a bundled Caddy proxy stands in for dev-nginx. - -## Relevant files -- `e2e-tests/global-setup.ts` (new) — network, infra, provisioning, config gen, app container -- `e2e-tests/global-teardown.ts` (new) — teardown -- `e2e-tests/.tmp-config/` (generated) — mounted at `/etc/grid` -- `e2e-tests/playwright.config.ts` — `globalSetup`/`globalTeardown`, `baseURL`, timeout -- `e2e-tests/package.json` — deps + scripts -- `e2e-tests/images/entrypoint.sh` — reference for `GRID_SERVICES`/ports -- `dev/cloudformation/grid-dev-core.yml` — resources to provision -- `dev/script/setup.sh` — reference for provisioning + seeding flow -- `dev/script/generate-config/service-config.js` — **reused** for config generation -- `dev/script/generate-config/config.json5` — default config values consumed by reuse -- `dev/config/*` — seed data (photographers, quotas, usage rights, usages.eml) - -## Verification -1. `cd e2e-tests && npm install` succeeds with new deps. -2. `npm test` starts ES + LocalStack, provisions the CFN - stack, seeds buckets, generates config via `service-config.js`, boots grid-all, kahuna - responds on 9005, and the smoke test passes. -3. Teardown removes all containers/network and `.tmp-config` (verify with `docker ps`). -4. Re-run to confirm no leaked containers/networks between runs. - -## Notes -- Reusing `service-config.js` keeps parity with dev. Watch for: it's CommonJS, relies on - `config.json5` (needs `json5`) and `DOMAIN`/env inputs; the `aws.local.endpoint` and - `es6.url` values are Guardian-domain/localhost-oriented and must be post-processed to - the Testcontainers network aliases. diff --git a/e2e-tests/steps/fixtures.ts b/e2e-tests/steps/fixtures.ts index ca6d986f83f..1714ccf884b 100644 --- a/e2e-tests/steps/fixtures.ts +++ b/e2e-tests/steps/fixtures.ts @@ -11,13 +11,6 @@ interface GridUrls { mediaApi: string; } -/** - * URL the browser loads the Kahuna SPA from. It is the real `https://media.` - * origin, served by the developer's dev-nginx, which routes it (and the other - * `https://*.media.` service domains the SPA follows via hypermedia links) to - * the grid-all container's fixed ports (see global-setup.ts / dev/nginx-mappings.yml). - * dev-nginx must be running with the media-service mappings for browser tests to work. - */ export const KAHUNA_APP_URL = `https://media.${DOMAIN}`; function readGridUrls(): GridUrls { @@ -37,7 +30,7 @@ interface TestContext { * Test fixture that exposes the Grid service URLs resolved by `global-setup.ts` and * points `baseURL` at Kahuna's fixed host port for API-level `request` tests. Browser * steps should navigate to `KAHUNA_APP_URL` so the page origin is a real Grid domain - * (required for the services' CORS origins to match) and dev-nginx does the routing. + * (required for the services' CORS origins to match). */ export const test = base.extend<{ gridUrls: GridUrls; testContext: TestContext }>({ gridUrls: async ({}, use) => { diff --git a/e2e-tests/steps/media-api.steps.ts b/e2e-tests/steps/media-api.steps.ts index cbc15b3d53d..d37bdc1a22c 100644 --- a/e2e-tests/steps/media-api.steps.ts +++ b/e2e-tests/steps/media-api.steps.ts @@ -1,9 +1,6 @@ import { When, Then, expect, KAHUNA_APP_URL } from './fixtures'; When('I open the Grid application', async ({ page, testContext }) => { - // Register the media-api listener before navigating so the request the SPA fires on - // load is captured. dev-nginx routes https://api.media. to the media-api port - // on the grid-all container, so this request should be served (200) by the container. testContext.mediaApiResponse = page.waitForResponse( (response) => response.url().startsWith('https://api.media.') && response.request().method() === 'GET', diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 83785e54f01..d6b5bfc9265 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -20,7 +20,6 @@ import { const GENERATE_CONFIG_DIR = path.join(REPO_ROOT, 'dev', 'script', 'generate-config'); -// The services packaged into the grid-all image (see e2e-tests/images/entrypoint.sh). const GRID_SERVICES = Object.keys(SERVICE_PORTS); type StackProps = Record; @@ -45,7 +44,7 @@ function rewriteEndpoints(conf: string): string { } /** - * Generate all service config files into `configDir`, reusing `service-config.js`. + * Generate all service config files into `configDir`. */ export function generateServiceConfig(configDir: string, coreStackProps: StackProps): void { // eslint-disable-next-line @typescript-eslint/no-var-requires diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 39d8d422305..b94cdf70f67 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -1,10 +1,8 @@ import * as path from 'path'; -/** Repository root (two levels up from this file: e2e-tests/testcontainers/ -> repo). */ export const REPO_ROOT = path.resolve(__dirname, '..', '..'); -/** Values mirror dev/.env — only used to shape generated config, not real infra. */ -export const DOMAIN = process.env.GRID_DOMAIN ?? 'local.dev-gutools.co.uk'; +export const DOMAIN = 'local.dev-gutools.co.uk'; export const EMAIL_DOMAIN = 'guardian.co.uk'; export const REGION = 'eu-west-1'; @@ -18,7 +16,7 @@ export const CORE_STACK_NAME = 'grid-dev-core'; export const PERMISSIONS_BUCKET = 'permissions-cache'; /** Pre-built application image (see e2e-tests/images/Dockerfile). Assumed to exist. */ -export const GRID_IMAGE = process.env.GRID_IMAGE ?? 'grid-all'; +export const GRID_IMAGE = process.env.GRID_IMAGE ?? 'grid-e2e-ci'; export const ELASTICSEARCH_IMAGE = 'docker.elastic.co/elasticsearch/elasticsearch:8.18.3'; export const LOCALSTACK_IMAGE = 'localstack/localstack:4.5.0'; /** Reverse proxy used in CI to stand in for the developer's dev-nginx (see global-setup). */ @@ -28,8 +26,8 @@ export const PROXY_IMAGE = 'caddy:2.8-alpine'; export const ELASTICSEARCH_ALIAS = 'elasticsearch'; export const LOCALSTACK_ALIAS = 'localstack'; export const LOCALSTACK_PORT = 4566; -/** Network alias the CI reverse proxy uses to reach the grid-all app container. */ -export const GRID_ALIAS = 'grid-all'; +/** Network alias the CI reverse proxy uses to reach the grid-e2e-ci app container. */ +export const GRID_ALIAS = 'grid-e2e-ci'; /** service -> http port, from e2e-tests/images/entrypoint.sh. */ export const SERVICE_PORTS: Record = { @@ -50,4 +48,4 @@ export const MEDIA_API_PORT = SERVICE_PORTS['media-api']; export const URLS_FILE = path.join(__dirname, '..', '.grid-urls.json'); /** The API key value uploaded to the KeyBucket (dev/.env API_KEY). */ -export const API_KEY = 'dev-'; +export const API_KEY = 'e2e-dev'; From 72ec295574f093287ae49e1011705f33d8a1d255 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 11:56:53 +0100 Subject: [PATCH 230/373] Add workflow_dispatch --- .github/workflows/playwright.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 0cce7c3ecae..9568b4cbae0 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -4,6 +4,7 @@ on: branches: [ main, master ] pull_request: branches: [ main, master ] + workflow_dispatch: permissions: contents: read jobs: From 7be08d49c228caa3b39efa7cf2decf41a4700b7b Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 12:13:30 +0100 Subject: [PATCH 231/373] Temporarily add branch to workflow --- .github/workflows/playwright.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 9568b4cbae0..faac5814655 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -1,9 +1,9 @@ name: Playwright Tests on: push: - branches: [ main, master ] + branches: [ main, master, jsh/e2e-tests ] pull_request: - branches: [ main, master ] + branches: [ main, master, jsh/e2e-tests] workflow_dispatch: permissions: contents: read From 31969c0404833dbed56914a6aafe2e685834d30f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 12:26:40 +0100 Subject: [PATCH 232/373] Update/trim documentation --- e2e-tests/README.md | 36 ++++++++++++++++++------------------ e2e-tests/images/README.md | 32 +++++--------------------------- 2 files changed, 23 insertions(+), 45 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 0c96f824316..481f42c3fd9 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -4,24 +4,9 @@ Playwright + [`playwright-bdd`](https://vitalets.github.io/playwright-bdd/) e2e run against a full Grid stack. Scenarios live in [`features/`](features) (Gherkin) with the step definitions in [`steps/`](steps). -## How to run tests +## Running the tests -Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses -[Testcontainers](https://testcontainers.com/) to stand up everything the tests need: - -1. a shared Docker network; -2. **Elasticsearch** and **LocalStack** (the backing infrastructure); -3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded - buckets, provisioned into LocalStack; -4. per-service config, generated by `dev/script/generate-config/service-config.js`; -5. the **`grid-e2e-ci`** image — a single container running all eight Play services. - -Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is -exposed to the tests. `globalTeardown` stops everything and cleans up. - -## Two ways to run - -### 1. CI / against the built stack (production mode) +### 1. CI (production mode) This is what [`.github/workflows/playwright.yml`](../.github/workflows/playwright.yml) does, and how you reproduce a CI failure locally: @@ -42,7 +27,7 @@ Grid. There is no dev-nginx in CI, so when `CI=true` `global-setup` also starts Caddy reverse proxy on `:443` that replays dev-nginx's subdomain routing for the `https://*.media.` browser origins. -### 2. Local iteration (live reload) +### 2. Dev (live recompilation) For iterating on failing tests you can run the services from source with live reload using the dev image (`docker build --target dev` from [`images/Dockerfile`](images/Dockerfile)), which runs @@ -58,3 +43,18 @@ npm run test:report # open the last HTML report Traces are captured `on-first-retry` (see [`playwright.config.ts`](playwright.config.ts)), so a failed test on CI leaves a trace you can open with `npx playwright show-trace`. + +## How they work + +Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses +[Testcontainers](https://testcontainers.com/) to stand up everything the tests need: + +1. a shared Docker network; +2. **Elasticsearch** and **LocalStack** (the backing infrastructure); +3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded + buckets, provisioned into LocalStack; +4. per-service config, generated by `dev/script/generate-config/service-config.js`; +5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running all eight Play services. + +Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is +exposed to the tests. `globalTeardown` stops everything and cleans up. diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index b4ab16a7b43..6fb2dc357aa 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,32 +1,14 @@ -# Grid all-in-one container +# Grid e2e container A single `Dockerfile` produces two single-container images that run eight Grid Play services plus Kahuna's frontend, selected with `--target`: -- **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled - artefacts and runs them in a production-style JRE. Used by the e2e-tests - testcontainers harness. -- **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services - under `sbt /run` (Play dev mode) with the repo bind-mounted, so source - changes recompile live. See - [Development image (live reload)](#development-image-live-reload) below. +- **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. +- ** Dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. `--target` is required — a plain `docker build` fails with a reminder. -Both expose the same service ports: - -| Service | Port | -| ----------------- | ---- | -| `media-api` | 9001 | -| `thrall` | 9002 | -| `kahuna` (+ UI) | 9005 | -| `cropper` | 9006 | -| `metadata-editor` | 9007 | -| `collections` | 9010 | -| `auth` | 9011 | -| `leases` | 9012 | - -## Build +## Build for CI Build from the **repository root** (the build context must be the repo root): @@ -46,7 +28,7 @@ The `grid-e2e-ci` build uses these stages: (`graphicsmagick`, `imagemagick`, `pngquant`, `exiftool`, `libgd3`) and the staged apps under `/usr/share/`. -## CI configuration +### Configuration Configuration is **not** baked into the image. At runtime each service loads `/etc/grid/common.conf` and `/etc/grid/.conf` (and reads the stage @@ -60,10 +42,6 @@ docker run --rm \ grid-e2e-ci ``` -The services still need their backing infrastructure (Elasticsearch, S3/ -DynamoDB/Kinesis/SNS/SQS, the image resizer, auth provider) to be reachable — -see the root `docker-compose.yml` for the local dev topology. - ## Selecting services Run a subset via `GRID_SERVICES` (space-separated), and append JVM options with From 245017f157326b10c047342c8948e6de8cf9f16d Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 13:16:02 +0100 Subject: [PATCH 233/373] Bump to latest GHA --- .github/workflows/playwright.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index faac5814655..4d242f121a0 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -15,8 +15,8 @@ jobs: run: working-directory: e2e-tests steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 with: node-version: lts/* - name: Install dependencies From 03f21ad4514ce44e75cae17af5161f1ae3aff73a Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 13:17:18 +0100 Subject: [PATCH 234/373] Remove GRID_JAVA_OPTS --- e2e-tests/global-setup.ts | 4 ---- e2e-tests/images/README.md | 4 +--- e2e-tests/images/entrypoint.dev.sh | 6 ------ e2e-tests/images/entrypoint.sh | 4 +--- e2e-tests/testcontainers/config.ts | 8 ++++++++ 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index e7bcf7c2b33..06cdf31dde6 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -157,10 +157,6 @@ async function globalSetup(): Promise { AWS_REGION: REGION, AWS_DEFAULT_REGION: REGION, AWS_CBOR_DISABLE: 'true', - // Staged Play apps run in prod mode, so an application secret must be provided. - // entrypoint.sh appends GRID_JAVA_OPTS to every service. Must be >= 256 bits. - GRID_JAVA_OPTS: - '-Dplay.http.secret.key=testcontainers-e2e-application-secret-0123456789', }) .withWaitStrategy(Wait.forHttp('/management/healthcheck', KAHUNA_PORT).forStatusCode(200)) .withStartupTimeout(startupTimeoutMs); diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index 6fb2dc357aa..9f329924070 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -44,8 +44,7 @@ docker run --rm \ ## Selecting services -Run a subset via `GRID_SERVICES` (space-separated), and append JVM options with -`GRID_JAVA_OPTS`: +Run a subset via `GRID_SERVICES` (space-separated): ```bash docker run --rm -e GRID_SERVICES="media-api kahuna auth" grid-e2e-ci @@ -96,7 +95,6 @@ All configurable via environment variables: | ----------------------- | -------------------- | -------------------------------------------------------------- | | `GRID_SERVICES` | `auth media-api kahuna` | Space-separated services to run. | | `GRID_EXTRA_CONFIG_DIR` | _(unset)_ | Dir of `.conf` overrides (sets `-DextraConfigDir`). | -| `GRID_JAVA_OPTS` | _(unset)_ | Extra JVM options forwarded to the run JVM (as `-J...`). | | `GRID_DEBUG` | _(unset)_ | If set, opens a JDWP debug server on port `5005` (also `EXPOSE`d). | ```bash diff --git a/e2e-tests/images/entrypoint.dev.sh b/e2e-tests/images/entrypoint.dev.sh index 0d1c6e63b55..9128055f913 100644 --- a/e2e-tests/images/entrypoint.dev.sh +++ b/e2e-tests/images/entrypoint.dev.sh @@ -12,7 +12,6 @@ # # Environment: # GRID_SERVICES space-separated services to run (default below) -# GRID_JAVA_OPTS extra JVM options, forwarded to the run JVM # GRID_EXTRA_CONFIG_DIR dir of .conf overrides (sets -DextraConfigDir) # GRID_DEBUG if non-empty, opens a JDWP debug server on port 5005 @@ -79,11 +78,6 @@ fi if [[ -n "${GRID_DEBUG:-}" ]]; then SBT_OPTS="$SBT_OPTS -jvm-debug 5005" fi -if [[ -n "${GRID_JAVA_OPTS:-}" ]]; then - for opt in $GRID_JAVA_OPTS; do - SBT_OPTS="$SBT_OPTS -J${opt}" - done -fi shutdown() { echo "Shutting down dev services..." diff --git a/e2e-tests/images/entrypoint.sh b/e2e-tests/images/entrypoint.sh index 6a30d7ddb92..4af4bb5d77c 100644 --- a/e2e-tests/images/entrypoint.sh +++ b/e2e-tests/images/entrypoint.sh @@ -9,7 +9,6 @@ # # Set GRID_SERVICES to a space-separated subset to run fewer services, e.g. # docker run -e GRID_SERVICES="media-api kahuna" grid-e2e-ci -# Extra JVM options can be appended via GRID_JAVA_OPTS. set -euo pipefail @@ -27,7 +26,6 @@ declare -A PORTS=( DEFAULT_SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall" SERVICES="${GRID_SERVICES:-$DEFAULT_SERVICES}" -EXTRA_JAVA_OPTS="${GRID_JAVA_OPTS:-}" pids=() @@ -57,7 +55,7 @@ for svc in $SERVICES; do echo "Starting $svc on port $port" # shellcheck disable=SC2086 - "$bin" -Dhttp.port="$port" $EXTRA_JAVA_OPTS & + "$bin" -Dhttp.port="$port" & pids+=("$!") done diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index d6b5bfc9265..3b6108faa37 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -76,6 +76,14 @@ export function generateServiceConfig(configDir: string, coreStackProps: StackPr // Mark the stage as DEV so services load `~/.grid/.conf` (see GridConfigLoader). fs.writeFileSync(path.join(configDir, 'stage'), 'DEV'); + // The staged apps run in Play prod mode, which requires an application secret (>= 256 + // bits). GridConfigLoader loads `common.conf` for every service, so set it once here + // rather than injecting it as a JVM option in the container. + fs.writeFileSync( + path.join(configDir, 'common.conf'), + 'play.http.secret.key = "testcontainers-e2e-application-secret-0123456789"\n', + ); + for (const service of GRID_SERVICES) { const conf = serviceConfigs[service]; if (!conf) { From aacdf30c3ed2cd15755a893f3693eb427d880fba Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 13:25:43 +0100 Subject: [PATCH 235/373] Remove unnecessary env args and comments --- e2e-tests/images/README.md | 16 ++------------ e2e-tests/images/entrypoint.dev.sh | 35 +++++------------------------- e2e-tests/images/entrypoint.sh | 6 +---- 3 files changed, 9 insertions(+), 48 deletions(-) diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index 9f329924070..36eb821fc73 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -42,17 +42,7 @@ docker run --rm \ grid-e2e-ci ``` -## Selecting services - -Run a subset via `GRID_SERVICES` (space-separated): - -```bash -docker run --rm -e GRID_SERVICES="media-api kahuna auth" grid-e2e-ci -``` - -If any running service exits, the container stops. - -## Development image (live reload) +## Build for development (live reload) The `--target dev` build is for local development: instead of staging compiled artefacts it runs the services under `sbt /run` (Play dev mode), so changed @@ -93,13 +83,11 @@ All configurable via environment variables: | Variable | Default | Purpose | | ----------------------- | -------------------- | -------------------------------------------------------------- | -| `GRID_SERVICES` | `auth media-api kahuna` | Space-separated services to run. | -| `GRID_EXTRA_CONFIG_DIR` | _(unset)_ | Dir of `.conf` overrides (sets `-DextraConfigDir`). | | `GRID_DEBUG` | _(unset)_ | If set, opens a JDWP debug server on port `5005` (also `EXPOSE`d). | ```bash docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" \ - -e GRID_SERVICES="media-api kahuna" -e GRID_DEBUG=1 \ + -e GRID_DEBUG=1 \ -p 9001:9001 -p 9005:9005 -p 5005:5005 \ grid-e2e-dev ``` diff --git a/e2e-tests/images/entrypoint.dev.sh b/e2e-tests/images/entrypoint.dev.sh index 9128055f913..8359167b68e 100644 --- a/e2e-tests/images/entrypoint.dev.sh +++ b/e2e-tests/images/entrypoint.dev.sh @@ -11,8 +11,6 @@ # docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" grid-dev # # Environment: -# GRID_SERVICES space-separated services to run (default below) -# GRID_EXTRA_CONFIG_DIR dir of .conf overrides (sets -DextraConfigDir) # GRID_DEBUG if non-empty, opens a JDWP debug server on port 5005 set -euo pipefail @@ -20,23 +18,9 @@ set -euo pipefail REPO=/build cd "$REPO" -DEFAULT_SERVICES="auth media-api kahuna" -SERVICES="${GRID_SERVICES:-$DEFAULT_SERVICES}" +SERVICES="auth media-api kahuna" -# service -> http port. Informational only: `sbt /run` binds the port from -# playDefaultPort in build.sbt, so these must stay in sync with it. -declare -A PORTS=( - [media-api]=9001 - [thrall]=9002 - [kahuna]=9005 - [cropper]=9006 - [metadata-editor]=9007 - [collections]=9010 - [auth]=9011 - [leases]=9012 -) - -# --- Kahuna frontend watcher ------------------------------------------------ +# --- Kahuna frontend watcher watch_pid="" if [[ " $SERVICES " == *" kahuna "* ]]; then echo "Starting Kahuna webpack watcher..." @@ -51,30 +35,23 @@ if [[ " $SERVICES " == *" kahuna "* ]]; then watch_pid=$! fi -# --- Assemble the sbt run command ------------------------------------------- +# --- Assemble the sbt run command run_tasks="" for svc in $SERVICES; do - if [[ -z "${PORTS[$svc]:-}" ]]; then - echo "Unknown service '$svc' (no port mapping); skipping." >&2 - continue - fi - echo "Will run $svc on port ${PORTS[$svc]}" + echo "Running $svc" run_tasks="$run_tasks ${svc}/run" done if [[ -z "$run_tasks" ]]; then - echo "No valid services selected in GRID_SERVICES='$SERVICES'." >&2 + echo "No valid services selected in '$SERVICES'." >&2 exit 1 fi # `all` runs the per-service `run` tasks in parallel from a single sbt session. SBT_COMMAND="all${run_tasks}" -# --- sbt / JVM options ------------------------------------------------------ +# --- sbt / JVM options SBT_OPTS="${SBT_OPTS:-}" -if [[ -n "${GRID_EXTRA_CONFIG_DIR:-}" ]]; then - SBT_OPTS="$SBT_OPTS -J-DextraConfigDir=${GRID_EXTRA_CONFIG_DIR}" -fi if [[ -n "${GRID_DEBUG:-}" ]]; then SBT_OPTS="$SBT_OPTS -jvm-debug 5005" fi diff --git a/e2e-tests/images/entrypoint.sh b/e2e-tests/images/entrypoint.sh index 4af4bb5d77c..452d3980fac 100644 --- a/e2e-tests/images/entrypoint.sh +++ b/e2e-tests/images/entrypoint.sh @@ -6,9 +6,6 @@ # port is passed explicitly per service via -Dhttp.port. Services run in the # background; if any one exits the container stops, and SIGTERM/SIGINT are # forwarded for a clean shutdown. -# -# Set GRID_SERVICES to a space-separated subset to run fewer services, e.g. -# docker run -e GRID_SERVICES="media-api kahuna" grid-e2e-ci set -euo pipefail @@ -24,8 +21,7 @@ declare -A PORTS=( [leases]=9012 ) -DEFAULT_SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall" -SERVICES="${GRID_SERVICES:-$DEFAULT_SERVICES}" +SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall" pids=() From a2311f746613f0e95067376f9030efad863c3fb3 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 13:43:50 +0100 Subject: [PATCH 236/373] Consolidate entrypoint scripts --- e2e-tests/images/Dockerfile | 1 + e2e-tests/images/README.md | 64 +++++++++++++-------------- e2e-tests/images/entrypoint.common.sh | 33 ++++++++++++++ e2e-tests/images/entrypoint.dev.sh | 25 +++++------ e2e-tests/images/entrypoint.sh | 27 ++--------- 5 files changed, 79 insertions(+), 71 deletions(-) create mode 100644 e2e-tests/images/entrypoint.common.sh diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index dbcb2ff0512..49e7fb4f7eb 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -108,6 +108,7 @@ RUN set -eux; \ mkdir -p "/var/log/$svc"; \ done +COPY e2e-tests/images/entrypoint-common.sh /usr/local/bin/entrypoint-common.sh COPY e2e-tests/images/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index 36eb821fc73..a7b12cfb0a2 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,36 +1,35 @@ -# Grid e2e container +# Grid E2E test images -A single `Dockerfile` produces two single-container images that run eight -Grid Play services plus Kahuna's frontend, selected with `--target`: +This directory provides **two** single-container images that run eight Grid Play +services plus Kahuna's frontend: -- **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. -- ** Dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. +- **CI image** — `Dockerfile`: stages pre-compiled artefacts and runs them in a + production-style JRE. Used by the e2e-tests testcontainers harness. +- **Local-dev image** — `Dockerfile.dev`: runs the services under `sbt /run` + (Play dev mode) with the repo bind-mounted, so source changes recompile live. + See [Development image (live reload)](#development-image-live-reload) below. -`--target` is required — a plain `docker build` fails with a reminder. - -## Build for CI +## Build Build from the **repository root** (the build context must be the repo root): ```bash -DOCKER_BUILDKIT=1 docker build --target ci -f images/Dockerfile -t grid-e2e-ci . +DOCKER_BUILDKIT=1 docker build -f images/Dockerfile -t grid-all . ``` -The `grid-e2e-ci` build uses these stages: +The build has three stages: -1. **frontend** — the official `node:22.12.0` image builds the Kahuna webpack - bundle (`npm ci && npm run dist`). -2. **toolchain** — `eclipse-temurin:11-jdk` installs sbt (shared with the dev - image). -3. **backend** — from `toolchain`, stages all eight services - (`sbt /stage`), packaging the frontend assets into Kahuna. -4. **ci** — `eclipse-temurin:11-jre` with the native image tools +1. **frontend** — `node:22.12.0` builds the Kahuna webpack bundle + (`npm ci && npm run dist`). +2. **backend** — `eclipse-temurin:11-jdk` installs sbt and stages all eight + services (`sbt /stage`), packaging the frontend assets into Kahuna. +3. **runtime** — `eclipse-temurin:11-jre` with the native image tools (`graphicsmagick`, `imagemagick`, `pngquant`, `exiftool`, `libgd3`) and the staged apps under `/usr/share/`. -### Configuration +## Runtime configuration -Configuration is **not** baked into the image. At runtime each service loads +Configuration is not baked into the image. At runtime each service loads `/etc/grid/common.conf` and `/etc/grid/.conf` (and reads the stage from `/etc/grid/stage`; defaults to `DEV`). Mount your environment's config in: @@ -39,27 +38,28 @@ docker run --rm \ -v "$PWD/my-config:/etc/grid:ro" \ -p 9001:9001 -p 9002:9002 -p 9005:9005 -p 9006:9006 \ -p 9007:9007 -p 9010:9010 -p 9011:9011 -p 9012:9012 \ - grid-e2e-ci + grid-all ``` -## Build for development (live reload) +The services still need their backing infrastructure (Elasticsearch, S3/ +DynamoDB/Kinesis/SNS/SQS, the image resizer, auth provider) to be reachable — +see the root `docker-compose.yml` for the local dev topology. + +## Development image (live reload) -The `--target dev` build is for local development: instead of staging compiled +`Dockerfile.dev` is for local development. Instead of staging compiled artefacts it runs the services under `sbt /run` (Play dev mode), so changed Scala sources are recompiled on the next request. When `kahuna` is selected its webpack bundle is rebuilt continuously via `npm run watch`. -Build from the **repository root**: +Build from the repository root: ```bash -DOCKER_BUILDKIT=1 docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . +DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile.dev -t grid-dev . ``` -The `dev` stage shares the `toolchain` (JDK + sbt) base with the CI build and -adds Node (installed from the official tarball to match kahuna's pinned -version) plus the native image tools. It warms the sbt dependency cache -(`sbt update`) and Kahuna's `node_modules` (`npm ci`) so the first run only has -to compile sources. +The build warms the sbt dependency cache (`sbt update`) and Kahuna's +`node_modules` (`npm ci`) so the first run only has to compile sources. Run with the repo bind-mounted over `/build` so host edits are picked up live, and your DEV config mounted at `/root/.grid`: @@ -69,11 +69,9 @@ docker run --rm \ -v "$PWD:/build" \ -v "$HOME/.grid:/root/.grid:ro" \ -p 9001:9001 -p 9005:9005 -p 9011:9011 \ - grid-e2e-dev + grid-dev ``` -Because the services run in Play **dev mode**, no `play.http.secret.key` is -required (it is auto-generated), unlike the CI image's production-style run. Edit a `.scala` file on the host and hit an endpoint to trigger a recompile; edit a Kahuna asset and the webpack watcher rebuilds `public/dist`. @@ -83,7 +81,7 @@ All configurable via environment variables: | Variable | Default | Purpose | | ----------------------- | -------------------- | -------------------------------------------------------------- | -| `GRID_DEBUG` | _(unset)_ | If set, opens a JDWP debug server on port `5005` (also `EXPOSE`d). | +| `GRID_DEBUG` | _(unset)_ | If set, opens a debug server on port `5005` (also `EXPOSE`d). | ```bash docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" \ diff --git a/e2e-tests/images/entrypoint.common.sh b/e2e-tests/images/entrypoint.common.sh new file mode 100644 index 00000000000..fac345f586e --- /dev/null +++ b/e2e-tests/images/entrypoint.common.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Shared definitions for the Grid container entrypoints (ci and dev). +# +# Sourced by entrypoint.sh and entrypoint.dev.sh to provide a single source of +# truth for the service list, their http ports, and the graceful-shutdown trap. + +# service -> http port +declare -A PORTS=( + [media-api]=9001 + [thrall]=9002 + [kahuna]=9005 + [cropper]=9006 + [metadata-editor]=9007 + [collections]=9010 + [auth]=9011 + [leases]=9012 +) + +export SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall" + +# Install a TERM/INT trap that kills the given child pids, waits for them, and +# exits cleanly. Pass the pids to tear down as arguments, e.g. +# trap 'shutdown_children "${pids[@]}"' TERM INT +shutdown_children() { + echo "Shutting down Grid services..." + local pid + for pid in "$@"; do + kill -TERM "$pid" 2>/dev/null || true + done + wait 2>/dev/null || true + exit 0 +} diff --git a/e2e-tests/images/entrypoint.dev.sh b/e2e-tests/images/entrypoint.dev.sh index 8359167b68e..b480faac53b 100644 --- a/e2e-tests/images/entrypoint.dev.sh +++ b/e2e-tests/images/entrypoint.dev.sh @@ -18,7 +18,10 @@ set -euo pipefail REPO=/build cd "$REPO" -SERVICES="auth media-api kahuna" +source "$REPO/e2e-tests/images/entrypoint.common.sh" + +# Default to the full production service list; GRID_SERVICES can narrow it. +SERVICES="${GRID_SERVICES:-$SERVICES}" # --- Kahuna frontend watcher watch_pid="" @@ -38,12 +41,16 @@ fi # --- Assemble the sbt run command run_tasks="" for svc in $SERVICES; do - echo "Running $svc" + if [[ -z "${PORTS[$svc]:-}" ]]; then + echo "Unknown service '$svc' (no port mapping); skipping." >&2 + continue + fi + echo "Will run $svc on port ${PORTS[$svc]}" run_tasks="$run_tasks ${svc}/run" done if [[ -z "$run_tasks" ]]; then - echo "No valid services selected in '$SERVICES'." >&2 + echo "No valid services selected in GRID_SERVICES='$SERVICES'." >&2 exit 1 fi @@ -52,18 +59,8 @@ SBT_COMMAND="all${run_tasks}" # --- sbt / JVM options SBT_OPTS="${SBT_OPTS:-}" -if [[ -n "${GRID_DEBUG:-}" ]]; then - SBT_OPTS="$SBT_OPTS -jvm-debug 5005" -fi -shutdown() { - echo "Shutting down dev services..." - [[ -n "$watch_pid" ]] && kill -TERM "$watch_pid" 2>/dev/null || true - [[ -n "${sbt_pid:-}" ]] && kill -TERM "$sbt_pid" 2>/dev/null || true - wait 2>/dev/null || true - exit 0 -} -trap shutdown TERM INT +trap 'shutdown_children "$watch_pid" "${sbt_pid:-}"' TERM INT echo "Running: sbt $SBT_OPTS \"$SBT_COMMAND\"" diff --git a/e2e-tests/images/entrypoint.sh b/e2e-tests/images/entrypoint.sh index 452d3980fac..84c4ef5d5e9 100644 --- a/e2e-tests/images/entrypoint.sh +++ b/e2e-tests/images/entrypoint.sh @@ -9,32 +9,12 @@ set -euo pipefail -# service -> http port -declare -A PORTS=( - [media-api]=9001 - [thrall]=9002 - [kahuna]=9005 - [cropper]=9006 - [metadata-editor]=9007 - [collections]=9010 - [auth]=9011 - [leases]=9012 -) - -SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall" +# Shared service list, port map, and shutdown helper. +source "$(dirname "$0")/entrypoint.common.sh" pids=() -shutdown() { - echo "Shutting down Grid services..." - for pid in "${pids[@]}"; do - kill -TERM "$pid" 2>/dev/null || true - done - wait - exit 0 -} - -trap shutdown TERM INT +trap 'shutdown_children "${pids[@]}"' TERM INT for svc in $SERVICES; do port="${PORTS[$svc]:-}" @@ -50,7 +30,6 @@ for svc in $SERVICES; do fi echo "Starting $svc on port $port" - # shellcheck disable=SC2086 "$bin" -Dhttp.port="$port" & pids+=("$!") done From ad881e8ca691bb9571316ee51c3c99a6f9d1b9bb Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 13:45:43 +0100 Subject: [PATCH 237/373] Use docker/build-push-action to add layer caching --- .github/workflows/playwright.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 4d242f121a0..4c8ed76b5d3 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -23,9 +23,19 @@ jobs: run: npm ci - name: Install Playwright Browsers run: npx playwright install --with-deps chromium + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Build CI image - working-directory: . - run: DOCKER_BUILDKIT=1 docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci . + uses: docker/build-push-action@v6 + with: + context: . + file: e2e-tests/images/Dockerfile + target: ci + tags: grid-e2e-ci + push: false + load: true + cache-from: type=gha + cache-to: type=gha,mode=max - name: Run Playwright tests run: npm test - uses: actions/upload-artifact@v4 From 4f690b7ab73f424402e3a9ef8cd177658de1993a Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 13:50:34 +0100 Subject: [PATCH 238/373] Add caches for npm and playwright browser deps --- .github/workflows/playwright.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 4c8ed76b5d3..aa7f8d34162 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -19,10 +19,30 @@ jobs: - uses: actions/setup-node@v7 with: node-version: lts/* + # Cache the npm download cache, keyed on the e2e-tests lockfile. + cache: npm + cache-dependency-path: e2e-tests/package-lock.json - name: Install dependencies run: npm ci + # Cache the downloaded Playwright browser binaries. The cache key is tied to + # the resolved Playwright version in the lockfile, so a version bump busts it. + - name: Cache Playwright browsers + id: playwright-cache + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('e2e-tests/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-playwright- - name: Install Playwright Browsers - run: npx playwright install --with-deps chromium + # On a cache hit the browsers are already present, so only install the OS + # dependencies; otherwise download the browsers too. + run: | + if [ "${{ steps.playwright-cache.outputs.cache-hit }}" = "true" ]; then + npx playwright install-deps chromium + else + npx playwright install --with-deps chromium + fi - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build CI image From 90eff74d260335abcd8b66c089ef3784618919d1 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 13:51:06 +0100 Subject: [PATCH 239/373] Correct entrypoint path --- e2e-tests/images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 49e7fb4f7eb..787a696d447 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -108,7 +108,7 @@ RUN set -eux; \ mkdir -p "/var/log/$svc"; \ done -COPY e2e-tests/images/entrypoint-common.sh /usr/local/bin/entrypoint-common.sh +COPY e2e-tests/images/entrypoint.common.sh /usr/local/bin/entrypoint.common.sh COPY e2e-tests/images/entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh From a367119900b17fc22db64c0145ae7e1e26be4a67 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 14:05:40 +0100 Subject: [PATCH 240/373] Remove media-api fixture and docs --- e2e-tests/global-setup.ts | 14 ++++++-------- e2e-tests/playwright.config.ts | 8 -------- e2e-tests/steps/media-api.steps.ts | 21 --------------------- 3 files changed, 6 insertions(+), 37 deletions(-) delete mode 100644 e2e-tests/steps/media-api.steps.ts diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 06cdf31dde6..49b99b2477e 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -91,7 +91,7 @@ async function globalSetup(): Promise { const network = await new Network().start(); - // --- Infrastructure: Elasticsearch + LocalStack -------------------------- + // --- Infrastructure: Elasticsearch + LocalStack const elasticsearch = await new GenericContainer(ELASTICSEARCH_IMAGE) .withNetwork(network) .withNetworkAliases(ELASTICSEARCH_ALIAS) @@ -133,13 +133,11 @@ async function globalSetup(): Promise { const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); generateServiceConfig(configDir, coreStackProps); - // --- Application: the pre-built grid-e2e-ci image --------------------------- - // All eight services run inside this single container and talk to each other over - // its localhost. Each is published on the *fixed* host port its dev-nginx mapping - // expects (dev/nginx-mappings.yml), so the developer's dev-nginx routes the - // https://*.media. domains straight into this container. This trades the - // previous dynamic-port isolation for compatibility with the existing nginx setup: - // it cannot run alongside a locally-running Grid that already owns these ports. + // All eight Grid services run inside this single container and talk to each + // other over its localhost. Each is published on the fixed host port its + // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's + // dev-nginx routes the https://*.media. domains straight into this + // container. let gridBuilder = new GenericContainer(GRID_IMAGE) .withNetwork(network) .withNetworkAliases(GRID_ALIAS) diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index d3579ebaeb1..0f53677ba8d 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -9,14 +9,6 @@ const testDir = defineBddConfig({ steps: './steps/**/*.ts', }); -/** - * Read environment variables from file. - * https://github.com/motdotla/dotenv - */ -// import dotenv from 'dotenv'; -// import path from 'path'; -// dotenv.config({ path: path.resolve(__dirname, '.env') }); - /** * See https://playwright.dev/docs/test-configuration. */ diff --git a/e2e-tests/steps/media-api.steps.ts b/e2e-tests/steps/media-api.steps.ts deleted file mode 100644 index d37bdc1a22c..00000000000 --- a/e2e-tests/steps/media-api.steps.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { When, Then, expect, KAHUNA_APP_URL } from './fixtures'; - -When('I open the Grid application', async ({ page, testContext }) => { - testContext.mediaApiResponse = page.waitForResponse( - (response) => - response.url().startsWith('https://api.media.') && response.request().method() === 'GET', - { timeout: 60_000 }, - ); - - await page.goto(KAHUNA_APP_URL); -}); - -Then('the page exposes a media-api URI link', async ({ page }) => { - const mediaApiLink = await page.getAttribute('link[rel="media-api-uri"]', 'href'); - expect(mediaApiLink).toContain('api.media.'); -}); - -Then('the media-api is served successfully', async ({ testContext }) => { - const response = await testContext.mediaApiResponse!; - expect(response.status()).toBe(200); -}); From 7794056dba0b8d1752c83dda6fd4374b4e4022b6 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 14:17:37 +0100 Subject: [PATCH 241/373] Remove redundant feature file --- e2e-tests/features/media-api.feature | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 e2e-tests/features/media-api.feature diff --git a/e2e-tests/features/media-api.feature b/e2e-tests/features/media-api.feature deleted file mode 100644 index a1b681ff104..00000000000 --- a/e2e-tests/features/media-api.feature +++ /dev/null @@ -1,10 +0,0 @@ -Feature: Frontend loads media-api - - The Kahuna SPA reads the media-api URL from a element and - immediately requests the media-api root to discover its hypermedia links. dev-nginx - routes https://api.media. to the media-api port on the grid-e2e-ci container. - - Scenario: The SPA discovers and requests media-api from the grid-e2e-ci container - When I open the Grid application - Then the page exposes a media-api URI link - And the media-api is served successfully From 30d09a6251239b0fe22419e9c5fa473beea2eea0 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 14:40:23 +0100 Subject: [PATCH 242/373] Turn off Docker layer cache for now which I think is adding more time than it saves --- .github/workflows/playwright.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index aa7f8d34162..212cf567fb7 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -54,8 +54,6 @@ jobs: tags: grid-e2e-ci push: false load: true - cache-from: type=gha - cache-to: type=gha,mode=max - name: Run Playwright tests run: npm test - uses: actions/upload-artifact@v4 From 712285aeace5e37ee1b995d382f675fb33b47aac Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 15:02:55 +0100 Subject: [PATCH 243/373] Remove branch-specific workflow trigger --- .github/workflows/playwright.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 212cf567fb7..b8ce84ca78d 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -1,9 +1,9 @@ name: Playwright Tests on: push: - branches: [ main, master, jsh/e2e-tests ] + branches: [ main ] pull_request: - branches: [ main, master, jsh/e2e-tests] + branches: [ main ] workflow_dispatch: permissions: contents: read From 6d39ad4e7caa3df0aaa34e05bcca3ae973991d91 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 15:25:29 +0100 Subject: [PATCH 244/373] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/playwright.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index b8ce84ca78d..799b51b67ba 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -15,8 +15,8 @@ jobs: run: working-directory: e2e-tests steps: - - uses: actions/checkout@v7 - - uses: actions/setup-node@v7 + - uses: actions/checkout@e8d4307400f9427dba7cb98e488d6ab85f1cec5f # v7.0.0 + - uses: actions/setup-node@v6 with: node-version: lts/* # Cache the npm download cache, keyed on the e2e-tests lockfile. From ebbb390ddf69f841b07255a1850205e9d57801c2 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 15:25:15 +0100 Subject: [PATCH 245/373] Fix a doc regression --- e2e-tests/images/README.md | 53 ++++++++++---------------------------- 1 file changed, 14 insertions(+), 39 deletions(-) diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index a7b12cfb0a2..b343e578f8f 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,66 +1,46 @@ # Grid E2E test images -This directory provides **two** single-container images that run eight Grid Play -services plus Kahuna's frontend: +A single `Dockerfile` produces two single-container images that run eight +Grid Play services plus Kahuna's frontend, selected with `--target`: -- **CI image** — `Dockerfile`: stages pre-compiled artefacts and runs them in a - production-style JRE. Used by the e2e-tests testcontainers harness. -- **Local-dev image** — `Dockerfile.dev`: runs the services under `sbt /run` - (Play dev mode) with the repo bind-mounted, so source changes recompile live. - See [Development image (live reload)](#development-image-live-reload) below. +- **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. +- **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. + +`--target` is required — a plain `docker build` fails with a reminder. ## Build Build from the **repository root** (the build context must be the repo root): ```bash -DOCKER_BUILDKIT=1 docker build -f images/Dockerfile -t grid-all . +DOCKER_BUILDKIT=1 docker build --target -f images/Dockerfile -t grid-e2e- . ``` -The build has three stages: - -1. **frontend** — `node:22.12.0` builds the Kahuna webpack bundle - (`npm ci && npm run dist`). -2. **backend** — `eclipse-temurin:11-jdk` installs sbt and stages all eight - services (`sbt /stage`), packaging the frontend assets into Kahuna. -3. **runtime** — `eclipse-temurin:11-jre` with the native image tools - (`graphicsmagick`, `imagemagick`, `pngquant`, `exiftool`, `libgd3`) and the - staged apps under `/usr/share/`. - ## Runtime configuration Configuration is not baked into the image. At runtime each service loads `/etc/grid/common.conf` and `/etc/grid/.conf` (and reads the stage -from `/etc/grid/stage`; defaults to `DEV`). Mount your environment's config in: +from `/etc/grid/stage`; defaults to `DEV`). Mount your environment's config with: ```bash docker run --rm \ -v "$PWD/my-config:/etc/grid:ro" \ -p 9001:9001 -p 9002:9002 -p 9005:9005 -p 9006:9006 \ -p 9007:9007 -p 9010:9010 -p 9011:9011 -p 9012:9012 \ - grid-all + grid-e2e-[ci|dev] ``` -The services still need their backing infrastructure (Elasticsearch, S3/ -DynamoDB/Kinesis/SNS/SQS, the image resizer, auth provider) to be reachable — -see the root `docker-compose.yml` for the local dev topology. - ## Development image (live reload) -`Dockerfile.dev` is for local development. Instead of staging compiled -artefacts it runs the services under `sbt /run` (Play dev mode), so changed -Scala sources are recompiled on the next request. When `kahuna` is selected its -webpack bundle is rebuilt continuously via `npm run watch`. +The `--target dev` build is for local development. Instead of staging compiled +artefacts it runs the services under `sbt /run` (Play dev mode), so changed Scala sources are recompiled on the next request. When `kahuna` is selected its webpack bundle is rebuilt continuously via `npm run watch`. Build from the repository root: ```bash -DOCKER_BUILDKIT=1 docker build -f e2e-tests/images/Dockerfile.dev -t grid-dev . +DOCKER_BUILDKIT=1 docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . ``` -The build warms the sbt dependency cache (`sbt update`) and Kahuna's -`node_modules` (`npm ci`) so the first run only has to compile sources. - Run with the repo bind-mounted over `/build` so host edits are picked up live, and your DEV config mounted at `/root/.grid`: @@ -69,12 +49,9 @@ docker run --rm \ -v "$PWD:/build" \ -v "$HOME/.grid:/root/.grid:ro" \ -p 9001:9001 -p 9005:9005 -p 9011:9011 \ - grid-dev + grid-e2e-dev ``` -Edit a `.scala` file on the host and hit an endpoint to trigger a recompile; -edit a Kahuna asset and the webpack watcher rebuilds `public/dist`. - ### Options All configurable via environment variables: @@ -91,6 +68,4 @@ docker run --rm -v "$PWD:/build" -v "$HOME/.grid:/root/.grid:ro" \ ``` The services still need their backing infrastructure reachable (see the root -`docker-compose.yml`). The image also works against the e2e-tests testcontainers -harness: mount the generated config the same way the CI image does (to -`/root/.grid` and/or `/etc/grid`) and join the same network. +`docker-compose.yml`). The image also works against the e2e-tests testcontainers harness: mount the generated config the same way the CI image does (to`/root/.grid` and/or `/etc/grid`) and join the same network. From 239823692cd78bafb8c5bedbdd87e53b4e691110 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 15:26:40 +0100 Subject: [PATCH 246/373] Remove generated features files --- .../features/kahuna.feature.spec.js | 29 ------------------- .../features/media-api.feature.spec.js | 24 --------------- 2 files changed, 53 deletions(-) delete mode 100644 e2e-tests/.features-gen/features/kahuna.feature.spec.js delete mode 100644 e2e-tests/.features-gen/features/media-api.feature.spec.js diff --git a/e2e-tests/.features-gen/features/kahuna.feature.spec.js b/e2e-tests/.features-gen/features/kahuna.feature.spec.js deleted file mode 100644 index c560c8d25e2..00000000000 --- a/e2e-tests/.features-gen/features/kahuna.feature.spec.js +++ /dev/null @@ -1,29 +0,0 @@ -// Generated from: features/kahuna.feature -import { test } from "../../steps/fixtures.ts"; - -test.describe('Kahuna service availability', () => { - - test('Healthcheck responds OK', async ({ When, Then, request, testContext }) => { - await When('I request the Kahuna healthcheck endpoint', null, { request, testContext }); - await Then('the response is successful', null, { testContext }); - }); - - test('Kahuna serves the application', async ({ When, Then, request, testContext }) => { - await When('I request the Kahuna root path without following redirects', null, { request, testContext }); - await Then('the response status is below 500', null, { testContext }); - }); - -}); - -// == technical section == - -test.use({ - $test: [({}, use) => use(test), { scope: 'test', box: true }], - $uri: [({}, use) => use('features/kahuna.feature'), { scope: 'test', box: true }], - $bddFileData: [({}, use) => use(bddFileData), { scope: "test", box: true }], -}); - -const bddFileData = [ // bdd-data-start - {"pwTestLine":6,"pickleLine":6,"tags":[],"steps":[{"pwStepLine":7,"gherkinStepLine":7,"keywordType":"Action","textWithKeyword":"When I request the Kahuna healthcheck endpoint","stepMatchArguments":[]},{"pwStepLine":8,"gherkinStepLine":8,"keywordType":"Outcome","textWithKeyword":"Then the response is successful","stepMatchArguments":[]}]}, - {"pwTestLine":11,"pickleLine":10,"tags":[],"steps":[{"pwStepLine":12,"gherkinStepLine":11,"keywordType":"Action","textWithKeyword":"When I request the Kahuna root path without following redirects","stepMatchArguments":[]},{"pwStepLine":13,"gherkinStepLine":12,"keywordType":"Outcome","textWithKeyword":"Then the response status is below 500","stepMatchArguments":[{"group":{"start":29,"value":"500"},"parameterTypeName":"int"}]}]}, -]; // bdd-data-end \ No newline at end of file diff --git a/e2e-tests/.features-gen/features/media-api.feature.spec.js b/e2e-tests/.features-gen/features/media-api.feature.spec.js deleted file mode 100644 index 398c869f67d..00000000000 --- a/e2e-tests/.features-gen/features/media-api.feature.spec.js +++ /dev/null @@ -1,24 +0,0 @@ -// Generated from: features/media-api.feature -import { test } from "../../steps/fixtures.ts"; - -test.describe('Frontend loads media-api', () => { - - test('The SPA discovers and requests media-api from the grid-all container', async ({ When, Then, And, page, testContext }) => { - await When('I open the Grid application', null, { page, testContext }); - await Then('the page exposes a media-api URI link', null, { page }); - await And('the media-api is served successfully', null, { testContext }); - }); - -}); - -// == technical section == - -test.use({ - $test: [({}, use) => use(test), { scope: 'test', box: true }], - $uri: [({}, use) => use('features/media-api.feature'), { scope: 'test', box: true }], - $bddFileData: [({}, use) => use(bddFileData), { scope: "test", box: true }], -}); - -const bddFileData = [ // bdd-data-start - {"pwTestLine":6,"pickleLine":7,"tags":[],"steps":[{"pwStepLine":7,"gherkinStepLine":8,"keywordType":"Action","textWithKeyword":"When I open the Grid application","stepMatchArguments":[]},{"pwStepLine":8,"gherkinStepLine":9,"keywordType":"Outcome","textWithKeyword":"Then the page exposes a media-api URI link","stepMatchArguments":[]},{"pwStepLine":9,"gherkinStepLine":10,"keywordType":"Outcome","textWithKeyword":"And the media-api is served successfully","stepMatchArguments":[]}]}, -]; // bdd-data-end \ No newline at end of file From 23f37dbd14910e186e865d6133154ddd5c5a075b Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 15:41:40 +0100 Subject: [PATCH 247/373] More doc tweaks --- e2e-tests/README.md | 6 +++--- e2e-tests/images/README.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 481f42c3fd9..10893ebc4b8 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -13,7 +13,7 @@ and how you reproduce a CI failure locally: ```bash # From the repo root: build the production image the harness runs. -DOCKER_BUILDKIT=1 docker build --target runtime -f e2e-tests/images/Dockerfile -t grid-e2e-ci . +DOCKER_BUILDKIT=1 docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci . cd e2e-tests npm ci @@ -30,11 +30,11 @@ Caddy reverse proxy on `:443` that replays dev-nginx's subdomain routing for the ### 2. Dev (live recompilation) For iterating on failing tests you can run the services from source with live reload using -the dev image (`docker build --target dev` from [`images/Dockerfile`](images/Dockerfile)), which runs +the dev image, which runs `sbt /run` (recompiling on change) and rebuilds Kahuna's webpack bundle via `npm run watch`. See [`images/README.md`](images/README.md) for how to build and run it. -Useful loop-level commands: +Useful commands: ```bash npm run test:headed # run with a visible browser diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index b343e578f8f..5fd9b1d7751 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -13,7 +13,7 @@ Grid Play services plus Kahuna's frontend, selected with `--target`: Build from the **repository root** (the build context must be the repo root): ```bash -DOCKER_BUILDKIT=1 docker build --target -f images/Dockerfile -t grid-e2e- . +DOCKER_BUILDKIT=1 docker build --target -f e2e-tests/images/Dockerfile -t grid-e2e- . ``` ## Runtime configuration From dbaabe78a6a288f125817f8df827e2036c7901b7 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 15:41:59 +0100 Subject: [PATCH 248/373] Assume e2e-ci when env CI is true, e2e-dev otherwise --- e2e-tests/testcontainers/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index b94cdf70f67..6db42366c3b 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -16,7 +16,7 @@ export const CORE_STACK_NAME = 'grid-dev-core'; export const PERMISSIONS_BUCKET = 'permissions-cache'; /** Pre-built application image (see e2e-tests/images/Dockerfile). Assumed to exist. */ -export const GRID_IMAGE = process.env.GRID_IMAGE ?? 'grid-e2e-ci'; +export const GRID_IMAGE = process.env.CI ? 'grid-e2e-ci' : 'grid-e2e-dev'; export const ELASTICSEARCH_IMAGE = 'docker.elastic.co/elasticsearch/elasticsearch:8.18.3'; export const LOCALSTACK_IMAGE = 'localstack/localstack:4.5.0'; /** Reverse proxy used in CI to stand in for the developer's dev-nginx (see global-setup). */ From 92f866bd14e3a2988d312050b5633750c22850fd Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 3 Aug 2026 17:17:57 +0100 Subject: [PATCH 249/373] Add test:ui --- e2e-tests/README.md | 1 + e2e-tests/global-setup.ts | 14 ++++++++------ e2e-tests/package.json | 3 ++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 10893ebc4b8..beda695272b 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -39,6 +39,7 @@ Useful commands: ```bash npm run test:headed # run with a visible browser npm run test:report # open the last HTML report +npm run test:ui # open browser and test suite, run tests at your leisure ``` Traces are captured `on-first-retry` (see [`playwright.config.ts`](playwright.config.ts)), diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 49b99b2477e..84b661dc9aa 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -91,7 +91,7 @@ async function globalSetup(): Promise { const network = await new Network().start(); - // --- Infrastructure: Elasticsearch + LocalStack + // Infrastructure: Elasticsearch + LocalStack const elasticsearch = await new GenericContainer(ELASTICSEARCH_IMAGE) .withNetwork(network) .withNetworkAliases(ELASTICSEARCH_ALIAS) @@ -118,7 +118,7 @@ async function globalSetup(): Promise { SERVICES: LOCALSTACK_SERVICES, DEFAULT_REGION: REGION, KINESIS_ERROR_PROBABILITY: '0.0', - // Make resource URLs (SQS queues, etc.) resolve via the network alias so the + // Make resource URLs resolve via the network alias so the // app container can reach them, and keep queue URLs path-style. LOCALSTACK_HOST: `${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`, SQS_ENDPOINT_STRATEGY: 'path', @@ -127,7 +127,7 @@ async function globalSetup(): Promise { .start(); started.push(localstack); - // --- Provisioning + config generation ------------------------------------ + // Provisioning + config generation const coreStackProps = await provisionCoreStack(localstack.getConnectionUri()); const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); @@ -156,7 +156,7 @@ async function globalSetup(): Promise { AWS_DEFAULT_REGION: REGION, AWS_CBOR_DISABLE: 'true', }) - .withWaitStrategy(Wait.forHttp('/management/healthcheck', KAHUNA_PORT).forStatusCode(200)) + .withWaitStrategy(Wait.forHttp('/management/healthcheck', MEDIA_API_PORT).forStatusCode(200)) .withStartupTimeout(startupTimeoutMs); if (process.env.GRID_DEBUG) { @@ -170,13 +170,15 @@ async function globalSetup(): Promise { const grid = await gridBuilder.start(); started.push(grid); - // --- Seed Elasticsearch with image fixtures ------------------------------ + // Seed Elasticsearch with image fixtures + // // The app creates the `images` index + `Images_Current` alias on startup; seed once the // stack is healthy so searches during the tests return the fixture documents. const esBaseUrl = `http://${elasticsearch.getHost()}:${elasticsearch.getMappedPort(9200)}`; await seedElasticsearch(esBaseUrl); - // --- CI routing: bundled reverse proxy ----------------------------------- + // CI routing: bundled reverse proxy + // // Locally, the browser reaches the https://*.media. domains via the developer's // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) // start a Caddy proxy that replays the same subdomain routing and terminates TLS with a diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 383dfebe8a7..0b8bb631486 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -6,7 +6,8 @@ "scripts": { "test": "bddgen && playwright test", "test:headed": "bddgen && playwright test --headed", - "test:report": "playwright show-report" + "test:report": "playwright show-report", + "test:ui": "bddgen && playwright test --ui" }, "keywords": [], "author": "", From c3b69429cb49764add027c32d9384a9881eaf0ea Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 7 Aug 2026 16:11:08 +0000 Subject: [PATCH 250/373] srv -> service --- e2e-tests/images/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 787a696d447..c13db239943 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -11,7 +11,7 @@ # docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci . # # * dev (grid-e2e-dev) — local development. Runs the services under -# `sbt /run` (Play dev mode) with the repo +# `sbt /run` (Play dev mode) with the repo # bind-mounted over /build so host edits recompile live. # docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . # @@ -104,8 +104,8 @@ COPY --from=backend /build/metadata-editor/target/universal/stage /usr/share/met COPY --from=backend /build/thrall/target/universal/stage /usr/share/thrall RUN set -eux; \ - for svc in auth collections cropper kahuna leases media-api metadata-editor thrall; do \ - mkdir -p "/var/log/$svc"; \ + for service in auth collections cropper kahuna leases media-api metadata-editor thrall; do \ + mkdir -p "/var/log/$service"; \ done COPY e2e-tests/images/entrypoint.common.sh /usr/local/bin/entrypoint.common.sh @@ -121,7 +121,7 @@ EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] # Stage: development image (grid-e2e-dev). JDK + sbt + Node + native tooling in a -# single stage with warmed caches, running services under `sbt /run`. +# single stage with warmed caches, running services under `sbt /run`. FROM toolchain AS dev ARG NODE_VERSION From 1a24f42a933d56af8fe2149592007cdac60ffc0a Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 7 Aug 2026 16:11:55 +0000 Subject: [PATCH 251/373] entrypoint.sh -> entrypoint.ci.sh --- .devcontainer/user/devcontainer.json | 76 +++++++++++++++++++ e2e-tests/images/Dockerfile | 6 +- .../{entrypoint.sh => entrypoint.ci.sh} | 0 e2e-tests/images/entrypoint.common.sh | 2 +- e2e-tests/testcontainers/constants.ts | 2 +- 5 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 .devcontainer/user/devcontainer.json rename e2e-tests/images/{entrypoint.sh => entrypoint.ci.sh} (100%) diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json new file mode 100644 index 00000000000..3410d4997c8 --- /dev/null +++ b/.devcontainer/user/devcontainer.json @@ -0,0 +1,76 @@ +{ + "name" : "grid", + "customizations" : { + "vscode" : { + "extensions" : [ + "hverlin.mise-vscode", + "scala-lang.scala", + "scalameta.metals", + "GitHub.copilot" + ] + }, + "jetbrains" : { + "plugins" : [ + "com.github.l34130.mise", + "org.intellij.scala", + "NodeJS", + "com.github.copilot" + ] + } + }, + "forwardPorts" : [ + 9001, + 9002, + 9003, + 9004, + 9005, + 9006, + 9007, + 9009, + 9010, + 9011, + 9012, + 6080 + ], + "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", + "remoteUser" : "vscode", + "features" : { + "ghcr.io/devcontainers/features/docker-in-docker:3" : { + "version" : "latest", + "moby" : false, + "dockerDashComposeVersion" : "v2" + }, + "desktop-lite" : { + "webPort" : "6080" + } + }, + "mounts" : [ + { + "source" : "devenv-coursier-cache-volume", + "target" : "/mnt/coursier-cache", + "type" : "volume" + }, + { + "source" : "devenv-ivy-cache-volume", + "target" : "/mnt/ivy-cache", + "type" : "volume" + } + ], + "containerEnv" : { + "DEVENV_IVY_CACHE_MOUNT_DIR" : "/mnt/ivy-cache", + "DEVENV_COURSIER_CACHE_MOUNT_DIR" : "/mnt/coursier-cache" + }, + "capAdd" : [ + "SYS_ADMIN" + ], + "securityOpt" : [ + "seccomp=unconfined" + ], + "runArgs" : [ + "--memory=16g", + "--cpus=8", + "--shm-size=512m" + ], + "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && sudo apt-get update && sudo apt-get install -y --no-install-recommends nginx graphicsmagick imagemagick pngquant libgd-dev libimage-exiftool-perl procps jq && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && echo 'net.ipv4.ip_unprivileged_port_start=443' | sudo tee /etc/sysctl.d/99-grid-e2e.conf && sudo sysctl -w net.ipv4.ip_unprivileged_port_start=443 && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" +} \ No newline at end of file diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index c13db239943..265365f3c4d 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -109,8 +109,8 @@ RUN set -eux; \ done COPY e2e-tests/images/entrypoint.common.sh /usr/local/bin/entrypoint.common.sh -COPY e2e-tests/images/entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod +x /usr/local/bin/entrypoint.sh +COPY e2e-tests/images/entrypoint.ci.sh /usr/local/bin/entrypoint.ci.sh +RUN chmod +x /usr/local/bin/entrypoint.ci.sh # The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. ENV AWS_CBOR_DISABLE=true @@ -118,7 +118,7 @@ ENV AWS_CBOR_DISABLE=true # media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +ENTRYPOINT ["/usr/local/bin/entrypoint.ci.sh"] # Stage: development image (grid-e2e-dev). JDK + sbt + Node + native tooling in a # single stage with warmed caches, running services under `sbt /run`. diff --git a/e2e-tests/images/entrypoint.sh b/e2e-tests/images/entrypoint.ci.sh similarity index 100% rename from e2e-tests/images/entrypoint.sh rename to e2e-tests/images/entrypoint.ci.sh diff --git a/e2e-tests/images/entrypoint.common.sh b/e2e-tests/images/entrypoint.common.sh index fac345f586e..52e2342b162 100644 --- a/e2e-tests/images/entrypoint.common.sh +++ b/e2e-tests/images/entrypoint.common.sh @@ -2,7 +2,7 @@ # # Shared definitions for the Grid container entrypoints (ci and dev). # -# Sourced by entrypoint.sh and entrypoint.dev.sh to provide a single source of +# Sourced by entrypoint.ci.sh and entrypoint.dev.sh to provide a single source of # truth for the service list, their http ports, and the graceful-shutdown trap. # service -> http port diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 6db42366c3b..6d8cdb7ecce 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -29,7 +29,7 @@ export const LOCALSTACK_PORT = 4566; /** Network alias the CI reverse proxy uses to reach the grid-e2e-ci app container. */ export const GRID_ALIAS = 'grid-e2e-ci'; -/** service -> http port, from e2e-tests/images/entrypoint.sh. */ +/** service -> http port, from e2e-tests/images/entrypoint.ci.sh. */ export const SERVICE_PORTS: Record = { 'media-api': 9001, thrall: 9002, From c78207a946504167439610a35567d7ed99b15aaf Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 7 Aug 2026 16:13:35 +0000 Subject: [PATCH 252/373] Docs improvement --- e2e-tests/images/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index 5fd9b1d7751..5ca9ecc9760 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,8 +1,7 @@ -# Grid E2E test images +c# Grid E2E test images A single `Dockerfile` produces two single-container images that run eight -Grid Play services plus Kahuna's frontend, selected with `--target`: - +Grid Play services plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter: - **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. - **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. From fe71706f9b63921081f20f3b1849cfd6b7772ea1 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 7 Aug 2026 16:14:27 +0000 Subject: [PATCH 253/373] Colocate comment with fn to describe purpose --- e2e-tests/testcontainers/provision.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e-tests/testcontainers/provision.ts b/e2e-tests/testcontainers/provision.ts index f5fa99443a6..07820f1479d 100644 --- a/e2e-tests/testcontainers/provision.ts +++ b/e2e-tests/testcontainers/provision.ts @@ -1,8 +1,3 @@ -/** - * Provisions the Grid core infrastructure inside LocalStack: applies the CloudFormation - * core stack, waits for completion, reads the created resource names, and seeds the - * buckets with the config files the services expect (mirroring dev/script/setup.sh). - */ import * as fs from 'fs'; import * as path from 'path'; import { @@ -29,6 +24,11 @@ function clients(endpoint: string) { return { cfn, s3 }; } +/** + * Provisions the Grid core infrastructure inside LocalStack: applies the CloudFormation + * core stack, waits for completion, reads the created resource names, and seeds the + * buckets with the config files the services expect (similar to dev/script/setup.sh). + */ async function createCoreStack(cfn: CloudFormationClient): Promise { const templateBody = fs.readFileSync( path.join(REPO_ROOT, 'dev', 'cloudformation', 'grid-dev-core.yml'), From 2739d66072b9ca2b4f1e953bd821cf2d1d07819c Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 7 Aug 2026 16:15:27 +0000 Subject: [PATCH 254/373] More readable list of localstack services --- e2e-tests/global-setup.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 84b661dc9aa..a470e837443 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -38,7 +38,16 @@ import { provisionCoreStack } from './testcontainers/provision'; import { seedElasticsearch } from './testcontainers/seed-elasticsearch'; import { setEnvironment } from './testcontainers/state'; -const LOCALSTACK_SERVICES = 'cloudformation,cloudwatch,dynamodb,kinesis,s3,sns,sqs,iam'; +const LOCALSTACK_SERVICES = [ + "cloudformation", + "cloudwatch", + "dynamodb", + "kinesis", + "s3", + "sns", + "sqs", + "iam", +].join(","); /** * Build a Caddyfile that reproduces the dev-nginx subdomain routing: each Grid service From e9ab22cec1eeca82d22c010270a3ab270bf6083c Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 7 Aug 2026 16:19:55 +0000 Subject: [PATCH 255/373] Add example output of caddyfile --- e2e-tests/global-setup.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index a470e837443..6431789a9ba 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -55,6 +55,41 @@ const LOCALSTACK_SERVICES = [ * its in-container port, and the S3 vanity domains proxy to localstack (prepending the * real bucket to the path). `tls internal` serves a self-signed cert per site; Playwright * runs with `ignoreHTTPSErrors`, so the internal CA does not need to be trusted. + * + * Example output: + * + * { + * auto_https disable_redirects + * } + * + * media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:9005 + * } + * + * api.media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:9001 + * } + * + * # ... etc for other similar services + * + * images.media.local.dev-gutools.co.uk { + * tls internal + * rewrite * /grid-dev-core-imagebucket-2b34bef1{uri} + * reverse_proxy localhost:4566 + * } + * + * public.media.local.dev-gutools.co.uk { + * tls internal + * rewrite * /grid-dev-core-imageoriginbucket-e8cd1fec{uri} + * reverse_proxy localhost:4566 + * } + * + * localstack.media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:4566 + * } */ function buildCaddyfile(coreStackProps: Record): string { const appServices: Record = { From 9f79df1c0deeed2e0ded2965c13cafb3d6bbcb3c Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 25 Aug 2026 15:47:05 +0000 Subject: [PATCH 256/373] Remove devcontainer setup as to not get ahead of oneself --- .devcontainer/user/devcontainer.json | 76 ---------------------------- 1 file changed, 76 deletions(-) delete mode 100644 .devcontainer/user/devcontainer.json diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json deleted file mode 100644 index 3410d4997c8..00000000000 --- a/.devcontainer/user/devcontainer.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "name" : "grid", - "customizations" : { - "vscode" : { - "extensions" : [ - "hverlin.mise-vscode", - "scala-lang.scala", - "scalameta.metals", - "GitHub.copilot" - ] - }, - "jetbrains" : { - "plugins" : [ - "com.github.l34130.mise", - "org.intellij.scala", - "NodeJS", - "com.github.copilot" - ] - } - }, - "forwardPorts" : [ - 9001, - 9002, - 9003, - 9004, - 9005, - 9006, - 9007, - 9009, - 9010, - 9011, - 9012, - 6080 - ], - "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", - "remoteUser" : "vscode", - "features" : { - "ghcr.io/devcontainers/features/docker-in-docker:3" : { - "version" : "latest", - "moby" : false, - "dockerDashComposeVersion" : "v2" - }, - "desktop-lite" : { - "webPort" : "6080" - } - }, - "mounts" : [ - { - "source" : "devenv-coursier-cache-volume", - "target" : "/mnt/coursier-cache", - "type" : "volume" - }, - { - "source" : "devenv-ivy-cache-volume", - "target" : "/mnt/ivy-cache", - "type" : "volume" - } - ], - "containerEnv" : { - "DEVENV_IVY_CACHE_MOUNT_DIR" : "/mnt/ivy-cache", - "DEVENV_COURSIER_CACHE_MOUNT_DIR" : "/mnt/coursier-cache" - }, - "capAdd" : [ - "SYS_ADMIN" - ], - "securityOpt" : [ - "seccomp=unconfined" - ], - "runArgs" : [ - "--memory=16g", - "--cpus=8", - "--shm-size=512m" - ], - "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", - "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && sudo apt-get update && sudo apt-get install -y --no-install-recommends nginx graphicsmagick imagemagick pngquant libgd-dev libimage-exiftool-perl procps jq && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && echo 'net.ipv4.ip_unprivileged_port_start=443' | sudo tee /etc/sysctl.d/99-grid-e2e.conf && sudo sysctl -w net.ipv4.ip_unprivileged_port_start=443 && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" -} \ No newline at end of file From 2652494cfeff97388cc3b965000c72dae4874b59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 27 Aug 2026 10:18:04 +0000 Subject: [PATCH 257/373] chore(deps): bump guardian/.github/.github/workflows/require-label.yaml Bumps [guardian/.github/.github/workflows/require-label.yaml](https://github.com/guardian/.github) from 2.0.0 to 3.1.2. - [Release notes](https://github.com/guardian/.github/releases) - [Commits](https://github.com/guardian/.github/compare/4cb5024736632ffcc564b7f4b772c38b8e5ce739...3522b599ee9e7d77089f505fed04f084c8b5fac2) --- updated-dependencies: - dependency-name: guardian/.github/.github/workflows/require-label.yaml dependency-version: 3.1.2 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/check-labels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-labels.yaml b/.github/workflows/check-labels.yaml index 75a4a1445bd..bc6ed7677bf 100644 --- a/.github/workflows/check-labels.yaml +++ b/.github/workflows/check-labels.yaml @@ -6,4 +6,4 @@ on: types: [opened, labeled, unlabeled, synchronize, reopened, edited] jobs: require-label: - uses: guardian/.github/.github/workflows/require-label.yaml@4cb5024736632ffcc564b7f4b772c38b8e5ce739 # v2.0.0 + uses: guardian/.github/.github/workflows/require-label.yaml@3522b599ee9e7d77089f505fed04f084c8b5fac2 # v3.1.2 From a3ac2fd4079ca98594cd5928f719ecdca7eabff7 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 23 Jul 2026 11:12:07 +0100 Subject: [PATCH 258/373] Clean up files and add documentation after review --- e2e-tests/testcontainers/config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 3b6108faa37..c9e2b65c4b9 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -20,6 +20,7 @@ import { const GENERATE_CONFIG_DIR = path.join(REPO_ROOT, 'dev', 'script', 'generate-config'); +// The services packaged into the grid-all image (see e2e-tests/images/entrypoint.sh). const GRID_SERVICES = Object.keys(SERVICE_PORTS); type StackProps = Record; From 2afdd13ad9d5491749dc0ae7b02eea64654e4c00 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 5 Aug 2026 15:32:48 +0000 Subject: [PATCH 259/373] Add SBT --- .devcontainer/.gitignore | 2 + .devcontainer/README.md | 39 ++++++++++++++++++ .devcontainer/devenv.yaml | 49 ++++++++++++++++++++++ .devcontainer/shared/devcontainer.json | 56 ++++++++++++++++++++++++++ .tool-versions | 7 +++- 5 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 .devcontainer/.gitignore create mode 100644 .devcontainer/README.md create mode 100644 .devcontainer/devenv.yaml create mode 100644 .devcontainer/shared/devcontainer.json diff --git a/.devcontainer/.gitignore b/.devcontainer/.gitignore new file mode 100644 index 00000000000..8ab6b2d9a06 --- /dev/null +++ b/.devcontainer/.gitignore @@ -0,0 +1,2 @@ +# User-specific devcontainer directory with merged personal preferences +user/ diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000000..f4163e57045 --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,39 @@ +# Dev container config + +This folder contains the configuration to work in a [dev container](https://containers.dev/); +an isolated, reproducible development environment that runs inside a container, typically Docker. + +The configuration skeleton is generated by the Guardian's [devenv](https://github.com/guardian/devenv) config tool. + +## Contents + +### `README.md` + +Edit this file as you see fit. It is created on the first run of `devenv init` but it won't be overwritten +by subsequent runs unless you delete it entirely. + +### `devenv.yaml` + +The source-of-truth configuration for this project's dev container. Edit this file to add/remove devenv modules, +or any other config settings. Then run: +```bash +devenv generate +``` +to regenerate the shared `devcontainer.json` file. + +### `shared/devcontainer.json` + +The main devcontainer configuration, committed to the repository and shared by all developers. +It is generated from `devenv.yaml`. + +> [!IMPORTANT] +> Do not edit this file directly! Instead, edit `devenv.yaml` and then run `devenv generate`. + +### `user/devcontainer.json` + +A personal copy of the devcontainer config, **git-ignored** (see `.gitignore`). +This file is intended for your own overrides: +extra VS Code extensions or Intellij plugins, personal settings or tweaks that shouldn't affect other contributors. +Modify this file freely without worrying about polluting shared config. +> [!IMPORTANT] +> But be aware that `devenv generate` will overwrite it! diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml new file mode 100644 index 00000000000..ad24a8db56f --- /dev/null +++ b/.devcontainer/devenv.yaml @@ -0,0 +1,49 @@ +# Devenv project configuration +# Edit this file to configure your project's devcontainer +# and then run `devenv generate` to create the devcontainer.json files + +# REQUIRED: Change this to your project name +name: "grid" + +# Modules: Built-in functionality +# - mise: Install and configure mise for dev tools management (https://mise.jdx.dev/) +# - docker-in-docker: Enable running Docker containers within the devcontainer +# - scala: Add IDE plugins and jar caching for Scala development +# - node: Add IDE plugins for Node.js development +# - github-copilot: Sets up GitHub Copilot for IDE and CLI use (requires the mise module) +# To disable, comment out or remove items from this list +modules: + - mise + - docker-in-docker # (disabled by default) + - scala # (disabled by default) + - node # (disabled by default) + - github-copilot + +# Optional: Ports to forward +# forwardPorts: +# - 8080 # same port on host and container +# - "8000:9000" # hostPort:containerPort + +# Optional: Mount directories from host to container +# mounts: +# - "source=${localEnv:HOME}/.gu/example,target=/home/vscode/.gu/example,readonly,type=bind,consistency=cached" + +# Optional: IDE plugins - shared project plugins like language support +# plugins: +# vscode: [] +# intellij: [] + +# Optional: Commands to run after container creation +# Each command has a 'cmd' field and a 'workingDirectory' field +# e.g. +# postCreateCommand: +# - cmd: "npm install" +# workingDirectory: "." +# - cmd: "make setup" +# workingDirectory: "/workspaces/project" +# postStartCommand: +# - cmd: "echo 'Container started successfully'" +# workingDirectory: "." +postCreateCommand: + - cmd: "sudo apt-get update && sudo apt-get install -y --no-install-recommends nginx graphicsmagick imagemagick pngquant libgd-dev libimage-exiftool-perl procps jq" + workingDirectory: "." diff --git a/.devcontainer/shared/devcontainer.json b/.devcontainer/shared/devcontainer.json new file mode 100644 index 00000000000..2f24aa7cc15 --- /dev/null +++ b/.devcontainer/shared/devcontainer.json @@ -0,0 +1,56 @@ +{ + "name" : "grid", + "customizations" : { + "vscode" : { + "extensions" : [ + "hverlin.mise-vscode", + "scala-lang.scala", + "scalameta.metals", + "GitHub.copilot" + ] + }, + "jetbrains" : { + "plugins" : [ + "com.github.l34130.mise", + "org.intellij.scala", + "NodeJS", + "com.github.copilot" + ] + } + }, + "forwardPorts" : [ + ], + "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", + "remoteUser" : "vscode", + "features" : { + "ghcr.io/devcontainers/features/docker-in-docker:3" : { + "version" : "latest", + "moby" : false, + "dockerDashComposeVersion" : "v2" + } + }, + "mounts" : [ + { + "source" : "devenv-coursier-cache-volume", + "target" : "/mnt/coursier-cache", + "type" : "volume" + }, + { + "source" : "devenv-ivy-cache-volume", + "target" : "/mnt/ivy-cache", + "type" : "volume" + } + ], + "containerEnv" : { + "DEVENV_IVY_CACHE_MOUNT_DIR" : "/mnt/ivy-cache", + "DEVENV_COURSIER_CACHE_MOUNT_DIR" : "/mnt/coursier-cache" + }, + "capAdd" : [ + "SYS_ADMIN" + ], + "securityOpt" : [ + "seccomp=unconfined" + ], + "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && sudo apt-get update && sudo apt-get install -y --no-install-recommends nginx graphicsmagick imagemagick pngquant libgd-dev libimage-exiftool-perl procps jq && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" +} \ No newline at end of file diff --git a/.tool-versions b/.tool-versions index 135d79b17d6..a54fe40beaf 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,5 @@ -java corretto-11.0.24.8.1 -nodejs 22.12.0 +java latest +nodejs 22.12.0 +github:guardian/devenv 20260721-123412 +aws-cli latest +sbt latest From a012810af5c59e8d71d0a527d15944572924a271 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 25 Aug 2026 09:57:19 +0000 Subject: [PATCH 260/373] Add configuration to devenv expose ports, add modules, and build containers on startup --- .devcontainer/devenv.yaml | 44 +++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml index ad24a8db56f..c2a6c43cb1a 100644 --- a/.devcontainer/devenv.yaml +++ b/.devcontainer/devenv.yaml @@ -2,27 +2,32 @@ # Edit this file to configure your project's devcontainer # and then run `devenv generate` to create the devcontainer.json files -# REQUIRED: Change this to your project name name: "grid" # Modules: Built-in functionality -# - mise: Install and configure mise for dev tools management (https://mise.jdx.dev/) -# - docker-in-docker: Enable running Docker containers within the devcontainer -# - scala: Add IDE plugins and jar caching for Scala development -# - node: Add IDE plugins for Node.js development -# - github-copilot: Sets up GitHub Copilot for IDE and CLI use (requires the mise module) -# To disable, comment out or remove items from this list modules: - mise - - docker-in-docker # (disabled by default) - - scala # (disabled by default) - - node # (disabled by default) + - docker-in-docker + - scala - github-copilot +features: + - ghcr.io/devcontainers/features/aws-cli:1.1.4 + # Optional: Ports to forward -# forwardPorts: -# - 8080 # same port on host and container -# - "8000:9000" # hostPort:containerPort +forwardPorts: + - 9001 + - 9002 + - 9003 + - 9004 + - 9005 + - 9006 + - 9007 + - 9009 + - 9010 + - 9011 + - 9012 + - 6080 # Optional: Mount directories from host to container # mounts: @@ -33,17 +38,6 @@ modules: # vscode: [] # intellij: [] -# Optional: Commands to run after container creation -# Each command has a 'cmd' field and a 'workingDirectory' field -# e.g. -# postCreateCommand: -# - cmd: "npm install" -# workingDirectory: "." -# - cmd: "make setup" -# workingDirectory: "/workspaces/project" -# postStartCommand: -# - cmd: "echo 'Container started successfully'" -# workingDirectory: "." postCreateCommand: - - cmd: "sudo apt-get update && sudo apt-get install -y --no-install-recommends nginx graphicsmagick imagemagick pngquant libgd-dev libimage-exiftool-perl procps jq" + - cmd: "docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev ." workingDirectory: "." From 1b48f8dafd72c7c8a884cd95e174b68c1aa2d15a Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 25 Aug 2026 09:57:43 +0000 Subject: [PATCH 261/373] Remove aws-cli as a dep from tool-versions, as it's taken care of by the devenv feature --- .tool-versions | 1 - 1 file changed, 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index a54fe40beaf..23922dffb01 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,5 +1,4 @@ java latest nodejs 22.12.0 github:guardian/devenv 20260721-123412 -aws-cli latest sbt latest From 5048e6ce2da3b0d4964554314d8f97fb83d3082c Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 25 Aug 2026 09:57:59 +0000 Subject: [PATCH 262/373] Use ui-port rather than --ui, as we're now in a devcontainer context --- e2e-tests/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 0b8bb631486..af46044d324 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -7,7 +7,7 @@ "test": "bddgen && playwright test", "test:headed": "bddgen && playwright test --headed", "test:report": "playwright show-report", - "test:ui": "bddgen && playwright test --ui" + "test:ui": "bddgen && playwright test --ui-port 6080" }, "keywords": [], "author": "", From e9a1ee27372968df80ee1ef0a2e40d10b6dd5068 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 25 Aug 2026 11:16:28 +0100 Subject: [PATCH 263/373] Revert Java version change --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index 23922dffb01..59808c18f04 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,4 +1,4 @@ -java latest +java corretto-11.0.24.8.1 nodejs 22.12.0 github:guardian/devenv 20260721-123412 sbt latest From a09c5681660761cb4595ffd5d8835cb29f734750 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 25 Aug 2026 11:19:51 +0100 Subject: [PATCH 264/373] Run Playwright tests on pull_request --- .github/workflows/playwright.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 799b51b67ba..3ff622adaae 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -3,7 +3,6 @@ on: push: branches: [ main ] pull_request: - branches: [ main ] workflow_dispatch: permissions: contents: read From 1ac810e149795fb9a2c1f3d098910981089059be Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 25 Aug 2026 10:58:56 +0000 Subject: [PATCH 265/373] Adjust whitespace in .tool-versions, which may fix setup-scala --- .tool-versions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.tool-versions b/.tool-versions index 59808c18f04..dd66c7e263f 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,4 +1,4 @@ -java corretto-11.0.24.8.1 -nodejs 22.12.0 +java corretto-11.0.24.8.1 +nodejs 22.12.0 github:guardian/devenv 20260721-123412 -sbt latest +sbt latest From 3be76caf30c034681609350bd5fda963718bf9ee Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 25 Aug 2026 11:47:58 +0000 Subject: [PATCH 266/373] Regenerate devcontainer config --- .devcontainer/devenv.yaml | 2 +- .devcontainer/shared/devcontainer.json | 18 ++++++- .devcontainer/user/devcontainer.json | 75 ++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/user/devcontainer.json diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml index c2a6c43cb1a..3d34c08c631 100644 --- a/.devcontainer/devenv.yaml +++ b/.devcontainer/devenv.yaml @@ -12,7 +12,7 @@ modules: - github-copilot features: - - ghcr.io/devcontainers/features/aws-cli:1.1.4 + ghcr.io/devcontainers/features/aws-cli:1.1.4: {} # Optional: Ports to forward forwardPorts: diff --git a/.devcontainer/shared/devcontainer.json b/.devcontainer/shared/devcontainer.json index 2f24aa7cc15..70b42b009f7 100644 --- a/.devcontainer/shared/devcontainer.json +++ b/.devcontainer/shared/devcontainer.json @@ -13,12 +13,23 @@ "plugins" : [ "com.github.l34130.mise", "org.intellij.scala", - "NodeJS", "com.github.copilot" ] } }, "forwardPorts" : [ + 9001, + 9002, + 9003, + 9004, + 9005, + 9006, + 9007, + 9009, + 9010, + 9011, + 9012, + 6080 ], "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", "remoteUser" : "vscode", @@ -27,6 +38,9 @@ "version" : "latest", "moby" : false, "dockerDashComposeVersion" : "v2" + }, + "ghcr.io/devcontainers/features/aws-cli:1.1.4" : { + } }, "mounts" : [ @@ -52,5 +66,5 @@ "seccomp=unconfined" ], "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", - "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && sudo apt-get update && sudo apt-get install -y --no-install-recommends nginx graphicsmagick imagemagick pngquant libgd-dev libimage-exiftool-perl procps jq && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" } \ No newline at end of file diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json new file mode 100644 index 00000000000..be2590a0ae9 --- /dev/null +++ b/.devcontainer/user/devcontainer.json @@ -0,0 +1,75 @@ +{ + "name" : "grid", + "customizations" : { + "vscode" : { + "extensions" : [ + "hverlin.mise-vscode", + "scala-lang.scala", + "scalameta.metals", + "GitHub.copilot" + ] + }, + "jetbrains" : { + "plugins" : [ + "com.github.l34130.mise", + "org.intellij.scala", + "com.github.copilot" + ] + } + }, + "forwardPorts" : [ + 9001, + 9002, + 9003, + 9004, + 9005, + 9006, + 9007, + 9009, + 9010, + 9011, + 9012, + 6080 + ], + "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", + "remoteUser" : "vscode", + "features" : { + "ghcr.io/devcontainers/features/docker-in-docker:3" : { + "version" : "latest", + "moby" : false, + "dockerDashComposeVersion" : "v2" + }, + "ghcr.io/devcontainers/features/aws-cli:1.1.4" : { + + } + }, + "mounts" : [ + { + "source" : "devenv-coursier-cache-volume", + "target" : "/mnt/coursier-cache", + "type" : "volume" + }, + { + "source" : "devenv-ivy-cache-volume", + "target" : "/mnt/ivy-cache", + "type" : "volume" + } + ], + "containerEnv" : { + "DEVENV_IVY_CACHE_MOUNT_DIR" : "/mnt/ivy-cache", + "DEVENV_COURSIER_CACHE_MOUNT_DIR" : "/mnt/coursier-cache" + }, + "capAdd" : [ + "SYS_ADMIN" + ], + "securityOpt" : [ + "seccomp=unconfined" + ], + "runArgs" : [ + "--memory=16g", + "--cpus=8", + "--shm-size=512m" + ], + "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" +} \ No newline at end of file From b40907339beb46f7d9cf94eb0c061bc01952bffe Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 26 Aug 2026 09:47:34 +0100 Subject: [PATCH 267/373] Tidy up after rebase --- e2e-tests/testcontainers/config.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index c9e2b65c4b9..3b6108faa37 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -20,7 +20,6 @@ import { const GENERATE_CONFIG_DIR = path.join(REPO_ROOT, 'dev', 'script', 'generate-config'); -// The services packaged into the grid-all image (see e2e-tests/images/entrypoint.sh). const GRID_SERVICES = Object.keys(SERVICE_PORTS); type StackProps = Record; From 8d396ff57a9225b9011c7ce2c49d7b27de87536b Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 28 Aug 2026 13:08:18 +0000 Subject: [PATCH 268/373] Add npm ci (in ./e2e-tests) to postCreateCommand --- .devcontainer/devenv.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml index 3d34c08c631..29218b922bd 100644 --- a/.devcontainer/devenv.yaml +++ b/.devcontainer/devenv.yaml @@ -41,3 +41,5 @@ forwardPorts: postCreateCommand: - cmd: "docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev ." workingDirectory: "." + - cmd: "npm ci" + workingDirectory: "./e2e-tests" From aa4238b476e85704cdd3dd3723bb94871008074e Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 28 Aug 2026 13:08:45 +0000 Subject: [PATCH 269/373] pin SBT to v1 in .tool-versions --- .tool-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.tool-versions b/.tool-versions index dd66c7e263f..b1348086897 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,4 +1,4 @@ java corretto-11.0.24.8.1 nodejs 22.12.0 github:guardian/devenv 20260721-123412 -sbt latest +sbt 1 From f0941794dd0ff3be47bdc2531060ba1be40da7eb Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 28 Aug 2026 13:25:14 +0000 Subject: [PATCH 270/373] Regenerate devenv --- .devcontainer/shared/devcontainer.json | 2 +- .devcontainer/user/devcontainer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/shared/devcontainer.json b/.devcontainer/shared/devcontainer.json index 70b42b009f7..0a134da5168 100644 --- a/.devcontainer/shared/devcontainer.json +++ b/.devcontainer/shared/devcontainer.json @@ -66,5 +66,5 @@ "seccomp=unconfined" ], "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", - "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd ./e2e-tests && npm ci && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" } \ No newline at end of file diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json index be2590a0ae9..832dc0f4d3a 100644 --- a/.devcontainer/user/devcontainer.json +++ b/.devcontainer/user/devcontainer.json @@ -71,5 +71,5 @@ "--shm-size=512m" ], "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", - "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd ./e2e-tests && npm ci && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" } \ No newline at end of file From efa01b5c27b66603b1fdcade0d6bc9e46b337474 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 28 Aug 2026 13:56:07 +0000 Subject: [PATCH 271/373] Ensure npm is on the PATH when running in postCreateCommand --- .devcontainer/devenv.yaml | 2 +- .devcontainer/shared/devcontainer-lock.json | 14 ++++++++++++++ .devcontainer/shared/devcontainer.json | 2 +- .devcontainer/user/devcontainer.json | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/shared/devcontainer-lock.json diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml index 29218b922bd..661a33352ad 100644 --- a/.devcontainer/devenv.yaml +++ b/.devcontainer/devenv.yaml @@ -41,5 +41,5 @@ forwardPorts: postCreateCommand: - cmd: "docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev ." workingDirectory: "." - - cmd: "npm ci" + - cmd: 'PATH="$HOME/.local/share/mise/shims:$PATH" npm ci' workingDirectory: "./e2e-tests" diff --git a/.devcontainer/shared/devcontainer-lock.json b/.devcontainer/shared/devcontainer-lock.json new file mode 100644 index 00000000000..20a580bcf17 --- /dev/null +++ b/.devcontainer/shared/devcontainer-lock.json @@ -0,0 +1,14 @@ +{ + "features": { + "ghcr.io/devcontainers/features/aws-cli:1.1.4": { + "version": "1.1.4", + "resolved": "ghcr.io/devcontainers/features/aws-cli@sha256:1f93c8315b7a6d76982ebb2269f8b0d50413fc0f965c032edf4aee0caceb73ef", + "integrity": "sha256:1f93c8315b7a6d76982ebb2269f8b0d50413fc0f965c032edf4aee0caceb73ef" + }, + "ghcr.io/devcontainers/features/docker-in-docker:3": { + "version": "3.1.0", + "resolved": "ghcr.io/devcontainers/features/docker-in-docker@sha256:62e04ec83d944da4fd2830f68a677d1f5466c0654aea1e4baa30232d16029ac1", + "integrity": "sha256:62e04ec83d944da4fd2830f68a677d1f5466c0654aea1e4baa30232d16029ac1" + } + } +} diff --git a/.devcontainer/shared/devcontainer.json b/.devcontainer/shared/devcontainer.json index 0a134da5168..da4c20e25bf 100644 --- a/.devcontainer/shared/devcontainer.json +++ b/.devcontainer/shared/devcontainer.json @@ -66,5 +66,5 @@ "seccomp=unconfined" ], "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", - "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd ./e2e-tests && npm ci && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd ./e2e-tests && PATH=\"$HOME/.local/share/mise/shims:$PATH\" npm ci && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" } \ No newline at end of file diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json index 832dc0f4d3a..59b78adfb87 100644 --- a/.devcontainer/user/devcontainer.json +++ b/.devcontainer/user/devcontainer.json @@ -71,5 +71,5 @@ "--shm-size=512m" ], "onCreateCommand" : "((printf \"\\033[1;34m[setup] Starting scalaOnCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgSlZNLXJlbGF0ZWQgY2FjaGVzIGluc2lkZSB0aGUgZGV2Y29udGFpbmVyIGFmdGVyIGNyZWF0aW9uLgoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApvaygpICAgIHsgcHJpbnRmICJcMDMzWzE7MzJtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgZ3JlZW4Kd2FybigpICB7IHByaW50ZiAiXDAzM1sxOzMzbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdvbGQKbG9nKCkgICB7IHByaW50ZiAiXDAzM1sxOzM2bVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGN5YW4KCmlmIFtbIC16ICRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0RJUiBdXTsgdGhlbgogICAgd2FybiAiREVWRU5WX0lWWV9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKaWYgW1sgLXogJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgXV07IHRoZW4KICAgIHdhcm4gIkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9ESVIgbm90IHNldCIKICAgIGV4aXQgMQpmaQoKIyBXZSB3aWxsIHVzZSB0aGlzIHRvIGNob3duIGJlbG93CkRFVkVOVl9DT05UQUlORVJfVVNFUj0kKHdob2FtaSkKCkRFVkVOVl9JVllfVVNFUl9ESVI9Ii9ob21lLyRERVZFTlZfQ09OVEFJTkVSX1VTRVIvLml2eTIiCkRFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSz0iJERFVkVOVl9JVllfVVNFUl9ESVIvY2FjaGUiCgpsb2cgIkxpbmsgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKcm1kaXIgIiRERVZFTlZfSVZZX0NBQ0hFX01PVU5UX0xJTksiCmxuIC1zZiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9JVllfQ0FDSEVfTU9VTlRfTElOSyIKCmxvZyAiRW5zdXJpbmcgY29ycmVjdCBvd25lcnNoaXAgb2YgdGhlIHNoYXJlZCBpdnkgZGF0YSB2b2x1bWUuIgpzdWRvIGNob3duIC1STCAiJERFVkVOVl9DT05UQUlORVJfVVNFUiI6IiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiICIkREVWRU5WX0lWWV9VU0VSX0RJUiIKCkRFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUj0iL2hvbWUvJERFVkVOVl9DT05UQUlORVJfVVNFUi8uY2FjaGUiCkRFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LPSIkREVWRU5WX0NPVVJTSUVSX1VTRVJfRElSL2NvdXJzaWVyL3YxIgoKbG9nICJMaW5rIHRoZSBzaGFyZWQgY291cnNpZXIgZGF0YSB2b2x1bWUgYXQgdGhlIGNvcnJlY3QgcG9pbnQuIgpta2RpciAtcCAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpybWRpciAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgpsbiAtc2YgIiRERVZFTlZfQ09VUlNJRVJfQ0FDSEVfTU9VTlRfRElSIiAiJERFVkVOVl9DT1VSU0lFUl9DQUNIRV9NT1VOVF9MSU5LIgoKbG9nICJFbnN1cmluZyBjb3JyZWN0IG93bmVyc2hpcCBvZiB0aGUgc2hhcmVkIGNvdXJzaWVyIGRhdGEgdm9sdW1lLiIKc3VkbyBjaG93biAtUkwgIiRERVZFTlZfQ09OVEFJTkVSX1VTRVIiOiIkREVWRU5WX0NPTlRBSU5FUl9VU0VSIiAiJERFVkVOVl9DT1VSU0lFUl9VU0VSX0RJUiI=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished scalaOnCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! scalaOnCreateCommand.sh\\033[0m\\n\")) 2>&1 | sudo tee /var/log/on-create.log", - "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd ./e2e-tests && npm ci && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" + "postCreateCommand" : "((printf \"\\033[1;34m[setup] Starting misePostCreateCommand.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAojIFNldHMgdXAgbWlzZSBpbnNpZGUgdGhlIGRldmNvbnRhaW5lciBhZnRlciBjcmVhdGlvbi4KIwojIEFsc28gbG9ncyB1c2VmdWwgaW5mbyAoaW5jbHVkaW5nIHRoZSBvdXRwdXQgZnJvbSBgbWlzZSBkb2N0b3JgKSB0byB0aGUgdGVybWluYWwgdG8gaGVscCB3aXRoIGRlYnVnZ2luZy4KCmVycm9yKCkgeyBwcmludGYgIlwwMzNbMTszMW1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyByZWQKb2soKSAgICB7IHByaW50ZiAiXDAzM1sxOzMybVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIGdyZWVuCndhcm4oKSAgeyBwcmludGYgIlwwMzNbMTszM21bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBnb2xkCmxvZygpICAgeyBwcmludGYgIlwwMzNbMTszNm1bLi4uXSAlc1wwMzNbMG1cbiIgIiQqIjsgfSAgIyBjeWFuCgppZiB3aGljaCBtaXNlICYmIG1pc2UgLS12ZXJzaW9uOyB0aGVuCiAgbG9nICJtaXNlIGlzIGFscmVhZHkgcHJlc2VudCBhdCAkKHdoaWNoIG1pc2UpLiIKZWxzZQogIGxvZyAiSW5zdGFsbGluZyBtaXNlLi4uIgogIGN1cmwgLWZzU0wgaHR0cHM6Ly9taXNlLnJ1bi9iYXNoIHwgc2gKZmkKCmxvZyAiQ2hlY2tpbmcgbWlzZSBpcyB3b3JraW5nIGNvcnJlY3RseSBhbmQgb24gdGhlIHBhdGguIgptaXNlIC0tdmVyc2lvbgoKbG9nICJVcGRhdGluZyBtaXNlIGlmIG5lZWRlZC4iCm1pc2Ugc2VsZi11cGRhdGUgLS15ZXMgfHwgd2FybiAiU2tpcHBpbmcgbWlzZSBzZWxmLXVwZGF0ZSAtIHlvdSBtYXkgYmUgb2ZmbGluZS4iCgpsb2cgIlRydXN0aW5nIG1pc2UgY29uZmlnIChzZWU6IGh0dHBzOi8vbWlzZS5qZHguZGV2L2NsaS90cnVzdC5odG1sKS4iCm1pc2UgdHJ1c3QgLS15ZXMgfHwgdHJ1ZQoKbG9nICJJbnN0YWxsaW5nIG1pc2UgdG9vbGluZy4iCm1pc2UgaW5zdGFsbCB8fCB3YXJuICJtaXNlIGluc3RhbGwgZmFpbGVkLiBZb3UgbWF5IG5lZWQgdG8gcnVuIG1pc2UgaW5zdGFsbCBtYW51YWxseSBpbnNpZGUgdGhlIGNvbnRhaW5lci4iCgpsb2cgIkVuc3VyZSB0aGF0IG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoIGlzIGluIGJhc2hyYy4iCnNlZCAtaSAncy9taXNlIGFjdGl2YXRlIGJhc2gvbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gvJyB+Ly5iYXNocmMKZ3JlcCAnbWlzZSBhY3RpdmF0ZSAtLXNoaW1zIGJhc2gnIH4vLmJhc2hyYyB8fCBlY2hvICdldmFsICIkKG1pc2UgYWN0aXZhdGUgLS1zaGltcyBiYXNoKSIgI0FkZGVkIGJ5IGRldmVudicgfCB0ZWUgLWEgfi8uYmFzaHJjCgpsb2cgIkNoZWNrIHRoZXJlIGlzIGV4YWN0bHkgb25lIGFjdGl2YXRlIGNvbW1hbmQgaW4gYmFzaHJjLiIKdGVzdCAiJChncmVwIC1jICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIiAtZXEgMSB8fCBlY2hvICdEaWQgbm90IGZpbmQgZXhhY3RseSBvbmUgYWN0aXZhdGUgY29tbWFuZCcKCmxvZyAiUnVuIHRoZSBhY3RpdmF0ZSBjb21tYW5kIGZyb20gYmFzaHJjLiIKZXZhbCAiJChncmVwICdtaXNlIGFjdGl2YXRlJyB+Ly5iYXNocmMpIgoKbG9nICJMaXN0IGluc3RhbGxlZCB0b29scy4iCm1pc2UgbGlzdAoKbG9nICJGaW5hbCBjaGVja3MuIgptaXNlIGRvY3Rvcgo=\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished misePostCreateCommand.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! misePostCreateCommand.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting githubCopilotPostCreate.sh\\033[0m\\n\" && (cd . && printf '%s' \"IyEvdXNyL2Jpbi9lbnYgYmFzaAoKZXJyb3IoKSB7IHByaW50ZiAiXDAzM1sxOzMxbVsuLi5dICVzXDAzM1swbVxuIiAiJCoiOyB9ICAjIHJlZApsb2coKSAgIHsgcHJpbnRmICJcMDMzWzE7MzZtWy4uLl0gJXNcMDMzWzBtXG4iICIkKiI7IH0gICMgY3lhbgoKIyBBZGRzIGBnaGAgKEdpdEh1YiBDTEkpIGFuZCBgY29waWxvdGAgKEdpdEh1YiBDb3BpbG90IENMSSkgdG8gdGhlIHVzZXIncyBQQVRIIHZpYSBtaXNlLgppZiBjb21tYW5kIC12IG1pc2UgPi9kZXYvbnVsbCAyPiYxOyB0aGVuCiAgbG9nICJJbnN0YWxsaW5nIEdpdEh1YiBDTEkiCiAgbWlzZSB1c2UgZ2hAbGF0ZXN0IC0tZ2xvYmFsCgogIGxvZyAiSW5zdGFsbGluZyBHaXRIdWIgQ29waWxvdCBDTEkiCiAgbWlzZSB1c2UgY29waWxvdEBsYXRlc3QgLS1nbG9iYWwKZWxzZQogIGVycm9yICJtaXNlIGlzIHJlcXVpcmVkOyBza2lwcGluZyB0b29sIGluc3RhbGxhdGlvbiBhbmQgY2hlY2tzLiIKICBleGl0IDEKZmkK\" | base64 -d | bash -euo pipefail && printf \"\\033[1;32m[setup] Finished githubCopilotPostCreate.sh\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! githubCopilotPostCreate.sh\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd . && docker build --target dev -f e2e-tests/images/Dockerfile -t grid-e2e-dev . && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\") && (printf \"\\033[1;34m[setup] Starting unknown\\033[0m\\n\" && (cd ./e2e-tests && PATH=\"$HOME/.local/share/mise/shims:$PATH\" npm ci && printf \"\\033[1;32m[setup] Finished unknown\\033[0m\\n\") || printf \"\\033[1;31m[setup] Errored! unknown\\033[0m\\n\")) 2>&1 | sudo tee /var/log/post-create.log" } \ No newline at end of file From 5e5178cb1a51f51b5f6ccb72af436b8f88f9f23f Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Fri, 28 Aug 2026 15:31:24 +0100 Subject: [PATCH 272/373] Enable recording download usage for local config creation --- dev/script/generate-config/service-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/script/generate-config/service-config.js b/dev/script/generate-config/service-config.js index d937984c866..b5820bba78e 100644 --- a/dev/script/generate-config/service-config.js +++ b/dev/script/generate-config/service-config.js @@ -38,7 +38,7 @@ function getCommonConfig(config) { |thrall.kinesis.lowPriorityStream.name="${config.coreStackProps.ThrallLowPriorityMessageStream}" |es.index.aliases.current="Images_Current" |es.index.aliases.migration="Images_Migration" - |image.record.download=false + |image.record.download=true |defaultShouldBlurGraphicImages=true |filters.shouldDisplayOrgOwnedCountAndFilterCheckbox=true |ai.search.enabled=${aiSearchEnabled} From ed5aa7627acbee2fa927970887531acf3cdbab9e Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Fri, 28 Aug 2026 17:14:20 +0100 Subject: [PATCH 273/373] Poll for usage updates on download image --- .../gr-downloader/gr-downloader.html | 1 + .../components/gr-downloader/gr-downloader.js | 33 ++++++++++++++++-- .../gr-image-usage/gr-image-usage.js | 34 +++++++++++++------ kahuna/public/js/image/controller.js | 5 ++- kahuna/public/js/services/image/usages.js | 1 - 5 files changed, 57 insertions(+), 17 deletions(-) diff --git a/kahuna/public/js/components/gr-downloader/gr-downloader.html b/kahuna/public/js/components/gr-downloader/gr-downloader.html index dd4a277554b..2974341eaf6 100644 --- a/kahuna/public/js/components/gr-downloader/gr-downloader.html +++ b/kahuna/public/js/components/gr-downloader/gr-downloader.html @@ -26,6 +26,7 @@ href="{{ ctrl.firstImageUris.uris.downloadUri }}" download rel="noopener" target="_blank" aria-label="Download image" class="side-padded" + ng-click="ctrl.onSingleDownloadClicked()" > Download diff --git a/kahuna/public/js/components/gr-downloader/gr-downloader.js b/kahuna/public/js/components/gr-downloader/gr-downloader.js index 95e9c560c88..54de2ed5374 100644 --- a/kahuna/public/js/components/gr-downloader/gr-downloader.js +++ b/kahuna/public/js/components/gr-downloader/gr-downloader.js @@ -3,18 +3,24 @@ import './gr-downloader.css'; import template from './gr-downloader.html'; import '../../services/image/downloads'; +import '../../services/api/media-api'; + export const downloader = angular.module('gr.downloader', [ - 'gr.image-downloads.service' + 'gr.image-downloads.service', + 'kahuna.services.api.media' ]); downloader.controller('DownloaderCtrl', [ '$window', '$q', + '$rootScope', '$scope', 'inject$', 'imageDownloadsService', + 'apiPoll', + 'mediaApi', - function Controller($window, $q, $scope, inject$, imageDownloadsService) { + function Controller($window, $q, $rootScope, $scope, inject$, imageDownloadsService, apiPoll, mediaApi) { let ctrl = this; @@ -60,6 +66,23 @@ downloader.controller('DownloaderCtrl', [ inject$($scope, uris$, ctrl, 'firstImageUris'); + function pollForDownloadUsageUpdate(image, downloadedAt) { + return mediaApi.getSession().then(session => { + return apiPoll(() => image.get().then(updatedImage => { + const hasNewUsageByCurrentUser = updatedImage.data.usages.data.some( + u => u.data.platform === 'download' && + u.data.downloadUsageMetadata && + u.data.downloadUsageMetadata.downloadedBy === session.user.email && + new Date(u.data.dateAdded) > downloadedAt + ); + if (!hasNewUsageByCurrentUser) { + return $q.reject(); + } + $rootScope.$emit('images-updated', [updatedImage]); + })); + }); + } + ctrl.download = (downloadKey) => { ctrl.downloading = true; @@ -94,6 +117,12 @@ downloader.controller('DownloaderCtrl', [ throw e; }); }; + + ctrl.onSingleDownloadClicked = () => { + if (window._clientConfig.recordDownloadAsUsage && ctrl.firstImageUris && ctrl.firstImageUris.uris.downloadUri) { + pollForDownloadUsageUpdate(ctrl.imagesArray()[0], new Date()).catch(() => {}); + } + }; }; }]); diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage.js b/kahuna/public/js/components/gr-image-usage/gr-image-usage.js index ff7b23b7789..62521ee2e04 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage.js +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage.js @@ -22,21 +22,32 @@ export const module = angular.module('gr.imageUsage', [ module.controller('grImageUsageCtrl', [ '$scope', + '$rootScope', '$state', 'inject$', '$window', 'imageUsagesService', - function ($scope, $state, inject$, $window, imageUsagesService) { + function ($scope, $rootScope, $state, inject$, $window, imageUsagesService) { const ctrl = this; + const bindUsages = (image) => { + const usages = imageUsagesService.getUsages(image); + const usages$ = usages.groupedByState$ + .map((grouped) => grouped + .map(list => list.sortBy(usage => usage.get('dateAdded')).reverse()) + .toJS() + ); + inject$($scope, usages$, ctrl, 'usages'); + inject$($scope, usages.count$, ctrl, 'usagesCount'); + inject$($scope, usages.hasSyndicationUsages$, ctrl, 'hasSyndicationUsages'); + }; + ctrl.$onInit = () => { ctrl.showSendToPhotoSales = $window._clientConfig.showSendToPhotoSales; - const usages = imageUsagesService.getUsages(ctrl.image); - const usages$ = usages.groupedByState$.map((u) => u.toJS()); - const usagesCount$ = usages.count$; + bindUsages(ctrl.image); // TODO match on `platform` rather than `type` as `platform` includes more detail ctrl.usageTypeToName = (usageType) => { @@ -72,17 +83,18 @@ module.controller('grImageUsageCtrl', [ }; ctrl.onUsagesDeleted = () => { - // a bit nasty - but it updates the state of the page better than trying to do that in - // the client. $state.go('image', {imageId: ctrl.image.data.id, crop: undefined}, {reload: true}); }; - const hasSyndicationUsages$ = - imageUsagesService.getUsages(ctrl.image).hasSyndicationUsages$; + const freeImagesUpdateListener = $rootScope.$on('images-updated', (e, updatedImages) => { + const maybeUpdatedImage = updatedImages.find(u => u.data.id === ctrl.image.data.id); + if (maybeUpdatedImage) { + ctrl.image = maybeUpdatedImage; + bindUsages(ctrl.image); + } + }); - inject$($scope, usages$, ctrl, 'usages'); - inject$($scope, usagesCount$, ctrl, 'usagesCount'); - inject$($scope, hasSyndicationUsages$, ctrl, 'hasSyndicationUsages'); + $scope.$on('$destroy', freeImagesUpdateListener); }; }]); diff --git a/kahuna/public/js/image/controller.js b/kahuna/public/js/image/controller.js index e2a6247a64d..35d522d7c2c 100644 --- a/kahuna/public/js/image/controller.js +++ b/kahuna/public/js/image/controller.js @@ -190,9 +190,6 @@ image.controller('ImageCtrl', [ const usageTab = ctrl.tabs.find(_ => _.key === 'usages'); usageTab.value = `Usages (${value > 0 ? value : 'None'})`; usageTab.disabled = value === 0; - - // stop watching - freeUsageCountWatch(); }); // TODO: we should be able to rely on ctrl.crop.id instead once @@ -319,6 +316,7 @@ image.controller('ImageCtrl', [ const maybeUpdatedImage = updatedImages.find(updatedImage => ctrl.image.data.id === updatedImage.data.id); if (maybeUpdatedImage) { ctrl.image = maybeUpdatedImage; + ctrl.usagesCount = ctrl.image.data.usages.data.length; } }); @@ -340,5 +338,6 @@ image.controller('ImageCtrl', [ freeImagesUpdateListener(); freeImageDeleteListener(); freeImageDeleteFailListener(); + freeUsageCountWatch(); }); }]); diff --git a/kahuna/public/js/services/image/usages.js b/kahuna/public/js/services/image/usages.js index b94e1df5fc5..07519cb329e 100644 --- a/kahuna/public/js/services/image/usages.js +++ b/kahuna/public/js/services/image/usages.js @@ -87,7 +87,6 @@ imageUsagesService.factory('imageUsagesService', [function() { }); }); - const groupedByState$ = usages$ .map((usagesList) => usagesList.groupBy(usage => usage.get('status'))); From d757666c9fe4d3d2c554228cb123125de8b4b9e0 Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Mon, 31 Aug 2026 12:31:00 +0100 Subject: [PATCH 274/373] Usages rendering --- .../gr-confirm-delete/gr-confirm-delete.css | 2 ++ .../gr-image-usage/gr-image-usage-list.html | 10 ++++--- .../gr-image-usage/gr-image-usage.css | 14 +++++++--- .../gr-image-usage/gr-image-usage.html | 2 +- .../gr-image-usage/gr-image-usage.js | 27 ++++++++++++++++--- 5 files changed, 44 insertions(+), 11 deletions(-) diff --git a/kahuna/public/js/components/gr-confirm-delete/gr-confirm-delete.css b/kahuna/public/js/components/gr-confirm-delete/gr-confirm-delete.css index 0b3b47bda1d..8d84a9848a8 100644 --- a/kahuna/public/js/components/gr-confirm-delete/gr-confirm-delete.css +++ b/kahuna/public/js/components/gr-confirm-delete/gr-confirm-delete.css @@ -8,6 +8,8 @@ gr-confirm-delete.gr-delete-image { line-height: inherit; width: 100%; height: 100%; + padding-top: 10px; + padding-bottom: 10px; } .gr-confirm-delete:hover { diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage-list.html b/kahuna/public/js/components/gr-image-usage/gr-image-usage-list.html index aae5504999e..ddcd7527f86 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage-list.html +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage-list.html @@ -1,9 +1,13 @@
- + -
    +
    • local_library phonelink diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage.css b/kahuna/public/js/components/gr-image-usage/gr-image-usage.css index df64db796ec..15927d6a3d8 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage.css +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage.css @@ -7,6 +7,14 @@ gr-image-usage-list .gr-image-usage__field_title { color: #999; vertical-align: top; text-align: left; + display: inline-flex; + align-items: center; + gap: 4px; +} + +gr-image-usage-list .gr-image-usage__field_title > button { + display: inline-flex; + align-items: center; } gr-image-usage-list .date-added { @@ -18,8 +26,9 @@ gr-image-usage-list .gr-image-usage__record { width: 90%; } -gr-image-usage-list .source-list__icon svg { - padding: 0 2px; +gr-image-usage-list .usage-list__icon { + padding-right: 4px; + font-size:1.4em; } gr-image-usage-list gr-frontend-icon svg { @@ -46,7 +55,6 @@ gr-image-usage-list .reference-list__icon:hover svg path { gr-image-usage-list gr-icon { position: relative; - top: -2px; color: #999; } diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage.html b/kahuna/public/js/components/gr-image-usage/gr-image-usage.html index 626ef8e3bc6..6e6ce00e6cb 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage.html +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage.html @@ -1,7 +1,7 @@
      - +
      diff --git a/kahuna/public/js/components/gr-image-usage/gr-image-usage.js b/kahuna/public/js/components/gr-image-usage/gr-image-usage.js index 62521ee2e04..ab5e0c8226e 100644 --- a/kahuna/public/js/components/gr-image-usage/gr-image-usage.js +++ b/kahuna/public/js/components/gr-image-usage/gr-image-usage.js @@ -32,17 +32,25 @@ module.controller('grImageUsageCtrl', [ const ctrl = this; - const bindUsages = (image) => { + const statusOrder = ['pending', 'published', 'unknown', 'removed', 'downloaded']; + + function bindUsages(image) { const usages = imageUsagesService.getUsages(image); const usages$ = usages.groupedByState$ .map((grouped) => grouped .map(list => list.sortBy(usage => usage.get('dateAdded')).reverse()) + .sortBy((_, status) => { + const index = statusOrder.indexOf(status); + return index === -1 ? statusOrder.length : index; + }) .toJS() ); - inject$($scope, usages$, ctrl, 'usages'); + inject$($scope, usages$.do(grouped => { + ctrl.hasMultipleStatusGroups = Object.keys(grouped).length > 1; + }), ctrl, 'usages'); inject$($scope, usages.count$, ctrl, 'usagesCount'); inject$($scope, usages.hasSyndicationUsages$, ctrl, 'hasSyndicationUsages'); - }; + } ctrl.$onInit = () => { ctrl.showSendToPhotoSales = $window._clientConfig.showSendToPhotoSales; @@ -116,6 +124,16 @@ module.controller('grImageUsageListCtrl', [ function (imageUsagesService) { const ctrl = this; + const collapseThreshold = 10; + + ctrl.$onInit = () => { + ctrl.isCollapsed = ctrl.usages.length > collapseThreshold && ctrl.hasMultipleStatusGroups; + }; + + ctrl.toggle = () => { + ctrl.isCollapsed = !ctrl.isCollapsed; + }; + ctrl.formatTimestamp = (timestamp) => { return moment(timestamp).fromNow(); }; @@ -137,7 +155,8 @@ module.directive('grImageUsageList', [function () { bindToController: true, scope: { type: '=', - usages: '=' + usages: '=', + hasMultipleStatusGroups: '<' } }; }]); From 2044353d2f407efeb2d27c5efd61a78ce2a768d1 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Fri, 8 May 2026 17:26:57 +0100 Subject: [PATCH 275/373] Changes to correct issues with search consistency around nonFree setting --- .../gr-collections-panel.js | 3 +- .../gr-image-metadata/gr-image-metadata.js | 2 +- .../gr-my-uploads/gr-my-uploads.tsx | 17 +- .../components/gr-photoshoot/gr-photoshoot.js | 2 +- .../gr-preset-labels/gr-preset-labels.js | 2 +- .../gr-sort-control/base-sort-control.tsx | 45 ++- .../gr-extended-sort-control.tsx | 76 ++-- .../gr-sort-control/gr-sort-control.tsx | 44 +-- kahuna/public/js/edits/image-editor.js | 2 +- kahuna/public/js/edits/list-editor.js | 2 +- kahuna/public/js/image/controller.js | 21 +- kahuna/public/js/preview/image.js | 2 +- kahuna/public/js/search/index.js | 256 ++++++------ kahuna/public/js/search/query-filter.js | 2 +- kahuna/public/js/search/query.js | 373 ++++++++++-------- kahuna/public/js/services/telemetry.ts | 5 +- kahuna/public/js/upload/controller.js | 20 +- 17 files changed, 462 insertions(+), 412 deletions(-) diff --git a/kahuna/public/js/components/gr-collections-panel/gr-collections-panel.js b/kahuna/public/js/components/gr-collections-panel/gr-collections-panel.js index 9a50ca7595d..f40d61316a3 100644 --- a/kahuna/public/js/components/gr-collections-panel/gr-collections-panel.js +++ b/kahuna/public/js/components/gr-collections-panel/gr-collections-panel.js @@ -183,8 +183,7 @@ grCollectionsPanel.controller('GrNodeCtrl', grCollectionTreeCtrl.onSelect({$collection: ctrl.node.data.data.path}); }; - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; - + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; }; ctrl.searchWithModifiers = searchWithModifiers; diff --git a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.js b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.js index 8edc2647be2..f8488c697e9 100644 --- a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.js +++ b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.js @@ -413,7 +413,7 @@ module.controller('grImageMetadataCtrl', [ return storage.getJs(generateStoreName(key)).hidden; }; - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; function isUsefulMetadata(metadataKey) { return ignoredMetadata.indexOf(metadataKey) === -1; diff --git a/kahuna/public/js/components/gr-my-uploads/gr-my-uploads.tsx b/kahuna/public/js/components/gr-my-uploads/gr-my-uploads.tsx index ceb54ca44b8..2eb74d27bcc 100644 --- a/kahuna/public/js/components/gr-my-uploads/gr-my-uploads.tsx +++ b/kahuna/public/js/components/gr-my-uploads/gr-my-uploads.tsx @@ -17,10 +17,6 @@ export interface MyUploadsWrapperProps { props: MyUploadsProps; } -//-- logo click event -- -interface LogoClickEventDetail { showPaid: boolean } -interface LogoClickEvent extends CustomEvent {optional?: string} - //-- filter change event -- interface Filter { uploadedByMe: boolean } interface FilterChangeEventDetail { filter: Filter } @@ -38,6 +34,10 @@ const MyUploads: React.FC = ({ props }) => { const [myUploads, setMyUploads] = useState(props.myUploads); + useEffect(() => { + setMyUploads(prev => prev === props.myUploads ? prev : props.myUploads); + }, [props.myUploads]); + const handleCheckboxClick = () => { setMyUploads(prevChkd => { props.onChange(!prevChkd); @@ -61,17 +61,18 @@ const MyUploads: React.FC = ({ props }) => { } }; - const handleLogoClick = (event: LogoClickEvent) => { - setMyUploads(false); + const handleLogoClick = () => { + setMyUploads(prev => prev ? false : prev); }; const handleFilterChange = (event: FilterChangeEvent) => { - setMyUploads(event.detail.filter.uploadedByMe); + const next = event.detail.filter.uploadedByMe; + setMyUploads(prev => prev === next ? prev : next); }; const handleUploadedBy = (event: UploadedByEvent) => { const matches: boolean = (event.detail.userEmail === event.detail.uploadedBy); - setMyUploads(matches); + setMyUploads(prev => prev === matches ? prev : matches); }; useEffect(() => { diff --git a/kahuna/public/js/components/gr-photoshoot/gr-photoshoot.js b/kahuna/public/js/components/gr-photoshoot/gr-photoshoot.js index b5ba6c7dca1..a4f5a954f92 100644 --- a/kahuna/public/js/components/gr-photoshoot/gr-photoshoot.js +++ b/kahuna/public/js/components/gr-photoshoot/gr-photoshoot.js @@ -77,7 +77,7 @@ photoshoot.controller('GrPhotoshootCtrl', [ return photoshootService.batchRemove({ images: ctrl.images }); }; - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; if (Boolean(ctrl.withBatch)) { const batchApplyEvent = 'events:batch-apply:photoshoot'; diff --git a/kahuna/public/js/components/gr-preset-labels/gr-preset-labels.js b/kahuna/public/js/components/gr-preset-labels/gr-preset-labels.js index 5294f160c6a..d05a1e77310 100644 --- a/kahuna/public/js/components/gr-preset-labels/gr-preset-labels.js +++ b/kahuna/public/js/components/gr-preset-labels/gr-preset-labels.js @@ -62,7 +62,7 @@ presetLabels.controller('GrPresetLabelsCtrl', [ ctrl.newLabel = ''; } - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; } ]); diff --git a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx index b102de908ba..b68920c665a 100644 --- a/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/base-sort-control.tsx @@ -5,7 +5,6 @@ import { DefaultSortOption, CollectionSortOption } from "./gr-sort-control-confi import "./gr-sort-control.css"; const SELECT_OPTION = "Select an option"; -const DEFAULT_OPTION = DefaultSortOption.value; const COLLECTION_OPTION = CollectionSortOption.value; const CONTROL_TITLE = "Sort by:"; const SORT_ORDER = "Sort order"; @@ -66,13 +65,13 @@ const hasClassInSelfOrParent = (node: Element | null, className: string): boolea }; export const BaseSortControl: React.FC = ({ - options, - startSelectedOption, - onSelect, - startHasCollection, - panelVisible, - isSimple - }) => { + options, + startSelectedOption, + onSelect, + startHasCollection, + panelVisible, + isSimple + }) => { const hasCollection = startHasCollection; const startSort:SortDropdownOption = startSelectedOption ? startSelectedOption : DefaultSortOption; @@ -82,11 +81,23 @@ export const BaseSortControl: React.FC = ({ const [currentIndex, setCurrentIndex] = useState(-1); const [isPanelVisible, setPanelVisible] = useState(panelVisible); + useEffect(() => { + const nextSort = startSelectedOption ? startSelectedOption : DefaultSortOption; + setSelection(nextSort); + if (!nextSort.isCollection) { + setPrevious(nextSort); + } + }, [startSelectedOption]); + + useEffect(() => { + setPanelVisible(panelVisible); + }, [panelVisible]); + const handleArrowKeys = (event:KeyboardEvent) => { if (event.key === 'ArrowDown' || - event.key === 'ArrowUp' || - event.key === 'Enter' || - event.code === 'Space') { + event.key === 'ArrowUp' || + event.key === 'Enter' || + event.code === 'Space') { event.preventDefault(); event.stopPropagation(); let rowCount = options.length; @@ -126,16 +137,16 @@ export const BaseSortControl: React.FC = ({ const handlePanelShow = (event: any) => { const panel = event.detail.panel; - if (panel === PANEL_IDENTIFIER) { - setPanelVisible(true); - } + if (panel === PANEL_IDENTIFIER) { + setPanelVisible(true); + } }; const handlePanelHide = (event: any) => { const panel = event.detail.panel; - if (panel === PANEL_IDENTIFIER || panel === SCROLL_IDENTIFIER) { - setPanelVisible(false); - } + if (panel === PANEL_IDENTIFIER || panel === SCROLL_IDENTIFIER) { + setPanelVisible(false); + } }; window.addEventListener("mouseup", autoHideListener); diff --git a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx index 95e3115d283..92a25e457cc 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-extended-sort-control.tsx @@ -24,25 +24,44 @@ export interface ExtendedSortWrapperProps { const checkForCollection = (query:string): boolean => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); +const deriveExtendedSortState = ( + query: string, + orderBy: string, + sortOptions: SortDropdownOption[], + noTakenDateClause: string +) => { + let selectedSort = DefaultSortOption; + if (!query.includes(noTakenDateClause)) { + selectedSort = sortOptions.find(o => o.value === orderBy) || DefaultSortOption; + } + return { + selectedSort, + hasCollection: checkForCollection(query) + }; +}; + const ExtendedSortControl: React.FC = ({ props }) => { const noTakenDateClause = "-has:dateTaken"; const takenDateClause = "has:dateTaken"; const sortOptions = SortOptions; - const orderBy = props.orderBy; - const query = props.query; - - let startSortOption = DefaultSortOption; - if (!query.includes(noTakenDateClause) && (sortOptions.filter(o => o.value === orderBy)).length > 0) { - startSortOption = sortOptions.find(o => o.value === orderBy); - } - - const startHasCollection = checkForCollection(query); + const orderBy = props.orderBy || ""; + const query = props.query || ""; + const { selectedSort: startSortOption, hasCollection: startHasCollection } = + deriveExtendedSortState(query, orderBy, sortOptions, noTakenDateClause); const [selSortOption, setSortOption] = useState(startSortOption); const [userTakenSelect, setUserTakenSelect] = useState(props.userTakenSelect); const noTakenDateCount = props.noTakenDateCount; const [hasCollection, setHasCollection] = useState(startHasCollection); + useEffect(() => { + const { selectedSort, hasCollection: nextHasCollection } = + deriveExtendedSortState(query, orderBy, sortOptions, noTakenDateClause); + setSortOption(selectedSort); + setHasCollection(nextHasCollection); + setUserTakenSelect(Boolean(props.userTakenSelect)); + }, [query, orderBy, props.userTakenSelect]); + const onSortSelect = (selOption: SortDropdownOption) => { setSortOption(selOption); setUserTakenSelect(selOption.isTaken); @@ -67,11 +86,11 @@ const ExtendedSortControl: React.FC = ({ props }) => { }; const handleQueryChange = (e: any) => { - const newQuery = e.detail.query ? (" " + e.detail.query) : ""; - setHasCollection(checkForCollection(newQuery)); - if (userTakenSelect && !newQuery.includes(takenDateClause)) { - props.onSortSelect(DefaultSortOption, 'with', false); - } + const newQuery = e.detail.query ? (" " + e.detail.query) : ""; + setHasCollection(checkForCollection(newQuery)); + if (userTakenSelect && !newQuery.includes(takenDateClause)) { + props.onSortSelect(DefaultSortOption, 'with', false); + } }; window.addEventListener("logoClick", handleLogoClick); @@ -87,24 +106,23 @@ const ExtendedSortControl: React.FC = ({ props }) => { return (
      - - + +
      ); }; export const extendedSortControl = angular.module('gr.extendedSortControl', []) .component('extendedSortControl', react2angular(ExtendedSortControl, ["props"])); - diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 4bef8d6132d..242d3dc6f9e 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -16,29 +16,36 @@ export interface SortWrapperProps { props: SortProps; } -const checkForCollection = (query: string): boolean => /~"[a-zA-Z0-9 #\-_.:/]+"/.test(query); +const checkForCollection = (query:string): boolean => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); + +const deriveSortState = (query: string, orderBy: string, sortOptions: SortDropdownOption[]) => { + const hasCollection = checkForCollection(query); + if (hasCollection) { + const collectionSort = sortOptions.find(o => o.isCollection) || DefaultSortOption; + return { hasCollection, selectedSort: collectionSort }; + } + + const selectedSort = sortOptions.find(o => o.value === orderBy) || DefaultSortOption; + return { hasCollection, selectedSort }; +}; const SortControl: React.FC = ({ props }) => { const sortOptions = SortOptions; - const orderBy = props.orderBy; - const query = props.query; - const startHasCollection = checkForCollection(query); - - let startSortOption = DefaultSortOption; - if (startHasCollection) { - if ((sortOptions.filter(o => o.isCollection)).length > 0) { - startSortOption = sortOptions.find(o => o.isCollection); - } - } else { - if ((sortOptions.filter(o => o.value === orderBy)).length > 0) { - startSortOption = sortOptions.find(o => o.value === orderBy); - } - } + const orderBy = props.orderBy || ""; + const query = props.query || ""; + const { hasCollection: startHasCollection, selectedSort: startSortOption } = + deriveSortState(query, orderBy, sortOptions); const [selSortOption, setSortOption] = useState(startSortOption); const [hasCollection, setHasCollection] = useState(startHasCollection); + useEffect(() => { + const { hasCollection: nextHasCollection, selectedSort } = deriveSortState(query, orderBy, sortOptions); + setHasCollection(nextHasCollection); + setSortOption(selectedSort); + }, [query, orderBy]); + const onSortSelect = (selOption: SortDropdownOption) => { setSortOption(selOption); props.onSortSelect(selOption); @@ -46,11 +53,6 @@ const SortControl: React.FC = ({ props }) => { // initialisation useEffect(() => { - const handleLogoClick = (e: any) => { - setSortOption(DefaultSortOption); - props.onSortSelect(DefaultSortOption); - }; - const handleQueryChange = (e: any) => { const newQuery = e.detail.query ? (" " + e.detail.query) : ""; const curHasCollec = e.detail.hasCollection ? e.detail.hasCollection : false; @@ -70,12 +72,10 @@ const SortControl: React.FC = ({ props }) => { } }; - window.addEventListener("logoClick", handleLogoClick); window.addEventListener("queryChangeEvent", handleQueryChange); // Clean up the event listener when the component unmounts return () => { - window.removeEventListener("logoClick", handleLogoClick); window.removeEventListener("queryChangeEvent", handleQueryChange); }; diff --git a/kahuna/public/js/edits/image-editor.js b/kahuna/public/js/edits/image-editor.js index 3d33f71c483..2f753318493 100644 --- a/kahuna/public/js/edits/image-editor.js +++ b/kahuna/public/js/edits/image-editor.js @@ -440,7 +440,7 @@ imageEditor.controller('ImageEditorCtrl', [ ); } - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; }; }]); diff --git a/kahuna/public/js/edits/list-editor.js b/kahuna/public/js/edits/list-editor.js index c26f84e5477..72037939028 100644 --- a/kahuna/public/js/edits/list-editor.js +++ b/kahuna/public/js/edits/list-editor.js @@ -114,7 +114,7 @@ listEditor.controller('ListEditorCtrl', [ } }; - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; const batchAddEvent = 'events:batch-apply:add-all'; const batchRemoveEvent = 'events:batch-apply:remove-all'; diff --git a/kahuna/public/js/image/controller.js b/kahuna/public/js/image/controller.js index e2a6247a64d..1cb73629f60 100644 --- a/kahuna/public/js/image/controller.js +++ b/kahuna/public/js/image/controller.js @@ -28,6 +28,9 @@ import '../components/gu-date/gu-date'; import {radioList} from '../components/gr-radio-list/gr-radio-list'; import {cropUtil} from '../util/crop'; import { List } from 'immutable'; + +const toNonFreeString = (val) => (val === true || val === 'true') ? 'true' : 'false'; + const image = angular.module('kahuna.image.controller', [ 'util.rx', 'util.storage', @@ -166,10 +169,6 @@ image.controller('ImageCtrl', [ ctrl.optimisedImageUri = optimisedImageUri; ctrl.lowResImageUri = lowResImageUri; - ctrl.maybeReplacedByMediaIds = image.data.usages.data - .filter(usage => usage.data.status === "replaced") - .map(usage => usage.data.childUsageMetadata.childMediaId); - ctrl.singleImageList = ctrl.image ? new List([ctrl.image]) : new List([]); editsService.canUserEdit(ctrl.image).then(editable => { @@ -224,9 +223,9 @@ image.controller('ImageCtrl', [ }; ctrl.shareImage = () => { - const sharedUrl = $window._clientConfig.rootUri + "/images/" + ctrl.image.data.id; - navigator.clipboard.writeText(sharedUrl); - globalErrors.trigger('clipboard', sharedUrl); + const sharedUrl = $window._clientConfig.rootUri + "/images/" + ctrl.image.data.id; + navigator.clipboard.writeText(sharedUrl); + globalErrors.trigger('clipboard', sharedUrl); }; ctrl.onCropsDeleted = () => { // a bit nasty - but it updates the state of the page better than trying to do that in @@ -239,11 +238,11 @@ image.controller('ImageCtrl', [ const showPaid = session.user.permissions.showPaid ? session.user.permissions.showPaid : undefined; const defaultNonFreeFilter = { isDefault: true, - isNonFree: showPaid ? showPaid : false + isNonFree: toNonFreeString(showPaid) }; storage.setJs("defaultNonFreeFilter", defaultNonFreeFilter, true); window.dispatchEvent(new CustomEvent("logoClick", { - detail: {showPaid: defaultNonFreeFilter.isNonFree}, + detail: {showPaid: defaultNonFreeFilter.isNonFree === 'true'}, bubbles: true })); scrollPosition.resetToTop(); @@ -292,8 +291,8 @@ image.controller('ImageCtrl', [ if (largestWidth != crop.master.dimensions.width) { const imageId = getImageIdFromCropResource(cropsResource); console.log('The largest cropped asset of ' + crop.id + ' available for image ' + imageId + - ' does not have the same dimensions as the master. Using the next largest cropped asset with width ' + largestWidth + - 'Please correct this inconsistency.'); + ' does not have the same dimensions as the master. Using the next largest cropped asset with width ' + largestWidth + + 'Please correct this inconsistency.'); } crop.downloadLink = largestAsset.downloadLink; } diff --git a/kahuna/public/js/preview/image.js b/kahuna/public/js/preview/image.js index d8b8ff2dea2..2b6156d1702 100644 --- a/kahuna/public/js/preview/image.js +++ b/kahuna/public/js/preview/image.js @@ -130,7 +130,7 @@ image.controller('uiPreviewImageCtrl', [ return collection.data.cssColour && `background-color: ${collection.data.cssColour}`; }; - ctrl.srefNonfree = () => storage.getJs("isNonFree", true) ? true : undefined; + ctrl.srefNonfree = () => storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; ctrl.orderTakenBy = () => { let orderBy = storage.getJs('orderBy', false); return (orderBy && orderBy.includes('taken')); diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 8897e6006ee..29ecf4b281c 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -32,25 +32,27 @@ import collectionsPanelTemplate from import {cropUtil} from '../util/crop'; import { COLLECTION_SORT_VALUE } from '../components/gr-sort-control/gr-sort-control-config'; +const toNonFreeString = (val) => (val === true || val === 'true') ? 'true' : 'false'; + export var search = angular.module('kahuna.search', [ - 'ct.ui.router.extras.dsr', - 'kahuna.search.query', - 'kahuna.search.results', - 'kahuna.preview.image', - 'kahuna.services.scroll-position', - 'data-structure.list-factory', - 'data-structure.ordered-set-factory', - 'gr.topBar', - 'gr.panels', - 'gr.keyboardShortcut', - 'gr.sortControl', - 'gr.extendedSortControl', - 'gr.permissionsFilter', - 'gr.myUploads', - 'gr-searchWrapper', - 'grInfoPanel', - 'grCollectionsPanel', - 'ui.router', + 'ct.ui.router.extras.dsr', + 'kahuna.search.query', + 'kahuna.search.results', + 'kahuna.preview.image', + 'kahuna.services.scroll-position', + 'data-structure.list-factory', + 'data-structure.ordered-set-factory', + 'gr.topBar', + 'gr.panels', + 'gr.keyboardShortcut', + 'gr.sortControl', + 'gr.extendedSortControl', + 'gr.permissionsFilter', + 'gr.myUploads', + 'gr-searchWrapper', + 'grInfoPanel', + 'grCollectionsPanel', + 'ui.router', cropUtil.name ]); @@ -58,110 +60,110 @@ export var search = angular.module('kahuna.search', [ // to render - similar to the image controller see: // https://github.com/guardian/media-service/pull/478 search.config(['$stateProvider', '$urlMatcherFactoryProvider', - function($stateProvider, $urlMatcherFactoryProvider) { + function($stateProvider, $urlMatcherFactoryProvider) { const zeroWidthSpace = /[\u200B]/g; function removeUtf8SpecialChars(val) { - return angular.isDefined(val) && val.replace(zeroWidthSpace, ''); + return angular.isDefined(val) && val.replace(zeroWidthSpace, ''); } $urlMatcherFactoryProvider.type('Query', { - encode: val => removeUtf8SpecialChars(val), - decode: val => removeUtf8SpecialChars(val), - //call decode value that includes zero-width-space character - is: val => angular.isDefined(val) && !zeroWidthSpace.test(val) + encode: val => removeUtf8SpecialChars(val), + decode: val => removeUtf8SpecialChars(val), + //call decode value that includes zero-width-space character + is: val => angular.isDefined(val) && !zeroWidthSpace.test(val) }); $stateProvider.state('search', { - // FIXME [1]: This state should be abstract, but then we can't navigate to - // it, which we need to do to access it's deeper / remembered chile state - url: '/?cropType&customRatio&defaultCropType', - template: searchTemplate, - deepStateRedirect: { - // Inject a transient $stateParams for the results state - // below to pick up and expose - fn: ['$dsr$', function($dsr$) { - const params = angular.extend({}, $dsr$.redirect.params, { - isDeepStateRedirect: true - }); - return {state: $dsr$.redirect.state, params}; - }] - }, - controllerAs: 'ctrl', - controller: [ - '$scope', '$window', '$stateParams', 'scrollPosition', 'panels', 'shortcutKeys', 'keyboardShortcut', - 'panelService', 'cropSettings', 'mediaApi', 'storage', '$state', - function($scope, $window, $stateParams, scrollPosition, panels, shortcutKeys, keyboardShortcut, - panelService, cropSettings, mediaApi, storage, $state) { + // FIXME [1]: This state should be abstract, but then we can't navigate to + // it, which we need to do to access it's deeper / remembered chile state + url: '/?cropType&customRatio&defaultCropType', + template: searchTemplate, + deepStateRedirect: { + // Inject a transient $stateParams for the results state + // below to pick up and expose + fn: ['$dsr$', function($dsr$) { + const params = angular.extend({}, $dsr$.redirect.params, { + isDeepStateRedirect: true + }); + return {state: $dsr$.redirect.state, params}; + }] + }, + controllerAs: 'ctrl', + controller: [ + '$scope', '$window', '$stateParams', 'scrollPosition', 'panels', 'shortcutKeys', 'keyboardShortcut', + 'panelService', 'cropSettings', 'mediaApi', 'storage', '$state', + function($scope, $window, $stateParams, scrollPosition, panels, shortcutKeys, keyboardShortcut, + panelService, cropSettings, mediaApi, storage, $state) { + + const ctrl = this; + + ctrl.canUpload = false; + ctrl.usePermissionsFilter = window._clientConfig.usePermissionsFilter; + ctrl.hasNotifications = window._clientConfig.announcements.length > 0; - const ctrl = this; + mediaApi.canUserUpload().then(canUpload => { + ctrl.canUpload = canUpload; + }); - ctrl.canUpload = false; - ctrl.usePermissionsFilter = window._clientConfig.usePermissionsFilter; - ctrl.hasNotifications = window._clientConfig.announcements.length > 0; + cropSettings.set($stateParams); - mediaApi.canUserUpload().then(canUpload => { - ctrl.canUpload = canUpload; + ctrl.onLogoClick = () => { + mediaApi.getSession().then(session => { + const showPaid = session.user.permissions.showPaid ? session.user.permissions.showPaid : undefined; + const defaultNonFreeFilter = { + isDefault: true, + isNonFree: toNonFreeString(showPaid) + }; + storage.setJs("defaultNonFreeFilter", defaultNonFreeFilter, true); + $state.go('search.results', {nonFree: defaultNonFreeFilter.isNonFree}); + window.dispatchEvent(new CustomEvent("logoClick", { + detail: {showPaid: defaultNonFreeFilter.isNonFree === 'true'}, + bubbles: true + })); + scrollPosition.resetToTop(); }); + }; - cropSettings.set($stateParams); - - ctrl.onLogoClick = () => { - mediaApi.getSession().then(session => { - const showPaid = session.user.permissions.showPaid ? session.user.permissions.showPaid : undefined; - const defaultNonFreeFilter = { - isDefault: true, - isNonFree: showPaid ? showPaid : false - }; - storage.setJs("defaultNonFreeFilter", defaultNonFreeFilter, true); - $state.go('search.results', {nonFree: defaultNonFreeFilter.isNonFree}); - window.dispatchEvent(new CustomEvent("logoClick", { - detail: {showPaid: defaultNonFreeFilter.isNonFree}, - bubbles: true - })); - scrollPosition.resetToTop(); - }); - }; - - if ($state.current.name === 'search') { - mediaApi.getSession().then(session => { - storage.setJs('isNonFree', session.user.permissions.showPaid, true); - }); - } + if ($state.current.name === 'search') { + mediaApi.getSession().then(session => { + storage.setJs('isNonFree', toNonFreeString(session.user.permissions.showPaid), true); + }); + } - ctrl.collectionsPanel = panels.collectionsPanel; - ctrl.metadataPanel = panels.metadataPanel; + ctrl.collectionsPanel = panels.collectionsPanel; + ctrl.metadataPanel = panels.metadataPanel; - panelService.setAndSaveState($scope, 'collections', ctrl.collectionsPanel); - panelService.setAndSaveState($scope, 'metadata', ctrl.metadataPanel); + panelService.setAndSaveState($scope, 'collections', ctrl.collectionsPanel); + panelService.setAndSaveState($scope, 'metadata', ctrl.metadataPanel); - keyboardShortcut.bindTo($scope).add({ - combo: shortcutKeys.get('metadataPanel'), - description: 'Toggle metadata panel', - callback: panels.metadataPanel.toggleHidden - }); + keyboardShortcut.bindTo($scope).add({ + combo: shortcutKeys.get('metadataPanel'), + description: 'Toggle metadata panel', + callback: panels.metadataPanel.toggleHidden + }); - keyboardShortcut.bindTo($scope).add({ - combo: shortcutKeys.get('collectionsPanel'), - description: 'Toggle collections panel', - callback: panels.collectionsPanel.toggleHidden - }); + keyboardShortcut.bindTo($scope).add({ + combo: shortcutKeys.get('collectionsPanel'), + description: 'Toggle collections panel', + callback: panels.collectionsPanel.toggleHidden + }); }], - resolve: { - shortcutKeys: [function() { - // keep the shortcut keys here to stop overriding - return new Map([ - ['metadataPanel', 'm'], - ['collectionsPanel', 'l'] - ]); - }], - panels: ['panelService', function(panelService) { - const collectionsPanel = panelService.createPanel(true); - const metadataPanel = panelService.createPanel(true); + resolve: { + shortcutKeys: [function() { + // keep the shortcut keys here to stop overriding + return new Map([ + ['metadataPanel', 'm'], + ['collectionsPanel', 'l'] + ]); + }], + panels: ['panelService', function(panelService) { + const collectionsPanel = panelService.createPanel(true); + const metadataPanel = panelService.createPanel(true); - return { collectionsPanel, metadataPanel }; - }] - } + return { collectionsPanel, metadataPanel }; + }] + } }); const extraQueryParamsNotUsedByGridDirectly = [ @@ -206,7 +208,9 @@ search.config(['$stateProvider', '$urlMatcherFactoryProvider', // Helper state to determine whether this is just // reloading a previous search state, or a new search isReloadingPreviousSearch: ['$stateParams', function($stateParams) { - return $stateParams.isDeepStateRedirect === true; + const isDeepStateRedirect = $stateParams.isDeepStateRedirect; + delete $stateParams.isDeepStateRedirect; + return isDeepStateRedirect === true; }], selection: ['orderedSetFactory', function(orderedSetFactory) { return orderedSetFactory(); @@ -309,36 +313,28 @@ search.config(['$stateProvider', '$urlMatcherFactoryProvider', $scope.$on('$destroy', () => sub.dispose()); }] } - } + } }); -}]); + }]); // FIXME: This is here if you go to another state directly e.g. `'/images/id'` // and then navigate to search. As it has no remembered `deepStateRedirect`, // we just land on `/`. See [1]. -search.run(['$rootScope', '$state', '$stateParams', '$timeout', function($rootScope, $state, $stateParams, $timeout) { - $rootScope.$on('$viewContentLoaded', (_, view) => { - if (view === 'results@search') { - // using a timeout of 0 to schedule the task for execution ASAP, but outside the ongoing transition - $timeout(() => { - $state.go('search.results', {isDeepStateRedirect: false}); - }); - } - }); - $rootScope.$on('$stateChangeSuccess', (_, toState) => { - if (toState.name === 'search') { - $state.go('search.results', null, {reload: true}); - } - }); - $rootScope.$on('$stateChangeStart', (_, toState, toParams) => { - if (toState.name === 'search.results') { - //If moving to a collection, sorts images by time added to a collection by default - //allows sorting by newest first if set by user. Need to account for 'With Taken Date' tab impacts on query - const checkForCollection = (query) => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); - const toQuery = toParams.query ? toParams.query : ""; - if (!checkForCollection(toQuery) && toParams.orderBy === COLLECTION_SORT_VALUE) { - delete toParams.orderBy; - } - } - }); +search.run(['$rootScope', '$state', function($rootScope, $state) { + $rootScope.$on('$stateChangeSuccess', (_, toState) => { + if (toState.name === 'search') { + $state.go('search.results', null, {reload: true}); + } + }); + $rootScope.$on('$stateChangeStart', (_, toState, toParams) => { + if (toState.name === 'search.results') { + //If moving to a collection, sorts images by time added to a collection by default + //allows sorting by newest first if set by user. Need to account for 'With Taken Date' tab impacts on query + const checkForCollection = (query) => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); + const toQuery = toParams.query ? toParams.query : ""; + if (!checkForCollection(toQuery) && toParams.orderBy === COLLECTION_SORT_VALUE) { + delete toParams.orderBy; + } + } + }); }]); diff --git a/kahuna/public/js/search/query-filter.js b/kahuna/public/js/search/query-filter.js index 763c9c99f29..11a0bdc55e1 100644 --- a/kahuna/public/js/search/query-filter.js +++ b/kahuna/public/js/search/query-filter.js @@ -42,7 +42,7 @@ queryFilters.factory('searchWithModifiers', const shift = $event.getModifierState('Shift'); if (alt || shift) { $event.preventDefault(); - const nonFree = storage.getJs("isNonFree", true) ? true : undefined; + const nonFree = storage.getJs("isNonFree", true) === 'true' ? 'true' : 'false'; return $state.go('search.results', { query: updateQueryWithModifiers(fieldName, fieldValue, alt, shift, $stateParams.query), diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index c0666a0fe25..15a7c5e11f4 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -27,18 +27,21 @@ import { TAKEN_SORT } from "../components/gr-sort-control/gr-sort-control-config"; +const toNonFreeString = (val) => (val === true || val === 'true') ? 'true' : 'false'; +const isNonFreeString = (val) => val === 'true'; + export var query = angular.module('kahuna.search.query', [ - // Note: temporarily disabled for performance reasons, see above - // 'ngAnimate', - eq.name, - guDateRange.name, - syntax.name, - grStructuredQuery.name, - 'util.storage', - 'gr.sortControl', - 'gr.extendedSortControl', - 'gr.permissionsFilter', - 'gr.myUploads' + // Note: temporarily disabled for performance reasons, see above + // 'ngAnimate', + eq.name, + guDateRange.name, + syntax.name, + grStructuredQuery.name, + 'util.storage', + 'gr.sortControl', + 'gr.extendedSortControl', + 'gr.permissionsFilter', + 'gr.myUploads' ]); query.controller('SearchQueryCtrl', [ @@ -59,23 +62,24 @@ query.controller('SearchQueryCtrl', [ ctrl.maybeOrgOwnedValue = window._clientConfig.maybeOrgOwnedValue; ctrl.canUpload = false; mediaApi.canUserUpload().then(canUpload => { - ctrl.canUpload = canUpload; + ctrl.canUpload = canUpload; }); ctrl.ordering = { - orderBy: $stateParams.orderBy + orderBy: $stateParams.orderBy }; ctrl.filter = { - uploadedByMe: false + uploadedByMe: false }; ctrl.dateFilter = { - // filled in by the watcher below + // filled in by the watcher below }; ctrl.usePermissionsFilter = window._clientConfig.usePermissionsFilter; ctrl.filterMyUploads = false; + let lastUploadedByEventKey; ctrl.initialShowPaidEvent = ($stateParams.nonFree === undefined && ctrl.usePermissionsFilter) ? false : true; ctrl.shouldDisplayAISearchOption = window._clientConfig.aiSearchEnabled; @@ -89,7 +93,7 @@ query.controller('SearchQueryCtrl', [ //--react - angular interop events-- function raisePayableImagesEvent(showPaid) { - const boolShowPaid = (showPaid === true || showPaid === "true") ? true : false; + const boolShowPaid = toNonFreeString(showPaid) === 'true'; const customEvent = new CustomEvent('setPayableImages', { detail: {showPaid: boolShowPaid}, bubbles: true @@ -115,8 +119,14 @@ query.controller('SearchQueryCtrl', [ function raiseUploadedByCheckEvent() { if (ctrl.user) { + const uploadedBy = ctrl.filter ? ctrl.filter.uploadedBy : undefined; + const eventKey = `${ctrl.user.email}|${uploadedBy || ''}`; + if (eventKey === lastUploadedByEventKey) { + return; + } + lastUploadedByEventKey = eventKey; const customEvent = new CustomEvent('uploadedByEvent', { - detail: { userEmail: ctrl.user.email, uploadedBy: $stateParams.uploadedBy }, + detail: { userEmail: ctrl.user.email, uploadedBy: uploadedBy }, bubbles: true }); window.dispatchEvent(customEvent); @@ -139,9 +149,9 @@ query.controller('SearchQueryCtrl', [ } else { if (sender === "selectMyUploads") { const shouldOverwriteUploadedBy = - !filter.uploadedBy || - filter.uploadedBy === (ctrl.user ? ctrl.user.email : undefined) || - ctrl.filterMyUploads; + !filter.uploadedBy || + filter.uploadedBy === (ctrl.user ? ctrl.user.email : undefined) || + ctrl.filterMyUploads; if (shouldOverwriteUploadedBy) { ctrl.filter.uploadedBy = (ctrl.user && ctrl.filterMyUploads) ? ctrl.user.email : undefined; ctrl.filter.uploadedByMe = ctrl.filterMyUploads; @@ -152,48 +162,50 @@ query.controller('SearchQueryCtrl', [ storage.setJs("isUploadedByMe", ctrl.filter.uploadedByMe, true); } - function manageDefaultNonFree(filter) { - const defaultNonFreeFilter = storage.getJs("defaultNonFreeFilter", true); - if (defaultNonFreeFilter && defaultNonFreeFilter.isDefault === true){ - let newNonFree = defaultNonFreeFilter.isNonFree ? "true" : undefined; - if (newNonFree !== filter.nonFree) { - storage.setJs("isNonFree", newNonFree ? newNonFree : false, true); - storage.setJs("defaultIsNonFree", newNonFree ? newNonFree : false, true); - storage.setJs("isUploadedByMe", false, true); - storage.setJs("defaultNonFreeFilter", {isDefault: false, isNonFree: false}, true); - ctrl.filter.orgOwned = false; - } - Object.assign(ctrl.filter, {nonFree: newNonFree, uploadedByMe: false, uploadedBy: undefined}); - raiseFilterChangeEvent(ctrl.filter); + function manageDefaultNonFree() { + const defaultNonFreeFilter = storage.getJs("defaultNonFreeFilter", true); + if (defaultNonFreeFilter && defaultNonFreeFilter.isDefault === true){ + const newNonFree = toNonFreeString(defaultNonFreeFilter.isNonFree); + storage.setJs("isNonFree", newNonFree, true); + storage.setJs("defaultIsNonFree", newNonFree, true); + storage.setJs("isUploadedByMe", false, true); + storage.setJs("defaultNonFreeFilter", {isDefault: false, isNonFree: newNonFree}, true); + ctrl.filterMyUploads = false; + if (ctrl.myUploadsProps) { + syncMyUploadsProps(); } + ctrl.filter.orgOwned = false; + Object.assign(ctrl.filter, {nonFree: newNonFree, uploadedByMe: false, uploadedBy: undefined}); + raiseFilterChangeEvent(ctrl.filter); + } } function manageOrgOwnedSetting(filter) { - const structuredQuery = structureQuery(filter.query) || []; - const orgOwnedIndexInQuery = structuredQuery.findIndex(item => item.value === ctrl.maybeOrgOwnedValue); - const queryHasOrgOwned = orgOwnedIndexInQuery >= 0; - if (ctrl.filter.orgOwned && !queryHasOrgOwned){ - // If the checkbox is ticked, ensure the chip is part of the search bar - const orgOwnedChip = { - type: "filter", - filterType: "inclusion", - key : "is", - value: ctrl.maybeOrgOwnedValue - }; - ctrl.filter.query = renderQuery([ - ...structuredQuery, - orgOwnedChip - ]); - } else if (!ctrl.filter.orgOwned && queryHasOrgOwned && !ctrl.usePermissionsFilter) { - // If the checkbox is unticked, ensure chip is no longer in the search bar - structuredQuery.splice(orgOwnedIndexInQuery, 1); - ctrl.filter.query = renderQuery(structuredQuery); - } + const structuredQuery = structureQuery(filter.query) || []; + const orgOwnedIndexInQuery = structuredQuery.findIndex(item => item.value === ctrl.maybeOrgOwnedValue); + const queryHasOrgOwned = orgOwnedIndexInQuery >= 0; + if (ctrl.filter.orgOwned && !queryHasOrgOwned){ + // If the checkbox is ticked, ensure the chip is part of the search bar + const orgOwnedChip = { + type: "filter", + filterType: "inclusion", + key : "is", + value: ctrl.maybeOrgOwnedValue + }; + ctrl.filter.query = renderQuery([ + ...structuredQuery, + orgOwnedChip + ]); + } else if (!ctrl.filter.orgOwned && queryHasOrgOwned && !ctrl.usePermissionsFilter) { + // If the checkbox is unticked, ensure chip is no longer in the search bar + structuredQuery.splice(orgOwnedIndexInQuery, 1); + ctrl.filter.query = renderQuery(structuredQuery); + } } function resetQuery() { - ctrl.filter.query = undefined; - ctrl.filter.orgOwned = false; + ctrl.filter.query = undefined; + ctrl.filter.orgOwned = false; } function checkForCollection(query) { @@ -247,8 +259,9 @@ query.controller('SearchQueryCtrl', [ function resolveNonFree() { let nonFreeCheck = ctrl.filter.nonFree; if (ctrl.usePermissionsFilter && nonFreeCheck === undefined) { - nonFreeCheck = storage.getJs("defaultIsNonFree", true); - } else if (!ctrl.usePermissionsFilter && (nonFreeCheck === 'false' || nonFreeCheck === false)) { + const defaultShowPaid = storage.getJs("defaultIsNonFree", true); + nonFreeCheck = defaultShowPaid === 'true' ? 'true' : 'false'; + } else if (!ctrl.usePermissionsFilter && nonFreeCheck === 'false') { nonFreeCheck = undefined; } ctrl.filter.nonFree = nonFreeCheck; @@ -262,9 +275,9 @@ query.controller('SearchQueryCtrl', [ // eslint-disable-next-line complexity function watchSearchChange(newFilter, sender) { - let showPaid = newFilter.nonFree ? newFilter.nonFree : false; - if (sender && sender == "filterChange" && !newFilter.nonFree) { - showPaid = ctrl.user.permissions.showPaid; + let showPaid = toNonFreeString(newFilter.nonFree); + if (ctrl.usePermissionsFilter && sender && sender === "filterChange" && newFilter.nonFree === undefined) { + showPaid = toNonFreeString(ctrl.user.permissions.showPaid); } storage.setJs("isNonFree", showPaid, true); @@ -290,7 +303,7 @@ query.controller('SearchQueryCtrl', [ //--update filter elements-- manageUploadedBy(newFilter, sender); - manageDefaultNonFree(newFilter); + manageDefaultNonFree(); manageOrgOwnedSetting(newFilter); emitQueryTelemetry(); @@ -307,8 +320,16 @@ query.controller('SearchQueryCtrl', [ } //-my uploads- + function syncMyUploadsProps() { + ctrl.myUploadsProps = { + ...ctrl.myUploadsProps, + myUploads: ctrl.filterMyUploads + }; + } + function selectMyUploads(myUploadsChecked) { ctrl.filterMyUploads = myUploadsChecked; + syncMyUploadsProps(); watchSearchChange(ctrl.filter, "selectMyUploads"); } @@ -327,7 +348,7 @@ query.controller('SearchQueryCtrl', [ ctrl.sortProps = { onSortSelect: updateSortChips, - query: ctrl.filter.query, + query: $stateParams.query, orderBy: ctrl.ordering ? ctrl.ordering.orderBy : "" }; //-end sort control- @@ -336,12 +357,12 @@ query.controller('SearchQueryCtrl', [ function updatePermissionsChips (permissionsSel, showChargeable) { ctrl.permissionsProps.selectedOption = permissionsSel; ctrl.filter.query = updateFilterChips(permissionsSel, ctrl.filter.query); - ctrl.filter.nonFree = showChargeable; + ctrl.filter.nonFree = toNonFreeString(showChargeable); watchSearchChange(ctrl.filter, "updatePermissionsChips"); } function chargeableChange (showChargeable) { - ctrl.filter.nonFree = showChargeable; + ctrl.filter.nonFree = toNonFreeString(showChargeable); watchSearchChange(ctrl.filter, "chargeableChange"); } @@ -349,12 +370,12 @@ query.controller('SearchQueryCtrl', [ let defOptVal = PermissionsConf.permissionsDefaultOpt(); let pfDefPerm = pfOpts.filter(opt => opt.value == defOptVal)[0]; ctrl.permissionsProps = { options: pfOpts, - selectedOption: pfDefPerm, - onSelect: updatePermissionsChips, - onChargeable: chargeableChange, - chargeable: ctrl.filter.nonFree ? ctrl.filter.nonFree : ($stateParams.nonFree == "true"), - query: ctrl.filter.query - }; + selectedOption: pfDefPerm, + onSelect: updatePermissionsChips, + onChargeable: chargeableChange, + chargeable: (ctrl.filter.nonFree || $stateParams.nonFree) === "true", + query: ctrl.filter.query + }; //-end permissions filter- ctrl.resetQuery = resetQuery; @@ -368,36 +389,34 @@ query.controller('SearchQueryCtrl', [ const pastYear = moment().subtract(1, 'years').toISOString(); ctrl.payTypeOptions = [ - {label: 'Free', value: 'free'}, - {label: 'Free and No Rights', value: 'maybe-free'}, - {label: 'All (inc. paid)', value: 'all'} + {label: 'Free', value: 'free'}, + {label: 'Free and No Rights', value: 'maybe-free'}, + {label: 'All (inc. paid)', value: 'all'} ]; ctrl.sinceOptions = [ - {label: 'Anytime'}, // value: undefined - {label: 'Today', value: lastMidnight}, - {label: 'Past 24 hours', value: past24Hours}, - {label: 'Past week', value: pastWeek}, - {label: 'Past 6 months', value: past6Months}, - {label: 'Past year', value: pastYear} + {label: 'Anytime'}, // value: undefined + {label: 'Today', value: lastMidnight}, + {label: 'Past 24 hours', value: past24Hours}, + {label: 'Past week', value: pastWeek}, + {label: 'Past 6 months', value: past6Months}, + {label: 'Past year', value: pastYear} ]; ctrl.filterDateFieldsOptions = [ - {label: 'Upload time', name: 'uploaded'}, // value: undefined - {label: 'Date taken', name: 'taken', value: 'taken'}, - {label: 'Last modified', name: 'modified', value: 'modified'} + {label: 'Upload time', name: 'uploaded'}, // value: undefined + {label: 'Date taken', name: 'taken', value: 'taken'}, + {label: 'Last modified', name: 'modified', value: 'modified'} ]; const dateFilterParams = [ - 'dateField', 'since', 'until', 'takenSince', 'takenUntil', - 'modifiedSince', 'modifiedUntil' + 'dateField', 'since', 'until', 'takenSince', 'takenUntil', + 'modifiedSince', 'modifiedUntil' ]; Object.keys($stateParams). - // Exclude date-related filters, managed separately in dateFilter - filter(key => dateFilterParams.indexOf(key) === -1). - // Exclude useAISearch, managed separately by its own dedicated watcher - filter(key => key !== 'useAISearch'). - forEach(setAndWatchParam); + // Exclude date-related filters, managed separately in dateFilter + filter(key => dateFilterParams.indexOf(key) === -1). + forEach(setAndWatchParam); // URL parameters are not decoded when taken out of the params. // Might be fixed with: https://github.com/angular-ui/ui-router/issues/1759 @@ -405,49 +424,49 @@ query.controller('SearchQueryCtrl', [ function valOrUndefined(str) { return str ? str : undefined; } function setAndWatchParam(key) { - //this value has been set on ctrl.order + //this value has been set on ctrl.order + if (key !== 'orderBy') { + ctrl.filter[key] = valOrUndefined($stateParams[key]); + } + + ctrl.collectionSearch = ctrl.filter.query ? checkForCollection(ctrl.filter.query) : false; + storeCollection(ctrl.filter.query); + + $scope.$watch(() => $stateParams[key], onValChange(newVal => { + // FIXME: broken for 'your uploads' + // FIXME: + they triggers filter $watch and $state.go (breaks history) if (key !== 'orderBy') { - ctrl.filter[key] = valOrUndefined($stateParams[key]); + ctrl.filter[key] = valOrUndefined(newVal); } - ctrl.collectionSearch = ctrl.filter.query ? checkForCollection(ctrl.filter.query) : false; - storeCollection(ctrl.filter.query); - - $scope.$watch(() => $stateParams[key], onValChange(newVal => { - // FIXME: broken for 'your uploads' - // FIXME: + they triggers filter $watch and $state.go (breaks history) - if (key !== 'orderBy') { - ctrl.filter[key] = valOrUndefined(newVal); - } - - // don't track changes to `query` as it would trigger on every keypress - if (key !== 'query') { - $rootScope.$emit( - 'track:event', 'Query', 'Change', 'Success', null, { field: key, value: newVal } - ); - } - })); + // don't track changes to `query` as it would trigger on every keypress + if (key !== 'query') { + $rootScope.$emit( + 'track:event', 'Query', 'Change', 'Success', null, { field: key, value: newVal } + ); + } + })); } // Init and apply date-related changes in $stateParams to ctrl.dateFilter $scope.$watchCollection(() => $stateParams, () => { - switch ($stateParams.dateField) { + switch ($stateParams.dateField) { case 'taken': - ctrl.dateFilter.since = $stateParams.takenSince; - ctrl.dateFilter.until = $stateParams.takenUntil; - ctrl.dateFilter.field = 'taken'; - break; + ctrl.dateFilter.since = $stateParams.takenSince; + ctrl.dateFilter.until = $stateParams.takenUntil; + ctrl.dateFilter.field = 'taken'; + break; case 'modified': - ctrl.dateFilter.since = $stateParams.modifiedSince; - ctrl.dateFilter.until = $stateParams.modifiedUntil; - ctrl.dateFilter.field = 'modified'; - break; + ctrl.dateFilter.since = $stateParams.modifiedSince; + ctrl.dateFilter.until = $stateParams.modifiedUntil; + ctrl.dateFilter.field = 'modified'; + break; default: // uploaded (default so represented as `undefined`) - ctrl.dateFilter.since = $stateParams.since; - ctrl.dateFilter.until = $stateParams.until; - ctrl.dateFilter.field = undefined; - break; - } + ctrl.dateFilter.since = $stateParams.since; + ctrl.dateFilter.until = $stateParams.until; + ctrl.dateFilter.field = undefined; + break; + } }); $scope.$watchCollection(() => ctrl.filter, onValChange(newFilter => { @@ -455,7 +474,7 @@ query.controller('SearchQueryCtrl', [ })); $scope.$watch(() => ctrl.ordering.orderBy, onValChange(newVal => { - $state.go('search.results', {...ctrl.filter, orderBy: newVal}); + $state.go('search.results', {...ctrl.filter, orderBy: newVal}); })); let aiSearchInitialised = false; @@ -478,81 +497,87 @@ query.controller('SearchQueryCtrl', [ vecWeight: ctrl.vecWeight }); } else { - $state.go('search.results', {...ctrl.filter, useAISearch: null}); + $state.go('search.results', {...ctrl.filter, useAISearch: undefined}); } }); $scope.$watchCollection(() => ctrl.dateFilter, onValChange(({field, since, until}) => { - // Translate dateFilter to actual state and query params - $state.go('search.results', {...ctrl.filter, ...{ - since: field === undefined ? since : null, - until: field === undefined ? until : null, - takenSince: field === 'taken' ? since : null, - takenUntil: field === 'taken' ? until : null, - modifiedSince: field === 'modified' ? since : null, - modifiedUntil: field === 'modified' ? until : null, - dateField: field + // Translate dateFilter to actual state and query params + $state.go('search.results', {...ctrl.filter, ...{ + since: field === undefined ? since : null, + until: field === undefined ? until : null, + takenSince: field === 'taken' ? since : null, + takenUntil: field === 'taken' ? until : null, + modifiedSince: field === 'modified' ? since : null, + modifiedUntil: field === 'modified' ? until : null, + dateField: field }}); })); // we can't user dynamic values in the ng:true-value see: // https://docs.angularjs.org/error/ngModel/constexpr mediaApi.getSession().then(session => { - //-uploaded by me- - const isUploadedByMe = storage.getJs("isUploadedByMe", true); - ctrl.user = session.user; - if (isUploadedByMe === null) { - ctrl.filter.uploadedByMe = ctrl.filter.uploadedBy === ctrl.user.email; + //-uploaded by me- + const isUploadedByMe = storage.getJs("isUploadedByMe", true); + ctrl.user = session.user; + if (ctrl.filter.uploadedBy === ctrl.user.email) { + ctrl.filter.uploadedByMe = true; + ctrl.filterMyUploads = true; + storage.setJs("isUploadedByMe", true); + } else if (isUploadedByMe === null) { + ctrl.filter.uploadedByMe = ctrl.filter.uploadedBy === ctrl.user.email; + ctrl.filterMyUploads = ctrl.filter.uploadedByMe; + storage.setJs("isUploadedByMe",ctrl.filter.uploadedByMe); + } else { + if ((ctrl.filter.uploadedBy === ctrl.user.email) && !isUploadedByMe ) { + ctrl.filter.uploadedByMe = true; ctrl.filterMyUploads = ctrl.filter.uploadedByMe; storage.setJs("isUploadedByMe",ctrl.filter.uploadedByMe); } else { - if ((ctrl.filter.uploadedBy === ctrl.user.email) && !isUploadedByMe ) { - ctrl.filter.uploadedByMe = true; - ctrl.filterMyUploads = ctrl.filter.uploadedByMe; - storage.setJs("isUploadedByMe",ctrl.filter.uploadedByMe); - } else { - ctrl.filter.uploadedByMe = isUploadedByMe; - ctrl.filterMyUploads = isUploadedByMe; - } + ctrl.filter.uploadedByMe = isUploadedByMe; + ctrl.filterMyUploads = isUploadedByMe; } + } - //-default non free- - const defNonFree = session.user.permissions ? session.user.permissions.showPaid : undefined; - storage.setJs("defaultIsNonFree", defNonFree ? defNonFree : false, true); - if (!ctrl.initialShowPaidEvent && (defNonFree === true || defNonFree === "true")) { - ctrl.initialShowPaidEvent = true; - raisePayableImagesEvent(defNonFree); - } + syncMyUploadsProps(); + raiseUploadedByCheckEvent(); + + //-default non free- + const defNonFree = session.user.permissions ? session.user.permissions.showPaid : undefined; + storage.setJs("defaultIsNonFree", toNonFreeString(defNonFree), true); + if (!ctrl.initialShowPaidEvent && toNonFreeString(defNonFree) === 'true') { + ctrl.initialShowPaidEvent = true; + raisePayableImagesEvent(defNonFree); + } + // If nonFree is provided in URL params, use that; otherwise use stored value + if ($stateParams.nonFree !== undefined) { + ctrl.filter.nonFree = toNonFreeString($stateParams.nonFree); + storage.setJs("isNonFree", ctrl.filter.nonFree, true); + } else { const isNonFree = storage.getJs("isNonFree", true); if (isNonFree === null) { - ctrl.filter.nonFree = $stateParams.nonFree; - storage.setJs("isNonFree", ctrl.filter.nonFree ? ctrl.filter.nonFree : (ctrl.usePermissionsFilter ? "false" : undefined), true); - } - else if (isNonFree === true || isNonFree === "true") { - ctrl.filter.nonFree = "true"; + ctrl.filter.nonFree = toNonFreeString($stateParams.nonFree); + storage.setJs("isNonFree", ctrl.filter.nonFree, true); } else { - ctrl.filter.nonFree = (ctrl.usePermissionsFilter ? "false" : undefined); + ctrl.filter.nonFree = isNonFreeString(isNonFree) ? 'true' : 'false'; } + } - //-org owned- - const structuredQuery = structureQuery(ctrl.filter.query); - const orgOwned = (structuredQuery.some(item => item.value === ctrl.maybeOrgOwnedValue)); - ctrl.filter.orgOwned = orgOwned; + //-org owned- + const structuredQuery = structureQuery(ctrl.filter.query); + const orgOwned = (structuredQuery.some(item => item.value === ctrl.maybeOrgOwnedValue)); + ctrl.filter.orgOwned = orgOwned; - watchSearchChange(ctrl.filter, "userPermissions"); + watchSearchChange(ctrl.filter, "userPermissions"); }); - - - const { nonFree, uploadedByMe } = ctrl.filter; - sendTelemetryForQuery(ctrl.filter.query, nonFree, uploadedByMe, ctrl.useAISearch); -}]); + }]); query.directive('searchQuery', [function() { - return { - restrict: 'E', - controller: 'SearchQueryCtrl as searchQuery', - template: template - }; + return { + restrict: 'E', + controller: 'SearchQueryCtrl as searchQuery', + template: template + }; }]); diff --git a/kahuna/public/js/services/telemetry.ts b/kahuna/public/js/services/telemetry.ts index bd1e0723087..ffa41e96f17 100644 --- a/kahuna/public/js/services/telemetry.ts +++ b/kahuna/public/js/services/telemetry.ts @@ -49,12 +49,11 @@ const sendFilterTelemetryEvent = (key: string, value: string, searchUuid: string }, 1); }; -export const sendTelemetryForQuery = (query: string, nonFree?: boolean | string, uploadedByMe?: boolean, useAISearch?: boolean ) => { +export const sendTelemetryForQuery = (query: string, nonFree?: string, uploadedByMe?: boolean, useAISearch?: boolean ) => { const structuredQuery = structureQuery(query || ""); const searchUuid = v4(); - // nonFree is unfortunately either a boolean, stringified boolean, or undefined - const freeToUseOnly = (!(nonFree === 'true' || nonFree === true)); + const freeToUseOnly = nonFree !== 'true'; const uploadedByMeOnly = (uploadedByMe); // Only log for true - matching how these filters work in Grid (only applied when true) diff --git a/kahuna/public/js/upload/controller.js b/kahuna/public/js/upload/controller.js index 56b103e9727..43fe44075ed 100644 --- a/kahuna/public/js/upload/controller.js +++ b/kahuna/public/js/upload/controller.js @@ -4,11 +4,13 @@ import './prompt/prompt'; import './recent/recent-uploads'; import '../services/scroll-position'; +const toNonFreeString = (val) => (val === true || val === 'true') ? 'true' : 'false'; + var upload = angular.module('kahuna.upload.controller', [ - 'kahuna.upload.prompt', - 'kahuna.upload.recent', - 'kahuna.services.scroll-position', - 'util.storage' + 'kahuna.upload.prompt', + 'kahuna.upload.recent', + 'kahuna.services.scroll-position', + 'util.storage' ]); upload.controller('UploadCtrl', ['uploadManager', 'mediaApi', 'scrollPosition', '$scope', 'storage', @@ -23,7 +25,7 @@ upload.controller('UploadCtrl', ['uploadManager', 'mediaApi', 'scrollPosition', $scope.$on("$locationChangeStart", function(event, _, current) { // handle route changes if (current.indexOf("/upload") > -1) { - ctrl.displayWarning(event); + ctrl.displayWarning(event); } }); @@ -38,7 +40,7 @@ upload.controller('UploadCtrl', ['uploadManager', 'mediaApi', 'scrollPosition', ctrl.systemName = window._clientConfig.systemName; mediaApi.canUserUpload().then(canUpload => { - ctrl.canUpload = canUpload; + ctrl.canUpload = canUpload; }); // TODO: Show multiple jobs? @@ -58,14 +60,14 @@ upload.controller('UploadCtrl', ['uploadManager', 'mediaApi', 'scrollPosition', const showPaid = session.user.permissions.showPaid ? session.user.permissions.showPaid : undefined; const defaultNonFreeFilter = { isDefault: true, - isNonFree: showPaid ? showPaid : false + isNonFree: toNonFreeString(showPaid) }; storage.setJs("defaultNonFreeFilter", defaultNonFreeFilter, true); window.dispatchEvent(new CustomEvent("logoClick", { - detail: {showPaid: defaultNonFreeFilter.isNonFree}, + detail: {showPaid: defaultNonFreeFilter.isNonFree === 'true'}, bubbles: true })); scrollPosition.resetToTop(); }); }; -}]); + }]); From 0db9942c6d78b21b8f52ff611174521775e31ad0 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Tue, 21 Jul 2026 15:54:18 +0100 Subject: [PATCH 276/373] Correcting UI sequencing issues related to use of back button --- .../gr-permissions-filter.tsx | 21 ++- .../gr-sort-control/gr-sort-control.tsx | 15 +- .../components/gu-date-range/gu-date-range.js | 7 +- kahuna/public/js/search/index.js | 13 +- kahuna/public/js/search/query.js | 176 ++++++++++++++++-- 5 files changed, 199 insertions(+), 33 deletions(-) diff --git a/kahuna/public/js/components/gr-permissions-filter/gr-permissions-filter.tsx b/kahuna/public/js/components/gr-permissions-filter/gr-permissions-filter.tsx index 1db69875ff1..170750c8a9a 100644 --- a/kahuna/public/js/components/gr-permissions-filter/gr-permissions-filter.tsx +++ b/kahuna/public/js/components/gr-permissions-filter/gr-permissions-filter.tsx @@ -106,8 +106,8 @@ const PermissionsFilter: React.FC = ({ props }) => { setIsChargeable(event.detail.showPaid); }; - const handleQueryChange = (event: QueryChangeEvent) => { - const newQuery = event.detail.query ? (" " + event.detail.query) : ""; + const syncSelectionFromQuery = (rawQuery: string | undefined) => { + const newQuery = rawQuery ? (" " + rawQuery) : ""; if (propsRef.current.query !== newQuery) { propsRef.current.query = newQuery; @@ -127,6 +127,10 @@ const PermissionsFilter: React.FC = ({ props }) => { } }; + const handleQueryChange = (event: QueryChangeEvent) => { + syncSelectionFromQuery(event.detail.query); + }; + useEffect(() => { window.addEventListener('queryChangeEvent', handleQueryChange); window.addEventListener('logoClick', handleLogoClick); @@ -134,7 +138,6 @@ const PermissionsFilter: React.FC = ({ props }) => { window.addEventListener('mouseup', autoHideListener); window.addEventListener('scroll', autoHideListener); window.addEventListener('keydown', autoHideListener); - setSelection(defPerms); // Clean up the event listener when the component unmounts return () => { @@ -148,6 +151,17 @@ const PermissionsFilter: React.FC = ({ props }) => { }; }, []); + useEffect(() => { + syncSelectionFromQuery(props.query); + }, [props.query]); + + useEffect(() => { + if (propsRef.current.chargeable !== props.chargeable) { + propsRef.current.chargeable = props.chargeable; + setIsChargeable(props.chargeable); + } + }, [props.chargeable]); + const handleOptionClick = (option: PermissionsDropdownOption) => { const payableDef = payableDefaults.filter(pd => pd.opt === option.value)[0]; if (payableDef.payable === 'false' || payableDef.payable === 'true') { @@ -162,6 +176,7 @@ const PermissionsFilter: React.FC = ({ props }) => { }; useEffect(() => { + propsRef.current.chargeable = isChargeable; props.onChargeable(isChargeable); }, [isChargeable]); diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 242d3dc6f9e..956ce963362 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -20,13 +20,15 @@ const checkForCollection = (query:string): boolean => /~"[a-zA-Z0-9 #-_.://]+"/. const deriveSortState = (query: string, orderBy: string, sortOptions: SortDropdownOption[]) => { const hasCollection = checkForCollection(query); - if (hasCollection) { + const matchedByOrderBy = sortOptions.find(o => o.value === orderBy); + + if (hasCollection && !matchedByOrderBy) { + // no explicit sort chosen yet for this collection query - use the collection default const collectionSort = sortOptions.find(o => o.isCollection) || DefaultSortOption; return { hasCollection, selectedSort: collectionSort }; } - const selectedSort = sortOptions.find(o => o.value === orderBy) || DefaultSortOption; - return { hasCollection, selectedSort }; + return { hasCollection, selectedSort: matchedByOrderBy || DefaultSortOption }; }; const SortControl: React.FC = ({ props }) => { @@ -53,6 +55,11 @@ const SortControl: React.FC = ({ props }) => { // initialisation useEffect(() => { + const handleLogoClick = (e: any) => { + setSortOption(DefaultSortOption); + props.onSortSelect(DefaultSortOption); + }; + const handleQueryChange = (e: any) => { const newQuery = e.detail.query ? (" " + e.detail.query) : ""; const curHasCollec = e.detail.hasCollection ? e.detail.hasCollection : false; @@ -72,10 +79,12 @@ const SortControl: React.FC = ({ props }) => { } }; + window.addEventListener("logoClick", handleLogoClick); window.addEventListener("queryChangeEvent", handleQueryChange); // Clean up the event listener when the component unmounts return () => { + window.removeEventListener("logoClick", handleLogoClick); window.removeEventListener("queryChangeEvent", handleQueryChange); }; diff --git a/kahuna/public/js/components/gu-date-range/gu-date-range.js b/kahuna/public/js/components/gu-date-range/gu-date-range.js index 6f73a1986db..c043025dc3b 100644 --- a/kahuna/public/js/components/gu-date-range/gu-date-range.js +++ b/kahuna/public/js/components/gu-date-range/gu-date-range.js @@ -122,7 +122,12 @@ guDateRange.directive('guDateRange', [function () { pikaStart.show(); }); - $scope.$watch('ctrl.guEndDate', resetView); + // Watch both start and end dates: many presets/filters only set + // one of the two (e.g. 'since' with 'until' left undefined), so + // watching guEndDate alone can miss changes to guStartDate and + // leave the display summary out of sync with the actual filter + // (e.g. after using the logo click or browser back button). + $scope.$watchGroup(['ctrl.guStartDate', 'ctrl.guEndDate'], resetView); $scope.$on('$destroy', function() { pikaStart.destroy(); diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 29ecf4b281c..25e3fc1523a 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -116,12 +116,13 @@ search.config(['$stateProvider', '$urlMatcherFactoryProvider', isNonFree: toNonFreeString(showPaid) }; storage.setJs("defaultNonFreeFilter", defaultNonFreeFilter, true); - $state.go('search.results', {nonFree: defaultNonFreeFilter.isNonFree}); - window.dispatchEvent(new CustomEvent("logoClick", { - detail: {showPaid: defaultNonFreeFilter.isNonFree === 'true'}, - bubbles: true - })); - scrollPosition.resetToTop(); + $state.go('search.results', {nonFree: defaultNonFreeFilter.isNonFree}).then(() => { + window.dispatchEvent(new CustomEvent("logoClick", { + detail: {showPaid: defaultNonFreeFilter.isNonFree === 'true'}, + bubbles: true + })); + scrollPosition.resetToTop(); + }); }); }; diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 15a7c5e11f4..e5e2612c345 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -49,10 +49,11 @@ query.controller('SearchQueryCtrl', [ '$scope', '$state', '$stateParams', + '$timeout', 'onValChange', 'storage', 'mediaApi', - function($rootScope, $scope, $state, $stateParams, onValChange, storage, mediaApi) { + function($rootScope, $scope, $state, $stateParams, $timeout, onValChange, storage, mediaApi) { const ctrl = this; ctrl.costFilterLabel = window._clientConfig.costFilterLabel; @@ -69,6 +70,8 @@ query.controller('SearchQueryCtrl', [ orderBy: $stateParams.orderBy }; + let lastRequestedOrderBy = $stateParams.orderBy; + ctrl.filter = { uploadedByMe: false }; @@ -162,6 +165,10 @@ query.controller('SearchQueryCtrl', [ storage.setJs("isUploadedByMe", ctrl.filter.uploadedByMe, true); } + // Guards the delayed disarming of the "pending default nonFree" flag, + // see manageDefaultNonFree() below for why this is needed. + let clearDefaultNonFreeFilterTimeout; + function manageDefaultNonFree() { const defaultNonFreeFilter = storage.getJs("defaultNonFreeFilter", true); if (defaultNonFreeFilter && defaultNonFreeFilter.isDefault === true){ @@ -169,7 +176,6 @@ query.controller('SearchQueryCtrl', [ storage.setJs("isNonFree", newNonFree, true); storage.setJs("defaultIsNonFree", newNonFree, true); storage.setJs("isUploadedByMe", false, true); - storage.setJs("defaultNonFreeFilter", {isDefault: false, isNonFree: newNonFree}, true); ctrl.filterMyUploads = false; if (ctrl.myUploadsProps) { syncMyUploadsProps(); @@ -177,9 +183,39 @@ query.controller('SearchQueryCtrl', [ ctrl.filter.orgOwned = false; Object.assign(ctrl.filter, {nonFree: newNonFree, uploadedByMe: false, uploadedBy: undefined}); raiseFilterChangeEvent(ctrl.filter); + + // IMPORTANT: don't disarm the "isDefault" flag immediately here. + // A single logo click triggers *several* back-to-back + // navigations/filter-change digests (onLogoClick's own $state.go, + // then gr-sort-control's handleLogoClick -> updateSortChips, and + // sometimes a delayed, spurious transition fired by + // ui-router-extras' Deep State Redirect mechanism that drops + // params such as `nonFree` back to their state defaults). + // If we disarm the flag on the very *first* of these passes + // (which may just be an incidental/harmless one - e.g. the echo + // of onLogoClick's own navigation - and not the actual rogue + // transition we need to correct), this function becomes a no-op + // for any later pass, so `nonFree` never gets corrected again if + // the rogue transition drops it afterwards, leaving it stuck at + // its non-default value. Instead, keep re-applying the default + // on every pass for a short grace period after being armed, and + // only disarm it once that period elapses. + if (clearDefaultNonFreeFilterTimeout) { + $timeout.cancel(clearDefaultNonFreeFilterTimeout); + } + clearDefaultNonFreeFilterTimeout = $timeout(() => { + storage.setJs("defaultNonFreeFilter", {isDefault: false, isNonFree: newNonFree}, true); + clearDefaultNonFreeFilterTimeout = undefined; + }, 1500); } } + $scope.$on('$destroy', () => { + if (clearDefaultNonFreeFilterTimeout) { + $timeout.cancel(clearDefaultNonFreeFilterTimeout); + } + }); + function manageOrgOwnedSetting(filter) { const structuredQuery = structureQuery(filter.query) || []; const orgOwnedIndexInQuery = structuredQuery.findIndex(item => item.value === ctrl.maybeOrgOwnedValue); @@ -210,7 +246,7 @@ query.controller('SearchQueryCtrl', [ function checkForCollection(query) { return /~"[a-zA-Z0-9 #-_.://]+"/.test(query); - }; + } function storeCollection(query) { const match = query ? query.match(/~"[a-zA-Z0-9 #-_.://]+"/) : undefined; @@ -220,13 +256,11 @@ query.controller('SearchQueryCtrl', [ } function getCollection() { - const collection = storage.getJs("currentCollection") ? storage.getJs("currentCollection") : ""; - return collection; + return storage.getJs("currentCollection") ? storage.getJs("currentCollection") : ""; } function getPriorOrderBy() { - const prior = storage.getJs("priorOrderBy") ? storage.getJs("priorOrderBy") : ""; - return prior; + return storage.getJs("priorOrderBy") ? storage.getJs("priorOrderBy") : ""; } function setPriorOrderBy(priorOrderBy) { @@ -244,7 +278,7 @@ query.controller('SearchQueryCtrl', [ function priorRevisedOrderBy(collectionSearch, newCollection, oldCollection) { const priorOrderBy = getPriorOrderBy(); if (collectionSearch && ((oldCollection !== newCollection) || ("" !== priorOrderBy))) { - if (priorOrderBy != "") { + if (priorOrderBy !== "") { setPriorOrderBy(""); return priorOrderBy; } else { @@ -257,10 +291,11 @@ query.controller('SearchQueryCtrl', [ } function resolveNonFree() { - let nonFreeCheck = ctrl.filter.nonFree; + let nonFreeCheck = ctrl.filter.nonFree !== undefined + ? toNonFreeString(ctrl.filter.nonFree) + : undefined; if (ctrl.usePermissionsFilter && nonFreeCheck === undefined) { - const defaultShowPaid = storage.getJs("defaultIsNonFree", true); - nonFreeCheck = defaultShowPaid === 'true' ? 'true' : 'false'; + nonFreeCheck = toNonFreeString(storage.getJs("defaultIsNonFree", true)); } else if (!ctrl.usePermissionsFilter && nonFreeCheck === 'false') { nonFreeCheck = undefined; } @@ -288,7 +323,7 @@ query.controller('SearchQueryCtrl', [ const newCollection = storeCollection(newFilter.query); if (ctrl.usePermissionsFilter) { - if (sender && ctrl.ordering["orderBy"] != $stateParams.orderBy) { + if (sender && ctrl.ordering["orderBy"] !== $stateParams.orderBy) { ctrl.ordering["orderBy"] = $stateParams.orderBy; } if ($stateParams.orderBy && $stateParams.orderBy.includes(TAKEN_SORT) && (!newFilter.query || !newFilter.query.includes(HAS_DATE_TAKEN))) { @@ -308,14 +343,61 @@ query.controller('SearchQueryCtrl', [ emitQueryTelemetry(); + // Helper to check if all keys in `goParams` already match the current + // $stateParams. If so, this navigation would be a no-op and firing it + // anyway has been observed to trigger a rogue transition from + // ui-router-extras DSR mechanism that drops params like nonFree. + function goParamsAlreadyCurrent(goParams) { + return Object.keys(goParams).every(key => { + const a = goParams[key]; + const b = $stateParams[key]; + // treat '' and undefined as equivalent to avoid false negatives + return (a || undefined) === (b || undefined); + }); + } + + // If every key in `goParams` other than `orderBy` already matches the + // current $stateParams, this transition's *only* purpose is to correct + // `orderBy` (e.g. auto-switching to the collection sort order after + // navigating into/out of a collection search). In that case the + // "real" navigation (e.g. a ui-sref collection link, or the browser's + // own back/forward navigation) has already happened and already + // pushed/popped a history entry - so this follow-up correction should + // patch the current history entry (`location: 'replace'`) rather than + // push a brand new one. Otherwise a single user action (e.g. clicking + // a collection) ends up creating *two* history entries, which means + // the back button has to be pressed twice to fully undo it - and on + // the first press, the collection filter is still applied. + function isOrderByOnlyCorrection(goParams) { + return Object.keys(goParams).every(key => { + if (key === 'orderBy') { return true; } + const a = goParams[key]; + const b = $stateParams[key]; + return (a || undefined) === (b || undefined); + }); + } + if (ctrl.collectionSearch && !curCollectionSearch) { storage.setJs("orderBy", CollectionSortOption.value); ctrl.ordering["orderBy"] = CollectionSortOption.value; - raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, CollectionSortOption.value); - $state.go('search.results', {...ctrl.filter, ...{orderBy: CollectionSortOption.value}}); + // Record synchronously *before* the ctrl.ordering.orderBy $watch can + // run in this same digest, so it recognises this orderBy change as + // already handled and doesn't fire its own extra navigation. + lastRequestedOrderBy = CollectionSortOption.value; + const goParams1 = {...ctrl.filter, ...{orderBy: CollectionSortOption.value}}; + if (!goParamsAlreadyCurrent(goParams1)) { + raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, CollectionSortOption.value); + const options1 = isOrderByOnlyCorrection(goParams1) ? {location: 'replace'} : undefined; + $state.go('search.results', goParams1, options1); + } } else { - raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, ctrl.ordering["orderBy"]); - $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering["orderBy"]}}); + lastRequestedOrderBy = ctrl.ordering["orderBy"]; + const goParams2 = {...ctrl.filter, ...{orderBy: ctrl.ordering["orderBy"]}}; + if (!goParamsAlreadyCurrent(goParams2)) { + raiseQueryChangeEvent(ctrl.filter.query, curCollectionSearch, ctrl.ordering["orderBy"]); + const options2 = isOrderByOnlyCorrection(goParams2) ? {location: 'replace'} : undefined; + $state.go('search.results', goParams2, options2); + } } } @@ -343,7 +425,9 @@ query.controller('SearchQueryCtrl', [ function updateSortChips (sortSel) { ctrl.ordering['orderBy'] = manageSortSelection(sortSel.value); storage.setJs("orderBy", ctrl.ordering["orderBy"]); - $state.go('search.results', {...ctrl.filter, ...{orderBy: ctrl.ordering['orderBy']}}); + + lastRequestedOrderBy = ctrl.ordering['orderBy']; + $state.go('search.results', {orderBy: ctrl.ordering['orderBy']}); } ctrl.sortProps = { @@ -351,6 +435,20 @@ query.controller('SearchQueryCtrl', [ query: $stateParams.query, orderBy: ctrl.ordering ? ctrl.ordering.orderBy : "" }; + + // Keep the React sort-control's props in sync with the underlying model. + // A *new* object reference is required each time, since react2angular's + // one-way ('<') binding only re-renders when the bound reference changes. + $scope.$watch( + () => `${ctrl.ordering.orderBy}||${ctrl.filter.query}`, + onValChange(() => { + ctrl.sortProps = { + ...ctrl.sortProps, + query: ctrl.filter.query, + orderBy: ctrl.ordering.orderBy + }; + }) + ); //-end sort control- //-permissions filter- @@ -373,9 +471,24 @@ query.controller('SearchQueryCtrl', [ selectedOption: pfDefPerm, onSelect: updatePermissionsChips, onChargeable: chargeableChange, - chargeable: (ctrl.filter.nonFree || $stateParams.nonFree) === "true", + chargeable: toNonFreeString(ctrl.filter.nonFree || $stateParams.nonFree) === "true", query: ctrl.filter.query }; + + // Keep the React permissions-filter's props in sync with the underlying + // model (e.g. after a reset via onLogoClick or the back button), for the + // same reason as ctrl.sortProps above: react2angular's one-way binding + // only re-renders on a *new* object reference. + $scope.$watch( + () => `${ctrl.filter.nonFree}||${ctrl.filter.query}`, + onValChange(() => { + ctrl.permissionsProps = { + ...ctrl.permissionsProps, + chargeable: toNonFreeString(ctrl.filter.nonFree) === "true", + query: ctrl.filter.query + }; + }) + ); //-end permissions filter- ctrl.resetQuery = resetQuery; @@ -436,7 +549,25 @@ query.controller('SearchQueryCtrl', [ // FIXME: broken for 'your uploads' // FIXME: + they triggers filter $watch and $state.go (breaks history) if (key !== 'orderBy') { - ctrl.filter[key] = valOrUndefined(newVal); + // For nonFree, ensure boolean true from URL decode is normalised to string 'true' + const val = valOrUndefined(newVal); + ctrl.filter[key] = (key === 'nonFree' && val !== undefined) ? toNonFreeString(val) : val; + } else { + ctrl.ordering.orderBy = valOrUndefined(newVal); + } + + if (key === 'query') { + const sq = structureQuery(valOrUndefined(newVal)) || []; + ctrl.filter.orgOwned = sq.some(item => item.value === ctrl.maybeOrgOwnedValue); + } + + // When uploadedBy changes externally (e.g. back button), sync uploadedByMe + // and filterMyUploads so manageUploadedBy doesn't re-set uploadedBy. + if (key === 'uploadedBy') { + const isMyUploads = ctrl.user ? valOrUndefined(newVal) === ctrl.user.email : false; + ctrl.filter.uploadedByMe = isMyUploads; + ctrl.filterMyUploads = isMyUploads; + syncMyUploadsProps(); } // don't track changes to `query` as it would trigger on every keypress @@ -474,7 +605,12 @@ query.controller('SearchQueryCtrl', [ })); $scope.$watch(() => ctrl.ordering.orderBy, onValChange(newVal => { - $state.go('search.results', {...ctrl.filter, orderBy: newVal}); + if ($stateParams.orderBy === newVal || lastRequestedOrderBy === newVal) { + return; + } + lastRequestedOrderBy = newVal; + const stateGoParams = {...ctrl.filter, orderBy: newVal}; + $state.go('search.results', stateGoParams); })); let aiSearchInitialised = false; From 980d3255168d5780e523a3572a7f29760fa00b11 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Wed, 22 Jul 2026 10:09:19 +0100 Subject: [PATCH 277/373] Correcting issue with resetting collection selection via back button when query.nonFree = undefined. Need to handle case of collapsing 'false' down to 'undefined' in the history sequence. --- kahuna/public/js/search/query.js | 37 ++++++++++++-------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index e5e2612c345..196bf79a8ba 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -343,37 +343,28 @@ query.controller('SearchQueryCtrl', [ emitQueryTelemetry(); - // Helper to check if all keys in `goParams` already match the current - // $stateParams. If so, this navigation would be a no-op and firing it - // anyway has been observed to trigger a rogue transition from - // ui-router-extras DSR mechanism that drops params like nonFree. + function normaliseParamForComparison(key, val) { + if (key === 'nonFree' && !ctrl.usePermissionsFilter && val === 'false') { + return undefined; + } + // treat '' and undefined as equivalent to avoid false negatives + return val || undefined; + } + function goParamsAlreadyCurrent(goParams) { return Object.keys(goParams).every(key => { - const a = goParams[key]; - const b = $stateParams[key]; - // treat '' and undefined as equivalent to avoid false negatives - return (a || undefined) === (b || undefined); + const a = normaliseParamForComparison(key, goParams[key]); + const b = normaliseParamForComparison(key, $stateParams[key]); + return a === b; }); } - // If every key in `goParams` other than `orderBy` already matches the - // current $stateParams, this transition's *only* purpose is to correct - // `orderBy` (e.g. auto-switching to the collection sort order after - // navigating into/out of a collection search). In that case the - // "real" navigation (e.g. a ui-sref collection link, or the browser's - // own back/forward navigation) has already happened and already - // pushed/popped a history entry - so this follow-up correction should - // patch the current history entry (`location: 'replace'`) rather than - // push a brand new one. Otherwise a single user action (e.g. clicking - // a collection) ends up creating *two* history entries, which means - // the back button has to be pressed twice to fully undo it - and on - // the first press, the collection filter is still applied. function isOrderByOnlyCorrection(goParams) { return Object.keys(goParams).every(key => { if (key === 'orderBy') { return true; } - const a = goParams[key]; - const b = $stateParams[key]; - return (a || undefined) === (b || undefined); + const a = normaliseParamForComparison(key, goParams[key]); + const b = normaliseParamForComparison(key, $stateParams[key]); + return a === b; }); } From 3631c75d7e29c244c598033231c11217ce78aae4 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Wed, 5 Aug 2026 12:11:31 +0100 Subject: [PATCH 278/373] Further amendments to ensure the checkbox filters are responsive immediately after logo click, but avoid unnecessary resets which were occurring - done through 'disarm function'. --- kahuna/public/js/search/query.html | 7 +++++-- kahuna/public/js/search/query.js | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/kahuna/public/js/search/query.html b/kahuna/public/js/search/query.html index b3711b67509..185dde529ce 100644 --- a/kahuna/public/js/search/query.html +++ b/kahuna/public/js/search/query.html @@ -45,6 +45,7 @@ when the option is off --> {{ searchQuery.costFilterLabel }} @@ -64,7 +65,8 @@
      • @@ -72,7 +74,8 @@
      • diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 196bf79a8ba..835894d5282 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -169,6 +169,21 @@ query.controller('SearchQueryCtrl', [ // see manageDefaultNonFree() below for why this is needed. let clearDefaultNonFreeFilterTimeout; + function disarmDefaultNonFreeFilter() { + if (clearDefaultNonFreeFilterTimeout) { + $timeout.cancel(clearDefaultNonFreeFilterTimeout); + clearDefaultNonFreeFilterTimeout = undefined; + } + const defaultNonFreeFilter = storage.getJs("defaultNonFreeFilter", true); + if (defaultNonFreeFilter && defaultNonFreeFilter.isDefault === true) { + storage.setJs( + "defaultNonFreeFilter", + {isDefault: false, isNonFree: defaultNonFreeFilter.isNonFree}, + true + ); + } + } + function manageDefaultNonFree() { const defaultNonFreeFilter = storage.getJs("defaultNonFreeFilter", true); if (defaultNonFreeFilter && defaultNonFreeFilter.isDefault === true){ @@ -401,6 +416,7 @@ query.controller('SearchQueryCtrl', [ } function selectMyUploads(myUploadsChecked) { + disarmDefaultNonFreeFilter(); ctrl.filterMyUploads = myUploadsChecked; syncMyUploadsProps(); watchSearchChange(ctrl.filter, "selectMyUploads"); @@ -444,6 +460,7 @@ query.controller('SearchQueryCtrl', [ //-permissions filter- function updatePermissionsChips (permissionsSel, showChargeable) { + disarmDefaultNonFreeFilter(); ctrl.permissionsProps.selectedOption = permissionsSel; ctrl.filter.query = updateFilterChips(permissionsSel, ctrl.filter.query); ctrl.filter.nonFree = toNonFreeString(showChargeable); @@ -451,10 +468,23 @@ query.controller('SearchQueryCtrl', [ } function chargeableChange (showChargeable) { + disarmDefaultNonFreeFilter(); ctrl.filter.nonFree = toNonFreeString(showChargeable); watchSearchChange(ctrl.filter, "chargeableChange"); } + ctrl.onNonFreeCheckboxChange = function() { + disarmDefaultNonFreeFilter(); + }; + + ctrl.onUploadedByMeCheckboxChange = function() { + disarmDefaultNonFreeFilter(); + }; + + ctrl.onOrgOwnedCheckboxChange = function() { + disarmDefaultNonFreeFilter(); + }; + let pfOpts = PermissionsConf.permissionsOptions(); let defOptVal = PermissionsConf.permissionsDefaultOpt(); let pfDefPerm = pfOpts.filter(opt => opt.value == defOptVal)[0]; From 96224f5c928cfe0845356d2ae4c4581cb957e195 Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Mon, 10 Aug 2026 15:25:54 +0100 Subject: [PATCH 279/373] Correct the reset for the 'Use AI search' checkbox on LogoClick. --- kahuna/public/js/search/query.html | 3 ++- kahuna/public/js/search/query.js | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/kahuna/public/js/search/query.html b/kahuna/public/js/search/query.html index 185dde529ce..5317c319c4e 100644 --- a/kahuna/public/js/search/query.html +++ b/kahuna/public/js/search/query.html @@ -3,7 +3,8 @@ ng-if="searchQuery.shouldDisplayAISearchOption"> diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index 835894d5282..b6931d1f666 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -196,6 +196,7 @@ query.controller('SearchQueryCtrl', [ syncMyUploadsProps(); } ctrl.filter.orgOwned = false; + ctrl.useAISearch = false; Object.assign(ctrl.filter, {nonFree: newNonFree, uploadedByMe: false, uploadedBy: undefined}); raiseFilterChangeEvent(ctrl.filter); @@ -485,6 +486,10 @@ query.controller('SearchQueryCtrl', [ disarmDefaultNonFreeFilter(); }; + ctrl.onAISearchCheckboxChange = function() { + disarmDefaultNonFreeFilter(); + }; + let pfOpts = PermissionsConf.permissionsOptions(); let defOptVal = PermissionsConf.permissionsDefaultOpt(); let pfDefPerm = pfOpts.filter(opt => opt.value == defOptVal)[0]; From ce4eb0e50381783dc50e73acfebe2ad3dd642eb9 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 11:26:22 +0000 Subject: [PATCH 280/373] Add localstack to devcontainer forwarded ports; mount project into container when not in CI --- .devcontainer/devenv.yaml | 1 + .devcontainer/shared/devcontainer.json | 1 + .devcontainer/user/devcontainer.json | 1 + e2e-tests/global-setup.ts | 4 ++++ 4 files changed, 7 insertions(+) diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml index 661a33352ad..f11896105a2 100644 --- a/.devcontainer/devenv.yaml +++ b/.devcontainer/devenv.yaml @@ -27,6 +27,7 @@ forwardPorts: - 9010 - 9011 - 9012 + - 4566 - 6080 # Optional: Mount directories from host to container diff --git a/.devcontainer/shared/devcontainer.json b/.devcontainer/shared/devcontainer.json index da4c20e25bf..8796a02a51c 100644 --- a/.devcontainer/shared/devcontainer.json +++ b/.devcontainer/shared/devcontainer.json @@ -29,6 +29,7 @@ 9010, 9011, 9012, + 4566, 6080 ], "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json index 59b78adfb87..3abef9f164f 100644 --- a/.devcontainer/user/devcontainer.json +++ b/.devcontainer/user/devcontainer.json @@ -29,6 +29,7 @@ 9010, 9011, 9012, + 4566, 6080 ], "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 6431789a9ba..4b6d11bb719 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -30,6 +30,7 @@ import { MEDIA_API_PORT, PROXY_IMAGE, REGION, + REPO_ROOT, SERVICE_PORTS, URLS_FILE, } from './testcontainers/constants'; @@ -192,6 +193,9 @@ async function globalSetup(): Promise { // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. { source: configDir, target: '/root/.grid', mode: 'ro' }, { source: configDir, target: '/etc/grid', mode: 'ro' }, + // Outside CI the grid-e2e-dev image runs services under sbt; mount the repo + // over /build so host edits recompile live. + ...(process.env.CI ? [] : [{ source: REPO_ROOT, target: '/build', mode: 'rw' as const }]), ]) .withEnvironment({ AWS_ACCESS_KEY_ID: 'test', From 477cd5c3f640c0afce5e61d9d595d6104d575aaf Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 26 Aug 2026 09:19:17 +0000 Subject: [PATCH 281/373] Add image-loader to the list of running services --- e2e-tests/README.md | 2 +- e2e-tests/global-setup.ts | 12 +++++++++--- e2e-tests/images/Dockerfile | 18 ++++++++++-------- e2e-tests/images/README.md | 4 ++-- e2e-tests/images/entrypoint.common.sh | 3 ++- e2e-tests/testcontainers/constants.ts | 1 + 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index beda695272b..09df0ac1206 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -55,7 +55,7 @@ Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses 3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded buckets, provisioned into LocalStack; 4. per-service config, generated by `dev/script/generate-config/service-config.js`; -5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running all eight Play services. +5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running all nine Play services. Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is exposed to the tests. `globalTeardown` stops everything and cleans up. diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 6431789a9ba..7248f9f68ab 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -6,7 +6,7 @@ * 2. Elasticsearch + LocalStack (infrastructure), * 3. the CloudFormation core stack + seeded buckets (provisioning), * 4. generated per-service config (reusing dev/script/generate-config), - * 5. the pre-built `grid-e2e-ci` image running all eight services. + * 5. the pre-built `grid-e2e-ci` image running all nine services. * * The Kahuna base URL is exposed to tests via `GRID_BASE_URL`, and the started * containers are stashed for `global-teardown.ts`. @@ -95,6 +95,7 @@ function buildCaddyfile(coreStackProps: Record): string { const appServices: Record = { [`media.${DOMAIN}`]: SERVICE_PORTS.kahuna, [`api.media.${DOMAIN}`]: SERVICE_PORTS['media-api'], + [`loader.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], [`cropper.media.${DOMAIN}`]: SERVICE_PORTS.cropper, [`thrall.media.${DOMAIN}`]: SERVICE_PORTS.thrall, [`media-metadata.${DOMAIN}`]: SERVICE_PORTS['metadata-editor'], @@ -177,7 +178,7 @@ async function globalSetup(): Promise { const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); generateServiceConfig(configDir, coreStackProps); - // All eight Grid services run inside this single container and talk to each + // All nine Grid services run inside this single container and talk to each // other over its localhost. Each is published on the fixed host port its // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's // dev-nginx routes the https://*.media. domains straight into this @@ -200,7 +201,12 @@ async function globalSetup(): Promise { AWS_DEFAULT_REGION: REGION, AWS_CBOR_DISABLE: 'true', }) - .withWaitStrategy(Wait.forHttp('/management/healthcheck', MEDIA_API_PORT).forStatusCode(200)) + .withWaitStrategy( + Wait.forAll([ + Wait.forHttp('/management/healthcheck', MEDIA_API_PORT).forStatusCode(200), + Wait.forHttp('/management/healthcheck', SERVICE_PORTS['image-loader']).forStatusCode(200), + ]), + ) .withStartupTimeout(startupTimeoutMs); if (process.env.GRID_DEBUG) { diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 265365f3c4d..430c2a911e8 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1 -# Single-container image that builds and runs eight Grid Play services -# (auth, collections, cropper, kahuna, leases, media-api, metadata-editor, -# thrall) together with Kahuna's frontend. +# Single-container image that builds and runs nine Grid Play services +# (auth, collections, cropper, image-loader, kahuna, leases, media-api, +# metadata-editor, thrall) together with Kahuna's frontend. # # This one Dockerfile produces two images, selected with `--target`: # @@ -75,6 +75,7 @@ RUN sbt \ "auth/stage" \ "collections/stage" \ "cropper/stage" \ + "image-loader/stage" \ "kahuna/stage" \ "leases/stage" \ "media-api/stage" \ @@ -97,6 +98,7 @@ RUN set -eux; \ COPY --from=backend /build/auth/target/universal/stage /usr/share/auth COPY --from=backend /build/collections/target/universal/stage /usr/share/collections COPY --from=backend /build/cropper/target/universal/stage /usr/share/cropper +COPY --from=backend /build/image-loader/target/universal/stage /usr/share/image-loader COPY --from=backend /build/kahuna/target/universal/stage /usr/share/kahuna COPY --from=backend /build/leases/target/universal/stage /usr/share/leases COPY --from=backend /build/media-api/target/universal/stage /usr/share/media-api @@ -104,7 +106,7 @@ COPY --from=backend /build/metadata-editor/target/universal/stage /usr/share/met COPY --from=backend /build/thrall/target/universal/stage /usr/share/thrall RUN set -eux; \ - for service in auth collections cropper kahuna leases media-api metadata-editor thrall; do \ + for service in auth collections cropper image-loader kahuna leases media-api metadata-editor thrall; do \ mkdir -p "/var/log/$service"; \ done @@ -115,8 +117,8 @@ RUN chmod +x /usr/local/bin/entrypoint.ci.sh # The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. ENV AWS_CBOR_DISABLE=true -# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases -EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 +# media-api, thrall, image-loader, kahuna, cropper, metadata-editor, collections, auth, leases +EXPOSE 9001 9002 9003 9005 9006 9007 9010 9011 9012 ENTRYPOINT ["/usr/local/bin/entrypoint.ci.sh"] @@ -173,9 +175,9 @@ RUN chmod +x /usr/local/bin/entrypoint.dev.sh # The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. ENV AWS_CBOR_DISABLE=true -# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases +# media-api, thrall, image-loader, kahuna, cropper, metadata-editor, collections, auth, leases # plus the JDWP debug port used when GRID_DEBUG is set. -EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 5005 +EXPOSE 9001 9002 9003 9005 9006 9007 9010 9011 9012 5005 ENTRYPOINT ["/usr/local/bin/entrypoint.dev.sh"] diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index 5ca9ecc9760..d3cec50f940 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,6 +1,6 @@ c# Grid E2E test images -A single `Dockerfile` produces two single-container images that run eight +A single `Dockerfile` produces two single-container images that run nine Grid Play services plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter: - **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. - **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. @@ -24,7 +24,7 @@ from `/etc/grid/stage`; defaults to `DEV`). Mount your environment's config with ```bash docker run --rm \ -v "$PWD/my-config:/etc/grid:ro" \ - -p 9001:9001 -p 9002:9002 -p 9005:9005 -p 9006:9006 \ + -p 9001:9001 -p 9002:9002 -p 9003:9003 -p 9005:9005 -p 9006:9006 \ -p 9007:9007 -p 9010:9010 -p 9011:9011 -p 9012:9012 \ grid-e2e-[ci|dev] ``` diff --git a/e2e-tests/images/entrypoint.common.sh b/e2e-tests/images/entrypoint.common.sh index 52e2342b162..f4c9666ceba 100644 --- a/e2e-tests/images/entrypoint.common.sh +++ b/e2e-tests/images/entrypoint.common.sh @@ -9,6 +9,7 @@ declare -A PORTS=( [media-api]=9001 [thrall]=9002 + [image-loader]=9003 [kahuna]=9005 [cropper]=9006 [metadata-editor]=9007 @@ -17,7 +18,7 @@ declare -A PORTS=( [leases]=9012 ) -export SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall" +export SERVICES="auth collections cropper image-loader kahuna leases media-api metadata-editor thrall" # Install a TERM/INT trap that kills the given child pids, waits for them, and # exits cleanly. Pass the pids to tear down as arguments, e.g. diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 6d8cdb7ecce..6e3386cad94 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -33,6 +33,7 @@ export const GRID_ALIAS = 'grid-e2e-ci'; export const SERVICE_PORTS: Record = { 'media-api': 9001, thrall: 9002, + 'image-loader': 9003, kahuna: 9005, cropper: 9006, 'metadata-editor': 9007, From 9208cccf71364a89ffc501ed59006188c7a8744b Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 26 Aug 2026 10:25:54 +0000 Subject: [PATCH 282/373] Add imgops service to testcontainers setup --- e2e-tests/global-setup.ts | 26 +++++++++++++++++++++++++- e2e-tests/testcontainers/constants.ts | 10 ++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 7248f9f68ab..1b3aaeb5a93 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -3,7 +3,7 @@ * * Boots the full local Grid stack with Testcontainers: * 1. a shared network, - * 2. Elasticsearch + LocalStack (infrastructure), + * 2. Elasticsearch + LocalStack + imgops (infrastructure), * 3. the CloudFormation core stack + seeded buckets (provisioning), * 4. generated per-service config (reusing dev/script/generate-config), * 5. the pre-built `grid-e2e-ci` image running all nine services. @@ -23,6 +23,10 @@ import { ELASTICSEARCH_IMAGE, GRID_ALIAS, GRID_IMAGE, + IMGOPS_ALIAS, + IMGOPS_CONTEXT, + IMGOPS_NGINX_CONF, + IMGOPS_PORT, KAHUNA_PORT, LOCALSTACK_ALIAS, LOCALSTACK_IMAGE, @@ -127,6 +131,11 @@ function buildCaddyfile(coreStackProps: Record): string { `localstack.media.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, ); + // On-the-fly image resizing (optimised / full-screen views) -> the imgops container. + blocks.push( + `media-imgops.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${IMGOPS_ALIAS}:80\n}`, + ); + return `${blocks.join('\n\n')}\n`; } @@ -172,6 +181,21 @@ async function globalSetup(): Promise { .start(); started.push(localstack); + // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to + // the `localstack` alias on 4566, so it shares this network. Published on the fixed host + // port dev-nginx maps `media-imgops` to; in CI the Caddy proxy routes to it instead. The + // Dockerfile doesn't bake in nginx.conf (docker-compose bind-mounts it), so copy it in. + const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(); + const imgops = await imgopsImage + .withNetwork(network) + .withNetworkAliases(IMGOPS_ALIAS) + .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) + .withExposedPorts({ container: 80, host: IMGOPS_PORT }) + .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) + .withStartupTimeout(120_000) + .start(); + started.push(imgops); + // Provisioning + config generation const coreStackProps = await provisionCoreStack(localstack.getConnectionUri()); diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 6e3386cad94..0b468f3bec9 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -22,6 +22,16 @@ export const LOCALSTACK_IMAGE = 'localstack/localstack:4.5.0'; /** Reverse proxy used in CI to stand in for the developer's dev-nginx (see global-setup). */ export const PROXY_IMAGE = 'caddy:2.8-alpine'; +/** + * imgops: standalone nginx on-the-fly image resizer, built from dev/imgops at setup time. + * Its nginx.conf proxies to the `localstack` alias on 4566, so it shares the stack network. + * Published on IMGOPS_PORT, the fixed host port dev-nginx maps `media-imgops` to. + */ +export const IMGOPS_ALIAS = 'imgops'; +export const IMGOPS_PORT = 9008; +export const IMGOPS_CONTEXT = path.join(REPO_ROOT, 'dev', 'imgops'); +export const IMGOPS_NGINX_CONF = path.join(IMGOPS_CONTEXT, 'nginx.conf'); + /** Network aliases the app container uses to reach the infrastructure containers. */ export const ELASTICSEARCH_ALIAS = 'elasticsearch'; export const LOCALSTACK_ALIAS = 'localstack'; From 57ad17cdf9518cfd58c3967f4026bf543f674b7d Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 09:44:45 +0100 Subject: [PATCH 283/373] Add localstack.media, and provide a unique configuration var for the presign service --- .../scala/com/gu/mediaservice/lib/aws/S3.scala | 6 ++++-- .../gu/mediaservice/lib/config/CommonConfig.scala | 15 +++++++++++++++ e2e-tests/global-setup.ts | 3 +++ e2e-tests/package.json | 3 ++- e2e-tests/testcontainers/config.ts | 10 +++++++++- 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 2b3cd2a09fa..73b63dd8dba 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -226,8 +226,10 @@ object S3Ops { .credentialsProvider(config.awsCredentials) .region(config.awsRegion) - config.awsLocalEndpointUri match { - case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint) + config.awsLocalPresigningEndpointUri match { + case Some(endpoint) if config.isDev => + println(s"Using localPresigningEndpointUri $endpoint") + builder.endpointOverride(endpoint) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build() case _ => builder.build() diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 60ebefabcf8..c155be5f100 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -30,6 +30,21 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui val awsLocalEndpoint: Option[String] = if(isDev) stringOpt("aws.local.endpoint").filter(_.nonEmpty) else None override val awsLocalEndpointUri: Option[URI] = awsLocalEndpoint.map(new URI(_)) + // Endpoint baked into presigned URLs handed to the browser. Distinct from `aws.local.endpoint` + // (used by the S3 client for container-internal calls) because the browser cannot resolve the + // container-network host. Falls back to `aws.local.endpoint` when unset. + println("PRESIGN ENDPOINTS") + println(stringOpt("aws.local.presigningEndpoint")) + println(awsLocalEndpoint) + val awsLocalPresigningEndpointUri: Option[URI] = + if (isDev) stringOpt("aws.local.presigningEndpoint") + .filter(_.nonEmpty) + .orElse(awsLocalEndpoint) + .map(new URI(_)) + else None + + println(awsLocalPresigningEndpointUri) + val useLocalAuth: Boolean = isDev && boolean("auth.useLocal") val localLogShipping: Boolean = sys.env.getOrElse("LOCAL_LOG_SHIPPING", "false").toBoolean diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 1b3aaeb5a93..4e759ba5d69 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -112,8 +112,11 @@ function buildCaddyfile(coreStackProps: Record): string { const imageBuckets: Record = { [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket, [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket, + [`localstack.media.${DOMAIN}`]: coreStackProps.IngestQueueBucket }; + console.log({coreStackProps}) + const blocks: string[] = []; for (const [siteHost, port] of Object.entries(appServices)) { diff --git a/e2e-tests/package.json b/e2e-tests/package.json index af46044d324..33f1ab316c2 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -7,7 +7,8 @@ "test": "bddgen && playwright test", "test:headed": "bddgen && playwright test --headed", "test:report": "playwright show-report", - "test:ui": "bddgen && playwright test --ui-port 6080" + "test:ui": "bddgen && playwright test --ui-port 6080", + "test:log": "bddgen && DEBUG=testcontainers* playwright test --ui-port 6080" }, "keywords": [], "author": "", diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 3b6108faa37..24ddf255799 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -34,13 +34,21 @@ function rewriteEndpoints(conf: string): string { const guardianLocalstackUrl = `https://${LOCALSTACK_ALIAS}.media.${DOMAIN}`; const legacyLocalstackUrl = 'http://localhost:4576'; const localLocalstackUrl = `http://localhost:${LOCALSTACK_PORT}`; - return conf + + const rewrittenConf = conf .split(guardianLocalstackUrl) .join(localstackUrl) .split(legacyLocalstackUrl) .join(localstackUrl) .split(localLocalstackUrl) .join(localstackUrl); + + // The S3 client reaches LocalStack over the container network (localstack:4566), but + // presigned URLs are handed to the browser, which can only reach LocalStack via the + // `localstack.media.` vanity domain (dev-nginx locally, the Caddy proxy in CI). + // Sign against that host so the URLs resolve outside the container network. + console.log(`${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`); + return `${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`; } /** From 8a8c2e0f0be8a5b1ff3e8838bb249bb285d48b50 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 11:26:22 +0000 Subject: [PATCH 284/373] Add localstack to devcontainer forwarded ports; mount project into container when not in CI --- .devcontainer/devenv.yaml | 1 + .devcontainer/shared/devcontainer.json | 1 + .devcontainer/user/devcontainer.json | 1 + e2e-tests/global-setup.ts | 4 ++++ 4 files changed, 7 insertions(+) diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml index 661a33352ad..f11896105a2 100644 --- a/.devcontainer/devenv.yaml +++ b/.devcontainer/devenv.yaml @@ -27,6 +27,7 @@ forwardPorts: - 9010 - 9011 - 9012 + - 4566 - 6080 # Optional: Mount directories from host to container diff --git a/.devcontainer/shared/devcontainer.json b/.devcontainer/shared/devcontainer.json index da4c20e25bf..8796a02a51c 100644 --- a/.devcontainer/shared/devcontainer.json +++ b/.devcontainer/shared/devcontainer.json @@ -29,6 +29,7 @@ 9010, 9011, 9012, + 4566, 6080 ], "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json index 59b78adfb87..3abef9f164f 100644 --- a/.devcontainer/user/devcontainer.json +++ b/.devcontainer/user/devcontainer.json @@ -29,6 +29,7 @@ 9010, 9011, 9012, + 4566, 6080 ], "image" : "mcr.microsoft.com/devcontainers/base:ubuntu26.04", diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 4e759ba5d69..d5779fdec94 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -34,6 +34,7 @@ import { MEDIA_API_PORT, PROXY_IMAGE, REGION, + REPO_ROOT, SERVICE_PORTS, URLS_FILE, } from './testcontainers/constants'; @@ -220,6 +221,9 @@ async function globalSetup(): Promise { // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. { source: configDir, target: '/root/.grid', mode: 'ro' }, { source: configDir, target: '/etc/grid', mode: 'ro' }, + // Outside CI the grid-e2e-dev image runs services under sbt; mount the repo + // over /build so host edits recompile live. + ...(process.env.CI ? [] : [{ source: REPO_ROOT, target: '/build', mode: 'rw' as const }]), ]) .withEnvironment({ AWS_ACCESS_KEY_ID: 'test', From 4c2d4cd750bca635267bd936990f9eb06c7c739f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 15:08:00 +0000 Subject: [PATCH 285/373] Add stable tag for imgops container to stop it rebuilding every startup --- e2e-tests/global-setup.ts | 5 ++++- e2e-tests/testcontainers/constants.ts | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index d5779fdec94..0952720a082 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -25,6 +25,7 @@ import { GRID_IMAGE, IMGOPS_ALIAS, IMGOPS_CONTEXT, + IMGOPS_IMAGE, IMGOPS_NGINX_CONF, IMGOPS_PORT, KAHUNA_PORT, @@ -189,7 +190,9 @@ async function globalSetup(): Promise { // the `localstack` alias on 4566, so it shares this network. Published on the fixed host // port dev-nginx maps `media-imgops` to; in CI the Caddy proxy routes to it instead. The // Dockerfile doesn't bake in nginx.conf (docker-compose bind-mounts it), so copy it in. - const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(); + const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { + deleteOnExit: false, + }); const imgops = await imgopsImage .withNetwork(network) .withNetworkAliases(IMGOPS_ALIAS) diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 0b468f3bec9..04c858250b2 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -31,6 +31,8 @@ export const IMGOPS_ALIAS = 'imgops'; export const IMGOPS_PORT = 9008; export const IMGOPS_CONTEXT = path.join(REPO_ROOT, 'dev', 'imgops'); export const IMGOPS_NGINX_CONF = path.join(IMGOPS_CONTEXT, 'nginx.conf'); +/** Stable tag for the imgops image so it persists across runs and Docker reuses cached layers. */ +export const IMGOPS_IMAGE = 'grid-e2e-imgops'; /** Network aliases the app container uses to reach the infrastructure containers. */ export const ELASTICSEARCH_ALIAS = 'elasticsearch'; From e7f3cb909c2fc07d4d729777fbb7d904bd4f80ca Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 15:08:52 +0000 Subject: [PATCH 286/373] Wait for all Grid services to come up before reporting the container as healthy --- e2e-tests/global-setup.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 0952720a082..aa8dea35458 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -236,10 +236,9 @@ async function globalSetup(): Promise { AWS_CBOR_DISABLE: 'true', }) .withWaitStrategy( - Wait.forAll([ - Wait.forHttp('/management/healthcheck', MEDIA_API_PORT).forStatusCode(200), - Wait.forHttp('/management/healthcheck', SERVICE_PORTS['image-loader']).forStatusCode(200), - ]), + Wait.forAll(Object.values(SERVICE_PORTS).map(port => + Wait.forHttp('/management/healthcheck', port).forStatusCode(200), + )) ) .withStartupTimeout(startupTimeoutMs); From 98240e0deed2e410a0f391a5857f166223c3b3fa Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 15:19:22 +0000 Subject: [PATCH 287/373] Tidy up after a manual review --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 6 ++---- e2e-tests/README.md | 2 +- e2e-tests/global-setup.ts | 2 +- e2e-tests/images/Dockerfile | 8 ++++---- e2e-tests/images/README.md | 4 ++-- e2e-tests/testcontainers/config.ts | 2 +- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 73b63dd8dba..2b3cd2a09fa 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -226,10 +226,8 @@ object S3Ops { .credentialsProvider(config.awsCredentials) .region(config.awsRegion) - config.awsLocalPresigningEndpointUri match { - case Some(endpoint) if config.isDev => - println(s"Using localPresigningEndpointUri $endpoint") - builder.endpointOverride(endpoint) + config.awsLocalEndpointUri match { + case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build() case _ => builder.build() diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 09df0ac1206..89dcb689940 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -55,7 +55,7 @@ Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses 3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded buckets, provisioned into LocalStack; 4. per-service config, generated by `dev/script/generate-config/service-config.js`; -5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running all nine Play services. +5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running Grid's Play services. Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is exposed to the tests. `globalTeardown` stops everything and cleans up. diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index aa8dea35458..4527b514dfc 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -209,7 +209,7 @@ async function globalSetup(): Promise { const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); generateServiceConfig(configDir, coreStackProps); - // All nine Grid services run inside this single container and talk to each + // All Grid services under test run inside this single container and talk to each // other over its localhost. Each is published on the fixed host port its // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's // dev-nginx routes the https://*.media. domains straight into this diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 430c2a911e8..579c271a5b4 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -1,14 +1,14 @@ # syntax=docker/dockerfile:1 -# Single-container image that builds and runs nine Grid Play services +# Single-container image that builds and runs all Grid Play services under test # (auth, collections, cropper, image-loader, kahuna, leases, media-api, # metadata-editor, thrall) together with Kahuna's frontend. # # This one Dockerfile produces two images, selected with `--target`: # -# * ci (grid-e2e-ci) — CI/production style. Stages pre-compiled artefacts -# and runs them under a slim JRE. -# docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci . +# * ci (grid-e2e-ci) — CI/production style. Stages pre-compiled artefacts and +# runs them under a slim JRE. docker build --target ci +# -f e2e-tests/images/Dockerfile -t grid-e2e-ci . # # * dev (grid-e2e-dev) — local development. Runs the services under # `sbt /run` (Play dev mode) with the repo diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index d3cec50f940..c23ff06cd75 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,7 +1,7 @@ c# Grid E2E test images -A single `Dockerfile` produces two single-container images that run nine -Grid Play services plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter: +A single `Dockerfile` produces two single-container images that run all +Grid Play services under test, plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter: - **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. - **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 24ddf255799..8021abf5527 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -47,7 +47,6 @@ function rewriteEndpoints(conf: string): string { // presigned URLs are handed to the browser, which can only reach LocalStack via the // `localstack.media.` vanity domain (dev-nginx locally, the Caddy proxy in CI). // Sign against that host so the URLs resolve outside the container network. - console.log(`${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`); return `${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`; } @@ -97,6 +96,7 @@ export function generateServiceConfig(configDir: string, coreStackProps: StackPr if (!conf) { throw new Error(`service-config.js did not produce config for '${service}'`); } + fs.writeFileSync(path.join(configDir, `${service}.conf`), rewriteEndpoints(conf)); } } From 3612f487dde771f62d58117c2458e4a4a88ac08a Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 15:46:33 +0000 Subject: [PATCH 288/373] Correct presigning config --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 2 +- .../scala/com/gu/mediaservice/lib/config/CommonConfig.scala | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 2b3cd2a09fa..ebfd333496e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -226,7 +226,7 @@ object S3Ops { .credentialsProvider(config.awsCredentials) .region(config.awsRegion) - config.awsLocalEndpointUri match { + config.awsLocalPresigningEndpointUri match { case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build() case _ => builder.build() diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index c155be5f100..669e49490c8 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -33,9 +33,6 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui // Endpoint baked into presigned URLs handed to the browser. Distinct from `aws.local.endpoint` // (used by the S3 client for container-internal calls) because the browser cannot resolve the // container-network host. Falls back to `aws.local.endpoint` when unset. - println("PRESIGN ENDPOINTS") - println(stringOpt("aws.local.presigningEndpoint")) - println(awsLocalEndpoint) val awsLocalPresigningEndpointUri: Option[URI] = if (isDev) stringOpt("aws.local.presigningEndpoint") .filter(_.nonEmpty) @@ -43,8 +40,6 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui .map(new URI(_)) else None - println(awsLocalPresigningEndpointUri) - val useLocalAuth: Boolean = isDev && boolean("auth.useLocal") val localLogShipping: Boolean = sys.env.getOrElse("LOCAL_LOG_SHIPPING", "false").toBoolean From 1c61f4ead7db069d323c9eba231e2017cc841a8d Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 16:10:20 +0000 Subject: [PATCH 289/373] Remove unnecessary caddy entry, the localstack entry already exists --- e2e-tests/global-setup.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 4527b514dfc..bca9bd18167 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -6,7 +6,7 @@ * 2. Elasticsearch + LocalStack + imgops (infrastructure), * 3. the CloudFormation core stack + seeded buckets (provisioning), * 4. generated per-service config (reusing dev/script/generate-config), - * 5. the pre-built `grid-e2e-ci` image running all nine services. + * 5. the pre-built `grid-e2e-ci` image running the Grid services under test. * * The Kahuna base URL is exposed to tests via `GRID_BASE_URL`, and the started * containers are stashed for `global-teardown.ts`. @@ -113,12 +113,9 @@ function buildCaddyfile(coreStackProps: Record): string { // S3 vanity domains that omit the bucket -> localstack, with the bucket prepended. const imageBuckets: Record = { [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket, - [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket, - [`localstack.media.${DOMAIN}`]: coreStackProps.IngestQueueBucket + [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket }; - console.log({coreStackProps}) - const blocks: string[] = []; for (const [siteHost, port] of Object.entries(appServices)) { @@ -187,9 +184,7 @@ async function globalSetup(): Promise { started.push(localstack); // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to - // the `localstack` alias on 4566, so it shares this network. Published on the fixed host - // port dev-nginx maps `media-imgops` to; in CI the Caddy proxy routes to it instead. The - // Dockerfile doesn't bake in nginx.conf (docker-compose bind-mounts it), so copy it in. + // the `localstack` alias on 4566, so it shares this network. const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { deleteOnExit: false, }); From 46fb21dfba893028b5d0aa659118ca9138fc584f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 16:23:26 +0000 Subject: [PATCH 290/373] Interleave image startup for a quicker spinup --- e2e-tests/global-setup.ts | 60 +++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index bca9bd18167..7633d27280f 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -142,13 +142,13 @@ function buildCaddyfile(coreStackProps: Record): string { } async function globalSetup(): Promise { - const started: StartedTestContainer[] = []; + const toStart: Promise[] = []; const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); const network = await new Network().start(); // Infrastructure: Elasticsearch + LocalStack - const elasticsearch = await new GenericContainer(ELASTICSEARCH_IMAGE) + const elasticsearch = new GenericContainer(ELASTICSEARCH_IMAGE) .withNetwork(network) .withNetworkAliases(ELASTICSEARCH_ALIAS) .withEnvironment({ @@ -162,9 +162,9 @@ async function globalSetup(): Promise { ) .withStartupTimeout(180_000) .start(); - started.push(elasticsearch); + toStart.push(elasticsearch); - const localstack = await new LocalstackContainer(LOCALSTACK_IMAGE) + const localstack = new LocalstackContainer(LOCALSTACK_IMAGE) .withNetwork(network) .withNetworkAliases(LOCALSTACK_ALIAS) // Pin to the fixed host port dev-nginx expects for the S3 vanity domains @@ -181,14 +181,14 @@ async function globalSetup(): Promise { }) .withStartupTimeout(120_000) .start(); - started.push(localstack); + toStart.push(localstack); // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to // the `localstack` alias on 4566, so it shares this network. const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { deleteOnExit: false, }); - const imgops = await imgopsImage + const imgops = imgopsImage .withNetwork(network) .withNetworkAliases(IMGOPS_ALIAS) .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) @@ -196,14 +196,35 @@ async function globalSetup(): Promise { .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) .withStartupTimeout(120_000) .start(); - started.push(imgops); + toStart.push(imgops); + + await Promise.all(toStart); // Provisioning + config generation - const coreStackProps = await provisionCoreStack(localstack.getConnectionUri()); + const coreStackProps = await provisionCoreStack((await localstack).getConnectionUri()); const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); generateServiceConfig(configDir, coreStackProps); + // CI routing: bundled reverse proxy + // + // Locally, the browser reaches the https://*.media. domains via the developer's + // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) + // start a Caddy proxy that replays the same subdomain routing and terminates TLS with a + // self-signed cert, published on the standard https port the domains resolve to. + if (process.env.CI) { + const proxy = new GenericContainer(PROXY_IMAGE) + .withNetwork(network) + .withExposedPorts({ container: 443, host: 443 }) + .withCopyContentToContainer([ + { content: buildCaddyfile(coreStackProps), target: '/etc/caddy/Caddyfile' }, + ]) + .withWaitStrategy(Wait.forListeningPorts()) + .withStartupTimeout(60_000) + .start(); + toStart.push(proxy); + } + // All Grid services under test run inside this single container and talk to each // other over its localhost. Each is published on the fixed host port its // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's @@ -245,35 +266,18 @@ async function globalSetup(): Promise { }); } + const started: StartedTestContainer[] = await Promise.all(toStart); const grid = await gridBuilder.start(); started.push(grid); + const startedESContainer = await elasticsearch; // Seed Elasticsearch with image fixtures // // The app creates the `images` index + `Images_Current` alias on startup; seed once the // stack is healthy so searches during the tests return the fixture documents. - const esBaseUrl = `http://${elasticsearch.getHost()}:${elasticsearch.getMappedPort(9200)}`; + const esBaseUrl = `http://${startedESContainer.getHost()}:${startedESContainer.getMappedPort(9200)}`; await seedElasticsearch(esBaseUrl); - // CI routing: bundled reverse proxy - // - // Locally, the browser reaches the https://*.media. domains via the developer's - // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) - // start a Caddy proxy that replays the same subdomain routing and terminates TLS with a - // self-signed cert, published on the standard https port the domains resolve to. - if (process.env.CI) { - const proxy = await new GenericContainer(PROXY_IMAGE) - .withNetwork(network) - .withExposedPorts({ container: 443, host: 443 }) - .withCopyContentToContainer([ - { content: buildCaddyfile(coreStackProps), target: '/etc/caddy/Caddyfile' }, - ]) - .withWaitStrategy(Wait.forListeningPorts()) - .withStartupTimeout(60_000) - .start(); - started.push(proxy); - } - const host = grid.getHost(); const baseUrl = `http://${host}:${grid.getMappedPort(KAHUNA_PORT)}`; const mediaApiUrl = `http://${host}:${grid.getMappedPort(MEDIA_API_PORT)}`; From 38096a406df3f588c36b2b71999705157670c9a5 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 3 Sep 2026 10:36:12 +0000 Subject: [PATCH 291/373] Await Grid and Caddy simultaneously; include imgops container build in promise chain to avoid unhandled promise --- e2e-tests/global-setup.ts | 72 ++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 38 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 7633d27280f..2ec08cc1760 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -45,14 +45,14 @@ import { seedElasticsearch } from './testcontainers/seed-elasticsearch'; import { setEnvironment } from './testcontainers/state'; const LOCALSTACK_SERVICES = [ - "cloudformation", - "cloudwatch", - "dynamodb", - "kinesis", - "s3", - "sns", - "sqs", - "iam", + "cloudformation", + "cloudwatch", + "dynamodb", + "kinesis", + "s3", + "sns", + "sqs", + "iam", ].join(","); /** @@ -142,13 +142,12 @@ function buildCaddyfile(coreStackProps: Record): string { } async function globalSetup(): Promise { - const toStart: Promise[] = []; const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); const network = await new Network().start(); // Infrastructure: Elasticsearch + LocalStack - const elasticsearch = new GenericContainer(ELASTICSEARCH_IMAGE) + const eventuallyElasticSearch = new GenericContainer(ELASTICSEARCH_IMAGE) .withNetwork(network) .withNetworkAliases(ELASTICSEARCH_ALIAS) .withEnvironment({ @@ -162,9 +161,8 @@ async function globalSetup(): Promise { ) .withStartupTimeout(180_000) .start(); - toStart.push(elasticsearch); - const localstack = new LocalstackContainer(LOCALSTACK_IMAGE) + const eventuallyLocalstack = new LocalstackContainer(LOCALSTACK_IMAGE) .withNetwork(network) .withNetworkAliases(LOCALSTACK_ALIAS) // Pin to the fixed host port dev-nginx expects for the S3 vanity domains @@ -181,27 +179,26 @@ async function globalSetup(): Promise { }) .withStartupTimeout(120_000) .start(); - toStart.push(localstack); // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to // the `localstack` alias on 4566, so it shares this network. - const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { + const imgopsImage = GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { deleteOnExit: false, }); - const imgops = imgopsImage + const eventuallyImgOps = imgopsImage.then(image => image .withNetwork(network) .withNetworkAliases(IMGOPS_ALIAS) .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) .withExposedPorts({ container: 80, host: IMGOPS_PORT }) .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) .withStartupTimeout(120_000) - .start(); - toStart.push(imgops); + .start() + ) - await Promise.all(toStart); + const [imgops, elasticsearch, localstack] = await Promise.all([eventuallyImgOps, eventuallyElasticSearch, eventuallyLocalstack]); // Provisioning + config generation - const coreStackProps = await provisionCoreStack((await localstack).getConnectionUri()); + const coreStackProps = await provisionCoreStack((await eventuallyLocalstack).getConnectionUri()); const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); generateServiceConfig(configDir, coreStackProps); @@ -212,25 +209,24 @@ async function globalSetup(): Promise { // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) // start a Caddy proxy that replays the same subdomain routing and terminates TLS with a // self-signed cert, published on the standard https port the domains resolve to. - if (process.env.CI) { - const proxy = new GenericContainer(PROXY_IMAGE) - .withNetwork(network) - .withExposedPorts({ container: 443, host: 443 }) - .withCopyContentToContainer([ - { content: buildCaddyfile(coreStackProps), target: '/etc/caddy/Caddyfile' }, - ]) - .withWaitStrategy(Wait.forListeningPorts()) - .withStartupTimeout(60_000) - .start(); - toStart.push(proxy); - } + + const maybeEventuallyProxy = !process.env.CI ? Promise.resolve() : new GenericContainer(PROXY_IMAGE) + .withNetwork(network) + .withExposedPorts({ container: 443, host: 443 }) + .withCopyContentToContainer([ + { content: buildCaddyfile(coreStackProps), target: '/etc/caddy/Caddyfile' }, + ]) + .withWaitStrategy(Wait.forListeningPorts()) + .withStartupTimeout(60_000) + .start(); + // All Grid services under test run inside this single container and talk to each // other over its localhost. Each is published on the fixed host port its // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's // dev-nginx routes the https://*.media. domains straight into this // container. - let gridBuilder = new GenericContainer(GRID_IMAGE) + let gridContainerDefinition = new GenericContainer(GRID_IMAGE) .withNetwork(network) .withNetworkAliases(GRID_ALIAS) .withExposedPorts( @@ -260,17 +256,17 @@ async function globalSetup(): Promise { if (process.env.GRID_DEBUG) { const logStream = fs.createWriteStream(path.join(os.tmpdir(), 'grid-boot.log')); - gridBuilder = gridBuilder.withLogConsumer((stream) => { + gridContainerDefinition = gridContainerDefinition.withLogConsumer((stream) => { stream.on('data', (line) => logStream.write(line)); stream.on('err', (line) => logStream.write(line)); }); } - const started: StartedTestContainer[] = await Promise.all(toStart); - const grid = await gridBuilder.start(); - started.push(grid); + const eventuallyGrid = gridContainerDefinition.start(); + + const [grid,] = await Promise.all([eventuallyGrid, maybeEventuallyProxy]); - const startedESContainer = await elasticsearch; + const startedESContainer = await eventuallyElasticSearch; // Seed Elasticsearch with image fixtures // // The app creates the `images` index + `Images_Current` alias on startup; seed once the @@ -285,7 +281,7 @@ async function globalSetup(): Promise { process.env.GRID_BASE_URL = baseUrl; fs.writeFileSync(URLS_FILE, JSON.stringify({ kahuna: baseUrl, mediaApi: mediaApiUrl })); - setEnvironment({ network, containers: started, configDir, baseUrl }); + setEnvironment({ network, containers: [elasticsearch, localstack, imgops, grid], configDir, baseUrl }); } export default globalSetup; From af88629510d5bd3dc3c58f543c3f8dd0a4ffafcc Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 3 Sep 2026 12:41:19 +0000 Subject: [PATCH 292/373] Add proxy to teardown containers --- e2e-tests/global-setup.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 2ec08cc1760..4eefc2955f9 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -264,7 +264,7 @@ async function globalSetup(): Promise { const eventuallyGrid = gridContainerDefinition.start(); - const [grid,] = await Promise.all([eventuallyGrid, maybeEventuallyProxy]); + const [grid, proxy] = await Promise.all([eventuallyGrid, maybeEventuallyProxy]); const startedESContainer = await eventuallyElasticSearch; // Seed Elasticsearch with image fixtures @@ -281,7 +281,7 @@ async function globalSetup(): Promise { process.env.GRID_BASE_URL = baseUrl; fs.writeFileSync(URLS_FILE, JSON.stringify({ kahuna: baseUrl, mediaApi: mediaApiUrl })); - setEnvironment({ network, containers: [elasticsearch, localstack, imgops, grid], configDir, baseUrl }); + setEnvironment({ network, containers: [elasticsearch, localstack, imgops, grid, ...(proxy ? [proxy] : [])], configDir, baseUrl }); } export default globalSetup; From 41ebdfdb555d2e40db27b212ad049562634659bf Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 3 Sep 2026 12:49:40 +0000 Subject: [PATCH 293/373] Remove unused import --- e2e-tests/global-setup.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 4eefc2955f9..f237a6dd95c 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -16,7 +16,6 @@ import * as os from 'os'; import * as path from 'path'; import { LocalstackContainer } from '@testcontainers/localstack'; import { GenericContainer, Network, Wait } from 'testcontainers'; -import type { StartedTestContainer } from 'testcontainers'; import { DOMAIN, ELASTICSEARCH_ALIAS, From 851a3d955313bfac915537fb5028b39ce52ebbd6 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 26 Aug 2026 09:19:17 +0000 Subject: [PATCH 294/373] Add image-loader to the list of running services --- e2e-tests/README.md | 2 +- e2e-tests/global-setup.ts | 12 +++++++++--- e2e-tests/images/Dockerfile | 18 ++++++++++-------- e2e-tests/images/README.md | 4 ++-- e2e-tests/images/entrypoint.common.sh | 3 ++- e2e-tests/testcontainers/constants.ts | 1 + 6 files changed, 25 insertions(+), 15 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index beda695272b..09df0ac1206 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -55,7 +55,7 @@ Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses 3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded buckets, provisioned into LocalStack; 4. per-service config, generated by `dev/script/generate-config/service-config.js`; -5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running all eight Play services. +5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running all nine Play services. Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is exposed to the tests. `globalTeardown` stops everything and cleans up. diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 4b6d11bb719..d104dda8a3e 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -6,7 +6,7 @@ * 2. Elasticsearch + LocalStack (infrastructure), * 3. the CloudFormation core stack + seeded buckets (provisioning), * 4. generated per-service config (reusing dev/script/generate-config), - * 5. the pre-built `grid-e2e-ci` image running all eight services. + * 5. the pre-built `grid-e2e-ci` image running all nine services. * * The Kahuna base URL is exposed to tests via `GRID_BASE_URL`, and the started * containers are stashed for `global-teardown.ts`. @@ -96,6 +96,7 @@ function buildCaddyfile(coreStackProps: Record): string { const appServices: Record = { [`media.${DOMAIN}`]: SERVICE_PORTS.kahuna, [`api.media.${DOMAIN}`]: SERVICE_PORTS['media-api'], + [`loader.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], [`cropper.media.${DOMAIN}`]: SERVICE_PORTS.cropper, [`thrall.media.${DOMAIN}`]: SERVICE_PORTS.thrall, [`media-metadata.${DOMAIN}`]: SERVICE_PORTS['metadata-editor'], @@ -178,7 +179,7 @@ async function globalSetup(): Promise { const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); generateServiceConfig(configDir, coreStackProps); - // All eight Grid services run inside this single container and talk to each + // All nine Grid services run inside this single container and talk to each // other over its localhost. Each is published on the fixed host port its // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's // dev-nginx routes the https://*.media. domains straight into this @@ -204,7 +205,12 @@ async function globalSetup(): Promise { AWS_DEFAULT_REGION: REGION, AWS_CBOR_DISABLE: 'true', }) - .withWaitStrategy(Wait.forHttp('/management/healthcheck', MEDIA_API_PORT).forStatusCode(200)) + .withWaitStrategy( + Wait.forAll([ + Wait.forHttp('/management/healthcheck', MEDIA_API_PORT).forStatusCode(200), + Wait.forHttp('/management/healthcheck', SERVICE_PORTS['image-loader']).forStatusCode(200), + ]), + ) .withStartupTimeout(startupTimeoutMs); if (process.env.GRID_DEBUG) { diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 265365f3c4d..430c2a911e8 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1 -# Single-container image that builds and runs eight Grid Play services -# (auth, collections, cropper, kahuna, leases, media-api, metadata-editor, -# thrall) together with Kahuna's frontend. +# Single-container image that builds and runs nine Grid Play services +# (auth, collections, cropper, image-loader, kahuna, leases, media-api, +# metadata-editor, thrall) together with Kahuna's frontend. # # This one Dockerfile produces two images, selected with `--target`: # @@ -75,6 +75,7 @@ RUN sbt \ "auth/stage" \ "collections/stage" \ "cropper/stage" \ + "image-loader/stage" \ "kahuna/stage" \ "leases/stage" \ "media-api/stage" \ @@ -97,6 +98,7 @@ RUN set -eux; \ COPY --from=backend /build/auth/target/universal/stage /usr/share/auth COPY --from=backend /build/collections/target/universal/stage /usr/share/collections COPY --from=backend /build/cropper/target/universal/stage /usr/share/cropper +COPY --from=backend /build/image-loader/target/universal/stage /usr/share/image-loader COPY --from=backend /build/kahuna/target/universal/stage /usr/share/kahuna COPY --from=backend /build/leases/target/universal/stage /usr/share/leases COPY --from=backend /build/media-api/target/universal/stage /usr/share/media-api @@ -104,7 +106,7 @@ COPY --from=backend /build/metadata-editor/target/universal/stage /usr/share/met COPY --from=backend /build/thrall/target/universal/stage /usr/share/thrall RUN set -eux; \ - for service in auth collections cropper kahuna leases media-api metadata-editor thrall; do \ + for service in auth collections cropper image-loader kahuna leases media-api metadata-editor thrall; do \ mkdir -p "/var/log/$service"; \ done @@ -115,8 +117,8 @@ RUN chmod +x /usr/local/bin/entrypoint.ci.sh # The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. ENV AWS_CBOR_DISABLE=true -# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases -EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 +# media-api, thrall, image-loader, kahuna, cropper, metadata-editor, collections, auth, leases +EXPOSE 9001 9002 9003 9005 9006 9007 9010 9011 9012 ENTRYPOINT ["/usr/local/bin/entrypoint.ci.sh"] @@ -173,9 +175,9 @@ RUN chmod +x /usr/local/bin/entrypoint.dev.sh # The Java SDK CBOR protocol is disabled for Localstack/Kinesis compatibility. ENV AWS_CBOR_DISABLE=true -# media-api, thrall, kahuna, cropper, metadata-editor, collections, auth, leases +# media-api, thrall, image-loader, kahuna, cropper, metadata-editor, collections, auth, leases # plus the JDWP debug port used when GRID_DEBUG is set. -EXPOSE 9001 9002 9005 9006 9007 9010 9011 9012 5005 +EXPOSE 9001 9002 9003 9005 9006 9007 9010 9011 9012 5005 ENTRYPOINT ["/usr/local/bin/entrypoint.dev.sh"] diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index 5ca9ecc9760..d3cec50f940 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,6 +1,6 @@ c# Grid E2E test images -A single `Dockerfile` produces two single-container images that run eight +A single `Dockerfile` produces two single-container images that run nine Grid Play services plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter: - **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. - **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. @@ -24,7 +24,7 @@ from `/etc/grid/stage`; defaults to `DEV`). Mount your environment's config with ```bash docker run --rm \ -v "$PWD/my-config:/etc/grid:ro" \ - -p 9001:9001 -p 9002:9002 -p 9005:9005 -p 9006:9006 \ + -p 9001:9001 -p 9002:9002 -p 9003:9003 -p 9005:9005 -p 9006:9006 \ -p 9007:9007 -p 9010:9010 -p 9011:9011 -p 9012:9012 \ grid-e2e-[ci|dev] ``` diff --git a/e2e-tests/images/entrypoint.common.sh b/e2e-tests/images/entrypoint.common.sh index 52e2342b162..f4c9666ceba 100644 --- a/e2e-tests/images/entrypoint.common.sh +++ b/e2e-tests/images/entrypoint.common.sh @@ -9,6 +9,7 @@ declare -A PORTS=( [media-api]=9001 [thrall]=9002 + [image-loader]=9003 [kahuna]=9005 [cropper]=9006 [metadata-editor]=9007 @@ -17,7 +18,7 @@ declare -A PORTS=( [leases]=9012 ) -export SERVICES="auth collections cropper kahuna leases media-api metadata-editor thrall" +export SERVICES="auth collections cropper image-loader kahuna leases media-api metadata-editor thrall" # Install a TERM/INT trap that kills the given child pids, waits for them, and # exits cleanly. Pass the pids to tear down as arguments, e.g. diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 6d8cdb7ecce..6e3386cad94 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -33,6 +33,7 @@ export const GRID_ALIAS = 'grid-e2e-ci'; export const SERVICE_PORTS: Record = { 'media-api': 9001, thrall: 9002, + 'image-loader': 9003, kahuna: 9005, cropper: 9006, 'metadata-editor': 9007, From fee3610549f4882f6073ead6bd56e1fa32f5baa9 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 26 Aug 2026 10:25:54 +0000 Subject: [PATCH 295/373] Add imgops service to testcontainers setup --- e2e-tests/global-setup.ts | 26 +++++++++++++++++++++++++- e2e-tests/testcontainers/constants.ts | 10 ++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index d104dda8a3e..a890fc78581 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -3,7 +3,7 @@ * * Boots the full local Grid stack with Testcontainers: * 1. a shared network, - * 2. Elasticsearch + LocalStack (infrastructure), + * 2. Elasticsearch + LocalStack + imgops (infrastructure), * 3. the CloudFormation core stack + seeded buckets (provisioning), * 4. generated per-service config (reusing dev/script/generate-config), * 5. the pre-built `grid-e2e-ci` image running all nine services. @@ -23,6 +23,10 @@ import { ELASTICSEARCH_IMAGE, GRID_ALIAS, GRID_IMAGE, + IMGOPS_ALIAS, + IMGOPS_CONTEXT, + IMGOPS_NGINX_CONF, + IMGOPS_PORT, KAHUNA_PORT, LOCALSTACK_ALIAS, LOCALSTACK_IMAGE, @@ -128,6 +132,11 @@ function buildCaddyfile(coreStackProps: Record): string { `localstack.media.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, ); + // On-the-fly image resizing (optimised / full-screen views) -> the imgops container. + blocks.push( + `media-imgops.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${IMGOPS_ALIAS}:80\n}`, + ); + return `${blocks.join('\n\n')}\n`; } @@ -173,6 +182,21 @@ async function globalSetup(): Promise { .start(); started.push(localstack); + // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to + // the `localstack` alias on 4566, so it shares this network. Published on the fixed host + // port dev-nginx maps `media-imgops` to; in CI the Caddy proxy routes to it instead. The + // Dockerfile doesn't bake in nginx.conf (docker-compose bind-mounts it), so copy it in. + const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(); + const imgops = await imgopsImage + .withNetwork(network) + .withNetworkAliases(IMGOPS_ALIAS) + .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) + .withExposedPorts({ container: 80, host: IMGOPS_PORT }) + .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) + .withStartupTimeout(120_000) + .start(); + started.push(imgops); + // Provisioning + config generation const coreStackProps = await provisionCoreStack(localstack.getConnectionUri()); diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 6e3386cad94..0b468f3bec9 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -22,6 +22,16 @@ export const LOCALSTACK_IMAGE = 'localstack/localstack:4.5.0'; /** Reverse proxy used in CI to stand in for the developer's dev-nginx (see global-setup). */ export const PROXY_IMAGE = 'caddy:2.8-alpine'; +/** + * imgops: standalone nginx on-the-fly image resizer, built from dev/imgops at setup time. + * Its nginx.conf proxies to the `localstack` alias on 4566, so it shares the stack network. + * Published on IMGOPS_PORT, the fixed host port dev-nginx maps `media-imgops` to. + */ +export const IMGOPS_ALIAS = 'imgops'; +export const IMGOPS_PORT = 9008; +export const IMGOPS_CONTEXT = path.join(REPO_ROOT, 'dev', 'imgops'); +export const IMGOPS_NGINX_CONF = path.join(IMGOPS_CONTEXT, 'nginx.conf'); + /** Network aliases the app container uses to reach the infrastructure containers. */ export const ELASTICSEARCH_ALIAS = 'elasticsearch'; export const LOCALSTACK_ALIAS = 'localstack'; From 1c3c007f776a0cf053006d89240236bf53912a7a Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 09:44:45 +0100 Subject: [PATCH 296/373] Add localstack.media, and provide a unique configuration var for the presign service --- .../scala/com/gu/mediaservice/lib/aws/S3.scala | 6 ++++-- .../gu/mediaservice/lib/config/CommonConfig.scala | 15 +++++++++++++++ e2e-tests/global-setup.ts | 3 +++ e2e-tests/package.json | 3 ++- e2e-tests/testcontainers/config.ts | 10 +++++++++- 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 2b3cd2a09fa..73b63dd8dba 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -226,8 +226,10 @@ object S3Ops { .credentialsProvider(config.awsCredentials) .region(config.awsRegion) - config.awsLocalEndpointUri match { - case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint) + config.awsLocalPresigningEndpointUri match { + case Some(endpoint) if config.isDev => + println(s"Using localPresigningEndpointUri $endpoint") + builder.endpointOverride(endpoint) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build() case _ => builder.build() diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index 60ebefabcf8..c155be5f100 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -30,6 +30,21 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui val awsLocalEndpoint: Option[String] = if(isDev) stringOpt("aws.local.endpoint").filter(_.nonEmpty) else None override val awsLocalEndpointUri: Option[URI] = awsLocalEndpoint.map(new URI(_)) + // Endpoint baked into presigned URLs handed to the browser. Distinct from `aws.local.endpoint` + // (used by the S3 client for container-internal calls) because the browser cannot resolve the + // container-network host. Falls back to `aws.local.endpoint` when unset. + println("PRESIGN ENDPOINTS") + println(stringOpt("aws.local.presigningEndpoint")) + println(awsLocalEndpoint) + val awsLocalPresigningEndpointUri: Option[URI] = + if (isDev) stringOpt("aws.local.presigningEndpoint") + .filter(_.nonEmpty) + .orElse(awsLocalEndpoint) + .map(new URI(_)) + else None + + println(awsLocalPresigningEndpointUri) + val useLocalAuth: Boolean = isDev && boolean("auth.useLocal") val localLogShipping: Boolean = sys.env.getOrElse("LOCAL_LOG_SHIPPING", "false").toBoolean diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index a890fc78581..d5779fdec94 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -113,8 +113,11 @@ function buildCaddyfile(coreStackProps: Record): string { const imageBuckets: Record = { [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket, [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket, + [`localstack.media.${DOMAIN}`]: coreStackProps.IngestQueueBucket }; + console.log({coreStackProps}) + const blocks: string[] = []; for (const [siteHost, port] of Object.entries(appServices)) { diff --git a/e2e-tests/package.json b/e2e-tests/package.json index af46044d324..33f1ab316c2 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -7,7 +7,8 @@ "test": "bddgen && playwright test", "test:headed": "bddgen && playwright test --headed", "test:report": "playwright show-report", - "test:ui": "bddgen && playwright test --ui-port 6080" + "test:ui": "bddgen && playwright test --ui-port 6080", + "test:log": "bddgen && DEBUG=testcontainers* playwright test --ui-port 6080" }, "keywords": [], "author": "", diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 3b6108faa37..24ddf255799 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -34,13 +34,21 @@ function rewriteEndpoints(conf: string): string { const guardianLocalstackUrl = `https://${LOCALSTACK_ALIAS}.media.${DOMAIN}`; const legacyLocalstackUrl = 'http://localhost:4576'; const localLocalstackUrl = `http://localhost:${LOCALSTACK_PORT}`; - return conf + + const rewrittenConf = conf .split(guardianLocalstackUrl) .join(localstackUrl) .split(legacyLocalstackUrl) .join(localstackUrl) .split(localLocalstackUrl) .join(localstackUrl); + + // The S3 client reaches LocalStack over the container network (localstack:4566), but + // presigned URLs are handed to the browser, which can only reach LocalStack via the + // `localstack.media.` vanity domain (dev-nginx locally, the Caddy proxy in CI). + // Sign against that host so the URLs resolve outside the container network. + console.log(`${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`); + return `${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`; } /** From 17da1054f7c6e013c72d7112be9eebf2089d7d6b Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 15:08:00 +0000 Subject: [PATCH 297/373] Add stable tag for imgops container to stop it rebuilding every startup --- e2e-tests/global-setup.ts | 5 ++++- e2e-tests/testcontainers/constants.ts | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index d5779fdec94..0952720a082 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -25,6 +25,7 @@ import { GRID_IMAGE, IMGOPS_ALIAS, IMGOPS_CONTEXT, + IMGOPS_IMAGE, IMGOPS_NGINX_CONF, IMGOPS_PORT, KAHUNA_PORT, @@ -189,7 +190,9 @@ async function globalSetup(): Promise { // the `localstack` alias on 4566, so it shares this network. Published on the fixed host // port dev-nginx maps `media-imgops` to; in CI the Caddy proxy routes to it instead. The // Dockerfile doesn't bake in nginx.conf (docker-compose bind-mounts it), so copy it in. - const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(); + const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { + deleteOnExit: false, + }); const imgops = await imgopsImage .withNetwork(network) .withNetworkAliases(IMGOPS_ALIAS) diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 0b468f3bec9..04c858250b2 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -31,6 +31,8 @@ export const IMGOPS_ALIAS = 'imgops'; export const IMGOPS_PORT = 9008; export const IMGOPS_CONTEXT = path.join(REPO_ROOT, 'dev', 'imgops'); export const IMGOPS_NGINX_CONF = path.join(IMGOPS_CONTEXT, 'nginx.conf'); +/** Stable tag for the imgops image so it persists across runs and Docker reuses cached layers. */ +export const IMGOPS_IMAGE = 'grid-e2e-imgops'; /** Network aliases the app container uses to reach the infrastructure containers. */ export const ELASTICSEARCH_ALIAS = 'elasticsearch'; From 77cce98cdd09f0733120fdc8a60eacca30e2fcef Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 15:08:52 +0000 Subject: [PATCH 298/373] Wait for all Grid services to come up before reporting the container as healthy --- e2e-tests/global-setup.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 0952720a082..aa8dea35458 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -236,10 +236,9 @@ async function globalSetup(): Promise { AWS_CBOR_DISABLE: 'true', }) .withWaitStrategy( - Wait.forAll([ - Wait.forHttp('/management/healthcheck', MEDIA_API_PORT).forStatusCode(200), - Wait.forHttp('/management/healthcheck', SERVICE_PORTS['image-loader']).forStatusCode(200), - ]), + Wait.forAll(Object.values(SERVICE_PORTS).map(port => + Wait.forHttp('/management/healthcheck', port).forStatusCode(200), + )) ) .withStartupTimeout(startupTimeoutMs); From 7de83bf1c740b7f33c453e89250846958968cf06 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 15:19:22 +0000 Subject: [PATCH 299/373] Tidy up after a manual review --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 6 ++---- e2e-tests/README.md | 2 +- e2e-tests/global-setup.ts | 2 +- e2e-tests/images/Dockerfile | 8 ++++---- e2e-tests/images/README.md | 4 ++-- e2e-tests/testcontainers/config.ts | 2 +- 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 73b63dd8dba..2b3cd2a09fa 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -226,10 +226,8 @@ object S3Ops { .credentialsProvider(config.awsCredentials) .region(config.awsRegion) - config.awsLocalPresigningEndpointUri match { - case Some(endpoint) if config.isDev => - println(s"Using localPresigningEndpointUri $endpoint") - builder.endpointOverride(endpoint) + config.awsLocalEndpointUri match { + case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build() case _ => builder.build() diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 09df0ac1206..89dcb689940 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -55,7 +55,7 @@ Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses 3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded buckets, provisioned into LocalStack; 4. per-service config, generated by `dev/script/generate-config/service-config.js`; -5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running all nine Play services. +5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running Grid's Play services. Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is exposed to the tests. `globalTeardown` stops everything and cleans up. diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index aa8dea35458..4527b514dfc 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -209,7 +209,7 @@ async function globalSetup(): Promise { const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); generateServiceConfig(configDir, coreStackProps); - // All nine Grid services run inside this single container and talk to each + // All Grid services under test run inside this single container and talk to each // other over its localhost. Each is published on the fixed host port its // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's // dev-nginx routes the https://*.media. domains straight into this diff --git a/e2e-tests/images/Dockerfile b/e2e-tests/images/Dockerfile index 430c2a911e8..579c271a5b4 100644 --- a/e2e-tests/images/Dockerfile +++ b/e2e-tests/images/Dockerfile @@ -1,14 +1,14 @@ # syntax=docker/dockerfile:1 -# Single-container image that builds and runs nine Grid Play services +# Single-container image that builds and runs all Grid Play services under test # (auth, collections, cropper, image-loader, kahuna, leases, media-api, # metadata-editor, thrall) together with Kahuna's frontend. # # This one Dockerfile produces two images, selected with `--target`: # -# * ci (grid-e2e-ci) — CI/production style. Stages pre-compiled artefacts -# and runs them under a slim JRE. -# docker build --target ci -f e2e-tests/images/Dockerfile -t grid-e2e-ci . +# * ci (grid-e2e-ci) — CI/production style. Stages pre-compiled artefacts and +# runs them under a slim JRE. docker build --target ci +# -f e2e-tests/images/Dockerfile -t grid-e2e-ci . # # * dev (grid-e2e-dev) — local development. Runs the services under # `sbt /run` (Play dev mode) with the repo diff --git a/e2e-tests/images/README.md b/e2e-tests/images/README.md index d3cec50f940..c23ff06cd75 100644 --- a/e2e-tests/images/README.md +++ b/e2e-tests/images/README.md @@ -1,7 +1,7 @@ c# Grid E2E test images -A single `Dockerfile` produces two single-container images that run nine -Grid Play services plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter: +A single `Dockerfile` produces two single-container images that run all +Grid Play services under test, plus Kahuna's frontend. You can select between CI and local-dev images using the `--target` parameter: - **CI image** (`--target ci`, tagged `grid-e2e-ci`): stages pre-compiled artefacts and runs them in a production-style JRE. Used by the e2e-tests testcontainers harness. - **Local-dev image** (`--target dev`, tagged `grid-e2e-dev`): runs the services under `sbt /run` (Play dev mode) with the repo bind-mounted, so source changes recompile live. See [Development image (live reload)](#development-image-live-reload) below. diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 24ddf255799..8021abf5527 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -47,7 +47,6 @@ function rewriteEndpoints(conf: string): string { // presigned URLs are handed to the browser, which can only reach LocalStack via the // `localstack.media.` vanity domain (dev-nginx locally, the Caddy proxy in CI). // Sign against that host so the URLs resolve outside the container network. - console.log(`${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`); return `${rewrittenConf}\naws.local.presigningEndpoint="${guardianLocalstackUrl}"\n`; } @@ -97,6 +96,7 @@ export function generateServiceConfig(configDir: string, coreStackProps: StackPr if (!conf) { throw new Error(`service-config.js did not produce config for '${service}'`); } + fs.writeFileSync(path.join(configDir, `${service}.conf`), rewriteEndpoints(conf)); } } From a852d148d0a2ed51b0690c2a5caac0bf7a3c2034 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 15:46:33 +0000 Subject: [PATCH 300/373] Correct presigning config --- .../src/main/scala/com/gu/mediaservice/lib/aws/S3.scala | 2 +- .../scala/com/gu/mediaservice/lib/config/CommonConfig.scala | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala index 2b3cd2a09fa..ebfd333496e 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/aws/S3.scala @@ -226,7 +226,7 @@ object S3Ops { .credentialsProvider(config.awsCredentials) .region(config.awsRegion) - config.awsLocalEndpointUri match { + config.awsLocalPresigningEndpointUri match { case Some(endpoint) if config.isDev => builder.endpointOverride(endpoint) .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build()).build() case _ => builder.build() diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala index c155be5f100..669e49490c8 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/CommonConfig.scala @@ -33,9 +33,6 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui // Endpoint baked into presigned URLs handed to the browser. Distinct from `aws.local.endpoint` // (used by the S3 client for container-internal calls) because the browser cannot resolve the // container-network host. Falls back to `aws.local.endpoint` when unset. - println("PRESIGN ENDPOINTS") - println(stringOpt("aws.local.presigningEndpoint")) - println(awsLocalEndpoint) val awsLocalPresigningEndpointUri: Option[URI] = if (isDev) stringOpt("aws.local.presigningEndpoint") .filter(_.nonEmpty) @@ -43,8 +40,6 @@ abstract class CommonConfig(resources: GridConfigResources) extends AwsClientBui .map(new URI(_)) else None - println(awsLocalPresigningEndpointUri) - val useLocalAuth: Boolean = isDev && boolean("auth.useLocal") val localLogShipping: Boolean = sys.env.getOrElse("LOCAL_LOG_SHIPPING", "false").toBoolean From ff462e0bbab9138dd7da1e2736bd2b87c787e4ca Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Wed, 2 Sep 2026 16:10:20 +0000 Subject: [PATCH 301/373] Remove unnecessary caddy entry, the localstack entry already exists --- e2e-tests/global-setup.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 4527b514dfc..bca9bd18167 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -6,7 +6,7 @@ * 2. Elasticsearch + LocalStack + imgops (infrastructure), * 3. the CloudFormation core stack + seeded buckets (provisioning), * 4. generated per-service config (reusing dev/script/generate-config), - * 5. the pre-built `grid-e2e-ci` image running all nine services. + * 5. the pre-built `grid-e2e-ci` image running the Grid services under test. * * The Kahuna base URL is exposed to tests via `GRID_BASE_URL`, and the started * containers are stashed for `global-teardown.ts`. @@ -113,12 +113,9 @@ function buildCaddyfile(coreStackProps: Record): string { // S3 vanity domains that omit the bucket -> localstack, with the bucket prepended. const imageBuckets: Record = { [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket, - [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket, - [`localstack.media.${DOMAIN}`]: coreStackProps.IngestQueueBucket + [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket }; - console.log({coreStackProps}) - const blocks: string[] = []; for (const [siteHost, port] of Object.entries(appServices)) { @@ -187,9 +184,7 @@ async function globalSetup(): Promise { started.push(localstack); // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to - // the `localstack` alias on 4566, so it shares this network. Published on the fixed host - // port dev-nginx maps `media-imgops` to; in CI the Caddy proxy routes to it instead. The - // Dockerfile doesn't bake in nginx.conf (docker-compose bind-mounts it), so copy it in. + // the `localstack` alias on 4566, so it shares this network. const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { deleteOnExit: false, }); From 5791d5e2d762bb4bb8b26f25410139a7504c7708 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 3 Sep 2026 13:51:24 +0100 Subject: [PATCH 302/373] Add loader-projection service to CI Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e-tests/global-setup.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index bca9bd18167..70acafad492 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -102,6 +102,7 @@ function buildCaddyfile(coreStackProps: Record): string { [`media.${DOMAIN}`]: SERVICE_PORTS.kahuna, [`api.media.${DOMAIN}`]: SERVICE_PORTS['media-api'], [`loader.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], + [`loader-projection.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], [`cropper.media.${DOMAIN}`]: SERVICE_PORTS.cropper, [`thrall.media.${DOMAIN}`]: SERVICE_PORTS.thrall, [`media-metadata.${DOMAIN}`]: SERVICE_PORTS['metadata-editor'], From e7ac0710fa7548098e0b46a9c8b842d769ec0eb9 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 3 Sep 2026 12:50:39 +0000 Subject: [PATCH 303/373] Add 9008 for image-serving vanity urls --- .devcontainer/devenv.yaml | 1 + .devcontainer/shared/devcontainer.json | 1 + .devcontainer/user/devcontainer.json | 1 + 3 files changed, 3 insertions(+) diff --git a/.devcontainer/devenv.yaml b/.devcontainer/devenv.yaml index f11896105a2..44306cce15c 100644 --- a/.devcontainer/devenv.yaml +++ b/.devcontainer/devenv.yaml @@ -23,6 +23,7 @@ forwardPorts: - 9005 - 9006 - 9007 + - 9008 - 9009 - 9010 - 9011 diff --git a/.devcontainer/shared/devcontainer.json b/.devcontainer/shared/devcontainer.json index 8796a02a51c..1bc566af490 100644 --- a/.devcontainer/shared/devcontainer.json +++ b/.devcontainer/shared/devcontainer.json @@ -25,6 +25,7 @@ 9005, 9006, 9007, + 9008, 9009, 9010, 9011, diff --git a/.devcontainer/user/devcontainer.json b/.devcontainer/user/devcontainer.json index 3abef9f164f..3c366bf3cdc 100644 --- a/.devcontainer/user/devcontainer.json +++ b/.devcontainer/user/devcontainer.json @@ -25,6 +25,7 @@ 9005, 9006, 9007, + 9008, 9009, 9010, 9011, From efbdaa91a34897f1c5e045f48f1f75e97c3a1f41 Mon Sep 17 00:00:00 2001 From: Mateusz Date: Fri, 4 Sep 2026 17:25:21 +0100 Subject: [PATCH 304/373] Update PhotographerRenamer.scala --- .../gu/mediaservice/lib/cleanup/PhotographerRenamer.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala index c55b9cf1b1f..a6bbcc588e1 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/cleanup/PhotographerRenamer.scala @@ -69,10 +69,12 @@ object PhotographerRenamer extends MetadataCleaner { "Aurelien Laudy" -> "Aurélien Laudy", "Aurelien Meunier" -> "Aurélien Meunier", "Aurelien Morissard" -> "Aurélien Morissard", + "Aurore Marechal" -> "Aurore Maréchal", "Aykut Unlupinar" -> "Aykut Ünlüpınar", "Aytac Unal" -> "Aytaç Ünal", "Azael Rodriguez" -> "Azael Rodríguez", "Balazs Mohai" -> "Balázs Mohai", + "Balint Szentgallay" -> "Bálint Szentgallay", "Bartlomiej Wojtowicz" -> "Bartłomiej Wojtowicz", "Bartlomiej Zborowski" -> "Bartłomiej Zborowski", "Bartosz Banka" -> "Bartosz Bańka", @@ -131,6 +133,7 @@ object PhotographerRenamer extends MetadataCleaner { "Daniel Garcia" -> "Daniel García", "Daniel Mihailescu" -> "Daniel Mihăilescu", "Daniel Muñoz" -> "Daniel Muñoz", + "Dante Fernandez" -> "Dante Fernández", "Darko Bandic" -> "Darko Bandić", "Darko Vojinovic" -> "Darko Vojinović", "Dave M Benett" -> "David M Benett", @@ -152,6 +155,7 @@ object PhotographerRenamer extends MetadataCleaner { "Eduardo Munoz" -> "Eduardo Muñoz", "Eduardo Munoz Alvarez" -> "Eduardo Muñoz", "Edwin Bercian" -> "Edwin Bercián", + "Efekan Akyuz" -> "Efekan Akyüz", "Egle Kazdailyte" -> "Eglė Každailytė", "Elif Ozturk" -> "Elif Öztürk", "Elin Hoyland" -> "Elin Høyland", @@ -399,6 +403,7 @@ object PhotographerRenamer extends MetadataCleaner { "Mario Arturo Martinez" -> "Mario Arturo Martínez", "Mario Cruz" -> "Mário Cruz", "Mario Vazquez" -> "Mario Vázquez", + "Marko Djokovic" -> "Marko Đoković", "Marko Djurica" -> "Marko Đurica", "Marko Drobnjakovic" -> "Marko Drobnjaković", "Martin Mejia" -> "Martín Mejía", @@ -426,6 +431,7 @@ object PhotographerRenamer extends MetadataCleaner { "Mitar Mitrovic" -> "Mitar Mitrović", "Moises Castillo" -> "Moisés Castillo", "Morne de Klerk" -> "Morné de Klerk", + "Muhammed Enes Yildirim" -> "Muhammed Enes Yıldırım", "Murat Ozgur Guvendik" -> "Murat Özgür Güvendik", "Mustafa Ciftci" -> "Mustafa Çiftçi", "Mustafa Ozturk" -> "Mustafa Öztürk", @@ -436,6 +442,7 @@ object PhotographerRenamer extends MetadataCleaner { "Niklas Halle’N" -> "Niklas Halle’n", "Nuno Andre Ferreira" -> "Nuno André Ferreira", "Octavio Passos" -> "Octávio Passos", + "Oisin Keniry" -> "Oisín Keniry", "Omer Urer" -> "Ömer Ürer", "Onur Coban" -> "Onur Çoban", "Orhan Seref Akkanat" -> "Orhan Şeref Akkanat", @@ -626,6 +633,7 @@ object PhotographerRenamer extends MetadataCleaner { "Zorana Jevtic" -> "Zorana Jevtić", "Zsolt Czegledi" -> "Zsolt Czeglédi", "Zsolt Szigetvary" -> "Zsolt Szigetváry", + "Zuzana Gogova" -> "Zuzana Gogová", "Zvonimir Barisin" -> "Zvonimir Barišin" ) From 7e59c9bba8269ac2a674f780852781d851da257e Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 3 Sep 2026 14:40:03 +0000 Subject: [PATCH 305/373] Migrate to ESM --- e2e-tests/global-setup.ts | 10 +++++----- e2e-tests/global-teardown.ts | 4 ++-- e2e-tests/package.json | 1 + e2e-tests/playwright.config.ts | 2 +- e2e-tests/steps/fixtures.ts | 2 +- e2e-tests/steps/kahuna.steps.ts | 2 +- e2e-tests/testcontainers/config.ts | 7 +++++-- e2e-tests/testcontainers/constants.ts | 4 ++-- e2e-tests/testcontainers/provision.ts | 2 +- e2e-tests/testcontainers/seed-elasticsearch.ts | 2 +- 10 files changed, 20 insertions(+), 16 deletions(-) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 395c8a98ee4..f093b6c49b2 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -37,11 +37,11 @@ import { REPO_ROOT, SERVICE_PORTS, URLS_FILE, -} from './testcontainers/constants'; -import { generateServiceConfig } from './testcontainers/config'; -import { provisionCoreStack } from './testcontainers/provision'; -import { seedElasticsearch } from './testcontainers/seed-elasticsearch'; -import { setEnvironment } from './testcontainers/state'; +} from './testcontainers/constants.ts'; +import { generateServiceConfig } from './testcontainers/config.ts'; +import { provisionCoreStack } from './testcontainers/provision.ts'; +import { seedElasticsearch } from './testcontainers/seed-elasticsearch.ts'; +import { setEnvironment } from './testcontainers/state.ts'; const LOCALSTACK_SERVICES = [ "cloudformation", diff --git a/e2e-tests/global-teardown.ts b/e2e-tests/global-teardown.ts index bbb6495382d..61c68a11b70 100644 --- a/e2e-tests/global-teardown.ts +++ b/e2e-tests/global-teardown.ts @@ -3,8 +3,8 @@ * (in reverse order), removes the shared network, and deletes the generated config. */ import * as fs from 'fs'; -import { URLS_FILE } from './testcontainers/constants'; -import { getEnvironment } from './testcontainers/state'; +import { URLS_FILE } from './testcontainers/constants.ts'; +import { getEnvironment } from './testcontainers/state.ts'; /** Run a best-effort teardown step, warning (not throwing) so the rest still runs. */ async function warnOnException(label: string, fn: () => unknown): Promise { diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 33f1ab316c2..eeaf5360083 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -3,6 +3,7 @@ "version": "1.0.0", "description": "", "main": "index.js", + "type": "module", "scripts": { "test": "bddgen && playwright test", "test:headed": "bddgen && playwright test --headed", diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index 0f53677ba8d..fc79c9bdda3 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -1,6 +1,6 @@ import { defineConfig, devices } from '@playwright/test'; import { defineBddConfig } from 'playwright-bdd'; -import { KAHUNA_PORT } from './testcontainers/constants'; +import { KAHUNA_PORT } from './testcontainers/constants.ts'; /* Generate Playwright test files from the Gherkin feature files and step definitions. Run via `bddgen` (see package.json scripts) before `playwright test`. */ diff --git a/e2e-tests/steps/fixtures.ts b/e2e-tests/steps/fixtures.ts index 1714ccf884b..58cec7d1eef 100644 --- a/e2e-tests/steps/fixtures.ts +++ b/e2e-tests/steps/fixtures.ts @@ -2,7 +2,7 @@ import { expect } from '@playwright/test'; import type { APIResponse, Response } from '@playwright/test'; import { test as base, createBdd } from 'playwright-bdd'; import * as fs from 'fs'; -import { DOMAIN, URLS_FILE } from '../testcontainers/constants'; +import { DOMAIN, URLS_FILE } from '../testcontainers/constants.ts'; interface GridUrls { /** Kahuna base URL on its fixed host port (for API-level `request` tests). */ diff --git a/e2e-tests/steps/kahuna.steps.ts b/e2e-tests/steps/kahuna.steps.ts index c81f964c8e9..4833a4a23cf 100644 --- a/e2e-tests/steps/kahuna.steps.ts +++ b/e2e-tests/steps/kahuna.steps.ts @@ -1,4 +1,4 @@ -import { When, Then, expect } from './fixtures'; +import { When, Then, expect } from './fixtures.ts'; When('I request the Kahuna healthcheck endpoint', async ({ request, testContext }) => { testContext.response = await request.get('/management/healthcheck'); diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 8021abf5527..20e06752fb8 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -6,6 +6,7 @@ */ import * as fs from 'fs'; import * as path from 'path'; +import { createRequire } from 'module'; import JSON5 from 'json5'; import { DOMAIN, @@ -16,7 +17,10 @@ import { REGION, REPO_ROOT, SERVICE_PORTS, -} from './constants'; +} from './constants.ts'; + +/** `service-config.js` is CommonJS and lives outside this package, so load it via require. */ +const require = createRequire(import.meta.url); const GENERATE_CONFIG_DIR = path.join(REPO_ROOT, 'dev', 'script', 'generate-config'); @@ -54,7 +58,6 @@ function rewriteEndpoints(conf: string): string { * Generate all service config files into `configDir`. */ export function generateServiceConfig(configDir: string, coreStackProps: StackProps): void { - // eslint-disable-next-line @typescript-eslint/no-var-requires const ServiceConfig = require(path.join(GENERATE_CONFIG_DIR, 'service-config.js')); const defaultConfig = JSON5.parse( fs.readFileSync(path.join(GENERATE_CONFIG_DIR, 'config.json5'), 'utf8'), diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 04c858250b2..2a9bbc2dde8 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -1,6 +1,6 @@ import * as path from 'path'; -export const REPO_ROOT = path.resolve(__dirname, '..', '..'); +export const REPO_ROOT = path.resolve(import.meta.dirname, '..', '..'); export const DOMAIN = 'local.dev-gutools.co.uk'; export const EMAIL_DOMAIN = 'guardian.co.uk'; @@ -58,7 +58,7 @@ export const KAHUNA_PORT = SERVICE_PORTS.kahuna; export const MEDIA_API_PORT = SERVICE_PORTS['media-api']; /** File (repo-relative to e2e-tests) where global-setup records the resolved service URLs. */ -export const URLS_FILE = path.join(__dirname, '..', '.grid-urls.json'); +export const URLS_FILE = path.join(import.meta.dirname, '..', '.grid-urls.json'); /** The API key value uploaded to the KeyBucket (dev/.env API_KEY). */ export const API_KEY = 'e2e-dev'; diff --git a/e2e-tests/testcontainers/provision.ts b/e2e-tests/testcontainers/provision.ts index 07820f1479d..0c5828d9378 100644 --- a/e2e-tests/testcontainers/provision.ts +++ b/e2e-tests/testcontainers/provision.ts @@ -7,7 +7,7 @@ import { waitUntilStackCreateComplete, } from '@aws-sdk/client-cloudformation'; import { CreateBucketCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; -import { API_KEY as API_KEY_PATH, CORE_STACK_NAME, PERMISSIONS_BUCKET, REGION, REPO_ROOT } from './constants'; +import { API_KEY as API_KEY_PATH, CORE_STACK_NAME, PERMISSIONS_BUCKET, REGION, REPO_ROOT } from './constants.ts'; const CREDENTIALS = { accessKeyId: 'test', secretAccessKey: 'test' }; diff --git a/e2e-tests/testcontainers/seed-elasticsearch.ts b/e2e-tests/testcontainers/seed-elasticsearch.ts index 7bf02112546..7b6c4ed4feb 100644 --- a/e2e-tests/testcontainers/seed-elasticsearch.ts +++ b/e2e-tests/testcontainers/seed-elasticsearch.ts @@ -12,7 +12,7 @@ */ import * as fs from 'fs'; import * as path from 'path'; -import { REPO_ROOT } from './constants'; +import { REPO_ROOT } from './constants.ts'; /** Alias the Grid app assigns to the live images index (dev service-config `es.index.aliases.current`). */ const IMAGES_ALIAS = 'Images_Current'; From 27c63d0de84298f454f8e3c61f4b10b6bbbb0b3c Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 3 Sep 2026 15:15:59 +0000 Subject: [PATCH 306/373] Add dev:e2e command, which spins up the test environment without running tests and delegate to a common script across dev and test commands --- e2e-tests/README.md | 25 ++- e2e-tests/dev.ts | 86 ++++++++ e2e-tests/global-setup.ts | 285 +------------------------ e2e-tests/global-teardown.ts | 32 +-- e2e-tests/package.json | 4 + e2e-tests/testcontainers/stack.ts | 344 ++++++++++++++++++++++++++++++ e2e-tests/testcontainers/state.ts | 2 + 7 files changed, 468 insertions(+), 310 deletions(-) create mode 100644 e2e-tests/dev.ts create mode 100644 e2e-tests/testcontainers/stack.ts diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 89dcb689940..2fd84488f9e 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -45,9 +45,29 @@ npm run test:ui # open browser and test suite, run tests at your leisure Traces are captured `on-first-retry` (see [`playwright.config.ts`](playwright.config.ts)), so a failed test on CI leaves a trace you can open with `npx playwright show-trace`. +## Running the stack without the tests + +`npm run dev` boots exactly the same stack the tests use, prints the service URLs and +holds it open until you press Ctrl-C, which tears it all down. Useful for poking at Grid +by hand, or for leaving the stack up while you iterate on step definitions. + +```bash +npm run dev # uses your local dev-nginx for the https://*.media. domains +npm run dev:proxy # no dev-nginx? start the bundled Caddy proxy on :443 instead +``` + +It uses the same image as the tests (`grid-e2e-dev` locally, `grid-e2e-ci` under `CI`), so +build that first. The first boot is slow because the dev image compiles from source; raise +`GRID_STARTUP_TIMEOUT_MS` (default `300000`) if it times out, and set `GRID_DEBUG=1` to +tee the container's boot logs to `$TMPDIR/grid-boot.log`. + +The stack binds fixed host ports (9001-9012, 4566, 9008), so `npm run dev` and `npm test` +cannot run at the same time. + ## How they work -Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses +Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) calls `startStack` +([`testcontainers/stack.ts`](testcontainers/stack.ts)), which uses [Testcontainers](https://testcontainers.com/) to stand up everything the tests need: 1. a shared Docker network; @@ -58,4 +78,5 @@ Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) uses 5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running Grid's Play services. Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is -exposed to the tests. `globalTeardown` stops everything and cleans up. +exposed to the tests. `globalTeardown` calls `stopStack`, which stops everything and +cleans up. `npm run dev` drives the same two functions. diff --git a/e2e-tests/dev.ts b/e2e-tests/dev.ts new file mode 100644 index 00000000000..0cabeaaf9b9 --- /dev/null +++ b/e2e-tests/dev.ts @@ -0,0 +1,86 @@ +/** + * Boots the Grid stack outside Playwright and holds it open until Ctrl-C. + * + * Run with `npm run dev`. Set `GRID_PROXY=true` to also start the bundled Caddy + * reverse proxy, which serves the https://*.media. domains for anyone + * without dev-nginx running locally. + */ +import { + DOMAIN, + ELASTICSEARCH_ALIAS, + GRID_IMAGE, + LOCALSTACK_PORT, + SERVICE_PORTS, + URLS_FILE, +} from './testcontainers/constants.ts'; +import { startStack, stopStack } from './testcontainers/stack.ts'; +import type { GridEnvironment } from './testcontainers/state.ts'; + +function banner(environment: GridEnvironment): string { + const services = Object.keys(SERVICE_PORTS) + .sort() + .map((service) => ` ${service.padEnd(16)} http://localhost:${SERVICE_PORTS[service]}`) + .join('\n'); + + return [ + '', + ' Grid is up.', + '', + ` kahuna (UI) ${environment.baseUrl}`, + ` media-api ${environment.mediaApiUrl}`, + '', + ' Services:', + services, + '', + ' Infrastructure:', + ` localstack http://localhost:${LOCALSTACK_PORT}`, + ` elasticsearch alias '${ELASTICSEARCH_ALIAS}' on the stack network`, + '', + ` Image: ${GRID_IMAGE}`, + ` Config: ${environment.configDir}`, + ` URLs: ${URLS_FILE}`, + ` Domains: https://media.${DOMAIN} (via dev-nginx or GRID_PROXY=true)`, + '', + ' Press Ctrl-C to tear everything down.', + '', + ].join('\n'); +} + +async function dev(): Promise { + let environment: GridEnvironment | undefined; + let shuttingDown = false; + + const shutdown = async (signal: NodeJS.Signals): Promise => { + // A second Ctrl-C exits immediately; Testcontainers' Ryuk reaper cleans up the rest. + if (shuttingDown) { + console.log('\nForcing exit.'); + process.exit(130); + } + shuttingDown = true; + + if (!environment) { + // Interrupted mid-boot: startStack owns the containers, so leave them to Ryuk. + console.log(`\nReceived ${signal} during startup; leaving cleanup to the Ryuk reaper.`); + process.exit(130); + } + + console.log(`\nReceived ${signal}, stopping the Grid stack...`); + await stopStack(environment); + console.log('Done.'); + process.exit(0); + }; + + process.on('SIGINT', shutdown); + process.on('SIGTERM', shutdown); + + environment = await startStack({ proxy: process.env.GRID_PROXY === 'true' }); + console.log(banner(environment)); + + // Hold the process open; the signal handlers are the only way out. + await new Promise(() => {}); +} + +dev().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index f093b6c49b2..778db410625 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -1,287 +1,14 @@ /** - * Playwright global setup. - * - * Boots the full local Grid stack with Testcontainers: - * 1. a shared network, - * 2. Elasticsearch + LocalStack + imgops (infrastructure), - * 3. the CloudFormation core stack + seeded buckets (provisioning), - * 4. generated per-service config (reusing dev/script/generate-config), - * 5. the pre-built `grid-e2e-ci` image running the Grid services under test. - * - * The Kahuna base URL is exposed to tests via `GRID_BASE_URL`, and the started - * containers are stashed for `global-teardown.ts`. + * Playwright global setup. Boots the Grid stack (see `testcontainers/stack.ts`) and + * stashes it for `global-teardown.ts`, exposing the Kahuna base URL via `GRID_BASE_URL`. */ -import * as fs from 'fs'; -import * as os from 'os'; -import * as path from 'path'; -import { LocalstackContainer } from '@testcontainers/localstack'; -import { GenericContainer, Network, Wait } from 'testcontainers'; -import { - DOMAIN, - ELASTICSEARCH_ALIAS, - ELASTICSEARCH_IMAGE, - GRID_ALIAS, - GRID_IMAGE, - IMGOPS_ALIAS, - IMGOPS_CONTEXT, - IMGOPS_IMAGE, - IMGOPS_NGINX_CONF, - IMGOPS_PORT, - KAHUNA_PORT, - LOCALSTACK_ALIAS, - LOCALSTACK_IMAGE, - LOCALSTACK_PORT, - MEDIA_API_PORT, - PROXY_IMAGE, - REGION, - REPO_ROOT, - SERVICE_PORTS, - URLS_FILE, -} from './testcontainers/constants.ts'; -import { generateServiceConfig } from './testcontainers/config.ts'; -import { provisionCoreStack } from './testcontainers/provision.ts'; -import { seedElasticsearch } from './testcontainers/seed-elasticsearch.ts'; +import { startStack } from './testcontainers/stack.ts'; import { setEnvironment } from './testcontainers/state.ts'; -const LOCALSTACK_SERVICES = [ - "cloudformation", - "cloudwatch", - "dynamodb", - "kinesis", - "s3", - "sns", - "sqs", - "iam", -].join(","); - -/** - * Build a Caddyfile that reproduces the dev-nginx subdomain routing: each Grid service - * domain (https://.media.) reverse-proxies to the grid-e2e-ci container on - * its in-container port, and the S3 vanity domains proxy to localstack (prepending the - * real bucket to the path). `tls internal` serves a self-signed cert per site; Playwright - * runs with `ignoreHTTPSErrors`, so the internal CA does not need to be trusted. - * - * Example output: - * - * { - * auto_https disable_redirects - * } - * - * media.local.dev-gutools.co.uk { - * tls internal - * reverse_proxy localhost:9005 - * } - * - * api.media.local.dev-gutools.co.uk { - * tls internal - * reverse_proxy localhost:9001 - * } - * - * # ... etc for other similar services - * - * images.media.local.dev-gutools.co.uk { - * tls internal - * rewrite * /grid-dev-core-imagebucket-2b34bef1{uri} - * reverse_proxy localhost:4566 - * } - * - * public.media.local.dev-gutools.co.uk { - * tls internal - * rewrite * /grid-dev-core-imageoriginbucket-e8cd1fec{uri} - * reverse_proxy localhost:4566 - * } - * - * localstack.media.local.dev-gutools.co.uk { - * tls internal - * reverse_proxy localhost:4566 - * } - */ -function buildCaddyfile(coreStackProps: Record): string { - const appServices: Record = { - [`media.${DOMAIN}`]: SERVICE_PORTS.kahuna, - [`api.media.${DOMAIN}`]: SERVICE_PORTS['media-api'], - [`loader.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], - [`loader-projection.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], - [`cropper.media.${DOMAIN}`]: SERVICE_PORTS.cropper, - [`thrall.media.${DOMAIN}`]: SERVICE_PORTS.thrall, - [`media-metadata.${DOMAIN}`]: SERVICE_PORTS['metadata-editor'], - [`media-collections.${DOMAIN}`]: SERVICE_PORTS.collections, - [`media-leases.${DOMAIN}`]: SERVICE_PORTS.leases, - [`media-auth.${DOMAIN}`]: SERVICE_PORTS.auth, - }; - - // S3 vanity domains that omit the bucket -> localstack, with the bucket prepended. - const imageBuckets: Record = { - [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket, - [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket - }; - - const blocks: string[] = []; - - for (const [siteHost, port] of Object.entries(appServices)) { - blocks.push(`${siteHost} {\n\ttls internal\n\treverse_proxy ${GRID_ALIAS}:${port}\n}`); - } - - for (const [siteHost, bucket] of Object.entries(imageBuckets)) { - blocks.push( - `${siteHost} {\n\ttls internal\n\trewrite * /${bucket}{uri}\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, - ); - } - - // Thumbnails / direct S3 access already include the bucket in the path. - blocks.push( - `localstack.media.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, - ); - - // On-the-fly image resizing (optimised / full-screen views) -> the imgops container. - blocks.push( - `media-imgops.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${IMGOPS_ALIAS}:80\n}`, - ); - - return `${blocks.join('\n\n')}\n`; -} - async function globalSetup(): Promise { - const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); - - const network = await new Network().start(); - - // Infrastructure: Elasticsearch + LocalStack - const eventuallyElasticSearch = new GenericContainer(ELASTICSEARCH_IMAGE) - .withNetwork(network) - .withNetworkAliases(ELASTICSEARCH_ALIAS) - .withEnvironment({ - 'discovery.type': 'single-node', - 'xpack.security.enabled': 'false', - ES_JAVA_OPTS: '-Xms1024m -Xmx1024m', - }) - .withExposedPorts(9200) - .withWaitStrategy( - Wait.forHttp('/_cluster/health', 9200).forStatusCodeMatching((code) => code < 300), - ) - .withStartupTimeout(180_000) - .start(); - - const eventuallyLocalstack = new LocalstackContainer(LOCALSTACK_IMAGE) - .withNetwork(network) - .withNetworkAliases(LOCALSTACK_ALIAS) - // Pin to the fixed host port dev-nginx expects for the S3 vanity domains - // (images.media / public.media / localstack.media -> 4566). - .withExposedPorts({ container: LOCALSTACK_PORT, host: LOCALSTACK_PORT }) - .withEnvironment({ - SERVICES: LOCALSTACK_SERVICES, - DEFAULT_REGION: REGION, - KINESIS_ERROR_PROBABILITY: '0.0', - // Make resource URLs resolve via the network alias so the - // app container can reach them, and keep queue URLs path-style. - LOCALSTACK_HOST: `${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`, - SQS_ENDPOINT_STRATEGY: 'path', - }) - .withStartupTimeout(120_000) - .start(); - - // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to - // the `localstack` alias on 4566, so it shares this network. - const imgopsImage = GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { - deleteOnExit: false, - }); - const eventuallyImgOps = imgopsImage.then(image => image - .withNetwork(network) - .withNetworkAliases(IMGOPS_ALIAS) - .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) - .withExposedPorts({ container: 80, host: IMGOPS_PORT }) - .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) - .withStartupTimeout(120_000) - .start() - ) - - const [imgops, elasticsearch, localstack] = await Promise.all([eventuallyImgOps, eventuallyElasticSearch, eventuallyLocalstack]); - - // Provisioning + config generation - const coreStackProps = await provisionCoreStack((await eventuallyLocalstack).getConnectionUri()); - - const configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); - generateServiceConfig(configDir, coreStackProps); - - // CI routing: bundled reverse proxy - // - // Locally, the browser reaches the https://*.media. domains via the developer's - // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) - // start a Caddy proxy that replays the same subdomain routing and terminates TLS with a - // self-signed cert, published on the standard https port the domains resolve to. - - const maybeEventuallyProxy = !process.env.CI ? Promise.resolve() : new GenericContainer(PROXY_IMAGE) - .withNetwork(network) - .withExposedPorts({ container: 443, host: 443 }) - .withCopyContentToContainer([ - { content: buildCaddyfile(coreStackProps), target: '/etc/caddy/Caddyfile' }, - ]) - .withWaitStrategy(Wait.forListeningPorts()) - .withStartupTimeout(60_000) - .start(); - - - // All Grid services under test run inside this single container and talk to each - // other over its localhost. Each is published on the fixed host port its - // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's - // dev-nginx routes the https://*.media. domains straight into this - // container. - let gridContainerDefinition = new GenericContainer(GRID_IMAGE) - .withNetwork(network) - .withNetworkAliases(GRID_ALIAS) - .withExposedPorts( - ...Object.values(SERVICE_PORTS).map((port) => ({ container: port, host: port })), - ) - .withBindMounts([ - // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. - { source: configDir, target: '/root/.grid', mode: 'ro' }, - { source: configDir, target: '/etc/grid', mode: 'ro' }, - // Outside CI the grid-e2e-dev image runs services under sbt; mount the repo - // over /build so host edits recompile live. - ...(process.env.CI ? [] : [{ source: REPO_ROOT, target: '/build', mode: 'rw' as const }]), - ]) - .withEnvironment({ - AWS_ACCESS_KEY_ID: 'test', - AWS_SECRET_ACCESS_KEY: 'test', - AWS_REGION: REGION, - AWS_DEFAULT_REGION: REGION, - AWS_CBOR_DISABLE: 'true', - }) - .withWaitStrategy( - Wait.forAll(Object.values(SERVICE_PORTS).map(port => - Wait.forHttp('/management/healthcheck', port).forStatusCode(200), - )) - ) - .withStartupTimeout(startupTimeoutMs); - - if (process.env.GRID_DEBUG) { - const logStream = fs.createWriteStream(path.join(os.tmpdir(), 'grid-boot.log')); - gridContainerDefinition = gridContainerDefinition.withLogConsumer((stream) => { - stream.on('data', (line) => logStream.write(line)); - stream.on('err', (line) => logStream.write(line)); - }); - } - - const eventuallyGrid = gridContainerDefinition.start(); - - const [grid, proxy] = await Promise.all([eventuallyGrid, maybeEventuallyProxy]); - - const startedESContainer = await eventuallyElasticSearch; - // Seed Elasticsearch with image fixtures - // - // The app creates the `images` index + `Images_Current` alias on startup; seed once the - // stack is healthy so searches during the tests return the fixture documents. - const esBaseUrl = `http://${startedESContainer.getHost()}:${startedESContainer.getMappedPort(9200)}`; - await seedElasticsearch(esBaseUrl); - - const host = grid.getHost(); - const baseUrl = `http://${host}:${grid.getMappedPort(KAHUNA_PORT)}`; - const mediaApiUrl = `http://${host}:${grid.getMappedPort(MEDIA_API_PORT)}`; - - process.env.GRID_BASE_URL = baseUrl; - fs.writeFileSync(URLS_FILE, JSON.stringify({ kahuna: baseUrl, mediaApi: mediaApiUrl })); - - setEnvironment({ network, containers: [elasticsearch, localstack, imgops, grid, ...(proxy ? [proxy] : [])], configDir, baseUrl }); + const environment = await startStack(); + process.env.GRID_BASE_URL = environment.baseUrl; + setEnvironment(environment); } export default globalSetup; diff --git a/e2e-tests/global-teardown.ts b/e2e-tests/global-teardown.ts index 61c68a11b70..10dc72408dd 100644 --- a/e2e-tests/global-teardown.ts +++ b/e2e-tests/global-teardown.ts @@ -1,37 +1,11 @@ /** - * Playwright global teardown. Stops the containers started in `global-setup.ts` - * (in reverse order), removes the shared network, and deletes the generated config. + * Playwright global teardown. Stops the stack started in `global-setup.ts`. */ -import * as fs from 'fs'; -import { URLS_FILE } from './testcontainers/constants.ts'; +import { stopStack } from './testcontainers/stack.ts'; import { getEnvironment } from './testcontainers/state.ts'; -/** Run a best-effort teardown step, warning (not throwing) so the rest still runs. */ -async function warnOnException(label: string, fn: () => unknown): Promise { - try { - await fn(); - } catch (error) { - console.warn(`${label}: ${(error as Error).message}`); - } -} - async function globalTeardown(): Promise { - const environment = getEnvironment(); - if (!environment) { - return; - } - - const { containers, network, configDir } = environment; - - for (const container of [...containers].reverse()) { - await warnOnException('Failed to stop container', () => container.stop()); - } - - await warnOnException('Failed to stop network', () => network.stop()); - await warnOnException('Failed to remove config dir', () => - fs.rmSync(configDir, { recursive: true, force: true }), - ); - await warnOnException('Failed to remove urls file', () => fs.rmSync(URLS_FILE, { force: true })); + await stopStack(getEnvironment()); } export default globalTeardown; diff --git a/e2e-tests/package.json b/e2e-tests/package.json index eeaf5360083..0a1ca0b37c5 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -5,12 +5,16 @@ "main": "index.js", "type": "module", "scripts": { + "dev:e2e": "node --experimental-strip-types --disable-warning=ExperimentalWarning dev.ts", "test": "bddgen && playwright test", "test:headed": "bddgen && playwright test --headed", "test:report": "playwright show-report", "test:ui": "bddgen && playwright test --ui-port 6080", "test:log": "bddgen && DEBUG=testcontainers* playwright test --ui-port 6080" }, + "engines": { + "node": ">=22.12" + }, "keywords": [], "author": "", "license": "ISC", diff --git a/e2e-tests/testcontainers/stack.ts b/e2e-tests/testcontainers/stack.ts new file mode 100644 index 00000000000..8f868c608cc --- /dev/null +++ b/e2e-tests/testcontainers/stack.ts @@ -0,0 +1,344 @@ +/** + * Boots and tears down the full local Grid stack with Testcontainers: + * 1. a shared network, + * 2. Elasticsearch + LocalStack + imgops (infrastructure), + * 3. the CloudFormation core stack + seeded buckets (provisioning), + * 4. generated per-service config (reusing dev/script/generate-config), + * 5. the pre-built `grid-e2e-ci` / `grid-e2e-dev` image running the Grid services. + * + * Used by Playwright's global setup/teardown and by `dev.ts`, which runs the same + * stack interactively outside the test runner. + */ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { LocalstackContainer } from '@testcontainers/localstack'; +import { GenericContainer, Network, Wait } from 'testcontainers'; +import type { StartedNetwork, StartedTestContainer } from 'testcontainers'; +import { + DOMAIN, + ELASTICSEARCH_ALIAS, + ELASTICSEARCH_IMAGE, + GRID_ALIAS, + GRID_IMAGE, + IMGOPS_ALIAS, + IMGOPS_CONTEXT, + IMGOPS_IMAGE, + IMGOPS_NGINX_CONF, + IMGOPS_PORT, + KAHUNA_PORT, + LOCALSTACK_ALIAS, + LOCALSTACK_IMAGE, + LOCALSTACK_PORT, + MEDIA_API_PORT, + PROXY_IMAGE, + REGION, + REPO_ROOT, + SERVICE_PORTS, + URLS_FILE, +} from './constants.ts'; +import { generateServiceConfig } from './config.ts'; +import { provisionCoreStack } from './provision.ts'; +import { seedElasticsearch } from './seed-elasticsearch.ts'; +import type { GridEnvironment } from './state.ts'; + +const LOCALSTACK_SERVICES = [ + "cloudformation", + "cloudwatch", + "dynamodb", + "kinesis", + "s3", + "sns", + "sqs", + "iam", +].join(","); + +export interface StartStackOptions { + /** + * Start the bundled Caddy reverse proxy instead of relying on the developer's + * dev-nginx. Defaults to true under CI, which has no dev-nginx. + */ + proxy?: boolean; + /** Seed Elasticsearch with the image fixtures. Defaults to true. */ + seed?: boolean; +} + +/** The subset of a started stack that teardown needs; a partially-booted stack also fits. */ +interface StoppableStack { + network: StartedNetwork; + containers: StartedTestContainer[]; + configDir?: string; +} + +/** + * Build a Caddyfile that reproduces the dev-nginx subdomain routing: each Grid service + * domain (https://.media.) reverse-proxies to the grid-e2e-ci container on + * its in-container port, and the S3 vanity domains proxy to localstack (prepending the + * real bucket to the path). `tls internal` serves a self-signed cert per site; Playwright + * runs with `ignoreHTTPSErrors`, so the internal CA does not need to be trusted. + * + * Example output: + * + * { + * auto_https disable_redirects + * } + * + * media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:9005 + * } + * + * api.media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:9001 + * } + * + * # ... etc for other similar services + * + * images.media.local.dev-gutools.co.uk { + * tls internal + * rewrite * /grid-dev-core-imagebucket-2b34bef1{uri} + * reverse_proxy localhost:4566 + * } + * + * public.media.local.dev-gutools.co.uk { + * tls internal + * rewrite * /grid-dev-core-imageoriginbucket-e8cd1fec{uri} + * reverse_proxy localhost:4566 + * } + * + * localstack.media.local.dev-gutools.co.uk { + * tls internal + * reverse_proxy localhost:4566 + * } + */ +function buildCaddyfile(coreStackProps: Record): string { + const appServices: Record = { + [`media.${DOMAIN}`]: SERVICE_PORTS.kahuna, + [`api.media.${DOMAIN}`]: SERVICE_PORTS['media-api'], + [`loader.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], + [`loader-projection.media.${DOMAIN}`]: SERVICE_PORTS['image-loader'], + [`cropper.media.${DOMAIN}`]: SERVICE_PORTS.cropper, + [`thrall.media.${DOMAIN}`]: SERVICE_PORTS.thrall, + [`media-metadata.${DOMAIN}`]: SERVICE_PORTS['metadata-editor'], + [`media-collections.${DOMAIN}`]: SERVICE_PORTS.collections, + [`media-leases.${DOMAIN}`]: SERVICE_PORTS.leases, + [`media-auth.${DOMAIN}`]: SERVICE_PORTS.auth, + }; + + // S3 vanity domains that omit the bucket -> localstack, with the bucket prepended. + const imageBuckets: Record = { + [`images.media.${DOMAIN}`]: coreStackProps.ImageBucket, + [`public.media.${DOMAIN}`]: coreStackProps.ImageOriginBucket + }; + + const blocks: string[] = []; + + for (const [siteHost, port] of Object.entries(appServices)) { + blocks.push(`${siteHost} {\n\ttls internal\n\treverse_proxy ${GRID_ALIAS}:${port}\n}`); + } + + for (const [siteHost, bucket] of Object.entries(imageBuckets)) { + blocks.push( + `${siteHost} {\n\ttls internal\n\trewrite * /${bucket}{uri}\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, + ); + } + + // Thumbnails / direct S3 access already include the bucket in the path. + blocks.push( + `localstack.media.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}\n}`, + ); + + // On-the-fly image resizing (optimised / full-screen views) -> the imgops container. + blocks.push( + `media-imgops.${DOMAIN} {\n\ttls internal\n\treverse_proxy ${IMGOPS_ALIAS}:80\n}`, + ); + + return `${blocks.join('\n\n')}\n`; +} + +/** Start the whole stack, tearing down anything already started if a later step fails. */ +export async function startStack(options: StartStackOptions = {}): Promise { + const { proxy = !!process.env.CI, seed = true } = options; + + const started: StartedTestContainer[] = []; + const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); + + let network: StartedNetwork | undefined; + let configDir: string | undefined; + + try { + network = await new Network().start(); + + // Infrastructure: Elasticsearch + LocalStack + const elasticsearch = await new GenericContainer(ELASTICSEARCH_IMAGE) + .withNetwork(network) + .withNetworkAliases(ELASTICSEARCH_ALIAS) + .withEnvironment({ + 'discovery.type': 'single-node', + 'xpack.security.enabled': 'false', + ES_JAVA_OPTS: '-Xms1024m -Xmx1024m', + }) + .withExposedPorts(9200) + .withWaitStrategy( + Wait.forHttp('/_cluster/health', 9200).forStatusCodeMatching((code) => code < 300), + ) + .withStartupTimeout(180_000) + .start(); + started.push(elasticsearch); + + const localstack = await new LocalstackContainer(LOCALSTACK_IMAGE) + .withNetwork(network) + .withNetworkAliases(LOCALSTACK_ALIAS) + // Pin to the fixed host port dev-nginx expects for the S3 vanity domains + // (images.media / public.media / localstack.media -> 4566). + .withExposedPorts({ container: LOCALSTACK_PORT, host: LOCALSTACK_PORT }) + .withEnvironment({ + SERVICES: LOCALSTACK_SERVICES, + DEFAULT_REGION: REGION, + KINESIS_ERROR_PROBABILITY: '0.0', + // Make resource URLs resolve via the network alias so the + // app container can reach them, and keep queue URLs path-style. + LOCALSTACK_HOST: `${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`, + SQS_ENDPOINT_STRATEGY: 'path', + }) + .withStartupTimeout(120_000) + .start(); + started.push(localstack); + + // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to + // the `localstack` alias on 4566, so it shares this network. + const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { + deleteOnExit: false, + }); + const imgops = await imgopsImage + .withNetwork(network) + .withNetworkAliases(IMGOPS_ALIAS) + .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) + .withExposedPorts({ container: 80, host: IMGOPS_PORT }) + .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) + .withStartupTimeout(120_000) + .start(); + started.push(imgops); + + // Provisioning + config generation + const coreStackProps = await provisionCoreStack(localstack.getConnectionUri()); + + configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'grid-config-')); + generateServiceConfig(configDir, coreStackProps); + + // All Grid services under test run inside this single container and talk to each + // other over its localhost. Each is published on the fixed host port its + // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's + // dev-nginx routes the https://*.media. domains straight into this + // container. + let gridBuilder = new GenericContainer(GRID_IMAGE) + .withNetwork(network) + .withNetworkAliases(GRID_ALIAS) + .withExposedPorts( + ...Object.values(SERVICE_PORTS).map((port) => ({ container: port, host: port })), + ) + .withBindMounts([ + // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. + { source: configDir, target: '/root/.grid', mode: 'ro' }, + { source: configDir, target: '/etc/grid', mode: 'ro' }, + // Outside CI the grid-e2e-dev image runs services under sbt; mount the repo + // over /build so host edits recompile live. + ...(process.env.CI ? [] : [{ source: REPO_ROOT, target: '/build', mode: 'rw' as const }]), + ]) + .withEnvironment({ + AWS_ACCESS_KEY_ID: 'test', + AWS_SECRET_ACCESS_KEY: 'test', + AWS_REGION: REGION, + AWS_DEFAULT_REGION: REGION, + AWS_CBOR_DISABLE: 'true', + }) + .withWaitStrategy( + Wait.forAll(Object.values(SERVICE_PORTS).map(port => + Wait.forHttp('/management/healthcheck', port).forStatusCode(200), + )) + ) + .withStartupTimeout(startupTimeoutMs); + + if (process.env.GRID_DEBUG) { + const logStream = fs.createWriteStream(path.join(os.tmpdir(), 'grid-boot.log')); + gridBuilder = gridBuilder.withLogConsumer((stream) => { + stream.on('data', (line) => logStream.write(line)); + stream.on('err', (line) => logStream.write(line)); + }); + } + + const grid = await gridBuilder.start(); + started.push(grid); + + // Seed Elasticsearch with image fixtures + // + // The app creates the `images` index + `Images_Current` alias on startup; seed once the + // stack is healthy so searches during the tests return the fixture documents. + if (seed) { + const esBaseUrl = `http://${elasticsearch.getHost()}:${elasticsearch.getMappedPort(9200)}`; + await seedElasticsearch(esBaseUrl); + } + + // CI routing: bundled reverse proxy + // + // Locally, the browser reaches the https://*.media. domains via the developer's + // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) + // start a Caddy proxy that replays the same subdomain routing and terminates TLS with a + // self-signed cert, published on the standard https port the domains resolve to. + if (proxy) { + const caddy = await new GenericContainer(PROXY_IMAGE) + .withNetwork(network) + .withExposedPorts({ container: 443, host: 443 }) + .withCopyContentToContainer([ + { content: buildCaddyfile(coreStackProps), target: '/etc/caddy/Caddyfile' }, + ]) + .withWaitStrategy(Wait.forListeningPorts()) + .withStartupTimeout(60_000) + .start(); + started.push(caddy); + } + + const host = grid.getHost(); + const baseUrl = `http://${host}:${grid.getMappedPort(KAHUNA_PORT)}`; + const mediaApiUrl = `http://${host}:${grid.getMappedPort(MEDIA_API_PORT)}`; + + fs.writeFileSync(URLS_FILE, JSON.stringify({ kahuna: baseUrl, mediaApi: mediaApiUrl })); + + return { network, containers: started, configDir, baseUrl, mediaApiUrl }; + } catch (error) { + // Leave nothing running if we failed part-way through the boot. + await stopStack(network ? { network, containers: started, configDir } : undefined); + throw error; + } +} + +/** Run a best-effort teardown step, warning (not throwing) so the rest still runs. */ +async function warnOnException(label: string, fn: () => unknown): Promise { + try { + await fn(); + } catch (error) { + console.warn(`${label}: ${(error as Error).message}`); + } +} + +/** Stop the containers (in reverse order), remove the network and the generated config. */ +export async function stopStack(environment: StoppableStack | undefined): Promise { + if (!environment) { + return; + } + + const { containers, network, configDir } = environment; + + for (const container of [...containers].reverse()) { + await warnOnException('Failed to stop container', () => container.stop()); + } + + await warnOnException('Failed to stop network', () => network.stop()); + if (configDir) { + await warnOnException('Failed to remove config dir', () => + fs.rmSync(configDir, { recursive: true, force: true }), + ); + } + await warnOnException('Failed to remove urls file', () => fs.rmSync(URLS_FILE, { force: true })); +} diff --git a/e2e-tests/testcontainers/state.ts b/e2e-tests/testcontainers/state.ts index 86406d78716..a844ec22568 100644 --- a/e2e-tests/testcontainers/state.ts +++ b/e2e-tests/testcontainers/state.ts @@ -15,6 +15,8 @@ export interface GridEnvironment { configDir: string; /** Base URL of the Kahuna UI, e.g. http://localhost:9005 */ baseUrl: string; + /** Base URL of the media-api, e.g. http://localhost:9001 */ + mediaApiUrl: string; } let environment: GridEnvironment | undefined; From e6a89cc604224eddd6dd3f86eb1282e920eeb105 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 3 Sep 2026 15:30:39 +0000 Subject: [PATCH 307/373] Pin Elasticsearch to 9200 --- e2e-tests/dev.ts | 4 ++-- e2e-tests/testcontainers/config.ts | 3 ++- e2e-tests/testcontainers/constants.ts | 2 ++ e2e-tests/testcontainers/stack.ts | 7 ++++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/e2e-tests/dev.ts b/e2e-tests/dev.ts index 0cabeaaf9b9..5f0280bcb42 100644 --- a/e2e-tests/dev.ts +++ b/e2e-tests/dev.ts @@ -7,7 +7,7 @@ */ import { DOMAIN, - ELASTICSEARCH_ALIAS, + ELASTICSEARCH_PORT, GRID_IMAGE, LOCALSTACK_PORT, SERVICE_PORTS, @@ -34,7 +34,7 @@ function banner(environment: GridEnvironment): string { '', ' Infrastructure:', ` localstack http://localhost:${LOCALSTACK_PORT}`, - ` elasticsearch alias '${ELASTICSEARCH_ALIAS}' on the stack network`, + ` elasticsearch http://localhost:${ELASTICSEARCH_PORT}`, '', ` Image: ${GRID_IMAGE}`, ` Config: ${environment.configDir}`, diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/testcontainers/config.ts index 20e06752fb8..68c8bbcb966 100644 --- a/e2e-tests/testcontainers/config.ts +++ b/e2e-tests/testcontainers/config.ts @@ -12,6 +12,7 @@ import { DOMAIN, EMAIL_DOMAIN, ELASTICSEARCH_ALIAS, + ELASTICSEARCH_PORT, LOCALSTACK_ALIAS, LOCALSTACK_PORT, REGION, @@ -75,7 +76,7 @@ export function generateServiceConfig(configDir: string, coreStackProps: StackPr coreStackProps, es6: { ...defaultConfig.es6, - url: `http://${ELASTICSEARCH_ALIAS}:9200`, + url: `http://${ELASTICSEARCH_ALIAS}:${ELASTICSEARCH_PORT}`, }, }; diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/testcontainers/constants.ts index 2a9bbc2dde8..45baf9fd226 100644 --- a/e2e-tests/testcontainers/constants.ts +++ b/e2e-tests/testcontainers/constants.ts @@ -36,6 +36,8 @@ export const IMGOPS_IMAGE = 'grid-e2e-imgops'; /** Network aliases the app container uses to reach the infrastructure containers. */ export const ELASTICSEARCH_ALIAS = 'elasticsearch'; +/** Fixed host port, so a stack started by `dev.ts` can be reached without a container handle. */ +export const ELASTICSEARCH_PORT = 9200; export const LOCALSTACK_ALIAS = 'localstack'; export const LOCALSTACK_PORT = 4566; /** Network alias the CI reverse proxy uses to reach the grid-e2e-ci app container. */ diff --git a/e2e-tests/testcontainers/stack.ts b/e2e-tests/testcontainers/stack.ts index 8f868c608cc..4445d2851c5 100644 --- a/e2e-tests/testcontainers/stack.ts +++ b/e2e-tests/testcontainers/stack.ts @@ -19,6 +19,7 @@ import { DOMAIN, ELASTICSEARCH_ALIAS, ELASTICSEARCH_IMAGE, + ELASTICSEARCH_PORT, GRID_ALIAS, GRID_IMAGE, IMGOPS_ALIAS, @@ -179,9 +180,9 @@ export async function startStack(options: StartStackOptions = {}): Promise code < 300), + Wait.forHttp('/_cluster/health', ELASTICSEARCH_PORT).forStatusCodeMatching((code) => code < 300), ) .withStartupTimeout(180_000) .start(); @@ -276,7 +277,7 @@ export async function startStack(options: StartStackOptions = {}): Promise Date: Thu, 3 Sep 2026 15:34:20 +0000 Subject: [PATCH 308/373] For test commands, only boot a stack if it doesn't already exist --- e2e-tests/dev.ts | 10 ++- e2e-tests/global-setup.ts | 9 ++- e2e-tests/testcontainers/stack.ts | 119 ++++++++++++++++++++++++++++-- e2e-tests/testcontainers/state.ts | 13 +++- 4 files changed, 136 insertions(+), 15 deletions(-) diff --git a/e2e-tests/dev.ts b/e2e-tests/dev.ts index 5f0280bcb42..ad3d6521c08 100644 --- a/e2e-tests/dev.ts +++ b/e2e-tests/dev.ts @@ -13,7 +13,7 @@ import { SERVICE_PORTS, URLS_FILE, } from './testcontainers/constants.ts'; -import { startStack, stopStack } from './testcontainers/stack.ts'; +import { probeStack, startStack, stopStack } from './testcontainers/stack.ts'; import type { GridEnvironment } from './testcontainers/state.ts'; function banner(environment: GridEnvironment): string { @@ -73,6 +73,14 @@ async function dev(): Promise { process.on('SIGINT', shutdown); process.on('SIGTERM', shutdown); + // The host ports are fixed, so a second stack cannot coexist with the first. + const { state, healthy } = await probeStack(); + if (state !== 'none') { + throw new Error( + `Ports ${healthy.join(', ')} are already serving Grid. Stop that stack before starting another.`, + ); + } + environment = await startStack({ proxy: process.env.GRID_PROXY === 'true' }); console.log(banner(environment)); diff --git a/e2e-tests/global-setup.ts b/e2e-tests/global-setup.ts index 778db410625..de7bb3b326c 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/global-setup.ts @@ -1,12 +1,13 @@ /** - * Playwright global setup. Boots the Grid stack (see `testcontainers/stack.ts`) and - * stashes it for `global-teardown.ts`, exposing the Kahuna base URL via `GRID_BASE_URL`. + * Playwright global setup. Attaches to a running Grid stack if there is one, otherwise + * boots a fresh one (see `testcontainers/stack.ts`), stashing it for `global-teardown.ts` + * and exposing the Kahuna base URL via `GRID_BASE_URL`. */ -import { startStack } from './testcontainers/stack.ts'; +import { ensureStack } from './testcontainers/stack.ts'; import { setEnvironment } from './testcontainers/state.ts'; async function globalSetup(): Promise { - const environment = await startStack(); + const environment = await ensureStack(); process.env.GRID_BASE_URL = environment.baseUrl; setEnvironment(environment); } diff --git a/e2e-tests/testcontainers/stack.ts b/e2e-tests/testcontainers/stack.ts index 4445d2851c5..deb6efec271 100644 --- a/e2e-tests/testcontainers/stack.ts +++ b/e2e-tests/testcontainers/stack.ts @@ -64,11 +64,17 @@ export interface StartStackOptions { seed?: boolean; } +/** How much of the stack is already listening on the fixed host ports. */ +type StackProbe = 'none' | 'healthy' | 'partial'; + +const PROBE_TIMEOUT_MS = 2_000; + /** The subset of a started stack that teardown needs; a partially-booted stack also fits. */ interface StoppableStack { - network: StartedNetwork; + network?: StartedNetwork; containers: StartedTestContainer[]; configDir?: string; + urlsFile?: string; } /** @@ -306,10 +312,17 @@ export async function startStack(options: StartStackOptions = {}): Promise unknown): Promise } } -/** Stop the containers (in reverse order), remove the network and the generated config. */ +/** Is a service answering its healthcheck on this fixed host port? */ +async function isServiceHealthy(port: number): Promise { + try { + const response = await fetch(`http://localhost:${port}/management/healthcheck`, { + signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), + }); + return response.ok; + } catch { + return false; + } +} + +/** + * Probe the fixed host ports for an already-running stack. + * + * The ports are pinned, so a live stack is always reachable at localhost:. + * That makes the probe, rather than the presence of the URLs file, the source of truth: + * the file outlives a `SIGKILL`ed run and would otherwise point tests at nothing. + */ +export async function probeStack(): Promise<{ state: StackProbe; healthy: number[]; ports: number[] }> { + const ports = Object.values(SERVICE_PORTS); + const results = await Promise.all(ports.map(isServiceHealthy)); + const healthy = ports.filter((_, index) => results[index]); + + if (healthy.length === 0) return { state: 'none', healthy, ports }; + if (healthy.length === ports.length) return { state: 'healthy', healthy, ports }; + return { state: 'partial', healthy, ports }; +} + +/** + * Attach to a running stack if there is a healthy one, otherwise boot a fresh one. + * + * The returned environment holds handles only for what this call created, so teardown + * never stops a stack started by `npm run dev:e2e`. + */ +export async function ensureStack(options: StartStackOptions = {}): Promise { + // Nothing pre-exists in CI, and silently attaching there would undermine the run. + const reuseAllowed = !process.env.CI && process.env.GRID_NO_REUSE !== 'true'; + + const { state, healthy, ports } = await probeStack(); + + if (state === 'partial') { + const missing = ports.filter((port) => !healthy.includes(port)); + throw new Error( + `A partial Grid stack is running: ports ${healthy.join(', ')} are healthy but ` + + `${missing.join(', ')} are not. Stop it (or wait for it to finish booting) and retry.`, + ); + } + + if (state === 'healthy') { + if (!reuseAllowed) { + throw new Error( + 'A Grid stack is already running and reuse is disabled (CI or GRID_NO_REUSE). ' + + 'Stop it before starting a fresh one; the fixed host ports cannot be shared.', + ); + } + return attachToStack(options); + } + + return startStack(options); +} + +/** + * Build an environment for a stack this process did not start. It owns no containers, + * so `stopStack` leaves everything running. + */ +async function attachToStack(options: StartStackOptions): Promise { + const baseUrl = `http://localhost:${KAHUNA_PORT}`; + const mediaApiUrl = `http://localhost:${MEDIA_API_PORT}`; + + console.log(`Reusing the Grid stack already running on ${baseUrl}`); + + // Re-seeding is opt-in: whoever started the stack already seeded it, and the fixtures + // are only reloaded on request because tests may have since changed the data. + if (options.seed === true || process.env.GRID_RESEED === 'true') { + await seedElasticsearch(`http://localhost:${ELASTICSEARCH_PORT}`); + } + + // Only claim the URLs file if it is missing, so teardown never deletes another stack's. + let urlsFile: string | undefined; + if (!fs.existsSync(URLS_FILE)) { + fs.writeFileSync(URLS_FILE, JSON.stringify({ kahuna: baseUrl, mediaApi: mediaApiUrl })); + urlsFile = URLS_FILE; + } + + return { containers: [], urlsFile, baseUrl, mediaApiUrl }; +} + +/** Stop what this process started and delete what it wrote; anything else is left alone. */ export async function stopStack(environment: StoppableStack | undefined): Promise { if (!environment) { return; } - const { containers, network, configDir } = environment; + const { containers, network, configDir, urlsFile } = environment; for (const container of [...containers].reverse()) { await warnOnException('Failed to stop container', () => container.stop()); } - await warnOnException('Failed to stop network', () => network.stop()); + if (network) { + await warnOnException('Failed to stop network', () => network.stop()); + } if (configDir) { await warnOnException('Failed to remove config dir', () => fs.rmSync(configDir, { recursive: true, force: true }), ); } - await warnOnException('Failed to remove urls file', () => fs.rmSync(URLS_FILE, { force: true })); + if (urlsFile) { + await warnOnException('Failed to remove urls file', () => fs.rmSync(urlsFile, { force: true })); + } } diff --git a/e2e-tests/testcontainers/state.ts b/e2e-tests/testcontainers/state.ts index a844ec22568..93147c97a6a 100644 --- a/e2e-tests/testcontainers/state.ts +++ b/e2e-tests/testcontainers/state.ts @@ -3,16 +3,23 @@ * `global-setup.ts`. Playwright runs global setup and global teardown in the same * Node process, so a module-level singleton is a reliable way to hand references * from setup to teardown. + * + * Ownership is structural: this process only holds handles to resources it created, + * so teardown stops what it holds and deletes what it wrote. When attaching to a + * stack someone else started, these are empty and teardown is a no-op. */ import type { StartedTestContainer } from 'testcontainers'; import type { StartedNetwork } from 'testcontainers'; export interface GridEnvironment { - network: StartedNetwork; - /** Every started container, in start order. Stopped in reverse on teardown. */ + /** Absent when attached to a stack this process did not start. */ + network?: StartedNetwork; + /** Every container this process started, in start order. Stopped in reverse on teardown. */ containers: StartedTestContainer[]; /** Absolute path to the generated config directory mounted into the app container. */ - configDir: string; + configDir?: string; + /** Set only when this process wrote the URLs file, so teardown does not delete another stack's. */ + urlsFile?: string; /** Base URL of the Kahuna UI, e.g. http://localhost:9005 */ baseUrl: string; /** Base URL of the media-api, e.g. http://localhost:9001 */ From d3f6d859a65da2cfd5520db60c15882edf9c8734 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 06:36:37 +0000 Subject: [PATCH 309/373] Update readme --- e2e-tests/README.md | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 2fd84488f9e..6af3791fec6 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -47,13 +47,13 @@ so a failed test on CI leaves a trace you can open with `npx playwright show-tra ## Running the stack without the tests -`npm run dev` boots exactly the same stack the tests use, prints the service URLs and +`npm run dev:e2e` boots exactly the same stack the tests use, prints the service URLs and holds it open until you press Ctrl-C, which tears it all down. Useful for poking at Grid by hand, or for leaving the stack up while you iterate on step definitions. ```bash -npm run dev # uses your local dev-nginx for the https://*.media. domains -npm run dev:proxy # no dev-nginx? start the bundled Caddy proxy on :443 instead +npm run dev:e2e # uses your local dev-nginx for the https://*.media. domains +GRID_PROXY=true npm run dev:e2e # no dev-nginx? start the bundled Caddy proxy on :443 instead ``` It uses the same image as the tests (`grid-e2e-dev` locally, `grid-e2e-ci` under `CI`), so @@ -61,13 +61,37 @@ build that first. The first boot is slow because the dev image compiles from sou `GRID_STARTUP_TIMEOUT_MS` (default `300000`) if it times out, and set `GRID_DEBUG=1` to tee the container's boot logs to `$TMPDIR/grid-boot.log`. -The stack binds fixed host ports (9001-9012, 4566, 9008), so `npm run dev` and `npm test` -cannot run at the same time. +The stack binds fixed host ports (9001-9012, 4566, 9008, 9200), so two stacks cannot run +at once. Starting a second one fails immediately rather than timing out. + +### Running the tests against a stack you already started + +The test commands reuse a running stack instead of booting their own, which turns a +multi-minute boot into a couple of seconds. Leave `npm run dev:e2e` running in one +terminal, then use `npm test`, `npm run test:ui` or any of the others as normal — they +attach automatically and leave the stack running when they finish. + +If only some services are up (usually because the stack is still booting), the run stops +straight away and names the ports it is waiting on. + +| Variable | Effect | +| --- | --- | +| `GRID_NO_REUSE=true` | Never attach; fail if a stack is already running. Always set under `CI`. | +| `GRID_RESEED=true` | Reload the Elasticsearch fixtures into the reused stack. | + +**Watch out for stale provisioning.** A reused stack picks up Scala changes (the repo is +bind-mounted and services run under `sbt run`), but *not* changes to anything applied at +boot: generated service config, the CloudFormation template, bucket contents, permissions +or the Elasticsearch fixtures. After changing any of those, restart `dev:e2e`. + +Reuse also means state carries over between runs. The current suite is read-only, so this +is harmless today, but a test that uploads or edits an image will want a fresh stack. ## How they work -Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) calls `startStack` -([`testcontainers/stack.ts`](testcontainers/stack.ts)), which uses +Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) calls `ensureStack` +([`testcontainers/stack.ts`](testcontainers/stack.ts)), which attaches to a healthy stack +if one is already running and otherwise uses [Testcontainers](https://testcontainers.com/) to stand up everything the tests need: 1. a shared Docker network; @@ -78,5 +102,6 @@ Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) calls `startSt 5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running Grid's Play services. Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is -exposed to the tests. `globalTeardown` calls `stopStack`, which stops everything and -cleans up. `npm run dev` drives the same two functions. +exposed to the tests. `globalTeardown` calls `stopStack`, which stops only the containers +this process started and deletes only the files it wrote — so attaching to someone else's +stack tears nothing down. `npm run dev:e2e` drives the same functions. From 265e2f47af362794730c530ddf949fb995653ff7 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 06:36:46 +0000 Subject: [PATCH 310/373] Hold process open indefinitely --- e2e-tests/dev.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/dev.ts b/e2e-tests/dev.ts index ad3d6521c08..5316c58ac03 100644 --- a/e2e-tests/dev.ts +++ b/e2e-tests/dev.ts @@ -84,8 +84,8 @@ async function dev(): Promise { environment = await startStack({ proxy: process.env.GRID_PROXY === 'true' }); console.log(banner(environment)); - // Hold the process open; the signal handlers are the only way out. - await new Promise(() => {}); + // Hold the process open indefinitely. + setInterval(() => {}, 2_000_000_000); } dev().catch((error) => { From 4ca4bd7f7589a0228b73692379d24d4ee7fba849 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 06:46:58 +0000 Subject: [PATCH 311/373] Regenerate package-lock.json --- e2e-tests/package-lock.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/e2e-tests/package-lock.json b/e2e-tests/package-lock.json index de39c578fd9..e9556f4aaa2 100644 --- a/e2e-tests/package-lock.json +++ b/e2e-tests/package-lock.json @@ -17,6 +17,9 @@ "json5": "^2.2.3", "playwright-bdd": "^9.2.0", "testcontainers": "^10.13.2" + }, + "engines": { + "node": ">=22.12" } }, "node_modules/@aws-sdk/checksums": { From 4a9c207b9f66c0d9059296c4dc78b6f39b1df4ec Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 06:47:13 +0000 Subject: [PATCH 312/373] Listen for ES and Localstack when probing stack --- e2e-tests/testcontainers/stack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/testcontainers/stack.ts b/e2e-tests/testcontainers/stack.ts index deb6efec271..e095fe061bc 100644 --- a/e2e-tests/testcontainers/stack.ts +++ b/e2e-tests/testcontainers/stack.ts @@ -356,7 +356,7 @@ async function isServiceHealthy(port: number): Promise { * the file outlives a `SIGKILL`ed run and would otherwise point tests at nothing. */ export async function probeStack(): Promise<{ state: StackProbe; healthy: number[]; ports: number[] }> { - const ports = Object.values(SERVICE_PORTS); + const ports = [...Object.values(SERVICE_PORTS), ELASTICSEARCH_PORT, LOCALSTACK_PORT]; const results = await Promise.all(ports.map(isServiceHealthy)); const healthy = ports.filter((_, index) => results[index]); From 70619f7d97aea04c2eca4c8a0c4e04e3cb25cb6d Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 06:55:14 +0000 Subject: [PATCH 313/373] Update script to remove reference to run command --- e2e-tests/dev.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e-tests/dev.ts b/e2e-tests/dev.ts index 5316c58ac03..1730c1e6a09 100644 --- a/e2e-tests/dev.ts +++ b/e2e-tests/dev.ts @@ -1,9 +1,9 @@ /** * Boots the Grid stack outside Playwright and holds it open until Ctrl-C. * - * Run with `npm run dev`. Set `GRID_PROXY=true` to also start the bundled Caddy - * reverse proxy, which serves the https://*.media. domains for anyone - * without dev-nginx running locally. + * Set `GRID_PROXY=true` to also start the bundled Caddy reverse proxy, which + * serves the https://*.media. domains for anyone without dev-nginx + * running locally. */ import { DOMAIN, From 54f9dbd9324c4be4c74279c0696c22af2e1e97d2 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 13:34:20 +0000 Subject: [PATCH 314/373] Move testcontainers folder to ./setup, per other projects --- e2e-tests/README.md | 2 +- e2e-tests/package.json | 2 +- e2e-tests/playwright.config.ts | 6 ++-- e2e-tests/{testcontainers => setup}/config.ts | 0 .../{testcontainers => setup}/constants.ts | 0 e2e-tests/{ => setup}/dev.ts | 6 ++-- e2e-tests/{ => setup}/global-setup.ts | 6 ++-- e2e-tests/{ => setup}/global-teardown.ts | 4 +-- .../{testcontainers => setup}/provision.ts | 0 .../seed-elasticsearch.ts | 0 e2e-tests/{testcontainers => setup}/stack.ts | 35 ++++++++++--------- e2e-tests/{testcontainers => setup}/state.ts | 0 e2e-tests/steps/fixtures.ts | 2 +- e2e-tests/steps/global-setup.ts | 15 ++++++++ 14 files changed, 47 insertions(+), 31 deletions(-) rename e2e-tests/{testcontainers => setup}/config.ts (100%) rename e2e-tests/{testcontainers => setup}/constants.ts (100%) rename e2e-tests/{ => setup}/dev.ts (93%) rename e2e-tests/{ => setup}/global-setup.ts (63%) rename e2e-tests/{ => setup}/global-teardown.ts (64%) rename e2e-tests/{testcontainers => setup}/provision.ts (100%) rename e2e-tests/{testcontainers => setup}/seed-elasticsearch.ts (100%) rename e2e-tests/{testcontainers => setup}/stack.ts (94%) rename e2e-tests/{testcontainers => setup}/state.ts (100%) create mode 100644 e2e-tests/steps/global-setup.ts diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 6af3791fec6..559ce3784c2 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -90,7 +90,7 @@ is harmless today, but a test that uploads or edits an image will want a fresh s ## How they work Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) calls `ensureStack` -([`testcontainers/stack.ts`](testcontainers/stack.ts)), which attaches to a healthy stack +([`setup/stack.ts`](setup/stack.ts)), which attaches to a healthy stack if one is already running and otherwise uses [Testcontainers](https://testcontainers.com/) to stand up everything the tests need: diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 0a1ca0b37c5..4c6109dd023 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -5,7 +5,7 @@ "main": "index.js", "type": "module", "scripts": { - "dev:e2e": "node --experimental-strip-types --disable-warning=ExperimentalWarning dev.ts", + "dev:e2e": "node --experimental-strip-types --disable-warning=ExperimentalWarning setup/dev.ts", "test": "bddgen && playwright test", "test:headed": "bddgen && playwright test --headed", "test:report": "playwright show-report", diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index fc79c9bdda3..15f9ca94c5b 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -1,6 +1,6 @@ import { defineConfig, devices } from '@playwright/test'; import { defineBddConfig } from 'playwright-bdd'; -import { KAHUNA_PORT } from './testcontainers/constants.ts'; +import { KAHUNA_PORT } from './setup/constants.ts'; /* Generate Playwright test files from the Gherkin feature files and step definitions. Run via `bddgen` (see package.json scripts) before `playwright test`. */ @@ -25,8 +25,8 @@ export default defineConfig({ /* Reporter to use. See https://playwright.dev/docs/test-reporters */ reporter: 'html', /* Boot the Grid stack with Testcontainers before tests, and tear it down after. */ - globalSetup: './global-setup.ts', - globalTeardown: './global-teardown.ts', + globalSetup: './setup/global-setup.ts', + globalTeardown: './setup/global-teardown.ts', /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL (Kahuna) to use in actions like `await page.goto('/')`. Set by global-setup. */ diff --git a/e2e-tests/testcontainers/config.ts b/e2e-tests/setup/config.ts similarity index 100% rename from e2e-tests/testcontainers/config.ts rename to e2e-tests/setup/config.ts diff --git a/e2e-tests/testcontainers/constants.ts b/e2e-tests/setup/constants.ts similarity index 100% rename from e2e-tests/testcontainers/constants.ts rename to e2e-tests/setup/constants.ts diff --git a/e2e-tests/dev.ts b/e2e-tests/setup/dev.ts similarity index 93% rename from e2e-tests/dev.ts rename to e2e-tests/setup/dev.ts index 1730c1e6a09..3ad0d8a2725 100644 --- a/e2e-tests/dev.ts +++ b/e2e-tests/setup/dev.ts @@ -12,9 +12,9 @@ import { LOCALSTACK_PORT, SERVICE_PORTS, URLS_FILE, -} from './testcontainers/constants.ts'; -import { probeStack, startStack, stopStack } from './testcontainers/stack.ts'; -import type { GridEnvironment } from './testcontainers/state.ts'; +} from './constants.ts'; +import { probeStack, startStack, stopStack } from './stack.ts'; +import type { GridEnvironment } from './state.ts'; function banner(environment: GridEnvironment): string { const services = Object.keys(SERVICE_PORTS) diff --git a/e2e-tests/global-setup.ts b/e2e-tests/setup/global-setup.ts similarity index 63% rename from e2e-tests/global-setup.ts rename to e2e-tests/setup/global-setup.ts index de7bb3b326c..1952354d6bc 100644 --- a/e2e-tests/global-setup.ts +++ b/e2e-tests/setup/global-setup.ts @@ -1,10 +1,10 @@ /** * Playwright global setup. Attaches to a running Grid stack if there is one, otherwise - * boots a fresh one (see `testcontainers/stack.ts`), stashing it for `global-teardown.ts` + * boots a fresh one (see `setup/stack.ts`), stashing it for `global-teardown.ts` * and exposing the Kahuna base URL via `GRID_BASE_URL`. */ -import { ensureStack } from './testcontainers/stack.ts'; -import { setEnvironment } from './testcontainers/state.ts'; +import { ensureStack } from './stack.ts'; +import { setEnvironment } from './state.ts'; async function globalSetup(): Promise { const environment = await ensureStack(); diff --git a/e2e-tests/global-teardown.ts b/e2e-tests/setup/global-teardown.ts similarity index 64% rename from e2e-tests/global-teardown.ts rename to e2e-tests/setup/global-teardown.ts index 10dc72408dd..f6079c8d495 100644 --- a/e2e-tests/global-teardown.ts +++ b/e2e-tests/setup/global-teardown.ts @@ -1,8 +1,8 @@ /** * Playwright global teardown. Stops the stack started in `global-setup.ts`. */ -import { stopStack } from './testcontainers/stack.ts'; -import { getEnvironment } from './testcontainers/state.ts'; +import { stopStack } from './stack.ts'; +import { getEnvironment } from './state.ts'; async function globalTeardown(): Promise { await stopStack(getEnvironment()); diff --git a/e2e-tests/testcontainers/provision.ts b/e2e-tests/setup/provision.ts similarity index 100% rename from e2e-tests/testcontainers/provision.ts rename to e2e-tests/setup/provision.ts diff --git a/e2e-tests/testcontainers/seed-elasticsearch.ts b/e2e-tests/setup/seed-elasticsearch.ts similarity index 100% rename from e2e-tests/testcontainers/seed-elasticsearch.ts rename to e2e-tests/setup/seed-elasticsearch.ts diff --git a/e2e-tests/testcontainers/stack.ts b/e2e-tests/setup/stack.ts similarity index 94% rename from e2e-tests/testcontainers/stack.ts rename to e2e-tests/setup/stack.ts index e095fe061bc..630ded1bd08 100644 --- a/e2e-tests/testcontainers/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -44,14 +44,14 @@ import { seedElasticsearch } from './seed-elasticsearch.ts'; import type { GridEnvironment } from './state.ts'; const LOCALSTACK_SERVICES = [ - "cloudformation", - "cloudwatch", - "dynamodb", - "kinesis", - "s3", - "sns", - "sqs", - "iam", + "cloudformation", + "cloudwatch", + "dynamodb", + "kinesis", + "s3", + "sns", + "sqs", + "iam", ].join(","); export interface StartStackOptions { @@ -337,14 +337,14 @@ async function warnOnException(label: string, fn: () => unknown): Promise } /** Is a service answering its healthcheck on this fixed host port? */ -async function isServiceHealthy(port: number): Promise { +const isServiceHealthy = (path: string) => async (port: number): Promise<{ port: number, healthy: boolean }> => { try { - const response = await fetch(`http://localhost:${port}/management/healthcheck`, { + const response = await fetch(`http://localhost:${port}/${path}`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), }); - return response.ok; + return { port, healthy: true }; } catch { - return false; + return { port, healthy: false }; } } @@ -356,9 +356,10 @@ async function isServiceHealthy(port: number): Promise { * the file outlives a `SIGKILL`ed run and would otherwise point tests at nothing. */ export async function probeStack(): Promise<{ state: StackProbe; healthy: number[]; ports: number[] }> { - const ports = [...Object.values(SERVICE_PORTS), ELASTICSEARCH_PORT, LOCALSTACK_PORT]; - const results = await Promise.all(ports.map(isServiceHealthy)); - const healthy = ports.filter((_, index) => results[index]); + const healthchecks = [...Object.values(SERVICE_PORTS).map(isServiceHealthy('management/healthcheck')), isServiceHealthy('_cluster/_health')(ELASTICSEARCH_PORT), isServiceHealthy('_localstack/health')(LOCALSTACK_PORT)]; + const results = await Promise.all(healthchecks); + const healthy = results.filter(({ healthy }) => healthy).map(({ port }) => port); + const ports = results.map(({ port }) => port); if (healthy.length === 0) return { state: 'none', healthy, ports }; if (healthy.length === ports.length) return { state: 'healthy', healthy, ports }; @@ -381,7 +382,7 @@ export async function ensureStack(options: StartStackOptions = {}): Promise !healthy.includes(port)); throw new Error( `A partial Grid stack is running: ports ${healthy.join(', ')} are healthy but ` + - `${missing.join(', ')} are not. Stop it (or wait for it to finish booting) and retry.`, + `${missing.join(', ')} are not. Stop it (or wait for it to finish booting) and retry.`, ); } @@ -389,7 +390,7 @@ export async function ensureStack(options: StartStackOptions = {}): Promise { + const environment = await ensureStack(); + process.env.GRID_BASE_URL = environment.baseUrl; + setEnvironment(environment); +} + +export default globalSetup; From 0c47b6e6cd32d79ddb239b1c538d51daaae2635c Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 15:00:34 +0100 Subject: [PATCH 315/373] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e-tests/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 559ce3784c2..a26b7411032 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -89,7 +89,7 @@ is harmless today, but a test that uploads or edits an image will want a fresh s ## How they work -Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) calls `ensureStack` +Playwright's `globalSetup` ([`setup/global-setup.ts`](setup/global-setup.ts)) calls `ensureStack` ([`setup/stack.ts`](setup/stack.ts)), which attaches to a healthy stack if one is already running and otherwise uses [Testcontainers](https://testcontainers.com/) to stand up everything the tests need: From f09d5649f3fe0076110b2543bf4b4e371c9f1457 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 13:34:20 +0000 Subject: [PATCH 316/373] Move testcontainers folder to ./setup, per other projects --- e2e-tests/README.md | 4 ++++ e2e-tests/setup/stack.ts | 10 ++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index a26b7411032..17bcccfb287 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -89,7 +89,11 @@ is harmless today, but a test that uploads or edits an image will want a fresh s ## How they work +<<<<<<< HEAD Playwright's `globalSetup` ([`setup/global-setup.ts`](setup/global-setup.ts)) calls `ensureStack` +======= +Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) calls `ensureStack` +>>>>>>> 9c098462b (Move testcontainers folder to ./setup, per other projects) ([`setup/stack.ts`](setup/stack.ts)), which attaches to a healthy stack if one is already running and otherwise uses [Testcontainers](https://testcontainers.com/) to stand up everything the tests need: diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index 630ded1bd08..ed231c65512 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -342,7 +342,7 @@ const isServiceHealthy = (path: string) => async (port: number): Promise<{ port: const response = await fetch(`http://localhost:${port}/${path}`, { signal: AbortSignal.timeout(PROBE_TIMEOUT_MS), }); - return { port, healthy: true }; + return { port, healthy: response.ok }; } catch { return { port, healthy: false }; } @@ -356,7 +356,13 @@ const isServiceHealthy = (path: string) => async (port: number): Promise<{ port: * the file outlives a `SIGKILL`ed run and would otherwise point tests at nothing. */ export async function probeStack(): Promise<{ state: StackProbe; healthy: number[]; ports: number[] }> { - const healthchecks = [...Object.values(SERVICE_PORTS).map(isServiceHealthy('management/healthcheck')), isServiceHealthy('_cluster/_health')(ELASTICSEARCH_PORT), isServiceHealthy('_localstack/health')(LOCALSTACK_PORT)]; + const healthchecks = [ + ...Object.values(SERVICE_PORTS).map(isServiceHealthy('management/healthcheck')), + isServiceHealthy('_cluster/health')(ELASTICSEARCH_PORT), + isServiceHealthy('_localstack/health')(LOCALSTACK_PORT), + isServiceHealthy('')(IMGOPS_PORT) + ]; + const results = await Promise.all(healthchecks); const healthy = results.filter(({ healthy }) => healthy).map(({ port }) => port); const ports = results.map(({ port }) => port); From 134d2e540f45f1a1b967975d29732859cf06dabc Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 18:49:09 +0000 Subject: [PATCH 317/373] Remove GRID_NO_REUSE, and amend docs --- e2e-tests/README.md | 32 +------------------------------- e2e-tests/setup/stack.ts | 4 ++-- 2 files changed, 3 insertions(+), 33 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 17bcccfb287..8d29acbe6cb 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -48,19 +48,13 @@ so a failed test on CI leaves a trace you can open with `npx playwright show-tra ## Running the stack without the tests `npm run dev:e2e` boots exactly the same stack the tests use, prints the service URLs and -holds it open until you press Ctrl-C, which tears it all down. Useful for poking at Grid -by hand, or for leaving the stack up while you iterate on step definitions. +holds it open until you press Ctrl-C, which tears it all down. ```bash npm run dev:e2e # uses your local dev-nginx for the https://*.media. domains GRID_PROXY=true npm run dev:e2e # no dev-nginx? start the bundled Caddy proxy on :443 instead ``` -It uses the same image as the tests (`grid-e2e-dev` locally, `grid-e2e-ci` under `CI`), so -build that first. The first boot is slow because the dev image compiles from source; raise -`GRID_STARTUP_TIMEOUT_MS` (default `300000`) if it times out, and set `GRID_DEBUG=1` to -tee the container's boot logs to `$TMPDIR/grid-boot.log`. - The stack binds fixed host ports (9001-9012, 4566, 9008, 9200), so two stacks cannot run at once. Starting a second one fails immediately rather than timing out. @@ -76,7 +70,6 @@ straight away and names the ports it is waiting on. | Variable | Effect | | --- | --- | -| `GRID_NO_REUSE=true` | Never attach; fail if a stack is already running. Always set under `CI`. | | `GRID_RESEED=true` | Reload the Elasticsearch fixtures into the reused stack. | **Watch out for stale provisioning.** A reused stack picks up Scala changes (the repo is @@ -86,26 +79,3 @@ or the Elasticsearch fixtures. After changing any of those, restart `dev:e2e`. Reuse also means state carries over between runs. The current suite is read-only, so this is harmless today, but a test that uploads or edits an image will want a fresh stack. - -## How they work - -<<<<<<< HEAD -Playwright's `globalSetup` ([`setup/global-setup.ts`](setup/global-setup.ts)) calls `ensureStack` -======= -Playwright's `globalSetup` ([`global-setup.ts`](global-setup.ts)) calls `ensureStack` ->>>>>>> 9c098462b (Move testcontainers folder to ./setup, per other projects) -([`setup/stack.ts`](setup/stack.ts)), which attaches to a healthy stack -if one is already running and otherwise uses -[Testcontainers](https://testcontainers.com/) to stand up everything the tests need: - -1. a shared Docker network; -2. **Elasticsearch** and **LocalStack** (the backing infrastructure); -3. the **CloudFormation core stack** (`dev/cloudformation/grid-dev-core.yml`) and seeded - buckets, provisioned into LocalStack; -4. per-service config, generated by `dev/script/generate-config/service-config.js`; -5. the Grid Docker image (either `grid-e2e-ci` or `grid-e2e-dev`) — a single container running Grid's Play services. - -Elasticsearch is then seeded with image fixtures, and the resolved Kahuna base URL is -exposed to the tests. `globalTeardown` calls `stopStack`, which stops only the containers -this process started and deletes only the files it wrote — so attaching to someone else's -stack tears nothing down. `npm run dev:e2e` drives the same functions. diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index ed231c65512..63027cfda85 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -380,7 +380,7 @@ export async function probeStack(): Promise<{ state: StackProbe; healthy: number */ export async function ensureStack(options: StartStackOptions = {}): Promise { // Nothing pre-exists in CI, and silently attaching there would undermine the run. - const reuseAllowed = !process.env.CI && process.env.GRID_NO_REUSE !== 'true'; + const reuseAllowed = !process.env.CI; const { state, healthy, ports } = await probeStack(); @@ -395,7 +395,7 @@ export async function ensureStack(options: StartStackOptions = {}): Promise Date: Fri, 4 Sep 2026 18:49:51 +0000 Subject: [PATCH 318/373] Correct imgops healthcheck --- e2e-tests/setup/stack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index 63027cfda85..718a1ee1212 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -360,7 +360,7 @@ export async function probeStack(): Promise<{ state: StackProbe; healthy: number ...Object.values(SERVICE_PORTS).map(isServiceHealthy('management/healthcheck')), isServiceHealthy('_cluster/health')(ELASTICSEARCH_PORT), isServiceHealthy('_localstack/health')(LOCALSTACK_PORT), - isServiceHealthy('')(IMGOPS_PORT) + isServiceHealthy('_')(IMGOPS_PORT) ]; const results = await Promise.all(healthchecks); From 97a15c6f62e030325d6af5f8575b4528ece5d882 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 19:59:20 +0100 Subject: [PATCH 319/373] Correct hostname of kahuna service in output Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e-tests/setup/dev.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/setup/dev.ts b/e2e-tests/setup/dev.ts index 3ad0d8a2725..d21a494ea7b 100644 --- a/e2e-tests/setup/dev.ts +++ b/e2e-tests/setup/dev.ts @@ -26,7 +26,7 @@ function banner(environment: GridEnvironment): string { '', ' Grid is up.', '', - ` kahuna (UI) ${environment.baseUrl}`, + ` kahuna (UI) https://media.${DOMAIN}`, ` media-api ${environment.mediaApiUrl}`, '', ' Services:', From 58e3abb641dbc610d708ddd96ee670c504be3bbb Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 4 Sep 2026 19:59:35 +0100 Subject: [PATCH 320/373] Remove missing whitespace Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e-tests/setup/stack.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index 718a1ee1212..6b3801ee451 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -395,7 +395,7 @@ export async function ensureStack(options: StartStackOptions = {}): Promise Date: Fri, 4 Sep 2026 19:15:55 +0000 Subject: [PATCH 321/373] Pin the config dir to a stable path, and clear it out if the running process owns it on exit --- e2e-tests/setup/constants.ts | 7 +++++++ e2e-tests/setup/stack.ts | 23 ++++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/e2e-tests/setup/constants.ts b/e2e-tests/setup/constants.ts index 45baf9fd226..ac8ba8429db 100644 --- a/e2e-tests/setup/constants.ts +++ b/e2e-tests/setup/constants.ts @@ -1,3 +1,4 @@ +import * as os from 'os'; import * as path from 'path'; export const REPO_ROOT = path.resolve(import.meta.dirname, '..', '..'); @@ -62,5 +63,11 @@ export const MEDIA_API_PORT = SERVICE_PORTS['media-api']; /** File (repo-relative to e2e-tests) where global-setup records the resolved service URLs. */ export const URLS_FILE = path.join(import.meta.dirname, '..', '.grid-urls.json'); +/** + * Generated per-service config, bind-mounted into the app container. The path is fixed + * rather than unique per run: the fixed host ports already allow only one stack at a time. + */ +export const CONFIG_DIR = path.join(os.tmpdir(), 'grid-e2e-config'); + /** The API key value uploaded to the KeyBucket (dev/.env API_KEY). */ export const API_KEY = 'e2e-dev'; diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index 6b3801ee451..9330ae051d7 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -16,6 +16,7 @@ import { LocalstackContainer } from '@testcontainers/localstack'; import { GenericContainer, Network, Wait } from 'testcontainers'; import type { StartedNetwork, StartedTestContainer } from 'testcontainers'; import { + CONFIG_DIR, DOMAIN, ELASTICSEARCH_ALIAS, ELASTICSEARCH_IMAGE, @@ -164,6 +165,20 @@ function buildCaddyfile(coreStackProps: Record): string { return `${blocks.join('\n\n')}\n`; } +/** + * Set while this process owns the config directory, so the exit hook below never deletes + * the config of a stack started elsewhere and still running (see `attachToStack`). + */ +let ownedConfigDir: string | undefined; + +// Catches the exits that bypass `stopStack`, such as a Ctrl-C mid-boot in dev.ts. +// Must stay synchronous: async work in an `exit` handler never runs. +process.on('exit', () => { + if (ownedConfigDir) { + fs.rmSync(ownedConfigDir, { recursive: true, force: true }); + } +}); + /** Start the whole stack, tearing down anything already started if a later step fails. */ export async function startStack(options: StartStackOptions = {}): Promise { const { proxy = !!process.env.CI, seed = true } = options; @@ -231,7 +246,12 @@ export async function startStack(options: StartStackOptions = {}): Promise fs.rmSync(configDir, { recursive: true, force: true }), ); + ownedConfigDir = undefined; } if (urlsFile) { await warnOnException('Failed to remove urls file', () => fs.rmSync(urlsFile, { force: true })); From 895c2413430df151303359f05cb6dc4250fa4d07 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 09:14:25 +0000 Subject: [PATCH 322/373] Remove explicit host port listing --- e2e-tests/README.md | 2 +- e2e-tests/setup/constants.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 8d29acbe6cb..29ccd1cbc7e 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -55,7 +55,7 @@ npm run dev:e2e # uses your local dev-nginx for the https://*.m GRID_PROXY=true npm run dev:e2e # no dev-nginx? start the bundled Caddy proxy on :443 instead ``` -The stack binds fixed host ports (9001-9012, 4566, 9008, 9200), so two stacks cannot run +The stack binds fixed host ports, so two stacks cannot run at once. Starting a second one fails immediately rather than timing out. ### Running the tests against a stack you already started diff --git a/e2e-tests/setup/constants.ts b/e2e-tests/setup/constants.ts index ac8ba8429db..1fd2617e88f 100644 --- a/e2e-tests/setup/constants.ts +++ b/e2e-tests/setup/constants.ts @@ -64,8 +64,7 @@ export const MEDIA_API_PORT = SERVICE_PORTS['media-api']; export const URLS_FILE = path.join(import.meta.dirname, '..', '.grid-urls.json'); /** - * Generated per-service config, bind-mounted into the app container. The path is fixed - * rather than unique per run: the fixed host ports already allow only one stack at a time. + * Generated per-service config, bind-mounted into the app container. */ export const CONFIG_DIR = path.join(os.tmpdir(), 'grid-e2e-config'); From afab9271885d71f2ba32d81c7d03487636b0b299 Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Thu, 3 Sep 2026 16:49:20 +0100 Subject: [PATCH 323/373] Do not record download usages from InDesign --- media-api/app/controllers/MediaApi.scala | 56 +++++++++++-------- media-api/test/controllers/MediaApiTest.scala | 37 ++++++++++++ 2 files changed, 71 insertions(+), 22 deletions(-) create mode 100644 media-api/test/controllers/MediaApiTest.scala diff --git a/media-api/app/controllers/MediaApi.scala b/media-api/app/controllers/MediaApi.scala index 1a3017e9495..1f068099831 100644 --- a/media-api/app/controllers/MediaApi.scala +++ b/media-api/app/controllers/MediaApi.scala @@ -35,6 +35,14 @@ import scala.concurrent.duration.DurationInt import scala.concurrent.{ExecutionContext, Future} import scala.util.Try +object MediaApi { + val InDesignIdentity = "InDesign (testing)" + + def shouldSkipUsageRecording(uri: String, user: String): Boolean = + // We don't want to record usages for InDesign downloads for now + uri.contains("download") && user == InDesignIdentity +} + class MediaApi( auth: Authentication, messageSender: ThrallMessageSender, @@ -518,29 +526,33 @@ class MediaApi( user: String, partnerName: Option[String] = None, startPending: Option[String] = None, - )(implicit logMarker: LogMarker) = { - - val baseRequest = ws.url(uri) - .withHttpHeaders(Authentication.originalServiceHeaderName -> config.appName, - HttpHeaders.ORIGIN -> config.rootUri, - HttpHeaders.CONTENT_TYPE -> ContentType.APPLICATION_JSON.getMimeType) - - val request = onBehalfOfPrincipal(baseRequest) - - val usagesMetadata = uri match { - case s if s.contains("download") => Map("mediaId" -> mediaId, - "dateAdded" -> printDateTime(DateTime.now()), - "downloadedBy" -> user) - case s if s.contains("syndication") => Map("mediaId" -> mediaId, - "dateAdded" -> printDateTime(DateTime.now()), - "syndicatedBy" -> user, - "startPending" -> startPending.getOrElse("false"), - "partnerName" -> partnerName.getOrElse( - throw new IllegalArgumentException("partnerName required for SyndicationUsageRequest")) - ) + )(implicit logMarker: LogMarker): Future[Unit] = { + if (MediaApi.shouldSkipUsageRecording(uri, user)) { + logger.info(logMarker, s"Skipping usages request to $uri for $user") + Future.successful(()) + } else { + val baseRequest = ws.url(uri) + .withHttpHeaders(Authentication.originalServiceHeaderName -> config.appName, + HttpHeaders.ORIGIN -> config.rootUri, + HttpHeaders.CONTENT_TYPE -> ContentType.APPLICATION_JSON.getMimeType) + + val request = onBehalfOfPrincipal(baseRequest) + + val usagesMetadata = uri match { + case s if s.contains("download") => Map("mediaId" -> mediaId, + "dateAdded" -> printDateTime(DateTime.now()), + "downloadedBy" -> user) + case s if s.contains("syndication") => Map("mediaId" -> mediaId, + "dateAdded" -> printDateTime(DateTime.now()), + "syndicatedBy" -> user, + "startPending" -> startPending.getOrElse("false"), + "partnerName" -> partnerName.getOrElse( + throw new IllegalArgumentException("partnerName required for SyndicationUsageRequest")) + ) + } + logger.info(logMarker, s"Making usages request to $uri") + request.post(Json.toJson(Map("data" -> usagesMetadata))).map(_ => ()) //fire and forget } - logger.info(logMarker, s"Making usages request to $uri") - request.post(Json.toJson(Map("data" -> usagesMetadata))) //fire and forget } diff --git a/media-api/test/controllers/MediaApiTest.scala b/media-api/test/controllers/MediaApiTest.scala new file mode 100644 index 00000000000..4560b513be8 --- /dev/null +++ b/media-api/test/controllers/MediaApiTest.scala @@ -0,0 +1,37 @@ +package controllers + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +class MediaApiTest extends AnyFunSpec with Matchers { + + describe("MediaApi.shouldSkipUsageRecording") { + it("skips recording when the URI is a download and the user is the InDesign API key") { + MediaApi.shouldSkipUsageRecording( + uri = "https://usage.example.com/usages/download", + user = MediaApi.InDesignIdentity + ) shouldBe true + } + + it("does not skip recording for a download from a real user") { + MediaApi.shouldSkipUsageRecording( + uri = "https://usage.example.com/usages/download", + user = "some-real-user@guardian.co.uk" + ) shouldBe false + } + + it("does not skip recording for a syndication request, even from the InDesign API key") { + MediaApi.shouldSkipUsageRecording( + uri = "https://usage.example.com/usages/syndication", + user = MediaApi.InDesignIdentity + ) shouldBe false + } + + it("does not skip recording for a syndication request from a real user") { + MediaApi.shouldSkipUsageRecording( + uri = "https://usage.example.com/usages/syndication", + user = "some-real-user@guardian.co.uk" + ) shouldBe false + } + } +} From ed615c0dc46970e6c78e69a9ce10ae12b8ca6fff Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 11:25:16 +0000 Subject: [PATCH 324/373] Pre-seed lease table to skip the long wait to establish a lease in e2e-tests --- e2e-tests/package-lock.json | 394 ++++++++++++++++++++++------------- e2e-tests/package.json | 2 + e2e-tests/setup/provision.ts | 75 ++++++- 3 files changed, 324 insertions(+), 147 deletions(-) diff --git a/e2e-tests/package-lock.json b/e2e-tests/package-lock.json index e9556f4aaa2..2c0e1b389c7 100644 --- a/e2e-tests/package-lock.json +++ b/e2e-tests/package-lock.json @@ -10,6 +10,8 @@ "license": "ISC", "devDependencies": { "@aws-sdk/client-cloudformation": "^3.658.0", + "@aws-sdk/client-dynamodb": "^3.1127.0", + "@aws-sdk/client-kinesis": "^3.1127.0", "@aws-sdk/client-s3": "^3.658.0", "@playwright/test": "^1.61.1", "@testcontainers/localstack": "^10.13.2", @@ -59,6 +61,48 @@ "node": ">=20.0.0" } }, + "node_modules/@aws-sdk/client-dynamodb": { + "version": "3.1127.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1127.0.tgz", + "integrity": "sha512-rV9TC7gqtwekWR9jXBdzuvCG65fnv7jphnYdJmFBfLX1b0Q+ma3xijzZaGGeN+US2pKw/XaQiZapJX/nZqtBJw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-node": "^3.972.82", + "@aws-sdk/dynamodb-codec": "^3.973.44", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.30", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-kinesis": { + "version": "3.1127.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kinesis/-/client-kinesis-3.1127.0.tgz", + "integrity": "sha512-8uAuIE20oxRFocQfD71v8xBhrIKaFO8xYsnuKcAuw0OP+5DsXUmDg4lOv8bsdYJPi7tJxE9HkSz7cHxEw+Y5jA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-node": "^3.972.82", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@aws-sdk/client-s3": { "version": "3.1089.0", "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1089.0.tgz", @@ -83,18 +127,18 @@ } }, "node_modules/@aws-sdk/core": { - "version": "3.975.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.3.tgz", - "integrity": "sha512-7ur3kCKuvPLqlsZ2XlvnNBVQ7KkpSu6Y6dOTwSPHLrFpTEfZM8isLBJc4cgv96WB7GifeVM436mpycwxBd2vEA==", + "version": "3.977.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.9.tgz", + "integrity": "sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.974.2", - "@aws-sdk/xml-builder": "^3.972.36", + "@aws-sdk/types": "^3.974.5", + "@aws-sdk/xml-builder": "^3.972.40", "@aws/lambda-invoke-store": "^0.3.0", - "@smithy/core": "^3.29.4", - "@smithy/signature-v4": "^5.6.5", - "@smithy/types": "^4.16.1", + "@smithy/core": "^3.33.3", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", "bowser": "^2.11.0", "tslib": "^2.6.2" }, @@ -103,16 +147,16 @@ } }, "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.59", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.59.tgz", - "integrity": "sha512-Ny5e4Mfh3QPmiAc0AiUe+cbTXDlxkU3Rc+EpWOfyWeWEy6yp7Fa1KmfNeCc+1a8by9zQ9gtohmiQUkMPScF3ng==", + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.70.tgz", + "integrity": "sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -120,18 +164,18 @@ } }, "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.61", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.61.tgz", - "integrity": "sha512-8jAjgStl5Ytq4+HF3X/9f+EmRinaRbGRRtQGktlPfBRVx73H+R1y48vIeXerQtYGFaUqkEp3fT6jP854rVO2yQ==", + "version": "3.972.72", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.72.tgz", + "integrity": "sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/fetch-http-handler": "^5.6.6", - "@smithy/node-http-handler": "^4.9.6", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -139,24 +183,24 @@ } }, "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.973.4", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.4.tgz", - "integrity": "sha512-e6ZvVsj90aRALf1kHP+J4iqC1496ZpVgqI/+u0LJ5HL7q7ATauGy4gdDvRCP13L1pN/fMiZLah162PGIYkbUVQ==", + "version": "3.973.15", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.15.tgz", + "integrity": "sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/credential-provider-env": "^3.972.59", - "@aws-sdk/credential-provider-http": "^3.972.61", - "@aws-sdk/credential-provider-login": "^3.972.66", - "@aws-sdk/credential-provider-process": "^3.972.59", - "@aws-sdk/credential-provider-sso": "^3.973.3", - "@aws-sdk/credential-provider-web-identity": "^3.972.65", - "@aws-sdk/nested-clients": "^3.997.33", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/credential-provider-imds": "^4.4.9", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-login": "^3.972.77", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -164,17 +208,17 @@ } }, "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.66", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.66.tgz", - "integrity": "sha512-g2fsqm87r/nKthLZ0VkkDBElkGg0PvSa8d97HQ6EilMbJTZ6hxa8FxkSZyJfgPfFdZn0TTmkOffQmTSUcAHIng==", + "version": "3.972.77", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.77.tgz", + "integrity": "sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/nested-clients": "^3.997.33", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -182,22 +226,22 @@ } }, "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.70", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.70.tgz", - "integrity": "sha512-3xzvkGdykBunxqh8WudmUpSyLWvIhfI6aBQo1b5rb3mDO5mNLadK+0hiI0qBQBMVynJbfLO+Ajy9dztMwy9O8w==", + "version": "3.972.82", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.82.tgz", + "integrity": "sha512-znDkEOGXB8W3kG1LJUKP3foBZY/9qLM0eil/DxWXSp37XsdsRLQHE/d/OaCGGVgKpA6znR38h/+INk8do1FjiA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.59", - "@aws-sdk/credential-provider-http": "^3.972.61", - "@aws-sdk/credential-provider-ini": "^3.973.4", - "@aws-sdk/credential-provider-process": "^3.972.59", - "@aws-sdk/credential-provider-sso": "^3.973.3", - "@aws-sdk/credential-provider-web-identity": "^3.972.65", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/credential-provider-imds": "^4.4.9", - "@smithy/types": "^4.16.1", + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-ini": "^3.973.15", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -205,16 +249,16 @@ } }, "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.59", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.59.tgz", - "integrity": "sha512-DlZF2/MhLlatDdlrIy3CUCpfdbLrKx+3SMjVo+WyHnPpwzkc/M3vwAHw4OVJf7DMvO+4vfRqSCMc/E9I1auN0g==", + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.70.tgz", + "integrity": "sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -222,18 +266,18 @@ } }, "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.973.3", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.3.tgz", - "integrity": "sha512-hmdDHoy2G5Es2e8IgelNMYUuSQI6uCIAKZMJ2u2PdKDhxvbk1uWD/g4+R7R5c/tJfKEB1+KjjWiaoCr/S+ZTiQ==", + "version": "3.973.14", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.14.tgz", + "integrity": "sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/nested-clients": "^3.997.33", - "@aws-sdk/token-providers": "3.1088.0", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/token-providers": "3.1116.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -241,17 +285,64 @@ } }, "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.65", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.65.tgz", - "integrity": "sha512-gHQb/Kt0chjk/JQDa/GJDqmAvEuVn8n7z10wK2h0LFM9TUDRkohgOO4aEF+s2sBLM0br7Cl5W6P7phgjrrJvLQ==", + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.76.tgz", + "integrity": "sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/nested-clients": "^3.997.33", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/dynamodb-codec": { + "version": "3.973.44", + "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.44.tgz", + "integrity": "sha512-eo27HNeBqMAfy9JBQug6ErU6DeG8OmOH6ou8+KmX3/fWdTmWUsHsfisz2tEQOG+GJ57bBC7kC6AjKAjpcxU4QA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.972.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", + "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.972.30", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.30.tgz", + "integrity": "sha512-0hmD7/NG2NoVOVHvUb6rtY5POQYr+/9gdHvrYhIBA1zmCqKOBd5Gz3dL+iZ15FHOJbs/5kLplGoePc8PHTlzYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/endpoint-cache": "^3.972.11", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -277,19 +368,19 @@ } }, "node_modules/@aws-sdk/nested-clients": { - "version": "3.997.33", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.33.tgz", - "integrity": "sha512-dVZOroI/r3/ENvqNGgjMPul+jjlz9GddfVusgTXlVjfZj5isibOxecLkGQbRPp8XOuX+RAfjXLFgPkD1JS5xrw==", + "version": "3.997.44", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.44.tgz", + "integrity": "sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/signature-v4-multi-region": "^3.996.41", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/fetch-http-handler": "^5.6.6", - "@smithy/node-http-handler": "^4.9.6", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/signature-v4-multi-region": "^3.996.46", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -297,15 +388,15 @@ } }, "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.41", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.41.tgz", - "integrity": "sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==", + "version": "3.996.46", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.46.tgz", + "integrity": "sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/types": "^3.974.2", - "@smithy/signature-v4": "^5.6.5", - "@smithy/types": "^4.16.1", + "@aws-sdk/types": "^3.974.5", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -313,17 +404,17 @@ } }, "node_modules/@aws-sdk/token-providers": { - "version": "3.1088.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1088.0.tgz", - "integrity": "sha512-4ObatWt2qpJg5FBk4LOOKrTQYzaqeewAtdO3r9ZO8lH9YqLtpTzLyIdy0mJ+nVdfYOnqISkKNfmzP22bNDhwyw==", + "version": "3.1116.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1116.0.tgz", + "integrity": "sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@aws-sdk/core": "^3.975.3", - "@aws-sdk/nested-clients": "^3.997.33", - "@aws-sdk/types": "^3.974.2", - "@smithy/core": "^3.29.4", - "@smithy/types": "^4.16.1", + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -331,13 +422,13 @@ } }, "node_modules/@aws-sdk/types": { - "version": "3.974.2", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.2.tgz", - "integrity": "sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==", + "version": "3.974.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.5.tgz", + "integrity": "sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.16.1", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -345,13 +436,13 @@ } }, "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.36", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.36.tgz", - "integrity": "sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==", + "version": "3.972.40", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.40.tgz", + "integrity": "sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.16.1", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -693,13 +784,13 @@ "license": "BSD-3-Clause" }, "node_modules/@smithy/core": { - "version": "3.29.5", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.5.tgz", - "integrity": "sha512-i0dk2t5B+CwV/dcJdUHILYkOQF5lof8f44dFCfDWToGCxjT9YQ+CgHqTAvJxzc3+zqQwm2QtVoJ5IqiNar/CnQ==", + "version": "3.33.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.3.tgz", + "integrity": "sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@smithy/types": "^4.16.1", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -707,14 +798,14 @@ } }, "node_modules/@smithy/credential-provider-imds": { - "version": "4.4.10", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.10.tgz", - "integrity": "sha512-MJenAe4OKRZUo1LdYYFDCsSHxaHvInIU/z52GsheO9vl1/VSySVCr0zkyKD6TFiGkSUaWGxvKZ/70OvgUZR5HQ==", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.29.5", - "@smithy/types": "^4.16.1", + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -722,14 +813,14 @@ } }, "node_modules/@smithy/fetch-http-handler": { - "version": "5.6.7", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.7.tgz", - "integrity": "sha512-3zpg8yqqyXzoK2TsRDdkqVOj2RDBFfLXwCczOZ5c7TWB4eiaebfSCsbMjDPYB3PJ9ihV62QaeadZ+wLadZtNGA==", + "version": "5.8.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.8.0.tgz", + "integrity": "sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.29.5", - "@smithy/types": "^4.16.1", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", "tslib": "^2.6.2" }, "engines": { @@ -737,14 +828,14 @@ } }, "node_modules/@smithy/node-http-handler": { - "version": "4.9.7", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.7.tgz", - "integrity": "sha512-wCU8HCLjAtAVqxxe0j2xff9LcEPw3yjBbg5IdQDIYFnxnPxbxcSLc7rgex7kqm9L/WYOnJEgaWQlfDkZleozMA==", + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.12.1.tgz", + "integrity": "sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.29.5", - "@smithy/types": "^4.16.1", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.18.0", "tslib": "^2.6.2" }, "engines": { @@ -752,14 +843,14 @@ } }, "node_modules/@smithy/signature-v4": { - "version": "5.6.6", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.6.tgz", - "integrity": "sha512-efP6DN3UTFrzIsGO42/xcabv8jU7+9nwEdphFUH7yL0k010ERyAWaO41KFQIDLcFZLZ8xzIQr4wplFxNzslSGQ==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.3.tgz", + "integrity": "sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==", "dev": true, "license": "Apache-2.0", "dependencies": { - "@smithy/core": "^3.29.5", - "@smithy/types": "^4.16.1", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", "tslib": "^2.6.2" }, "engines": { @@ -767,9 +858,9 @@ } }, "node_modules/@smithy/types": { - "version": "4.16.1", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", - "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", + "version": "4.18.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.18.0.tgz", + "integrity": "sha512-CgB6HHWer/vrKps24ulRIbpcpb7K4xAU7SkZ7YHzBPlwHsvsrCJFEXK421s+cJzX+ZrqtA/TuU5w1HzI7k9N8A==", "dev": true, "license": "Apache-2.0", "dependencies": { @@ -2083,6 +2174,16 @@ "dev": true, "license": "MIT" }, + "node_modules/mnemonist": { + "version": "0.38.3", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", + "dev": true, + "license": "MIT", + "dependencies": { + "obliterator": "^1.6.1" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -2108,6 +2209,13 @@ "node": ">=0.10.0" } }, + "node_modules/obliterator": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", + "dev": true, + "license": "MIT" + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 4c6109dd023..91399f8863c 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -20,6 +20,8 @@ "license": "ISC", "devDependencies": { "@aws-sdk/client-cloudformation": "^3.658.0", + "@aws-sdk/client-dynamodb": "^3.1127.0", + "@aws-sdk/client-kinesis": "^3.1127.0", "@aws-sdk/client-s3": "^3.658.0", "@playwright/test": "^1.61.1", "@testcontainers/localstack": "^10.13.2", diff --git a/e2e-tests/setup/provision.ts b/e2e-tests/setup/provision.ts index 0c5828d9378..608fd50c837 100644 --- a/e2e-tests/setup/provision.ts +++ b/e2e-tests/setup/provision.ts @@ -6,6 +6,14 @@ import { DescribeStackResourcesCommand, waitUntilStackCreateComplete, } from '@aws-sdk/client-cloudformation'; +import { + CreateTableCommand, + DynamoDBClient, + PutItemCommand, + ScanCommand, + waitUntilTableExists, +} from '@aws-sdk/client-dynamodb'; +import { KinesisClient, ListShardsCommand } from '@aws-sdk/client-kinesis'; import { CreateBucketCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { API_KEY as API_KEY_PATH, CORE_STACK_NAME, PERMISSIONS_BUCKET, REGION, REPO_ROOT } from './constants.ts'; @@ -21,7 +29,9 @@ function clients(endpoint: string) { credentials: CREDENTIALS, forcePathStyle: true, }); - return { cfn, s3 }; + const dynamo = new DynamoDBClient({ endpoint, region: REGION, credentials: CREDENTIALS }); + const kinesis = new KinesisClient({ endpoint, region: REGION, credentials: CREDENTIALS }); + return { cfn, s3, dynamo, kinesis }; } /** @@ -104,14 +114,71 @@ async function provisionPermissionsBucket(s3: S3Client): Promise { return PERMISSIONS_BUCKET; } +/** + * Pre-create the KCL lease table for a stream, with an unowned lease per shard. + * + * Thrall consumes each stream with the Kinesis Client Library, which names its DynamoDB + * lease table after the KCL application name — and thrall sets that to the stream name. + * Left to itself on a cold stack, KCL spends ~70s before it reads a single record: + * + * - `Scheduler.initialize()` calls `shouldInitiateLeaseSync()`, which sleeps a random + * 1-30s (polling every 3s) for as long as the lease table is empty. It is anti-stampede + * jitter for a real fleet, is not configurable, and is pure cost for a single worker. + * - It then runs the initial shard sync synchronously, and `ShardSyncTask` finishes with + * `Thread.sleep(shardSyncIntervalMillis)` — 60s on KCL's defaults. + * + * Seeding the leases here makes `isLeaseTableEmpty()` false on the very first check, so + * neither the jitter nor the shard sync runs: the lease taker picks up the unowned leases + * on its first pass instead. + */ +async function seedKclLeaseTable( + dynamo: DynamoDBClient, + kinesis: KinesisClient, + streamName: string, +): Promise { + await dynamo.send( + new CreateTableCommand({ + TableName: streamName, + KeySchema: [{ AttributeName: 'leaseKey', KeyType: 'HASH' }], + AttributeDefinitions: [{ AttributeName: 'leaseKey', AttributeType: 'S' }], + BillingMode: 'PAY_PER_REQUEST', + }), + ); + await waitUntilTableExists({ client: dynamo, maxWaitTime: 60 }, { TableName: streamName }); + + const { Shards = [] } = await kinesis.send(new ListShardsCommand({ StreamName: streamName })); + + for (const shard of Shards) { + await dynamo.send( + new PutItemCommand({ + TableName: streamName, + Item: { + leaseKey: { S: shard.ShardId! }, + leaseCounter: { N: '0' }, + checkpoint: { S: 'TRIM_HORIZON' }, + checkpointSubSequenceNumber: { N: '0' }, + ownerSwitchesSinceCheckpoint: { N: '0' }, + // Without the hash range KCL's lease auditor reports the stream as having holes. + startingHashKey: { S: shard.HashKeyRange!.StartingHashKey! }, + endingHashKey: { S: shard.HashKeyRange!.EndingHashKey! }, + }, + }), + ); + } +} + /** * Apply the core stack and seed its buckets. Returns the LogicalResourceId -> * PhysicalResourceId map used to generate service config. - */ -export async function provisionCoreStack(localstackEndpoint: string): Promise { - const { cfn, s3 } = clients(localstackEndpoint); + */export async function provisionCoreStack(localstackEndpoint: string): Promise { + const { cfn, s3, dynamo, kinesis } = clients(localstackEndpoint); const props = await createCoreStack(cfn); await seedBuckets(s3, props); props.PermissionsBucket = await provisionPermissionsBucket(s3); + + for (const stream of [props.ThrallMessageStream, props.ThrallLowPriorityMessageStream]) { + await seedKclLeaseTable(dynamo, kinesis, stream); + } + return props; } From ac1daa11c2f7c62a43120e3ba752bf0a53128199 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 12:48:37 +0000 Subject: [PATCH 325/373] Use listr2 to manage task DAG and give nice output, removing the URL task in the process (it's redundant after moving to static ports) --- .gitignore | 3 - e2e-tests/package-lock.json | 341 +++++++++++++++ e2e-tests/package.json | 1 + e2e-tests/setup/constants.ts | 3 - e2e-tests/setup/dev.ts | 17 +- e2e-tests/setup/progress.ts | 39 ++ e2e-tests/setup/provision.ts | 96 ++--- e2e-tests/setup/seed-elasticsearch.ts | 10 +- e2e-tests/setup/stack.ts | 594 +++++++++++++++++--------- e2e-tests/setup/state.ts | 2 - e2e-tests/steps/fixtures.ts | 29 +- 11 files changed, 842 insertions(+), 293 deletions(-) create mode 100644 e2e-tests/setup/progress.ts diff --git a/.gitignore b/.gitignore index e28f06d7bb7..5b6dcd2b913 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,3 @@ image-embedder-lambda/__tests__/embedder/test-data /e2e-tests/blob-report/ /e2e-tests/playwright/.cache/ /e2e-tests/playwright/.auth/ - -# Testcontainers global-setup artifacts -/e2e-tests/.grid-urls.json diff --git a/e2e-tests/package-lock.json b/e2e-tests/package-lock.json index 2c0e1b389c7..2512b809255 100644 --- a/e2e-tests/package-lock.json +++ b/e2e-tests/package-lock.json @@ -17,6 +17,7 @@ "@testcontainers/localstack": "^10.13.2", "@types/node": "^26.1.1", "json5": "^2.2.3", + "listr2": "^10.1.2", "playwright-bdd": "^9.2.0", "testcontainers": "^10.13.2" }, @@ -973,6 +974,22 @@ "node": ">=6.5" } }, + "node_modules/ansi-escapes": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", + "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", + "dev": true, + "license": "MIT", + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/ansi-regex": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", @@ -1344,6 +1361,22 @@ "dev": true, "license": "MIT" }, + "node_modules/cli-cursor": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-5.0.0.tgz", + "integrity": "sha512-aCj4O5wKyszjMmDT4tZj93kxyydN/K5zPWSCe6/0AV/AA1pqe5ZBIw0a2ZfPQV7lL5/yb5HsUreJ6UFAF1tEQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "restore-cursor": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/cli-table3": { "version": "0.6.5", "resolved": "https://registry.npmjs.org/cli-table3/-/cli-table3-0.6.5.tgz", @@ -1405,6 +1438,40 @@ "node": ">=8" } }, + "node_modules/cli-truncate": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-5.2.0.tgz", + "integrity": "sha512-xRwvIOMGrfOAnM1JYtqQImuaNtDEv9v6oIYAs4LIHwTiKee8uwvIi363igssOC0O5U04i4AlENs79LQLu9tEMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "slice-ansi": "^8.0.0", + "string-width": "^8.2.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-truncate/node_modules/string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -1519,6 +1586,13 @@ "dev": true, "license": "MIT" }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true, + "license": "MIT" + }, "node_modules/commander": { "version": "13.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz", @@ -1760,6 +1834,19 @@ "once": "^1.4.0" } }, + "node_modules/environment": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", + "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/escalade": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", @@ -1780,6 +1867,13 @@ "node": ">=6" } }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "dev": true, + "license": "MIT" + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -1874,6 +1968,19 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/get-port": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/get-port/-/get-port-7.2.0.tgz", @@ -2056,6 +2163,58 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/listr2": { + "version": "10.1.2", + "resolved": "https://registry.npmjs.org/listr2/-/listr2-10.1.2.tgz", + "integrity": "sha512-ENXj5KYVtdZigbaWY0+lUCzRZNvKDEbxree+93oaCeSz8GICxgJyOMi6U9mHuiFd3hJ9y0RIDQh8RYttBKOnaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "cli-truncate": "^5.2.0", + "colorette": "^2.0.20", + "eventemitter3": "^5.0.1", + "log-update": "^6", + "rfdc": "^1.4.1", + "wrap-ansi": "^10.0.0" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/listr2/node_modules/string-width": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.2.2.tgz", + "integrity": "sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.5.0", + "strip-ansi": "^7.1.2" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/listr2/node_modules/wrap-ansi": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-10.0.1.tgz", + "integrity": "sha512-M0N4xzyzosiIok3svYlEo1sdLZts/8FPgYH/GPC3wvlmPoRvnoManGMrE54waYj3tISA8w6lsdesfVv67qSr8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.3", + "string-width": "^8.2.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/lodash": { "version": "4.18.1", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", @@ -2077,6 +2236,102 @@ "dev": true, "license": "MIT" }, + "node_modules/log-update": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/log-update/-/log-update-6.1.0.tgz", + "integrity": "sha512-9ie8ItPR6tjY5uYJh8K/Zrv/RMZ5VOlOWvtZdEHYSTFKZfIBPQa9tOAEeAWhd+AnIneLJ22w5fjOYtoutpWq5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-escapes": "^7.0.0", + "cli-cursor": "^5.0.0", + "slice-ansi": "^7.1.0", + "strip-ansi": "^7.1.0", + "wrap-ansi": "^9.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "dev": true, + "license": "MIT" + }, + "node_modules/log-update/node_modules/is-fullwidth-code-point": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", + "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/slice-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.2.tgz", + "integrity": "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "is-fullwidth-code-point": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/log-update/node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-update/node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/long": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", @@ -2128,6 +2383,19 @@ "url": "https://opencollective.com/express" } }, + "node_modules/mimic-function": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/mimic-function/-/mimic-function-5.0.1.tgz", + "integrity": "sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/minimatch": { "version": "9.0.9", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.9.tgz", @@ -2226,6 +2494,22 @@ "wrappy": "1" } }, + "node_modules/onetime": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-7.0.0.tgz", + "integrity": "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-function": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/package-json-from-dist": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.1.tgz", @@ -2504,6 +2788,23 @@ "node": ">=0.10.0" } }, + "node_modules/restore-cursor": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-5.1.0.tgz", + "integrity": "sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==", + "dev": true, + "license": "MIT", + "dependencies": { + "onetime": "^7.0.0", + "signal-exit": "^4.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", @@ -2514,6 +2815,13 @@ "node": ">= 4" } }, + "node_modules/rfdc": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.4.1.tgz", + "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", + "dev": true, + "license": "MIT" + }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -2578,6 +2886,39 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/slice-ansi": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-8.0.0.tgz", + "integrity": "sha512-stxByr12oeeOyY2BlviTNQlYV5xOj47GirPr4yA1hE9JCtxfQN0+tVbkxwCtYDQWhEKWFHsEK48ORg5jrouCAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.3", + "is-fullwidth-code-point": "^5.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", + "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 91399f8863c..4887d15bb6f 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -27,6 +27,7 @@ "@testcontainers/localstack": "^10.13.2", "@types/node": "^26.1.1", "json5": "^2.2.3", + "listr2": "^10.1.2", "playwright-bdd": "^9.2.0", "testcontainers": "^10.13.2" } diff --git a/e2e-tests/setup/constants.ts b/e2e-tests/setup/constants.ts index 1fd2617e88f..5c001d5e77f 100644 --- a/e2e-tests/setup/constants.ts +++ b/e2e-tests/setup/constants.ts @@ -60,9 +60,6 @@ export const SERVICE_PORTS: Record = { export const KAHUNA_PORT = SERVICE_PORTS.kahuna; export const MEDIA_API_PORT = SERVICE_PORTS['media-api']; -/** File (repo-relative to e2e-tests) where global-setup records the resolved service URLs. */ -export const URLS_FILE = path.join(import.meta.dirname, '..', '.grid-urls.json'); - /** * Generated per-service config, bind-mounted into the app container. */ diff --git a/e2e-tests/setup/dev.ts b/e2e-tests/setup/dev.ts index d21a494ea7b..457f647ef00 100644 --- a/e2e-tests/setup/dev.ts +++ b/e2e-tests/setup/dev.ts @@ -8,15 +8,13 @@ import { DOMAIN, ELASTICSEARCH_PORT, - GRID_IMAGE, LOCALSTACK_PORT, SERVICE_PORTS, - URLS_FILE, } from './constants.ts'; import { probeStack, startStack, stopStack } from './stack.ts'; import type { GridEnvironment } from './state.ts'; -function banner(environment: GridEnvironment): string { +function banner(elapsedMs: number): string { const services = Object.keys(SERVICE_PORTS) .sort() .map((service) => ` ${service.padEnd(16)} http://localhost:${SERVICE_PORTS[service]}`) @@ -24,10 +22,7 @@ function banner(environment: GridEnvironment): string { return [ '', - ' Grid is up.', - '', - ` kahuna (UI) https://media.${DOMAIN}`, - ` media-api ${environment.mediaApiUrl}`, + ` Grid is up in ${Math.round(elapsedMs / 1000)}s at https://media.${DOMAIN}`, '', ' Services:', services, @@ -36,11 +31,6 @@ function banner(environment: GridEnvironment): string { ` localstack http://localhost:${LOCALSTACK_PORT}`, ` elasticsearch http://localhost:${ELASTICSEARCH_PORT}`, '', - ` Image: ${GRID_IMAGE}`, - ` Config: ${environment.configDir}`, - ` URLs: ${URLS_FILE}`, - ` Domains: https://media.${DOMAIN} (via dev-nginx or GRID_PROXY=true)`, - '', ' Press Ctrl-C to tear everything down.', '', ].join('\n'); @@ -81,8 +71,9 @@ async function dev(): Promise { ); } + const startedAt = Date.now(); environment = await startStack({ proxy: process.env.GRID_PROXY === 'true' }); - console.log(banner(environment)); + console.log(banner(Date.now() - startedAt)); // Hold the process open indefinitely. setInterval(() => {}, 2_000_000_000); diff --git a/e2e-tests/setup/progress.ts b/e2e-tests/setup/progress.ts new file mode 100644 index 00000000000..a4db342d513 --- /dev/null +++ b/e2e-tests/setup/progress.ts @@ -0,0 +1,39 @@ +/** + * Renders long-running work as a listr2 task tree, so booting or tearing down the stack + * shows which container or gate is currently being waited on instead of a silent terminal. + * + * Interactive terminals get the animated renderer; CI and piped output fall back to the + * line-oriented `simple` renderer, which prints one line per task transition. + */ +import { Listr, PRESET_TIMER } from 'listr2'; +import type { ListrTask } from 'listr2'; + +export type { ListrTask }; + +export interface RunTasksOptions { + /** Keep going after a failed task. Used by teardown, where every step is best-effort. */ + exitOnError?: boolean; +} + +/** Run `tasks` against `context`, which they mutate as they go, and return it. */ +export async function runTasks( + tasks: ListrTask[], + context: Ctx, + options: RunTasksOptions = {}, +): Promise { + const runner = new Listr(tasks, { + ctx: context, + exitOnError: options.exitOnError ?? true, + // The animated renderer redraws in place, which CI logs record as a wall of escape codes. + fallbackRendererCondition: () => !!process.env.CI, + rendererOptions: { + timer: PRESET_TIMER, + // Keep finished containers and services on screen: the completed list is the summary. + collapseSubtasks: false, + collapseErrors: false, + }, + fallbackRendererOptions: { timer: PRESET_TIMER }, + }); + + return runner.run(); +} diff --git a/e2e-tests/setup/provision.ts b/e2e-tests/setup/provision.ts index 608fd50c837..33af4d8ef9b 100644 --- a/e2e-tests/setup/provision.ts +++ b/e2e-tests/setup/provision.ts @@ -19,9 +19,9 @@ import { API_KEY as API_KEY_PATH, CORE_STACK_NAME, PERMISSIONS_BUCKET, REGION, R const CREDENTIALS = { accessKeyId: 'test', secretAccessKey: 'test' }; -type StackProps = Record; +export type StackProps = Record; -function clients(endpoint: string) { +export function provisioningClients(endpoint: string) { const cfn = new CloudFormationClient({ endpoint, region: REGION, credentials: CREDENTIALS }); const s3 = new S3Client({ endpoint, @@ -39,15 +39,16 @@ function clients(endpoint: string) { * core stack, waits for completion, reads the created resource names, and seeds the * buckets with the config files the services expect (similar to dev/script/setup.sh). */ -async function createCoreStack(cfn: CloudFormationClient): Promise { +export async function createCoreStack(cfn: CloudFormationClient): Promise { const templateBody = fs.readFileSync( path.join(REPO_ROOT, 'dev', 'cloudformation', 'grid-dev-core.yml'), 'utf8', ); await cfn.send(new CreateStackCommand({ StackName: CORE_STACK_NAME, TemplateBody: templateBody })); + // LocalStack applies the stack in seconds; the SDK default would sit out its 30s minimum delay. await waitUntilStackCreateComplete( - { client: cfn, maxWaitTime: 180 }, + { client: cfn, maxWaitTime: 180, minDelay: 1, maxDelay: 5 }, { StackName: CORE_STACK_NAME }, ); @@ -72,23 +73,23 @@ async function putObject( await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })); } -async function seedBuckets(s3: S3Client, props: StackProps): Promise { +export async function seedBuckets(s3: S3Client, props: StackProps): Promise { const devConfig = path.join(REPO_ROOT, 'dev', 'config'); - // API key used by the machine authentication provider. - await putObject(s3, props.KeyBucket, API_KEY_PATH, 'DEV Key'); - - // Static config consumed by the services. - for (const file of ['photographers.json', 'rcs-quota.json', 'usage_rights.json']) { - await putObject(s3, props.ConfigBucket, file, fs.readFileSync(path.join(devConfig, file))); - } - - await putObject( - s3, - props.UsageMailBucket, - 'usages.eml', - fs.readFileSync(path.join(devConfig, 'usages.eml')), - ); + await Promise.all([ + // API key used by the machine authentication provider. + putObject(s3, props.KeyBucket, API_KEY_PATH, 'DEV Key'), + // Static config consumed by the services. + ...['photographers.json', 'rcs-quota.json', 'usage_rights.json'].map((file) => + putObject(s3, props.ConfigBucket, file, fs.readFileSync(path.join(devConfig, file))), + ), + putObject( + s3, + props.UsageMailBucket, + 'usages.eml', + fs.readFileSync(path.join(devConfig, 'usages.eml')), + ), + ]); } /** @@ -96,7 +97,7 @@ async function seedBuckets(s3: S3Client, props: StackProps): Promise { * permissions fixture so the real authorisation provider can read `permissions.json`. * Returns the bucket name so it can be added to the stack props map. */ -async function provisionPermissionsBucket(s3: S3Client): Promise { +export async function provisionPermissionsBucket(s3: S3Client): Promise { await s3.send( new CreateBucketCommand({ Bucket: PERMISSIONS_BUCKET, @@ -131,7 +132,7 @@ async function provisionPermissionsBucket(s3: S3Client): Promise { * neither the jitter nor the shard sync runs: the lease taker picks up the unowned leases * on its first pass instead. */ -async function seedKclLeaseTable( +export async function seedKclLeaseTable( dynamo: DynamoDBClient, kinesis: KinesisClient, streamName: string, @@ -144,41 +145,28 @@ async function seedKclLeaseTable( BillingMode: 'PAY_PER_REQUEST', }), ); - await waitUntilTableExists({ client: dynamo, maxWaitTime: 60 }, { TableName: streamName }); + await waitUntilTableExists({ client: dynamo, maxWaitTime: 60, minDelay: 1 }, { TableName: streamName }); const { Shards = [] } = await kinesis.send(new ListShardsCommand({ StreamName: streamName })); - for (const shard of Shards) { - await dynamo.send( - new PutItemCommand({ - TableName: streamName, - Item: { - leaseKey: { S: shard.ShardId! }, - leaseCounter: { N: '0' }, - checkpoint: { S: 'TRIM_HORIZON' }, - checkpointSubSequenceNumber: { N: '0' }, - ownerSwitchesSinceCheckpoint: { N: '0' }, - // Without the hash range KCL's lease auditor reports the stream as having holes. - startingHashKey: { S: shard.HashKeyRange!.StartingHashKey! }, - endingHashKey: { S: shard.HashKeyRange!.EndingHashKey! }, - }, - }), - ); - } + await Promise.all( + Shards.map((shard) => + dynamo.send( + new PutItemCommand({ + TableName: streamName, + Item: { + leaseKey: { S: shard.ShardId! }, + leaseCounter: { N: '0' }, + checkpoint: { S: 'TRIM_HORIZON' }, + checkpointSubSequenceNumber: { N: '0' }, + ownerSwitchesSinceCheckpoint: { N: '0' }, + // Without the hash range KCL's lease auditor reports the stream as having holes. + startingHashKey: { S: shard.HashKeyRange!.StartingHashKey! }, + endingHashKey: { S: shard.HashKeyRange!.EndingHashKey! }, + }, + }), + ), + ), + ); } -/** - * Apply the core stack and seed its buckets. Returns the LogicalResourceId -> - * PhysicalResourceId map used to generate service config. - */export async function provisionCoreStack(localstackEndpoint: string): Promise { - const { cfn, s3, dynamo, kinesis } = clients(localstackEndpoint); - const props = await createCoreStack(cfn); - await seedBuckets(s3, props); - props.PermissionsBucket = await provisionPermissionsBucket(s3); - - for (const stream of [props.ThrallMessageStream, props.ThrallLowPriorityMessageStream]) { - await seedKclLeaseTable(dynamo, kinesis, stream); - } - - return props; -} diff --git a/e2e-tests/setup/seed-elasticsearch.ts b/e2e-tests/setup/seed-elasticsearch.ts index 7b6c4ed4feb..e6778c79586 100644 --- a/e2e-tests/setup/seed-elasticsearch.ts +++ b/e2e-tests/setup/seed-elasticsearch.ts @@ -67,15 +67,19 @@ async function waitForAlias(esBaseUrl: string, timeoutMs = 60_000): Promise { +export async function seedElasticsearch( + esBaseUrl: string, + report: (message: string) => void = console.log, +): Promise { const fixture = JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) as EsSearchFixture; const hits = fixture.body?.hits?.hits ?? []; if (hits.length === 0) { - console.warn('No image fixtures found to seed into Elasticsearch'); + report('No image fixtures found to seed into Elasticsearch'); return; } + report(`Waiting for the '${IMAGES_ALIAS}' alias`); await waitForAlias(esBaseUrl); // NDJSON bulk body: an index action line (targeting the document's `_id`) followed by @@ -107,5 +111,5 @@ export async function seedElasticsearch(esBaseUrl: string): Promise { throw new Error(`Elasticsearch bulk seed reported errors: ${JSON.stringify(firstError)}`); } - console.log(`Seeded ${hits.length} image fixture(s) into Elasticsearch alias '${IMAGES_ALIAS}'`); + report(`Seeded ${hits.length} image fixture(s) into Elasticsearch alias '${IMAGES_ALIAS}'`); } diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index 9330ae051d7..cf44f5e6dda 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -13,6 +13,7 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { LocalstackContainer } from '@testcontainers/localstack'; +import type { StartedLocalStackContainer } from '@testcontainers/localstack'; import { GenericContainer, Network, Wait } from 'testcontainers'; import type { StartedNetwork, StartedTestContainer } from 'testcontainers'; import { @@ -37,10 +38,18 @@ import { REGION, REPO_ROOT, SERVICE_PORTS, - URLS_FILE, } from './constants.ts'; import { generateServiceConfig } from './config.ts'; -import { provisionCoreStack } from './provision.ts'; +import { runTasks } from './progress.ts'; +import type { ListrTask } from './progress.ts'; +import { + createCoreStack, + provisioningClients, + provisionPermissionsBucket, + seedBuckets, + seedKclLeaseTable, +} from './provision.ts'; +import type { StackProps } from './provision.ts'; import { seedElasticsearch } from './seed-elasticsearch.ts'; import type { GridEnvironment } from './state.ts'; @@ -68,14 +77,30 @@ export interface StartStackOptions { /** How much of the stack is already listening on the fixed host ports. */ type StackProbe = 'none' | 'healthy' | 'partial'; +const PROBE_OUTCOMES: Record = { + none: 'No running Grid stack found', + healthy: `Found a Grid stack already running on http://localhost:${KAHUNA_PORT}`, + partial: 'Found a partially running Grid stack', +}; + const PROBE_TIMEOUT_MS = 2_000; +/** Names a container by its role in the stack, since Docker otherwise assigns a random one. */ +const ROLE_LABEL = 'uk.co.guardian.grid.role'; + /** The subset of a started stack that teardown needs; a partially-booted stack also fits. */ interface StoppableStack { network?: StartedNetwork; containers: StartedTestContainer[]; configDir?: string; - urlsFile?: string; +} + +/** What the boot tasks build up. Each task mutates it in place for the ones that follow. */ +interface BootContext extends StoppableStack { + elasticsearch?: StartedTestContainer; + localstack?: StartedLocalStackContainer; + grid?: StartedTestContainer; + coreStackProps?: StackProps; } /** @@ -179,183 +204,340 @@ process.on('exit', () => { } }); -/** Start the whole stack, tearing down anything already started if a later step fails. */ -export async function startStack(options: StartStackOptions = {}): Promise { - const { proxy = !!process.env.CI, seed = true } = options; +function elasticsearchContainer(network: StartedNetwork): GenericContainer { + return new GenericContainer(ELASTICSEARCH_IMAGE) + .withNetwork(network) + .withNetworkAliases(ELASTICSEARCH_ALIAS) + .withLabels({ [ROLE_LABEL]: 'Elasticsearch' }) + .withEnvironment({ + 'discovery.type': 'single-node', + 'xpack.security.enabled': 'false', + ES_JAVA_OPTS: '-Xms1024m -Xmx1024m', + }) + .withExposedPorts({ container: ELASTICSEARCH_PORT, host: ELASTICSEARCH_PORT }) + .withWaitStrategy( + Wait.forHttp('/_cluster/health', ELASTICSEARCH_PORT).forStatusCodeMatching((code) => code < 300), + ) + .withStartupTimeout(180_000); +} - const started: StartedTestContainer[] = []; - const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); +function localstackContainer(network: StartedNetwork): LocalstackContainer { + return new LocalstackContainer(LOCALSTACK_IMAGE) + .withNetwork(network) + .withNetworkAliases(LOCALSTACK_ALIAS) + .withLabels({ [ROLE_LABEL]: 'LocalStack' }) + // Pin to the fixed host port dev-nginx expects for the S3 vanity domains + // (images.media / public.media / localstack.media -> 4566). + .withExposedPorts({ container: LOCALSTACK_PORT, host: LOCALSTACK_PORT }) + .withEnvironment({ + SERVICES: LOCALSTACK_SERVICES, + DEFAULT_REGION: REGION, + KINESIS_ERROR_PROBABILITY: '0.0', + // Make resource URLs resolve via the network alias so the + // app container can reach them, and keep queue URLs path-style. + LOCALSTACK_HOST: `${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`, + SQS_ENDPOINT_STRATEGY: 'path', + }) + .withStartupTimeout(120_000); +} - let network: StartedNetwork | undefined; - let configDir: string | undefined; +/** + * imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to + * the `localstack` alias on 4566, so it shares the stack network. + */ +function imgopsContainer(image: GenericContainer, network: StartedNetwork): GenericContainer { + return image + .withNetwork(network) + .withNetworkAliases(IMGOPS_ALIAS) + .withLabels({ [ROLE_LABEL]: 'imgops' }) + .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) + .withExposedPorts({ container: 80, host: IMGOPS_PORT }) + .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) + .withStartupTimeout(120_000); +} - try { - network = await new Network().start(); - - // Infrastructure: Elasticsearch + LocalStack - const elasticsearch = await new GenericContainer(ELASTICSEARCH_IMAGE) - .withNetwork(network) - .withNetworkAliases(ELASTICSEARCH_ALIAS) - .withEnvironment({ - 'discovery.type': 'single-node', - 'xpack.security.enabled': 'false', - ES_JAVA_OPTS: '-Xms1024m -Xmx1024m', - }) - .withExposedPorts({ container: ELASTICSEARCH_PORT, host: ELASTICSEARCH_PORT }) - .withWaitStrategy( - Wait.forHttp('/_cluster/health', ELASTICSEARCH_PORT).forStatusCodeMatching((code) => code < 300), - ) - .withStartupTimeout(180_000) - .start(); - started.push(elasticsearch); - - const localstack = await new LocalstackContainer(LOCALSTACK_IMAGE) - .withNetwork(network) - .withNetworkAliases(LOCALSTACK_ALIAS) - // Pin to the fixed host port dev-nginx expects for the S3 vanity domains - // (images.media / public.media / localstack.media -> 4566). - .withExposedPorts({ container: LOCALSTACK_PORT, host: LOCALSTACK_PORT }) - .withEnvironment({ - SERVICES: LOCALSTACK_SERVICES, - DEFAULT_REGION: REGION, - KINESIS_ERROR_PROBABILITY: '0.0', - // Make resource URLs resolve via the network alias so the - // app container can reach them, and keep queue URLs path-style. - LOCALSTACK_HOST: `${LOCALSTACK_ALIAS}:${LOCALSTACK_PORT}`, - SQS_ENDPOINT_STRATEGY: 'path', - }) - .withStartupTimeout(120_000) - .start(); - started.push(localstack); - - // imgops: standalone nginx image resizer, built from dev/imgops. Its nginx.conf proxies to - // the `localstack` alias on 4566, so it shares this network. - const imgopsImage = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build(IMGOPS_IMAGE, { - deleteOnExit: false, - }); - const imgops = await imgopsImage - .withNetwork(network) - .withNetworkAliases(IMGOPS_ALIAS) - .withCopyFilesToContainer([{ source: IMGOPS_NGINX_CONF, target: '/etc/nginx/nginx.conf' }]) - .withExposedPorts({ container: 80, host: IMGOPS_PORT }) - .withWaitStrategy(Wait.forHttp('/_', 80).forStatusCode(200)) - .withStartupTimeout(120_000) - .start(); - started.push(imgops); - - // Provisioning + config generation - const coreStackProps = await provisionCoreStack(localstack.getConnectionUri()); - - // Reaching here means the probe found no live stack, so recreating the shared path is - // safe and clears anything a killed run left behind. - configDir = CONFIG_DIR; - fs.rmSync(configDir, { recursive: true, force: true }); - fs.mkdirSync(configDir, { recursive: true }); - ownedConfigDir = configDir; - generateServiceConfig(configDir, coreStackProps); - - // All Grid services under test run inside this single container and talk to each - // other over its localhost. Each is published on the fixed host port its - // dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's - // dev-nginx routes the https://*.media. domains straight into this - // container. - let gridBuilder = new GenericContainer(GRID_IMAGE) - .withNetwork(network) - .withNetworkAliases(GRID_ALIAS) - .withExposedPorts( - ...Object.values(SERVICE_PORTS).map((port) => ({ container: port, host: port })), - ) - .withBindMounts([ - // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. - { source: configDir, target: '/root/.grid', mode: 'ro' }, - { source: configDir, target: '/etc/grid', mode: 'ro' }, - // Outside CI the grid-e2e-dev image runs services under sbt; mount the repo - // over /build so host edits recompile live. - ...(process.env.CI ? [] : [{ source: REPO_ROOT, target: '/build', mode: 'rw' as const }]), - ]) - .withEnvironment({ - AWS_ACCESS_KEY_ID: 'test', - AWS_SECRET_ACCESS_KEY: 'test', - AWS_REGION: REGION, - AWS_DEFAULT_REGION: REGION, - AWS_CBOR_DISABLE: 'true', - }) - .withWaitStrategy( - Wait.forAll(Object.values(SERVICE_PORTS).map(port => - Wait.forHttp('/management/healthcheck', port).forStatusCode(200), - )) - ) - .withStartupTimeout(startupTimeoutMs); - - if (process.env.GRID_DEBUG) { - const logStream = fs.createWriteStream(path.join(os.tmpdir(), 'grid-boot.log')); - gridBuilder = gridBuilder.withLogConsumer((stream) => { - stream.on('data', (line) => logStream.write(line)); - stream.on('err', (line) => logStream.write(line)); - }); - } +/** + * All Grid services under test run inside this single container and talk to each + * other over its localhost. Each is published on the fixed host port its + * dev-nginx mapping expects (dev/nginx-mappings.yml), so the developer's + * dev-nginx routes the https://*.media. domains straight into this + * container. + */ +function gridContainer( + network: StartedNetwork, + configDir: string, + startupTimeoutMs: number, +): GenericContainer { + const container = new GenericContainer(GRID_IMAGE) + .withNetwork(network) + .withNetworkAliases(GRID_ALIAS) + .withLabels({ [ROLE_LABEL]: 'the Grid services' }) + .withExposedPorts( + ...Object.values(SERVICE_PORTS).map((port) => ({ container: port, host: port })), + ) + .withBindMounts([ + // DEV stage reads ~/.grid; /etc/grid is honoured for non-DEV stages. Mount both. + { source: configDir, target: '/root/.grid', mode: 'ro' }, + { source: configDir, target: '/etc/grid', mode: 'ro' }, + // Outside CI the grid-e2e-dev image runs services under sbt; mount the repo + // over /build so host edits recompile live. + ...(process.env.CI ? [] : [{ source: REPO_ROOT, target: '/build', mode: 'rw' as const }]), + ]) + .withEnvironment({ + AWS_ACCESS_KEY_ID: 'test', + AWS_SECRET_ACCESS_KEY: 'test', + AWS_REGION: REGION, + AWS_DEFAULT_REGION: REGION, + AWS_CBOR_DISABLE: 'true', + }) + // Waiting on the healthchecks here would collapse every service into one opaque wait; + // the first line of output is enough to hand over to the per-service checks below. + .withWaitStrategy(Wait.forLogMessage(/./)) + .withStartupTimeout(startupTimeoutMs); + + if (!process.env.GRID_DEBUG) { + return container; + } - const grid = await gridBuilder.start(); - started.push(grid); + const logStream = fs.createWriteStream(path.join(os.tmpdir(), 'grid-boot.log')); + return container.withLogConsumer((stream) => { + stream.on('data', (line) => logStream.write(line)); + stream.on('err', (line) => logStream.write(line)); + }); +} - // Seed Elasticsearch with image fixtures - // - // The app creates the `images` index + `Images_Current` alias on startup; seed once the - // stack is healthy so searches during the tests return the fixture documents. - if (seed) { - const esBaseUrl = `http://${elasticsearch.getHost()}:${elasticsearch.getMappedPort(ELASTICSEARCH_PORT)}`; - await seedElasticsearch(esBaseUrl); - } +/** + * Locally, the browser reaches the https://*.media. domains via the developer's + * dev-nginx. CI has no dev-nginx, so a Caddy proxy replays the same subdomain routing and + * terminates TLS with a self-signed cert, published on the https port the domains resolve to. + */ +function proxyContainer(network: StartedNetwork, caddyfile: string): GenericContainer { + return new GenericContainer(PROXY_IMAGE) + .withNetwork(network) + .withLabels({ [ROLE_LABEL]: 'the reverse proxy' }) + .withExposedPorts({ container: 443, host: 443 }) + .withCopyContentToContainer([{ content: caddyfile, target: '/etc/caddy/Caddyfile' }]) + .withWaitStrategy(Wait.forListeningPorts()) + .withStartupTimeout(60_000); +} - // CI routing: bundled reverse proxy - // - // Locally, the browser reaches the https://*.media. domains via the developer's - // dev-nginx. CI has no dev-nginx, so when running under CI (GitHub Actions sets CI=true) - // start a Caddy proxy that replays the same subdomain routing and terminates TLS with a - // self-signed cert, published on the standard https port the domains resolve to. - if (proxy) { - const caddy = await new GenericContainer(PROXY_IMAGE) - .withNetwork(network) - .withExposedPorts({ container: 443, host: 443 }) - .withCopyContentToContainer([ - { content: buildCaddyfile(coreStackProps), target: '/etc/caddy/Caddyfile' }, - ]) - .withWaitStrategy(Wait.forListeningPorts()) - .withStartupTimeout(60_000) - .start(); - started.push(caddy); +/** Poll a fixed host port until its healthcheck passes, reporting how long it has waited. */ +async function waitForHealthy( + port: number, + healthPath: string, + timeoutMs: number, + report: (message: string) => void, +): Promise { + const startedAt = Date.now(); + const deadline = startedAt + timeoutMs; + + for (;;) { + const { healthy } = await isServiceHealthy(healthPath)(port); + if (healthy) { + return; } + if (Date.now() > deadline) { + throw new Error(`localhost:${port}/${healthPath} did not pass its healthcheck within ${timeoutMs}ms`); + } + + report(`waiting on localhost:${port} (${Math.round((Date.now() - startedAt) / 1000)}s)`); + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } +} + +/** + * Start LocalStack and provision everything held in it. Sequential: each step needs the + * resources the one before it created. + */ +function localstackTasks(): ListrTask[] { + let clients: ReturnType; + + return [ + { + title: 'Start container', + task: async (ctx) => { + ctx.localstack = await localstackContainer(ctx.network!).start(); + ctx.containers.push(ctx.localstack); + clients = provisioningClients(ctx.localstack.getConnectionUri()); + }, + }, + { + title: 'Apply CloudFormation template', + task: async (ctx) => { + ctx.coreStackProps = await createCoreStack(clients.cfn); + }, + }, + { + title: 'Seed config buckets', + task: (ctx) => seedBuckets(clients.s3, ctx.coreStackProps!), + }, + { + title: 'Create permissions bucket', + task: async (ctx) => { + ctx.coreStackProps!.PermissionsBucket = await provisionPermissionsBucket(clients.s3); + }, + }, + { + title: 'Seed KCL lease tables', + task: async (ctx) => { + await Promise.all( + [ + ctx.coreStackProps!.ThrallMessageStream, + ctx.coreStackProps!.ThrallLowPriorityMessageStream, + ].map((stream) => seedKclLeaseTable(clients.dynamo, clients.kinesis, stream)), + ); + }, + }, + ]; +} - const host = grid.getHost(); - const baseUrl = `http://${host}:${grid.getMappedPort(KAHUNA_PORT)}`; - const mediaApiUrl = `http://${host}:${grid.getMappedPort(MEDIA_API_PORT)}`; +/** Start the whole stack, tearing down anything already started if a later step fails. */ +export async function startStack(options: StartStackOptions = {}): Promise { + const { proxy = !!process.env.CI, seed = true } = options; - fs.writeFileSync(URLS_FILE, JSON.stringify({ kahuna: baseUrl, mediaApi: mediaApiUrl })); + const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); + const context: BootContext = { containers: [] }; + + const tasks: ListrTask[] = [ + { + title: 'Create network', + task: async (ctx) => { + ctx.network = await new Network().start(); + }, + }, + { + // These three share only the network, and Elasticsearch is by far the slowest to come + // up, so provisioning LocalStack costs nothing beyond it. + title: 'Start infrastructure', + task: (_, task) => + task.newListr( + [ + { + title: 'Elasticsearch', + task: async (ctx) => { + ctx.elasticsearch = await elasticsearchContainer(ctx.network!).start(); + ctx.containers.push(ctx.elasticsearch); + }, + }, + { + // nginx resolves the `localstack` alias per request, so this need not wait for it. + title: 'imgops', + task: (_, imgopsTask) => { + let image: GenericContainer; + + return imgopsTask.newListr( + [ + { + title: 'Build image', + task: async () => { + image = await GenericContainer.fromDockerfile(IMGOPS_CONTEXT).build( + IMGOPS_IMAGE, + { deleteOnExit: false }, + ); + }, + }, + { + title: 'Start container', + task: async (ctx) => { + ctx.containers.push(await imgopsContainer(image, ctx.network!).start()); + }, + }, + ], + // Subtasks inherit the concurrency of the group above, which these cannot use. + { concurrent: false }, + ); + }, + }, + { + title: 'LocalStack', + task: (_, task) => task.newListr(localstackTasks(), { concurrent: false }), + }, + ], + { concurrent: true }, + ), + }, + { + title: 'Generate service config', + task: (ctx) => { + // Reaching here means the probe found no live stack, so recreating the shared path is + // safe and clears anything a killed run left behind. + fs.rmSync(CONFIG_DIR, { recursive: true, force: true }); + fs.mkdirSync(CONFIG_DIR, { recursive: true }); + ctx.configDir = CONFIG_DIR; + ownedConfigDir = CONFIG_DIR; + generateServiceConfig(CONFIG_DIR, ctx.coreStackProps!); + }, + }, + { + title: 'Start Grid services', + task: (_, task) => + task.newListr([ + { + title: 'Start container', + task: async (ctx) => { + ctx.grid = await gridContainer(ctx.network!, ctx.configDir!, startupTimeoutMs).start(); + ctx.containers.push(ctx.grid); + }, + }, + { + title: 'Wait for services', + task: (_, services) => { + const readiness: ListrTask[] = [ + ...Object.entries(SERVICE_PORTS).map(([service, port]) => ({ + title: service, + task: (_: BootContext, serviceTask: { output: string }) => + waitForHealthy(port, 'management/healthcheck', startupTimeoutMs, (message) => { + serviceTask.output = message; + }), + })), + { + // Waits for the `Images_Current` alias the app assigns on startup, so this + // only needs the container running, not every service healthy. + title: 'Seed Elasticsearch', + skip: () => !seed && 'seeding not requested', + task: async (ctx, seedTask) => { + const esBaseUrl = `http://${ctx.elasticsearch!.getHost()}:${ctx.elasticsearch!.getMappedPort(ELASTICSEARCH_PORT)}`; + await seedElasticsearch(esBaseUrl, (message) => { + seedTask.output = message; + }); + }, + }, + ]; + + return services.newListr(readiness, { concurrent: true }); + }, + }, + ]), + }, + { + title: 'Start reverse proxy', + skip: () => !proxy && 'using dev-nginx', + task: async (ctx) => { + const caddy = await proxyContainer(ctx.network!, buildCaddyfile(ctx.coreStackProps!)).start(); + ctx.containers.push(caddy); + }, + }, + ]; + + try { + await runTasks(tasks, context); + + const host = context.grid!.getHost(); return { - network, - containers: started, - configDir, - urlsFile: URLS_FILE, - baseUrl, - mediaApiUrl, + network: context.network, + containers: context.containers, + configDir: context.configDir, + baseUrl: `http://${host}:${context.grid!.getMappedPort(KAHUNA_PORT)}`, + mediaApiUrl: `http://${host}:${context.grid!.getMappedPort(MEDIA_API_PORT)}`, }; } catch (error) { // Leave nothing running if we failed part-way through the boot. - await stopStack({ network, containers: started, configDir }); + await stopStack(context); throw error; } } -/** Run a best-effort teardown step, warning (not throwing) so the rest still runs. */ -async function warnOnException(label: string, fn: () => unknown): Promise { - try { - await fn(); - } catch (error) { - console.warn(`${label}: ${(error as Error).message}`); - } -} - /** Is a service answering its healthcheck on this fixed host port? */ const isServiceHealthy = (path: string) => async (port: number): Promise<{ port: number, healthy: boolean }> => { try { @@ -402,7 +584,21 @@ export async function ensureStack(options: StartStackOptions = {}): Promise> } = {}; + await runTasks( + [ + { + title: 'Look for a running Grid stack', + task: async (ctx, task) => { + ctx.result = await probeStack(); + task.title = PROBE_OUTCOMES[ctx.result.state]; + }, + }, + ], + probe, + ); + + const { state, healthy, ports } = probe.result!; if (state === 'partial') { const missing = ports.filter((port) => !healthy.includes(port)); @@ -433,22 +629,26 @@ async function attachToStack(options: StartStackOptions): Promise !reseed && 'reseeding not requested', + task: async (_, task) => { + await seedElasticsearch(`http://localhost:${ELASTICSEARCH_PORT}`, (message) => { + task.output = message; + }); + }, + }, + ], + {}, + ); - return { containers: [], urlsFile, baseUrl, mediaApiUrl }; + return { containers: [], baseUrl, mediaApiUrl }; } /** Stop what this process started and delete what it wrote; anything else is left alone. */ @@ -457,22 +657,34 @@ export async function stopStack(environment: StoppableStack | undefined): Promis return; } - const { containers, network, configDir, urlsFile } = environment; + const { containers, network, configDir } = environment; + + const tasks: ListrTask[] = [ + // Reverse start order, so nothing is stopped before whatever depends on it. + ...[...containers].reverse().map((container) => ({ + title: `Stop ${container.getLabels()[ROLE_LABEL] ?? container.getName()}`, + task: async () => { + await container.stop(); + }, + })), + ...(network ? [{ title: 'Remove network', task: () => network.stop() }] : []), + ...(configDir + ? [ + { + title: 'Remove generated config', + task: () => { + fs.rmSync(configDir, { recursive: true, force: true }); + ownedConfigDir = undefined; + }, + }, + ] + : []), + ]; - for (const container of [...containers].reverse()) { - await warnOnException('Failed to stop container', () => container.stop()); + if (tasks.length === 0) { + return; } - if (network) { - await warnOnException('Failed to stop network', () => network.stop()); - } - if (configDir) { - await warnOnException('Failed to remove config dir', () => - fs.rmSync(configDir, { recursive: true, force: true }), - ); - ownedConfigDir = undefined; - } - if (urlsFile) { - await warnOnException('Failed to remove urls file', () => fs.rmSync(urlsFile, { force: true })); - } + // Every step is best-effort: a failure is reported against its task and the rest still run. + await runTasks(tasks, {}, { exitOnError: false }); } diff --git a/e2e-tests/setup/state.ts b/e2e-tests/setup/state.ts index 93147c97a6a..1c16a90b51f 100644 --- a/e2e-tests/setup/state.ts +++ b/e2e-tests/setup/state.ts @@ -18,8 +18,6 @@ export interface GridEnvironment { containers: StartedTestContainer[]; /** Absolute path to the generated config directory mounted into the app container. */ configDir?: string; - /** Set only when this process wrote the URLs file, so teardown does not delete another stack's. */ - urlsFile?: string; /** Base URL of the Kahuna UI, e.g. http://localhost:9005 */ baseUrl: string; /** Base URL of the media-api, e.g. http://localhost:9001 */ diff --git a/e2e-tests/steps/fixtures.ts b/e2e-tests/steps/fixtures.ts index 4e8fd548b63..081db695d80 100644 --- a/e2e-tests/steps/fixtures.ts +++ b/e2e-tests/steps/fixtures.ts @@ -1,22 +1,10 @@ import { expect } from '@playwright/test'; import type { APIResponse, Response } from '@playwright/test'; import { test as base, createBdd } from 'playwright-bdd'; -import * as fs from 'fs'; -import { DOMAIN, URLS_FILE } from '../setup/constants.ts'; - -interface GridUrls { - /** Kahuna base URL on its fixed host port (for API-level `request` tests). */ - kahuna: string; - /** media-api base URL on its fixed host port (for API-level `request` tests). */ - mediaApi: string; -} +import { DOMAIN } from '../setup/constants.ts'; export const KAHUNA_APP_URL = `https://media.${DOMAIN}`; -function readGridUrls(): GridUrls { - return JSON.parse(fs.readFileSync(URLS_FILE, 'utf8')) as GridUrls; -} - /** * Mutable per-scenario state shared between steps (e.g. the response captured in a * `When` step and asserted on in a `Then` step). @@ -27,18 +15,11 @@ interface TestContext { } /** - * Test fixture that exposes the Grid service URLs resolved by `global-setup.ts` and - * points `baseURL` at Kahuna's fixed host port for API-level `request` tests. Browser - * steps should navigate to `KAHUNA_APP_URL` so the page origin is a real Grid domain - * (required for the services' CORS origins to match). + * Browser steps should navigate to `KAHUNA_APP_URL` so the page origin is a real Grid + * domain (required for the services' CORS origins to match). API-level `request` tests + * use `baseURL` from the Playwright config, which points at Kahuna's fixed host port. */ -export const test = base.extend<{ gridUrls: GridUrls; testContext: TestContext }>({ - gridUrls: async ({}, use) => { - await use(readGridUrls()); - }, - baseURL: async ({}, use) => { - await use(readGridUrls().kahuna); - }, +export const test = base.extend<{ testContext: TestContext }>({ testContext: async ({}, use) => { await use({}); }, From 92d92b7ef5a5b91c7c99b5caf82a7ad10b65c7fe Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Mon, 7 Sep 2026 14:37:26 +0100 Subject: [PATCH 326/373] Fix collection modal overlay z-index --- .../components/gr-collection-overlay/gr-collection-overlay.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kahuna/public/js/components/gr-collection-overlay/gr-collection-overlay.css b/kahuna/public/js/components/gr-collection-overlay/gr-collection-overlay.css index f57453b833f..adb36b6c7b7 100644 --- a/kahuna/public/js/components/gr-collection-overlay/gr-collection-overlay.css +++ b/kahuna/public/js/components/gr-collection-overlay/gr-collection-overlay.css @@ -5,7 +5,7 @@ position: fixed; top: 0; left: 0; - z-index: 40; + z-index: 200; } .collection-overlay__button:hover { From dde259964e3d6afea99260bcfea5a450babda879 Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Fri, 4 Sep 2026 17:03:10 +0100 Subject: [PATCH 327/373] Spike C2PA detection --- .../gu/mediaservice/model/FileMetadata.scala | 11 ++- .../app/lib/imaging/C2PADetector.scala | 91 +++++++++++++++++++ .../app/lib/imaging/FileMetadataReader.scala | 7 +- 3 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 image-loader/app/lib/imaging/C2PADetector.scala diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala index b465e689fc2..ff39ebf296c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala @@ -15,7 +15,8 @@ case class FileMetadata( icc: Map[String, String] = Map(), getty: Map[String, String] = Map(), colourModel: Option[String] = None, - colourModelInformation: Map[String, String] = Map() + colourModelInformation: Map[String, String] = Map(), + hasC2PA: Boolean = false ) { def toLogMarker: LogMarker = { val fieldCountMarkers = Map ( @@ -31,7 +32,7 @@ case class FileMetadata( val totalFieldCount = fieldCountMarkers.foldLeft(0)(_ + _._2) val markers = fieldCountMarkers + ("totalFieldCount" -> totalFieldCount) - MarkerMap(markers) + MarkerMap(markers + ("hasC2PA" -> hasC2PA)) } def readXmpHeadStringProp: (String) => Option[String] = (name: String) => { @@ -56,7 +57,8 @@ object FileMetadata { (__ \ "icc").readNullable[Map[String,String]].map(_ getOrElse Map.empty) ~ (__ \ "getty").readNullable[Map[String,String]].map(_ getOrElse Map.empty) ~ (__ \ "colourModel").readNullable[String] ~ - (__ \ "colourModelInformation").readNullable[Map[String,String]].map(_ getOrElse Map.empty) + (__ \ "colourModelInformation").readNullable[Map[String,String]].map(_ getOrElse Map.empty) ~ + (__ \ "hasC2PA").readNullable[Boolean].map(_ getOrElse false) )(FileMetadata.apply _) @@ -68,6 +70,7 @@ object FileMetadata { (JsPath \ "icc").write[Map[String,String]] and (JsPath \ "getty").write[Map[String,String]] and (JsPath \ "colourModel").writeNullable[String] and - (JsPath \ "colourModelInformation").write[Map[String,String]] + (JsPath \ "colourModelInformation").write[Map[String,String]] and + (JsPath \ "hasC2PA").write[Boolean] )(unlift(FileMetadata.unapply)) } diff --git a/image-loader/app/lib/imaging/C2PADetector.scala b/image-loader/app/lib/imaging/C2PADetector.scala new file mode 100644 index 00000000000..45ecfbda1e7 --- /dev/null +++ b/image-loader/app/lib/imaging/C2PADetector.scala @@ -0,0 +1,91 @@ +package lib.imaging + +import com.drew.imaging.jpeg.{JpegSegmentReader, JpegSegmentType} +import com.drew.imaging.png.{PngChunkReader, PngChunkType} +import com.drew.imaging.tiff.TiffMetadataReader +import com.drew.lang.StreamReader +import com.gu.mediaservice.model.{Jpeg, MimeType, Png, Tiff} + +import java.io.{BufferedInputStream, File, FileInputStream} +import java.util.Collections +import scala.jdk.CollectionConverters._ +import scala.util.control.NonFatal + +/** + * Lightweight detector for embedded C2PA (Content Credentials) manifests, built entirely on + * top of the `metadata-extractor` library already used elsewhere in Grid (see FileMetadataReader). + * + * This does NOT parse or validate the manifest itself - `metadata-extractor` has no support for + * C2PA/JUMBF (ISO/IEC 19566-5). It only checks for the presence of the container structures C2PA + * uses to embed a manifest in JPEG (an APP11 marker segment), PNG (a "caBX" ancillary chunk) and + * TIFF (a private IFD tag), as defined in the C2PA Technical Specification, Appendix A.2 + * "Embedding manifests into non-BMFF-based assets": + * https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_non_bmff_based_assets + * + * The mime type is taken from the caller (already known/validated by Grid at upload time) rather + * than re-detected from the file's bytes: `metadata-extractor`'s own byte-sniffing (FileTypeDetector) + * is ambiguous for TIFF-based formats - e.g. a plain TIFF whose first IFD starts at the common + * offset of 8 bytes can be misidentified as a Sony ARW raw file, since the two signatures overlap. + */ +object C2PADetector { + + // An APP11 segment payload is prefixed with CI (2 bytes, the ASCII common identifier "JP" for + // any JPEG-XT/APP11 extension - not specific to C2PA), EN (2 bytes, box instance number) and Z + // (4 bytes, packet sequence number) - an 8 byte prefix - followed by the JUMBF box itself: + // LBox+TBox ("jumb", 8 bytes), then its description box's LBox+TBox ("jumd", 8 bytes), then a + // 16-byte UUID identifying the box's content type. So the UUID starts at offset 24 within the + // segment payload. C2PA's manifest store UUID begins with the ASCII mnemonic "c2pa" - checking + // for that (rather than just the generic "JP" CI prefix) is what distinguishes an actual C2PA + // manifest from some other, non-C2PA use of the same JPEG APP11/JUMBF embedding mechanism, and + // matches the check the reference C2PA Rust SDK itself performs (see C2PA_MARKER in + // https://github.com/contentauth/c2pa-rs/blob/main/sdk/src/asset_handlers/jpeg_io.rs). + // See the C2PA Technical Specification, Appendix A.2.1 "Embedding manifests into JPEG": + // https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_jpeg + private val JumbfUuidOffset = 24 + private val C2PAManifestStoreUuidMnemonic: Array[Byte] = "c2pa".getBytes("US-ASCII") + + // PNG stores JUMBF/C2PA data in an ancillary chunk named "caBX" (case is significant). + // See the C2PA Technical Specification, Appendix A.2.2 "Embedding manifests into PNG": + // https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_png + private val PngC2PAChunkType = new PngChunkType("caBX") + + // TIFF stores the raw JUMBF box directly (no "JP" identifier) as the data of a tag with ID + // 52545 decimal / 0xCD41 hex, tag type 7 (UNDEFINED). See the C2PA Technical Specification, + // Appendix A.2.5 "Embedding manifests into TIFF-based assets": + // https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_tiff_based_assets + private val TiffC2PATag = 0xCD41 + + def hasC2PAManifest(file: File, mimeType: MimeType): Boolean = { + try { + mimeType match { + case Jpeg => hasJpegApp11Jumbf(file) + case Png => hasPngC2PAChunk(file) + case Tiff => hasTiffC2PATag(file) + } + } catch { + case NonFatal(_) => false + } + } + + private def hasJpegApp11Jumbf(file: File): Boolean = { + val segments = JpegSegmentReader.readSegments(file, Collections.singletonList(JpegSegmentType.APPB)) + segments.getSegments(JpegSegmentType.APPB).asScala.exists { payload => + payload.length >= JumbfUuidOffset + C2PAManifestStoreUuidMnemonic.length && + payload.slice(JumbfUuidOffset, JumbfUuidOffset + C2PAManifestStoreUuidMnemonic.length) + .sameElements(C2PAManifestStoreUuidMnemonic) + } + } + + private def hasPngC2PAChunk(file: File): Boolean = { + val in = new BufferedInputStream(new FileInputStream(file)) + try { + val chunks = new PngChunkReader().extract(new StreamReader(in), Collections.singleton(PngC2PAChunkType)) + chunks.asScala.nonEmpty + } finally in.close() + } + + private def hasTiffC2PATag(file: File): Boolean = { + val metadata = TiffMetadataReader.readMetadata(file) + metadata.getDirectories.asScala.exists(_.containsTag(TiffC2PATag)) + } +} diff --git a/image-loader/app/lib/imaging/FileMetadataReader.scala b/image-loader/app/lib/imaging/FileMetadataReader.scala index f81b9ce3a7f..be75c2eb482 100644 --- a/image-loader/app/lib/imaging/FileMetadataReader.scala +++ b/image-loader/app/lib/imaging/FileMetadataReader.scala @@ -58,6 +58,8 @@ object FileMetadataReader extends GridLogging { metadata <- readMetadata(image) } yield getMetadataWithIPTCHeaders(metadata, imageId) // FIXME: JPEG, JFIF, Photoshop, GPS, File + // Note: hasC2PA is left at its default (false) here since the mime type is unknown/unsupported + // in this code path, and C2PA detection needs to know the container format to parse correctly. def fromIPTCHeadersWithColorInfo(image: ImageWrapper)(implicit logMarker: LogMarker): Future[FileMetadata] = fromIPTCHeadersWithColorInfo(image.file, image.id, image.mimeType) @@ -67,7 +69,10 @@ object FileMetadataReader extends GridLogging { metadata <- readMetadata(image) colourModelInformation <- getColorModelInformation(image, metadata, mimeType) } - yield getMetadataWithIPTCHeaders(metadata, imageId).copy(colourModelInformation = colourModelInformation) + yield getMetadataWithIPTCHeaders(metadata, imageId).copy( + colourModelInformation = colourModelInformation, + hasC2PA = C2PADetector.hasC2PAManifest(image, mimeType) + ) private def getMetadataWithIPTCHeaders(metadata: Metadata, imageId:String): FileMetadata = FileMetadata( From 5466c472a3cbf5661ebf5668b20ef349113d8cb8 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 16:04:55 +0000 Subject: [PATCH 328/373] Address a few nits after a review --- e2e-tests/setup/progress.ts | 1 + e2e-tests/setup/seed-elasticsearch.ts | 3 +-- e2e-tests/setup/stack.ts | 25 +++++++++++++------------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/e2e-tests/setup/progress.ts b/e2e-tests/setup/progress.ts index a4db342d513..f95bbbefab7 100644 --- a/e2e-tests/setup/progress.ts +++ b/e2e-tests/setup/progress.ts @@ -32,6 +32,7 @@ export async function runTasks( collapseSubtasks: false, collapseErrors: false, }, + registerSignalListeners: false, fallbackRendererOptions: { timer: PRESET_TIMER }, }); diff --git a/e2e-tests/setup/seed-elasticsearch.ts b/e2e-tests/setup/seed-elasticsearch.ts index e6778c79586..4695f397932 100644 --- a/e2e-tests/setup/seed-elasticsearch.ts +++ b/e2e-tests/setup/seed-elasticsearch.ts @@ -75,8 +75,7 @@ export async function seedElasticsearch( const hits = fixture.body?.hits?.hits ?? []; if (hits.length === 0) { - report('No image fixtures found to seed into Elasticsearch'); - return; + throw new Error('No image fixtures found to seed into Elasticsearch. This is unexpected - failing early'); } report(`Waiting for the '${IMAGES_ALIAS}' alias`); diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index cf44f5e6dda..af35598f12c 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -52,6 +52,7 @@ import { import type { StackProps } from './provision.ts'; import { seedElasticsearch } from './seed-elasticsearch.ts'; import type { GridEnvironment } from './state.ts'; +import { ListrTaskFn } from 'listr2'; const LOCALSTACK_SERVICES = [ "cloudformation", @@ -293,7 +294,7 @@ function gridContainer( // Waiting on the healthchecks here would collapse every service into one opaque wait; // the first line of output is enough to hand over to the per-service checks below. .withWaitStrategy(Wait.forLogMessage(/./)) - .withStartupTimeout(startupTimeoutMs); + .withStartupTimeout(10_000); if (!process.env.GRID_DEBUG) { return container; @@ -331,7 +332,7 @@ async function waitForHealthy( const startedAt = Date.now(); const deadline = startedAt + timeoutMs; - for (;;) { + for (; ;) { const { healthy } = await isServiceHealthy(healthPath)(port); if (healthy) { return; @@ -483,9 +484,9 @@ export async function startStack(options: StartStackOptions = {}): Promise { const readiness: ListrTask[] = [ - ...Object.entries(SERVICE_PORTS).map(([service, port]) => ({ + ...Object.entries(SERVICE_PORTS).map(([service, port]): { title: string, task: ListrTaskFn } => ({ title: service, - task: (_: BootContext, serviceTask: { output: string }) => + task: (_, serviceTask) => waitForHealthy(port, 'management/healthcheck', startupTimeoutMs, (message) => { serviceTask.output = message; }), @@ -667,17 +668,17 @@ export async function stopStack(environment: StoppableStack | undefined): Promis await container.stop(); }, })), - ...(network ? [{ title: 'Remove network', task: () => network.stop() }] : []), + ...(network ? [{ title: 'Remove network', task: () => { network.stop() } }] : []), ...(configDir ? [ - { - title: 'Remove generated config', - task: () => { - fs.rmSync(configDir, { recursive: true, force: true }); - ownedConfigDir = undefined; - }, + { + title: 'Remove generated config', + task: () => { + fs.rmSync(configDir, { recursive: true, force: true }); + ownedConfigDir = undefined; }, - ] + }, + ] : []), ]; From 47ec1fe9a605add80556bf117f4f26b58f535dd1 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 16:05:42 +0000 Subject: [PATCH 329/373] Define ES and Kahuna URLs statically, rather than relying on mapped ports --- e2e-tests/setup/stack.ts | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index af35598f12c..17166bfff5c 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -75,12 +75,18 @@ export interface StartStackOptions { seed?: boolean; } +// Every service is bound to a fixed host port, so its URL is the same whether this process +// started the stack or attached to one already running. +const KAHUNA_URL = `http://localhost:${KAHUNA_PORT}`; +const MEDIA_API_URL = `http://localhost:${MEDIA_API_PORT}`; +const ELASTICSEARCH_URL = `http://localhost:${ELASTICSEARCH_PORT}`; + /** How much of the stack is already listening on the fixed host ports. */ type StackProbe = 'none' | 'healthy' | 'partial'; const PROBE_OUTCOMES: Record = { none: 'No running Grid stack found', - healthy: `Found a Grid stack already running on http://localhost:${KAHUNA_PORT}`, + healthy: `Found a Grid stack already running on ${KAHUNA_URL}`, partial: 'Found a partially running Grid stack', }; @@ -98,9 +104,7 @@ interface StoppableStack { /** What the boot tasks build up. Each task mutates it in place for the ones that follow. */ interface BootContext extends StoppableStack { - elasticsearch?: StartedTestContainer; localstack?: StartedLocalStackContainer; - grid?: StartedTestContainer; coreStackProps?: StackProps; } @@ -416,8 +420,7 @@ export async function startStack(options: StartStackOptions = {}): Promise { - ctx.elasticsearch = await elasticsearchContainer(ctx.network!).start(); - ctx.containers.push(ctx.elasticsearch); + ctx.containers.push(await elasticsearchContainer(ctx.network!).start()); }, }, { @@ -476,8 +479,9 @@ export async function startStack(options: StartStackOptions = {}): Promise { - ctx.grid = await gridContainer(ctx.network!, ctx.configDir!, startupTimeoutMs).start(); - ctx.containers.push(ctx.grid); + ctx.containers.push( + await gridContainer(ctx.network!, ctx.configDir!, startupTimeoutMs).start(), + ); }, }, { @@ -496,9 +500,8 @@ export async function startStack(options: StartStackOptions = {}): Promise !seed && 'seeding not requested', - task: async (ctx, seedTask) => { - const esBaseUrl = `http://${ctx.elasticsearch!.getHost()}:${ctx.elasticsearch!.getMappedPort(ELASTICSEARCH_PORT)}`; - await seedElasticsearch(esBaseUrl, (message) => { + task: async (_, seedTask) => { + await seedElasticsearch(ELASTICSEARCH_URL, (message) => { seedTask.output = message; }); }, @@ -523,14 +526,12 @@ export async function startStack(options: StartStackOptions = {}): Promise { - const baseUrl = `http://localhost:${KAHUNA_PORT}`; - const mediaApiUrl = `http://localhost:${MEDIA_API_PORT}`; - // Re-seeding is opt-in: whoever started the stack already seeded it, and the fixtures // are only reloaded on request because tests may have since changed the data. const reseed = options.seed === true || process.env.GRID_RESEED === 'true'; @@ -640,7 +638,7 @@ async function attachToStack(options: StartStackOptions): Promise !reseed && 'reseeding not requested', task: async (_, task) => { - await seedElasticsearch(`http://localhost:${ELASTICSEARCH_PORT}`, (message) => { + await seedElasticsearch(ELASTICSEARCH_URL, (message) => { task.output = message; }); }, @@ -649,7 +647,7 @@ async function attachToStack(options: StartStackOptions): Promise Date: Mon, 7 Sep 2026 16:06:44 +0000 Subject: [PATCH 330/373] Define localstack as local var --- e2e-tests/setup/stack.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index 17166bfff5c..dd34b4dc1e5 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -361,9 +361,9 @@ function localstackTasks(): ListrTask[] { { title: 'Start container', task: async (ctx) => { - ctx.localstack = await localstackContainer(ctx.network!).start(); - ctx.containers.push(ctx.localstack); - clients = provisioningClients(ctx.localstack.getConnectionUri()); + const localstack = await localstackContainer(ctx.network!).start(); + ctx.containers.push(localstack); + clients = provisioningClients(localstack.getConnectionUri()); }, }, { From ac31d44a0b8c349b72a7c6d7757d9556aa714b65 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 16:12:15 +0000 Subject: [PATCH 331/373] Small helper for reportTo --- e2e-tests/setup/progress.ts | 2 ++ e2e-tests/setup/seed-elasticsearch.ts | 2 +- e2e-tests/setup/stack.ts | 16 ++++------------ 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/e2e-tests/setup/progress.ts b/e2e-tests/setup/progress.ts index f95bbbefab7..6e8a6ac223b 100644 --- a/e2e-tests/setup/progress.ts +++ b/e2e-tests/setup/progress.ts @@ -38,3 +38,5 @@ export async function runTasks( return runner.run(); } + +export const reportTo = (task: { output: string }) => (message: string) => { task.output = message; }; \ No newline at end of file diff --git a/e2e-tests/setup/seed-elasticsearch.ts b/e2e-tests/setup/seed-elasticsearch.ts index 4695f397932..181cc729191 100644 --- a/e2e-tests/setup/seed-elasticsearch.ts +++ b/e2e-tests/setup/seed-elasticsearch.ts @@ -69,7 +69,7 @@ async function waitForAlias(esBaseUrl: string, timeoutMs = 60_000): Promise void = console.log, + report: (message: string) => void, ): Promise { const fixture = JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) as EsSearchFixture; const hits = fixture.body?.hits?.hits ?? []; diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index dd34b4dc1e5..f77bc2637c7 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -13,7 +13,6 @@ import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { LocalstackContainer } from '@testcontainers/localstack'; -import type { StartedLocalStackContainer } from '@testcontainers/localstack'; import { GenericContainer, Network, Wait } from 'testcontainers'; import type { StartedNetwork, StartedTestContainer } from 'testcontainers'; import { @@ -40,7 +39,7 @@ import { SERVICE_PORTS, } from './constants.ts'; import { generateServiceConfig } from './config.ts'; -import { runTasks } from './progress.ts'; +import { reportTo, runTasks } from './progress.ts'; import type { ListrTask } from './progress.ts'; import { createCoreStack, @@ -104,7 +103,6 @@ interface StoppableStack { /** What the boot tasks build up. Each task mutates it in place for the ones that follow. */ interface BootContext extends StoppableStack { - localstack?: StartedLocalStackContainer; coreStackProps?: StackProps; } @@ -491,9 +489,7 @@ export async function startStack(options: StartStackOptions = {}): Promise } => ({ title: service, task: (_, serviceTask) => - waitForHealthy(port, 'management/healthcheck', startupTimeoutMs, (message) => { - serviceTask.output = message; - }), + waitForHealthy(port, 'management/healthcheck', startupTimeoutMs, reportTo(serviceTask)), })), { // Waits for the `Images_Current` alias the app assigns on startup, so this @@ -501,9 +497,7 @@ export async function startStack(options: StartStackOptions = {}): Promise !seed && 'seeding not requested', task: async (_, seedTask) => { - await seedElasticsearch(ELASTICSEARCH_URL, (message) => { - seedTask.output = message; - }); + await seedElasticsearch(ELASTICSEARCH_URL, reportTo(seedTask)); }, }, ]; @@ -638,9 +632,7 @@ async function attachToStack(options: StartStackOptions): Promise !reseed && 'reseeding not requested', task: async (_, task) => { - await seedElasticsearch(ELASTICSEARCH_URL, (message) => { - task.output = message; - }); + await seedElasticsearch(ELASTICSEARCH_URL, reportTo(task)); }, }, ], From d9748efe8fa512177a6b9cc3057c93b80edf9bd3 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 16:12:51 +0000 Subject: [PATCH 332/373] Remove listr for ensureStaack for consistency with dev.ts --- e2e-tests/setup/stack.ts | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index f77bc2637c7..78008270f6f 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -580,21 +580,8 @@ export async function ensureStack(options: StartStackOptions = {}): Promise> } = {}; - await runTasks( - [ - { - title: 'Look for a running Grid stack', - task: async (ctx, task) => { - ctx.result = await probeStack(); - task.title = PROBE_OUTCOMES[ctx.result.state]; - }, - }, - ], - probe, - ); - - const { state, healthy, ports } = probe.result!; + const { state, healthy, ports } = await probeStack(); + console.log(PROBE_OUTCOMES[state]); if (state === 'partial') { const missing = ports.filter((port) => !healthy.includes(port)); From 584dcb96ab33561ef3006b197b44c30670370d05 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 16:19:40 +0000 Subject: [PATCH 333/373] Minor tidies after a manual review --- e2e-tests/setup/dev.ts | 3 +++ e2e-tests/setup/provision.ts | 3 ++- e2e-tests/setup/stack.ts | 3 +-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/e2e-tests/setup/dev.ts b/e2e-tests/setup/dev.ts index 457f647ef00..c493652c7f8 100644 --- a/e2e-tests/setup/dev.ts +++ b/e2e-tests/setup/dev.ts @@ -6,6 +6,7 @@ * running locally. */ import { + CONFIG_DIR, DOMAIN, ELASTICSEARCH_PORT, LOCALSTACK_PORT, @@ -31,6 +32,8 @@ function banner(elapsedMs: number): string { ` localstack http://localhost:${LOCALSTACK_PORT}`, ` elasticsearch http://localhost:${ELASTICSEARCH_PORT}`, '', + ` Config at ${CONFIG_DIR}`, + '', ' Press Ctrl-C to tear everything down.', '', ].join('\n'); diff --git a/e2e-tests/setup/provision.ts b/e2e-tests/setup/provision.ts index 33af4d8ef9b..9b577b99d9a 100644 --- a/e2e-tests/setup/provision.ts +++ b/e2e-tests/setup/provision.ts @@ -46,9 +46,10 @@ export async function createCoreStack(cfn: CloudFormationClient): Promise Date: Mon, 7 Sep 2026 16:34:01 +0000 Subject: [PATCH 334/373] Make timeout for ES seeding consistent with Grid startup timeout, and lower startup timeout --- e2e-tests/setup/seed-elasticsearch.ts | 5 +++-- e2e-tests/setup/stack.ts | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/e2e-tests/setup/seed-elasticsearch.ts b/e2e-tests/setup/seed-elasticsearch.ts index 181cc729191..c950979d57b 100644 --- a/e2e-tests/setup/seed-elasticsearch.ts +++ b/e2e-tests/setup/seed-elasticsearch.ts @@ -44,7 +44,7 @@ const sleep = (ms: number): Promise => new Promise((resolve) => setTimeout * Poll the alias until the Grid app has created the index and assigned it, so the bulk * insert below has a valid write target. Throws if it never appears within the timeout. */ -async function waitForAlias(esBaseUrl: string, timeoutMs = 60_000): Promise { +async function waitForAlias(esBaseUrl: string, timeoutMs: number): Promise { const deadline = Date.now() + timeoutMs; let lastStatus = 'no response'; while (Date.now() < deadline) { @@ -69,6 +69,7 @@ async function waitForAlias(esBaseUrl: string, timeoutMs = 60_000): Promise void, ): Promise { const fixture = JSON.parse(fs.readFileSync(FIXTURE_PATH, 'utf8')) as EsSearchFixture; @@ -79,7 +80,7 @@ export async function seedElasticsearch( } report(`Waiting for the '${IMAGES_ALIAS}' alias`); - await waitForAlias(esBaseUrl); + await waitForAlias(esBaseUrl, startupTimeoutMs); // NDJSON bulk body: an index action line (targeting the document's `_id`) followed by // the source document, repeated for every hit. diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index 299fd99c325..73c54ddf39f 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -398,7 +398,7 @@ function localstackTasks(): ListrTask[] { export async function startStack(options: StartStackOptions = {}): Promise { const { proxy = !!process.env.CI, seed = true } = options; - const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 300_000); + const startupTimeoutMs = Number(process.env.GRID_STARTUP_TIMEOUT_MS ?? 120_000); const context: BootContext = { containers: [] }; const tasks: ListrTask[] = [ @@ -496,7 +496,7 @@ export async function startStack(options: StartStackOptions = {}): Promise !seed && 'seeding not requested', task: async (_, seedTask) => { - await seedElasticsearch(ELASTICSEARCH_URL, reportTo(seedTask)); + await seedElasticsearch(ELASTICSEARCH_URL, startupTimeoutMs, reportTo(seedTask)); }, }, ]; @@ -618,7 +618,7 @@ async function attachToStack(options: StartStackOptions): Promise !reseed && 'reseeding not requested', task: async (_, task) => { - await seedElasticsearch(ELASTICSEARCH_URL, reportTo(task)); + await seedElasticsearch(ELASTICSEARCH_URL, 60_000, reportTo(task)); }, }, ], From c4dc40b7ff91e0c6a0ca3c0976c1bc272e28597d Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 7 Sep 2026 16:35:37 +0000 Subject: [PATCH 335/373] await result of network.stop() --- e2e-tests/setup/stack.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index 73c54ddf39f..e59582c0277 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -640,11 +640,9 @@ export async function stopStack(environment: StoppableStack | undefined): Promis // Reverse start order, so nothing is stopped before whatever depends on it. ...[...containers].reverse().map((container) => ({ title: `Stop ${container.getLabels()[ROLE_LABEL] ?? container.getName()}`, - task: async () => { - await container.stop(); - }, + task: container.stop, })), - ...(network ? [{ title: 'Remove network', task: () => { network.stop() } }] : []), + ...(network ? [{ title: 'Remove network', task: network.stop }] : []), ...(configDir ? [ { From c1fb7486cf3ccd36b741f2b4d889243f491e1d82 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:59:13 +0000 Subject: [PATCH 336/373] chore(deps): bump picomatch from 2.3.1 to 2.3.2 in /dev/oidc-provider Bumps [picomatch](https://github.com/micromatch/picomatch) from 2.3.1 to 2.3.2. - [Release notes](https://github.com/micromatch/picomatch/releases) - [Changelog](https://github.com/micromatch/picomatch/blob/master/CHANGELOG.md) - [Commits](https://github.com/micromatch/picomatch/compare/2.3.1...2.3.2) --- updated-dependencies: - dependency-name: picomatch dependency-version: 2.3.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- dev/oidc-provider/package-lock.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dev/oidc-provider/package-lock.json b/dev/oidc-provider/package-lock.json index ac971507a8a..52a18029516 100644 --- a/dev/oidc-provider/package-lock.json +++ b/dev/oidc-provider/package-lock.json @@ -3383,10 +3383,11 @@ "dev": true }, "node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, + "license": "MIT", "engines": { "node": ">=8.6" }, @@ -6534,9 +6535,9 @@ "dev": true }, "picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true }, "pirates": { From 93af2eff1f5ffbd981935e91534a48cd33c49386 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:27:39 +0000 Subject: [PATCH 337/373] chore(deps-dev): bump handlebars from 4.7.8 to 4.7.9 in /cdk Bumps [handlebars](https://github.com/handlebars-lang/handlebars.js) from 4.7.8 to 4.7.9. - [Release notes](https://github.com/handlebars-lang/handlebars.js/releases) - [Changelog](https://github.com/handlebars-lang/handlebars.js/blob/v4.7.9/release-notes.md) - [Commits](https://github.com/handlebars-lang/handlebars.js/compare/v4.7.8...v4.7.9) --- updated-dependencies: - dependency-name: handlebars dependency-version: 4.7.9 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- cdk/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdk/package-lock.json b/cdk/package-lock.json index f798c6935f8..f4a94597b83 100644 --- a/cdk/package-lock.json +++ b/cdk/package-lock.json @@ -7936,9 +7936,9 @@ "license": "MIT" }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "dev": true, "license": "MIT", "dependencies": { From d48ef58b7eef07dbf7c875d55b600c685319af7f Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Tue, 8 Sep 2026 12:13:47 +0100 Subject: [PATCH 338/373] Store c2pa availability under an object --- .../lib/elasticsearch/MappingTest.scala | 3 +- .../gu/mediaservice/model/FileMetadata.scala | 11 ++-- .../mediaservice/model/FileMetadataTest.scala | 49 ++++++++++++++ .../app/lib/imaging/C2PADetector.scala | 49 +++++++------- .../app/lib/imaging/FileMetadataReader.scala | 4 +- .../test/resources/c2pa/c2pa-absent.jpg | Bin 0 -> 2529 bytes .../test/resources/c2pa/c2pa-absent.png | Bin 0 -> 1067 bytes .../test/resources/c2pa/c2pa-absent.tiff | Bin 0 -> 360140 bytes .../test/resources/c2pa/c2pa-present.jpg | Bin 0 -> 38312 bytes .../test/resources/c2pa/c2pa-present.png | Bin 0 -> 36850 bytes .../test/resources/c2pa/c2pa-present.tiff | Bin 0 -> 395956 bytes .../scala/lib/imaging/C2paDetectorTest.scala | 60 ++++++++++++++++++ kahuna/public/js/search/query-filter.js | 2 +- 13 files changed, 143 insertions(+), 35 deletions(-) create mode 100644 common-lib/src/test/scala/com/gu/mediaservice/model/FileMetadataTest.scala create mode 100644 image-loader/test/resources/c2pa/c2pa-absent.jpg create mode 100644 image-loader/test/resources/c2pa/c2pa-absent.png create mode 100644 image-loader/test/resources/c2pa/c2pa-absent.tiff create mode 100644 image-loader/test/resources/c2pa/c2pa-present.jpg create mode 100644 image-loader/test/resources/c2pa/c2pa-present.png create mode 100644 image-loader/test/resources/c2pa/c2pa-present.tiff create mode 100644 image-loader/test/scala/lib/imaging/C2paDetectorTest.scala diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala index f4935c04c53..45550775985 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/elasticsearch/MappingTest.scala @@ -109,7 +109,8 @@ object MappingTest { icc = Map("icc1" -> "value1"), getty = Map("getty1" -> "value1"), colourModel = Some("my-model"), - colourModelInformation = Map("colourModel1" -> "value1") + colourModelInformation = Map("colourModel1" -> "value1"), + c2pa = FileMetadata.C2paAvailable ), userMetadata = Some(Edits( archived = true, diff --git a/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala b/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala index ff39ebf296c..9e1dba21d0c 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/model/FileMetadata.scala @@ -16,7 +16,7 @@ case class FileMetadata( getty: Map[String, String] = Map(), colourModel: Option[String] = None, colourModelInformation: Map[String, String] = Map(), - hasC2PA: Boolean = false + c2pa: Map[String, JsValue] = Map() ) { def toLogMarker: LogMarker = { val fieldCountMarkers = Map ( @@ -32,7 +32,7 @@ case class FileMetadata( val totalFieldCount = fieldCountMarkers.foldLeft(0)(_ + _._2) val markers = fieldCountMarkers + ("totalFieldCount" -> totalFieldCount) - MarkerMap(markers + ("hasC2PA" -> hasC2PA)) + MarkerMap(markers + ("c2paMetadataAvailable" -> c2pa.keys.nonEmpty)) } def readXmpHeadStringProp: (String) => Option[String] = (name: String) => { @@ -47,6 +47,9 @@ case class FileMetadata( } object FileMetadata { + val NoC2PA: Map[String, JsValue] = Map() + val C2paAvailable: Map[String, JsValue] = Map("isAvailable" -> JsBoolean(true)) + // TODO: reindex all images to make the getty map always present // for data consistency, so we can fallback to use the default Reads implicit val ImageMetadataReads: Reads[FileMetadata] = ( @@ -58,7 +61,7 @@ object FileMetadata { (__ \ "getty").readNullable[Map[String,String]].map(_ getOrElse Map.empty) ~ (__ \ "colourModel").readNullable[String] ~ (__ \ "colourModelInformation").readNullable[Map[String,String]].map(_ getOrElse Map.empty) ~ - (__ \ "hasC2PA").readNullable[Boolean].map(_ getOrElse false) + (__ \ "c2pa").readNullable[Map[String,JsValue]].map(_ getOrElse Map.empty) )(FileMetadata.apply _) @@ -71,6 +74,6 @@ object FileMetadata { (JsPath \ "getty").write[Map[String,String]] and (JsPath \ "colourModel").writeNullable[String] and (JsPath \ "colourModelInformation").write[Map[String,String]] and - (JsPath \ "hasC2PA").write[Boolean] + (JsPath \ "c2pa").write[Map[String,JsValue]] )(unlift(FileMetadata.unapply)) } diff --git a/common-lib/src/test/scala/com/gu/mediaservice/model/FileMetadataTest.scala b/common-lib/src/test/scala/com/gu/mediaservice/model/FileMetadataTest.scala new file mode 100644 index 00000000000..f6a5f48b81f --- /dev/null +++ b/common-lib/src/test/scala/com/gu/mediaservice/model/FileMetadataTest.scala @@ -0,0 +1,49 @@ +package com.gu.mediaservice.model + +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers +import play.api.libs.json.{JsObject, JsSuccess, Json} + +class FileMetadataTest extends AnyFunSpec with Matchers { + + private def baseJson(extraFields: (String, Json.JsValueWrapper)*): JsObject = Json.obj( + "iptc" -> Json.obj(), + "exif" -> Json.obj(), + "exifSub" -> Json.obj(), + "xmp" -> Json.obj() + ) ++ Json.obj(extraFields: _*) + + describe("c2pa Reads") { + it("should read an available C2PA manifest") { + val json = baseJson("c2pa" -> Json.obj("isAvailable" -> true)) + + json.validate[FileMetadata] shouldBe JsSuccess(FileMetadata(c2pa = FileMetadata.C2paAvailable)) + } + + it("should read an explicitly empty c2pa object as unavailable") { + val json = baseJson("c2pa" -> Json.obj()) + + json.validate[FileMetadata] shouldBe JsSuccess(FileMetadata(c2pa = FileMetadata.NoC2PA)) + } + + it("should default to unavailable when the c2pa key is absent entirely (e.g. a legacy document indexed before this field existed)") { + val json = baseJson() + + json.validate[FileMetadata] shouldBe JsSuccess(FileMetadata(c2pa = FileMetadata.NoC2PA)) + } + } + + describe("c2pa Writes") { + it("should write an available C2PA manifest") { + val fileMetadata = FileMetadata(c2pa = FileMetadata.C2paAvailable) + + (Json.toJson(fileMetadata) \ "c2pa").as[JsObject] shouldBe Json.obj("isAvailable" -> true) + } + + it("should write an unavailable C2PA manifest as an empty object") { + val fileMetadata = FileMetadata(c2pa = FileMetadata.NoC2PA) + + (Json.toJson(fileMetadata) \ "c2pa").as[JsObject] shouldBe Json.obj() + } + } +} diff --git a/image-loader/app/lib/imaging/C2PADetector.scala b/image-loader/app/lib/imaging/C2PADetector.scala index 45ecfbda1e7..85d43f2038f 100644 --- a/image-loader/app/lib/imaging/C2PADetector.scala +++ b/image-loader/app/lib/imaging/C2PADetector.scala @@ -23,39 +23,33 @@ import scala.util.control.NonFatal * https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_non_bmff_based_assets * * The mime type is taken from the caller (already known/validated by Grid at upload time) rather - * than re-detected from the file's bytes: `metadata-extractor`'s own byte-sniffing (FileTypeDetector) - * is ambiguous for TIFF-based formats - e.g. a plain TIFF whose first IFD starts at the common - * offset of 8 bytes can be misidentified as a Sony ARW raw file, since the two signatures overlap. + * than re-detected from the file's bytes. */ -object C2PADetector { +object C2paDetector { - // An APP11 segment payload is prefixed with CI (2 bytes, the ASCII common identifier "JP" for - // any JPEG-XT/APP11 extension - not specific to C2PA), EN (2 bytes, box instance number) and Z - // (4 bytes, packet sequence number) - an 8 byte prefix - followed by the JUMBF box itself: - // LBox+TBox ("jumb", 8 bytes), then its description box's LBox+TBox ("jumd", 8 bytes), then a - // 16-byte UUID identifying the box's content type. So the UUID starts at offset 24 within the - // segment payload. C2PA's manifest store UUID begins with the ASCII mnemonic "c2pa" - checking - // for that (rather than just the generic "JP" CI prefix) is what distinguishes an actual C2PA - // manifest from some other, non-C2PA use of the same JPEG APP11/JUMBF embedding mechanism, and - // matches the check the reference C2PA Rust SDK itself performs (see C2PA_MARKER in + // An APP11 segment payload carrying a JUMBF box starts with an 8-byte APP11 prefix (CI/EN/Z), + // followed by the JUMBF box's own LBox+TBox (8 bytes) and its description box's LBox+TBox + // (8 bytes) - after which comes a 16-byte UUID identifying what the box's content actually is. + // That puts the UUID we care about at a fixed offset of 24 bytes into the payload. A C2PA + // manifest store's UUID begins with the ASCII mnemonic "c2pa" - checking for that (rather than + // just the generic "JP" CI prefix at offset 0) is what tells an actual C2PA manifest apart from + // some other, non-C2PA use of the same JPEG APP11/JUMBF embedding mechanism. This matches the + // check the reference C2PA Rust SDK itself performs (see C2PA_MARKER in // https://github.com/contentauth/c2pa-rs/blob/main/sdk/src/asset_handlers/jpeg_io.rs). - // See the C2PA Technical Specification, Appendix A.2.1 "Embedding manifests into JPEG": - // https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_jpeg + // Spec reference - https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_jpeg private val JumbfUuidOffset = 24 - private val C2PAManifestStoreUuidMnemonic: Array[Byte] = "c2pa".getBytes("US-ASCII") + private val C2paManifestStoreUuidMnemonic: Array[Byte] = "c2pa".getBytes("US-ASCII") - // PNG stores JUMBF/C2PA data in an ancillary chunk named "caBX" (case is significant). - // See the C2PA Technical Specification, Appendix A.2.2 "Embedding manifests into PNG": - // https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_png + // PNG stores JUMBF/C2PA data in an ancillary chunk named "caBX". + // Spec reference - https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_png private val PngC2PAChunkType = new PngChunkType("caBX") // TIFF stores the raw JUMBF box directly (no "JP" identifier) as the data of a tag with ID - // 52545 decimal / 0xCD41 hex, tag type 7 (UNDEFINED). See the C2PA Technical Specification, - // Appendix A.2.5 "Embedding manifests into TIFF-based assets": - // https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_tiff_based_assets + // 52545 decimal / 0xCD41 hex, tag type 7. + // Spec reference - https://spec.c2pa.org/specifications/specifications/2.1/specs/C2PA_Specification.html#_embedding_manifests_into_tiff_based_assets private val TiffC2PATag = 0xCD41 - def hasC2PAManifest(file: File, mimeType: MimeType): Boolean = { + def hasC2paManifest(file: File, mimeType: MimeType): Boolean = { try { mimeType match { case Jpeg => hasJpegApp11Jumbf(file) @@ -70,9 +64,12 @@ object C2PADetector { private def hasJpegApp11Jumbf(file: File): Boolean = { val segments = JpegSegmentReader.readSegments(file, Collections.singletonList(JpegSegmentType.APPB)) segments.getSegments(JpegSegmentType.APPB).asScala.exists { payload => - payload.length >= JumbfUuidOffset + C2PAManifestStoreUuidMnemonic.length && - payload.slice(JumbfUuidOffset, JumbfUuidOffset + C2PAManifestStoreUuidMnemonic.length) - .sameElements(C2PAManifestStoreUuidMnemonic) + val mnemonicLength = C2paManifestStoreUuidMnemonic.length + val mnemonicEndOffset = JumbfUuidOffset + mnemonicLength + val isLongEnoughToContainMnemonic = payload.length >= mnemonicEndOffset + val mnemonicInPayload = payload.slice(JumbfUuidOffset, mnemonicEndOffset) + + isLongEnoughToContainMnemonic && mnemonicInPayload.sameElements(C2paManifestStoreUuidMnemonic) } } diff --git a/image-loader/app/lib/imaging/FileMetadataReader.scala b/image-loader/app/lib/imaging/FileMetadataReader.scala index be75c2eb482..777ecf8dad3 100644 --- a/image-loader/app/lib/imaging/FileMetadataReader.scala +++ b/image-loader/app/lib/imaging/FileMetadataReader.scala @@ -58,8 +58,6 @@ object FileMetadataReader extends GridLogging { metadata <- readMetadata(image) } yield getMetadataWithIPTCHeaders(metadata, imageId) // FIXME: JPEG, JFIF, Photoshop, GPS, File - // Note: hasC2PA is left at its default (false) here since the mime type is unknown/unsupported - // in this code path, and C2PA detection needs to know the container format to parse correctly. def fromIPTCHeadersWithColorInfo(image: ImageWrapper)(implicit logMarker: LogMarker): Future[FileMetadata] = fromIPTCHeadersWithColorInfo(image.file, image.id, image.mimeType) @@ -71,7 +69,7 @@ object FileMetadataReader extends GridLogging { } yield getMetadataWithIPTCHeaders(metadata, imageId).copy( colourModelInformation = colourModelInformation, - hasC2PA = C2PADetector.hasC2PAManifest(image, mimeType) + c2pa = if (C2paDetector.hasC2paManifest(image, mimeType)) FileMetadata.C2paAvailable else FileMetadata.NoC2PA ) private def getMetadataWithIPTCHeaders(metadata: Metadata, imageId:String): FileMetadata = diff --git a/image-loader/test/resources/c2pa/c2pa-absent.jpg b/image-loader/test/resources/c2pa/c2pa-absent.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e114f92fd44233d24f84cb1aab76d22d05aa1fc4 GIT binary patch literal 2529 zcmex=``2_j6xdp@o1cgOJMMZh|#U;coyG6VInuyV4pa*FVB^NNrR z{vTivbFC}Pw&aUqAY)5e3MK^H%$7$+4qadL@?OGrwo zs;O&eYMGdtnOj&|IlH*JxqEne1&4%&g-1k2C8wmOrDtSj6_=Ejl~+_&HMg|3wRd!O zO`1Gq>a^)IX3ko)c*)Xb%U7&iwQ2K~t=qQm*tzS_;UhlG{sQ@nk%1ZFEg%W;7@EHXf&OA*VPR%r2lxYE z#}NLy#lXYN2#h>tK?Zw6u5*IGf5-=6&_QGasrx-W(TmPL*$ub(|M-vMqlw*8Tal z2@oJafB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly PK!5-N0t5&U_>;g7@4@;T literal 0 HcmV?d00001 diff --git a/image-loader/test/resources/c2pa/c2pa-present.jpg b/image-loader/test/resources/c2pa/c2pa-present.jpg new file mode 100644 index 0000000000000000000000000000000000000000..0976984646a0438a21260f00cfa30972405493bf GIT binary patch literal 38312 zcmeI4X;f2Z8prRFm4qb_77--uh!8H9kOWQ{Sro*B8{h(+Q@lxTLMQ}G!s5bIup)?U z#nw{kv}Z)E%k=bA1*=l0Dpsp@Izv@Noz4`k)^?6lP_$}ihIwxYOSh@^OwXAQ&kH#r z_df6P|3A;o?+fP{?i>CT^O&8Por&Q%hT-T38~%Iq`?Kd0GK~9-HFuR%mg&&nQ_#CX zPggLQ4D(oxVS6y`tIJ5o1r)W}8I?5>Rrc+uN~ZPZXYt%gmc)GD;sk{6xJO&wW|v3+k-C1bO(R=c^}VnY)a{na@klbeuwR!VoR zpY_^KNp$bfGZ|(tZf~W%7zwd3=2E4;lrfi~S(u-mwiO$1G`MfvKY7v1WZSwND>V&W zm+OoM9cC_LOl(|91#9AVSfw)?bi)(;B1Zi>ipfMI#=}Tlh7mHHl;OiyunbHj5C{c) zkx(cUi$yL{xvNwnkxugTa+e4C2Tcj|4+v0%DI*mjF`)qgQJSeSRJ=;13W`ih(b7*S z6I3*}2rd?jr4p&1tE(R!91u)D_82~a1)+2DamU~^a|0(x9$z37iCrWppwj~*aFQf= zB%jaYp}V!{KE{*rJ%i)Z1YQdmVaPH$UDvWz6q>&Gn0Ic^Kv;s#S}%6-@%5YJA081I zH8on5sMctck~3z^%*>jVJv%Rd;iAP$3JUcG)@Uj=m)Pu;Rn;}iSFCwlpYu~ng#~V9$?cTF*e@EwmcMcvpe7yU_`zKF*aQe*0=gwc~z1Vl@a{tw9 zpIyK4`OPoB9K3yJ=~6R>B5oYqvHp4p>*H`k4N%^oGzTG=7h_5{NQ+jXW9ZG zv&<`mt`o`ATej{!CJs%=9q`s!dt7|NRQ=(DoHU2BM-x{6vy_bq8`pIilae?pBT|M< z$A16RgHP9Da%?`H%7-_w0IC2U1o$q1uR|aL&=6<{Gz1y~4S|M0L!cqh5NHTA1R4Sj zfrdaspdru@Xb3a}8UhW0hCoB0A8?Z9YxHRHR+BThuH88DSE-{xuHMQrcGj@WU^bcUOle+u zrB%=7FRx&0Qi|>NisZOBeU;K&Vb?3mt)@5&TW!;q8(3SMGv4Nix1;#cD;Z3Mz0%5} zeY>3kQh!`PF=H!M8W=k>cK+%Idu#8Wt-U|_E!jVQT|4J(bYmrV{v9Q3jlQ(f#`XTK zYiDhC#%h-vOIeG_UOcJ!l)=K3u@-JaqoRzln2jhx&y<>swqk}(R2#UBmMAvOuU*l2 zXr+Hx#WzO|D|Y)t-~3nMspe0fay$NJ({0&Dh58D{kj)*%IS8v$LZm|ituLLJ14kd^ zk-ows86YyalkO}rThQ?=dNzATO-xN$MY7p!NY=86ns_xu#U|?F4Y8^u+7PRy8Dp%L z(kIY*TBB8~DZ8Ubagm9&uvW%iZY?rfjOE)L(u%oc*~`mIO;xOwGr>eDX^oPms!>{$ z`f`h%wb+?Td$E!WEVA0r{Uj7ZnQdm1h2b(%Gsni38bci(rNnVzagWua*)c^qww%>j zIYr|=nd{p1mDbXlsT1~b)Yy2$$Qol=ccz85_jTndYmC~k!lm-$#Fn+4;u&WShM$f8 zYnIp2Ayw61 z-rZnzJ#|yu>$B&|$l}pIwfOk+=Umq=o0EAY|4G~ZQ~TrIJN&`(o7Tu`Fs3Hxuvz}O{7)EhVJJ)$T%2{zg&;)xd1t^S?aV7Whvju#9 zJ09T|Twg#mQ}x6;^iuwgM4gZcc}3WbuqyswZ_(te$_3{i{){c^O;Hr6E3&3g@o5NT+1b^V1X# zDcPt3t!1pijJ{12x#i_{1)4mG^5p_0Qjg4exr+RmdHLCMvx?@;%_)fYqkOnqq}2Ue zGc(etDZZ{WO{wFlc&$pUq6&Ro6DUectDPUIy49m6e8l4hQGp}J^%*~I9(pRw7Ly_a zwG;nu$COi^Bgd3D=5o_mD^);w34{*+2;6BTi3`bejOSkbhn9_Ezl}Sz_t)R%N5qHy zvHzORc4?ncG~V;tH`FzMXx@-6MNJS2l{>ZPL@ZzI*|*y?$Jz~Az`s+KI@3V_7@NqREZ_%nh?)Z0mT&0gxSJn`3c__QL z=_!VCAFDw=g*vVef4ki9h6&~NgVi*#+@9KUu%W+Z@}-LxU9rqwl7iQlOYZ>^fbKG`$af?^z|Bcre7VJCO$ zb8fATj;XDVQ&w4;=TAR7(6?eMhwy^vj!}ZcJ2(IifCJzFH~Z0ojT-ZyDZ{?%LA+o?`m&6)XcI34;4YgNJ_kB0itT7K7j>c7|zEXW7I{QKxY&PMPe5T3h4B}2%IDd9?6{=Mc1PH7*EFc4319| zcr9RrA)M!Ie6&s@$M7vpFH)!=`$anJAa|~V&A39{a3GjcKycZH^2CD@b;adyZ64n z|KK5~3&+quNjt}gvhV1UAzcLePko+{(}fe&=tatS{NQ+jXW9ZGv&<`mt`o`ATej{! zCJs%=9q`s!dt7|NRQ=(DoHU2BM-x{6vy_bq8`pIilae?}9x217V<+0zPrG~Wc53_j aC=6cU0n`B00Mr1~0Mx)QQUm1hrT+lGjA_gO literal 0 HcmV?d00001 diff --git a/image-loader/test/resources/c2pa/c2pa-present.png b/image-loader/test/resources/c2pa/c2pa-present.png new file mode 100644 index 0000000000000000000000000000000000000000..87c9a3ccc2f4162a16429012c1e4ca8a5b3a81ae GIT binary patch literal 36850 zcmeI4c~BI07RP_xJqH7WFn|ciF$jndm}!~;W-vr9L9zrDNCbB)*wfQ<;4lL-Jupc$ z3L3;~)isHUIZ9C1V=JcOkwjS&5?vF`u`w}%Nj4RasB4u8#v>+c$^Lp65qFbKvb9^a zTlJdiY5Ldi_j|wheXslT-&;DPsysL_CJ-SMTrsV*8X;meJ~IMHJYxLV^#nr8dpWka zPP{gHnv8fj9$!ry)660j3SEfMcBFr%h4Kwxt0(+c<-=A@cUpN|j(pKFKaV%FY_^fk z)amuPnL0DaXBzc7V`g49uVc8poNOJPgMHPl^)Ht;yd3NPC#M?L?dDyA&FOIC2@ibk zpHL=FD11lB^e&tB+?Js9<5w1x*uq8M8o`PKaj-VKhO@J_COiv8ag5ta_RmXid2{8m zj*I;kp`v>{9*;1ZO-5vEVlBM3v6;7s9_ATsCgae+&=)B5-m;1^L}0B5{D+46kSc&e z5=0RyM5+i%MGSq4N>G4ICYMPA zSv&}ml1OFp07YOBHrNx2NP?nB2_=3rk{2WPC(y+v=B3bxMRz9I6f@xpV9gtM~ z(i>x|j}9hh8(kfWz{qh?(J?6#QzxaR>vD4Q^oIPB(y3+T(<<(*slES!S#|ZZITLTT zSZ$4N!PDC2edysuiy!M;(zSHi@76y4#FOiudU}1&#!Z`lzh&#T?K^kv-t+P+ukPLV z=7EEMI`r1thmU;t(Xo$@_Ya)-9qnm8;jjz5d_k&Og)JFZWVk|J;!Q7SYA{o%pyzIY5ppczD=6fPhEQ~|sQ@GXGP zArJv*2s8v50u6zNKtrG*&=6<{Gz1y~4S|M0L!cqh5NHTA1R4Sjfrdaspdru@Xb3a} z8UhW0hCoB0Ag2%;UzX$vQ6#jSsR@QCRm{@@w*}v(G?wPkg@9ke7 za_XynftgkK!6V}SH#hQL&hBxGy?@(#c(=g1gs4Qbop)FSYxMHBOb)h*cZeR%>L%7< zGh-8uwOhH#HP(tPeOn#h1I*@xdB%$PjpSfnJr&LO<^UVdD!XAalQ znx=_+89qptKZI0Y2n=T*odaJVdo*rHRs!GrhEf$ z&N0&Zyv!Uf&wx82;~%3TkIAGBIk~K!=k$4uQSkLBZm{qU-o*+|SA)%Ac5d`VW)+hq zIGuJ&EAJ9Tu+SPNPs7k{I4v5^=@58_z92suyfi40hdf^QqqFLbTDF4wF_BmoL}^JKtD~>r-8X+gXpUQiN#nZ5KOlPx4pX z4>Z9ZNdY#aRbt6~xVT;_?I9*cC3e)4%jphs2|hI&DBKB|jA%|io7^hBbi84Fxo76F z?{Dn6xaKS!C<{pMoc4Z!gd*wo6e$L5(Lz)8jgd_yg0pE_&*b`V>Gp-UcK9>5 zn?T16Z#VMxc5CokVRKm2CAgjVzuIO59X7npAm3bZ8t2R6cryoi9g`@~MRg5Gd z*4NgoRz$7dtiQhO8|lQXJvknL?4e5K?rGquBvKMF&hY1tw0XbnTB%QcuDG&hH^t-}p%X;&Bnj4iVZ@kuD_!Q-dk%kz z9{+leW8tZn$7tt{E(xj}(WRu?ua* zv&PQ$pz4>?#=Nt2#ps+<6^Je)6VaaIZGWpOes^}k(W}o}R!s7)3og_Umpf9{wZ`+m zK6+^lHu-q!!0dh%m43Pg9ooWGUVJP)V}6@f)9P40W6C>&0}pqL1g}ru?FTqqKmkwy z6aWQ40Z;%G00lq+PyiGF1wa8%02BZPKmkwy6aWQ40Z;%G00lq+PyiGF1wa8%02BZP zKmkwy6aWQ40Z;%G00lsS|C$2V>Iy$Uo3m`K_xz2|c6Xc$jGDe>FHn1mQFt7p&VfgUN0Km(|Q g|HF5oZ|im9{W9vtAOQqKjvye1BpHT40*|a9$l-c|RaB zU*}3K>M6DQv!y!QTV`iV&Q8@-*VM$P*OsU1YpTnu5_MJOwbkiV`OI1A>e^ZHWV$95 z5BplaE!W)Sp5~Gr|E<%CWJgE3J=4_M(h*kp+P~x~EX-Dz`$BQzvFjJ#eowzCuO3)8 zziDu`?}|)gIFXiQQ*%YCIoY&2tQ8DTB{~{kTfXOnKJj0!`;R}_{a3-vSr3Qzht$^B z)do$glMU(0Rc+~pY!8>#Hr3T0+I{F3LCGl#mn;k-p?M^`2#1~wO8dluyl`PkgJ@|a zRvJ0<^I(3^r=Xy)AiqyxVPR2GpT5OI`WN@>SA6uK!2^ek8Zl<{s1YMamW{8NSa$5x zaU(}gs-8SGKCP;%YRtr0b7~SNR7|f*WE+VT6%`ftD<0mz|M0}vkz*5I`ElskU`#m9 z{7CmBJLl{p5{>2M7Zmm>>f0|Yu%jfqzhcq6Sblz9Uij%{;pahKY5t(G)8-WnJ}X&x z?1e)TS8m+WXWU8mJ$2O5m-daHUfaIDsPE8W!;c;@;kb#DCQqrFF|)d6*6jHUPF}cZ z@sd-Poqf*N&RxFZyi{GfzM-*cRY#_C&DySuF8;mu&$}0DU)no*4o*g zN}qjETe@pbVVfZukA>+)}=ctbLBg_%V)y!pWd0DY|C`E zr^9~V$W37G7f+xu+0j^0m&_y&kALLftSo)2{-gc%>wYxqlc9Tdgdg0R9sk{{(p{lDd(=82|#=Ktl_U7&#ya zpj2D3ZgF;qJ&$O6Ze|m^XO>7c_gM;G^a9Ktnw?J+}vHI;&5~Axec}Q$WhH#ZcTNz zH+N0$)lc`cWDsBvp)ja1Gy*VcHHNjjeU3t!ea5# z?3Ua0%@z6iTO-E}AG>}Kyq|Q+xU0JugQP^)s@FA>OAXbpZs>m zuQt9D?_1Dk%GHbiaZX+=8o%;}Y5n3w1+jDT3I<2dS@!9hKE=`b^QH}omt;RFEG}Bs zx;ni!yA0DN#wTR&^(`JfKXK-~vhJxY33p)o>U3RG_#;!cw6!%;7FIqwJ}g_XUvbI8 zGnST}ee$ximz=(6s^M>`G z9*@@~X6D|;ue{>Z5q`x=91p#dpZrb=4_x$>?{A-{8`Q!JW zaB0iF-9z6#xZ$*8elzl#wkMz2{9N$iB{P~5n=%EDtlam|{Nsat@7~!JUlT75yBsw* z5;>F?jR)OMqb6kE6=#?2_<~4&eqk&xJ|J6aVD^5vhy_n}Pjb|NyrFqRhRyn?mn*Ap zy7tDJiMOA0>ay*z#Ej?TW3t7^<%RB73>k3gjqUY6+`9Aa;I?1yXt`qVg99or=)EOk zi-P!qXxXfr4?ZyC&HI<1IQ`g@8=4=hSyy)L!NK<*ci&-{1Vv=85<2EV<*@@#W9IwS8*vz=~)~xD!`}&w;t!cv;(PBiB7T;-@p` z&OLYchMlip_gtd0ch~W?UvYIu#D`~BsBdxUk=6YY`>&+iFWi5Hxx!zx@2U^)yZ>Kb zlP}(b7p2F2nChlGdlhCm)_qP7J235-RHel8k_Rwvf#OUQm6jvswq=1TU%MNre(vK zC%&+6_r+VX3BF>=_FRB_AwYlt0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FqeJ6F9hh?g=BtozeJ%XJ7u)l1E;-^_lOzzk1(m-}>>Je|OsC9Xl&F u&wSy)+*_NEf9Q96SKM@Q>(C2M9&r2z2QQjX^x)Hx-|d`r>BhmQ4Eir>A33N1 literal 0 HcmV?d00001 diff --git a/image-loader/test/scala/lib/imaging/C2paDetectorTest.scala b/image-loader/test/scala/lib/imaging/C2paDetectorTest.scala new file mode 100644 index 00000000000..0fe7fe792f3 --- /dev/null +++ b/image-loader/test/scala/lib/imaging/C2paDetectorTest.scala @@ -0,0 +1,60 @@ +package test.lib.imaging + +import com.gu.mediaservice.model.{Jpeg, Png, Tiff} +import lib.imaging.C2paDetector +import org.scalatest.funspec.AnyFunSpec +import org.scalatest.matchers.should.Matchers + +import java.io.File + +class C2paDetectorTest extends AnyFunSpec with Matchers { + + import test.lib.ResourceHelpers._ + + describe("hasC2paManifest") { + it("should detect a C2PA manifest in a signed JPEG") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-present.jpg"), Jpeg) shouldBe true + } + + it("should not detect a C2PA manifest in an unsigned JPEG") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-absent.jpg"), Jpeg) shouldBe false + } + + it("should not detect a C2PA manifest in a JPEG with no APP11/JUMBF segment at all") { + C2paDetector.hasC2paManifest(fileAt("getty.jpg"), Jpeg) shouldBe false + } + + it("should detect a C2PA manifest in a signed PNG") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-present.png"), Png) shouldBe true + } + + it("should not detect a C2PA manifest in an unsigned PNG") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-absent.png"), Png) shouldBe false + } + + it("should not detect a C2PA manifest in a PNG with no caBX chunk at all") { + C2paDetector.hasC2paManifest(fileAt("IndexedColor.png"), Png) shouldBe false + } + + it("should detect a C2PA manifest in a signed TIFF") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-present.tiff"), Tiff) shouldBe true + } + + it("should not detect a C2PA manifest in an unsigned TIFF") { + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-absent.tiff"), Tiff) shouldBe false + } + + it("should not detect a C2PA manifest in a TIFF with no private C2PA tag at all") { + C2paDetector.hasC2paManifest(fileAt("flower.tif"), Tiff) shouldBe false + } + + it("should gracefully return false, rather than throw, when the file doesn't match the given mime type") { + // a PNG parsed as though it were a JPEG - the underlying metadata-extractor call will fail + C2paDetector.hasC2paManifest(fileAt("c2pa/c2pa-present.png"), Jpeg) shouldBe false + } + + it("should gracefully return false, rather than throw, when the file doesn't exist") { + C2paDetector.hasC2paManifest(new File("/no/such/file.jpg"), Jpeg) shouldBe false + } + } +} diff --git a/kahuna/public/js/search/query-filter.js b/kahuna/public/js/search/query-filter.js index 763c9c99f29..f33026a7197 100644 --- a/kahuna/public/js/search/query-filter.js +++ b/kahuna/public/js/search/query-filter.js @@ -16,7 +16,7 @@ export function maybeQuoted(value) { } export function fieldFilter(field, value) { - const cleanValue = stripDoubleQuotes(value); + const cleanValue = stripDoubleQuotes(String(value)); const valueMaybeQuoted = maybeQuoted(cleanValue); return `${maybeQuoted(field)}:${valueMaybeQuoted}`; } From ecb15875297afdddec2ee16f3c9e1c4c45509cc0 Mon Sep 17 00:00:00 2001 From: hk-15 <168672047+hk-15@users.noreply.github.com> Date: Tue, 8 Sep 2026 09:53:05 +0100 Subject: [PATCH 339/373] Add grid id lookup to AlamyCleanUp script --- scripts/src/main/resources/alamy-grid-ids.csv | 408 ------------------ scripts/src/main/resources/alamy-refs.csv | 0 .../mediaservice/scripts/AlamyCleanUp.scala | 39 +- 3 files changed, 32 insertions(+), 415 deletions(-) delete mode 100644 scripts/src/main/resources/alamy-grid-ids.csv create mode 100644 scripts/src/main/resources/alamy-refs.csv diff --git a/scripts/src/main/resources/alamy-grid-ids.csv b/scripts/src/main/resources/alamy-grid-ids.csv deleted file mode 100644 index af38bd4c402..00000000000 --- a/scripts/src/main/resources/alamy-grid-ids.csv +++ /dev/null @@ -1,408 +0,0 @@ -c25c83b14e64c512846b1c0087d5b6a1af8600a7 -721907529e555d8fc165921790954efb6c03c9d3 -798080a703b54464d0c2c7a85b7051095bf7cb05 -a1a0d1d0c502abe7b82d797e955f2c9872494371 -070d69226d3b60b02fef9668d9ba6aef3b1e3cc9 -f8bb084cab9b18780ae293a557312c652bff4d98 -a927c37cc045e06a1379b8db6ee4e4836a713e66 -336c0d568faee1644a3aed23f5b86867c509609f -dede3c95bf1847f02f0bf07f86d1d4e88e05f36c -cce40acf10c1e0c6fdec7def7e883d0c523d8a78 -212a20eed262bbc07b65506dcd4b174c57e383e2 -23681f09f71647d44cc0bf7b6ec97350d23e6b82 -0b972c6af65573094c0ecab591e86e72d52e6d69 -469a9a9d323aca5436fa58139a1b9f1bc5d8170c -d043dbb6b0b07f54e588c08335bb013ece594c72 -603f82954faefeee93748b0f9fba83c76737057d -81ad6539ed1e5e988c5f271d006f565445178d54 -f7f5011ec5c3ab731fb2f5eb3b5a4bebf2f07295 -12026b219936ceb74b61ad7be521245cc81fb9c4 -f6e39eb25745cabe8f4038b1d2c4628a0e077742 -244d192c9774875d02a9ea3a692bb83dea3d34e6 -3222a0c07de556f870191dcd33b8934ec09a094c -b1381980282e2022ceb6c16b9f9809f1fd7ef7cd -f5bc2738fe0d1a1eb21c4c050aa78bd62a0c73e4 -5c2fd0854ed4255492760dac3cefbb758de1a637 -a24841264e94a6a2649e4dde2035933bbfff1179 -8d1173806807ac6d5aaa6b0c6bb5fb75e3c6f620 -7688359c6635869b46aab378d38aeaf6974adedb -ce8a1a85f31cf377abec0ae72d5e990e47e4b4dc -d70c24da93fcfa039df9d2952abc31992ba551f9 -7ed5b3497b08c48f0b4f5accedec7bf6744910af -d6465e4ab0fab58a549357adbca5c175cf3201b6 -fabb0bc8bd87fdd73e32e4e0eae880a8041bb6c9 -74bbeec351056b5e0f5494646ca0014eba8ef5f4 -d907f17d4d14f0a9fb921c4ac2cd2cf0d2c28cc7 -e84c2e1e5c25b14f3843b4fb8a1afe25fbaca6be -20cf8404c8cb4d0a5822ab17532d1425899b72b4 -5551c3a698937e37106b2474b178c91551c05fff -5f68bd0efa47784b3b34294fde0309568a3480a2 -53d9a8b96d37b4b5559c6dd7be805e80bada5835 -cce31eabf26cb4d45cb72413d68400423377bbba -54da8b48e0bf5d1e71ec22aef32d2474a8556783 -8e6d4623d65ab4245f8912b7ec1d07699010d64c -3c3206c447aeda924e4e4f53c07bf8ca00fab911 -436758ffd587e7a5317f590dd5383ea8485506be -0a3c9c34209474ad04107084f0d0c1a8f25fa756 -9a0ad16d4ba3e327dcd6e13aa36292525e9dd4d0 -a4eaca0f0781c06cb30bcac7893a992aae6a2a0b -f9bc7bd11b56882090eca258216292e1846e767a -be2428822e83768a771005c9eeaf19d5a9826550 -21f2f352d4e68012f5788bbf1a4c9ef7029f49ee -6f4d23e836e293450ad2c6c06ec38300c9bbf55f -4e126536a072a1ebcdfec6b71d6fb3721a500aba -b03f7b5387001ebc410031eedf00ea02d66b69b4 -f311305e5203b69e162441b531569c077fbfefaa -50a45eead6c0a1411c1928a3dfab202bb07cd663 -6583e32ad457d9b0d6071da903d4401e2bf4fe06 -c9bff191836ea8665d5a5393308a9fde156611b9 -6514c9ca0700ca4b69007b70bec2406a873a017a -b47c87878854e94e582100cab3997a82086680df -74e5fcd012a6a3b18a78197a5984e9d9f1f9c403 -4ab6010b4fd8c1fde9bfee34610cc0f9bd7acfbb -33be2e7540d11454c5d2cab8c5781f4779a1be31 -b5c4aa837dbd73d746f564251008ef6669d56a86 -51c63060c595f007e17ebe3356ad4b6ad1d7495b -3fd6976999c64eb3c3acb781b04ce7651fa99401 -186f5c845e754cf95e3a343df25777653f3c6da9 -afe8a58664fbf2b25a9e522dec1654f860c76c83 -ee428a9abcab65637f1516fd215dd72f2fc3d1b8 -0400e8b9201a1de3f77be664401a76264894f7af -815807d3b802eb07323ebcc37bd2447b9619c257 -4e6fd8229501d89997660f80facf81b2df8e3f5d -d84d9550a530d6c92c4f89e295abc1992481b85c -9cc968ed5bfbed46b9521296fcf4b5512107428a -bcf040519a5be80595dc7e87922a8137e4e416b1 -672ee5d594cafd68b51a79676d590d84fd3703a1 -850b3ea2a57086dd01b2de7930e8f154a8c5c5dc -579ce130320c2dfb215d2cc7678c94c83b24c675 -f041c683c0f6eeec97383f6a2ccbdb45a7e1a82f -68db57f2ed049f7061218bc29a183606072fd59a -f23eda9654c4c20c1a2c156880de0b84d407b8ab -510f149925a66bd5dd204912a0d085bcdd0af5cf -add237d61e4b9c5b5d675c0605d605b51ca1e675 -0aea68374ff6bb4612f4e04635c810b9396a7f92 -db22ae65330227b72a9eb11ed954516209f6b2ca -5b6f84e342df55e0a09c71e79fe4451f40efdfac -60c61927616fb8b7cdca6ee363fdcf873499aa55 -603f82954faefeee93748b0f9fba83c76737057d -81ad6539ed1e5e988c5f271d006f565445178d54 -a4130e8495780b082a1a295607cfee3c8591b8bf -5a7e9c512a9a31c5024c5dc143609762589a9579 -343cc9405e0516d2295e60096e0976c2bc1d8af2 -949f7064366d6cd1c830f4089ab76d6392ef2a0d -734d8665caac1f8c9acfcf6ac7fa76316e21f282 -cf2a26132d785504f2d3f1598157389e1cdba89e -1b697213601ccd2ddbf7c53baff30b08ec74a0b2 -d134d03cbbdc45f5eeaca05df0320e1a712fe348 -e718794c29078b9ab8b557e52e2413677e50220d -9f0d31e2e30aa0f94e751ddc554ac197b6700816 -5a94ce41203c91bb390b84bdd79385e580836c8d -14a80ce02afc9aa71407a8bfa5c9258ee395a24c -e7310ee958d6118f931bc1140b5e406f28d2babf -846f64a664a2fecd64bfab0b85cd52e5da493ebb -739db6ea9547844e2146103f0a6fb789023a38cc -5849aa16572223dc81e924237afbe6e25bb2e014 -dea57ddae41c2cab985ff1d30548651ea51d9c2d -b1381980282e2022ceb6c16b9f9809f1fd7ef7cd -916b77bbd8a762d915a14bef06335d4ba8f41645 -be2428822e83768a771005c9eeaf19d5a9826550 -0a3c9c34209474ad04107084f0d0c1a8f25fa756 -736a485f91a47524989ca47d56a8d8122e6a9a71 -c6a4d1044cd25267f1834fe492d5b0dfcfd6f66b -ce8a1a85f31cf377abec0ae72d5e990e47e4b4dc -28a01d34678a07ec1b6c7f243b493b499c5ded7d -27b96e7381a2d555d513066a159dbc4992dd9cb6 -250e952b6cde8bd2db1265254779911ac53eedc1 -916b77bbd8a762d915a14bef06335d4ba8f41645 -dbf3fc049b76378fdc26eb9832aa8a777e4b5cd8 -a449bc8d0aa453a6a1678ee9e4a315ac56573ecc -18da3ccbb65db9d1a04df85c039e900aaa4a37c5 -bcf040519a5be80595dc7e87922a8137e4e416b1 -40f664b1ec94283eca210fa92a380307ed45a2eb -d555d18d2ab277d151f5e2f7983169aedee3a93b -28a01d34678a07ec1b6c7f243b493b499c5ded7d -ea6483a5f87f7bce5a6ccae68c70fe9850af65b6 -e27e5d26d0ec1654dc9a22403187cf45a5005ecc -d6731f5fc06a6790390d46427d4e85ee2058feeb -2929ec258e7d012d1a0850177b8db4c6f892d06c -9f0d31e2e30aa0f94e751ddc554ac197b6700816 -d040b5953ba51905a638ba7c12db87bfb3f1b953 -736a485f91a47524989ca47d56a8d8122e6a9a71 -2db26bdffd1e339baec4c20d73ec04058fa3a1fe -b9a5b8634c0ead531df3b2e220ff08dde6ddcf56 -f23eda9654c4c20c1a2c156880de0b84d407b8ab -a87b38b208c9cd539ea00b4d48ab95d5ded98871 -8b9a7e4d5a0623e0df0246ff0ff1184b2a4eac80 -69c59cc1aa4dbda0fbfa25d3da9a9e4d8d452b95 -15710e501152599e2666f449fa57886fdc7066e6 -8a8747ef54144550db05c16905e95044407320c6 -c3902c51a8552f5139421d8cfc19465b1c9fe31b -2fe6bfa01b1a7999c911018ec8b3f94ef4254a06 -164d60054e3eedb51fc3e25ea02ce280d541634e -784ff703e263f7182dfc7972434be1800c720da2 -813b7668bcebc18361abeb614175c7b04b66ae3a -8cd1eddcd6ac922be4acf4963d0e6a0cfbd15487 -53d9a8b96d37b4b5559c6dd7be805e80bada5835 -436758ffd587e7a5317f590dd5383ea8485506be -9092e7304c3a871e6a03e6353983571cbb8154d0 -b7ab57d433a8ebdfc28568b8d7a284246ac53297 -9c95a6b639203dee64c7b5d4269f805f8eea2bab -bf07d426fd3e88fc1c749e541c4f3f368da4c4ed -3c3206c447aeda924e4e4f53c07bf8ca00fab911 -565bdc2842e4b2a70319af1f06be93054b5c3adb -4fbbc00359ab6c02928d71551a3b17b961d1cb79 -409f0b2ac2bded6151635236e851d0436e7cad32 -88f3993b3f4bc3a5a204bc00241e21991589a0bc -00fa29f4bc850374b1e142e9e2a7e2e8ec116225 -4acc80126af2142677241e08247aba08bf43390e -6670a757716611a50df5d890b93b6fa43e67c8cf -ac75780112f4fc647c3b9871c23b19814d292606 -27b96e7381a2d555d513066a159dbc4992dd9cb6 -6f4d23e836e293450ad2c6c06ec38300c9bbf55f -c25c83b14e64c512846b1c0087d5b6a1af8600a7 -a1a0d1d0c502abe7b82d797e955f2c9872494371 -5f68bd0efa47784b3b34294fde0309568a3480a2 -3222a0c07de556f870191dcd33b8934ec09a094c -4569becbc6615c9d8172bbadc9b7ac92eeb41738 -56fdbbaa95c2a35597df108d0df42c5b16103f8a -a87b38b208c9cd539ea00b4d48ab95d5ded98871 -c514f54661dcbe6fcfcde537e03033cc506c5941 -fd13dc4196edbe95e083d255077eba90fdf82706 -54da8b48e0bf5d1e71ec22aef32d2474a8556783 -db5c001671006f354e4aa9185ea9da4efef0c0e1 -40f664b1ec94283eca210fa92a380307ed45a2eb -42302003c8aae5309f704ed08db93aad952f19c3 -0abeefd476d4673e2e61838f86136dca290dbf91 -f1ada0201f2f49011f77fa752fa98a1edfe185dd -9c78355fa1f5d0d47eebebd859fd655dc8f226d7 -95cb715d92dd34b62ee38795ca10a82b0a7a8d77 -85ec62654a739fbdfcda5e5ada82929cde803303 -cb6841ff33e2817cbff4efae3ae18b4056954eb3 -c514f54661dcbe6fcfcde537e03033cc506c5941 -00d10a93b1fc29abcdd92868a1a9bf62c7555b8b -7fd4213b7a6cb7990adb5f28dedb776973d2c8f6 -88eeeb0127735539d2b1190b47b3d962f2acf617 -a9cd174c61d7c5d68525a297b82fba830e10456e -5751b33336190ed56ba36ef878794dbea1ef4e9d -6d1638b9389df49461531f2b461aab9eac6d9147 -da6ab02347425a235cf663e28e9527a08f68c1de -517f6b1ec0142f8a6f3e2e22fab7c6c035402dbe -510ffb6acd703a7e5ebcf64a2b0fc984d8ad466b -37be64d29826f5285e5ca49955c9f7c6e27e4994 -b4dafd2b8617c32c122e0405a3b93026477d9a04 -f041c683c0f6eeec97383f6a2ccbdb45a7e1a82f -fd13dc4196edbe95e083d255077eba90fdf82706 -f7f5011ec5c3ab731fb2f5eb3b5a4bebf2f07295 -4832fc9b197101f80822e2765bdad01ed30b69a0 -d70c24da93fcfa039df9d2952abc31992ba551f9 -f6fa4f9ad569d07380da29255537dc3dc602dee0 -24de44a4e9c717ecdceb03ebe1a8452a8ddc0cea -95cb715d92dd34b62ee38795ca10a82b0a7a8d77 -77cde73a2bbb3a64a3bf7cf94c088b990b2544fc -44d85ede47587d5c4e2b43db2ed8795e7ee9dfaa -b7fb2e93c164b90651538aba2f0afa41604653f9 -7b35dde59c1630b103a616ab30ebbb795b4d3bb9 -515c1ac2389b8d807b6aa609964aaa4411fd9811 -026256ab77bb4f1935c2120a4f5800d51700a636 -caddf5f0df825159e83273dc492ec154256feb46 -00bb6450cc1e1fdafcd143ac23353a5760343285 -76e48294efd9aa35732bf9a8a1bdf82133185525 -92917951118c2890636e8569d1db24ee6248da8d -228ba9592d100aa095ebe58393bf03718158b109 -70c6a410276396550048cda447a51e54403a736f -b123fca30a06bbc876fa533d05f4460dfaedc3c7 -a0274263f122af9e4ea39dd8516d5cf9546e2ad6 -c1ccd3a4d62cdaac0fa29b64cb5d96a316db62aa -151cf6964ed44672001ceacda85de58e03886c1f -a9cd174c61d7c5d68525a297b82fba830e10456e -5751b33336190ed56ba36ef878794dbea1ef4e9d -68a96ae4cd16098e5263db284a40040b7c814673 -91f750d9b18664cc4c9ee9e2b5b9fca790297cd5 -162614497729b3b91e3411002cf7c4abd5c52a7f -6caedb53e865db1be613b303ba7be71629cae502 -fea7958aecbc221453f91d2ddf4edd1b5732dd71 -da6ab02347425a235cf663e28e9527a08f68c1de -e84403c4dcedca3b39f7f44c12fe8d1b77c93a2f -7c2c85b07c99e53c37a42a7f329dceed5eb2005c -8cd1eddcd6ac922be4acf4963d0e6a0cfbd15487 -435d76522a9e764000e5e5705d6a77ab13256368 -b4cdd43306609d595be6fd40120c8d18a5b1db09 -42302003c8aae5309f704ed08db93aad952f19c3 -bc0b357c38c4ea60f9d0bd68c4da2530520b282b -2118e3b84064e50a11fa6291de8eea3e37b645d1 -2182a83bed1ffe7fae739fc355ccfbcadc71ebe5 -568b9735b7d30eec8544f906f2cb00d330c1213a -c1ccd3a4d62cdaac0fa29b64cb5d96a316db62aa -8817aa4c209eeb2214d422b06098328ac6215b3f -4d211209a3709163d2ba03e4ba7167af8e86254d -05d65c0bdc78e48b0b5ad160d7c51b106445688a -a479fc00637314dd7853571637f6aa28b47214ca -63b65fd0825f9ee86172de35bdb2179301e26cb6 -2c115fd2009449e184e3794beb6957b2b531da1a -57849b2330fa12fac5967943c8cbb6a9658cd832 -7a260a423e00650d4e6f69d2c04554093871ed79 -c408aca70325a19f8910cf1dad9774c9a83c64dc -8e96f9fd0cb2b6def27768d34f7a3ace0c87bf83 -182b7ccea1b7989e4527209cdad453d6f1f5ea23 -0b2154431fd17517a5ac5014d7657b1f2d0da4d5 -88eeeb0127735539d2b1190b47b3d962f2acf617 -f2a79ead963baa0e4222090939d426966bf42dbe -0be6e4e9ab5f1f959b0fa11955bafabf819b3dc7 -59dda01b16b40e6f9d154383eeb16f221afdc963 -8d7bc5faf261165a98b3d2d111d489cbc93c906e -0993a1606fac39ecba7a05e68133c4db56939f11 -912f4fb080686f24f48e2d9e43491e2aeef47230 -7328785f395384900fcf8aeb09e0f852b10b2583 -42bba42ed6279175d1e4c1d5bb0caa4bb9d967d4 -58cdeacd45b50585c724e70f9f5b9f97945ae461 -b1e2bd69213cad42492c3c24c07c12c59a594ddc -49042ac1d19defc89c1102be916fa984507d4e36 -2118e3b84064e50a11fa6291de8eea3e37b645d1 -3587e7ef23265cf4d28b100d4d8e28024966c180 -3cf696cbd30a65a1010e8cb57374afc1a14e4cf3 -62b49b70e5b85d1c2224e51fe5ce130d621a93e3 -e91f7f6b5bb8260bc584f6e55af20403defd6931 -568b9735b7d30eec8544f906f2cb00d330c1213a -c67dbdc73e2fa4beaece1a2116bc22c6934e4542 -6caedb53e865db1be613b303ba7be71629cae502 -542a48d5109a224438c4c079e6e149c925b17614 -17270ab6bb3be5c39fdce0177834c6529ebd499f -6ee32945c43038c66252e3fc57ea5b462ea8dfb4 -277146a1c43695ec22b36c1ebc152c314a7f55fb -00bb6450cc1e1fdafcd143ac23353a5760343285 -46b81549144216ab4a1b93328f690e3037cbc626 -9d63341247b5a0f549b9248f3d4aeaf1f2d4ee3c -9076458b66ae81915a9e98a7ccb7b0e1075231e6 -e22a1b8f9c1033df5f1a3e99d0ef48f3338b8ec5 -78d0ac4727032fba4393142ad19cfe5ec13f2169 -70bdb822fe2b93ecfeca63fc0297d2170a5e3360 -9f10606311e057b20a0b745bcb4242e332eecb79 -66e882f4517e9074e6e96e8c92ce8528b15fbe09 -b701f304b1c48787eadab89af79d3b4d1aebe913 -a46fda9d5e382a4926d90f6cfbf2164db3c10be6 -d76d7b1b567f9e3abbdf874aff891fda8a902d2e -cc94292154b2de93e771d86b1cac36d5ca522c6b -288209f6ed22f148a42e1e82f278b72de52f3d6a -c0c1c011f426c076a378e72ea8dd5668d64563d6 -079ededfd3bbd73a68c50647cd6081ab02fd5f8c -18b50e5482c9df8f658b9a9850f8ca4d3b8bb78a -92d017c5b9f8edd6dc3c0cd316ce3ea26d65da1b -bc0b357c38c4ea60f9d0bd68c4da2530520b282b -7c2c85b07c99e53c37a42a7f329dceed5eb2005c -91137733e1887c1f48ac98447e97f0339eb6c3a6 -9076458b66ae81915a9e98a7ccb7b0e1075231e6 -f0effd7ea133e867a15b5a404c1cf7f9b6633679 -76e48294efd9aa35732bf9a8a1bdf82133185525 -bb9213f7dd723778772c21b56bdbebd61427f3dc -a23f82710b8c0c032e007405bbc851ebf6ac1499 -c6a9e3c2f8769564eddb2eaaa37f74459a69ef89 -1ee42db1738a5da1871c78b3d351d170861c9e04 -a3fbdd87f856aeea224fc5b96e8a103fd80dba1e -15a9ce47d34b7af68828e2cedac904fe2e06506c -882fb103cf7b13488a8e91ae9d24c2bd7e5b93fb -444837367956c98e8f83a122c1cc28ff8e93d5e2 -ad0a0f8e52f4c31a1f1d86343a4aa3318b1485ac -ab7bbc6cb63b0fc27f7f54c6e764ea512089a565 -ea28b30e69c7e886d6659df6734c1871ccf5b694 -cc05402d2c735d2b5fba129f778a953bb822d987 -413f699b598e6b8009fa622ce3de9b9e8f09d223 -286d0a8fdd1448a4b440cc231cc05287d2bfba0d -4fbbc00359ab6c02928d71551a3b17b961d1cb79 -d12fcecc5f90c43b806e4f64387f38e5a2820652 -8455dc3424dd8a36ff65ef51ed279cc8403a192f -53bbddb8cc7783c219f67918399e4ae2ba52ea74 -b03f7b5387001ebc410031eedf00ea02d66b69b4 -8d7bc5faf261165a98b3d2d111d489cbc93c906e -dea57ddae41c2cab985ff1d30548651ea51d9c2d -d76d7b1b567f9e3abbdf874aff891fda8a902d2e -454da4578c278092ebfa4e5bcb1b2fe8554b5547 -defbfb41ffe7bc7f4f62f7afd550da4761a50e7c -4c58de0c8c8e382803aa5342b379c426af339a1a -53ba6f81e9bf68714f7794ecfb5bc0a750491ef1 -cacc7532b904ec7a7d3f3e678274769a572bc101 -19d6ff005775f473a2750782ea80b3a3c8a1e798 -75c4b42259f6d3222a5cb0cc3e38143d75d20084 -964499de61d342468b6dd3651c1a3bf1468f3904 -3aa7bcc1351407073b579ca167e944308a561688 -00fa29f4bc850374b1e142e9e2a7e2e8ec116225 -00fa29f4bc850374b1e142e9e2a7e2e8ec116225 -21650c987654f61aad6fa2d3b412f0a698238252 -7c55968ba4d4d23216c2b3aa2d4ffac1f01bdf1b -db396015b2047e1a65ea397bab28e04a5c4c8805 -129198cba973c3eb26f9348eaadc0e6d2fea05d8 -db396015b2047e1a65ea397bab28e04a5c4c8805 -876da6b56746d1cf1645c2ffc006d26129e439a6 -5e8c0832b716ae4cfa35c0c8210b96cd62df3c89 -3f84d4ce868160f92127cf410db6957bb9a6c380 -c0d97b2a48318382ddcacfdbfb7f7e47250f7edd -ccb53cb85dc61775dd73f0502cad79e5ab021da2 -00bb6450cc1e1fdafcd143ac23353a5760343285 -74bbeec351056b5e0f5494646ca0014eba8ef5f4 -c23dab5287b5c9f02eef09325ad7c9039466a76d -d6731f5fc06a6790390d46427d4e85ee2058feeb -f4ef8109bfc90a24f3bbf6e506fc086d8051f5b3 -bd275e36bb0bac0794d722e0f10093eb10073e90 -e25223066fb12a1692d4c233f0884f038740fdee -8bfb1848d3f8c6c742f4502b194bd44c08f0e446 -2db26bdffd1e339baec4c20d73ec04058fa3a1fe -ccb53cb85dc61775dd73f0502cad79e5ab021da2 -2189df30b4ab8a83d304a9a2da19b23d8604e2ed -55bb3a36caeca7bf041541eb19da841872ccffd3 -d695e7a7109f85b790608f7d93574f17cd8314e1 -da5836a94660cd6dd878eec5e4ede2ce2893d8b6 -54c9d2fbf544ce6fc0335b47520e860263724986 -3cbeb457d4f6b709f94d2487f7cf9adf6b846c0f -e0472dfbc39c7f5e64c01bb9aad4454baa00864e -0be6e4e9ab5f1f959b0fa11955bafabf819b3dc7 -30af47926c10c0fd042ca28171cdd9613a794c67 -9d8e745eb057f0ae9173ec9f48daeb86697a2130 -1d75b922cba6731e7ba8b93cc797515dbfc1e053 -edc44df8d7e41cfacd11b9f233fe4fe77e3d6b77 -687e2fa86ee95aaa52f55a22cf23bd7d28d3ed27 -f254704460f106c65a8a327680e31105e702534c -04187a6918579e7a3296239eda427090cbce64ee -7784d52cea97ba62cb3441464a6323df163782ca -4ada6ba371ce8622fceeba282a25f57c3b773b20 -3c6ac68cac0a2b5d8aa14549a7275d4c44d430b3 -b632cc71af273988bdc9f46978e7876cb5ac3bd2 -672ee5d594cafd68b51a79676d590d84fd3703a1 -850b3ea2a57086dd01b2de7930e8f154a8c5c5dc -b3999d0d8dc0dbfe77132a239914f3a16363dc19 -5d281433bf2bc962af39ed6310e9e85bf8092016 -d5db14ef1529f8e24d92e0185b1cf9e56fa7f137 -a3fbdd87f856aeea224fc5b96e8a103fd80dba1e -ac8f8eb331b3ef2c59e47f58d9de44f883e7d37f -0c18c1f54e04a6d60d9cbf7f1f88f6efdf5163cc -8dd11ae26c2ea122e787dbe1c1a0a0d47e30549a -da5836a94660cd6dd878eec5e4ede2ce2893d8b6 -d1a5f78adb659412c47551591ae1f97e74556a12 -2cdc3e75b7e58e48e81b67de1b86223195e73ead -b15d4da5f64cf90e8e2a1ae4ec1a3e1e9ac4359b -2f2431d792f46d721c6e9e0eb5b81acb3cc8a077 -882fb103cf7b13488a8e91ae9d24c2bd7e5b93fb -2fa6199bd03a7e4919a09501bea5420f840499f3 -81c359b2a92fa218bc5898719c4e39f751c82596 -b3999d0d8dc0dbfe77132a239914f3a16363dc19 -260e73a587c9a40ea07da7a6ac0992f6ac7b9706 -c4aa9ffad392f406ae2f4dc1754fe896b8d0761e -b9f2a8657483ac5adc13c9d929b5c0d4f5d2a0f7 -4d764805ff97b3b1e0be5f6d4619397ee13ff92d -aab61e75fef2268d6eccb8d77350617a5230b2b6 -6596686264483bb1f659c935134c02eeae1a1380 -b4dafd2b8617c32c122e0405a3b93026477d9a04 -cfb3aca33e74cd09b3c01b7ffc99191c2ec535cf -d72c17b01d1d254f36912f032c01bc39b0f82cc6 -954a6bc26eea5b4150e4b7ff0912a8e82a817619 -032595162df436ea28175212fad9cbb3c7390b59 -66bfebea85dd6e6ef24e687a5f01de1e553c6685 -e90a3eaf526e9ca2854c60994ecb2122bf7de5f0 -a30281ded338ad7894efcf1310ba6417c2f39a13 -526426f6058f3d038e6e170bd30053a571a0a53d -f0478a5faffddd09dca85331f0cba61ef7ec5770 -70b1ea2d3e3f09cd9364c42d8a294b89a0d077ac -270a2442a0ea8287342f7357b4167cd79dc0a0fd -4e6fd8229501d89997660f80facf81b2df8e3f5d -d034d4b5ba63260bf62af87d3f44b54b0f2b9103 -81c359b2a92fa218bc5898719c4e39f751c82596 -f0effd7ea133e867a15b5a404c1cf7f9b6633679 \ No newline at end of file diff --git a/scripts/src/main/resources/alamy-refs.csv b/scripts/src/main/resources/alamy-refs.csv new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala b/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala index 1bfbeebeb6f..9a015a6c5ce 100644 --- a/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala +++ b/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala @@ -1,11 +1,11 @@ package com.gu.mediaservice.scripts -import play.libs.ws.WSClient +import play.api.libs.json.{JsValue, Json} import java.net.URI import java.net.http.HttpRequest.BodyPublishers import java.net.http.HttpResponse.BodyHandlers -import java.net.http.{HttpClient, HttpRequest, HttpResponse} +import java.net.http.{HttpClient, HttpRequest} import scala.io.Source object AlamyCleanUp extends App { @@ -13,13 +13,38 @@ object AlamyCleanUp extends App { val STAGE = sys.env.getOrElse("STAGE", throw new RuntimeException("Must set a STAGE env variable")) val GRIDDOMAIN = if(STAGE == "PROD") "gutools.co.uk" else "test.dev-gutools.co.uk" - val ids = if(STAGE == "PROD") { - val resource = Source.fromResource("alamy-grid-ids.csv") - resource.getLines().toList + println(s"Running for stage $STAGE with domain $GRIDDOMAIN") + + val supplierRefs = if(STAGE == "PROD") { + val resource = Source.fromResource("alamy-refs.csv") + resource.getLines().drop(1).map(_.takeWhile(_ != ',')).toList } else { - List("3b2a8f1845359b84c805634fada85f008e6be297", "278c8c9801a5c8da10f07297289cc3eb26d16ff7", "fd41ca6b9b4ba27cef603bfd127841f6bf537f90") + List("2BW0WK7", "AA63AP", "F208HC", "D9CNPG") + } + + if (supplierRefs.isEmpty) { + println("No supplier refs found in alamy-refs.csv, exiting script") + sys.exit(1) + } + + val ids = { + supplierRefs.zipWithIndex.flatMap({case (ref, index) => + println(s"Fetching grid ids, checking supplierRef ${index + 1} of ${supplierRefs.size}") + val client = HttpClient.newHttpClient() + val request = HttpRequest.newBuilder(new URI(s"https://api.media.$GRIDDOMAIN/images?q=suppliersReference%3A$ref")).headers("X-Gu-Media-Key", GRIDKEY).build() + val response = client.send(request, BodyHandlers.ofString()) + val json = Json.parse(response.body()) + (json \ "data").as[Seq[JsValue]].map(imageMetadata => (imageMetadata \ "data" \ "id").as[String]) + }) } - println(s"Running for stage $STAGE with domain $GRIDDOMAIN with ${ids.size} ids") + + if (ids.isEmpty) { + println("No Grid ids found, exiting script") + sys.exit(1) + } + + println(s"Found ${ids.size} ids, now running deletion") + val usagesBody = BodyPublishers.ofString("""{"data":{"restrictions":"No longer available from Alamy","category":"chargeable"}}""") val labelBody = BodyPublishers.ofString("""{"data":["a2g"]}""") val outcomes = ids.zipWithIndex.map({case (id, index) => From 71f5fcab8857e661d6d88a5767a86070655371f9 Mon Sep 17 00:00:00 2001 From: hk-15 <168672047+hk-15@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:21:11 +0100 Subject: [PATCH 340/373] Read csv from args instead --- scripts/src/main/resources/alamy-refs.csv | 0 .../gu/mediaservice/scripts/AlamyCleanUp.scala | 18 +++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) delete mode 100644 scripts/src/main/resources/alamy-refs.csv diff --git a/scripts/src/main/resources/alamy-refs.csv b/scripts/src/main/resources/alamy-refs.csv deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala b/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala index 9a015a6c5ce..a23b7b79f13 100644 --- a/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala +++ b/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala @@ -15,9 +15,21 @@ object AlamyCleanUp extends App { println(s"Running for stage $STAGE with domain $GRIDDOMAIN") - val supplierRefs = if(STAGE == "PROD") { - val resource = Source.fromResource("alamy-refs.csv") - resource.getLines().drop(1).map(_.takeWhile(_ != ',')).toList + val supplierRefs = if (STAGE == "PROD") { + val csvPath = args.headOption.getOrElse { + throw new IllegalArgumentException("Usage: AlamyCleanUp ") + } + val source = Source.fromFile(csvPath) + try { + source + .getLines() + .drop(1) + .map(_.takeWhile(_ != ',').trim) + .filter(_.nonEmpty) + .toList + } finally { + source.close() + } } else { List("2BW0WK7", "AA63AP", "F208HC", "D9CNPG") } From 7a8d5df0ba9dec27af4927317dacdc44b1d7fc89 Mon Sep 17 00:00:00 2001 From: hk-15 <168672047+hk-15@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:47:07 +0100 Subject: [PATCH 341/373] Update error messaging --- .../main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala b/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala index a23b7b79f13..22a8920e45d 100644 --- a/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala +++ b/scripts/src/main/scala/com/gu/mediaservice/scripts/AlamyCleanUp.scala @@ -17,7 +17,7 @@ object AlamyCleanUp extends App { val supplierRefs = if (STAGE == "PROD") { val csvPath = args.headOption.getOrElse { - throw new IllegalArgumentException("Usage: AlamyCleanUp ") + throw new IllegalArgumentException("To run the script pass in a path to a csv file as an argument") } val source = Source.fromFile(csvPath) try { @@ -35,7 +35,7 @@ object AlamyCleanUp extends App { } if (supplierRefs.isEmpty) { - println("No supplier refs found in alamy-refs.csv, exiting script") + println("No supplier refs found in supplied csv, exiting script") sys.exit(1) } From 4ff9c335f9af0769ec74ef08eb0720f0929647cc Mon Sep 17 00:00:00 2001 From: Lindsey Dew Date: Thu, 10 Sep 2026 12:27:57 +0100 Subject: [PATCH 342/373] Upgrade amazon sdk (#4927) --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 6091b4ac353..dad2f1577af 100644 --- a/build.sbt +++ b/build.sbt @@ -77,7 +77,7 @@ Global / concurrentRestrictions := Seq( ) val awsSdkVersion = "1.12.797" -val awsSdkV2Version = "2.49.5" +val awsSdkV2Version = "2.54.13" val elastic4sVersion = "8.19.1" val awsKclVersion = "3.4.3" val okHttpVersion = "3.12.1" From 25c349e41bd5d1e43c4ccef8b4d6e2ce7fd553f4 Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Thu, 10 Sep 2026 12:36:44 +0100 Subject: [PATCH 343/373] Make metadata arrows consistent with usages and collections --- .../js/components/gr-display-crops/gr-display-crops.css | 3 +++ .../js/components/gr-display-crops/gr-display-crops.html | 6 +++--- .../js/components/gr-image-metadata/gr-image-metadata.html | 6 +++--- kahuna/public/stylesheets/main.css | 7 +++++++ 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/kahuna/public/js/components/gr-display-crops/gr-display-crops.css b/kahuna/public/js/components/gr-display-crops/gr-display-crops.css index b392fac8b07..5b9ec9b8c12 100644 --- a/kahuna/public/js/components/gr-display-crops/gr-display-crops.css +++ b/kahuna/public/js/components/gr-display-crops/gr-display-crops.css @@ -12,3 +12,6 @@ justify-content: flex-start; } +.gr-display-crops + dd { + width: 100%; +} \ No newline at end of file diff --git a/kahuna/public/js/components/gr-display-crops/gr-display-crops.html b/kahuna/public/js/components/gr-display-crops/gr-display-crops.html index 80db07d7aab..6d58b199db0 100644 --- a/kahuna/public/js/components/gr-display-crops/gr-display-crops.html +++ b/kahuna/public/js/components/gr-display-crops/gr-display-crops.html @@ -1,10 +1,10 @@
      -
      +
      diff --git a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html index b30edde7c3a..876edd9eb64 100644 --- a/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html +++ b/kahuna/public/js/components/gr-image-metadata/gr-image-metadata.html @@ -735,12 +735,12 @@
      -
      +
      diff --git a/kahuna/public/stylesheets/main.css b/kahuna/public/stylesheets/main.css index a7a54f31d7d..7b5716c24b2 100644 --- a/kahuna/public/stylesheets/main.css +++ b/kahuna/public/stylesheets/main.css @@ -2202,6 +2202,13 @@ FIXME: what to do with touch devices position: relative; } +.image-info__group--expandable { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 10px; +} + .image-info__title { color: #eee; } From dc16a5198cb87581de7441f169c77fe1fd819b30 Mon Sep 17 00:00:00 2001 From: Junyuan Xue Date: Tue, 8 Sep 2026 12:13:54 +0100 Subject: [PATCH 344/373] Support matching via existence in ES --- .../mediaservice/lib/config/FieldAlias.scala | 15 +++- kahuna/public/js/search/query-filter.spec.js | 46 ++++++++++ media-api/app/lib/ImageResponse.scala | 6 +- .../app/lib/elasticsearch/QueryBuilder.scala | 30 +++++-- media-api/test/lib/ImageResponseTest.scala | 44 ++++++++- .../lib/elasticsearch/QueryBuilderTest.scala | 89 ++++++++++++++++++- 6 files changed, 217 insertions(+), 13 deletions(-) create mode 100644 kahuna/public/js/search/query-filter.spec.js diff --git a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/FieldAlias.scala b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/FieldAlias.scala index a50f590ae2f..fa4468ed556 100644 --- a/common-lib/src/main/scala/com/gu/mediaservice/lib/config/FieldAlias.scala +++ b/common-lib/src/main/scala/com/gu/mediaservice/lib/config/FieldAlias.scala @@ -10,7 +10,15 @@ case class FieldAlias(elasticsearchPath: String, displayInAdditionalMetadata: Boolean, displaySearchHint: Boolean, alias: String, - searchHintOptions: List[String]) + searchHintOptions: List[String], + // Some fields are only ever indexed when "true" (e.g. a field that is present + // only to support `has:`/`-has:` queries, such as fileMetadata.c2pa.isAvailable). + // For these fields a literal `alias:false` term query can never match, since "false" + // is never actually indexed - it is only ever implied by the field's absence. + // Setting this flag translates `alias:true`/`alias:false` search queries into + // exists/not-exists queries instead of literal term matches, so both values behave + // intuitively for users (e.g. via a searchHintOptions dropdown of "true"/"false"). + matchViaExistence: Boolean = false) object FieldAlias { implicit val FieldAliasWrites: Writes[FieldAlias] = @@ -26,6 +34,8 @@ object FieldAlias { config.getBoolean("displaySearchHint") else false val searchHintOptions = if (config.hasPath("searchHintOptions")) config.getStringList("searchHintOptions").asScala.toList.filter(_.nonEmpty) else List.empty + val matchViaExistence = if (config.hasPath("matchViaExistence")) + config.getBoolean("matchViaExistence") else false FieldAlias( config.getString("elasticsearchPath"), @@ -33,7 +43,8 @@ object FieldAlias { displayInAdditionalMetadata, displaySearchHint, config.getString("alias"), - searchHintOptions + searchHintOptions, + matchViaExistence ) } ).toSeq diff --git a/kahuna/public/js/search/query-filter.spec.js b/kahuna/public/js/search/query-filter.spec.js new file mode 100644 index 00000000000..d32fc18d41b --- /dev/null +++ b/kahuna/public/js/search/query-filter.spec.js @@ -0,0 +1,46 @@ +// query-filter.js registers itself as an Angular module as a side effect of import +// (`angular.module(...)`), which needs a real DOM/angular runtime to succeed. Since +// fieldFilter/maybeQuoted are plain functions with no Angular dependency, stub out +// angular.module here so the module can be imported in isolation under Jest. +jest.mock('angular', () => { + const chainableModule = { + factory: () => chainableModule, + filter: () => chainableModule + }; + return { __esModule: true, default: { module: () => chainableModule } }; +}); + +import { fieldFilter, maybeQuoted } from './query-filter'; + +describe('maybeQuoted', () => { + it("leaves a value without reserved characters untouched", () => { + expect(maybeQuoted('true')).toBe('true'); + }); + + it("wraps a value containing whitespace in quotes", () => { + expect(maybeQuoted('some value')).toBe('"some value"'); + }); + + it("wraps a value containing a colon in quotes", () => { + expect(maybeQuoted('a:b')).toBe('"a:b"'); + }); +}); + +describe('fieldFilter', () => { + it("builds a simple field:value filter for a plain string value", () => { + expect(fieldFilter('credit', 'Getty Images')).toBe('credit:"Getty Images"'); + }); + + it("quotes the field name too, if it contains reserved characters", () => { + expect(fieldFilter('some field', 'value')).toBe('"some field":value'); + }); + + it("strips double quotes already present in the value before re-quoting", () => { + expect(fieldFilter('credit', '"Getty Images"')).toBe('credit:"Getty Images"'); + }); + + it("builds a correct field:value filter when the value is a boolean, not a string", () => { + expect(fieldFilter('c2paMetadataAvailable', true)).toBe('c2paMetadataAvailable:true'); + expect(fieldFilter('c2paMetadataAvailable', false)).toBe('c2paMetadataAvailable:false'); + }); +}); diff --git a/media-api/app/lib/ImageResponse.scala b/media-api/app/lib/ImageResponse.scala index fb50aa33511..04c1fa95e41 100644 --- a/media-api/app/lib/ImageResponse.scala +++ b/media-api/app/lib/ImageResponse.scala @@ -428,8 +428,10 @@ object ImageResponse { config.fieldAliasConfigs.flatMap { config => val parts = config.elasticsearchPath.split('.').toList.filter(_.nonEmpty) val lookupResult = nestedLookup(JsDefined(source.source), parts) - lookupResult.toOption.map { - config.alias -> _ + lookupResult.toOption match { + case Some(value) => Some(config.alias -> value) + case None if config.matchViaExistence => Some(config.alias -> JsBoolean(false)) + case None => None } } } diff --git a/media-api/app/lib/elasticsearch/QueryBuilder.scala b/media-api/app/lib/elasticsearch/QueryBuilder.scala index ed8989282b1..b8932dbc11f 100644 --- a/media-api/app/lib/elasticsearch/QueryBuilder.scala +++ b/media-api/app/lib/elasticsearch/QueryBuilder.scala @@ -17,11 +17,25 @@ import scalaz.syntax.std.list._ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agency], config: MediaApiConfig) extends ImageFields with GridLogging { - def resolveFieldPath(field: String): String = { - config.fieldAliasConfigs.find(_.alias == field) match { - case Some(x) => x.elasticsearchPath - case None => getFieldPath(field) - } + // Users can search by either the friendly alias (e.g. "c2paMetadataAvailable") or the underlying + // Elasticsearch path it resolves to (e.g. "fileMetadata.c2pa.isAvailable"), so field alias configs + // are looked up by whichever of the two was typed. + private def findFieldAliasConfig(field: String) = + config.fieldAliasConfigs.find(fieldAlias => fieldAlias.alias == field || fieldAlias.elasticsearchPath == field) + + def resolveFieldPath(field: String): String = + findFieldAliasConfig(field).map(_.elasticsearchPath).getOrElse(getFieldPath(field)) + + private def isMatchViaExistence(field: String): Boolean = + findFieldAliasConfig(field).exists(_.matchViaExistence) + + private def isBooleanPhrase(value: String): Boolean = + value.equalsIgnoreCase("true") || value.equalsIgnoreCase("false") + + private def booleanValueOf(value: Value): String = value match { + case Words(v) => v + case Phrase(v) => v + case _ => "" } // For some sad reason, there was no helpful alias for this in the ES library @@ -52,6 +66,12 @@ class QueryBuilder(matchFields: Seq[String], overQuotaAgencies: () => List[Agenc case AnyField => makeMultiQuery(condition.value, matchFields) case MultipleField(fields) => makeMultiQuery(condition.value, fields) case SingleField(field) => condition.value match { + // Some fields are only ever indexed when true (see FieldAlias.matchViaExistence) - for these, + // translate a literal true/false value query into an exists/not-exists query so both values + // behave intuitively, rather than a literal term match (which could never match "false"). + case v @ (Words(_) | Phrase(_)) if isMatchViaExistence(field) && isBooleanPhrase(booleanValueOf(v)) => + val existsQ = boolQuery().filter(existsQuery(resolveFieldPath(field))) + if (booleanValueOf(v).equalsIgnoreCase("true")) existsQ else boolQuery().not(existsQ) // Force AND operator else it will only require *any* of the words, not *all* case Words(value) => matchQuery(resolveFieldPath(field), value).operator(Operator.AND) diff --git a/media-api/test/lib/ImageResponseTest.scala b/media-api/test/lib/ImageResponseTest.scala index 2e16e8e6e86..d9e2e4c5f35 100644 --- a/media-api/test/lib/ImageResponseTest.scala +++ b/media-api/test/lib/ImageResponseTest.scala @@ -35,6 +35,13 @@ class ImageResponseTest extends AnyFunSpec with Matchers with Fixtures { "alias" -> "captionWriter", "label" -> "Caption Writer / Editor", "displaySearchHint" -> true + ), + Map( + "elasticsearchPath" -> "fileMetadata.c2pa.isAvailable", + "alias" -> "c2paMetadataAvailable", + "label" -> "C2PA Metadata Available", + "displaySearchHint" -> true, + "matchViaExistence" -> true ) ) ) ++ MOCK_CONFIG_KEYS.map(_ -> NOT_USED_IN_TEST).toMap), @@ -114,14 +121,17 @@ class ImageResponseTest extends AnyFunSpec with Matchers with Fixtures { val extractedFields = ImageResponse.extractAliasFieldValues(mediaApiConfig, sourceWrapper) extractedFields.nonEmpty shouldEqual true - extractedFields should have length 3 + // 3 regular fields plus c2paMetadataAvailable, which is always present for matchViaExistence + // fields (as an explicit `false` here, since this image has no c2pa data - see below) + extractedFields should have length 4 extractedFields.contains("orgProgrammeMaker" -> JsString("xmp programme maker")) shouldEqual true extractedFields.contains("auxLens" -> JsString("xmp aux lens")) shouldEqual true extractedFields.contains("captionWriter" -> JsString("the editor")) shouldEqual true + extractedFields.contains("c2paMetadataAvailable" -> JsBoolean(false)) shouldEqual true } - it("should return empty set of extract configured alias fields from sourcewrapper if fields do not exist in image") { + it("should return only the matchViaExistence alias fields (as false) if no other configured alias fields exist in image") { val image = createImage( id = "test-image-with-no-filemetadata", agency, @@ -132,6 +142,34 @@ class ImageResponseTest extends AnyFunSpec with Matchers with Fixtures { val extractedFields = ImageResponse.extractAliasFieldValues(mediaApiConfig, sourceWrapper) - extractedFields.isEmpty shouldEqual true + extractedFields shouldEqual Seq("c2paMetadataAvailable" -> JsBoolean(false)) + } + + it("should extract a matchViaExistence alias field as true when the underlying field is present") { + val image = createImage( + id = "test-image-with-c2pa", + agency, + fileMetadata = Some(FileMetadata(c2pa = FileMetadata.C2paAvailable)) + ) + val json = Json.toJson(image) + val sourceWrapper = SourceWrapper[Image](json, image, fromIndex="test_index") + + val extractedFields = ImageResponse.extractAliasFieldValues(mediaApiConfig, sourceWrapper) + + extractedFields.contains("c2paMetadataAvailable" -> JsBoolean(true)) shouldEqual true + } + + it("should extract a matchViaExistence alias field as false, rather than omitting it, when the underlying field is absent") { + val image = createImage( + id = "test-image-without-c2pa", + agency, + fileMetadata = Some(FileMetadata(c2pa = FileMetadata.NoC2PA)) + ) + val json = Json.toJson(image) + val sourceWrapper = SourceWrapper[Image](json, image, fromIndex="test_index") + + val extractedFields = ImageResponse.extractAliasFieldValues(mediaApiConfig, sourceWrapper) + + extractedFields.contains("c2paMetadataAvailable" -> JsBoolean(false)) shouldEqual true } } diff --git a/media-api/test/lib/elasticsearch/QueryBuilderTest.scala b/media-api/test/lib/elasticsearch/QueryBuilderTest.scala index ace13ce3005..137cb560371 100644 --- a/media-api/test/lib/elasticsearch/QueryBuilderTest.scala +++ b/media-api/test/lib/elasticsearch/QueryBuilderTest.scala @@ -5,7 +5,7 @@ import com.sksamuel.elastic4s.ElasticDsl import com.sksamuel.elastic4s.requests.common.Operator import com.sksamuel.elastic4s.requests.searches.queries._ import com.sksamuel.elastic4s.requests.searches.queries.matches.{MatchPhraseQuery, MatchQuery, MultiMatchQuery, MultiMatchQueryBuilderType} -import lib.querysyntax.Negation +import lib.querysyntax.{Match, Negation, Phrase, SingleField} import com.gu.mediaservice.lib.config.GridConfigResources import com.sksamuel.elastic4s.handlers.searches.queries.QueryBuilderFn import com.sksamuel.elastic4s.requests.searches.queries.compound.BoolQuery @@ -183,6 +183,93 @@ class QueryBuilderTest extends AnyFunSpec with Matchers with ConditionFixtures w } } + describe("field alias matchViaExistence queries") { + // e.g. `fileMetadata.c2pa.isAvailable` is only ever indexed as `true`, so `:true`/`:false` are + // translated into exists/not-exists queries rather than literal term matches (see FieldAlias.matchViaExistence). + val c2paFieldAliasConfig = Map( + "elasticsearchPath" -> "fileMetadata.c2pa.isAvailable", + "alias" -> "c2paMetadataAvailable", + "label" -> "C2PA Metadata Available", + "matchViaExistence" -> true + ) + val configWithC2paAlias = new MediaApiConfig(GridConfigResources( + Configuration.from(commonConfigurations ++ Map("field.aliases" -> Seq(c2paFieldAliasConfig))), + null, + new ApplicationLifecycle { + override def addStopHook(hook: () => Future[_]): Unit = {} + override def stop(): Future[_] = Future.successful(()) + } + )) + val queryBuilderWithC2paAlias = new QueryBuilder(matchFields, () => Nil, configWithC2paAlias) + + def existsClauseOf(query: Query): ExistsQuery = + query.asInstanceOf[BoolQuery].filters.head.asInstanceOf[ExistsQuery] + + it("alias:true is expressed as an exists query on the underlying elasticsearch path") { + val condition = Match(SingleField("c2paMetadataAvailable"), Phrase("true")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + val existsClause = existsClauseOf(query.must.head) + existsClause.field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("alias:false is expressed as a not-exists query on the underlying elasticsearch path") { + val condition = Match(SingleField("c2paMetadataAvailable"), Phrase("false")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + query.must.size shouldBe 1 + val notClause = query.must.head.asInstanceOf[BoolQuery] + val existsClause = existsClauseOf(notClause.not.head) + existsClause.field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("the underlying elasticsearch path:true also resolves to the same exists query as the alias") { + val condition = Match(SingleField("fileMetadata.c2pa.isAvailable"), Phrase("true")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + val existsClause = existsClauseOf(query.must.head) + existsClause.field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("the underlying elasticsearch path:false also resolves to the same not-exists query as the alias") { + val condition = Match(SingleField("fileMetadata.c2pa.isAvailable"), Phrase("false")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + query.must.size shouldBe 1 + val notClause = query.must.head.asInstanceOf[BoolQuery] + val existsClause = existsClauseOf(notClause.not.head) + existsClause.field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("boolean values are matched case-insensitively, e.g. alias:TRUE and alias:False") { + val trueQuery = queryBuilderWithC2paAlias.makeQuery( + List(Match(SingleField("c2paMetadataAvailable"), Phrase("TRUE"))) + ).asInstanceOf[BoolQuery] + existsClauseOf(trueQuery.must.head).field shouldBe "fileMetadata.c2pa.isAvailable" + + val falseQuery = queryBuilderWithC2paAlias.makeQuery( + List(Match(SingleField("c2paMetadataAvailable"), Phrase("False"))) + ).asInstanceOf[BoolQuery] + falseQuery.must.size shouldBe 1 + val notClause = falseQuery.must.head.asInstanceOf[BoolQuery] + existsClauseOf(notClause.not.head).field shouldBe "fileMetadata.c2pa.isAvailable" + } + + it("a non-boolean value on a matchViaExistence field falls back to a literal phrase match, not an exists query") { + val condition = Match(SingleField("c2paMetadataAvailable"), Phrase("maybe")) + val query = queryBuilderWithC2paAlias.makeQuery(List(condition)).asInstanceOf[BoolQuery] + + query.must.size shouldBe 1 + val phraseClause = query.must.head.asInstanceOf[MatchPhraseQuery] + phraseClause.field shouldBe "fileMetadata.c2pa.isAvailable" + phraseClause.value shouldBe "maybe" + } + + it("fields with no matching alias or elasticsearch path still fall back to getFieldPath, unaffected by the new lookup") { + queryBuilderWithC2paAlias.resolveFieldPath("someUnrelatedField") shouldBe "someUnrelatedField" + } + } + describe("is search filter") { it("should correctly construct an is owned photo query") { val query = queryBuilder.makeQuery(List(isOwnedPhotoCondition)).asInstanceOf[BoolQuery] From 84f519841d5bfa78ea6b98ddbe7533a7a2c9cfac Mon Sep 17 00:00:00 2001 From: AndyKilmory Date: Thu, 10 Sep 2026 17:49:02 +0100 Subject: [PATCH 345/373] Restore four upstream changes lost during development Reverted on this branch, mostly in 2044353d2: - 49aa90b4d: re-exclude useAISearch from the generic $stateParams watcher (it has its own, so it was watched twice) - 6c4b6704c: escape the hyphen in the collection regex, which was acting as a character range and matching most quoted searches - 9c493ffca: drop the in-flight $stateParams mutation and restore the $viewContentLoaded handler; the mutation caused a 4x page load - e052e487f: restore ctrl.maybeReplacedByMediaIds, without which the 'replaced' banner could never render Checked the whole branch against main; nothing else is missing. --- .../components/gr-sort-control/gr-sort-control.tsx | 2 +- kahuna/public/js/image/controller.js | 4 ++++ kahuna/public/js/search/index.js | 14 ++++++++++---- kahuna/public/js/search/query.js | 2 ++ 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx index 956ce963362..82c1ae71675 100644 --- a/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx +++ b/kahuna/public/js/components/gr-sort-control/gr-sort-control.tsx @@ -16,7 +16,7 @@ export interface SortWrapperProps { props: SortProps; } -const checkForCollection = (query:string): boolean => /~"[a-zA-Z0-9 #-_.://]+"/.test(query); +const checkForCollection = (query: string): boolean => /~"[a-zA-Z0-9 #\-_.:/]+"/.test(query); const deriveSortState = (query: string, orderBy: string, sortOptions: SortDropdownOption[]) => { const hasCollection = checkForCollection(query); diff --git a/kahuna/public/js/image/controller.js b/kahuna/public/js/image/controller.js index 1cb73629f60..5ebf9e7e4bd 100644 --- a/kahuna/public/js/image/controller.js +++ b/kahuna/public/js/image/controller.js @@ -169,6 +169,10 @@ image.controller('ImageCtrl', [ ctrl.optimisedImageUri = optimisedImageUri; ctrl.lowResImageUri = lowResImageUri; + ctrl.maybeReplacedByMediaIds = image.data.usages.data + .filter(usage => usage.data.status === "replaced") + .map(usage => usage.data.childUsageMetadata.childMediaId); + ctrl.singleImageList = ctrl.image ? new List([ctrl.image]) : new List([]); editsService.canUserEdit(ctrl.image).then(editable => { diff --git a/kahuna/public/js/search/index.js b/kahuna/public/js/search/index.js index 25e3fc1523a..d8348a24cbf 100644 --- a/kahuna/public/js/search/index.js +++ b/kahuna/public/js/search/index.js @@ -209,9 +209,7 @@ search.config(['$stateProvider', '$urlMatcherFactoryProvider', // Helper state to determine whether this is just // reloading a previous search state, or a new search isReloadingPreviousSearch: ['$stateParams', function($stateParams) { - const isDeepStateRedirect = $stateParams.isDeepStateRedirect; - delete $stateParams.isDeepStateRedirect; - return isDeepStateRedirect === true; + return $stateParams.isDeepStateRedirect === true; }], selection: ['orderedSetFactory', function(orderedSetFactory) { return orderedSetFactory(); @@ -321,7 +319,15 @@ search.config(['$stateProvider', '$urlMatcherFactoryProvider', // FIXME: This is here if you go to another state directly e.g. `'/images/id'` // and then navigate to search. As it has no remembered `deepStateRedirect`, // we just land on `/`. See [1]. -search.run(['$rootScope', '$state', function($rootScope, $state) { +search.run(['$rootScope', '$state', '$stateParams', '$timeout', function($rootScope, $state, $stateParams, $timeout) { + $rootScope.$on('$viewContentLoaded', (_, view) => { + if (view === 'results@search') { + // using a timeout of 0 to schedule the task for execution ASAP, but outside the ongoing transition + $timeout(() => { + $state.go('search.results', {isDeepStateRedirect: false}); + }); + } + }); $rootScope.$on('$stateChangeSuccess', (_, toState) => { if (toState.name === 'search') { $state.go('search.results', null, {reload: true}); diff --git a/kahuna/public/js/search/query.js b/kahuna/public/js/search/query.js index b6931d1f666..9bb85973c95 100644 --- a/kahuna/public/js/search/query.js +++ b/kahuna/public/js/search/query.js @@ -555,6 +555,8 @@ query.controller('SearchQueryCtrl', [ Object.keys($stateParams). // Exclude date-related filters, managed separately in dateFilter filter(key => dateFilterParams.indexOf(key) === -1). + // Exclude useAISearch, managed separately by its own dedicated watcher + filter(key => key !== 'useAISearch'). forEach(setAndWatchParam); // URL parameters are not decoded when taken out of the params. From e53fead8c6a54c73eb2a0ff2937bab6409547402 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 11 Sep 2026 07:44:59 +0000 Subject: [PATCH 346/373] Wrap network and container stop methods in closures to ensure local scope is correct --- e2e-tests/setup/stack.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index e59582c0277..c7f3512d903 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -640,9 +640,9 @@ export async function stopStack(environment: StoppableStack | undefined): Promis // Reverse start order, so nothing is stopped before whatever depends on it. ...[...containers].reverse().map((container) => ({ title: `Stop ${container.getLabels()[ROLE_LABEL] ?? container.getName()}`, - task: container.stop, + task: () => container.stop(), })), - ...(network ? [{ title: 'Remove network', task: network.stop }] : []), + ...(network ? [{ title: 'Remove network', task: () => network.stop() }] : []), ...(configDir ? [ { From 234388c8cfdab540cc8250d13e3aadb392adf308 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 28 Aug 2026 13:06:48 +0000 Subject: [PATCH 347/373] Add first pass at upload features --- e2e-tests/features/upload.feature | 322 ++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 e2e-tests/features/upload.feature diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature new file mode 100644 index 00000000000..641d49c5858 --- /dev/null +++ b/e2e-tests/features/upload.feature @@ -0,0 +1,322 @@ +Feature: Uploading images to the Grid + This lets an authorised user get images into the Grid by selecting files, + dragging and dropping files or URLs, importing Witness contributions, and + then completing the required metadata while tracking each upload's progress. + + Background: + Given the application stack is running + And I am signed in through pan-domain auth + And I have opened the image upload page + + # --------------------------------------------------------------------------- + # Upload page shell (view.html + controller.js) + # --------------------------------------------------------------------------- + + Scenario: An authorised user sees the upload tools + Given I am permitted to upload images + When the upload page loads + Then I should see the file upload prompt + And I should see my past 50 uploads + And the drag-and-drop uploader should be active + # Evidence: kahuna/public/js/upload/view.html + # Evidence: kahuna/public/js/upload/controller.js + + Scenario: An unauthorised user is told they cannot upload + Given I am not permitted to upload images + When the upload page loads + Then I should see a message that I am not authorised to upload images + And I should see a link to email support + And I should not see the file upload prompt + # Evidence: kahuna/public/js/upload/view.html + # Evidence: kahuna/public/js/upload/controller.js + + Scenario: Returning to search from the upload page + When I choose "Back to search" from the top bar + Then I should be taken to the image search page + # Evidence: kahuna/public/js/upload/view.html + + Scenario: The current uploads section only appears while an upload is running + Given I have an upload in progress + When the upload page loads + Then I should see my current uploads section + # Evidence: kahuna/public/js/upload/view.html + # Evidence: kahuna/public/js/upload/controller.js + + Scenario: Viewing all of my uploads + When I choose "View all your uploads" + Then I should be taken to a search filtered to images I uploaded + # Evidence: kahuna/public/js/upload/view.html + + Scenario: Warning before leaving the page with uploads in progress + Given I have an upload in progress + When I try to navigate away from the upload page + Then I should be warned that uploads are in progress and asked to confirm + # Evidence: kahuna/public/js/upload/controller.js + + # --------------------------------------------------------------------------- + # File upload prompt (prompt/prompt.html + prompt.js) + # --------------------------------------------------------------------------- + + Scenario: The prompt explains how to upload + When the upload page loads + Then I should see a message telling me to drag and drop or click to upload to the system + # Evidence: kahuna/public/js/upload/prompt/prompt.html + # Evidence: kahuna/public/js/upload/prompt/prompt.js + + Scenario: The prompt suggests an example label when no labels are applied + Given I have not applied any preset labels + When the upload page loads + Then I should see a suggested example label to apply to all uploads + # Evidence: kahuna/public/js/upload/prompt/prompt.html + # Evidence: kahuna/public/js/upload/prompt/prompt.js + + Scenario: Preset labels are applied to all uploads + When I add a preset label in the prompt + Then that label should be applied to all my uploads + # Evidence: kahuna/public/js/upload/prompt/prompt.html + # Evidence: kahuna/public/js/upload/prompt/prompt.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + + # --------------------------------------------------------------------------- + # Select-files uploader (file-uploader.html + file-uploader.js) + # --------------------------------------------------------------------------- + + Scenario: Opening the file picker from the Upload button + When I click the "Upload" button + Then the system file picker should open + # Evidence: kahuna/public/js/upload/file-uploader.html + # Evidence: kahuna/public/js/upload/file-uploader.js + + Scenario: Selecting files queues them for upload + When I select one or more image files to upload + Then those files should be queued for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/file-uploader.html + # Evidence: kahuna/public/js/upload/file-uploader.js + # Evidence: kahuna/public/js/upload/manager.js + + Scenario: Files above the size limit are skipped with a warning + Given an upload size limit is configured + When I select a file that is larger than the size limit + Then I should be warned that the oversized file will be skipped + And only the files within the limit should be queued for upload + # Evidence: kahuna/public/js/upload/manager.js + + # --------------------------------------------------------------------------- + # Drag-and-drop uploader (dnd-uploader.html + dnd-uploader.js) + # --------------------------------------------------------------------------- + + Scenario: Dragging valid content over the page shows the dropzone + When I drag files over the upload page + Then the dropzone overlay should appear with an explanation + # Evidence: kahuna/public/js/upload/dnd-uploader.html + # Evidence: kahuna/public/js/upload/dnd-uploader.js + + Scenario: The dropzone hides when I stop dragging + Given the dropzone overlay is showing + When I drag away from the upload page + Then the dropzone overlay should disappear + # Evidence: kahuna/public/js/upload/dnd-uploader.html + # Evidence: kahuna/public/js/upload/dnd-uploader.js + + Scenario: Dropping files uploads them + When I drop one or more image files onto the page + Then those files should be queued for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/dnd-uploader.js + # Evidence: kahuna/public/js/upload/manager.js + + Scenario: Dropping a URL loads the image from that URL + When I drop an image URL onto the page + Then the image at that URL should be loaded for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/dnd-uploader.js + # Evidence: kahuna/public/js/upload/manager.js + + Scenario: Dropping a Witness contribution imports it + When I drop a Witness contribution URL onto the page + Then the importing overlay should be shown + And the Witness image should be imported with its metadata and usage rights + And I should be taken to the imported image's page + # Evidence: kahuna/public/js/upload/dnd-uploader.html + # Evidence: kahuna/public/js/upload/dnd-uploader.js + + Scenario: A failed Witness import is reported + Given I drop a Witness contribution URL onto the page + When the Witness import fails + Then I should see an alert that importing the Witness contribution failed + # Evidence: kahuna/public/js/upload/dnd-uploader.js + + Scenario: Dropping invalid content is rejected + When I drop something that is not a valid file or URL + Then I should see an alert that I must drop valid files or URLs + # Evidence: kahuna/public/js/upload/dnd-uploader.js + + Scenario: Dragging a Grid image back onto the page is ignored + When I drag an image that is already in the Grid over the page + Then the dropzone overlay should not appear + # Evidence: kahuna/public/js/upload/dnd-uploader.js + + # --------------------------------------------------------------------------- + # Current uploads list (jobs/upload-jobs.html + upload-jobs.js) + # --------------------------------------------------------------------------- + + Scenario: Current uploads show how many remain + Given I have several uploads in progress + When I view my current uploads + Then I should see a count of how many uploads remain + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + + Scenario: An uploading job shows a preview with its name and size + Given a file is uploading + When I view my current uploads + Then I should see a preview thumbnail with the file name and size + And I should see the job's status + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/manager.js + + Scenario: A failed upload shows the error and can be removed + Given an upload has failed + When I view my current uploads + Then the job should be marked as an upload error with the error message + And I should be able to remove the failed job after confirming + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + + Scenario: An unsupported file type gives a helpful error + When I upload a file that is not a JPG, PNG or TIFF + Then the job should show an error explaining only JPG, PNG and TIFF are supported + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + + Scenario: A completed upload becomes an editable image + Given an upload has completed + When I view my current uploads + Then the job should switch to the image metadata editor + And I should be able to delete the image if I am permitted + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + + Scenario: A missing description defaults to the file name + Given an uploaded image has no description + When the upload completes + Then the description should default to the file name without its extension + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + + Scenario: Deleting an image removes it from current uploads + Given an uploaded image is shown in my current uploads + When the image is deleted + Then it should be removed from my current uploads + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + + Scenario: A failed deletion is reported + Given an uploaded image is shown in my current uploads + When deleting the image fails + Then I should see an alert explaining the deletion failed + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + + # --------------------------------------------------------------------------- + # Required metadata editor (jobs/required-metadata-editor.html + .js) + # --------------------------------------------------------------------------- + + Scenario: Editing required metadata for an uploaded image + Given an uploaded image is shown in the metadata editor + When I fill in the description, byline and credit + Then the metadata should be saved automatically + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + + Scenario: Description and credit are required + Given an uploaded image is shown in the metadata editor + When I leave the description or credit empty + Then those fields should be marked as required + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + + Scenario: The description placeholder gives guidance + Given an uploaded image with no description + When I view the description field + Then I should see placeholder guidance about who, what, where, when and why + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + + Scenario: Choosing an image type when image types are configured + Given image types are configured + When I view the metadata editor + Then I should be able to choose an image type from a dropdown + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + + Scenario: The credit field suggests existing values + Given an uploaded image is shown in the metadata editor + When I type into the credit field + Then I should see suggestions matching existing credits + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + + Scenario: Copyright only shows when it was already present + Given an uploaded image that already has a copyright value + When I view the metadata editor + Then I should see the copyright field + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + + Scenario: Existing usage instructions are shown with room for more + Given an uploaded image that already has usage instructions + When I view the metadata editor + Then I should see the existing usage instructions + And I should be able to add further special instructions + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + + Scenario: Applying a field value to all uploads in a batch + Given I am uploading more than one image + And I am permitted to edit + When I apply a field value to all uploads + Then that value should be applied to the same field on every current upload + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + + Scenario: Metadata editing is disabled without edit permission + Given I am not permitted to edit the image + When I view the metadata editor + Then the metadata fields should be disabled + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + + Scenario: Applying a metadata template makes fields read-only + Given an uploaded image is shown in the metadata editor + When a metadata template is selected + Then the affected fields should be populated and made read-only + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + + # --------------------------------------------------------------------------- + # Recent uploads (recent/recent-uploads.html + recent-uploads.js) + # --------------------------------------------------------------------------- + + Scenario: Past uploads are loading + When my past uploads have not yet loaded + Then I should see a loading message + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js + + Scenario: I have not uploaded anything yet + Given I have never uploaded an image + When my past uploads load + Then I should see a message that I haven't uploaded anything yet + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js + + Scenario: My past uploads are listed + Given I have uploaded images before + When my past uploads load + Then I should see each of my past uploaded images + And I should be able to delete an image I am permitted to delete + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js + + Scenario: A failed deletion of a past upload is reported + Given a past uploaded image is listed + When deleting the image fails + Then I should see an alert explaining the deletion failed + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js From 3088ac91827d40c938268342b0372eb4c16a79a3 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 28 Aug 2026 14:44:14 +0000 Subject: [PATCH 348/373] Add .pending as we are missing the tests; add a few todos for amends and missing features --- ...{upload.feature => upload.feature.pending} | 149 +++++++++--------- 1 file changed, 75 insertions(+), 74 deletions(-) rename e2e-tests/features/{upload.feature => upload.feature.pending} (74%) diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature.pending similarity index 74% rename from e2e-tests/features/upload.feature rename to e2e-tests/features/upload.feature.pending index 641d49c5858..9af1ef94011 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature.pending @@ -18,8 +18,8 @@ Feature: Uploading images to the Grid Then I should see the file upload prompt And I should see my past 50 uploads And the drag-and-drop uploader should be active - # Evidence: kahuna/public/js/upload/view.html - # Evidence: kahuna/public/js/upload/controller.js + # Evidence: kahuna/public/js/upload/view.html lines 11-12, 19-27, 31 + # Evidence: kahuna/public/js/upload/controller.js lines 40-45 Scenario: An unauthorised user is told they cannot upload Given I am not permitted to upload images @@ -27,31 +27,31 @@ Feature: Uploading images to the Grid Then I should see a message that I am not authorised to upload images And I should see a link to email support And I should not see the file upload prompt - # Evidence: kahuna/public/js/upload/view.html - # Evidence: kahuna/public/js/upload/controller.js + # Evidence: kahuna/public/js/upload/view.html lines 11, 29 + # Evidence: kahuna/public/js/upload/controller.js lines 36-41 Scenario: Returning to search from the upload page When I choose "Back to search" from the top bar Then I should be taken to the image search page - # Evidence: kahuna/public/js/upload/view.html + # Evidence: kahuna/public/js/upload/view.html lines 3-6 Scenario: The current uploads section only appears while an upload is running Given I have an upload in progress When the upload page loads Then I should see my current uploads section - # Evidence: kahuna/public/js/upload/view.html - # Evidence: kahuna/public/js/upload/controller.js + # Evidence: kahuna/public/js/upload/view.html lines 14-17 + # Evidence: kahuna/public/js/upload/controller.js lines 45 Scenario: Viewing all of my uploads When I choose "View all your uploads" Then I should be taken to a search filtered to images I uploaded - # Evidence: kahuna/public/js/upload/view.html + # Evidence: kahuna/public/js/upload/view.html lines 22-23 Scenario: Warning before leaving the page with uploads in progress Given I have an upload in progress When I try to navigate away from the upload page Then I should be warned that uploads are in progress and asked to confirm - # Evidence: kahuna/public/js/upload/controller.js + # Evidence: kahuna/public/js/upload/controller.js lines 18-34, 47-54 # --------------------------------------------------------------------------- # File upload prompt (prompt/prompt.html + prompt.js) @@ -60,22 +60,21 @@ Feature: Uploading images to the Grid Scenario: The prompt explains how to upload When the upload page loads Then I should see a message telling me to drag and drop or click to upload to the system - # Evidence: kahuna/public/js/upload/prompt/prompt.html - # Evidence: kahuna/public/js/upload/prompt/prompt.js + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 1-3 + # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 20 Scenario: The prompt suggests an example label when no labels are applied Given I have not applied any preset labels When the upload page loads Then I should see a suggested example label to apply to all uploads - # Evidence: kahuna/public/js/upload/prompt/prompt.html - # Evidence: kahuna/public/js/upload/prompt/prompt.js + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 + # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 21 Scenario: Preset labels are applied to all uploads When I add a preset label in the prompt Then that label should be applied to all my uploads - # Evidence: kahuna/public/js/upload/prompt/prompt.html - # Evidence: kahuna/public/js/upload/prompt/prompt.js - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 43, 113-114 # --------------------------------------------------------------------------- # Select-files uploader (file-uploader.html + file-uploader.js) @@ -84,23 +83,23 @@ Feature: Uploading images to the Grid Scenario: Opening the file picker from the Upload button When I click the "Upload" button Then the system file picker should open - # Evidence: kahuna/public/js/upload/file-uploader.html - # Evidence: kahuna/public/js/upload/file-uploader.js + # Evidence: kahuna/public/js/upload/file-uploader.html lines 9-15 + # Evidence: kahuna/public/js/upload/file-uploader.js lines 41-42 Scenario: Selecting files queues them for upload When I select one or more image files to upload Then those files should be queued for upload And I should be taken to the upload progress view - # Evidence: kahuna/public/js/upload/file-uploader.html - # Evidence: kahuna/public/js/upload/file-uploader.js - # Evidence: kahuna/public/js/upload/manager.js + # Evidence: kahuna/public/js/upload/file-uploader.html lines 4-7 + # Evidence: kahuna/public/js/upload/file-uploader.js lines 20-26 + # Evidence: kahuna/public/js/upload/manager.js lines 68-83 Scenario: Files above the size limit are skipped with a warning Given an upload size limit is configured When I select a file that is larger than the size limit Then I should be warned that the oversized file will be skipped And only the files within the limit should be queued for upload - # Evidence: kahuna/public/js/upload/manager.js + # Evidence: kahuna/public/js/upload/manager.js lines 36-47 # --------------------------------------------------------------------------- # Drag-and-drop uploader (dnd-uploader.html + dnd-uploader.js) @@ -109,53 +108,53 @@ Feature: Uploading images to the Grid Scenario: Dragging valid content over the page shows the dropzone When I drag files over the upload page Then the dropzone overlay should appear with an explanation - # Evidence: kahuna/public/js/upload/dnd-uploader.html - # Evidence: kahuna/public/js/upload/dnd-uploader.js + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1-6 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 113, 116, 158-171 Scenario: The dropzone hides when I stop dragging Given the dropzone overlay is showing When I drag away from the upload page Then the dropzone overlay should disappear - # Evidence: kahuna/public/js/upload/dnd-uploader.html - # Evidence: kahuna/public/js/upload/dnd-uploader.js + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 117, 173-180 Scenario: Dropping files uploads them When I drop one or more image files onto the page Then those files should be queued for upload And I should be taken to the upload progress view - # Evidence: kahuna/public/js/upload/dnd-uploader.js - # Evidence: kahuna/public/js/upload/manager.js + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 33-37, 182-205 + # Evidence: kahuna/public/js/upload/manager.js lines 68-83 Scenario: Dropping a URL loads the image from that URL When I drop an image URL onto the page Then the image at that URL should be loaded for upload And I should be taken to the upload progress view - # Evidence: kahuna/public/js/upload/dnd-uploader.js - # Evidence: kahuna/public/js/upload/manager.js + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 64-66, 219-220 + # Evidence: kahuna/public/js/upload/manager.js lines 86-96 Scenario: Dropping a Witness contribution imports it When I drop a Witness contribution URL onto the page Then the importing overlay should be shown And the Witness image should be imported with its metadata and usage rights And I should be taken to the imported image's page - # Evidence: kahuna/public/js/upload/dnd-uploader.html - # Evidence: kahuna/public/js/upload/dnd-uploader.js + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 8-12 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 40-62, 69-78, 206-217 Scenario: A failed Witness import is reported Given I drop a Witness contribution URL onto the page When the Witness import fails Then I should see an alert that importing the Witness contribution failed - # Evidence: kahuna/public/js/upload/dnd-uploader.js + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 79-85 Scenario: Dropping invalid content is rejected When I drop something that is not a valid file or URL Then I should see an alert that I must drop valid files or URLs - # Evidence: kahuna/public/js/upload/dnd-uploader.js + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 222-231 Scenario: Dragging a Grid image back onto the page is ignored When I drag an image that is already in the Grid over the page Then the dropzone overlay should not appear - # Evidence: kahuna/public/js/upload/dnd-uploader.js + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 135-150, 158-160 # --------------------------------------------------------------------------- # Current uploads list (jobs/upload-jobs.html + upload-jobs.js) @@ -165,56 +164,55 @@ Feature: Uploading images to the Grid Given I have several uploads in progress When I view my current uploads Then I should see a count of how many uploads remain - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 2 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 45, 79, 88 Scenario: An uploading job shows a preview with its name and size Given a file is uploading When I view my current uploads Then I should see a preview thumbnail with the file name and size And I should see the job's status - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js - # Evidence: kahuna/public/js/upload/manager.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 9-31 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 61-72 + # Evidence: kahuna/public/js/upload/manager.js lines 16-21 Scenario: A failed upload shows the error and can be removed Given an upload has failed When I view my current uploads Then the job should be marked as an upload error with the error message And I should be able to remove the failed job after confirming - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 24-38 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 77-78, 179-185 Scenario: An unsupported file type gives a helpful error When I upload a file that is not a JPG, PNG or TIFF Then the job should show an error explaining only JPG, PNG and TIFF are supported - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 155-163 Scenario: A completed upload becomes an editable image Given an upload has completed When I view my current uploads Then the job should switch to the image metadata editor - And I should be able to delete the image if I am permitted - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 40-48 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 84-92 Scenario: A missing description defaults to the file name Given an uploaded image has no description When the upload completes Then the description should default to the file name without its extension - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 105-110 Scenario: Deleting an image removes it from current uploads Given an uploaded image is shown in my current uploads When the image is deleted Then it should be removed from my current uploads - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 Scenario: A failed deletion is reported Given an uploaded image is shown in my current uploads When deleting the image fails Then I should see an alert explaining the deletion failed - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 197-203 # --------------------------------------------------------------------------- # Required metadata editor (jobs/required-metadata-editor.html + .js) @@ -224,71 +222,74 @@ Feature: Uploading images to the Grid Given an uploaded image is shown in the metadata editor When I fill in the description, byline and credit Then the metadata should be saved automatically - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 1, 30-56, 61-72, 85-104 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 33, 40-65 Scenario: Description and credit are required Given an uploaded image is shown in the metadata editor When I leave the description or credit empty Then those fields should be marked as required - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 36, 96 + @todo necessary? static placeholder Scenario: The description placeholder gives guidance Given an uploaded image with no description When I view the description field Then I should see placeholder guidance about who, what, where, when and why - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 34-40 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 157-161 + @todo how to verify writes Scenario: Choosing an image type when image types are configured Given image types are configured When I view the metadata editor Then I should be able to choose an image type from a dropdown - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 5-27 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 38 Scenario: The credit field suggests existing values Given an uploaded image is shown in the metadata editor When I type into the credit field Then I should see suggestions matching existing credits - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 87-104 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 67-71 Scenario: Copyright only shows when it was already present Given an uploaded image that already has a copyright value When I view the metadata editor Then I should see the copyright field - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 114-134 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 36 Scenario: Existing usage instructions are shown with room for more Given an uploaded image that already has usage instructions When I view the metadata editor Then I should see the existing usage instructions And I should be able to add further special instructions - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 140-167 Scenario: Applying a field value to all uploads in a batch Given I am uploading more than one image And I am permitted to edit When I apply a field value to all uploads Then that value should be applied to the same field on every current upload - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 20-27, 49-53, 76-80, 107-111, 129-133, 156-160, 179-183 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 108-119 Scenario: Metadata editing is disabled without edit permission Given I am not permitted to edit the image When I view the metadata editor Then the metadata fields should be disabled - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 14, 44, 71, 101, 124, 152, 173 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 27-29 + @todo how do we apply metadata templates? Scenario: Applying a metadata template makes fields read-only Given an uploaded image is shown in the metadata editor When a metadata template is selected Then the affected fields should be populated and made read-only - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 15, 45, 72, 102, 125, 153, 175 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 84-98 # --------------------------------------------------------------------------- # Recent uploads (recent/recent-uploads.html + recent-uploads.js) @@ -297,26 +298,26 @@ Feature: Uploading images to the Grid Scenario: Past uploads are loading When my past uploads have not yet loaded Then I should see a loading message - # Evidence: kahuna/public/js/upload/recent/recent-uploads.html - # Evidence: kahuna/public/js/upload/recent/recent-uploads.js + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 1-2 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 34 Scenario: I have not uploaded anything yet Given I have never uploaded an image When my past uploads load Then I should see a message that I haven't uploaded anything yet - # Evidence: kahuna/public/js/upload/recent/recent-uploads.html - # Evidence: kahuna/public/js/upload/recent/recent-uploads.js + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 4-5 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-34 Scenario: My past uploads are listed Given I have uploaded images before When my past uploads load Then I should see each of my past uploaded images And I should be able to delete an image I am permitted to delete - # Evidence: kahuna/public/js/upload/recent/recent-uploads.html - # Evidence: kahuna/public/js/upload/recent/recent-uploads.js + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 7-13 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-51 Scenario: A failed deletion of a past upload is reported Given a past uploaded image is listed When deleting the image fails Then I should see an alert explaining the deletion failed - # Evidence: kahuna/public/js/upload/recent/recent-uploads.js + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 63-69 From 6bd534af268a3e74082bc0762e93ae3556762a7f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 11 Sep 2026 16:35:37 +0000 Subject: [PATCH 349/373] Add list of editable metadata fields from required-metadata-editor --- e2e-tests/features/upload.feature.pending | 112 ++++++++++++++++++++-- 1 file changed, 102 insertions(+), 10 deletions(-) diff --git a/e2e-tests/features/upload.feature.pending b/e2e-tests/features/upload.feature.pending index 9af1ef94011..25089c96a68 100644 --- a/e2e-tests/features/upload.feature.pending +++ b/e2e-tests/features/upload.feature.pending @@ -1,3 +1,5 @@ +# Uploads mutate the single shared e2e user's upload history, so scenarios must not race. +@mode:serial Feature: Uploading images to the Grid This lets an authorised user get images into the Grid by selecting files, dragging and dropping files or URLs, importing Witness contributions, and @@ -5,7 +7,6 @@ Feature: Uploading images to the Grid Background: Given the application stack is running - And I am signed in through pan-domain auth And I have opened the image upload page # --------------------------------------------------------------------------- @@ -21,18 +22,22 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/view.html lines 11-12, 19-27, 31 # Evidence: kahuna/public/js/upload/controller.js lines 40-45 + # Needs a stack booted with enforceUploadImagesPermission=true. + @todo Scenario: An unauthorised user is told they cannot upload Given I am not permitted to upload images When the upload page loads Then I should see a message that I am not authorised to upload images And I should see a link to email support And I should not see the file upload prompt + And I should not see the drag-and-drop uploader # Evidence: kahuna/public/js/upload/view.html lines 11, 29 # Evidence: kahuna/public/js/upload/controller.js lines 36-41 Scenario: Returning to search from the upload page When I choose "Back to search" from the top bar Then I should be taken to the image search page + And my previous search should be intact # Evidence: kahuna/public/js/upload/view.html lines 3-6 Scenario: The current uploads section only appears while an upload is running @@ -49,7 +54,10 @@ Feature: Uploading images to the Grid Scenario: Warning before leaving the page with uploads in progress Given I have an upload in progress - When I try to navigate away from the upload page + When I try to navigate away from the upload page via the following buttons: + | Back to search | + | Home | + | View all your uploads | Then I should be warned that uploads are in progress and asked to confirm # Evidence: kahuna/public/js/upload/controller.js lines 18-34, 47-54 @@ -76,6 +84,11 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 43, 113-114 + @todo + Scenario: Preset labels are stored in local storage + When I add preset label(s) in the prompt + Then those label(s) should already be selected when I reload the page + # --------------------------------------------------------------------------- # Select-files uploader (file-uploader.html + file-uploader.js) # --------------------------------------------------------------------------- @@ -105,12 +118,14 @@ Feature: Uploading images to the Grid # Drag-and-drop uploader (dnd-uploader.html + dnd-uploader.js) # --------------------------------------------------------------------------- + @todo Scenario: Dragging valid content over the page shows the dropzone When I drag files over the upload page Then the dropzone overlay should appear with an explanation # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1-6 # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 113, 116, 158-171 + @todo Scenario: The dropzone hides when I stop dragging Given the dropzone overlay is showing When I drag away from the upload page @@ -118,6 +133,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1 # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 117, 173-180 + @todo Scenario: Dropping files uploads them When I drop one or more image files onto the page Then those files should be queued for upload @@ -125,6 +141,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 33-37, 182-205 # Evidence: kahuna/public/js/upload/manager.js lines 68-83 + @todo Scenario: Dropping a URL loads the image from that URL When I drop an image URL onto the page Then the image at that URL should be loaded for upload @@ -132,6 +149,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 64-66, 219-220 # Evidence: kahuna/public/js/upload/manager.js lines 86-96 + @todo can remove this Scenario: Dropping a Witness contribution imports it When I drop a Witness contribution URL onto the page Then the importing overlay should be shown @@ -140,17 +158,20 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 8-12 # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 40-62, 69-78, 206-217 + @todo can remove this Scenario: A failed Witness import is reported Given I drop a Witness contribution URL onto the page When the Witness import fails Then I should see an alert that importing the Witness contribution failed # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 79-85 + @todo not implemented afaics Scenario: Dropping invalid content is rejected When I drop something that is not a valid file or URL Then I should see an alert that I must drop valid files or URLs # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 222-231 + @todo Scenario: Dragging a Grid image back onto the page is ignored When I drag an image that is already in the Grid over the page Then the dropzone overlay should not appear @@ -160,6 +181,7 @@ Feature: Uploading images to the Grid # Current uploads list (jobs/upload-jobs.html + upload-jobs.js) # --------------------------------------------------------------------------- + @todo Scenario: Current uploads show how many remain Given I have several uploads in progress When I view my current uploads @@ -167,6 +189,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 2 # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 45, 79, 88 + @todo Scenario: An uploading job shows a preview with its name and size Given a file is uploading When I view my current uploads @@ -176,6 +199,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 61-72 # Evidence: kahuna/public/js/upload/manager.js lines 16-21 + @todo Scenario: A failed upload shows the error and can be removed Given an upload has failed When I view my current uploads @@ -184,11 +208,13 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 24-38 # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 77-78, 179-185 + @todo Scenario: An unsupported file type gives a helpful error When I upload a file that is not a JPG, PNG or TIFF Then the job should show an error explaining only JPG, PNG and TIFF are supported # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 155-163 + @todo Scenario: A completed upload becomes an editable image Given an upload has completed When I view my current uploads @@ -196,18 +222,44 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 40-48 # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 84-92 + @todo Scenario: A missing description defaults to the file name Given an uploaded image has no description When the upload completes Then the description should default to the file name without its extension # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 105-110 + @todo Scenario: Deleting an image removes it from current uploads Given an uploaded image is shown in my current uploads When the image is deleted Then it should be removed from my current uploads # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + # Permissions? e.g. delete image permission + @todo + Scenario: Uploading a previously deleted image displays it for undeletion + Given an image is uploaded + And then deleted + And the same image is uploaded again + Then it should be present in my current uploads + And I should be able to undelete it + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + + # Bug: the UI still presents this button even if permission to delete is not present + @todo + Scenario: Uploading a previously deleted image uploaded by a different user does not display undeletion button without the appropriate permission + Given an image is uploaded by a user that is not me + And then deleted + And the same image is uploaded again by me + And I do not have delete permissions + Then it should be present in current uploads + And I should see the undelete button + And the undelete button is disabled + And there should be a message indicating that I do not have permission to undelete + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + + @todo Scenario: A failed deletion is reported Given an uploaded image is shown in my current uploads When deleting the image fails @@ -218,6 +270,7 @@ Feature: Uploading images to the Grid # Required metadata editor (jobs/required-metadata-editor.html + .js) # --------------------------------------------------------------------------- + @todo Scenario: Editing required metadata for an uploaded image Given an uploaded image is shown in the metadata editor When I fill in the description, byline and credit @@ -225,13 +278,14 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 1, 30-56, 61-72, 85-104 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 33, 40-65 + @todo Scenario: Description and credit are required Given an uploaded image is shown in the metadata editor When I leave the description or credit empty Then those fields should be marked as required # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 36, 96 - @todo necessary? static placeholder + @todo Scenario: The description placeholder gives guidance Given an uploaded image with no description When I view the description field @@ -239,7 +293,8 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 34-40 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 157-161 - @todo how to verify writes + # How to verify writes? + @todo Scenario: Choosing an image type when image types are configured Given image types are configured When I view the metadata editor @@ -247,6 +302,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 5-27 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 38 + @todo Scenario: The credit field suggests existing values Given an uploaded image is shown in the metadata editor When I type into the credit field @@ -254,13 +310,25 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 87-104 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 67-71 - Scenario: Copyright only shows when it was already present - Given an uploaded image that already has a copyright value + # How to verify writes? + @todo + Scenario: Metadata only shows when it was already present for some fields + Given an uploaded image that already has metadata values for the following fields: + | byline | + | credit | + | copyright | + | specialInstructions | + | description | + | domainMetadata | + | usageInstructions | + | imageType | When I view the metadata editor - Then I should see the copyright field + Then I should see the metadata values in the appropriate fields # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 114-134 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 36 + # This also applies to any textual fields — we should pull them from img meta + @todo Scenario: Existing usage instructions are shown with room for more Given an uploaded image that already has usage instructions When I view the metadata editor @@ -268,14 +336,26 @@ Feature: Uploading images to the Grid And I should be able to add further special instructions # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 140-167 - Scenario: Applying a field value to all uploads in a batch + @todo + Scenario: Applying a field value to all current uploads in a batch Given I am uploading more than one image And I am permitted to edit - When I apply a field value to all uploads + When I apply the following field values to all current uploads: + | Leases | + | Image type | + | Description | + | Byline | + | Credit | + | Special instructions | + | Collections | + | Labels | + | Keywords | + | Photoshoot | Then that value should be applied to the same field on every current upload # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 20-27, 49-53, 76-80, 107-111, 129-133, 156-160, 179-183 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 108-119 + @todo Scenario: Metadata editing is disabled without edit permission Given I am not permitted to edit the image When I view the metadata editor @@ -283,7 +363,8 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 14, 44, 71, 101, 124, 152, 173 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 27-29 - @todo how do we apply metadata templates? + # How do we apply metadata templates? + @todo Scenario: Applying a metadata template makes fields read-only Given an uploaded image is shown in the metadata editor When a metadata template is selected @@ -291,16 +372,25 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 15, 45, 72, 102, 125, 153, 175 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 84-98 + @todo + Scenario: Removing a metadata template restores previously edited fields + Given an uploaded image is shown in the metadata editor + And a field is edited + When a metadata template is selected and then removed + Then the previously overridden fields are restored + # --------------------------------------------------------------------------- # Recent uploads (recent/recent-uploads.html + recent-uploads.js) # --------------------------------------------------------------------------- + @todo Scenario: Past uploads are loading When my past uploads have not yet loaded Then I should see a loading message # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 1-2 # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 34 + @todo Scenario: I have not uploaded anything yet Given I have never uploaded an image When my past uploads load @@ -308,6 +398,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 4-5 # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-34 + @todo Scenario: My past uploads are listed Given I have uploaded images before When my past uploads load @@ -316,6 +407,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 7-13 # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-51 + @todo Scenario: A failed deletion of a past upload is reported Given a past uploaded image is listed When deleting the image fails From 0c3ab3391f0d93fa422b0f5a236d55f2ce2cb5cf Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 11 Sep 2026 16:38:12 +0000 Subject: [PATCH 350/373] Tidy up comments --- e2e-tests/features/upload.feature.pending | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/e2e-tests/features/upload.feature.pending b/e2e-tests/features/upload.feature.pending index 25089c96a68..442aefa549f 100644 --- a/e2e-tests/features/upload.feature.pending +++ b/e2e-tests/features/upload.feature.pending @@ -149,7 +149,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 64-66, 219-220 # Evidence: kahuna/public/js/upload/manager.js lines 86-96 - @todo can remove this + @todo can remove this functionality in another PR Scenario: Dropping a Witness contribution imports it When I drop a Witness contribution URL onto the page Then the importing overlay should be shown @@ -158,7 +158,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 8-12 # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 40-62, 69-78, 206-217 - @todo can remove this + @todo can remove this functionality in another PR Scenario: A failed Witness import is reported Given I drop a Witness contribution URL onto the page When the Witness import fails @@ -236,17 +236,17 @@ Feature: Uploading images to the Grid Then it should be removed from my current uploads # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 - # Permissions? e.g. delete image permission @todo Scenario: Uploading a previously deleted image displays it for undeletion Given an image is uploaded And then deleted And the same image is uploaded again + And I have delete permission Then it should be present in my current uploads And I should be able to undelete it # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 - # Bug: the UI still presents this button even if permission to delete is not present + # Existing bug: the UI still presents this button even if permission to delete is not present @todo Scenario: Uploading a previously deleted image uploaded by a different user does not display undeletion button without the appropriate permission Given an image is uploaded by a user that is not me @@ -363,7 +363,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 14, 44, 71, 101, 124, 152, 173 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 27-29 - # How do we apply metadata templates? + # This requires configuration, which is already present in CODE @todo Scenario: Applying a metadata template makes fields read-only Given an uploaded image is shown in the metadata editor From b7802edd8a5dc3f6efa691bc7b322a59caee0b0a Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 11 Sep 2026 16:40:37 +0000 Subject: [PATCH 351/373] When I view the metadata editor -> When I view the metadata editor for an image I did not upload cc. @paperboyo @ https://github.com/guardian/grid/pull/4929#discussion_r3988893799 --- e2e-tests/features/upload.feature.pending | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/features/upload.feature.pending b/e2e-tests/features/upload.feature.pending index 442aefa549f..c9a5f93a9ea 100644 --- a/e2e-tests/features/upload.feature.pending +++ b/e2e-tests/features/upload.feature.pending @@ -358,7 +358,7 @@ Feature: Uploading images to the Grid @todo Scenario: Metadata editing is disabled without edit permission Given I am not permitted to edit the image - When I view the metadata editor + When I view the metadata editor for an image I did not upload Then the metadata fields should be disabled # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 14, 44, 71, 101, 124, 152, 173 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 27-29 From f57579dbf59565b67177a08e88e9e3bfbff6d27f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 14 Sep 2026 17:11:01 +0100 Subject: [PATCH 352/373] Apply suggestion from @jonathonherbert --- e2e-tests/features/upload.feature.pending | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/features/upload.feature.pending b/e2e-tests/features/upload.feature.pending index c9a5f93a9ea..9e8b9f42825 100644 --- a/e2e-tests/features/upload.feature.pending +++ b/e2e-tests/features/upload.feature.pending @@ -79,7 +79,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 21 Scenario: Preset labels are applied to all uploads - When I add a preset label in the prompt + When I add a preset label via the 'apply label to all uploads' button Then that label should be applied to all my uploads # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 43, 113-114 From 72126b3e231cef7a9ce8cdb72bdc1135ed7b999f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 14 Sep 2026 17:14:47 +0100 Subject: [PATCH 353/373] Apply suggestion from @jonathonherbert --- e2e-tests/features/upload.feature.pending | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/features/upload.feature.pending b/e2e-tests/features/upload.feature.pending index 9e8b9f42825..4fa0d9412c2 100644 --- a/e2e-tests/features/upload.feature.pending +++ b/e2e-tests/features/upload.feature.pending @@ -232,7 +232,7 @@ Feature: Uploading images to the Grid @todo Scenario: Deleting an image removes it from current uploads Given an uploaded image is shown in my current uploads - When the image is deleted + When the image is deleted using the delete button at the bottom of the job form Then it should be removed from my current uploads # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 From f11c22fc3a27dba0b207a7f3aaff2e30b3f4a261 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 14 Sep 2026 17:20:54 +0100 Subject: [PATCH 354/373] Apply suggestion from @jonathonherbert --- e2e-tests/features/upload.feature.pending | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/features/upload.feature.pending b/e2e-tests/features/upload.feature.pending index 4fa0d9412c2..9d3c2ad112b 100644 --- a/e2e-tests/features/upload.feature.pending +++ b/e2e-tests/features/upload.feature.pending @@ -239,7 +239,7 @@ Feature: Uploading images to the Grid @todo Scenario: Uploading a previously deleted image displays it for undeletion Given an image is uploaded - And then deleted + And then deleted using the delete button at the bottom of the job form And the same image is uploaded again And I have delete permission Then it should be present in my current uploads From ac5e0364289ce44cd941a500cc4e5ca667c9a53d Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Fri, 11 Sep 2026 16:44:33 +0000 Subject: [PATCH 355/373] Remove redundant spec --- e2e-tests/features/upload.feature | 408 ++++++++++++++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 e2e-tests/features/upload.feature diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature new file mode 100644 index 00000000000..4d2707b0121 --- /dev/null +++ b/e2e-tests/features/upload.feature @@ -0,0 +1,408 @@ +# Uploads mutate the single shared e2e user's upload history, so scenarios must not race. +@mode:serial +Feature: Uploading images to the Grid + This lets an authorised user get images into the Grid by selecting files, + dragging and dropping files or URLs, importing Witness contributions, and + then completing the required metadata while tracking each upload's progress. + + Background: + Given the application stack is running + And I have opened the image upload page + + # --------------------------------------------------------------------------- + # Upload page shell (view.html + controller.js) + # --------------------------------------------------------------------------- + + Scenario: An authorised user sees the upload tools + Given I am permitted to upload images + When the upload page loads + Then I should see the file upload prompt + And I should see my past 50 uploads + And the drag-and-drop uploader should be active + # Evidence: kahuna/public/js/upload/view.html lines 11-12, 19-27, 31 + # Evidence: kahuna/public/js/upload/controller.js lines 40-45 + + # Needs a stack booted with enforceUploadImagesPermission=true. + @todo + Scenario: An unauthorised user is told they cannot upload + Given I am not permitted to upload images + When the upload page loads + Then I should see a message that I am not authorised to upload images + And I should see a link to email support + And I should not see the file upload prompt + And I should not see the drag-and-drop uploader + # Evidence: kahuna/public/js/upload/view.html lines 11, 29 + # Evidence: kahuna/public/js/upload/controller.js lines 36-41 + + Scenario: Returning to search from the upload page + When I choose "Back to search" from the top bar + Then I should be taken to the image search page + And my previous search should be intact + # Evidence: kahuna/public/js/upload/view.html lines 3-6 + + Scenario: The current uploads section only appears while an upload is running + Given I have an upload in progress + When the upload page loads + Then I should see my current uploads section + # Evidence: kahuna/public/js/upload/view.html lines 14-17 + # Evidence: kahuna/public/js/upload/controller.js lines 45 + + Scenario: Viewing all of my uploads + When I choose "View all your uploads" + Then I should be taken to a search filtered to images I uploaded + # Evidence: kahuna/public/js/upload/view.html lines 22-23 + + Scenario: Warning before leaving the page with uploads in progress + Given I have an upload in progress + When I try to navigate away from the upload page via the following buttons: + | Back to search | + | Home | + | View all your uploads | + Then I should be warned that uploads are in progress and asked to confirm + # Evidence: kahuna/public/js/upload/controller.js lines 18-34, 47-54 + + # --------------------------------------------------------------------------- + # File upload prompt (prompt/prompt.html + prompt.js) + # --------------------------------------------------------------------------- + + Scenario: The prompt explains how to upload + When the upload page loads + Then I should see a message telling me to drag and drop or click to upload to the system + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 1-3 + # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 20 + + Scenario: The prompt suggests an example label when no labels are applied + Given I have not applied any preset labels + When the upload page loads + Then I should see a suggested example label to apply to all uploads + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 + # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 21 + + @todo + Scenario: Preset labels are applied to all uploads + When I add a preset label in the prompt + Then that label should be applied to all my uploads + # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 43, 113-114 + + @todo + Scenario: Preset labels are stored in local storage + When I add preset label(s) in the prompt + Then those label(s) should already be selected when I reload the page + + # --------------------------------------------------------------------------- + # Select-files uploader (file-uploader.html + file-uploader.js) + # --------------------------------------------------------------------------- + + Scenario: Opening the file picker from the Upload button + When I click the "Upload" button + Then the system file picker should open + # Evidence: kahuna/public/js/upload/file-uploader.html lines 9-15 + # Evidence: kahuna/public/js/upload/file-uploader.js lines 41-42 + + Scenario: Selecting files queues them for upload + When I select one or more image files to upload + Then those files should be queued for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/file-uploader.html lines 4-7 + # Evidence: kahuna/public/js/upload/file-uploader.js lines 20-26 + # Evidence: kahuna/public/js/upload/manager.js lines 68-83 + + Scenario: Files above the size limit are skipped with a warning + Given an upload size limit is configured + When I select a file that is larger than the size limit + Then I should be warned that the oversized file will be skipped + And only the files within the limit should be queued for upload + # Evidence: kahuna/public/js/upload/manager.js lines 36-47 + + # --------------------------------------------------------------------------- + # Drag-and-drop uploader (dnd-uploader.html + dnd-uploader.js) + # --------------------------------------------------------------------------- + + @todo + Scenario: Dragging valid content over the page shows the dropzone + When I drag files over the upload page + Then the dropzone overlay should appear with an explanation + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1-6 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 113, 116, 158-171 + + @todo + Scenario: The dropzone hides when I stop dragging + Given the dropzone overlay is showing + When I drag away from the upload page + Then the dropzone overlay should disappear + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 117, 173-180 + + @todo + Scenario: Dropping files uploads them + When I drop one or more image files onto the page + Then those files should be queued for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 33-37, 182-205 + # Evidence: kahuna/public/js/upload/manager.js lines 68-83 + + @todo + Scenario: Dropping a URL loads the image from that URL + When I drop an image URL onto the page + Then the image at that URL should be loaded for upload + And I should be taken to the upload progress view + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 64-66, 219-220 + # Evidence: kahuna/public/js/upload/manager.js lines 86-96 + + @todo + Scenario: Dropping a Witness contribution imports it + When I drop a Witness contribution URL onto the page + Then the importing overlay should be shown + And the Witness image should be imported with its metadata and usage rights + And I should be taken to the imported image's page + # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 8-12 + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 40-62, 69-78, 206-217 + + @todo + Scenario: A failed Witness import is reported + Given I drop a Witness contribution URL onto the page + When the Witness import fails + Then I should see an alert that importing the Witness contribution failed + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 79-85 + + @todo + Scenario: Dropping invalid content is rejected + When I drop something that is not a valid file or URL + Then I should see an alert that I must drop valid files or URLs + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 222-231 + + @todo + Scenario: Dragging a Grid image back onto the page is ignored + When I drag an image that is already in the Grid over the page + Then the dropzone overlay should not appear + # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 135-150, 158-160 + + # --------------------------------------------------------------------------- + # Current uploads list (jobs/upload-jobs.html + upload-jobs.js) + # --------------------------------------------------------------------------- + + @todo + Scenario: Current uploads show how many remain + Given I have several uploads in progress + When I view my current uploads + Then I should see a count of how many uploads remain + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 2 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 45, 79, 88 + + @todo + Scenario: An uploading job shows a preview with its name and size + Given a file is uploading + When I view my current uploads + Then I should see a preview thumbnail with the file name and size + And I should see the job's status + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 9-31 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 61-72 + # Evidence: kahuna/public/js/upload/manager.js lines 16-21 + + @todo + Scenario: A failed upload shows the error and can be removed + Given an upload has failed + When I view my current uploads + Then the job should be marked as an upload error with the error message + And I should be able to remove the failed job after confirming + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 24-38 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 77-78, 179-185 + + @todo + Scenario: An unsupported file type gives a helpful error + When I upload a file that is not a JPG, PNG or TIFF + Then the job should show an error explaining only JPG, PNG and TIFF are supported + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 155-163 + + @todo + Scenario: A completed upload becomes an editable image + Given an upload has completed + When I view my current uploads + Then the job should switch to the image metadata editor + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 40-48 + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 84-92 + + @todo + Scenario: A missing description defaults to the file name + Given an uploaded image has no description + When the upload completes + Then the description should default to the file name without its extension + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 105-110 + + @todo + Scenario: Deleting an image removes it from current uploads + Given an uploaded image is shown in my current uploads + When the image is deleted + Then it should be removed from my current uploads + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + + @todo + Scenario: Uploading a previously deleted image displays it for undeletion + Given an image is uploaded + And then deleted + And the same image is uploaded again + And I have delete permission + Then it should be present in my current uploads + And I should be able to undelete it + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + + # Existing bug: the UI still presents this button even if permission to delete is not present + @todo + Scenario: Uploading a previously deleted image uploaded by a different user does not display undeletion button without the appropriate permission + Given an image is uploaded by a user that is not me + And then deleted + And the same image is uploaded again by me + And I do not have delete permissions + Then it should be present in current uploads + And I should see the undelete button + And the undelete button is disabled + And there should be a message indicating that I do not have permission to undelete + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 + + @todo + Scenario: A failed deletion is reported + Given an uploaded image is shown in my current uploads + When deleting the image fails + Then I should see an alert explaining the deletion failed + # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 197-203 + + # --------------------------------------------------------------------------- + # Required metadata editor (jobs/required-metadata-editor.html + .js) + # --------------------------------------------------------------------------- + + @todo + Scenario: Editing required metadata for an uploaded image + Given an uploaded image is shown in the metadata editor + When I fill in the description, byline and credit + Then the metadata should be saved automatically + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 1, 30-56, 61-72, 85-104 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 33, 40-65 + + @todo + Scenario: Description and credit are required + Given an uploaded image is shown in the metadata editor + When I leave the description or credit empty + Then those fields should be marked as required + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 36, 96 + + @todo + Scenario: The description placeholder gives guidance + Given an uploaded image with no description + When I view the description field + Then I should see placeholder guidance about who, what, where, when and why + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 34-40 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 157-161 + + # How to verify writes? + @todo + Scenario: Choosing an image type when image types are configured + Given image types are configured + When I view the metadata editor + Then I should be able to choose an image type from a dropdown + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 5-27 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 38 + + @todo + Scenario: The credit field suggests existing values + Given an uploaded image is shown in the metadata editor + When I type into the credit field + Then I should see suggestions matching existing credits + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 87-104 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 67-71 + + # How to verify writes? + @todo + Scenario: Metadata only shows when it was already present for some fields + Given an uploaded image that already has metadata values for the following fields: + | byline | + | credit | + | copyright | + | specialInstructions | + | description | + | domainMetadata | + | usageInstructions | + | imageType | + When I view the metadata editor + Then I should see the metadata values in the appropriate fields + And I should be able to edit those fields with other values + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 114-134 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 36 + + @todo + Scenario: Applying a field value to all current uploads in a batch + Given I am uploading more than one image + And I am permitted to edit + When I apply the following field values to all current uploads: + | Leases | + | Image type | + | Description | + | Byline | + | Credit | + | Special instructions | + | Collections | + | Labels | + | Keywords | + | Photoshoot | + Then that value should be applied to the same field on every current upload + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 20-27, 49-53, 76-80, 107-111, 129-133, 156-160, 179-183 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 108-119 + + @todo + Scenario: Metadata editing is disabled without edit permission + Given I am not permitted to edit the image + When I view the metadata editor for an image I did not upload + Then the metadata fields should be disabled + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 14, 44, 71, 101, 124, 152, 173 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 27-29 + + # This requires configuration, which is already present in CODE + @todo + Scenario: Applying a metadata template makes fields read-only + Given an uploaded image is shown in the metadata editor + When a metadata template is selected + Then the affected fields should be populated and made read-only + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 15, 45, 72, 102, 125, 153, 175 + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 84-98 + + @todo + Scenario: Removing a metadata template restores previously edited fields + Given an uploaded image is shown in the metadata editor + And a field is edited + When a metadata template is selected and then removed + Then the previously overridden fields are restored + + # --------------------------------------------------------------------------- + # Recent uploads (recent/recent-uploads.html + recent-uploads.js) + # --------------------------------------------------------------------------- + + @todo + Scenario: Past uploads are loading + When my past uploads have not yet loaded + Then I should see a loading message + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 1-2 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 34 + + @todo + Scenario: I have not uploaded anything yet + Given I have never uploaded an image + When my past uploads load + Then I should see a message that I haven't uploaded anything yet + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 4-5 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-34 + + @todo + Scenario: My past uploads are listed + Given I have uploaded images before + When my past uploads load + Then I should see each of my past uploaded images + And I should be able to delete an image I am permitted to delete + # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 7-13 + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-51 + + @todo + Scenario: A failed deletion of a past upload is reported + Given a past uploaded image is listed + When deleting the image fails + Then I should see an alert explaining the deletion failed + # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 63-69 From f6b1db7affa23a068633361cb248a408575e196f Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 10 Sep 2026 10:39:50 +0000 Subject: [PATCH 356/373] Tag unimplemented upload scenarios @todo and skip them The upload feature file specifies far more than we are wiring up in the first pass. Tag everything we are not implementing yet with @todo and exclude those tags from bddgen, so its "undefined step" errors stay meaningful for the scenarios we do implement. Gherkin tags cannot contain whitespace, so the pre-existing @todo annotations were failing to parse; their notes move to comments. Uploads mutate the single shared e2e user's upload history, so the feature runs @mode:serial. --- e2e-tests/features/upload.feature | 23 +- e2e-tests/features/upload.feature.pending | 415 ---------------------- e2e-tests/playwright.config.ts | 3 + 3 files changed, 18 insertions(+), 423 deletions(-) delete mode 100644 e2e-tests/features/upload.feature.pending diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index 4d2707b0121..9d3c2ad112b 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -78,9 +78,8 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 21 - @todo Scenario: Preset labels are applied to all uploads - When I add a preset label in the prompt + When I add a preset label via the 'apply label to all uploads' button Then that label should be applied to all my uploads # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 43, 113-114 @@ -150,7 +149,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 64-66, 219-220 # Evidence: kahuna/public/js/upload/manager.js lines 86-96 - @todo + @todo can remove this functionality in another PR Scenario: Dropping a Witness contribution imports it When I drop a Witness contribution URL onto the page Then the importing overlay should be shown @@ -159,14 +158,14 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 8-12 # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 40-62, 69-78, 206-217 - @todo + @todo can remove this functionality in another PR Scenario: A failed Witness import is reported Given I drop a Witness contribution URL onto the page When the Witness import fails Then I should see an alert that importing the Witness contribution failed # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 79-85 - @todo + @todo not implemented afaics Scenario: Dropping invalid content is rejected When I drop something that is not a valid file or URL Then I should see an alert that I must drop valid files or URLs @@ -233,14 +232,14 @@ Feature: Uploading images to the Grid @todo Scenario: Deleting an image removes it from current uploads Given an uploaded image is shown in my current uploads - When the image is deleted + When the image is deleted using the delete button at the bottom of the job form Then it should be removed from my current uploads # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 @todo Scenario: Uploading a previously deleted image displays it for undeletion Given an image is uploaded - And then deleted + And then deleted using the delete button at the bottom of the job form And the same image is uploaded again And I have delete permission Then it should be present in my current uploads @@ -325,10 +324,18 @@ Feature: Uploading images to the Grid | imageType | When I view the metadata editor Then I should see the metadata values in the appropriate fields - And I should be able to edit those fields with other values # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 114-134 # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 36 + # This also applies to any textual fields — we should pull them from img meta + @todo + Scenario: Existing usage instructions are shown with room for more + Given an uploaded image that already has usage instructions + When I view the metadata editor + Then I should see the existing usage instructions + And I should be able to add further special instructions + # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 140-167 + @todo Scenario: Applying a field value to all current uploads in a batch Given I am uploading more than one image diff --git a/e2e-tests/features/upload.feature.pending b/e2e-tests/features/upload.feature.pending deleted file mode 100644 index 9d3c2ad112b..00000000000 --- a/e2e-tests/features/upload.feature.pending +++ /dev/null @@ -1,415 +0,0 @@ -# Uploads mutate the single shared e2e user's upload history, so scenarios must not race. -@mode:serial -Feature: Uploading images to the Grid - This lets an authorised user get images into the Grid by selecting files, - dragging and dropping files or URLs, importing Witness contributions, and - then completing the required metadata while tracking each upload's progress. - - Background: - Given the application stack is running - And I have opened the image upload page - - # --------------------------------------------------------------------------- - # Upload page shell (view.html + controller.js) - # --------------------------------------------------------------------------- - - Scenario: An authorised user sees the upload tools - Given I am permitted to upload images - When the upload page loads - Then I should see the file upload prompt - And I should see my past 50 uploads - And the drag-and-drop uploader should be active - # Evidence: kahuna/public/js/upload/view.html lines 11-12, 19-27, 31 - # Evidence: kahuna/public/js/upload/controller.js lines 40-45 - - # Needs a stack booted with enforceUploadImagesPermission=true. - @todo - Scenario: An unauthorised user is told they cannot upload - Given I am not permitted to upload images - When the upload page loads - Then I should see a message that I am not authorised to upload images - And I should see a link to email support - And I should not see the file upload prompt - And I should not see the drag-and-drop uploader - # Evidence: kahuna/public/js/upload/view.html lines 11, 29 - # Evidence: kahuna/public/js/upload/controller.js lines 36-41 - - Scenario: Returning to search from the upload page - When I choose "Back to search" from the top bar - Then I should be taken to the image search page - And my previous search should be intact - # Evidence: kahuna/public/js/upload/view.html lines 3-6 - - Scenario: The current uploads section only appears while an upload is running - Given I have an upload in progress - When the upload page loads - Then I should see my current uploads section - # Evidence: kahuna/public/js/upload/view.html lines 14-17 - # Evidence: kahuna/public/js/upload/controller.js lines 45 - - Scenario: Viewing all of my uploads - When I choose "View all your uploads" - Then I should be taken to a search filtered to images I uploaded - # Evidence: kahuna/public/js/upload/view.html lines 22-23 - - Scenario: Warning before leaving the page with uploads in progress - Given I have an upload in progress - When I try to navigate away from the upload page via the following buttons: - | Back to search | - | Home | - | View all your uploads | - Then I should be warned that uploads are in progress and asked to confirm - # Evidence: kahuna/public/js/upload/controller.js lines 18-34, 47-54 - - # --------------------------------------------------------------------------- - # File upload prompt (prompt/prompt.html + prompt.js) - # --------------------------------------------------------------------------- - - Scenario: The prompt explains how to upload - When the upload page loads - Then I should see a message telling me to drag and drop or click to upload to the system - # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 1-3 - # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 20 - - Scenario: The prompt suggests an example label when no labels are applied - Given I have not applied any preset labels - When the upload page loads - Then I should see a suggested example label to apply to all uploads - # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 - # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 21 - - Scenario: Preset labels are applied to all uploads - When I add a preset label via the 'apply label to all uploads' button - Then that label should be applied to all my uploads - # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 43, 113-114 - - @todo - Scenario: Preset labels are stored in local storage - When I add preset label(s) in the prompt - Then those label(s) should already be selected when I reload the page - - # --------------------------------------------------------------------------- - # Select-files uploader (file-uploader.html + file-uploader.js) - # --------------------------------------------------------------------------- - - Scenario: Opening the file picker from the Upload button - When I click the "Upload" button - Then the system file picker should open - # Evidence: kahuna/public/js/upload/file-uploader.html lines 9-15 - # Evidence: kahuna/public/js/upload/file-uploader.js lines 41-42 - - Scenario: Selecting files queues them for upload - When I select one or more image files to upload - Then those files should be queued for upload - And I should be taken to the upload progress view - # Evidence: kahuna/public/js/upload/file-uploader.html lines 4-7 - # Evidence: kahuna/public/js/upload/file-uploader.js lines 20-26 - # Evidence: kahuna/public/js/upload/manager.js lines 68-83 - - Scenario: Files above the size limit are skipped with a warning - Given an upload size limit is configured - When I select a file that is larger than the size limit - Then I should be warned that the oversized file will be skipped - And only the files within the limit should be queued for upload - # Evidence: kahuna/public/js/upload/manager.js lines 36-47 - - # --------------------------------------------------------------------------- - # Drag-and-drop uploader (dnd-uploader.html + dnd-uploader.js) - # --------------------------------------------------------------------------- - - @todo - Scenario: Dragging valid content over the page shows the dropzone - When I drag files over the upload page - Then the dropzone overlay should appear with an explanation - # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1-6 - # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 113, 116, 158-171 - - @todo - Scenario: The dropzone hides when I stop dragging - Given the dropzone overlay is showing - When I drag away from the upload page - Then the dropzone overlay should disappear - # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 1 - # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 117, 173-180 - - @todo - Scenario: Dropping files uploads them - When I drop one or more image files onto the page - Then those files should be queued for upload - And I should be taken to the upload progress view - # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 33-37, 182-205 - # Evidence: kahuna/public/js/upload/manager.js lines 68-83 - - @todo - Scenario: Dropping a URL loads the image from that URL - When I drop an image URL onto the page - Then the image at that URL should be loaded for upload - And I should be taken to the upload progress view - # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 64-66, 219-220 - # Evidence: kahuna/public/js/upload/manager.js lines 86-96 - - @todo can remove this functionality in another PR - Scenario: Dropping a Witness contribution imports it - When I drop a Witness contribution URL onto the page - Then the importing overlay should be shown - And the Witness image should be imported with its metadata and usage rights - And I should be taken to the imported image's page - # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 8-12 - # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 40-62, 69-78, 206-217 - - @todo can remove this functionality in another PR - Scenario: A failed Witness import is reported - Given I drop a Witness contribution URL onto the page - When the Witness import fails - Then I should see an alert that importing the Witness contribution failed - # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 79-85 - - @todo not implemented afaics - Scenario: Dropping invalid content is rejected - When I drop something that is not a valid file or URL - Then I should see an alert that I must drop valid files or URLs - # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 222-231 - - @todo - Scenario: Dragging a Grid image back onto the page is ignored - When I drag an image that is already in the Grid over the page - Then the dropzone overlay should not appear - # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 135-150, 158-160 - - # --------------------------------------------------------------------------- - # Current uploads list (jobs/upload-jobs.html + upload-jobs.js) - # --------------------------------------------------------------------------- - - @todo - Scenario: Current uploads show how many remain - Given I have several uploads in progress - When I view my current uploads - Then I should see a count of how many uploads remain - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 2 - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 45, 79, 88 - - @todo - Scenario: An uploading job shows a preview with its name and size - Given a file is uploading - When I view my current uploads - Then I should see a preview thumbnail with the file name and size - And I should see the job's status - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 9-31 - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 61-72 - # Evidence: kahuna/public/js/upload/manager.js lines 16-21 - - @todo - Scenario: A failed upload shows the error and can be removed - Given an upload has failed - When I view my current uploads - Then the job should be marked as an upload error with the error message - And I should be able to remove the failed job after confirming - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 24-38 - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 77-78, 179-185 - - @todo - Scenario: An unsupported file type gives a helpful error - When I upload a file that is not a JPG, PNG or TIFF - Then the job should show an error explaining only JPG, PNG and TIFF are supported - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 155-163 - - @todo - Scenario: A completed upload becomes an editable image - Given an upload has completed - When I view my current uploads - Then the job should switch to the image metadata editor - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.html lines 40-48 - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 84-92 - - @todo - Scenario: A missing description defaults to the file name - Given an uploaded image has no description - When the upload completes - Then the description should default to the file name without its extension - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 105-110 - - @todo - Scenario: Deleting an image removes it from current uploads - Given an uploaded image is shown in my current uploads - When the image is deleted using the delete button at the bottom of the job form - Then it should be removed from my current uploads - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 - - @todo - Scenario: Uploading a previously deleted image displays it for undeletion - Given an image is uploaded - And then deleted using the delete button at the bottom of the job form - And the same image is uploaded again - And I have delete permission - Then it should be present in my current uploads - And I should be able to undelete it - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 - - # Existing bug: the UI still presents this button even if permission to delete is not present - @todo - Scenario: Uploading a previously deleted image uploaded by a different user does not display undeletion button without the appropriate permission - Given an image is uploaded by a user that is not me - And then deleted - And the same image is uploaded again by me - And I do not have delete permissions - Then it should be present in current uploads - And I should see the undelete button - And the undelete button is disabled - And there should be a message indicating that I do not have permission to undelete - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 187-195 - - @todo - Scenario: A failed deletion is reported - Given an uploaded image is shown in my current uploads - When deleting the image fails - Then I should see an alert explaining the deletion failed - # Evidence: kahuna/public/js/upload/jobs/upload-jobs.js lines 197-203 - - # --------------------------------------------------------------------------- - # Required metadata editor (jobs/required-metadata-editor.html + .js) - # --------------------------------------------------------------------------- - - @todo - Scenario: Editing required metadata for an uploaded image - Given an uploaded image is shown in the metadata editor - When I fill in the description, byline and credit - Then the metadata should be saved automatically - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 1, 30-56, 61-72, 85-104 - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 33, 40-65 - - @todo - Scenario: Description and credit are required - Given an uploaded image is shown in the metadata editor - When I leave the description or credit empty - Then those fields should be marked as required - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 36, 96 - - @todo - Scenario: The description placeholder gives guidance - Given an uploaded image with no description - When I view the description field - Then I should see placeholder guidance about who, what, where, when and why - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 34-40 - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 157-161 - - # How to verify writes? - @todo - Scenario: Choosing an image type when image types are configured - Given image types are configured - When I view the metadata editor - Then I should be able to choose an image type from a dropdown - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 5-27 - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 38 - - @todo - Scenario: The credit field suggests existing values - Given an uploaded image is shown in the metadata editor - When I type into the credit field - Then I should see suggestions matching existing credits - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 87-104 - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 67-71 - - # How to verify writes? - @todo - Scenario: Metadata only shows when it was already present for some fields - Given an uploaded image that already has metadata values for the following fields: - | byline | - | credit | - | copyright | - | specialInstructions | - | description | - | domainMetadata | - | usageInstructions | - | imageType | - When I view the metadata editor - Then I should see the metadata values in the appropriate fields - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 114-134 - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 36 - - # This also applies to any textual fields — we should pull them from img meta - @todo - Scenario: Existing usage instructions are shown with room for more - Given an uploaded image that already has usage instructions - When I view the metadata editor - Then I should see the existing usage instructions - And I should be able to add further special instructions - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 140-167 - - @todo - Scenario: Applying a field value to all current uploads in a batch - Given I am uploading more than one image - And I am permitted to edit - When I apply the following field values to all current uploads: - | Leases | - | Image type | - | Description | - | Byline | - | Credit | - | Special instructions | - | Collections | - | Labels | - | Keywords | - | Photoshoot | - Then that value should be applied to the same field on every current upload - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 20-27, 49-53, 76-80, 107-111, 129-133, 156-160, 179-183 - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 108-119 - - @todo - Scenario: Metadata editing is disabled without edit permission - Given I am not permitted to edit the image - When I view the metadata editor for an image I did not upload - Then the metadata fields should be disabled - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 14, 44, 71, 101, 124, 152, 173 - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 27-29 - - # This requires configuration, which is already present in CODE - @todo - Scenario: Applying a metadata template makes fields read-only - Given an uploaded image is shown in the metadata editor - When a metadata template is selected - Then the affected fields should be populated and made read-only - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 15, 45, 72, 102, 125, 153, 175 - # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.js lines 84-98 - - @todo - Scenario: Removing a metadata template restores previously edited fields - Given an uploaded image is shown in the metadata editor - And a field is edited - When a metadata template is selected and then removed - Then the previously overridden fields are restored - - # --------------------------------------------------------------------------- - # Recent uploads (recent/recent-uploads.html + recent-uploads.js) - # --------------------------------------------------------------------------- - - @todo - Scenario: Past uploads are loading - When my past uploads have not yet loaded - Then I should see a loading message - # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 1-2 - # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 34 - - @todo - Scenario: I have not uploaded anything yet - Given I have never uploaded an image - When my past uploads load - Then I should see a message that I haven't uploaded anything yet - # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 4-5 - # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-34 - - @todo - Scenario: My past uploads are listed - Given I have uploaded images before - When my past uploads load - Then I should see each of my past uploaded images - And I should be able to delete an image I am permitted to delete - # Evidence: kahuna/public/js/upload/recent/recent-uploads.html lines 7-13 - # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 25-51 - - @todo - Scenario: A failed deletion of a past upload is reported - Given a past uploaded image is listed - When deleting the image fails - Then I should see an alert explaining the deletion failed - # Evidence: kahuna/public/js/upload/recent/recent-uploads.js lines 63-69 diff --git a/e2e-tests/playwright.config.ts b/e2e-tests/playwright.config.ts index 15f9ca94c5b..b2506c2ed07 100644 --- a/e2e-tests/playwright.config.ts +++ b/e2e-tests/playwright.config.ts @@ -7,6 +7,9 @@ import { KAHUNA_PORT } from './setup/constants.ts'; const testDir = defineBddConfig({ features: './features/**/*.feature', steps: './steps/**/*.ts', + /* @todo marks scenarios that are specified but not yet implemented. Excluding them keeps + bddgen's "undefined step" failures meaningful for the scenarios we do implement. */ + tags: 'not @todo', }); /** From 6520661d878a46c7e49bf5e239f5b5b8a2324694 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 10 Sep 2026 11:21:50 +0000 Subject: [PATCH 357/373] Implement the runnable upload e2e scenarios Wires up the upload page shell, prompt and select-files scenarios against the real stack: eight scenarios covering the upload tools rendering, navigation to search and to a filtered search, the current-uploads section, the prompt copy, the file picker, queueing selected files, and the upload size limit. Locators live in one place (steps/support/upload-page.ts) and prefer roles and accessible names. `` gains a region role so "the drag-and-drop uploader should be active" has a hook. Two things needed handling to make the page testable at all: * Kahuna's first-run graphic-image blur explainer covers the top bar, so the background accepts the default via its cookie. * An upload reaches a terminal state within about a second, so the scenario that needs one in flight holds the transfer to the ingest bucket open. Two more scenarios are tagged @todo because the app cannot currently satisfy them, with the reason recorded against each: the "uploads in progress" leave confirmation never fires (ui-router destroys the controller scope before $locationChangeStart), and the prompt's example label is guarded by a `ctrl` that does not exist in filePrompt's isolated scope. --- e2e-tests/features/upload.feature | 6 + e2e-tests/fixtures/images/test-card-f.jpg | Bin 0 -> 51942 bytes e2e-tests/fixtures/images/test.jpg | Bin 0 -> 70488 bytes e2e-tests/steps/common.steps.ts | 28 +++++ e2e-tests/steps/fixtures.ts | 7 +- e2e-tests/steps/support/upload-page.ts | 50 ++++++++ e2e-tests/steps/upload.steps.ts | 141 ++++++++++++++++++++++ kahuna/public/js/upload/view.html | 2 +- 8 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 e2e-tests/fixtures/images/test-card-f.jpg create mode 100644 e2e-tests/fixtures/images/test.jpg create mode 100644 e2e-tests/steps/common.steps.ts create mode 100644 e2e-tests/steps/support/upload-page.ts create mode 100644 e2e-tests/steps/upload.steps.ts diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index 9d3c2ad112b..e87e4b9d141 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -52,6 +52,9 @@ Feature: Uploading images to the Grid Then I should be taken to a search filtered to images I uploaded # Evidence: kahuna/public/js/upload/view.html lines 22-23 + # Currently unreachable: ui-router destroys the UploadCtrl scope before broadcasting + # $locationChangeStart, so the controller's warning listener never runs on a ui-sref click. + @todo Scenario: Warning before leaving the page with uploads in progress Given I have an upload in progress When I try to navigate away from the upload page via the following buttons: @@ -71,6 +74,9 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 1-3 # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 20 + # Currently unreachable: prompt.html guards the example label with `ctrl.presetLabels`, + # but filePrompt has an isolated scope and no controller, so `ctrl` is always undefined. + @todo Scenario: The prompt suggests an example label when no labels are applied Given I have not applied any preset labels When the upload page loads diff --git a/e2e-tests/fixtures/images/test-card-f.jpg b/e2e-tests/fixtures/images/test-card-f.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a57b764b234eddb94913aaef880a4b4e912e77ec GIT binary patch literal 51942 zcmbSz2{@GR`|p&BuNDy%VTurnRF;TIg-~RjvQ0&0NwSA{D~TkNy%dvm>{50yE%r5g z_R2almY5l1ws+2>@AtR-uXCMqo$HLNnRj{K=eeKl{@nNHzTddtxjzs)jr0xm5gRrl z5F5ZBf(s*dUbyDxsH(1{c0x(zs0xBRf;fxd+qrYs&Rx6ze8l#O?c29c41BBK-Q0=TxPgaf<0hWXn>TF&)dYikh)sf=hN0k(E0nuW&+DOjVvs! ztkIXR*f=`fxOvOj#ns!#*U$f6Kw#+OuqWXWPa|XF;u8{+UcO3B&&bTm#^>bbz5h^J zR$ftARsH2_eM4hYb4zPaZ(skw;P;{7pT8!jre|j7h@|Be3YE4>Ut=%@fyf&-ZQ|L) zyDsvEjsD;tkKm@w2UNBQ>6-Cg^V}tM>>;1Ndd`w1b)w^3yymuG_%Vxu5}M-agsMr`BR08YR$3Ne=-UR@sA z<|Xy)6s5#_a8P74I{l;QqRhnpanD*Vf+~H!M@A^)n}r%y*Q7)EiMlftncy?*V@7x> zDp>8PAV3l6M4M&@Gf-ni^QdJhuK`%Rc1P0uH>hhbVJVD@;J0w+WiLO2*D$~HYEYd8iR^rW54{l+6Ufi7v(m-?fs{}E=c_}FZo z8*w?q?vRu9CiTs_$U^`0feH9t0~Zm}%T|2SOPHM231^_wBbm3QOj2|2bfq+w$S>~K zSwYKTWS$yY-s=CELQH!NuI3DD^G=&)7V&gz+AAFzo8A#Iaj0>cmvB3BzV5%PZ*f79 zWw;1$=Ae%v`Dm`D5z&`8bHDCL8QTB7JSP?i_4SYJ|5K=tP>1k?LCRHsG&ePM(Su9Y5|&6&Z3K zkoT02>9)~e?c}?u%}Y~l$xFJLckJ>5-dxqC0l)NtO%YxOhlA~Ch0>#m=wQw6chd_KlCN zpN{b^*}OJ%Z*k1o`Eo9T@&(CS#r?dXwtn0Q))?-mjZcr_hgj{T(E#?!BD@-)*XR5} zga7TVcOoh4>!n;7HP~{+cR9f)SJ`4dhl|*u6S}^CvoZ^IWYqL<5q&5Y7tvFlJWVa8 z32+his4QRA)yvS&v)|U(=;f>x_M;3{nG=K8W>YZ%a4lJ>3{*%M_@Ma9m5UH^zz=uB z;=!9T@#faNSH_4C3B%zci1vEknrLPc7g2=ksg7hakzd2pme-0Wji(u#^<7;f2F*SN zgv~PP1$linT!aJd6=t99S}aXx0Q-A#?s0lzY*lpJAK3_hWDA_}>7pHhfd(o{t*Z+M zW$r(bTkBzj=$a+RzI4m^b!vdZ@wUF`It~>FwiQPgvJC@qw+z-isY0=dku#bmy~bPw zT8{OlI&Y@*GYiNSSnbM;M1Fm`@)Gx}ZJMz)IjS!~lxTlz!|Wa$Ir+V!^%BQ>!2W&; zda13CC5GArdYF}2Eo6iX3Z+i%1h=iu;QEz-|qQ_0j>ooYPM$ zjZXHNR%2MrUn*+5P!hT9q>PJ?PNj7OY$qvpmkRS$$QtnH?Tm|}HN)}oaW_{o%oP^Z z&R21sx-BvKzn4Z_(~60VRq3;Q|5`C#ewCo4+nsPZB!zT&_c(ELzTM($-rJF3mnTUk zrj){7Pk-At?y?1|L9v{0I2lrazC%R(O5V}h0_Oqxp6eziLu$v~t;)&i^o{tHmyZb# zUU%|diG5z>4G#o{hz!yq3GJT24oNG?sH zF1Kxwb8$@a+i_H*sn#&0OZ;^1{>245{FQd-ixw}N8b>XRpc0N)iFEGCHc$;+t-&(p z`fzTQTtw<~+b>?(OA^~|nW&0n7Dc~iX%h%cb1vdm1)Ym9nudqGxd?4BZ22e`(PdqP zE9i$kK8xe5ve<-YntOU*^*!g@BVI=S#u|)YK^-s)E}w-1lXXVcAKHFRGcI@8)r0>M z{&)TkF5+>ZGGjrG2v!x}ya8-$$l!0Ss7j&nJ&bv0*xbcCt?Dqo@>4ecFAP#$bh6NU zU&_WcUy+BqP@5!AFW9Sp$FRPx6xl8eqA1^bICW&u3@PkNFF~>7G4V`RRl>Y4>YB$= zNIVXFN6w&1l?bzGneex2&_!I9Pw>rR=(G65AF)*zPh_X~y;51+d>y$%fU(F$(3|mG zgtI;9p=nPFC&L%)#-6HMCoTOG@fYM9Q)9k`!h)Z<2o%`q_ajZ$40zdWb!J2%oEg-)JrTlRZie*HXD_vrcee?4j@0JQm&@ipo1><5%c0N>SwOBXh%`s29 zMSpKxD`^}{&@58i2MZ3NX5X>%ZCU1I4z@LpWj+gjD)Dk=V<|7PbMucxd8`>^fxfpf z%%LyZ>TH^6Yc1OIX!xY8_C&Pe6S1sLO#_BlYh%4%;ED3ib|>rBg`&q-T1aK#oP19X zfM03je;wF=E7o*NRI5m9GJmX~)uP3GO>m6schPB*%NvkeE$gB?vY*SX7B+!j?ApMD zP5v}poQtr?8sbbVc8CQ%VKDgerwcq@W75uuED2^b-ZTp zw=X>w7D;S0UEMs$raq%(fYn{R)WGa#(6qA3%U&E?3}#jSSU6?<#kE(&+!6ai8){5% z?t6BzJG1cEJUjeFGJ*^%BZvnwUU=FO%MCMOohP-zGRv zS(4v+TF26nT!ZS&ga?w~fF|ZQURBSx?M~i;7avbK{fJsE;3DStbzLSbW!U3a3kZ}> z)F9hUiHuI!r4WJF4p7~F_T_4A;b5(n4$@-|qSw~G%ODRxvKSJmL3}t-r{J)sv_pHd&fw4J{fldmuN{b!jlmMQgtl^bQu@mfd2f>`l zg|3mVmSWI^(mSA7z8}3p=tAV>tM58I)!P@DDJ_Tc zkHmmg$j3!+)IC^J05QItLe17d6H+y=iIRM;l0uUUZ>_$90u--7VL=sx!x>XoaPC>p zoL3oP@4aI;yKu0a4D)gOrdyITr+}A3mt!j?m+op8TO?UW3bwx0m|$=b0+gaK2TyC$ zb6!SaC$-A_I3}U<(1OMGm+&%2G5PVu+za-ig$r@{PLD!fO`=+3uShZ>sV;?l`g9As zg^M_er$qu6thf@F-veXXV=oGw+*=aU7_4Fiy-Ncd2g`^|CY=s*KT>f~{%59~9LlU0 znRP``BC=^$^Uc&V$2-q(vIEA#CurR(E)ow?7OB(~nHBC?H{ zbAfBPA6?UoWHnYu2hU%Q5l@kKZ*(rFrfjz^J{^t zFgD~`4R5W^Kz~*x>-y+fX(j*1SMQ#0+@T>tLhim!oH}3OI$u&9_VryeQe}3p>gFHY z4<0Cg(fZct?FMO{mL^AAwN-by*V7kWJ42XpHNY^*uFaUxtf%k(a=vcf+4JE9|4C-g z&JPM%3th?Sy5hPoB=zrVviPSZnZLA~Sq^Gk#I+EN@29;*1NX#?KfrO@yu>$e`|9l( zu;}Q0qPu3?F;y8x{`-Z~Yw7ft6cmauZ^#PNci};imdSUq3h@M=R z;J9c~3VwgCm)n#XpX>Vj&c!TQM^ya3$X7S(>!L;?&0hU*SLrvF5-r&IdYAWM4IS~o;U{m*e7g3 z+^!PS)0^w^pw-6420;Zzm$oz)5kN=@XwRU>LUj|JwbB>32<&+-VjP-v@ISa1Dxe{Es4ikhfSd^GH969#ndOUxe@FgBaSqca9OVL9F8vqwCizr`J1SU=sKzMJ6 z!(U;bOBD(A1+|nT7g2DS?c`e~e{jn-^Is{Z*=7>L%tIRKC>CP!Y?ja0BPLfll~yyYSc!#T^ABpRN5 z|17+g{WM78Ngk>j$N9-7cjot~V%ApU{ytV&l}g9yT@w5io0a{IM^T8fD3Dt?-0+bV zN;Z6uG_gYDBJB1s2HwGs-Y->O0nWk(HrWo+H73yd;?<%oi1P6Uk#?5x<{M90hrQ$? z&OhV)w9GH=osT}(bC>s!7S4M?_b}!Ay%vQ<79nQ8%)*qP zS(0N5D}OZkm%ID(A@#wg+h^m?e)9OlMetGzzFsRIXqbIJ>V(I^ zV^>lc4G9rL{Oi}XVhtFwDTkjqr#F7u6H~{Eu39vd#J$CLC=BJ4yt7{+OnB0tF2!o2 zjG)ER%kNZqOC>cx|lA7e9cJ6+m*pd9Fcd|%r6UxwbMyI23R z-nWxdq0^Ky(V7)0^UfkD(BFiec`K!A71hAbQ=-8=DV(Pt9tFR9*jm$+L z3X7t5aY&To4!K48?R#7k8C9P;0;(QCjA=Q}ikocDScdU!v%L3yg&rTefWhJfXrYU@ zx`bmU1CQO(7*$C&GV{={qlzU_|Id1`dRxPyViS_%2)vjHreLw-f1ec@)e5Bltp?qF zlF{s7Te@G-2U^oeld}JfygY2eZif+GTd`^7#tV zZ4dpS!N(!5VcY_wi9Xjay~hvVTJ>ySyze$Gq}{4EkvFMO--_TJAYgab5LvG>!Z71N zpU=RAAIA=5u3%R~-X6n8WTD<3grB?;en0miVApiv8+IY53&Q8*n>AO*mL>cc^1}I6 zdo+1Q!*yli=8mw;@B?!Fu~hg$Y84gr5YGvW;{-yVv|Qds&zzFf6}6V+d@j~Z-t%y^ z;zy7m+4^U){cevBqfR3w3?vITD4DTRw0cjv3JZoH;TltL!63~bA?89p<;-uLMz%l> zO~kq&UF?EoW%f5|1px-%XZSa)p9jb%q&Ww}=+7Zc9pGh=Ij($bA74VHTtsJ*$1BwD z?a4;LN3NW+OgJJdk7zw>-s|#kj)`Fh%3a1n&72;#M+QM+od? z)7FQnXJhpB0w?}(O|c=C6Yp)Iu7m8{hf|~0!ClM`4&AH99*n7k7SifB12^fzuJ~v@ zxp!UKIQZ0IKwl;?1`05qs4p3UOcqO|n7v|?mU=@k(EocOG#d+-;;xkeIgOd$j<>r>{ZZyaI$zzu#tSFg-6?b5 zW<3(KF{RCLo_FiIB2Ppv0;4C*nn^|KUiO#f2z`Zzp*gGh^u`hF7`D2$D{E^N@;7w6 zgXry$w7M0;m_i*l{^$2XMu)s&Z^&n0pqvVyarxJQGf#WobTM;{FoBtM2<71ZxlMMw z9ko~g*bW1^LKTrR(=~b$3E;tJvP11bmtzT@F9IXe{#DlXG-X@2h0(^A{bmouA86-^ z@vqZMo79Ryr<>|gONHk7tAyoc0T$V^V%PTt-+yC*X&^Ni4Qo$v5%(ZXO5TGx13jJC zmyPBYxra6+JP-Q9Khki0BKR{c_o^Bz3&**OnYEc@D=Kgiusw?Wh>Oq!PAu7+1dOrF zr-qTSi8)U9(s7aG`P;{z-VlfQQ{mZ>;Rn(sC1c>!*GikyRpvfE-gHB9h7>}gqd2a< z9IBt$;+@N!{zd2$dXM|;MwC|s^h{z0s%Cwl-Qze15(l#9Uy~_+kl`+1W;VdS z%1gdgrk4wTor-DnWJ`=_P z%DHF?9IH*3-r$cf;x5oT58pn0J}%soewua=igj6OEq6V9c+dUfuLN^bFH^?G#4FbQ z`nan#(;jsk%4i(yX})X6)7||X+Gwq*McntGTa+wLU^wm0`0Z;erRPiP2fgT|fO)M+ zMxW=G>cH0{z14ox(x(>K0?@R%0l!P0hq4Ll$@EUZa|Tz5!dBv!_C4}g>ipiM(ip_t zwXiV$0RA|^-l%a+{on*9rDTU=6rHL;g?$=gCoNuH&@sOE(w%x;Ynvx-+GOL!n`aL` zZg3H0uJ?3y7;l^Xu_}72(}>4knbpKNT`V&Gc3~PZ89{k;82(xqWIIB7m#@@ zUx_|gsHn?zWl1P%E0jN2h)%eN%c|%nZ`%dvnQ#C&hBtfGFlnEN&@#&du&%r-s!LWs zs{e~W><@_uxK0-C6K${}h7_X9rtE9%)xZ`C#6~A1p z$KOCbzW@vvO~T)x8vEVx3o@gZ9l|}C{IgNWW8YS@o@VyijTE)7Ub$xYEh8$oNLJ2W zzN5CO%SCR?kuA#cBq!=(k@`t@ISX&_hcy?v zJO_a%Vtm*GUs+qqnf@4hZ2ug(?-qVg-11Vh51l14*l~xgKf8AP`{Wx);pLw08w!lr zP7B&l>SVEGM(z6Y*bu`(QO?d__xT<<1!y+899Tx0&5ZPkDasNXAHpg9LVc|LD)3W# z1FZi_N+(WkxDXh%?n>JtG_TSn0;ufgI?hotz7nUN=p#RxwAX&-O`}apVadHPf>4ZE zI}kz3d5`iA;Ju_V1?^mwTfys-^BM>I9dGmne1#x@MiYG@CqA63$g2)}I&>^}dMT9z z%%ixohXea`ZeyxyLUVXu7@X0(8-)CVCByuCsDzf7ud;07aQHZJ8-A^JOK|XC|zp8OBsf%`B=cA z&baAZ)rsuz3hYwz3K5w?-3FdQYEOvozR|Az!#y=T-;>>=`T-0bfhH1bi~8VSNahr- zwa!_uy|p+odisp)lpuuvs=8>SZ6A;hDQVxT}cb42|2YEG=v$tn;Y&HLC!nMOph| z#iP^{uSJUi3kr6SH@?bv7TSK+TQy2SUQ@C`<=J9ZQqXA zRt-`v$8^m)3xJBsA5dA}&&CiW5fy}OpIt02UgMuCbXT7>>oaYmX8`?raL4$w@wC`u z!{0hm&R)_m$rYWHE|F%{h1Q<5w!d>$I!@$`0kK9R-1N7ILP7P^6BpBjTcSUFCZfOM ze+BtvCZhYDcKhL~%dr4O&S0By*ZTS(Yp_kAi*SU~`?~C3oy`G$%TZd<51}*AD}2Y% zimiAexmusKG@3evgX=cyxYQr;Svcu68xDocHw`X-!fTyP*PGU3k#q0eY&1lf+>;0o zDTN%;I04G)Gpfl)gp`#~6ZUOU6##*JCut{Q>XU6=w|%Xf2J7!f*B!vmBx>{;o%Q#- z--e$?pBy$N;w*+=ZQ62yB=+OuN8Xwj=U+)>Up^xrJaj~SZ-r3?tmk$&d^Jxtkb=b`7Hzdl z3Mb>IpO_Am3LOpQdCO7k@R&}J*&q?`SmwADD2I6v7`oJjt}ZjqY4_S8aCR=(Evh_z z1Ub)|NS6I2FQA(mX^Kx)5IrH^_n0$M@)=+!fQg9Ej5>%d?$xTeu4NCMfY8V zO6=LIJ)ZkgpQbi-LkY^n0EQq3rF>$S{(zG5{JY=H#}7JYD3c7RCs+|5eMEzDG2s1e0{x zXDhD_l2BqkB}JLff}QTyMAWTt5j(H-A&De;5Kv9M4kzK_ghV$p%ffz-+{JlzEl*Z1 zRECs^B#b8CCS0#U$$AQhr!xiUrbKIjgd*LVh&w+>nj+HGvujX-{lgo5Qc>gQGV8~J zTCl%#LI+ELuN6+4V@X)PXi}cAx*fQhr{4-9^TlNNMBB5+s#yq0A;gKpA zi8qYLPRu~Gil?_`$04E04bg~G7CeP;+j=XvV5Y}%21{uW4#2U9`Qa@4F7_(ai7f|6 z+I5Q!c0}JcfXeR+p~?w*uNf!7jnfERyyu?C+tuLlE+9k3LIs90ZraP1e>l;AANlm_ zob|Ta9h2i_o^}#p9rZ~n`K|4EKix(Fk@3juLXkGWi)e6zmH~2`vB~8{il8N|Ix{lq zUH8wXLbXh7mxMw(8FdQfg<{U& znA`8SU`!h5yWlE39BOND^X%9jdFQO`jpjYw7j|4J%^7qT8@L}hn7Pb~MNYDXuvu}`-9)QJz9C^auC5=A+4ZqmQsy z4B>+u+6jf2_QbKI1eOe$Xw)j}kMI{+!GmlX>IES45p&S{t1uOpbvJqJ zjbDqXd3|gkM-j6?$58so| z(-j5QGvA}r_j=kp*H*P~$iBn|9k>9yqPbQ|zV}8=G=dn{80=|A5_{|?A4fUK$|cjN zB7pVu9RMSG@UH5hx_4NArN}OB%8(w|62;U%juAt1| zLCUPG*7=rCo$8Yn9`#g-T(U5;ed)-+wX(2??ipRB|yssTD}ZmfLg2Lk(l zxJ!gnC0KosVMqavNa)^{XsGJ)_uY3AAnv6677ne|45$zC8B+~$q>l#D504!u1^ML4 z;ckwpn>Yk&VIAf>dUfy>x+eb5&V@HW2&gds-M+<8j+n}O4L~3X*l2>p)xk5Xfkh@^^ zdCNRW^C>-2n)p429i~kFh8bM@IPv29{;?E;OQ(DzrN8?f^kwegkYo&1Uu|#rtTL>B zPeOfig?&*WGr9iBuOBZLqz~rBJzrz4(ohfbdM_#!A1W`GwEuWtWv<46@xwn;hl_Z| z+4;UrsYGsQAw0odl1Ke&2K%?x;M?KQ zU5S;%WnV{Fcoj*F8E@N}U{roLa5Ac&ui)py3aP795(rK9E})T=A>halze`5PRA?RL z4@};2d)(FRie~d&*-~u_Scngr zeSN*M5MyTibElQ!2Y*c9EGfP=2md(B#MR7%+D1Vq7QD3{S4mB&QC7F8UZEJOh8Vb8O`<+!!o<61qyRz zNZK=M0YV|tQL*M-Ed^l6EW8_4uPspPDV(BdwKGt4A$Hs#^s7Mi7)UbUZ!df4Am6(b zzI3|&4uqAQT?_Xxv)Qfw$aA_Gx~GX;!Y-%7pX`+k+N4&ZG)_u#5v?!om{r7&sILao zJ9c?)EBqEHuZ^Hmvomy_%Rjo^a!a0JINR1$aRK>@-JtWAh1K(d#m2UHR>3{x=t)!q zNsETJ?HIh+wH;!0rX0x4*K-78ZX^!kCOz?_2NL1Ukm zFL}|trk~KscB9&Di>G#c?aBYt@L5$m4h;#pBojwFWZHbqP|VBQ7OhY+a)!B zZ!V9vds9K5oT^RMf&ly6Ml+6gM*A2%4^nJ(lQNh2)umX+_eAdq&P=$c-c-31k@yPzg2UuV&xOR z8;H~_9&Fzm)3{}a;y2!KN7Tx=!vm_rB+)KmkR5ax0*K9&2>OEMY=#p4k_wV^2mK!f z*o|p{zNGwpyu2=l!|TWpg@WVlyiWj~i+lRFIHecgZi@@s%GhREiSOO{5^vspVN-gf z4Pxal9X5!(=~)=U?@j5$83k=%DvlhLD%4V5V}x@^X*PamISNR6UW~G1c0V*Z3d{$^ zM@3wOC0GQ|+TJ3*zNqL}oZ;zhnGMGJheWq~Cjo?j#7K&-`nHHbVu_oQ!`b)22DQ6Z z*f~=I=%-1N0CRG?hyMwG+5sQx!RN!W+whyvnKq+^t+N>d9|?WvWUGiN=aC0i|)B ztv>dDV8Y+SLnQNzMq|Ht+!&VrdK~xf^Vq-=n1ZyKEH1u3+9djAM{6WC zu3Mc9vTDUEyT(hS|H~;0D&F)z^uPIL3BCJZ8Kmhr7?)w#VVu>BK&@ z#96CDM*h}=y+AD4!8_9Grv5hkW)odu zzD19Do$jaL-2oG;@J*A%uAY4TmVz%=Uk3Pl`p=o5o(6FY3hxZr7xpmEXUt$YFN3Ut zYERf1i+5K&(c7Qo@E0;X@UWr2fX?dfvYKc9@t^NcvW>L6v}!qu1v>U0gEvZT+c3xL z3bpY|-sL@rn+fg+SQzWIs}SEmrybnEBUN4Hme{3-^nq;J@n*W`e7-2S-jR=JKrJ(P zSwuF;ke7LXVyWx|mRbWGTpUxxP?`N0&?mups1Ep%GT3}tB5xaeEK7$G(aEl^MppyX zo&zmW2zXoUN|6Z}){zA%%GC=y5d?}J%hKyp0c zglP%FF7{kdY`G?GAz(exs1P{}_Epy*kZ3fW^Y)enr&yUyF&)4z1gxiv6?*l>ZzDlv zAmos0$Q(rFkOZ`A()2`|q*YB(aNd>MMYp2npLPb`!p@!*?C1cU**!rUu%Rjjn++Ta zPsH=jIVH>YTV z(gJA?*&q83ZN=qV70ylxN&BS(_`^E6759AF?AT z#6?Is^YE~)5I~UVW>v5?qiFx|CFS;$TWSV+7O-q8CIh17Z&^A=u@Gzvg(pz0aiX(m zXnElpJPFJgE23LoIqzb)-qM0y|6+{qlui1f`1*o<9ug+^I!>k~ihDdCeAs7xkc27& zd0YaWDeIy(hxj(#%*+!VgxruIaHNS@0D_zfpS24+av%j-1$2ArSxWvTSN$tyr{1cI z-_lkLFH}UWtKMZxW37?=PX-iO2P0y_yKrA{{2HT~?#3Rfi(*YgMwZe0w9tfl(EG{ISGkXwSRVqU$Mo zo1dpAMqWK@za?b=LwDO!eMGB(u-GH>#v1H9B=Wb|6gjV-FgXkJ*p*#x+h;kwfOd_q z%{?IY-$9iBJqcUcb3J{BjN|}s`P=g#IZp?`0RjCu?lI#QUEa(*MF>oNMiVuc8^_SY zDcQY)OUkUTj6wcUW{|TVGikzWa8rM`CbW&q#xe+p2zVkTPfKq$~mdOnO zMf%BmUf|1Bu(R;IpuZJr8Y8fPWtnCP_IR_t2QUKSU>(r!2~!R~J_$Z!0U*C01VD9< zpSCD5*_b5aC~9bK22z8bfTR=Nbmu0H2cPR7HFrWvns=YS8@K)gRGguq!ts+#<$Bc?uHo5=6%(yahnxohn zzSavg5WmrfZO2ST%nY?TynTaiTav`@x8850iUU<7=2+FJx9I~Hk$+~KL(n8$?|CqT z?qbO|k?jx1>G6Hu54}e*`y=QD6L;M?7v1+7+4Fg+PCnYp+ZtIn8^=g?v(wj_CE9(y z-gGd|(B3tYw0NW4qim3lrsRb59-tz_Zd#B-&KEtlPhvM|U+z_}$YqlRjnPh07duFh z=DnC&(^)H`Yr*Ev$D?USQ)(VS{2h2pFMRjrj)$7`O;#`FMxBsguMZaR`#KTzlG@we zf46PxSHoE`pd^vDwyf4SjbC&*9i28A*d*8|S(fJP?YW2b zE38r*Bpd+?4d}ylUY?DH@}Q;~@VZCJT~+O~Z{tf#FCEQFPdUB4&=}Zx&p4#^KL+13 z&JyvW(5j}r1?Mz)620BNGZ%gf)hR~g?j62h`7-^1bc0%U_VtG`gGX|xrj6BKTbh4W z*&C#Xs2=`n-L_i=lP9oJ6k5d`*iL6dfRnNN30o1!ri9$B)Lp`*BqI>V#zh(-N`#BI zZUKtJhH4zp^V5Gv#*%<3To#b3__miY2vXKA9>aJI6e^~t%YX9n5BycBIZyLoIMP%b z`?eX$Q(tEDxRlAr%5RYHvAM@Z*uo3m{XH%W&JwHiapzXxn_%PM1+V@-5U5yz#zfk! zw|#TBNGh|>GhbyppG*F~?^=n5d#qskdbZlD;%gww2KTR=nX)WM~Vkkq_J-wXTN9r_uqg;wRncmuBiEbO{`H*`70Mmko<%+BLGY6Jyr zABWmF3WI%N7;dTr_IU&dq!h!o$nSI{g|df3TVFBI{4$5afM~>!PpCRb8Lrg|5EbJh zzEc1Qw`1sFd>oQkk{%4kGjN4GfLaX%lLccqftY_i_+7tZ^&Ejh6ZlhMKCekt<93o` zZvc@oAn<(;>Q7;^>~FSNeb-F$`AdP@%s6+pU{);&6#CyXWUyp04A$TI*#oHWdswp| z!2^K6J=pTHe%I=S^@6iN)(n<{DT5vUM-FvRtZ5~S#s8<+{RiLS);JiF0;51+QgK73 zfd7nvCV-gD{}c%hTeqtHR(Rfq8cSXNKKA7<$@5KIXm8kcu&VH9pzUyBAc?-4^AONesI3g-ag`^4gySY~%f-g!vDPjk%~$jICMDd&f-n9^m=sjptnDwzIrJ`Ck+qQp2Nlmi zjjuVO02{OiDX2;NTdQBeTyFwH*R^KGc?X(lfR-{~;rZin@aH;UG>{p%beSn207Qh-yRRlnT{GI;E3;K=*PY*%I4LM;DsNRC2I>1HTu3-Phut3x! zjOVh~>D?s5IUaEU?1!aai#g=I-EjBl7L`hBj>XwpC6LKd`(4QNhsI2;yRnFUt4IVR z>Evf>#|=CC0*nH?gUR2oSCO4w<*J?Nsj%fsPNV7J@!2ZXXR1`}3%v7s@b1KiyAJwn zD9dx##PQf&)0lyp07ug+uB@HKM+ApnAxN&Ol`RL*_{w7KT_=}}>{}*f-}z3hej{2= zk^)`Nn_j~e+FJ8QKdH1mpWPfX=2odtq)p1WLX+>VYe@{#`S8^cX(BF zpPtwxp4+OG?rD-kq~*RS1xR(KH@B38h)qe3PB|@P(QwwB=kzW`!Dfy8FLt!a@eg{M zAL1=5B)^P}6t-Tg)cDzC=5WWS!$|(P@Wb*q`A!1gW@?Yv75DZOZBv)}NbGU5a9Rv- zO04et>eigO>&*@53oxf{y4K}`5s#$VJ{8;BmLBjJ+Tq#Y4fZ?r6Ep#FTxUvdNpoK1 z0i^;hrNP3`P1lKWQ`S5b<_C=+sbH>=U?ZDDx1rt*e=G!APDPHm|EfbF2Uyj0;b09U zf%?J2CHA=0CeBeLeF7NcNi0YBB?QzA$JJW25xPNIuXGLyJD+@~WZxF_X@j$spampo z;cqFbJaYFvHAfv(nv7cPL>c+f1RD)+6t*5+NfYfF$NbJSxX>TIjOS2MUuwq8IrD;| zg0|j{v3fW4a=WxkUOTz9ZSx9Qsx^7wg&Z0#rVqkQ%A!QP8z=0_aR!_o%E1pxZG+%292R3mL&}Pg2&q1qp zF)z|TgaC5lqQErJWjUiamsTC{V3Rn44mBvRS_9rkJDrOlJbL$^QW-21oUFcHK6?v& z)j9(d`Snlne;t!?Fi!#t8nwn zpqf8I6xv>txZ|X7-vph(@lv?nZmgjV{a?ShwXqGI{L+d(~I2fp4d%M19 z%2Q=HA-BE0~CsS~<({Wm){0r1T)GjixtPs zSl(qUyeIOsfpKK8%u%7(6ypoq{vr%Cz+eUe;B;c@2pFP+1Zw>S>v0T+4_(zpA0%@Q z+1_s+`M+CnB$^4RUT17L_SQk}Oy!n~9ya zU#MUXLF8mW(En7fta?pCCCGgXZVz-n)$0;<4ZQg|^#ak7y-@+Y9Q@b89~&IvZaI}_ zcwcP}Ol_G*W?sAS3luP zReMkOiw!ynv#)C26lE#-HR@zno$7x&MLBqrhRU)Q`PrXz;?>aR>r-cCux|^U zO?DoYj_a+x-jq^o>#VjQ8A>ZUd3-QU(na}IdC_T+Y=PVnHhG82jM{N*U|GyA@uIsL za><_Gsd*+Uy~U0<zA zg}2_(!E5!X6YPH1#Fs*$#uxD4c!>QHNA}BhXsPbRU*aKtjyTaOzacXk`F-$6@U@y_ z^*VP7>Ht*Q40N~}$Sw+CX6;nvY5I`{*xvl|zQg`EFNbSi2@Y0KSzacBY7OwN$NS|YBXEaMl720W}f*92g{J-x4t%WPo$s<0V&kJ^#O*SFDk|QJs0tUz~0V9B%r3{G6_Xt z!FEt4o(0s#nZge5h{<&gpWsNEq06V)mn5oF>?JPdIoZ?nfSZig1q!-|B@1^%2dyq4duLUQT5A!B%2 zIQvG~}LP>Ys_OOJO z29#NIz_@U{9<7tpHnLpATEB*BA5_h`+;(f45oWURePluJ!L$D}r$~wbX&(P5g5wUK zN6}aU6KHN=lox4^31qfatiSW)E^Ouy+?A`n8#ueV@EmN_5lQqQ^p9{6)!_dN_)^<< zf4k|rD!5=48FDD^=q_P`O^Cd+;P$}B;MK;(30gCxKsk)YUj&@g0&rErSJLK9E!t}M zjqWZ0MmsA#>XJ&SMTz0V(^31dv=WecPhjpW_Dx!n188%jm>*2*6(p+-_|E+FD|IWu z(5j7_$1ClS)ABSve6Q#=yJ91;GqlaZe10z5Wlr>p%KT^RuRBX5j~v))%oZw4f~$%t zMW7R~P8J)LAS=-xX=G<1ALZ^J=$`sb&Xaxo=FAZh8P?&&7Aic#1VUD$9N zEtsq0w3XA%Otz9~+pUrR;mwJ7i?I8&Efqv7|Jb4#>a&1<>!N~Y9!IM=wth;+G{^{Km3d`%Ht7#Vl6U< zh+|l^8yQGFdl68Hb&-3q+1B{+{l&e~Y1(cXvEL97&JL||SQ33woaN0N$g`I7eqZ{= z&LEZinNFAN$2$byP-r-C5chhzX`puN-eT7`jF{;@tfDl&+q5X|jXX8J^pkv_IsfS+ znz4~rbrN42p5G)Jr;+~h?q-#Jp3TTmdabrwMBi<8#>`uW={V1tX>m`s^PQ=?B()fN zU=;c$kKtaS;=@P3mb&BCd)RVrH{#WNXOSf~Z-q%(2T zmwl`7P*Weqf}Qsw^($Ks&O7E941U10h}DTgjzjW9 zYt6Smjm39qA}J{@|UAh{2SQwryx=iFfSqB(^iSBucm}a ztzR^D&VJy`oPsmPu@kg@NnN%YL~VFvlBu6%Vm#;g+*wR1>iMDS6bY6SaBP&OCGvWp zfo8BJfLA4_`_sE%0M=L~PAWsx@iKX;bRysCXXW%BH8Rx8+D5R*Vg17KOp-9Y+|3GO zs!|lK>lzxNpByUl4kG!d0>$RQTIa%AHTY`1t+s;d=XH^pipu}p;#YfE7ewr&fB0Vh zS=Z}Gc);yqv@_fNdHq(v!oLk-WQ8?(yp?^x8Jjx9O?IZkxs2Mid4yKoUk+2(&5hq_ zuI|guAhPsCvV01@D1!6QT&C1wI*k(brSdc-hE8$k}_`k)OAeHLZ>q)58!3CaSLe3P+Td!R?r z-EfhcE@EUvE1^m*smJpWvGL~`47^sFv|U-+82!`pLMsT5|jO`SHgpnKXmvT7YG zDL*}a;;rxj^?Yl$jb4s|BqzMG9L6GKYPzsKJL?-cXU8{GV`*W>Fv_xNLD#N0RNm06 ze$F(LyUszJn%*AEXId|wTfilsFJr&I*3Ei&E@sl-pI~E>g>q{@^E#nEAfY?>#FciR zs`dx^+jf9F!GqEuGS-+kKwoKu+c@vM)4;W{Z?RFa@vpM8veNBl(+zKK%lSF_I{o2o z{MpoRNwEpuuO%V}Dj!q&Rs(Iw-6U1RyDW0pT^A>w8cl%L(1SvjReFd9!Yvkzwl< z#%sF@#;>C!%f0aNht6A@blo!BCD2Cq4+vhirwue%EevM~K%Q6jROF3SPgi*D9kmui z^E`n;E&@HDz{-DrKaJ)2m*ZE#M_1Y&U{$RFUZ`eCtwITESF}GR9)M2}#lfaKg2;B5 zJrMuZxt{j<2tH@OL~1ZwnZiqb<*fv#=R%|r99i6n%;IFD_vJ!O#%x*6abos)mfGYyR6L|kqs3_vpPv8x<74ddw)3!A!-sWaee=I zt?z#tiQNW3Bk{jZ#ks6aY~eMJ8o``=ex7SrU0}>DhbWMaxxoxaDx4YJwSVN7#)RUc z!9L&tcEXtc03&V3Rd~>NDRWw6_(ddbHZ2moi2DFPm5o>769{%6d<|^)6CfNk#+4lh z#UTc1GhFom0CHb@^C~tCKB3EsKFeH3FhIl~uFc8^wWG~snIpW+>OVJJ2hL`(J;?5W zzx*t11@k%cjAR1L!^LgUIZHTsjqk@YC;l;<<{!%PJ zD~x8CSwjVgjyQ%Hk~W=I!~y#WTp#7Q%}=1u#=CCb0U~0-hwG)dwRX6r>%afyNMZ=% zWfZj{)ueX(nWHAQCQyGP}BsZAUcT0?yDxd7QIFsVq! zZ}99q(atHA!kNhSI_U82YA;H|kjkH+gZ}&7{diC&@hOo|eG=WQ+uXrN3|Bw#E9|pf z^Vu`W;^L7`LyFDkza<&*?6gOgpB5RY6eM}=9Mi(ew-CZ?Mzy@;)5iwNb(q8=y9iJg zxcdS2t#;b)iJJKR9nKaS9TV-Q^`=Gs7h|FxYN>Z(bi~&u2+{hc6J2tvTr}Uny(b&iisLe|Z(AtKEy) zVeB6-!stn7Lf>j4WGG@Di}{06E3x$R?6Ty+VfCdS?G#0J%(d<%(=@x%Y^*|ni!oKi zl%70b=U9OP$%$<&KC{39F?U%4hYI=Im`797qU~wwVQp%b-3Gz#K5q35ppV8Nhc@vwem zeAmv-(^{D}ge8%OcB1 zNef(S#%?jL3ror331FSCNr8an5!ViM)YdiCoLmhJwwz1v^ zC-VQC}}EXl!(VgCzNP5};WF$BQDF>Sq#xQQHy#kaVb#K5vUHteP? zAlif#<19GHSZ`t6Ly(3>fMR$rC5!ztZA;LFON$|fn*m`m@hsEHtfy&6Rw4fQlSb*# z>Bi4MMFlZXzJN`gOR?Zr!#qvDRpw>yq9e@ko+;o4Um&RI(?bn3M9*<}Wwt zUUV`9ptjeQ>0JyjzX$=Ug)R&0v8nMR&e6vPpqfM6Sah-)WF=U;j?@T$?x{j7`fm!G zn5#eZ5o@^;Pf(5gFfvHT~o{TRlTU(Rt)y7DirvBa5mTPSFr z1u@c4Os>HtXkuW?F@C+nlTd%*5l8)Y4&|K^iCEPWLfLIY%ehPEGNN*u&ShuioC?YI zyW9B-&vIwSOvlWj*~*)TA5wo%v=a8e2(|qds(vU%SL4z#eYBA=S6m%qg{8H|zfgg5 zhlAYqto(#I&JT{I@GtoM5Z6Rdfh|_eM7O2lM1+iGd(f1j+3koa`khrnU1XkS`s)u5{{rn}G zrIAZgk9{yYtMJRTF6i)SwkKmx)vvkFzL)b-_a;PsNx|t%%tW_G4)rAm?C|j^JQ=j| zOHFDOE6aW9Neq?XEJJBa<~}% zZ*6q+hmw1=%XMs$mZjmwhCSVu!|yXfov*&J6n}cpVe&R%`AYqw!2y%^elN5p9d93M ztQj%y3!)J|wu`@=c<1>pFjOM4cbs@}dF-7F-1b|kh%@y|Ra0RqMq&nt?JTd#+FV;0 zhHJTVP`K*6&Aio)^*kDWh1ajzZZ4I(bVwW!4uv>-YKEj@N0mooii^!pEjd~4*p1`z zo~G5HvfTuzg=>rYZyN&;exxFsn~|K{Gm0~1_jK=xRSwNV`m~}uOI--vlrxDp)*}=6 zVDXgoWvdboRMElHjrsYPUmfM}%(J&?#XI)sQ)MDz#bapt$`Yp=Z|U4qUX^%!LgL+9 z)KKMOl+$apdbQG;l!%j2SX;%}+J`<<167isTksSUit+Qu{Ly*z_K?)Yi0^ zu%^_sv$TXK5I*N4uogSK1-0mv5DDPUtSR0qa@{F=+3nOSZfJ-A(^>HJ-Y006UvFCX z#wA9V%-RVA#9n-vF?I#JQ5c+>`ik;seCVOLKx8=4D#^68a{kNM=c;WD0jmn!{=+TLPP}m+Awd7JnLABUHEy78W=uWLetuYmWHo?BUb?xg&Bc#Y0YFGC((2x$ z!Dn!)&rr94LvMwD7)v??u$p^GGW4FK$BnJrcaFCQ)g=Xa^<1%MZ^KkMh+qyi|$XN!k z5$Qd1YiF^|$#?iZ&~@(bYtU!_te0IYIvR=wzB$}_S-go8tUk8qY$h?kY9Sb!ymVd; zRp-VYhm;6q2yzW!J)C=@8N5Le6=1CGu9^m(Mi|{@^EaBU=g_L~_YL4mGo*p+EW=La zW=Bhmcs(Zg)UQHGHJkhcUdm1m5m%IVJQ0wg6(0snZ3lOJ->8JqdwCZXiYE5^KZ|pKIPsgTG&m*fVR%4`)peiAP zCSPJfw5Zct1tFeJ2eEhM{rl#kLo;99xwOR z=FymcnAu%ad&~joxH$VkfmsoX8K4R_gaA-(^tmA&7kNQIRH??Zx)v{vt%;#n^JstR&JT)QxfC*{;)SoUL?rG_$SCJmA%;nxYi!e8MqH`6^xgMA z#^PS=^XEJHWMe9hOvauqFOM3lH^h6xJ$Z}#ZpD?a&vQCAXmZ#hCgu#aS4Zmh*Q!ky zTjDO_^~D<{nq1_szO6K#lFFdKjjgKXe-w$_XlnzlgIB)tcNJ)~v@Tys6?D9)Kj^76 zr*ezH$;DoONZ-rS^W(=4GbP&XT5k%J=*W_79UWcXp#PNgc7K=FpU;Q5tTS@jP2IYS ziP?`dXBNl&ukE7?#`Ll-ce8x*n0e}^hhN(S?6XyZlG%L{@EH>wl9oOUzAHHn0Ywx)6g3> z?$jq12|-^2*a@b{}A~T@&f<4ANr|o#97e{(Y8~`~9c;Bu?KL_G+C(8-l z^_Qc+f>j0}fRtD+` zcp)g5SKWd?7yY!f<%PeKeF8&0102 zyD9-HMzkc>Jif?(dGmnYjImP{|4@e$tTTDAxY2tM{r`<~UiN#2%{#H1Ec`78k1XJZ zRH>jh!vTF@SITFK z%az@u)xL#ub7R(WGmIYIE_3-T770Z)rV(s2T>_frZ}?oxS)>l%f(G$ZCLTA?GFK?x zY|pQ@Z7Te;?sdNu<>jxRZ~Mv@JyG__3maw>{GzkRMZ zln^_kAp5pi%W1oBWTc=y7PN0Dvqn)#0t61yw469#pfJmsXLERLv?z2MZByhdL!OES|q z8X*%P`y77Cno0GrPD_b^S1SdYCQaH{ClFwd)^+x*?EWFX@v?-oablR^Z|;y$f87^# z?D&Dd9J069m7Lntb$B$d0Kyf*)sL*!!f}H`{M}wpe3^re;F`QkQ)s^tN)##U|E0ky zmd306*;>B*S`R*B=M_4^#`9G7v9_*aW86|(AZiK5AJ=vJW3n#G01CG-O@(NbV^!3nc8))flH!dFI z9#{cl%w+*`!8~v+rvq@9+(aY}IFGiw^?$=z-*MTZI6o4*)()LU31NRF_7nm38hcoa zLfg$t1XzaMeXfjz2K%s-IFJG(f(nW8OxAkYvC%8q#!r|GXQ zCp*M|KHaEL4zIgsnJ-TQ>JdD}`9w@O?4KXbd)a3A%j^JU3D9BSH>=?RJQ`Rp^o7!8 zBKh1>$5JQ#xVEld`cCtQ|3DzK;GO)?91iGmkEvc*DfkBiTd>}>z3J63nK4{#B{$8( z*4!t%)2TxNP5p2}rL3bH2nOF@+~}XrR&Z^6F8BYV1X&p5AG&hXS*$wtdN7y&RObjx znn@Z8F|>v5=QdPjL_fB_$^pG(;#cKbwGejb?~aPclPMm}Ol67uZTVuh{F))ch5C1I z{>hpoaBhn1tu^B;$<9W;zjxg=y>e|z9a5TrmzKeov_OAFRQj=$bsBsoEKm#dYkLyc zzT!kIk%LNDsxg~|-spiIIgSE%KCmZj{40o_uWA1Vi4@1ixySyk*F;6py^tk%){+_-8HZBs6z#Nqw1l0vr==TS|$G$ z`{O^pstzjMivVlDP-wMm;m)~=XS^oziqRm4l=c9S4tO{Eg97puci10QRxw2f6qtdG zFleRbP^~cpSAet%xS~5nmmPp)Z-BfJNJcsI<7TjcnW+gDn=&qj1Nc#g;WSG0*6({J zOu%gx@MU0kY_rxV2-*^)gHYH!8BFWO#<%1Q87B0471kc?mLB#eVCOAXdxNP2nt4=G zyV^04ktf5&D;uHY6tDQ}m5MY1x~Bl$W)pbp>sPW|54xw4wal7kw!{4)U!3gqj>6!% zo1A2fl?V7yDR7#-7Wg1u*tU00$Fy#ryHygSotEhEVDs#AWnJy<$0Ojc;3?kRxSmQ$ zZn29p9#rYXWnOi1(Xq2{KTiTX52c)wYlKPD(`Ixeep!jcEVVj1@vPfk1~Q26p6@1O zl$&FkF*<8g;Wa6InQbkqK7y6`+&WMAFJV(YU;-q?%_yga5})T4U8*YBDc-Duf1G#z z^hjQd-4~}P0o{v{w`0E_D{A;1D3?_At^YL{_9jo*w_PqmC6i_x7C>}EHNM#E-_W?L zkhfG@U<97xrGKpnmf?rg*BSHgxIt&AEJF`J(h)#9F}|}RCeszL zqSLJ8?V?j}bc)<;h@VVMJ(`tLuK8~9mx)J%Nr9oZsASdAOwXp@{df2{TO4Wyep^__SITAUh(4%u_9Dg41!!6*A(h*i{>njhL8srmcT?oP$_ zJvk}bN_uKnjeX9?J~YPj00n+a|I-`~YCr5BeXv8;95xpPxLtC|WuJskR(`(4SL&|X zVY9|8K=(=QD1FnqS87`G_{H?ZHKm0zeY+Ak8L%vTx-_H~o!$qU2B}0!ng(bt(%ZhX zfYzHCzfxubE$Erf>ZQ?zk>Vf8fnHr<=d~M+PgISAicfls?IrC-MfrCb*#%c;J9I}h z6CLt*$>_(EZOi)V%h#D2N7h&~Gc5p0opwb@l2uZ zW0`TT?-e_{%rBHBGMYdgH*fZk_PVakMAUXC+IUTHAE?UKh8fjpNvpjSs+J9W#zX0) zqJX!Vkw|qNDTxSB=2arF=L5i?lJ;Rc_8AzZ@gR6a1DY-uLRfMb+Z{X7my;6KQa6OS z=Fd~uI82TW4okY%M)X@|*Jhv(uQm6wRlgNn=RERjMtXRLc#K4f==%W^g(zdE;6P6m zVmtxYFG+o*jm_kw5?i8(%N7t2fEg%&)2krTH>tkOavjf5D`;fv6T)g^R5kNEhct(qNo@^q z_NAnNMG^e^tbGskupi`yf8mxR*>Xe?^M8R%Yok9Pj<`k!Qu4h5*&UR_uTzHVrmmZ260rRB0bIQg3dge~Ckw z-oFj}KR|wi+KnF6b&va$sXueE>UhL1a%TQvvDohhR&Q+;g{FQ62fg5(&}VFRzWn>0 z3Ta|hvD2m&!_Ssc@xM77T`yAEfmUp~;|CeSuQG_X4g{Qm%EvFznLyn-G+kZB%}fEZ z0?6eHBE+I0m@eN(m`UNsXE>=FEvkZ0>8-4GK- z4JR>0m`*LO7sV={^jt>3Rmp|bvAX}?F^Xr?W-f-bS-GWdrDC@KAE}tgM{^g-w4E=Z zsgZz-Nr9=el+3Y~wyNG$;Uy6T!vPx4D3+HCTd!$o8`t^5EVEjav4)I1P;RfbQO^MC z!q)wUceQo()72dSxrbB7wUVgQf~4($vz|dZ{Ygs-J;hqg7pTpjHscf{y{mT_S#;CE z0#|mPN{o9h7vHeShtjcjX4X^B`Plc00xKF`v6uT$s;8FKaHB5sxqzF(r+_&qdwx)SJD z0lR2|sU9hrfl4mJr7b}lJ?$|2VU+gw%D)`WxD+|l)~sLGGP^-(yTSk9fJE1ffAOo< zF=^86dS{qznY&%x%A6^}lE%ETEHeA#Q}MK*__xppm&CoW(y3%a2Mya3Wu{vEcf9jv zJ!4PVI;r|SNa7%oCrD=S5R$P+Q{KwKv>LMOjKL??XnUElpggdRX*6u?l5*)R4nA=M9_wr4}Dcj&-^vbULHjA^! z!{gdsQ=Pphns9*89{zhy8ukM4>5M9%@9f=ST^+0Ufs5q|fG)9$!@0rxGkgRuk8=vh z)gRkF$vf^`P!9++j~^2*jCiz4&-Wo>G)C92jJVzxyYK6Xoc4HsTiU>Cs?&v1vA%=j zS!4Oq>DBA^=aLVasvjP#`luhH;V&DDQ8;|eqGj|g6nWcXWi_p%B)jeGjM0~OS72z+ zW>x<*``9{v=6BUa&x7~@t3lfWCQ6|KKRD%1`(aPu6uJK^#9hy?G1J~&>**JZU0=C2 zLb821cRamzuu)k&lY4(f4<4#8Ez=p*AQ?_utHJ9|^D_OuMA)sRc{e;g5eZ9xT$Eux ztW{at@pi_n7BvMmDN&LEX#FAzxTXm&XoC)FF__m?VJ_uF%8Caw+;b4e47d$o$SI+m z3I4L-#H- zv!qBXyPrn5(pQ3M{YfYq2y`kCbEZ>ABcRPwp{WyWTXs3rgT;1Ifb8qrGk|XiXF;mK z$o2%Xs4OpdM^*C|CJ}s4&~z4dlR+SEm0n|50o?*jt{0%X`Mxl18K6fY(D>qMW>d*U zmMv#eG$R4b6mV98OpYMOy8%(;>cJ2erKlPo%vLjizTgJXpOh1H;2W$s7M1PQO}Hw+ z4Af;aKv8$jK{7ZOFco&?bAD)vN_5~9_NM#LUxP@XGI3E3q<^TO*)ghxP+fSW>ewid z1cCL9*J4NpsS8ZJ!(zlvx7qq%;5 z3v?&`H|TPMJW=?6@MsXC)mo?IB9_($ zd{n-(KhaxQgYN^!{P}a~8s|6DX+0&-^daRX-qoTUoA2B>{y_xVN-R@TM3Q)t0}{;gX)DF zn+}POls@kC!!2e8#$)`icgA@Tmc^{gqFu6O(r6Ab%@K)xlZhAaD!@0_iHMO$} zuV}gC^%8Ia&-Uy2wxNrB@&;2F2U~r`hC2p!{C~HPO4}bQCXS_jJzBkd# zIm#s;^fO$Zi?80zq-Wx2Q$fLNe2znG;)4Y)*WYCyBj6$y?{RH^Ic}LnEVH=z&#gMT zm^Jc_8Re2x8?}1g2h4X(U3nWloxikA(pf(_v@KG_#qVBlPnJpOoY~nIcWybiMD`!= z^P8#j;N~wD+W0)@gW<7d7u$q;_v_D>$AC|sT@az17Fk$cC-y)Y{Hs|Poq3ycX`SO|KI$~cyASs)8rwef zcen2tPU{LRo4aosm+I8+btO{8Fx03<{D@Xc(^2V>fsc)j=Jyt(ZE{lVDh}&hnGc1? zAf9kUsbuxJv?oa*OB&Nz(^I%_b=!7dD-7-;Tv8gyC?B(O&&x?Oo)*^XZ13w%bd`Os z$gdxj)>FDQrp6;Fspdj3>Q5fubUshAklEz$Crzi0GG|4Zgz=+ysRY7k-7Y~RE*?GW z*_X*S-BWB~l!B2qAPJ1aliX}WtrY%9Q4Kq%&kvXx4(`^0S|4!3Mz}t$7l!#>8_`4E zIi~enLZSP_!u|FWE>rug*eZq4sC=81*Creh;7tTX0qAx{F+h!8%6Wjmt{#CzEW93| zfCDPMj-LWoVL>OJgBBeeeR6`7oN*)N;G9$1^rOEVv3BgQXj(s&#x~+5t%xNpAwaln z_Lrm0gbH7!f~){Je=VRDMPDmt-$R4dABUsQIk3d5nVcZOE8kdeFIymIKZz?G51&nkcEWnOw6kNQpr7^(% zaG+D52B^GCF{VojG)XK^+;U@e_{C|i@q{!hkT|JEEb z{Qp*dqVrZ_k2@*0R<*jf0}`zuDO=Ojwbfa2BWg0Us>3WL66O$+MtfBD%%Xu!m@zwD z`bH`B>0RPz%>WM?e0!t_-05(S*NHred`IZ4{HZptPV*1mMR6a+_&%7t3lIavJuO2D z|C|CeFZ!3`iFr~)CQ#V+Q(k8JH?tSdepW_Y2HHH9579lbK@r(}AF6V;_3B-fd;R=M z9{BdtywqsGt>7naguD!6S3^3#?@!P3Qo(~!!z}+mdc%S57_$XokyAAh4n7q{S{gU; z%dVDsizLJD>&>ahY&be&naPDl`T@Tf){>QJ-(T4ooMK>E06#neSp6lxp z#S2Z>FbXP*KCHF-Ug(wUxJ}K!(VSJMH*%TK4oSZ@ReD$h1U$N~%d0i3sq&zfmQ*IG zMUD$9Dz(Eo%u;^yv)ih>9|gG)g#$>jnuDt3E0`dx!rpTy(=4ZR^HppS33AqlL5b^h z%vpZ3)E$10CR~k({FnHx_nMF__wB!Cnme(!>sW)KU1H6CAD4w4A|{w_(T>yM#B1l^ z*-xqs+k2u%tl3^g`K{$*Z@fG@85zZ;*bO?8YDF`M^x~%aKn2NuW9hHy^_CAAx3Da2s&?P2BGz!af4r zaD|@}D)X8*b71kw$;c6Wyi6D4I+qE%Twy4HP#MR;0`mW6@ft~?yI3447b73i1d(FU>{m9ikyLC5>oUI$vU5!hntXB?l+E>#RMcvb zQfm@h%>%S{$2btK>CP*~W@Z#}Plr15am+8srO}+pfYIaSt%P8f)1}t(Zxp{qv+JN? zj9&!>zBx_9nv|A%?z|G1ZtxuZD}LR}yPS(91iCP-HGC|6=)r83jKw&?uz99+6;6bI zBaT_rz!)gtoL6j^_q@`|*81efga6>8;_`D-=Z*mkR0-PX^5i8_-$5p6eqV4`py`y# znWGFijEKG$i{A5p#k(xv>2HaodmfPTTjT1-Z7HIo3GqIz)TB-F75Pda1-?k^n=0t? z(uEN>#XE8xx9W31y`zV$s6Wjf7T<-4*004dUTVtyPF44ktci*yhLwAlOZclfC}_1_kL&W=%m68ji86aX=6yn zd^xV{G`Q3qnkY>h1a5if75EBqC2ObJ9N~(Pl;LR2)%d~NDFVmthBgWo+6PU+6@|?z z9)3_iUUh6xHWYDBUxIOQ224|$1p&@ghWG?%%i3%m0_{a;<|@=?1UX4DS6thSwxQNd z@ze}v=Y}z^7z3{pm?n;yN6*#pkn>_&R%2_r&X9sKnU?*<#w>Levj@ZgT&%Kn@iHLck<)hyb%J?CCw%j+ZOh`##!R1;Ucw0gwrnkBKmD>Z+KOgH~QmfId>Nuc#nnthK*e_wjY16UQqQ(lNJpdh2<7_JZlZ^ zGgsWbFE!*Wf0yf;cl1}0v?61{?&ReDFdP0Jpa{3<%q65=bL;6b5#bHYlH@7T$V-b; z5!PT&Us2ziORiUDuJZdgySFb+>48>LZ=WDzWmyL~TLMR-xVu zH89yApK#ZCf1fd_-4iQv4&&?SIpDfVdzimpnET57Zyc;*hr#GkZe zYdb+~+;lCT%Z#u8vGLm+)X7@GBF=^f!RCAfwR;zXjNX|XOaH|GG5>_-7-6(@!YEgn zR+x)QWPgt)^D8eBer1eavWVb2k;eXLQSPVsbWMkalU12PIORR-lsnhRXP2H=BsbxF zF1mf+b4_WI^+rD?0Bn;lh>d~va3b!{s2lch8AhK2X@m4Wry)`RGvAnsVtE#WCFgiaDZmwq z;RSzJ4d^h@_+d*h`ji9khRpAS3ivX8K_c805Wd1ULADV!z+l(#V6Zc^kMRD$5Pl(; zL;tkiP~b|Z?ThNMEQL)n?R3?Rgd)n@gzKGcSLT{9m5v@0OTgy>9Rh_%8ocbFf4Tn- zXm4zz^1D zWy7iWwjOF1*FW4};Bt1f4KB9@Q8^Tk&RHP;y?PSR@SQDewB2w|R7>~X@#{KcT>VKw zhWpTtZxfM;?f>wBBMT}B#X~yzLAxYURom?$Cu_)CstMeAvlaG>ck^?s%B5``n6Jun zZ%)6w2Sh%=$irdC8@_oR2Z9CisX*I8L~9)sK|4*sEJ1dEC3^uG3DT_aG8cG_z$Dc8 z`~qYTNXMYDVt`UsRsQbd9pdGu<5j3dFov*U>oH+sd&ZVTE{v(}L&XLOY9=5uvUYv8 z)fIo2jM2utD?vros(cEWprqe%Z(jQoG-o$FSOymZG9a_(7xy&y;^4Nw-C)gjHB$qzXA2*qnNhS7vdjL!2-(LV0e@$D&&osmy6 z2NAPQ9s76q4;zV1UQuo&Ai?J%Uxx?eIJi$!wj?A#W^Zt_g7k~3THZE~HwShCwyN5z4I*@{}Pp*yV5nJu!wZ&OR2H9%q zBt7Iy+Gn_WI6B%r;AZf2rf4Y79ZpcYbOE{LqlKNb>fD)=SJQ`r<7e&-ke<&pm6 zoQlN{6K=Cg{%PzouOgjLckd;u2x-APlxtOZ4$74Mxq?$E&z)*S>E|qbupGM&RXGL!>Rl4A~5`F#<@s z@E#G_*AtSWyc>%Rnpuln5e8Tm+oNW6(VP{y4NL+(Ju%74g?4!$ zW41FjX29WLLSQhBaNfo&jtZWJC!}=nLUTVvKyNm+wVUe}Kv)EocMap1WtW?km9wo| zSNr;In~nrFFeM%7yT?I8|E@ z7?7b0S;OAwgZJrf82RtXo8bdfJvsa6_m$%J7$$mM>ft)fMuHqL34qM(GgWFPz04z% z4KMol%r$TZKw_Tqm&c$HplvoIwC%30=J}fqT*q~_b+4bmjipf(;FNB7j3j<@V3}O! zJ}4h76U40V?yS9wd)|k z&kZ;U6W@E85-sD#e8D2`pExxqUxC!;&-br)m&cW(pN3m5vO*sdUIiEk$ZKD6s#nUCLCAg-sdI%Htb!b)d4l?cJcLje@^oYWZIeUDu(ct z2{u0wf9OYz!5y9)PmfjXWp!uASkkM>)FKi)Kmx&6N-uT0JaqVIPOoe z_6J$AG(sGq(DGKvw^vUGw?)os7*3D7Hm_aQkn0Hzl+aC6x}jmLNGZ@u?63=MZ%X3a zhPI7KsS!0Bw`#F4KiYX^fxGUZfKP^0Ox(#gO+RP3MF+@0Us?mj`P6Ve;QWOB!PO@P z&1Z4A4W2ctg4pV49}Q$BJdaLrja)RyZo*KZAd8V~Z}0h$fRJo4EkvF&XFu*g*gK?9ONW4WmT5s{ zSBKD!*v+Tih9a|U!`KTD{ZgMB1;Mz>axjm9Z8|)e`;fFCxE#DEFR8VVmRr9laC+Kwwl?DFU5f4)IwYZKtZs5tF zd?B8-%Kvc!0W@{I63>?!aII-v4(0H6c-ialfioTs{3aHe?S&iXt%LO!q#`!hzj3R8f;E8Gbil5J?gC% zctJ?NaS3cmpWB;hl@@ktJ6kT%$n1Wj{{EFK@zniC??j}p%U8axPD(PVJT^NL??b1D zPVIH&a=wcS#LqglCJQO^8p-n`&i?G#Stw~_R&ZzF%z-k@1Xhy z?i;nX&46QJUOXMI@5risw}28wY1*+rr>KPTQi|leB^kqDWj_nD4GYrj)-!)!W%zC1 zkv6ZG&b}ZkGy46~8#KMwE-HVn(AR7PVvfycuI93vxov{wt3gJo0E&!$;n=9c(WKb) zyH|T>+2EjzM}(tPakeFP@~P3+7P_1V=C@~Ig7hE^3*%51+X*QiqA*}1_$09l>-c9*#j)tV4I-Ptse*+8erJo=NCSOYs1NR(3^S;3@0!bVs%?D z4E(8Dpp|v8I^{6yK7r*HOfwg{eFP4+T^Od~2E2CDh8-}KA}wt}iHG?$$4vk!uu!me zT-<=f2MQx?0)GQm`o~w2V3%0Vd3J&+MxY-j9us{FUJFs-KzZZbJaJRG_zB_SRdZ0h zC<`>a)!^06EDHHWv*v30CP&+WAl1`E%)<^;3<=VcRX=QrNq8c+HJST^YQ;R zp4nqwgd^bD8-?_4;LRZg7~j4`Aoq|J!{~J_c$pWw>oJJMIaVX)yHRb_?oXez&I(&- z%Wn=-_bvsAJRpwP$?IS7QFn2I?OP@_)Cn?B1K4o=Kb~bFY!++<4LKq~U{@=`{*|_p zw=HA{#Hy_VAPO{M6(ecC!5BU0)g#E!5x&+a#aM?L;}{4d(8fsx#=tg+@5t(t`ml$; zgQ>`hF{?F_fN|#8pUE!EA@H8ueOe~>IQDdMO#vrm;3c+5Gdn&%h@8IPzZ29uwkVV^ z=J{tsL>dKRP{phm4+=m$3Xg#~_9K`TaP)@Jwu%I>7B&e`funKW>ChB#ftN$_zI20T zozCxpK=SWyn#sR}CLFzQ*F)wl1hWRt`qQ_V2rw+}Aqb=OWZU{s%vv04-Jym8qtT<; zYyts0KClza{d{9tDF0t}b42eKXvaq|K;YreI|%QdeBk%_~0ypPqDmd;syR zueh#{;Gr4aROyZHsOM0Lu~7jr+Ws{5!C7dgaC26H5tuQr3?yX1o)8ArkEv@(z^#%p zf{3o#lb{CC*_TEg>I)k9&sA&4KWOM19ISg~F(!d2fnvxLn44qa18^HX0buS;pgs8P zNleGr-q6n@xV0W6V-EV3WwWg<)fT7^njc&Ti1WW#P%3x@CKC>ZDrq%o72p18^|PSx zG2VAa#U*7W)7av&2N%9axd*xc0UO^kOo-SI)%L#N3z{ZX05hy?nZinh&tw{Km$MSTGd@G$A_v@qPwWMd)7-oYz5w21Dv)1D4&E3H zzfN7g%+JA%dI$7t5FN?S@?N)PGInlfX~|DRf5KHiDWqKXc45O1+3toGzPGSK#UlIP zp|=}Oc1ej4vPvIy7*8V+v3|a3zqR4|MsFan7RcjO8fiYSjJ4z4* zhWI=|5ZC5@Zd;IXY=f>?;b$@pUqRtdS+v%Zuf7^>)GyQMSYuDEXF1CB?~&KOQMJdS z^DK^^FN)*JqP0ysczH2?J}RYUk$yM2olq(UnM;#_@|Qmzv3i1W?;AIz6F#;IBn*_~ zE$N87tBQP^Q$B#bsnvwh-*YFXw94{#-bDwS-APqwQ+~%Z`It8C5=6Usc7By84egTu z3;phM{V}@n!HW(PsXO#P`PgmjFLOvZZT;Bs$)iqzj{+RjDvvzV!X%*7R zb-CNwE_WxQ%4>UIS=%Vlx5X~Roohh#J5(Pa5n9pnG1XqnLFK{ZK$2BIR86KN0;0YPdaN{BE*L0Ty(L8JtPp@=9o z0cl}$hjcR$1?kuTX{AQT*oejRJNJCw_nh~{=llcPp17a5?<>C7b$u^3%dLWO>1nbT zm}v1-r|_nJ@n^GB#gYnvb6~-pGn?MAflq;UiegsGk4AGMho^Ue6pK*k!upG2Hh`tc zL2zBIMFQarL^PA-YTkc7KKAX4&mn`9t!XDn+Q-*5y9NLmvld#kSms?ca50$R<>}Bi zg#L4%*<>^a6o3ESuAlqs^0S}MhC*(0zoa<<(em$|jMge22_~MY4CEs6!8;@F_&TpV zaj)sIw{&q!WDAF63|5njXx9KX9z=DDXzUj+9x}Ets4MHRT&#LcA90#K;w@j~s5F?Z zihjs0mGN9(M7aZ!*bYI=W@Pe60@79$q`APqp;q%S&Gbf*de4<)SCfpv90evZ0w?lN^HPE14#6!9uf#B#AfCkV-BadECBq;@V4qp?>Z-HqOJ9R zi_m`azwY-e<%BxJDD)nc)_soEgiD4(pu171_o057Q`a3zrXDT4mI`-?abg}UN0yVA z_bEZpKMWw$Td1HHu9dIxcGj!qABMz;fkD;6RW1 z##5T?uW6x?$Tx`6nA_iFwFAd*lXvZ`5j!I^_EU z3i@a^| zP~E4_#m-LU6&F0AM2I2>PGVTnu2D%aF`Vilybc;n68L;I7Z~vR5NqFvW#vd2k6 z{gCI>@`+@hYsZ(&Y_kwoIx77K*-?)fi+!Kqm5JAVQ&lz1xk>S#UVW#80p&$!2c_z>oy>&(Z#ONBd z;sTp=j@t7z)Rs@^QwuS8qr)i)L)HhptL6}1VE+nVld$YILJj3D#GU)(W$C=N?~y*b z&QTn>psh%XXdl3QHxhSqbw4AJX~pQt&YFLr7+?xTx1be55?204LHg6H3Coh`d9&V6 z7GvC^Me&I5?OWMM!BKDJpY8N`W~s>)0i6=z zctd47`8U1?17h7CN7ercs5(r}F8+dg6?FDwPRr-W#F?}h!{8Sz+%z)NTI5-=nV5j2 zi}IxzH-;zG&C003_X&3y_7nMFameE5XP1r6+8pGAtwsVwmA|a(ETT?mez!iX}@uMWW%3RxD6SX7V8{bAfq6l!pacjbnhi2! z`@9$F5a_3SJ9h;JdK1P5`vLwu0}_fJV#HV$A1)m@0a!?NC2=g;xw;3;4O0#sT1aI( z%=`+F7amDLCw}01)f56W*ox7U*o%jQ`#O_TZGIrMM40%7~J`bBN{X)uV=e+ml zflU(e5wEGo*EZybCVQ4E*Rt!MS>`L}4%fvt);89&j9GV~5^bKul6|+wS@mapJlP|% z%f6vVMP}D~YeR3--qfyk`UXvK9QI+AuaPL~shdJS2kuYv zP;ine_kC%*1bY@Wi9G<30AtqyxLv4GK=+HpYck6202x(E%DCz(2%t=0t-?S%4CGgN z4w1^l0$~#ql(g>QPh(&nDoNc3hiL!{!xeAgC;L}EmEOX_X9wYcb8IA~8|f!OS<_q> zAV)2v=-Pc236G1B?#4QtOqRy>(?k-pgo5s?m%&JUP%R47)rNbpf}k`rtqev!^>R~@ z3E?IlWfh8&w#&2mRKMvR;C28;JKUQ@E#k_@zK*tz61+d%P)h?IY zSKs2mHM6BBa z-!*9e;en@i`LsRcMwA(GWTZ@iPa`eAIK+;;3dX zGucIo#hix6zNnc`l1NOfTh}ZO{m}%lo4x)vaj?Wv6rft<>QDpSBH#peguFKmynxX{ zmq<|S-Tg^ZOK))KJY0G?H1x2hNi}?n#7vnUEEm&nhm_NSQYJqX|Gp#NWc07KZ{!Yq zEufRD-_(jmrYXl|X6IbLchAl~?e=hQDsu2&l$~X{Tb^t<01JTP?j;!n3HT=FVxc`= z9HrzeaF4M+aG&HuZLc~6l0_8#7KEWzyVjO#bw|g6ke>!K3 zmAMhs+~r9yljuA?+uiO8$O@iW7=LJ+|I@y*ifpU!8{>O?980)4!xVk2g$q_J;9V1E%&Q;^|4bhchep$NS`B9pY}vC$+%sXYGqXelse04ZmQVSSS|q&!X=FiUa&#Hwa7 z9bsN@eJ3aI|40AbPnS`Bi2tNF%MgW^#qhv7G8Uo$!y=u-mN$0P9=Sy#s+HVmFlUC8 zO`*CZjm(T@IEE_5&(Yrm$G}yLjdL8kSTBodv;e8|+FOC)sOBu|Vs+fNQJ?v~Ddp5} zi2{XAp4Im~t#^~j%#I6JOKh9sSG}4LwWwpQ78qKVNwaTSy;ev1e4>cBag#I{vn(iJshq@NFH zrwacj8AVcBVzI%__%-BfB$)-QR4biU0JHd&yjGcGC&LQO|#z8Hdn#$4pWHFeR}k7~zuE{z=UvwN2Of)Ik)^)J*Jn^|7g zbd?J|a*=78fQ6OvV1hlix>zSvUY{<4(;oLrb>vr7_}LBj)cJ%Hl~dckNNF4$ch6IS z_8_&)5IU1NW;=Jx9mz>Co*D;AP+BKzh(5W=eesncEXZGuM8yJ(!Q(uS$`pc zCp!n(4)T5A+oiJcG-otvpm3madXE^kJl)$kXh0c-1A(czes6q1(7|0#-#|DuS#Kdm z`hc>-+s%Mj9*3+=`$JIT)&mrRg~yhIo>~vhE(E^pzYWoYQ2<_QE?SM#}0&?xd7zp&pB_I_E#dB zl&TCOaUk*rvcF2@f0mSdgHG6|*oa5}4nMgSS{B`0_`Ia^;A5!+?b$Ep!!8Kr^FyAT z^%WJTgA*W+o*`kIp$I>yaI0ASTe-wBNk}^3>@>0fkXeL4J*KySIr1IIk%wS1k6@r4 z6Q5j=0(|d^{*Zs`aKyJf8{#VXl;hydY{F1x68Bg3{lhR=Cd+IxT=rxq&j*4pX}f6s zS^)tC*iOIEL2WO3QHDB@N+9}xzQD?NMIXnRFU#~8k>uid5a+^E`L!Zx3_v}z%M$s} z!(Nx-o|~A#89-$G;DaCc)dHjknlxA&KJv5>Hjzv7DB)CcQqDeS7aAm;9IX3c5xhkE zgJyt%c6SyyVn1B%>c;yT(fHadDN|h_4Ws@_!(+u)h7EyWS{uVE3aYwVIk87(Y#|TGaRg?-N$p@s}=xyW= z*`u9)$q86UQ7bs~$^f{ozNy9QvCdt#BIMORw-Lf3PV3xKq(l)S(e=_%jrQA^d?+L3 zNUO2zFNwB2)A!j*46C`^P2^D2?%2uN1yd0OUrF~aw_m_1Ewbg@51I@)!{Ij@uP67- z>P}_CZr}5-Tj%%rAu!Mbt^;q&1iQF%kFPAm=7ydm+b`opDi9Cm% z@`v;B>EO6s8b>af2qU7Ag4#gh_Uvet#@X&aL~YHM&9W1fR(#j{b*{^~^#d8Zz|A{i zbtO&J`^+=+O(z zAQ*g;#Vo?Tylsfx@8RE^?%>*p{|7(wf2vc8@BiTbzneWa`yv5)4KNBaOO8IfdGFif zsR+<&!6~Hl?&jV;Fh4wzDaSHzuTOa=hPHJpIR=*3!wDUQ0}*p>)mRK)UVDf!8*P_m}f31 zC7s*T5MQHT2eEFDvOXLMO&-VAdo4(>R483zR=~tctUw1~Bc;iQA7UR&nZMfA_x}Ml zgXVZL3WD7Y5cgzs#7n6+%#AVzjmO z+V3+8S6_m9?g2BA#>lBQ0m-IEt?>Xm(xQUzl$=2eZ64GkyDTw4kWe*D<-BVv#CABe64wPiO5aPYvB# zHXsYl*eQXK; z8>fi3r$+mDB5w1N_Ich*Yhdt|=#TmEvL^s48H8?wX;|rkX@>J^%Lk7D6i7uy4Zp(= z+a9+3X@Dtv2Jqx8O!i~{<`u{n{nH>|(i4azQ(}%7x6qDw&F(xrR?dDHGxZ6Q>84e$ zKz=e(^rrTsO9vJ=$Fo?992y+)>?=LC+=Bhl_o_Adw;vRfgon=vdS9*UV;5gCt81#| zkCF~NtHHjgB=UM3X=4mH)iE2H2NNX);PAAw}+E8m=P2KG?fsv4VyK)+H|DRE3KW&(Hm8-<_q&}FJL>fzf{TPK2j`r zST@l5{>9(l6T!1t8I8zJou%aY0wAE2ulB7@b-_i{>;Kl(0}in*{Tk_8K>&e1JOFArwomGXKK4cZ*CqOzniL)^ z`+jUG{ORUGV@Z*!$h*5APsZMC*|>!=h!)Pe=xGQd$fBjc*5LSXOZ--eJb`EySdgvu zLQ@ex_yu`xKk2Bi_kXr#t{S326l;5LBr!l}TY3U*u%C4J|Ft=RzPXW%?(S#I^$*2D zFMojEHc{f@12z15Z{%5U0af!8xq%weRmQ9{LP3$)%+{ z1$TT7v>7uip8t(@OtWRc3zhiG6)F9lE|-+fQRMremqMT=8ESO}zW*By9xKOo)8DOE z|3%rs6GBU-pM?#aMH}j&KQA#by-MK|bl10g0Qh{((}`)PiA&W;(7KF1DdpEHbu}5| zlHEsOdL}%;yY|OOnWbiQ_wi`L&B&o~M^^{C$nR(f{ z2;DQ!-Pk}oOz)nLPw@fGId5;%Se6@{f-#}XgKmD#^E=^+>gg80tv@40`$Cr=CEVC? zwNvCE9<$TLzJTSeD@90tmj05Hl|gQ4EPM|UHaV4AU_XgZv=Tgjg~}0JdDpN}lkUAr z>W-!jH=S)|#xLxVRNJG<(rk*reRl9lX9f6l9#66-o zi}T7+m8)Y!;NJ0bsf$1oh=i?%evml@yn#0o*bNl*Ktl5EPf>|Q*@~}wyFr|&9k{xK zJtrR3V!kN+aZBXmW+FcZakz)lzjFK~p&!1AUEVV{nR(TLG4>Sam37(tv~~wL8h#y> z?1Fr69Zu$XM|U+&!?5Pm0V{jMeM6|FS_kd(@>J@dko})|_gYK7>l~JKqs=d2v5K*@ zyZm3OGEb=@cxS+1Fb~vi#{1sur<5NV=ss-z;(FDC(lt;q!9s0v08gT2eAAzrcr_)j z8pK8{Y;6sKy&$-qqWhxXh=H3eYs-58vB=BZUYC!ZnYwe6TMBibpuQ~Xub}UV zEVp=7F2S@$wdh7H`e9FDiX#lN#?dp#-Xhf}tPK}#@AGjDzujuc|Fy<>`Yn=g?a_r7 zI(9a(34ahr6V(L@97fLRa0pD?n7#Sq#q_8wX1j}2>eVCuDvpm`Ki2rJ@F;gws3ZOj zSkxB4t(O@7;`&UmdUPRYaBL5$vzpjFnm{9f_bsB9NaxG-_IAbzA- z?ecrq#Jz`Hq>XlIi>a^&2Lc>8->P)*58n>Po|({j)s^piy1{+ay>GZ#gM_)Z?~&If zyHP4g)ezbpyjA5^Ry$78a0WsfM&T82a4Bd1= z&k*@}B&kb*H;N#S_O41)+7bqqqvU|Jcmqm#elrz}B_|i6qlcvkya0wMn?1jQeiVbA z7`^RFoQ`a**?VBqTq^jTHuPEXtg`tph;%hJ z%#OD63m_G~Ex>Z-$uywRxrunO_5{~@7dVr*yP(_FqaYPQrKn4S6b-?9E3Y8uE+Z#m z=s%51G)e4lXqCY>Psb%EJrlSdf-O)S?B^xEOHLBL_~}#S*x{=V#7+~2xadsKsVd)5 z-t!PlpW#v9QO!#S)*YD^J!%`rEBo$rp<8oTw2w8=P~M>G7$AAM_WZ0uW82co@J9~JubKc5c~S>0X^O^^j~Abj zes}%&7z6)j=ys_?O-$JP*{kd2+*y+1<9U7e%8IoUtx=S~I`U5X;?ZYSZ$~uDZ!mDW zc58I({2q?q5g*wF(xe#x-seoN$Jpjvexq8BvaVN!Sle&a$TFA9Mny{Pj_2aO5n=W6fK|K8wXq^!&{Ll_`3b-FMauor?`*$OgwKa zfB){p1yT9g$lN7!i5>Cw-9OZEs_3Z;irs^<)bjgyT5f^t*xUHix62-l1Rmh?u4}=M zMYc>}XfYB9<#C7t|6}kEgOwRr)*49q*e!A2>@OLQ+cR?6tg|mAO%1GPT~A;S6VX@G z@roj?-B;rt`WzRmTFyP%Hi1XwvBsDm&6h}gachYqn2IpL`|7g~j4cgLsxOFLj`)6R z6c3|UMOpO$bB~XP{8qnsd(1>|{jl4*YyR8_oL0xebytWl(unGq`bM%hB|ON}S+ebf zyWxm>RN!U%_lj?>tB{Q@1USdwq7k#^t0vziFJB7YEMR+snbkOkDFg2`HJ=RRid7dF zzvpoc=(&Ocw#?=Pd z&%FNiG>Pcs7o=#V9el{+)Cf;ihtpYVSvzfEBH&F(8sEEZR(r*j7NL3=cZPj-R) znbMuY2N24YRIG^b0b=FYy~E1U$@kN&dtTIsOaZ8q6+0qgE9Gmlf#98oQXx53N&6tjA;4sVXcs#*F^5J0?VRf$_-%3waS|b&QpzQ$rK}2 zA6xOy80YdoA#^-7xV_(#C!_pN%J1ht12W5a-Zk;C56*i_P>=hSf_P!fr5XN6B@;<+ zyD4oP(nqEBppsi(|L&r_VVG%ntK6BXyK9wYKyQMKD>#S$+NbJrT6x#JTUVRB+y)^s z6Zhb1loJar2e&Wl_M-DNCp-k~d7OGmo#Z-njwVEw{L(8x*R@=3Ts>5aM7xbezulZ4 zVoN&C8;pt0RK8TNr)7Pc-WNn(@^xQLBHP_XYm$+LFVQW_% zqPhg57lv1T;!3~WWFJMB(uc~kjaQ554cQ%zCUoH#{354S_c$y^(vXH53my*D4jhL2zl9`6!1KCvqofomYX?GR7oKc zw{L|qQ1P*QMK^MSwhwKfEZS?cX~-5;C}v}>eTf{_23(fLee;*Y3Hz;Q?Vy&nBAN8u<8A&yvApZClO zHKFcuLt7H@bH3O1?vx5{!L?lXxP6;ImnX6*P$x@|F;l$T0KqcOYGzw}?)zT}t3xlZ z#{XBsD#E^NrENO*^#_)n$FoKy! z#qpVG&&Y`zMmz|yUI~D!$t|ZjTs?rfajf>dh-?^dptGz0~PO&kh z5M~XCc&4V-yWLXZA3~5%^roipl4>s@LD)|taTU{ZK_CMtU*xrf$@_sM2kF4{C*Xx` ztZ`w;r|dwSvI}l*K_gsu)uygqjhlZ)Eg66~n zw2SE4eC+&>@6lJT7zH*)-l94te8Jrs^!~{qt)N3 zqa5_wEG5gz_;)z*sct|}5-HQM*jc=f;SSr@51+!Nnvd7U!uJ+CsN^{gKP~d&RkY8v zBir*^4+T&?6PmvyzgpmBSYv-QW*GBT6B^69p7``V37tKs(^MOCwWw~Hzy&4tc(U17 zUpOkgLBKaic}hDgx2G!XhxgHT#K}YB8jUOOV}{xSf?|yQ#VQ7@yUVp`-@nqAB^lSB zxY%2G$@cLvCDeT*mVzIL&zy?7T=9kBb&vO9Nion5_r zcD*=09MiI=m`2=W3g1K9OPdOYtksDgHs(JE@*Y3JGamZVvSAUSXyN=qP? z6@B}l2#+vhR9}s<_JpO6O3TlUcI#iz(O7ey*e z1^DwP4`gdD&by$f%t>#*3?{fP(U=(&HDO(*#Oy+X4PhQgEWrmpoZRb0=b`L#;5nN3 z#px7&SHZ}Iu{oc(axO6*ucs^HxPZ%OuKYsY$_5_uLI2@4S(vjoiJOeO)@w~a@YB}$ zTmUuDm>n%g@{ktjO-6aG%np1DWkvI*N3R;Q+xQ|<-Z)*PMi-lW38cq_dngw?2mlw# z(Q!3GDC)W5xF&Pwq!*q(&;w1br2A-xYT8=m=Xy_0gj;HATRdqIzB1BtPLsx%85@*R=Ff2X)KVTG;pBIVD4r4Mb25Tl(q86VZnS zcuwPSFRwODzF*rGdNZ#3z%qY51X~-&&>OwTv1MtD zNRKi3%zw9bdHBD#cJcT5D&c0KI*Iv9EKusrI@`V>ONZ>zImlc`fzkvu8mYOrP1dAK zE?6&x{`uj}Va#Jzw}f7@s1|`1EJqA3zS6=uW;&j>MbA~g-Na1uE%l13q&y@by&YKA z+NfUKas~6r21pUO_cOL>SIZ~ZVlBR8{DrvbhBuNQxV(6Ra3eob8&+tqKJM7z<_MQ* zla*!)r}rS1!8kS9@GEO@?~}+CE-zSjj9$Zj)}zr^d6*Sgzc+tsPRf;3UDFf1K=sg+ z9^t9L<;#GY;J+i|?yq52)g*W4h$Jh-9`1T)wZ@gzwtdSD*%yAP$e%e~mmRjkOkJlC zLennusNtGQ3X6Sq!a4|``XbBW?M=K$t)CT7tl~6jOQstbBF}VhI?jdT8q3Q`9K|rn zOssRDRN<&56x+2Nf=4ZBev0%>(H&e!2s?<@waYph2m|RG>4L9W=}KuIEUc32hMw{2 zRMB-bRKEx^IT-;vR^NIJ?g7ZXUhV)esh_l6M&{*&j_rG=j3$I9gbzAO>AIYEIsf*) zHhVDg${_^l?^<`&w~?M0KCO_Y zG7R6X6XvozJ$P#vZBw2pbyg=*PE-E}?&-=}=v-qR^Co=G{!lJ7)&vod(+6WHb`;20 zh_+ePdkiF+rn>GMygdiow*z^+4E7U_Y(^sG>iTuP2M=H;3@$aN;;wyovwrDw-p%&m zd|E1Ue-7-Yu!;qqj}SHQkTt~uoUD{avE0mpMgTaR3S5c+kh(3lS_9h#H!GUy_RJ&i zUfVud@3}mD&1qwQ8NLw=Jf_y`b^denH7N|VeENr>jP8dC(%@|g7ci9@ zNIZb_o7MLoYP#gsGHCHWzIo_%-ya{{gn$d!>gQ6xLji{UiYBv zZ|oUjMz881J&+*g`dwjd1ndGTIuO;9^xW_z<#u;~RH7E(ZKyKv3;o<_Hg*IMCxC-K z1-2v99b|9Js<$QJg9gV>)Atbsm0}}xwa-=W&9Pu*v}F8NLLP7U>I#c@rV>&?!yMwa(jR`+RGQqEnngAb@iCYHApP%R%JY6qwjk z{><|#s@qygD4~AnLE;WPmO(pd5{n|=rU9NkSGdU-Hkl>>2W_(u)&ngiqEJg~SfN3= z&D?9MWe6f~!;o~`f|oPBg&A@;@!EBGQ3n%4HDYjn57rq)@0j5HVs>TSJS|cbc4p!4 zbJay19i2Chc76I|nKPrD0+Oq={U84b?kO9+&b39g#8Mp-C7r85Vp_$>-%5`!A1Z^8YB|Gi48z z58d>xeY`F?Fil4lHxA{MrdrZ$Kwn%?)Cc&P4KQL=|1jJt_lpC`SZ|sXH22roQCnrO zSRYm|FO!V1Z+ya25yvY z-INo;fnI5QYb?}dds1$^g7qPuoZ9ucgLh=N$CA*YLa_vl&1`7wD8K4U2Ya_E?BIhdt(0@!XJ3Ko1b5hBX zD|-Ej{O8aerDF~sQTz7OxqCSD=nidy9(CFd;-j0e@-B!Mh*`ZgzJ(Y5)|A{V|93b~ zSE_G3h{DWc$fP+e{>dEV`MM6%IGtFaug!fXAK($ftHr!mjo72^eXg+>RQka-OOIvex~J`y6+M#t);zl! zLKSV_sVaRoP4DW-w7G&HwJtj476~XQhoB9!(uRu5mXKpRQvl_AHHdUp0lIRx$LDKy z-cD&b71=$vvrV8xE|FFq{XDgB62SY~hW{|Ueb+bqx7B8_k2ZeFq@yHQEEAh6Bs`f4 zG}7@SXqq&T{ih_T{Dx@a2w)qT!96rvRO%J?J?{NX^Yn!0Y_Tykir8P6PVr%V4N|*^ zteB*44>|nd=Q;}w-BM!uH;~mvkdG{0TfUR7RH!R&Gi7*&b}UD_7j{Sr(6OcI?~-jl z?n~^kJjDFr3xi&E?1yeEph>w)55%(XE%7wnm@s%+1PYqe*{GX<(g=Lggi*JUpgJi= zPiFz&{L8V-l1;ZLJW~O2(^3y&#krIw6}y-W+r;|j&-q5qZm|-t=x61Xd@hXW+5_m7 zr3%N9{NS?tJ#HpHHb!R%I4Z%7$lW8Tr|)`(-=(vRjp;e3msDsmxP6fO-3N4A^A8_e zfv0Xf`Tzc@0Tgng_B4$ke*j#<{#ZQJeE3``N8GQ+d`j(R`xLP5AEMSu$-uWj0SdG3 zRlv>(+buNt;>qD^-JY-SmaGY>JJHIKcod`rvL}Jf;Edj%{ny@MIZ0>-2|m~}<$oBW z8mIV~GziQ*C~fefqLBDW%tpF}rO>o%v8uVOQbI$PZw;0I16coS090o3rt*U;RD?lf z);VJ3y1k#G$zrUm&tWgh&{K-d0SM$sPCs1mLN6nKt5rEA;=KKMlOhcHuRK)BQ@N=k^-sUdZm7m|Dt6O?T;pS0Q{EQZHR88Pnx^q%Z{Q; z3&g+Iu|eg3j_ebG-~0G7nt2~D@RBB=P+>5hF=8F-eQdf!eh9q=>i$uN4xbtG7(Y%v=(!x_cvzFXcJJJV}n z=Z4Mb+|t2vj>Ftpe>-FbzYG>&yR!QOw`OLe8AX&~8*|nI9thIUp~vT0F1fU=>aHE$ zB_vUPFg&S$gTs&Q=Ho2#M+g3-PDE2x*-DI=__ZW6NDcWYK5CpZ?5KHijH`Cm=YR+N z!y4BjIKuG^TWFIlV`YlJBaf1pX=;)AphB_5o!Cs_mU^O0D^|o&`$)kHuR>3K9h~^r z;r|YSt$Zb7_GmSKlY}{TJ8|@+&A>+2X4T4Bzire`gx}h8j>rovU z9^^LHT~vNkC%?kT;4hZye_9k&3f&V4+|qiK`s6mU*6C=C3vs;`~ZgyT7}95YEafVwP)bSgiVB%4S~3(ZE}_s<)@Er+L@j zD=7}Wf;?m{<9^}q|7#_h8wABy>ff7=A36UiHVXfF9zO-U*HNs6X;hAlH6Iw|nTNeT zosoRkc-Zw9DP~bDy~MX2G6`VqXPQBOfB6!HnhVjpE1>fI1>nZj)-4>N|1DWuMWtnL z^5QDTL?hFOmXcWUrgqB3FcRQ*?JjsZXP;tznq7l$vq0yYa%kMch!bUB4+%D2Sf4M+ zc3=(&8t^mH&Udt05FV6@_$?g-vOm4IOHRjJT)y{A{L+Q!=;%OsKR8L9wme*vEJ6Q2 zQgYL!4_!=E6S#(5dRK8XQWViVZ4#Ot4`y9@38{>zyQjN+@1969w+xzGy2ny%(oq58 z$)FLpOqtn>y~J%&^NsUZL|%_dx0^nF)yVLp{hO;;y@~1rt$$SWg|l*;Jue`aQIw(L zM?04E|D|TvC(&@zy>OFoVlMWSX1Cjmzsv0k{lHV+UdoX$CT`87zhi>w3&wP$kNE&O zRqQzDSAkzavII+G^K-KBa!UbVRx%}7f=ezne^0*nO4wgOfK?5zWe2SMsbZ6#;5b9@ z!FS%T;w!jGd3v3ZzB`xM-=QAGZ&VqUG>7!J{<%GfwNINMK2siMe>RlJ8aW&7#AsX- z$t+|Q+`BqER^KB8DXCKyzQ}B&`5;D<^VLr!vVm*!pO)eOM=!$eH;oj7e;ADR(0Gd= z3x6Ow3#nj^yB_M*zN?=US|RYkv>M~tobjd5YXmr(hEn6}*9teKOhSc=5GUUu>Yd_B za_E1H|KK(&N6F|*I3%r|54W^hn{Sp7iv zl>e6T!$P0kAGSs|H`JTd|EQ=eA~}SWm}d<=*()nQRdvAMZ!QJV`0|ZEL4esu)K@yw*Ay^0`KyZRva0%}2?(UXp zzW}Ed$aqL7cu0TR0aO405-KtZ zLfn6BNXS4CItB_V8YZIL2?9VuMny(N!NJ7^qGO|>AzFrvD8oYsf}YTDz+XFJ;1dwi za`N#Tx|IDKCL$)~;S~^9cXD zxfPa3kdT}s@is*5xTQO=8xA7jWF%l&=>6N*^*7WZ zsI%V~3YdVN3508|2w4IyUR_GG1`U{t8AKA^D-og@!=MoEL(z4#%pVsaFHiV7;hI2Q zWhNv*(2!|~75a;Nwm>vJ^O;s46asX@dKOFVyt3#)g-W=Qy%Xg9>+Le5pam*2MmhAB zU#wv+7_CpV|LW7P9&8>!06cv?#+*WEk-8RLDZ@8C$9p`SX1E3l0f}IW<=-gzNgT&2 zrHJ1+hTM*279%PFxOT0;&C87|vq63_9dJz#C4>}#xSF1jw-jmn{#hqNtwF_`#O8Hf zwf2FiMu;BoX8&S}Ewi zGF6e$>-X)};1-U279rH})FEgFh_*8z6e0HSieUREMY#NDMJP)rum0SiIh^!RHbUqQ z1ONH0*pSF<14u++e?KViwihC~g^G-8jC1oV+x<)o6Qf#INnI6u%J`yt7ty^)d#MLu zA!um>ek~*~ZZeoKI}z=%T)t#vK~t(=Qi}V@N_5+NL_oOleJ2#5hgAYfZUh>pV|~-87wq6c8`NfZ@{6YEWQb}9PP)pY zDGy`jf*A%y2S!D9+96wW2n=gEG|^mjGxXE=^0(gaC~F`pK?c{{$ud9KxK$Y9 z4qxTHz{OZch(WW9ON+JqF}>tWU&cdOLMu_^(@?orHfFs<-YYeLU3q5bxZS~15|aFN znaY$ri{12LB;>O}CVuwuH}gk-|pjlaUhZo(Vc*mz1YJhT^# zuhwjJoEdaup2)HzO_Cq)?)$MhP)gq#Ft!0v?;SfGQmH9pUR3?>gHJO!+zyd>9114V0KL+p;wQimA-cHZ|A*(l7;x`0rJNtUfEE&vB{Uh-1`0r!P(Dq-YPr z88!QHq&fIAl?z|9LGxe*j@gxO3j3h|G|Vr^FsdQOh%>UUL=pU(24LyjJ&u!xb(s{t zkU&ApM1~)y9BQ?@N|HEv&muF^S~yP0%&17R2jvtoR0aiUd8^tJhd$UJ-@FkbY=ljl~Rz>Ed$>c+93Mbh!<0{)A@Mi=D@^eQZW?J52pI(qaEQzJkFI_E*? z0Bt)%)muB?mj=blIuT&g#-qU%I!b2<-+pZ+pzr9)^0US-7u|JKVUyLTW7c~Q{@k2m z0Vk=wBGf2FVQ)@Dp2~6P*wqFiVe}*M)s9b}nrrBz0vPj61&Hm)^YOZd|GQfsm9!Z} zWoA_kCii;kn3;CUX^si+@!`DL7qoIqJ6d=bf=pj9J+@c721omlqY_t+JuX$V0r?k| zlcSU66Cb;Xqo-u3Ky>eT)T{xs7m6!E9$!AUp-5U$-`9n7 zTx{V!y(78L40$1&`!U3taVC(teR-xEXTCx9eMROrVN3;3mYK7KOOeCNio{ z$%hb9|JMzOqrwEJZBx^6Z^aOL&fvh^X_4$<7bBvSN9yFc)@N&$bOP;gOq5|UG>fvV z6hxCV^DRe}iNF*|*>p$7w*R|`z`tV#Kd1J!ZL(SZh65gFFiUg!wbR{Xw)i*Y{3qiY z(l@i&GWC9vR0r0I2K=OzWMI0i!Tzz@HX8%!aiS$S+1VtMyXWWp2-mKx|I#msRh|rR z`oHx{S!b*Cg7W?I6?BSxiKt<0$ijA@!R}rihBbdqsZXPL=-ey$)E|;A$fh?$ti=W1 zc{FjVJx;9bjKe`5?B!uWGHv?_1>L?TVT%`o~RfY=776Rm< zD-+uvU*{c+Z9J)x=C9O}^XCKFUF1k}VHX7h=?X@<#T|4}#DPL9f~ z>T=2tk>3G9Q>N{5CO%m?l3!u(pDRF5^Ip#vUhFS3GDSN+alR>P-dCSXzm1jp+AblO z3SYp*M5|_m>C`EkkjUbS2%;}uTeJD8J~@b;5J;32v7hoM17XgOq1As!a>uHgDH0ka z1HW_#_xg1)xOD5|^tTwC1ATq$WY=Zt`*10sZf&}^p z&|CnSf-Dy6;&d||3^?`f*#kfn@@Y11fwe72cH1z#a!UkRf=?M@8~f!S{Q;aM5Y|~{ z=i;aOk2bk8fyv$F01;qfcLL3B_S3PciwYo4P^MQ-znkY7TA?ZqHAA{)wxC_FLQc9&s+*UjdNO{3_MbXJKITlJ@GeB$q(yD>-8E@c_$&1KdLq@gc*ru`Rl{6 z+~shkBgK+jGV&Kq+ZGREmAoW0#~oUBZJ9mYQ3(hBhB;)|6!^qLnNAkcQB|$u9IK5z zC3a-i%7+>uQR=olmRX2;??P$Kj}sP%xqf56xVTC{!3ZV*xf9?Q!)*Gudu?v9Gov`VO#7r3i;prpLi@YH zIrFc_K~DzAsg2vo8xb6R;`BDN=KQJ11Tf49`xBp~8dFuQnzYK(tU*BvB9b61o(OSl zYp|&sbGyoL=FgX7rJZwvj^Fb*S7S9BaLRv59@KdL#2`|FmQH`~J!^Hd7;r8dtR3wv zuEwbk7ZIR{Y>ij*lzaF5cR7_d)SF}zi<~^Oha*~7`FLDf(d-A6nbyeAISGla3yER3 zFzwRn@W-!{)8nYPr@^fZmg(sXHEX~$BD_0J_gh)~0U$^(H=9ACr1TRObl_MN@NK1# zP9b}Yt|ROPjh&kzRo;{qI>-qH7|$XkOx`%JITs9W7eL{C0cq$}EbR{WeJfR;Ux)2v zPIK&1$tm2DlJFi(j@3rCqPy#WiWjNN@P4_c@t6_6FmxjRbU?A!A+wZ6#TW3jt;Cwo z*z2Gc%>exz`?^s2;VD*=-=zD0;KOi}6BLswma4%*TqL-@ zm42T2?fL@;e007Rfe@ZG5-LBoLq2GWGj6%xv%{akV1zw2>RgHLo+Of1y1l&hH(q?o z5cX9wh=Kwbf_#(-@gne7C&;)9TZLwvg`TZqUH*XUK|o+skl<<$LWHTNvBBULgrSDu z$3O%>&O-$ql%7C;k@C&_0Z6U6hoI>XunxM$I4vTBIuKIC8vX;PmjA#h63Wv3Jk}!l zGTijEtt=GllCA^v0=oQI8;<+Te0R{$irp}#*i$h~%w`0Q7fDst+QXYs32%}ODv1V5 zBH97b@_F~)@XYb$PQARm>HuOQ0saE#gmLW*I2F(r;H@gT7gn7wKoncqPGX-l?ph=?u-GZAm#*5;VtJM2xj2?I93%g*Rt{ePITWoxYW|SDFHpTOrOi)?MLOzrN4d_sx$5$KfzPALk zu~GbH&8OkG3LDxtJ1uR-;H!ldm+P{!%79=3GXrF;n{mixWM4|DAET6SSOk2 zqt<%|nK~%-{_Y{j5LbKO)EF|9{Kml2%2=H8mlu*Yvo%m!t&@Q;$|4_zipqX+*FgAM z$JaN;hVH{}>9AF>WNj$*tL}K=IFiGcI0A80J>RCBi#12BQQCu^7Owqb`~D_pGv&9( zN$h);Fq}#6kP90)alZL1E~e?N*y?S85QdiqRspd8?5l3ZLlwo zHW;*ohfNiK{^ZeC-$u9x==*~AmL5)>u!-4k`BwCA8QI5>k^VxP!=aDg8mXw~h1=&t zb!eFM5<;tIj1L9dBGRwAzUpY zhHG#iC+EtCi0(s7%%(IQ=Z9kHS)OX@C!o3ui<)0$m0ieawG_lKYwAXXo8l=_zaUV} zh(Xh~rBU(sJ2|}(0{^r+i8R{_s$Rw(jSr>({BauzrkW?~c!{PHrF6y4)$Wjj?#r04tRffGdygGR%A9;y09SvfPt62a;h1!|t{_Uar(6%HTE6x1X~HRQ~A zY<{k=?zh{%##5E<&qZ=>FS@Qn7Th2&Db-RDET1dVGO6HCtq_d$018o8>VGh!r~jT) z_aa?eEr1vdkU>)IrKL4W?yKuM6psy=AZxc5oIuN`D5?=dv)+J5P)WpT z6;7&$9NI+v`^xZX6ZEp%p@3SU01$JPxL0iFvspG=vyxiI7cN5%1p{`|ly@Q{!SA#u ze5Vco{L08Tol81_$XNV9-H3GM(Q+${L77RE)YA(-mIP=tOCG0IWKzBcmtUEPcFYjB zXZXN-^rt{;;YHV0j2GCZR3vS&3&yR4$#Z9tA!LU2hH>pdZ+5)DKwxie4=a&MQx~mJ zOsg~5DpL1_hR9hOk;(B~A(cmF@;WA<_u~16d7&6q4(xa1|7b*< zId+_>msEKHg^>ZuDyyL?Mk_0Js|}er;bn}FfZd9HpcgJvkwn&Hrg*S+Zd*psi072I zS6gQgPN4~ozzGj&ov9+@7r}wVL>VK*f9YA4E|7I}04+%dbdIZknk0>hL*p=iVwL#{IA8ILYp(2kmZ^ zk;6jnnQg^>XHZteTmJkh(!@~)0qKF3&7h5-?3D456ZE(9$flSXS|MsVYf*u!t>mzt z>L$pAQm@g0ewLp=(V3jM;`95+%$%i}-wAEeW@(r4o9`9J%w-fBnYipDe!%J$%_L|S zwl(bJ&BeV(uq($S+4r+X+BKtY3C&z-3rEv2*IrI6$yudUzqs~#={%=CkrB+~CGUhK zRZ>(Q_ewxb=#@{EBwlAf(X2UEhDg563az=8=fDWh4{eHRzTr;=&C0xHR_wM4%S+I-7n_AlmPbmgB4?izMsgp^(I5!p)I|v8It@~F$U(AADu{Z8-6*>I8GJFU?6jgQ2LG=+KkUkUU~OuN>S5At5uYAxt`R< zK90%&Z`Q!)t8V7eQ!Zad`=W7;@8(qLNbJB!BTw~&?fQ;q!AX|MpEu*CBCwRVN3VW* z$q{@{nTm{mnu^5|18h6g9RKd3DXE}xYRG{@*IL?RRA7}u?z4@8$+q9D!%l6W#CaxgTJiKbe6v8HZ z(lncJI1Hhr?b)^MRu8s-D>v)g@Mrm=xNVw#Kc0BS$%iRLU+jn+UB(78J`F{xyM?cU zc*=4%(kt(uoO)vzdMTpEbvT{p8|NBHrzErY(NY%V1K_faWZ34{6xqH450b!ac;p`d zU4nhCSa-mg1;>{)r>L{dYY0^oI@ewF>tAVq0JJE>nVtS|J$$2I)?i-TB!edc(l^d` z;keSlqz^KTi;DhnNLxe3V%_WH+k+=ik^di6j)rJa`r#&@^zAA?nAU<|!=^TpDtPSf89d)#P&ICwC&_Bls{K2YeuhCP?JUcE8-z_3hqRiJJy7 z7tCm0P!3Ya(x$;bddQVX*nHhkzjS*u!dvq z;H_r#K|2ZXC1(xNU0J&0@-49w$~RBl%_nolg0`)uEGGDN%H7K^F>GLrgf-`Gq7b|< zMBpC=KS6F@!AW+71A@G4X*L?tjy^Om%5v^31cM?HCjiEzEA~|Z7`Hu`l(_ZCag8uOq#b{vOX~>CXX$bwDdx;tpR*Y$lKHXnxmIga0iO*2 zSuu%6AUw(?=EZ%j;yoVHh;Yfps?eDv4!1&)0}g+y+a1f+t8mUeEScVGoAKPA2=IYlg;lg<$k<+owr&^UyMEvpkMm9ZrWduUl3PD;rRn)!>DnH4J{s3f4_XHQ4 zXVe4+f0LaRFtvr-v8Nj@8Zy>0wj3T?CMmcI>IF-As_9-*j-(Y;&6$e$QI`=vUAnxx zl<_iGU}wzmxdbPyn^!+S%wl}N_1}s<^E3?fKAqc4$X&PjZlrEwj4>&SZPt#%nM!Hw z+NDnH^E0bJZiO?%r>wd0`oP@d7H_>uC^e@erQP|B2nO!!h6C-JUWrJ%O`@5q@+6rg zP2whz?#pkV_4ZrcxAMo0uM`FxRWGTumR|1I2{JN}F@ur+XgIWSJU{k9Tqv^3_mcm36#v30hYBrc$_? zD_`pIb6R)*njx*_V;h$zW!SU)agPy#33TVr$qtoi`>DJvDXBUyw2cZx`d4};Iv09U z<)2Tp2MJ0g_CwQ5g=H4c=tl}S7_&@08_Od1Yed^&jdVSK0Cp-(lCbE^b?a2)NOPt8 zVYISd4}xjktQ=VOt6yK0EQEK2rLAgbJz`bwR%XAPhuyNMUqPf%jAh9`AOV7*t-c=- zE*gKX)+{$QjPUmbbS~XH&h=oVwpaJNSboz&WSw-^wwxj2LYVEQ*ow-MJxF`9OG2ZJ zutnR{gmnxK%FA|9kk9OX#;@ zd##z-OCk!naGqu&JlkddjRwV6okQ9pm_C?8yciaKglqdWvr)6{N1rOV-(n4y*eS&`AH$b^1>V91=QaPUOCJ5WFb65 z{@4|b=l5W0KbMWvFff;GH+>W}_*(`moh3WkZ7PDA}t*Ml&|9=wE>(iU1g8;kZz&xu&BT*e(& zg2Rm-KTXf5m`%VEF$o;1P~w41$hiYst4KG^cI&d}3)gg~#ie@C?Dg!BHI|wrJrZ-x z{Mzd&?bcVcFDBJ=ogMnp04PS1QXC0%0+FW^Uq%D){8GOU&dg4sQg8H3(PD zUJq4VmsLwMla1C5-Ki}S^M}_Bk6loa?(9#KL+wz7^U^%0H{Wtou)kO9;&uPqC#OSs zGR2xXPs0(vq^9svd3MlqXeWO!csw1Zwmq1w>Dt35H2ICrIlFZ&p<4}Jy!+Z>Zq5}r zQP4cUkK5Xi{*9`kde>V9S0dk}q+JSZdxWx5hAqlNN2f5cfh)78=%B9e!^ zX2%OR=k^aUTJF=pu4oM-Kxi&)P+)S-fDs1UIA8W7zx=B0 zX1*t1dVsc~eRW=AvZe~Pu4Wqlg)xsb@7`TR0a~aeEt%;JuBEvu_{EwLA)_ktKGdtO zs~oef)l*~76cN>=di2i-2>jhYqxWzA8O7gM{!LoTuQddTqJ7A7l+5n2c^w6B`ls^D>Lq?jwf_TvKt~b5^#2SBEf(BH z=d^^ojlkO>-=x0#QQ~<5idP7L9-z#_9Nqb#fO1`a@EL%d#X9;aq)N00Ls&~Q_)U(2 z#J9m9g%*|_eU@?q_?z7^!Dz~Mj_t&k?sA=>tNEKaF%+oKfI^+Uj+$_8 zhl+%_E}l)fBOkN)c+GJ-F+q4{6Rj3e4+0MA*VLm??X@E(_f1>oVjo8dC8_Lus`fS! zJ>%&Rgd?Tkf-~^{Dz6~d)6I5n^b+S26lQ&?5TZ(UP)QWk-=*wp)qt}w zbob)-41BlOue!p;`0RZ4=ey4J!XsP98b;0klY@_9$e<%RHH;0JJX?gEoPU1=1}ct-2mG|DKV`qTXRyEe4Vj z@BvJJfWRrD<>IU?zWO`J_gZ$l$A|Fnl{B2N<9DQP6VH0TV6?aSaOLk_Dm}q;jA4H! zfx5zH(#K07D_3o5hyR&Y0 zvYqXAnNe!wibfeuGR__9$3_jrV8yG@sG$nkH?uSnEzPr{eCMq0>u!#>{}QNRm|(in zN6zPU;uh<5vRJfFNqjHjwpUu7G!oeES({Bg!eSn56V>)NAyuCu#71(CW}F~@s0$R{ zU+bMAkIt(W7Cv>tSi~~y=G5j;&+9LG+4qzFSH&EkuWj`*BNU?#MK))|c#Mhky?8=y z^E)G%{I?2{M`{Mil1 zG^vsB!7=ONX4^@ok~AM3GcbMQ+4S9&x4#wInehqdT3UU3SmM!L9#Vfy_xyr;azZqV zho34in=-EHW1sx*;z1@X`K*mJQvdH}1}nls!KIDbVLZ}tHgoQSNlA_0W?H*hvypt; zWLanH15tTCejXPe_~6Eyk~vDhJ#;hxjgm^mS8vcPjfzN}W29FyVZ{lb@`$xsRnkK* z$Cc~bZAt!!ZloM+gh}!G6EaT`%}UCevxj{4@w+t7t7bmRR6MXC$0NM zs88Pc?#fRU)3a=vWVM?0n040`0SMs_lRSembMUYg6(&#H@KyN+JSW3lzB=ALMox;r ze&Om7-Y|_kLmR0VAqaQ!+?0!UyW_AKWd2E8Alht?0u6F-ikTbL)7RR-!g$on$OnED8pkRwrDXN?(rw(;H%W`8OhQ ze`WTvv!Z^=F-s)!t*QDXHn6G-3%G5_>?q$YKl3{`+EZW{S}oe7+3_7E0v{`oH)s4PMfTLA?&;YdhSV`Q3}3AqX)SH&&vI z>v!{Tg*=t)x$Vp844 z2z|B=LUiyBt^O-=1!UO>8T{&?K6JF1FLUk>fa9ax$Iqp@vJXlU?qg(`GS`Hg z|KuN|;2y9}1{;fwIhl z2yhxKsB5cvt9BisJ*Coo?WF$6PW&IiitQi2B!yE*=t~%LlkW)5&UnQ7Qg5MsnQ|98 zHt^w(T`Tjc2*e*dfw4hty~!{0o;I`@gWZkDWA%pVR|BAJJTeQ_YcQ*rM*Hr-x zL6c*abTvU&Yg;H##;l8}{jfwdLj?b?bV92`*o0C`I(7W9Oh~|{5Fy6p6}RXpv%x4S z-7Rf_p;w%*%Dccs=X^PK6a`rR|AqL>IAhZfM&bys2(XJP;kWnC`q}tD0L2D3!*MU? ztqDq}sDbls(A?Hh?K4h7Wy>jy<;plaoGxV>&x9)q&9`5_?k_L=9qxZK6BRdR-HIM3 z>K1Y72i%zVz&CbYa;XX_UouimzNMjC%OpqctFwbSgH^p$wrj)7eqnz8J=c&HA=~V9 z%C>dg&Bh${nttDfPscE<6=Lmo|0?#mBjc0%=71&#DBBzesUp*@tW;Rof={B)L~urF z!~Fg!n|MZtBaWr1RtXUwC48^mlI{;q%%gVk8Km36#@i^9}@NEr*_y85{HW0}P zr?WMnwhpaH#o8e=?q>;iS(4J`vlkQwUWHa_`f^;lo@3$@ee4hghFNbtYe=x?G|#sp zT-g-KaVIwZ@2i~fd=(N`_%%Nns z_d9$KEdQeSWkcxyy$xD%U5qsR5$Gf_d#|=|`7dS%?IHaIozZ>aS2rDlpGe3;)_-Nw z22ccKeXN*XARzSSuNRpR=|v>A5Fmz8c!&_g^25D($X_TtvimHOdbHTm)M#jbw^8!m zb4!(O^8fyTJL#HNVP_{8ysNfkcM-O%uQ-CMuXEm*olTuvJfA?%haia72;4Q#rq`_n z+E&4TMDd>PLKMunC)5pVubxaf>0`YW;G(0}?BB&HU9_VO)&|l(!jZo^UgJ{!VYJ|>TEjHnGCc&G_VMx1mI`yl6 zcSg7&oDm&H$?;Gf&4>dgu9?E$@`~BIyg}au_6(*L=2_ywQ&{~U4)qmWe>5&I^&;F> zmIKZ%Wy#wKc4&bdypB2{UUFswjDsS@AH#zv(=cg^Ec!_mV%}>B{<;;umYv#d5PSGVBqBDSJM44;OBd-vl%S%Mr9viHLuM)SJ&#}osCun3PEb)X}BlS~FYP_;* zntasnI#gI1!Y_wbx(dshTnMxQ5iueLDr2G%Z=Ty4FOoliTTI-qH4?j->CrQQt5{_g zq>$2#*akl@uFHoJtn<>SFe?vx)IrS1*UrpWjk@mJbu}rW23PL|Sg_B2Y5BiCOG`9?MS8NVs>!?KJE(FZB2P%V5$#(u3 zl&wg}d!;ZfB*V^t($)KLqB{_P_hF3)g~oaJiAjdBL~4Ba_8cD<#HAE=fKP6Pa$D>#wc=^2NsfPyrn}EzIy{v(y zw?PQ5vf>u^pmob4AW?y(XZy|ZlKB0;k;S0S_iL_!&#`Q_^#o7z{*8DRNn2l(Ev>d$ z_%R)Nf5{pi$2Bs{C@$M;E!~?~UKQ!M)q*>Al?NiZ9Y3_=I0qOmT0262Y4PKZA(EV` zc7^SGP5OYra+Y(g?9K1Z>}ZoDn$_1YFJGlkFiW+`Yx*sM>KUQ)vXy5aibC*% zFZ=1|imK1~s~wewj$mhv_J+OZ_btRgow_mwS=f@>oU3+y*OT|rj5%C+8$eMm9jt;6 zm*_u9r&!f~%y@hK0ODKprty5hVP%d&28ED-Vi&G+s+J;-ir0n|6t_e4i>w+n3h?n- zaxJCNJ{YO1dNh)ab@hit;X7B5duAlE*`1D}duq51Bq1W$2)q9ez+8PG)Dn2sH0S%W zcHZd~y4vF9g{P|8cMoj)N<=AN9GMgcspcO*1MUw#A>Wdodyxz^r5EhDJAVLWYoc*~ z05bPWU0a{iH}E4I)(X0*BT3iQqu82PPev^ty5h(r6dtUZnN;)&D5 z>Q>d>eF2Jsg7k#H_JBE)2fJ=x%XIoX&NsE;>=&d-?S#qf`z$Yp`|Zkv*pBVhuRK5tM~%(89!6)amq1ln6=u(9}SD&QuHJLp+kpR-rgYF0|HvF@M( zm&ULdrCEmP;PbbfHAc_SB5g)>X;Z`E-B8J~!unZ;j-)5tjJn+4z(m;m`oo#P;dS;F zi%I;>{OiX#oDqueb-T)41s!ZszYLl#czxQN_j^jTbt-ozGBIXn*lBI=Yxk)Uwegd- zu8FAiFcYBy?uRV3XKUG@*OlLOd!Mki21%YSHB(2$^d|tnpd;Pc_}OH>6=iM$YMtl0 z3e66D0F82-{p!}u$r+`09C-0wK3F+i3#mWR`Ot6Q^d?xQALgb$D)ae+3n*4&p6uLmDe z{}y%L%bx*%mjrWexfyLasL<_@Vk|)UqjK0n6~=N8RUfl0n*(LfUKz5F@8fW#rG{%4Yp;If~Dv_6eRCj4@2x z4tYP0L(KSs+s;xtoZXBX%xWl&XDD2@w*@>z@8%!mE|Dl5wrP*1YTVk5d1Qdw z*|pJwiPq22r~53tHjyOg^UE(>_|XMDwS{AJ25w{2oFD@2TX*Oh;AjuUl953dm>p`f zp{`#}U~q$C<&qqy2a@ENna}n+BXu7$|3+Q0e3p~>`fF0G$&>+N7b==90uA}y(vQez z=N`kOH(sab?eUsRWTSY~!Q|wG7lD)o^_>xbNGA~q%krva`Z=%E?#+HPdDjvw@G=kR zQpeoq)@l=LQ42^9-Qt5eWs`*No-tRz1P6c4R$pZH9ueuTr0Cne-ceD8N+bjlYtt9D z=_1xRWY`-}ybLc7MYy+E(79^^CTfT2D&OhUsBecE5$j+TtFV3?Q9vPXTA?C}5MC`J zz8+pRVi}4bu$cAxU^Y1O^K51Ed5>96Dj7pm`#SI2gSUbTwpRpYe*lf;Jvoln89tlb zTNxr5y<(SCF3Gb;&+WgL_Y~Y|Zyvr_%MiI2yJScQ^RW`gf2tLc#ec_fM2AS@x=`I_<#2wSK%*RugHhO#c?KsnOg8j~&xTPszPW#ECQ00+Ux7Nn%sllj9*Nu9{Qd zkjhYvhgCw?M?a#?0lx_K1bT70A5F((xF;gmd{bzj&uK#*C-!pzL1*z`&+Y^3ToRDW z8|m}E2%2D>iB+{Q`!I6*n~O`ge+m%T`_@l9{}4|Lu4O6R_3+NG9O(Nf{rj`C{C?ic zsEO!ae~TBx*GwUvRdFh&*IiV?%*oAZ8^W@zF7-&vp2R!ACo@~&5tN#XEIW~R6mNUs8auGPq7MopqO1;Cy9A(CfPXTQ&7;d1s5V{DSk-uzzA;$dNnY5lKJYE;w42Mj(J?=BpkT?yc#at!(e~? zeD+$|=XB0yDp3CY%Dini&O}WePMx#ho)Z_9%^$!<=^ZTCi0BVsKK8EVC4O^k2H4H1AW%X2nv1Rl*VLGPC;Lq3p2szBaUrZ>zbAmDg_20n#Eb-u?svPy$hG zFA~U}jVdGcqj$`T4a1L>)d-L=H0ZqToer04LLdTn+Z&DX76JjA~owciCOemViUlNgl z5b$V|*#ZGs%tnO3ricS?d1cJY`PFx1TR^+{uRu5H?;;BMy-F%MS-%xw9xjYS`kEK9 z_NDL0=Z0m3vdu!HIoS(Hem>Kn7$q+d!|q^qCiUhrz1UlUyPu3$S1uZLau8N5PQ{4BzSN6@76!2TlH*N0o z%XlOu%_?2hR5$b^)0fkeuUB%lvjr%L1+Hwm1_>s9;7Ugc3C`)rBfE+8&2dyc7vf2z z)BZg1`FVkQk;(iwFas-V;*a)QP1bbh?ZqG9a{~8cl>P#-|+x*cV&O@3`Q+_?$ z{Hh$ITc+?5!tWZQ{xgHs@~wiY$bAO8`bB2Jv`vA4H_j59hj}>BtPC&txbT#EOIuHI znKFp%Hr!6MbApitOmE>A#BKY|D%`O6S6d}|{MqE-o(N6S=(W65%oB6GzQ!Cs?h2!} zr`q;;2SV@gC&I!W*c$LlD)^>YI=SjyvUGk1kI{ii$Yd%ZvAB8ZCyl1*zFvntR z?Bh))i;5%q8@<{y+c)axW|Of;9uF@fB6!2xtvGU5c|}AWh*}c>1J^iZ4zs3Y)bO%g z=7}<4W(7Qk7ia0q!hPb zD!lZ-{H=`{#}$D`2l_%R2$c|5k#&fvsg+&Wgp3OYRhfh?fP)&pI*DO+9c9*ZqvD)o zj)igSlve?-NEi6w#qW?Yg6W(oul2r#Qg%2(6%>EVcOeU>fjWZN-CF!mtmlCvzmx|t zldKGGQl$%@H+Sz*Fi+U-`Kgj7J!a`I-x7w?EdRA;olL9%^Ks33kXoO*JA*H+h(H_V zjt^GRS+4KLY){zf1?8KfwjyovoSwkqxdQbo1{a{~B5j2Oo!6LIlQk=NEo`Ous?O`b z{rQjUs^69+fKWkRZQTgpBUFa|k z=(l-bEs(CDRSoxto-6SZmO7ji6e8Ba*HEh1UcmXyc{#ruh?+K~$TceB&{I^DZdr^{ znu}H|FGglw3wBNQl*OFdRYzD4(@5oy(POQOV)1BupoKv}eC}V@8*$U{=J4$AzY#gu zkk#KdFrxnN2ruxy(C9Y)AAqKLB*a%??e#CF-T)a@69ZSeg;oq)5mv4Z108d5#7c3` zdiZPwPWl)S@oWMC7OP(saEflJ(72uFsy1upmgEKMr#$AO3n2{ z0PLMWb7j`X9Efp~J52Ucx;qUk_DZo@zF=ro4kJwlon-QJLQ3dyUaN0(e;#+;uC)#P zS30hUy$|7U{7$&J@2B)bKG_i@6!O-=w7U~dkWvwEUWhg z--looi~pMJ_t|>i1{z*L*@UQ)xWlM6D1|bDi6(sfpgcM?7Z%WfCe$4&HFR=rpDdl!;Wf-3A1} zmCsi8kqhqVP&(n++{kDmPB=?jFJ_48jtc!Tu_6Oti%dAO&QkS42L6HRkJB__;m=c? z-4kMRL7LRYA^6$qCs)rpFj-6GXDat$$n%r+uPYDD&;%~^A~xN=D?(DOJZGHubl9Dn zW*5Fy@9+u=H+A(XYvjdML=T^(X{iztCd&lUo#ZFim`=EyAh2i6~H|`@dVNi()DoMl| zOrf`14bexLC8TAc$EX)>Zqk0_a!_B)XSyC{8}prugtcs89}!LFj0cB*&IQM;*c1 z2CS|=r6ZC^6*~~=HUANZ=2|5$^VK-LQFv2n$UOHx89E!3Yer|XR7KnH<$J~kkO!9; z0T94NmX2=_pt|%}X%W~$y6w93U+;(V$%+;(|K6MW3#pzxz z{!PYd>*Q%+`Pyqp+t1gyX`?ri##+41m9*&-?)Am2L>R19ZX4e+N^!86@KZ*;vc8((j{elhbY51_V&Cm z-RStuJQqyS8u7_>LXv#OgxaCHw+jRHuC3H3SL0+-&pp2jGpZ4>#2>jglg)R%-$g$nsLum|0RXHP0IZ7 zayY+}9oZ%bsoM}Maz_ek?H^mZ_bZQLhhc%>f!hBeQLpQ zPa;3{2O*vQD)vQskP>SvSwvj#?;4|WLPR@6A4?=3DE%IoU3ZiXFg@8$xbH#KDFLplv_OC-&CG;)z4@pF*y}6BZSi@mh)$} zAnZ4g#9qbZ)rcZNjEaRXZ?-iQ_Bmsx6YF5W!=pv? zZkn@c*vz;#!&Ow-HS6s)$xdXlz+znhop~n`YZFmd?gUug*>!l6eLouPHr=;#Rm^m~ zGD?HOW~?H*Y{Y`@oKAJAYy^Tco2{u&te5HT@+^+!iUhQ}61;u%77LWXR<96$3Y}Htvv4* z3+1$o?uOM7Wi%&c=kNkT3?bK{leZH3s@R8suQoC|b1VBVKU3Mr1!r!3x>U$)#RCSz zGvPV6W(K>?=$3#U=w|T{lg>GORqPe-{4&uG1<*42~ zNYDTK@?TefJrn8gs~;p&_@6Pja)PYl)^k<$~>s^h^6BEj) z`27ymliroouQw#QqtuKP2Rd{)n{qzuxd$gFS;latm}{}X~x(#?a6$(lK=CW1s3KQ=>U_0Jqa6nRj{D}TZk)9#0{IbbZ#%-FVVDF z;^Jpp$C}?dyW!^_IgKsPk4yJ@xoJGU!u%^})5S?QZH7zX#{M9Tv1AQ zRKOgEml~isi^hKPIlJ1pDPm;Mx9Z9$y`U+~YXnGZuiE0-gOUv~a&?rScjqW0CgWT(ZG4#d}u0c8p+ zPBCxYP;y5p!_?e|t_V94;^O4(w$VDm!_DL~+j9gh78@+nKjJ@vv%P$g?fuB&q|bvd zuo)2nNrf7JDVvSjj|QEIb?m9fk^WaS3=)(k<5^@puQsMIhQ1`t)A>Rg^W;noW{7HK zb!BzfW~})eFIt;di&N*1%3xt`3Uw1RIJk~dwEEsS(bnc~F2vIqf@pM^CQGM*l}ZeG zt3*3z1}RO>+hC!TK$IwH2KV$x__4zHtUbxZx4JnO20sH44V_XJ9c!YGi&9AbSVmbB z#RhFwr2ktav!?!wbRLw|vK`&5Lczq4t*rM0g$0p$$@xF_dqAN{*}^3cFZ*s2Y`NFC ze2ZbZZuRDV0u|bCnWCTCIMI!371)C*0iI2RdOy^fI@|`}&K<2bu64J4k z!&%ZWc2Zi&Kg~~&fEk|(^Q=P`r|(hgtg16mA6gs_d@JbM*iGq0p3;h@;&bd6=2;H8 zhp|Mc53sGFPZ%(N)g*Z`=7N@_=l)Va{}(z{jw^IC(lS-eW_N-WQrP%cnvDjn%WA2M z=>=?{nz9A;(pt<(x8lOoGMu$JQgiI!v23+!#>yE&xLBUpm#|Ay{WYTEO8LLmVG-Qj z@a(0D>OV)TQeE3i$~d@yf-@8!diIHGN^$XZIcQiK6re)VFBchyDENyU$+=VN$ZD zSFDL2$tn(2jYu^HQRu!hzv+eAQb8{XOJVWQ?XEtU`v~FDj*$f>+OO&?r(L!T8hhbT z$FH`c4Bp(sd?3{x<9y`BcAjHk?=1~gZe7ugYR?66Rr=a@=W?-pT=4T3gPNEU2`w$Q z5=!?baQJieO?_8PPH1y)LSl)Q{=Cam-Rfwgkt0|N+Z-n1W#w^Bx0rnrPYHxXPrt)22Ni%Ft;FO} zN^PwBGIeBvm&J+QgUU#wC(R9lgEJTNC3+6k;_odFKqtK=!RO@A(L2IlB&RmD!OCk8Sn2&3t#< z&q(rBnIvYT9D-g$wcebQl_VHwxlCx9$#H{+XVYD;QS(*qVM6A*G8#Zs|Gwp1XQF{} zvq#{I8Gqr!l{6a&AYS91Vf!~g-^Oy0o`H`o-9f9C>k@#PlI8AJeZIuiK0N*f>!I?FNIBGDKFHS@H{ye8gsbp6~Rb0a2W#kn(Y8<*BY!84AD{ zscj&da+L108}tyeZB30J{YgRyZYBIx0~VYkpqT375K(xOP?oCvSL9Y^zt*jb>6PH( zIF3f2OMSK{Q}p?nOPl8O$5Nw;qV*jnpCaqi0MK2!rX}&DHR~~Xz6QAeGqPp9wq5nN zTG&t4OOoDfZuJ<=gN!tr2^Y*mJ|1c%D5g%^F&Cw^{dD}B7Ou6&PPS~annw}-8$l)G zFLC2g6&8nWnwoMsK~JyJd*~MJE_QRSWURC5Zmaj(q1v(bKTqovhjx&w13y;uxiojU z@61?sIYM0bz4CHcsHd-S!*#H_6FRM=N8i<>PHYFwj@FCWn76^5Iwd)CMsc#~0eb)N zH(=cafzSlL2zP>=f!^vi8IHwI*t<#|;o6*1y82qv~G+N{uIFN}F;NM?iX$b<^h3@DXKmE(nKsafh^l2BnSW z?M|bL={m)n46N?wn#h*)snIdfiAF?O}=FswZj zHke-aYiK!n}EY{?Z^rr^kk4K9hQb zJ$lAh(^}Z+3wx5-qUU<>+x6HL*>Wcfk{V*f+Vq=l&aXXoIBk|`euezO*W`@gr?>{- zd>PyLh=bCA$#KQdd2hScODXHh9A46MwUng2vQZ_YdY7>9RvVgi5@h7_$0j5&tu_?4 zLx1~-&E0(l#}eVQU0c$e()B%)ukp(l$Ao*B796gdu$uIN&f28@C_{*5y=TS^!?37U zlJwxIxkvPk_`o0)kwC80V+ON%nTZIm&3G9=X@7&esh^^aOB#-hl#|b&IV{33#qLkUoOSujb>qJ~Tb`8e&FKgR9 zwaH>N3Ginz7{xwswSTcpGX@Zo9|{wT_GGU_PT7)~tO|D&tBx3IMp7OB3a+2G%*>fa zB>g*fzL?+Q^98fsHR)4#fr7^Q^^Gb=k;!@_pOfd3E?5$ia#eXfM! zc#0CK`z)pY2O%|yY!B#SNcrY}IGIy6Q*5DygS~A5DKaTD=ERLcEsaQT55{kQ!{68# zR1&^w_->z&BLg>fOaaHm?7;qLJbWhw(xWj#qwd!-3#-R#v68JhvtzGEHx|b&l_omCjp>Os-G)eJ%D~E!al>e*_#J{6Eg=Oo*%WFjwm~Za6hpC;~ z*>5jBp99!4&#zywo#)@OA3lu)(mm)v^H2B;>yd8G=B-NdRh-+OB%X|zETH*&uyi+D zy50gs=d1J${osxQXHsQ9y8|}#hg*P-gb>yk3p@5-cxK%&csoU1L-{4V$h~#%VP>+) z4&Zi!W{Vs?g=Xki`*a1u(O$k+1(55D+{mfJ$l!k7=Z5}VVFo;!%!mz)$ z(*=b|Vt$^vN%z9HZO`dKCVu>>gRe7eP*{lITH^QtWC zd_@5NwN{sjo%0C!Xfo4$?)QcTN#`kUj%YLieJIT=H}qQr=szLmDt_+00|0ZOxqk=D z9RSD#0zn_az#`rt#t2!b)?rtYMIqJ7ATojJMi;3E>S%XN7x)uo1L4PA2+jUzm%|;) zcMZR5kb`r?X7@S*@4!GY{cn`{H`4kCjHk^uQn3-Q)<{cypZhhhUh%GwYQ>a_J*YSN zi3``iqVeMIYkVAw+m*-h#>cH{a!Qdtg9!#?g>CwJ5m^G(%&k9?alVXqn>eFgA6!Di z=``(TxX@AsvhJoKr``iIHd_W}(=lyRdRFkOOO2A%xx5u11+Ey|hsrcy#vbj=^o5FD zs1!+T?G^5?dW$3j6>v^FCOxkZeU7s;jF;5XDaxH zX7C=Sv1|b#!T#|2yL;$9f0aqpl~u~JTa~O$PyvMJ+R36 zg>LE$7z(9&@nJ39ezkh@Rl9T@#kShG&9guWC5*d!l6k_CDuMhVff9Cq%lnfO*mw8 z%Kd;5jFG%jWf``(F??qDM>0|s3wuB^8LQ^Y6NiasJbvFk`#P39B$1r|NFselMyAIZ z)ffD?{pv%lbg$oMxNjGQ&3peSmbNO4v%4)yKYMHQ3E9B6Wc%G7z-h;A8HxQs`m92? z7>K>Tt9G0hfokC>r6%|I>36AtooiCsr##|3RH(M3_b~QdmlZzi822y;p~pt>BAH`oWRbnKQ!L%=A0PSYt|zqhGqTeFYgKcFl}ejk0lw3C)^a zuQ~O2;p?<^)=f#PCtn<@SX_oWmL9vP{@_J>P*@b?)%} zbrHAG?rj@eOaJZZh}xf#;oeTs?jc>?!(4=&hWC`K&QYke7K3|$|Dm=V*Rdi|tUD?qiu29$ zdAJ2Iv!?<~o;qZ%$h(w__SaQu^Z7?oheTKRFn4oQ_b@(;AhGqxOc$+?KTQ5Px|O$B zvek}frYPINrot;YlH%OB&?ZqR(V3O*D_7OwL{@9bUFwoi9cwK16m`hjtxe5A4ThZX z_v84ss|+1l|5j*9Xl8zfMNZz}2WBNuj{c^CUeB?%@8L)%rD{de?U6f^!V=jMN8+AaqSBrt~^-%aj0sRG;;A_MR~dZLu`eXz-5H-wDf z=%*VPms8icw5?+DIs@qwRzkx7Y0mq-?DkGep=ZV_dP{`3)?@j0!ZIT}7H)MF#jZS@ zeb1g4w=mE-`tubpG0W5owz%V0bmZsWBIcp-g{hxc-NSriL#PZq7Qh-bh9i8hU%|Wf zYLw9NH6iL957jRNB{7RYw(mimn?TFao~>2W5_zb4=Rxw*V}}vkD^kS#k6;!{7N&Ah zMdVcq#Q2=2Xu*UUMDIR7SdWJM`f-jR_#}>IZKVAjwi*WYA z81-|OtXA8~N{TFn#4k?nVT7zyiDWSAq1rJlEN?9n4ew!|hJq=c-?%GWf#hiBc!k$* zxxM0@$PR9Ju2^rG?qU27f4D`TOfYRrA@=4^1tE|Q!`z(xqSV4UO75x_smcL zSrsa9Y0?^Oz)Qb<1-J{!j$-H)bz-$y2NdO)VR=>SWQpS*29fO4e6d?mj66t2;T}e# z`3`u7)tV+cExRTEqNSw|`kX)&hm`P@fPbg=v$E7gvupnPpSDbXbL(Nla#MRnGy1u_ zHJ`K|%=)PtuqnFSZJN`5U$p-Tk!>?;u!Rii!X(^n`uI8q2zY5a#w7(R?eP}Eul_WD z{>u{p$55aR=&At0y%D=>1=gjK0zZ*wFzQ#R=J8&B1D3fDHxRuY zBIh%Yw?>5?wWP@TEMl@0$R>otmF}7E>LA?+nL_QABO)! zRhr=XmRH@XCVehT8av1PWH!U~c4eCKXy4haJ~zaeiH}=@<-;?|0j3gYlqruK;YtCU zS^RrLDq}*CIEQr<-bigstvLVj21UqU_>Cs>nS5Ll=zoZSOkfck&_oapuOrHie!}bW z-~>U#0Xxa{4Ebjb1}ay1sI*W4^D?dZ2s>39gByNOtQG~GQ(fr03*@4_+qEg34s1_V1PI!e1nL=|?jhH6awm@g z7Wpvyur5M=OAC#!J==7F@FhlSdEFV!bxjeuqcqPHBPL`6Zp^Z&gbqO2>HdI;${Zixq z`E3$#dW_MxU9WJ#5j&o3wx>#G;CI|l>O;JyFnSEd9ZDJPRs#(1V}ssmgVvfQmkKHt zrL7Z;-(200EWNHtGk9e4a;B(7Pu3P3Hq%wQ*1IuWV{j!7pxvUCi}NotsYNB0-+h*~ z>2XVYyzNQ7@L>b;$7|1KTjf8bL;iQDc>ZHnqOUTpZ0*r`fQzr?*jq4(%vwq`urmUH zN*=j%vtO`yqHtB;8wQQyd_8MW{@y+YM4AtBmk=ZvPNyb+GD%-Z8q>2ZM%RES!nVeL zCY&MYbkD-rFh1Sxv;tL|EV5(oFKe-I_1(NH_`Ux{sI;b`CtB1frz~tJZH>7_wUfJ@ zUev8d^XJXIYX*Y^BLOiD4$!ZiEYI@ zSLnP>8JgT$i!b=K3bGTv{+7By1FhGnuNbz9B&nOcUKDqEe1-Du9ww)xIo?xxh;amu zq4Nc_iHE?u0%6>2WO;#lo*U^2_X?W-*_r#dMU~v-YEnBQzxL*EDl!D&!^c-`!?S%y{!{I2iDJIvg|Bvq4 z?L#Z)3}Eh`vT>;ltnhUK;YKMMYfuaUlxtGgkq=Uqhti7dimYPV*!}MH3b;;O%>DpM zQ!)Anr)B#WtmpDi5K!urI;Hm>8$FlK!AOQfge?W!s_gW)2fx~v`-fkxjnbk$`*u(m zeUUJ_BhI)b)z0&APX{%(Pi9dWg06|4_43gp%fPxo;=sqj@jlcr^MIpWaO`sj9IY2Q zg+Q>@L;Ohxt09NjZFR{CeX?LJp-yX0nVBKKW<`k3B@(FOdO{poI{a*E=VP&EOkol| z?&hPJ@XsyS&OVIk(_~lX+f#vK;K#Qw>~@4nG&{=UuAh4vr!epxr7WkXi4t<}5c3~43>7R(nk63&n zfHUHLM^eS{1*1f6@>%u++gk{hb!wXKR=tm6=Zw}Cp*$^YpX{vsKnhxC#_ylb2fuvu zI8y8&@L5;H{1Oqsr|{I6v2bvx+B5r)z-l$W#2tBtcRYdj&8GMYk(JIw_?PMxx-4|j z5}adyxX+j=i+?;iZcp%AS@(|=HeU=ovmf^aBQR8k5!upoZi7;hR|zHh+Ig00vw%tnW258QSs3?cPB}i_>4&=9~+y z0|)AvLx$Rslr!1zRoKBvN6FfVY9N+R^Y5Ewxx^I|0vktgjmV*@`yAij89umb`Q#V6 zfszgFUwTea>2=|ehXJwrwiU3)iq}v1^eSzm(FF*(qO8d>W{(_uWy>YDja?P>Ys?IY zQCvsr`Mt^O%DP;5Urp4~xhvS>q9oGDn6I)*g~m(BxN;S96iwPf*5jadvG^S3#8@H#a%Fs%K|4kVL;=%ed{lQRPqan{*H72mY@A6r`PYGEiN~;}ZI~ z1d-1!%y?DwX40M~VGQM)WtF`%M#L^5S81(px*uZwFTun5TT{ zN~!EDsLWNN21i`cyT)Czrh6RUgyP($lnI1Y?FDiiTF5z&TId*UVKR^S$iXeQ{-`HE z*!JC8U#>|RS}w$`UX<)N@n*#$SWOjL;it|0$t^t#Vej~yFIz#U`UtM=s-YN&o6z3F zywKorD=Vw4s4jqj@}UVqGrE@rmvxyqHG_85Jc$ztMia+X>7fVq>zBB-0)Zf8MrojX zR)g^b%dFH;9iuxLDl%SxDZ2E`>H8h?Xan9Iz1d#0y~Z)4#+_3%^5OCN&TZ(|&mApr zST&bsb4UERM;LkLbQq#BznufHrRN6) zEp5~$@*QTBOi7bx&0W)B^oW*0}8E3`W!4X_@)ybn!x@55b5{I8uR{ z?V}YoAHG%ZL0?i=mZ)%#O6)sobSV(kIaHqO2(vnMAlKAG&6FKAq@`7v2*-@txuM>} zWIYDxqY?gjM zrr?SA;p(l>aEs%eC{}h{QPV7i0O};MT#kmbNVL89A*75{Z)3HIBV<)=WCo)3!>(s> z;c>wTWsfD#H>Y}HF#@xK0vlSP0i+608WOwW_v z5)99K7AST(;fnR}&ClNwi}V)Q3mq0K9L(NeNPJSGqsQsbu3CTGxCyO}sb0&^04*7s zfTPP+u;3fJ72Hc3NCihb@aNYN-Vf4=Go<8x1u;`HqAA_OWHFQd7$7V?bUYXCHqFfz zwP1HxZdb!>^rJRT;Qz5kg=X85Mak(kR59uOx{eQWiCa@st%|WT05Z0L0COL6MqgM{ zFcN=eOinFaS1J2U_Z?VTM#9(6EZ|v+ih&M36G?edkOnU@k?w1J?d+@pO(?;@AUylT z7h=qjar4kFTY`bTh&7H=mu1?k@NAFerqIAki2x5#UEpXDF}(#GT?e$n9s+mShp$2e zqT@|EmG6H1{swRBu$yAMt*SKH`HT2b+_TSzIL<;hVQno8EgQz_t~plJ-2}&eHd)=c zx&2bgTSrH>nM`{O0h6CmUfF5O9YrDckI?M-UE+O&3=)<$-~>u>V-vH1vA~kVO(x=g z)N1xb7)xDqQgZv+%l`T&`YxU+kYET^ScXnSn@uB-#HpxLhZf5u8)5M(LyZYm?|zV} z=SqF{$aUGELf?*bTBS32&eoC7q=9(R-lSYegqvYk$O)=_m+Z`ux%NTt2IuWpf6cE6 z?aXX=;)%8vrx_jfmVq!j9~+K}gXT)A)D)d_4B~@kSlYIv;^;4`N47<$8OEd)FbLLp zaSpH4mgM&p?Rb+<`%mv-I1U*DGFx<8xLmpoZ(freSwoG2`HFwsY)4%-2YaxQ9Hp&8 zSPrV3d&OeazRu^YSnrp}t)0C-7<2CPPU9SGA6}p&p*zIazmxf~ZG-BzO?78&;Ef{g zk$HI>_RwDWRAc<;rDH~_nDLXaPXwE@_EO;-T#N*r%xdLV>2JdCVSob7JIaUl1kyMs z+7%XP=1$cO5^ldnO)pqs4%F?4tW2*vO0%>Xk-k&mC z)IeHy#i9uwI#H6f>YbCE2c@J74(7WaEgCu$`N1W|cH7BQQUaaqTyKq*qMdt*(j3DT z-CmV;FO?r>;Hib(ys9TZ!~#xFST_;argdcsTTvfS&0X`e>5S~MWm?`GOqj8M!hBV$ zb(j(7lAA{Fe7Tz?(3Nw|uCyh?-Gasi-E}NLKHjU4xsswmgBhp&=pebxvi`cfXNjC7 zm+Fv*FO7?3aIf|vR`FU?wU?;&^tHe=E%5~p|0UI6!?4rzt8aJZD&d%oZ!de_-o0e% z%Vfl=+>hf(gKC{pgwu$VmMb4g?xyyw;&{Denyi2x((+$wHpO?_spTesvr^d#^n{VkC*OSl{bbom){$B>`|XeobcHsA40@!jj_ zyJ6k9U*eQ|n-5sVR zTVuAQQ@gSrT_he=gm57`(w-#gm{YX1P`yUNKZ2~y+e41sH2QN38e;c3s^eIjt@xaU zm}7dowzW!(E7jhmqgqoNyo}fD6t6D9Zl!{UQ1(KJW%oqPos-Y<- zUT5L&1~-jnsN|w4>?%r=TTmS7QJ$kSlhxOpGI32gdtH?;$$Ny$NDM8Tp~<<2C=sJr zF_7?9E)|<3M%SLT28#nzZ<^M#kRE>2!4|niT(1K_moEf-Or!DZ=h%ZI5`D3q4$PLXYBY4JT^xpw)HUOB%` z6+kazTe5GWFB7h~JIay_1T@-lLst*I1!2AAq z+^Y@n4o{wj`?EW0n--V_g{_%z^@p7na*C#UtYQ5Y{Fdyuj))()3%F^uSgf%jYg0ed zvWAnsdvRDFUJGb}jG9cet@UT&;uqKe0HD8rky4S_rnc3-I%<21$^J689Um!y?jnlr@hhIMbA>vo2IXKQx3}o# zlg8v4uq1RK2z*H`fHq0Wy!@k-^SaKeN>#@H(ZAJlE-#x86r+iUQL^!f#T5%Wa9W}p z?zNPh&YuhXi)8rI>vdl-v`6*s5`;5?^^?GwmG>YojvR6oQ^d}_lCJybf)jBuj|XR>+03o8pKBi zaXMzJG!Kd6v6cg=H%U}duRp@NZ57EI+zIqPkm6ZA=AO_zWKd;rLBdo6QHwpUP4UtM zxz4Qw!zJEkMnlbd1+!NO^>{7^xZ6CEHKs>}hue|KdN)R{Uu6!$3BiYJHzJ{ixXBV` zmn=?36~>BdmEOhacL;+~PHSC11?yi4W|&V*5#!u6(>z|H<1Q>5P)xloglN6$_t(nI zCK#0ZN>uZu$>vjE*jXRt`s+#;*|s`bMb<;jqt#~5asVQd8%THN4dGo|*uv38Og1Ms zfqPOXotXu6X$F67S4c#L#bxYl*NZS*P zlj5p38e7F21uer+8nNB(CdEr`CDe1v-j*&Ra^Gn4UH0W*mKeK;ub~zw!Bu_Sa&@TH zdlJ*w>qi=F23}I9>8_8zWf>(Rdu-==AaSH((yg1|z*ts`bxxOv))6BnCkQs*vAG7% z5i4?yIqmMLt{Ig}KoYc?VM;f0Rwq%*pC!(8odyGpyH)s<4yCSwc9*+OLi#8?%BhK!9 zRw*H)7&2U=JW4NC8*V(IS~+-wnxGx#U75W8HFpu`D7_@bX!O_=a7|~QLbZ}$I~m|-3dR_~LXP@i~?(V1ar zFgBpb)eXH)0syCr{n?<}%b_4~aDnBl41sKeZ%W2WPRFtAs1*Ra6Z3u<2;-bL9baJJ zL^{Gf!p}xv3l>OnV1z_;l74WCkKJ}d6<**hMgntRD#xxqdZK12lVid@dtg~XL1{G? zm}I^)Q?09cv>PpZ8&!t)$Y9hUgWX<6!_j$HH`Qdb#(Dh_>R5}qQV-+!NP!0_c*|}?k zg5mFoxOPgXIlJ4iZ=mJsYnPD6ZLnHIc~-yy>!|{3@WJ{;Y}&QULTW63hCko@3V(`M zgv0vnjJv!~xLH+y`?lAiGZ2%)N&j5LaDG^({6*G5u@}ISuhH?yD`R@R*`3DN?vjR4 zq`P+ZP@qhO-;qMiw4yftW=z)ey-r~G1+Ux=j#eHoN^L!WXs{)oA2RsJ;R(u5%%F-7 ze5GB=rizu%H?raf&y_E!p+Qj|0d5G~|5yaA_{uO*P|Tc2$&k~>l4&fhhbi~|Ig4dY zZ6301ZW2m+a7bQQOm&Ghj-M~eUJ@-Ap;r$f{3b!raUhmzeihPNNsn7ey3Wc49iR3! zuFj7_j@^(a_@&?0vj(fDInD!gFCsbZc~6p_&!7&6`|qwjT+kwU<-kAq%pzkvESeC9 zFab9^z$R+Nd^cFl52Yx~;4!QOyA5!bwcKZxeukh`7cMPP;rmm@+&I!6%47$ox(LY?`(3M`j9DTuD4maSgpIWN^ zbE=gPsH0AGlaAM4MpFot{X;0Zzjyc0gV+Mg@6K0F_*j$P^xr^+-z^`D0G-g^1znM> z$7;)(s3y9xF>B_kdvnM|uCoO(IW7wnBPgAcH^?it4V;X82)#_sM7w#o2koEj^Zn}y zd_95Qn*V&Bk3*)e-<&*FbJ8wo@s}D%7~qg(DoHRfQOuvIqL?z6R%%!|MSgtJO$f93 zS_yokE+5@-;w&Ytyk^+WQ2O1B&O?x2Q$3AZq<2C%Q?4;)ZzI z#d#uZ!bKzM(551`WQtmMbTDr1W2|JbSa`8*9S;)HJcUFIMb(n4yyqQxOJErx3VWKq7!1Cjo&D z6;LG0M1QP-T|%(C#GQQq&dgqr|;4|9rcP!xf<*~VbyPsHfLjIGel9KFxnbg0IZ>Gej|>sgaU%QBqZ1T_ksR3ng3Ew zKy&^z>~PjBgAy4(;Yl*J09fiTgCi(Lv;TgKmBrke6+Or{RDQfA=QkR}yCX_PpgLehwQMx z1X)b~HZruoTVtIDiK%m{x=CVvlt3s|bg!#MDr{PAA3jbHlT>DxQT$W0NuF#t0id-+ z`_#q2ycuGlhpD`W8SqUFOpW57FkP4@@Lm#hBL%RFe>xEaa@W*Enfh8Qk__ONm(N-T zZ+_6}1iN9cuj$we*)8BNnUgT+1is)sAL#Pa@Wa_9-t92_Wf`vqEFYbGF5ZL<-#X&- zioB$r-Wlse;BI9YS>vZo9s%57JNxtY+ked2a>8yFWHHdRb_p==qthRZd%;tbpN}{w z%i*xztQdWuTrYUcTltz&of+BD(oXFikqf$JB70dkb|Yz#tjJ+ZjdXVr2KE+M2{I|+ zrm`Hy@u?gh*|iBBE3v?uWZH@~ zRVyNy68%m3XzPZ!2O9rCL(6ny(0NAT&F9sZ!Q(mWWJ7gLhAA_dS4(B3@`M_ z3dqic_hcq4DiEJia#ljP6~(+Nh>i)rZiE@mPDvq`;S?;f$pr*@n9AU?*COrGn+5l| zp)2@4zSa|ItwY9QQR=Uv=xaCOHVtb?PeFsdl@{26BTFEK@0pF*fPaG3^6U-5J&ZIB zSh4U;wzEaGLTZ)?6rsb9arpgtY=DLVhgJHr*074hh_X`y)KE(lBUKx{>~aIKZ-v0B2?oA?629UY7TB2jtY7(7AMYZP)cjC3|L#51^5B#1l8|RrP02)d71Pia9nz^V zR_0IBh>$f|z7p5xQh z!tCZJ?bgdk0k96mJQKYZ>aG~$;qw;!oMs&p0P*OhAyZQ;4;rl`v@HpvBr93W10k2X zJEP0Dvc>1l(5s#TQV6;hb@5M`w-&twd^D;K~0cGTPsU<}j3r?`+Auvc^Wq z;K3z*eBb*_CYanTJZ30ki6%?Us`E%_z~e?3Z6c}70n<29?K6htvHWqwlV-1MM!4z< zB2?lNvFc2PMi>MGq~^bM!dN0aVKICe(V!0QDpew5f`84E(&@1xyf$|Oe@=#pM6{NR z9iR)Q*HoDi%pIkN9EfUpL3TF%&ft`%`B%ulS00Ia6#j_LpLyLss>H6LgbjxBC-Cl zho2%ZXU(}aGY&w9wpa=0grAw#If1%@dV2!j@}5}Q!mk{qfAjkPp9|h`gEY_|8Uc5j zR5B5G4}*G2|4pmjiA{1v;)&+vya+jfQNUFU*`qHZwwUWE3MP~) z<>)nh!_pvD-}r2HHG2LLBroJz8GOh}Pj!q-%qy;BNEA#d_~1bh1iT>iPdxvoImEN7 zA&4`<1?e6y0DofHns6a6J0VE+g-55w( zt8fgcONzd!XHbRDoR#sLPPVZ3i<2xRVYTB`yFM+*uUmL&A^=9cUO564Hflhn_u?&h zlai+UCzbZ*_dJ%0%S0^$#5q*fjgmQ)DXluYvt;dCkU&fD#b@D#p&FESF$y@!#TYf3 z6J3c5@-?a!Ht(cnu8~(#BOxxg`@sCY)^!dq=gP4qvec-&8Sw!KbtdrBf0u^@{8=6X z?6j;RaqJtwKzX7T;u!8$;xg13oqmwC6K<~O8bF?i&AVQBq_mm2M|cW(ezfkl0iT%w@hmo`zK>R))yz}VP#U%clSSxyC= zN_r29fJ)jrN#4T%5_0F4DJ+}Sx2 z9Bim%ojg(SjAanxn(uJPM?{@REb~D)z1vv2H2|PI2!xXl%vDOXI@a5+l z7Q3H?T@*?=zZpn8J?jFciRF!)l5+)H8#Id~h+`5973fnR3OF3+)ptZJpXnL!g0?{!G zD_c$XWlB=n($$8LApG2?smIgH-VB|H9Tt<%<>4BDvlT|)rri-Y$XEM-pQe$G56R1B zrq&irdKyxPbVg_6MxScsXIr4aHGJjKWP;eS%3Qy3AV7(4OX?m z2fCNe=AOs6&$3^sVgsr_uFUuG7W??I3r5r%)-Ona2CfVWzi=l{o+yY%n3XP@4JLAw zLtGZJV#Pd#GRMxyNgHUcrG7*J?@cH7+^nYD!;ox{SE0>5LTx^KK8wfF(jzvjM}H3k z?g{ZTP?F-o8%cJ{pnS1Ix=B4nwbaIy8GCFW>)S|ZR?hpvT{AKzj%VMTA9A*G^^(pq zgImAoxYcu_jCyvjtO!nrI@Qbtfo3^dZLPL46y*e~Z`fj9Sl9X#ub1TIt&sa|U&}th zk^)B1j8irYkbJAD8Dgim40}#7F)Mq5>}(6LkGpbziVH}63mmu|L;xoW*Dxnl6QKkg zxD8o7;hoD9DOI4sQWhjCy1+QqJbM29$|8nOxeP3y)`A7ZI2z1ifFnGVt^@@gMrrtp zmH8w6COcw?%f0l5XUGG$>(c%>tY?3 zkL_E+pE~B6!S60UPkmDsG3pb{#wpRRRo1fSyB~h=X;~`5wGv7zk%)z_qswqmSkA|9 z?1f?I+F`V0zLla~tL-nx&YddQbS0B(?zJHrqm@G!gi2I^n!d2(+gWro7(4sZp|UD_ z070UBXjR7PL%Sh(mS_;gSBUtz4NTeLVR}UZG-WJ#@P~A0yV4eK7qb*@NuEVw#X|EyKwa6>F;9+>C>PB7Dr zA7y#;a@=J4HcT1snNBs=zl{}nTHlEye*q_EAHAODpV?&=$?^=t_ie~0A zi?fx@*<-#_U0$xMsI;^FFu7KvB*^jNe36RHFRGqL8A3N6=x^pI;ilCV47oh%kTAh} zj7*({>g&=WzS9ujsa#|tRn+GHs9YLR%?h=ARVZwQJwUCJpBW2d4UQZ~F|UF(h%p#b z$CxN|X=~@XIzmho;VT2)^RQm~oly%`;*z|Rs> z)BJdpy6}Y!!s+>*Vj#gTrO2P7KL1#6$cJgmjyy`COJRIS_i0XpsxI|Y&r@y3T5F-+ z6b*6%kJ$Kc%*7$pr-tu8qdsGy=|K@b)^|@oKTZ<{sYem=FNU=jE^QgfN><_P_4Rky z1c41LYbX8cS<>ej;)Y}DlWjfOPVQ_6-Oxpx`Cvn?BV`B?O>ebw4W4pKwbW8$p)H!) z!y8BAyS?PEBzb0@G6NnGQn${iAs->fU-pm^?)-{PMZ%F74l$lw=kBHrudz!%I68i- zB@>^hrNkT{j``yU^flZ92f_cx-dl#nwQbv?coE#)0t5&_g1giT2`<6i-6gmM3ikwe z3-0a`Jh;0AcX!R3EZJwTefE7n&bjZt@7;I#_`x@s1yytOHu~tTb@A)uv#PfO`>!98 z?yJH|C3#5=&%C0gpK!2>w|t=a@*<}zbio_u1>KHDNOF#befE$M`(?&oBTrQu5( zD-D}QbXl5etFnnI?WRadm5s1Cb$z^^e91jh*Y8ZLm>uM}hgSJ-TD^_=1FX#;5$8mS zvf&b6N)@HL2vv{JJYE(O>x)t%y2Ag9Vr<_LJiON7if=(iTIF>?4Aq4tGf^ZyM|T4r z{9fES%PJ!wBsJV7@Cm^jA5|O1mo9s!Ea(jnc>=*29`zR=T7Mc~9y8bht@=Bpxh@uz zafoTh&j1~J^Lfq91rol;IxcTJKS-bs?kFCGQ=BihuL9RL|=?Q36oTBXI%h zis6NEX=xr_Y3(Fh*Bya=Gc!KN982iyDl$FON?h?FG3)zR(Qb_;CXS5@);@L!{x_9A zJP9ZIYlWW-ASa*VUL`RG9G7(@Le&y`r!d#ROtCX|e`<_lj0+BZU4aUMl3Yg|w&YT$ zCTPcm^XArqk4pX#QkI$V$z!*=yiDw4^ZAOx6#wd3q>*1f@$I&=J<|G{OecoKLix?a zklm6`Pg){9VRixgtg!Hd*bh&8aM^$%6yfsYGlK9Y^hn-+&aJQn2@;nep+JfIdZ$=b zz}-ft44`xmGZK}T8jYXh1xn`4yjmi{Del5gi27`26%4BPmA)W!knhysc)K5n;e+*r zgU7|-ngopefHBodz187Kz%gqK;Dj)u$(WyQHcrs9s%f$bcYnhlm%Y3~;D zzOcPikEfBJz`k6Y6p7k+R$zwD7in{c&A4h=d`OeWBAF~OxXy+icGP4@c+KO|c55iz zGBtUD94xqLS-RC~g^stbPHMx=(nW5id4F2t~GO;qTg(J!=(>zv|N zAjKGV+Ll6w^}@-~G1I!u4%8ssY^W_}lL;Qa_g!_@iAs5pzn_n+YUL3;=UlR#_Svt4 zEFsTK9yx!+Z7=F(8zO;?l~3(=M}3eU)SD&3F-opr2^F*c ziv8}YtBrZRZ!t6&>VhT1V-)*hUb|FQccAkI63!13yGS6Vrf8*GfC818yr?lh&oXcV zLaJQ?l73v8F73!mMyJsBq?xD@ax?@L@Ns;G<3`Iv;+=uWF2qdls-tV^JkjqZ#XwU~Q z51(Rq%c#HQPgWk>7e+G*o=gKgI-&IEi1~b=-*Dd`Ea z0*%q77saD{MR!2l=1tv}C$-P$*jF=-tr;-CkG<=FnDdXPHnLCIR9`fbf2Ig&NCiDH z9*mHXf`~ih^oT${+u@1cNK!XDytmT(=Uvz2#~2RVy=8S$*HurHXr6+eECVBcM$NXq zA0`HD*c9lFpGrXy816O9>Q*iDPAgwPk(nwbgo$3~_`UXs=fceLF6(VH{LU2nJMFUB zkl@;T=ye8SktOW*iDf>U53-zB{JDn;ckM z;U@({<_m`+mr`)00D^!yL)s0;FQ_#fyk7r5lj=e6^%xwBi0o`b-!8?!rtY$>BC;!g z2BuK?+;cF=0u7H>v*NTzPdJ0vUo^5r?1nrb$c_+<)MFwpa{D5Sf8gq2&;A9a8Myw6 zZ|v}`py^7HX7vpxtsk?cJ@$_C@}y_#@sI1o+WmlP{IY_@qlRVNC|s9Z*X+;sg$fO_ zdM(8&bzrFykm=3~Z%mJDq2T7&vEDS3s*vIrPpt z}>#IM}45QqWB6;KLNQ5xOeAYU=Z^X$M$Wuyj^=vVhCh>mcBSJuZpLWf^=TDc)6slKdzx!-dtYord@e=foCVS zCrFxaK4|nJQd{wRT1jPJ*OPIBk0x8pY9C1Nqg?KA-Nszwk6ApNDI(cnE?-6Pof-7n zqqT;SwW85n!}icoq*4obHuIefGL!Z(Dzvz|2ql{If!nW^~ger8sfmqr_0jT4(S1q@M7QH?Dff}Xa|HQ<OC zB`-lH?zH=!E=U%WC}o;Crx#7;={ zIsOAWX_Qpx{ohaM{%;t&E?cj$y6(PZ6EJAB@^7Xd=5=YZWzm_0E1bned9qA}lMmB; zm+<*6Aa6zsg{!y?3-v=>xsTN%9d`=DVR97Z|1tjmosA#CGxjTMtN@j*l3*s&kA~MF z%>;HCUHFzC;*otZJjulrsLgPA%#Bz-#@u-n!Es}aY8bz>s)K?oWZzZf*ghtr_eeNr z@eKa2z($$LUyeZczvl>a*#;_dQ8&ow59nRX&)~Pmt@sqKm03Fbiy!)ja?&*Vw(x>+ zUCwI=#%a+zJwG7pv18&6*{Hh*vC~g98F#k!5$#(fTmKXWN^%ig<2;iArOcD71G0Np zv~Tp0Z*G|qJ(-_s;ZXWv%Q-^M^W_$Li7VF;=^DlD(GS5DRUO210*aNM*riOM$b9J9 zC7&HNOT~tX?2*Q-29tAg9ZsX*_27s=e|{^5iGAZi3bD)L5hd0bDz>LM`2=3VaQ z6Ge}0`E4Of1#S~oH+S?A2l`Zq^lGE?|HNfPIc-iQrSPXYlC66qsQm>if_ z%FiGOT)+Q~5g8?X5CxpO?u)+eX- zb#EV`OC0Q4xXYI6m>c`-RdBS7Q?)7Nc1q&JkC2S{k}*|iTt0D~17=I*8?evUA>sQ6 zJv#tKzyS!^ZU+#7Owb=WyiaECt4Zb>uB{Wj6I30(qrz6bnHYL4H0z1n-bGU1HYM{3 z==$Ag5;FOMZ?CrL5&xx2CTxLV(Oc&Wt;ACagrAX$bI6Y*h0AZEEF-~@>Y=o)sNOgJ z@D+c01t_1=pWw_iE80S-*kkc@L~vF$;PQQ^Onu_92*Y!#ft2T)$y3n)`nOVekP)(8 zO<+)&joolnFq6ZwbWm*A%n*LszGaO@@9 zJXEL2kF#?QIU5`#JyLj$V^Nui9 zyxqaIN3%jn#FSxTyI5OQI7O8YS8~|hA zz%5#4^Wc;wJj{8!dP->-EZcYn=B5nyb&r?af3WW3#vXCRX#*8zuv{nia5lK;9R@usW-5YSxKlYg$g~liaN)SeCm#+57;t;fwfb~t;}4p+ zRGIG2a;|C?A?I`vt(#B29XuwN-%;9szopaMeY)t&KZ-mV$9?=GlC+Qa?ycJmck&`SL5A@FU#87O-C>KizZ`qCx_?wwle1jt_(U6p+KCg#(epN&q^e8eECv2eGi)`An z2<`B=d8M7R{;KtfE#f!Wxjh7!#KZI22&IR$l}II^4@4RlX!+Z)75sKQjNG?QeTc9h z=PO*7j_>7+Zpz*@ejx2?;E)Xr-J-g!37fTB)P)|+DCb=dhmRWTeNBC)kc1t1BI2*l zXswP%Waz+!M_|yH<)lin*`BpHqD5T~E}0Y)B0`CCsL2)~A1+k!qv$*JF0kBv`WiDr zn2*oeLOmY+STEPi03Q0MxQUP3dl$$f4QN}+*cuvX`ZVy+(V2#5N}8yhW+wjBxucCh zvDy7p=#Y2EF~P5#A-J;oeyIj&%Q_edk>i4M%k9(I*;GE&gc!P^xJ{}F zoLB&PL3E5Ee{@?wghO?PK}9(?6OcD7bCbIH4HY-zxdDgpmgVN6_~KEC!TiHQfhtcW z`d4h*#=3-sOGlTcd@d$aSKa9^Tm6nDEwvI_Ap7`t1IqelAj!4ezA6}_q^KoEUFn@%}fP#>0?=9PKCi0d7^_JiJ)? z*Izd)^xjQ#{Tue8?VGFmxJ_7I#8SGxjrs(hv)Ebd%(($y9VC;BLgp(l#{R`|7R5V} z{v~-d)-+HHh6YJJL?*7E*N}*MeGg{~GMwFIU6fiE4F7SM%9DbAWIxNi_UI<+ede^^ zPp|rk<_tMbWMVWC_(&p@z!#q#B+=t9b&Si=wx<5%GsK{)fl_1ZuX& zZH~`co-g&}D(wsVj{8PXD^~}0I1|s-h+;8<8^%G|_aSdZC_M5h>6vR4DJlEWq2yK(l%I@dEc|F`H_;; zHC^Q9jldegOS}qs=JbfzU_AZP^Qp zT$d8z=++{OwlRDB5Debvy(*CR`?++S3~R(o>8Q4`{*tu@guV-c%lu0BpEJN&NKOL> zaOl9H5d)Fv2#$zwYuxSI5ABP@6F;Ii&lkCtvaHps7^^?z1Sp0}&CJgSze;M})@lKg zZDR0@J30vVteWkv<=oiyJWM2*_X6K{);SbPIA@2#Y=slAYgU`ik4Xbj;rzqDM1|OG zp7%vF7e~&TU(}#gLQImlLG(F%F;~R%-F~ES_6?=;%kcxV-XA@_{3{s(BP(FmI?Bm8 zpsxSJ(}dfnP<-CU(q)vReeWx3czu)(N$Qlf@ECa3@)!x}d&-B~dli1(k`s#1cBn-P zMHqB%#H)`PWSIOw-^FJ+eI{K?S#SKu%4l25KdI^NVjKYaps~!Kg>*g7CD;|>)fYYS zBSQ89+mX`(G zGp344Y{9>vPW}(t(7Jk`=tz=p%tarbv35CPPyUk*nEtrE*`}216@&jkMt<`ij)z({ zBRFUTv8Wqx=X9@7hV2M_vP7M%$!}gFC;WX>|A)Zk z`I+3)3x`k}p<|nH8-cXLG8xOdP=-}iil+|avfc`qkT-*{+UdudjIL+EHWm<1`NMz& z* zHmey_lQJEJh9!sQHx%@zQs<&jn~Hh%X`Sw>W!Ao~>fF3AI$b){?EgaT!?f2wglgyy zw8KEdaBBsAfj{{%JaflP)l5f!_0R33o*^ZZ;k?iPT2{bxIq7`85}vum+il8DNC53> z)JA#!D$2a0Nsi**gQU&Q;4@wVRh&PdnQy>rza4oH6jB*BRHP^de zq~Q?HOuU=E&bp3{GHU}bFPaKm@Y8YT4AC8$7Cw-ow`08LACA=UX1gEe^2_M>gmw`6 zpZIx`fAI4?>4FKH+a0XHR6;7V^?h?A@|O0~4s@rSed)TN$BU5O(W2?l!8X^>^#(ZC z4EzEHyOn4%z9`2F-OCd@u6i;SFx>9T{lnFj06mu@uzBkK!1sp%L0-gPP>k+4W>zQF z$2Ud@#KA45f8k0Qz*Xg6xT>@EiV-+Qf1vaw{(2czLEdrd4a`M)@s6#DVkdG1@G##I zSqnj>j|nSDZ#`bYN%{!bXi>Kk=ikT~PNbF)Q9L=yH@`#>4Skd}k-dn?br}+Fh~1@$ zNk8oHU)4vmT)YpJY8zSPSq<=+>-+^Jt@FU~bO_?K_FTx-*}@|^c_)0(e{n7$dqw`+ z2@8mc`BEWOXsk4Gtp-NsuA~$gCf}z`C%h+HI7s@4Z|ouOQ%^O;`5Q1OoxUz)gP3a6 zU^?r^F+}I-CD;;J;y)agMLmY8NwWA}^egvqlMW2i5-3P)Mm@x{85(|K)EQdu!6Zj< zaF`-Cl;mTM$N?l_ZMycY)4=H&#Zq+m8a5Cj@m<&`m3G^pV0j*OZ(n;Hm9l35@sN8_M7*Fw25Zy*`V+c`jUO$Mro&KLfhx-RfdRy7 zMpfhf4RAEVho!Z z2>^IdbEI`NpZA>+3Q=KqVuUB6f0xPJ)lGeN+&%`)yCG&wI6-{x^xJ>l)9Gx2Ysk#F ze?dV66z30jKM5@Qz$pxFFAV-1g02syTexuo)B#^&Hf+|;! z%rhAF+wH%gB8`)5tTPUt8qtQ+`OMW^t5hDc8QYfpu$a>D_=lvj@fU{wh)L%LQ>XYe zU#~O(>X}jtV#)y5!f@82w^|?75LD!u0*P%zSiEvp%oa0r>=jFIlQ_<1ZFW$p(&@4D$oYE>Ha=6sqJ;V1ip?{X} zvdE%9dL|-k5^Ca**Pxc?Hsbekk{kfUQ2ZmQ)Q>^9jlo=-;24?Od5#@lZZ!HF;!|Wl zLOd2Iy_6kTyK$LBgWf5CXi}ZZG81*E-u6f8jI_Y8H(vfh%4+SD;`)aa7f?1;%HDG*QcWYv>Ejg57#=nvlL_@cn~lHi zrKqL*6YLmozvzmW^Ftz{+xC)f^)bTt;@;<@>Rso6GC$z=;AGc-G8A~){ezbhMJVMS zP+n0@vXs<_VYWE?Q?xt$L$s?JzTCfa=BxbABF4fVcK>0Mn+~(h0rOX)_jkV6_pzSK z99@qFR`W5i4GS9O$gw3Ay4*G}KaYM#InkQq3sh<~z?bf$_+|E=-h~L7nzty%#`0WL*|7p;%Ai9LgYQl4DM-&S}y7*lGv8ZT0$aE&M!4AdD}5 zyfWb~@-0g&jYaAZ%iS4xmY0V+U zFlv`GQU{+5%)bjROK411cT@wJZL9I8jp&a2dZKRY)ba}cp1vkDyeuVD*vqcx>6g%i zRCSI&ye7u_WNkWxUnR@GD$K>01Lo|-(W|nqmi9+z=E-Q}7N4xdQemU3QP%fRlMI?u zcqQ#sZdr45sRUdfs?u_(EjqdX*2ff87k z-2LzKnY810szgSbbJIF!?LQdDvk@tkC*#=^*OPV;XoqsozoTTA@18}*?C&mA+&epc z`8|^Ms@|yAxaJ1K;Ero%v(f@j|HlFN#U%lUT3^M_PqgLej_q9FZeNS?x4rD)b7MR& z#^{J@v8?Xx_DX$jqc!7tQpSErt>tlyPa3l?t8*>5?azUjzRrdy@EV^~OP~(fMFW$oI9H|2?x-BwEH^ zm`hoJQ5~0BSLj1NH!B2A@$5VaTX|?NZ1TlC^-omGYVz7pEICoalCL#`?n9(G+qg_# z>hcgq!5=|itKiPhh}Oi&>0jjB*c=S9)gEDaQ5=x;&GWM*MuJ+A4p( zKp;vgHuH|V4Q3`HAu=Y-?T$-|55DRvbr6Qn=Sy^mcho5iivuZ$(`8nmkxq^x55_H+bT5MI;7FRJ#2pNnOQic7_9ILuP?*Y*Zh(eU);w^h7>U~ zNn}$(f~eOb+JoVCF+^TL9g%WV#LjC)ix;ez`Rfegu*3Bku|-Juk8 zvIf~mfsjIM1+0&mE6T|FY2SptfXZ01Wf($Zct``?94KktYMIM8Wjxk~&C(X4Y=ZZc zFFtkS(lZIfK~%q56eU)?4$TQIGnNEXG!nO^Sppdprk5cs@NC>ag^R?#J24MRSW|yx zVZKbj>drY?dRVs#Lb1snrtQX>JDcOJj)9F}VWiKK*^a;K;*r*JYHM_KqS)wl6I}C9 zT%r&uQU9PbbS%NEHDYrm1m4@nq%KN4D-Ei_K+1eG1R1Ro}~0Q1A4<0=Ojy;+Q3Lo4bHvKU2-qtE4_HnBX5| zW@+U7I_Irl&GDqvcPL(}wj@iR2_mxf`kKO+kt8Hl!(kaV_;TKRnj{AOd%qV{nakX< zC8Ngr`t$~weH#~=gc!(66S8oK*|$O1XgdxOX1M$fksLv06`6Vm#irgSC*YtMh$ZzldLt{A61s?UP03#zDb0ain=os%N8lK{F2%x?r;xJ->u?oPZPo*#_ev{@I%`w^3VI)16RqqWYrno&F(Jab0V)B|cmCc!?-a+6C zCo=cKw0Zyi{wm#&xpL>FLyvQY2oh=(Q$C%9s6VRA zS}jAk>(I$zppL06+pwTiS-ULASbZ2PvhInjVc>~h9AzL36Gu=(?H{?7t_sKDQ3CnD zXaes7VUSgPUz`_zQ=LDl$1s95A`hwRjZTL3!EZ2QybVq>Ws4Q;gc2k!i^>BjF1} zxoV--xm_p!AtAIh{9)n#KbEbZ4ctjgMObMqaVrA4zd&YFAm9pUsSRc*#jBgex)%C3 z$^ooW&b>M!F=Km8E`xdP)(~tx1`+TUA`BeGYZoY5)i^C882JF3=o5dE$QzcCDoJ$45{Iszama6|c=y_fbpl1m@P|flBtQ4Q* z0Cgd579$aTIIWkD`$}C87tivqhQ&O%253+b5(l8qiXyP6#Z>%h4A$J?@*)1vtXCu2i{#_@iobLWNtfY@-eW7brC3ICP&J| zCkVXY;sD9@nl0G1JI}#M7P&*W1qQ=%{m$WoD+g;1;QjCWD24?f|DZ{p2e{*v!8yIj zz8^=)ek1|`r2g-V3r;o~F-EevY(``hMHJ315&mn<9lYjOK4GW@%SycR+-!v-^=Y=J zu>VW*`*Q`^t7ii-9}b(MR2D@;!WAUKScKy-8WCg+RNwpEpLcv1mHWW6J-2MqqUuXZ z>cwSBnMlQ@3nDx0qo$k}bCYSkZzfl|C$o);H?mJQtxE;x3o|DKMld5qNe&;Dk4 z`pqG3|ME%vDgF=Czv_jE*1Rfe4<(>i`HM94JD_36%FifTd3fdlHt897TK{W+H&Lgm zVO_A$3d;Yyv5X%%6I!!^fR24!Mw-iZS*kXf;3oN*>^u;qKCAxq6qGFBK= zJXr~;33EHcJfkd0>F=(FW-i5yj8w*JkYOQ8xM$&oQ!NfiB5!krBkV0V|Lf@^x^?#X zPW_DN{^acjj`7YG-uMeqY?ciN-=&}GeBkp}5pf*zasNadn$3d7kuNgK>s^tu=DK!B zIah=T+VkK43*f*HhIe1*ER9|1vkd+{mh_)rS0rp=K~(`~y?8$i=GhkqtDAJ+B^v}=4v4RGJ3i~N z2UUVTgu#;5*Dtj5?W~>|x9q*GS*9yW+y}kcqS@rE22#Lngqsm_)wIcIX^5tjL=(5jJxpZIJtjw zomm@c`EuvLUr;71;XzskG|;g)Yr`XuvHliu_((EBU!okNK)97qRh`&xmbNQ<+03Yt z3TVH98>_zsD+CZ0FMDuslD954nDrNOWOi5aM7;G$jwKlShUOdR5OF$YRqEO#b zfjvH6#*Qo+CK^8vo?uDNVvGnh3G8?DP6G_ZQ32x zLlCT3F=UtjdW?Jnye-A^)oze*ruKs_a)JVFYBL%iX+G~GG1ZMn-AG;k=bbt7&dpZO zRz=W(U~Gzo z&)Gd92WpgG#~tr@DWc*&2!7~)WLjeJAf8ipC zvn}v;pIIl@vGYj_Ju((PHoB``cl?#39S8NJQO0sMa_ADp!R@zBSTEXdI;#ZQil8J* zmcz6!cRl#A&|C2+?#AD|D*QKvU2{62G}p}Ew_^Hlf>&bl6$(*;_8dPbXY~*Zj8~xs z5xF;h%Qysl=kHt0PvL;O5(-Vy$1V~({UBRP-a|A4ao(dT{Ga$>+0!)Sp8B%4CPgsm z%=EJXL;hnW6wsCTH*)s64E5dYDGOPXc}5xUVgiYd9E{c1&tM=6Lxb2_rCSe!mpqlj z@jFhw`a>Lk9wLP^O6*FQ$#J2)C9i?*kH4Tq3|V_|HHyFDlgiEwF5D~S7E(H=A)Im; z;mF!~?4Oz-Q-hDUT$MO>c10mI*R+kIsa_2CKgjdg`jaj$($HoW({`P98!SRzo#Oa) zDfiVxZK+<9MHu%$s*fc~7+?TZ8v@eZ@Likac>A+H=MUAxR+UxwG2f7I;{GjrCKSH*C8 zj2oe!o~_tc2>S&E7gqIeiRo)XxGLpNZnp9@`^T{uis~hkjJVj8t(Lb9k6)A z?NtYIevf=slviB52V)r#VG+}bJuoxxsYwfeKi`shYK)+LWt<9@jW?e<72+e6xf<0` zQFn?mYq<)=^>a$W#6o1oqxoJq(I8fpYGt6kku9ATd`tDT_A;&v^0Sf*zNm zJ%6y>HUvBwwrXd!FvH~^kD|JCkgcN37iCwVWI4l|!~JsQB+u%xVAacPs_DLBk=I1A zEw{?g*7NHzy^Go|hxOBeGg{&#(2iBc^YxSBN!7Ud(&OwA>nr(^VKQRHltu4Nrn7Gg zGng?n$;ikOS071-65AtLb}aSia``{Ly+L6LFLg|<3%8!3&n;?}YHo^DDhQn)!#M~R zE=f%%6-!9~$`?I~t`2?_zvoS1wHF`hi7x#yNy&+FG1ydwBJ&-yQxuVCMy~QyH)jd;DtPe#Zm>~C`K@x4PU*S(NJf-?$S`FqVF>LnI&WTtsL9nG7(DE$<)i?wzVB? zkr!JwDDJA?yh{Ba=y3D~Sxg1(K{LYOaWBfg!W25~fu0UAw!=&L+(|yPZnKL;njZ+(hsz&39W?76>8t2v8MEpNszs3S}vpgBx7_Pt@D zry$UGgnv@s4!A0jHvzawp!LiLI_v4RKW_96MgaOch1Vd3O9CGF)rV|0{u@U#+vVVh zowz-2CMSx;y+03usD0Q5SscT&j59jAV1=sCxAZLg6ROJK9=H^2v!Fre1({QQE{e~C zi*vfw2c2uE+W>AM0t4a&`H9=Y2D#aSFT$sO`y>yZ^`#vjce6)`SCD&^mrDW0dOQ4$ zZrGIkrMl-`afx#aZss(O#wzb+m)+0ejv_@y2OZ@U$7`}++qy&5!V+VP@RF_Jj-Igd zT)rTBrMC?9RS`!Q3J-amU&EE05oIX=LN=D+%PT^BfIwu?kycqyu!8?mIFLHE{ZaK8=;6;Ko5 zX{z)W6C6m#B^-e}1dG7@Ja>;kK9Dw#u6A~rQ2$s>Q=MlPNv`fpDYLZKHG|vLRqGBv z%s)6t^I`>o+~0Fmz{3Go6DNMPp4_Dth z3EztavmOj>*Y&x6O<2>8f{8l{pF|8ff*StnVAhAZ0*GR^l6fD*Kv-H1KthaULWfP1 zS`RwBy7tN}xY6&A=tuHW*DibquU8cf3utB`H694|VG+)HkNF0D&w$@MlY}B8|1_&B zlKZBLMe`<-YsRi+)Hd$mWfKtqi#H?}o!^eOe?c|CN#{&>uX=Khm4AZc$k%66)Yxl5 z&m3ceh;$t%Dq|6kK-ao%EmRk`qw zoEDYnZ08--U@MAr)L3l~JA_{)KNKD2;Eed>Qe_Da`lnSZ;VPQpaq1MNoTFW<)o5QSN%*nh24H6f@58#l2W?n z+F$(BB9Df&1fyCF6vsHZV4&6bfJg%28z39zc5W52N6eN?TjUu&38*2F9oO3_LAiV- zP@uH!xE`x-;L(J@9dk?<1u5d&WnXD-YOhiiDmiLvkX>RwR<4>6ioY(aQ}(h>%Qwun zL)UCrw-@{Nw;k!fKx1eRzqEWR;tDST1f(#8>F90%dvnj<816TsmHtquZN^m{S!r%# zEgaiJJ-YHo3~u<&JvAP(!@UYE)x=ZNE#HVces5QT2#1Mg;|3PHUh{n+WI+dAPk{jx z!J0c$Qq2d9P(XG}S6^C0IH23~)Fr29PfVUo3+)$q)p%QXSX;J9^CgYI*4p{&`dMHw zSNWY#pr#C9aLm=TOycv=n9ErImbmwAKa7rLkra>Z=w-Fn#>1Rt*cjVaeFzzftC#|Y zK4KIls;Ldgz-Lj(AywMUKa0)Y7TYQfmnYZYrPr!_ew%Fu^CBJcHBwloP3=!DHlMTg zAx-kr8WmaStbUgqvFWV8Ea4gR;Gtnny+BPXg?KQ}hd!>FmS zE}ug^XVeBolN#)l53VFTGNvs@+*8@FQ{qx8m)9w~e?$Dk4uQYr^Wg%GvGuSC!xQnG zfQc2E2eoc0=kU>=FnnUAUeWil!nv@jGPlT1w#u8$aS8VY* z*@lg-NC~$ImT3+=>$*CH4la`nnGMlwY;37Yt1Lv+#@`be0w{S`AwhN&-;b@jVB6*J ztP7ydD7aX+O&o%OIw`7$yVdFNHl3AIir|Ze!_JgrM2nAIk2GW!7^vTh`d#;(bD$=&Bz>(yXF1+n8yj3OA7&qTW_~}N0Y7Hy#NU=jLU&EF6t2B zf32nAFBQ7-XbbvJafBS&kk$Rw8KYGw2`$jODh6TrtxqWh`{D^eW%GC!Jca7nnYhTQ zgw1p&_`37cN3$xL75s3nsSQTLWL)a;36%OF_{x|xu|pOl2Epo9aZLnr4gq;meAwLDZE zw)2h$ISJUo00y+S#2@1b?QpQ4@VF4h_R86RE1k&h`{2)qon}7J66e}xC#6!r6ZwX| z0*@_Q)IV_l;`Po&LZC8^^Y>SyGKUYSNENI43j(*<@)oI~B)7YCHrx6}pSkp=ou&$w z*^53{+Tf~b=rHV-rbc5cTHf;PHmdi`h7In9rLW5Uf~v*`#m>{7UU@4ruFXU`iAos) ztV-ZxMw*Igdhgt$oJ-C6(w7NBp$?#G+m!atTgez*t_?r+h|WoO!q#t`HN6)U9N2?& zACn#UeEizrk`zEPVoo&DVxpe$y_sW^N$@|o{uY;m@HBTaEo$tLkM}U7(Ms{$G zUU1Fl{VE~y(&8a_Ga%%_Dt!tRsx2^gM6Ph;^d!kbzBhA=J%EaPvmjK6j@Rs7wJ4ak zc=!;fFEmwlMjJ=;=CsyZ@%bZ|)O*S{EZj15c^Bzy$TG02tXl%_R{iv7#P%-Tq3%o# zX#A<%)|aMgTGsRDqexr56x_DisKR_I@&3YJKBg01b(e8!(c2`@x#u~&^2323fqUB5VT-KqAyC;ah z+tfOkIWIqfXDeAPI5q^B zrr;u@{)ydaH0-YKo(rClMEZ)LML_9_OFLudlE{9*&6}V;rw#VT9QOF3@F21NDB(Ur zRbBSeN-)vB3UN(e9UjecpBQ>rdTO_;{4S5a_ZNBg-uEUCj_FfK((^0+tS0~QoZk!nxn`07d#(S+>?3EZTBS9%Ur-A=8$g@3 ztTxiW%q)c6dw8PS$e5~pt}+R*!SpxPD(l71@LO4mySV+TDukf{tv`e@*2GARv&d^ zdcnk5VZTu%S*@H9!tXZ1G)dciOrrO5%~BLE6hftFR*7kW?cYO!Ct0Oc;#iBlr~yBM zQNPA1NVIAtU4mI@oH%0hF2J-p*D2$nEHKnuU3hT%lsas(;u8q3kqyxR{%(57Fy=%3 zF^azU;8GWxydvT3*_d%?(kWb&D~7(8L4``q+)4r+I?HU){KfGg4k%$!8JY@R7V8Kb zZVtgBUeyFs!bBQD-#VD<4Ko_Y3hqge^$|OUcDz8ng6Ar#Id}TE&M|f3JqdUDqT&5s z`mUZ^CltHa%O78Yl%2=T+FYG|jLW?>mb~rL8_kwWOiI0(k9DkzAC?59*q4>W)$?w#xs{z7yzeudKD_^a+O1+SVGcwmW{% znjU7^Brm-FXUsob44e~McglfJpot(iJp4p|fp`JQmv53X?ZLFH?T4lx5elrAsd4w& zPmN#YY-`DXNL5da`Dy@ElzY}=hzeS>1jD6>^R8?KC1Y?KG#l9{I;3-dK6eglsTlTH zL$^U**HmLmxKL%oy{$hX=j>^S2rfVTJr6MI{3mpRV%E4*zoYXi*WaVf8;HBlAE^)& z7$Q6^yYN&iA&AtE#Tf`cKMT(s1neV|DoojC3FKQ*Mt@BGcFtYyx5;UARwaPez7xjD zH+}AK;pUd6-uviiGaa}P$&%XgahQCZc|vBZs{NSDWnh71d-x3;_dY>9@|t+hwTN^+ zX8k@P#{vuW=3{8X)+F8=(2NDPI`xeC1P9`nx!mnmvHej&?W)Y151LAo2F?1a&|MA? zzj>NVmqPoLP1U3A@sYcFvn<^aOvxM=59GhHt0W_>m2Y8(AYnm1TGxuS2YM z!}jw0KlXSzvZ+WeaXLPI+UN@pg$7cbX9A9DhWLHDTino9?vB)=Lhn{`z}5uq$z6=EG`(8)cF{t;IAZ z8{r^0;AQIwOjxF;Tkr_!;w_F5LW@inzNbAQsZ;$;ZH@+AY6L*Iaio6a*SkRWrK~s1 z#j;9T9ZvvRh%UpQd*dXe$e(h!PkF9T*|`?lx%bpv(%)A)U~bJgA0WfVGN9!yeU)}T zeQHmXPgUC`I$nSD`Nw0zrkDP2Tw-c^x?91JD&R2r=Fp4TP}0xg4FU zFEV|=Z<1a5Qv-@GLeucoIEkvpC6DHnve@MH4T7b=QvkD+zEdW~?&cahS9nMV=e%8F zJn||w+veJryPmyPu`%WJSyML%Rxo+bhDZ2h{-kN27RPz*Qdtp}iy@lqp-4m7s@=dw zi+aSoEw|=KQC8%F7P;!i6F6p9KY=qQ1(2@n7bC;!S~V?BHC|djykAuaQWS4v#uh3W z3=h0e;eHpiA9`96-qBex%+~$YwKM2dPEnQo_~@3HA)U?aCFnS;DFNKf z)^B+=^vy@j_Qg^*@w&ZOshe0BjwYTo$=C5Ncov|`mYM6<_qnNi({9^mPnTDFBSRBY zC^6PA_v87k#c_Q8xh`+NcxvyH79wkhd9(nYgx)j~-^S8McI#?hwbUsckj{{zIaU}VTzV7u>EK9tcU zrNQ|c4$&0*ij#dXzfj6nx<_&BOV?S&O#`!QTp?E$o)GgZ+#u@(AOz}mDmAU+Mpz?H zTz%FvDat2axo*5W*7l9CH>7h=x+ZR1XR(*7z%+dPSIp~g5J#zzL<`TGvEZdu%x#sI zl15$tB4&3__Ey7pw;w4h84hcs11*-velz<*?7?{3%+O3vp??{X>0qP^eMK^3h3iLx z4#QvHd%O+mx2ws#DCd6bQBrDt#$p z-M%}NnObO)LtEP`G?DRQo5FlXNbJiy&9iz(XPVx?#n3Ho)HR?xOumj7%;{7)HfASx zF*OdH$?&AX2gW@5E*}%lnW*JctJefTYtwx)T3Vtv0eqUFoA14vCL^wuD2}BRs#WPN z_-+M-fzlZA3w}L4?_#}T9XFfgC@04tnTO1obD(#V>;oRtGbdQvnVRVrfcs#d%UAUK z;>GO%b1aXq^9HD!ddfxmWq3}l;8fCCzD~w;oK|G)(awVvjn*7l!qWY`$$+DMw2?P= z4b68yq}?Xl^v4YEOcvW0^e&ryvOmSbB;Reser1mTPt|j^I*)z3|GU~b><>p}4$=Rm z3xKC+izGsF$Pb4FK(zw3E(s+xG0UuR0hIIcxDKEPk$M@3tS?H>_{H~Iie&h?+8O|;T%p^}n zryrXc+o7?hPK&dEGqyk-)VGkgJ6HMmF!H$`oJ!Kx$fHQXw;9i3JX`YnAyY^t(i$R@ zLoJ<8Mq%c`uciK~SFkUAslM;jD8{18?4W=-G;%Y6mGPOc6(b4@R&9d&F`G&gsJB*{ z^uTTM$!CfUASatSmM*_O=hmvHFOPhSsJg!uxw>q`D|8UMlRmbNY;0*LXD_f2A6JT1 z^BPIo8bl_mGyNM_Oy+;czVT#(e9uMlV0|w}2^uT4;a~&q)O7ixDs>Wp(ouCV+!b=G zE%rTUud?m4;!l@$<6U9LGgrf#T**Bk;argMmMmHj%~4q9+lg*@2pUba5jDbfbOpym z=2C3Um7sav>^x6f-Nw$QIdVbuLHeD;>r4L%cY$c774gS8QcRJgOfB=C6(%#fdb%@{ z7FJXmDgzOlvBMZYZ0gI_4;=2JI~$%1kGjEIf?jFR3}7vo*DYf9D)YlqH>!FGz&sH+n=znHKlQ{N z8!^^xd`#5+?=j0-v_*<5n|Xv6Z7BwJjV-=pFi6qm{!e19!>)m#0Sp`mhZfmd3)pnj zlfj`iikq+Iq|;bUr7b+75S{hf<^>1q11${-IT&oXUvUa-# z2^r_`nsQf*gG+767MwK?UQhT?ID5gl!3362)+}12sbxh~C7~3xRyh^IjDq|c(PG+d zcc--=j{S9=rJ!N9)HbrZx-i4AZ!b2nEv~)|IWw8*WW|2SwfGR+tIo<4@vd_D)|T?x z{_$pC=q46Lwr=42svK_a;td`C(M@Dmv4M$QU5V&?aWaq;V*5&Io$4Qc8nqi$3=v3; zS=Zg#hAi;CbU`3txyQZwO>bKr#T3zSYZ$LZrCaAiFoyve!j$&Gd%{3`IOE4hD_YPJ zLlcI2{+J`}hSxLwIJTdBC)!~?pE9S4kDCt%tN7cELTD8|lU~TcGfp5*i$=?Or3{F3 zllj6uh-Zo3cGl4K^jwLwh^nT5dSiIG;bGIm*Z+?kc9k3V*bAj>dn?Vv^2@61aB1S@ z4Oh&jH@!NZsIiHCypS+SV<&WFIQVt#dm(?%CG~`+GeT;_a`N6`Q3S+FOj(djTMgp6 zkU$3(YJ1@nIMd#nS7t&|U{@QNzX+kwdZI(4B)-9sDoeX!{_8E;gl49*%~Nju-eR!+ z<--8`FT!7b#vP94ZX2XO?`TW<^U8diWR!W{=?V!*3A`_-*MMem5s+N;nChx3b{Dt* zoT*IVl_#|3@e`)oUjAdkqud@DZGqROqiSGytiC~fIX8zLR2Uk-MIR}+V3 zPVCGOh=h^UnfI{P&bA*bH%OCe4v4v(Ze%3yUowhkRAAftB_yU0T2V4yN}%L8Pfc%! zjF54>fw;2A*?Z%&jZHIN=M4Xu|Hu1p01SkhXeSu^_an6H(!T@&6&v>TA0fYsDbOnr zSgH0V94PA+g~{1L#Fk#u0-O^`F-65JezJ))9*>iTsXoIu&>4H-8>0ZS$ZIfPOY6Cw zdcV&h3CXdK8wBnVO*fXnk1TU|vo1ER&m)};5zISLmuK>bj1mM7St7Swk&E2+TQd;8 zQ85ck|4(F})ZZ2eKYc6P?Y-UDU~h~w*)Cs2N7D$8m@hs+xCjc%DF|fJITY_ksB3G& zLEo4#q&0(y+`Vl}X!NZ(b2%<|tZcv)E+WBRXe(fFdd#ZC`&oPPE%;}fk|QTZ5f@rc zR#u#wxv1j5w-USisystr!R3aFE}zLysyfCX^)p}J_zum*YnL;%kEVyb3M%I%xo+TM z47vzu$MC)IQ0;azwlp1$O)?A*s%{gEr-x{lEm2pahqEPnmsHVLi^o8rBT}|oi2zkr z_E%~K)l=8$DbLHAVi731b4dvY{eZ}aJq;l(vUq~CQ|7GbJ4T_W=Uv%iH_z27Ua$QJq`c{$!X@&a(GAZjV zl&vzf6O$7&692`FT?$8Gtjx5|kB%LEPB=_RgOy~cxyudENs-L<>Mw?^AYdIWQe+vZriMxh`z?TvaovQ#lH5c>KAM| zI#j``rI40^b~tjed=U#$x9a8+zF&YC&wQ)yWSSxc`YuH}mgiQV zdx0?qe7vw{#`kau_u#YtL!qtrEIYUvE$x;2%!M&VE2VX-?YMO{QJ!7O;yKi~g~I@R zdy8(((8arDGSj_pEd8-oeg>ufB!8m!MgB*R;LA6N9&gvTk4_;E9L3`v9l5?({R1Vi zO%=DiqNU#}{goMI&7>xjA4`n&+o4C+sf8Waiw*w$^MBu1S)lz>(&oJjccsv_CP|OC z6ciOU=B2xFl%R20WHMMZEIeIU+%O{1tww*T{CTon+@3%Ps-wNxhESthTUUEOG_B!` z(L^#v+haBpl!39tj6P;g@dpY(Z6@oKfL#+_BTY8CIw+Ut!fYxJv7@JAAKxbH)ZEBC z*)6fOp%FbSbGl~^am#p|orcC$-Vy_A^zPu$OapN?R4C_{#m$tPlH8l^t)G3!yXc@^ zWI@B6!W(1EIaeXMI%Y-b^1`2>*xmjX=4IF4q2o=LCR*PqI$(R?AGX9CYas1*aCUcd zp4?OwO-GH+{6wzXwbMOs&etSC*#361$5`^J<%iIEn~xNIXQOdI)*~iTv$tr+r0V7O z;q5DAug4Z7gjbhJ+OWLWSLnSqA{Ae{H!r3L4nXPyetxf@zco5im1uoB&ds7YR|^2o zkZa1nzBv6cp!|~FdnWvVT4!tBn|y0xFO&XoAHMWZEFn?0l+xE7(4lE{ z7{{S5%G%jE3$|^17T?EYr@24OFbd5zHDTbmT2USsNYrQcAi`2vFNsCh+j$&<$dCP- z7ygqj(P)vq?+x8LfUhy8S4UIAt9WQ(6I-%|?OG`x-aNgBZE-f|gQIBvgx{P~gLv-S zt#+@g{o6EW)HRv@jjE_WPyn`vY>|PWk&#mUfuLQtkQeiEX^s@O1yMgCc0(LXnKMB6 zl5}_H22cjp8#(kn$@2hd#PYM0S;@sDVLO+z%tHGP;+3{8Fv${@6 zSs$ug`b0AyJyA>{`520C)NkOQk{}ckUmA$tq~Q=g>owvkrYzJq6PHo;X^6C@Hm2B8 z{lTUykvS4xiyX!<c`luzUr)`4Axk%{6V+bo4 zzPO$WUexMr{GU6E5_s%w;I)B~4Y>pAX0%pc}r+#HbMT(wFt2&)$NoZOBSYTcGbrk60xEgQ`@pEilrHT-Qz;R1W~XQER}rfnnUrP2KA5{bGDm|ImLl65Yb6)TiAbo4#V$*0 zoGpvD!f|a!~AP33wQ^t|V&hUP`DlXQReS2rxq8DmL6wZcF^ke zX8s%#A_a5~&W28gWYyk&Sjei^ z_r7vPCf-!Q@1*i>B!ha$uc(c>R(?jwM;80?$2XqmR|E0x+L{m!om%Mf?uuq+2P|-iLKE+4TF#>90Xzs zuctOGW6RY*8{(bQ6N#?$l7=80hn5|6n#7KfP>3=deVY!i3cnoSM!enR-0T23gpLu6 zvpas1LBiC;dm_>7>(?kv7{`vDQQ7WXEiUaOQ8ZgFh@@8{$CkdciMvx=*l2{%$c9~z`1;SP~WF=MMD-zFjayBfPqaO=x z+$J4RuX=opbRSLF1h#BpgyZ=T&Qd!3m7JC!;x@_H!QSdfX+CG&PdUDX@19QEGEV#% zR-EuU9mg8!zUGA=*5qr5O)A88PEe9~cxDA|MPh?1t3j-8wLZtorXET!M)P2@&TE@& zyp6r9yvC=)=y6R=BftW3Q_=xmRa(#4v#g5md zKm2wJO6Zj5@h8%LY~y)s5U;BJ;~6}6$)KS~jVb<^2sj5jU;l|Q)V?zFjJe$cq7024 z<-A7C?K`u%-F1DEk@Qd&ZfaOZ+ zN~;hQMpJw_slji6hc3QBR%PM5BXu2VT-0L3wceuGAm@}qk*}#yU-UIh%jK3Z4UJq+ z_f{~HI9`s;J-wu@)-_?_asTdiRqMpcgq&4|OGi;`J&bE-^=K-ZlPc6GEl8MiI^x~o zO9Do{5!w=@r~)&DDQw!@yEwHNF~*U;Ar3U(32U+mtgp{78i%~w5`y`Y@T-5MI?O>O zw`-eTHo7%6weZO8TG)&h_u+`iZGKR^U0duiSXzIxJ8$u2&xQL_=lm6)ak`+nSJ|dL5m-BVCkM#VYS(h_Pu*eZP#d@A9 z1TS)NFVes9qh?6HnUS-*S3fnjt_vfVXuPr>5b8Ull*Czk@Wga(tcL_RT_n>$djR{T zd^$uKvXjx@86rJIb9wcBo9^)J&`~DQDw|-eC_2{aoq{@dhceyBP!mb_GLN}cwNGKl7UheU2ufgu z8=ciuZ}NtzlD@-=E~&I_3FO4Iq^ONmK^$-%0L{mm-?`{y;Q zAFUSFMgMjdKf8VeySbU|r8ioW4y8+w{s`4^-Ga+t)<|83mYvY7lz4jf6idCa zEk1hYISpS3teNJeH{G`k`s$kUQRc%k(3i;6xO)9DvX&nWRtBFHi- zExA;$flk0pimI?}K*=3kELP9-sJ%r+L@Eu+O2u^#6sQo(#`SB_N2b$}C5y=Uvva1) ziO|J*d&ufewm!72o^|g&<+5?u&15}&`5P`jb!)tthPa_vqMZ2ie$01Tm2h1Ms; z-_*ORc|cHDwK}l<+#+&w_PFSA9DXT+g;YfidO-Vxk zG(+spr@_5iz0%#!0KsF2c(V?!9sDwi|L#R4#@3ETJqKLApn}Z!J0W@w%-4XRtR?o` zQ|A$-`|PxGa=Mk4Mv}rS<_u+L8*gGb%}QQF z-pco9MP2X{X%UP)@0Z%ZM5)#QWX^|;<&XHi~v9xzr2 zGiZ`69Q}cUD>Dfk_?1(CCl=QJH)%BizvG=yUKWe=X61aRW@bbG7)my{UL7+XrKAJ-1Zi4mfCp`i`n(?6ub zBHPIhzrM^QCtZ)vNVK6_3r2scd&nqyoi;8-v{0!^D}SM;=XZcizTi5r{YcJO3ZsTm; zO&o}4g`{Vq`~ceP#oN`F=7quS@(8+>xsqQ-p;1yMHujaIQm+?EVkRp40}Qdr%96dB zpe5zeXuP~pd)9nNyh|IEKt|qIKi^PvDY%e0HM-C&qZzkQsrOEd95zu#U1{6RmzbWG z5MlOGA(BU>r`TQ_MBKwSuMN&6t|~x)7s#mA)(TpzSN2Y>8hr8cM$UI5?CTmQ=u&Gl27aHl3;dgG{N;bEu~WM{7#W@>%Y8I?Jay7Z}S^-@xFU$Z%N;flE z1nY}6Kf)*@EG|nO1h;}tpO~xmJPdFvzGRE=ohrPR+4MYd@Xh`?apDMLA^qyM<)f=1 zw#nbrGul?s+41~yl=1pEh=Xmc-clk_)Y0vGeN#kLI8x_(^-c3cjdYqrV{HfT(K9UZ zgc7=SV_KU2xv8)nBTt_-n&(a)h|l%)2ngH(Dang#TpM=j)2eUG&5|J~nDQ#8LX!Hf zirLtx{{!WY$)}2*vv7@9fr#4u0G}$QUiBJR`Vw=zQ*Hl*n4!E*Cj->kY!B_Ycjh^| zn4=}Fy1n(TneMPx{a-Ii+U9iHjg_{wz6`suO|5=zbbppfm+h#n&zAc{xYxt;v1l=y zXol|Sqf1M~qKH=2TI;Rpw6&{(K;_rhH5-$km#xZE-LNPxL5LD09FKuDJhyL0w1aH5l zEL}gxR((&8-UO_@_tm%+OdwGZkc`0sk})&!f1n@+?>*^{{Pnlxj^9~%vF{T-6AVpQ zs+s;7>)5n4T)}kcR;(x3XlVVB*trU)CcjA{aJOF=)y42H7M?SfM*s^H>-t;-iq#W< z_B>v1XJ*%`8+-9alOk+EeJ=?c7pNDae~9BVIS_pD_Iv%u6NAIwwJV}!eeGur@BsdSQU|2q4aX!EQ4k`VeedYLJX&B%Uy^{xSHUH* zo4zngS{_JSaiZ(`|ADgm)Q-Ruj6J$4-}7wwq!2iwsjOkuN_olnht{X<$XGjrK2E7dlsTPzmeE1~eq2vr z7e_-kKv!0~&=b%TfPwYbqYm{QR?g7sON{U9pa)wSPG!4@>Ob3u!0zM>8zqG#b&XnQf?mHUgq4&NwxMlu-?Ol6zTYM~X> ztHp3nodB{n^t$jR)I+H9?+>Ec31&TO88a54-@=#7Af$J06x0oi7JxZPD&oOt=j;Oq z)u?;+fr1uv1CNr>{o17}8y$c~lOoFb&O9OR6@Kl86utf!Au(o!5*3LOArO zjx|^1q;dL{H}&WJLHO_O%H=DnP@rSg zy`3BNbZ}?arn`j}c^H+Hk63%4`e{1NnILx^h7c{jPl`kPxXKA3g2CS@7R`>RPl02#4R5?nNjt~G|B(nJ z$oPbe)M$l`YLB4df-LvH{q$~Ak5B75xhfq(cjBkba2Vi|mf#d4JnzEFs6vCe%Vk!DJ z;~CTx2bf%T5-7I~j#Fe|HY>Q|PtbY<1uuMFSK)yaKtZ&pTw7`~zza-?v3{JgXI$l_ zVW13C^UVFYaS=G0!jvdJl&U|ma0SyZ7F#; zV*E>D^j?|(x4g(yMG`daD3N$ThwtoBG+TJS^+<}vih%K1Y~{WTUjHyPjV;-|2WX$S zpF2NXa$MLHZT(y1*nHNI0%V{PAizv7AqQu=l&OMwUCP8djEW@DoGgQ3n^gS(ii8{m z`zqHQr9@AR2l5>K9bznkAKr3KAPMq@ z720bmI@J^wf9doT(CPiZUx)5@V7)^1Ys#N^+t0UIaU^Y^w@=|gQQ4j32 z*Wd!Q@hh-TLJ~?ZLezT%wEBBf5T;e|G z)O&eCn(a?x-1Hg2=KcA`j!;U7tiaL?ms(!qdnB?{`mt6BAI<2GRrgBAxg8nI%kSz$ zH%qU7D>o-V^=IJkmqcHsx94S~MgFqFXB%%@cx_bBeSyW3u~BjqfAaEknRw`9(#&OU zW}mJ)3}RQTNYj#(+JYrlNBbaPvJS3*-9o*B{$+C5nH0o(+O%gd7}wEseCT=gRSK{NT>d?gXJ8=%`=KkNOGWfCtk^Lez~CUN zD6G90$fO5RicRRmkO11m92~w1l204vXN=)iE~kWaX6~79iV_$IK_NNKS7RbFaKLc4 z>Mwm?qFr~1u+#l*&p{FgU$PRn^ezT*r8)SGo-{$^RW$Ivs^^~Yd&(&->CeC5H#Co9 zKcdRkPmB(>nViUeMcFQ4uq`KH0Ysz!wJ83Q8(?|PeTl%#m)bEFundB#@>#PXCpH{@ z_f-mC^+sF1>T0i?37H=;)KSh|AJ(JVdBmibN=)s2(=Eh|ZxD#X|Qq}=^$ zUxD8r`n8+T|GgoAYUcx=M@MoTw03DQHXU=>8Gm#9rgB2M=whbmkCiL?c8@lxtXT+c zUEJJHYFP#!&@4_z#woj>#XrUiOxSv@MTgghhw}R)6hW*H`m{S8`5E=?a}5a?+hE~K zMN0+K!4C^)+WC(eG%^NzC7V*Vi{@AwZ{8P!#SYqc6{>XQ~XfdE5{Ms!&iksH0tTO;{oHTU_(7#?mm|c85CeoixJzpid`DNVP{=nlGIZJ4MQtfeld+< zS35P+lT*5iU1Pkd*z)@F_}9P-z0cXXF;#0hRTJ6;@Va)QFo6N;-PG+q?G4O7U@RHF z)&&T!FaWPr;xDfi2$|(?1^c#}tdi3S0XVVgc>i>hMMvO&Dud;X(vk%hH4xJoJ?gKC zquc*DAHp^!H99G|1tZ>^2T+ytp$u?;whpF?0k$@#Hqq|eEj1lEpw|Im zQ=*BBFOP|Ee*nYyk4?t3=2v#kbj3AJF8@#8bb0L;vlr&)0oB){V?d9^ zlO&3EtI3xgfq^k<&QM~YkXrmLkp*(}Sc;saTy&=?!uNEM{6R5L%Plt;Fm^)Zv5!Wi z0rGuQEOi7U5eEWhGd%$nUjRO#Zs4p%22VzB9mG2lp8|&B%X*#tUB*f;GqZ7-91NQxAYLQN&jd8EcSLvlAbr6;`0709zjr Xv { + const response = await request.get('/management/healthcheck'); + expect(response.ok()).toBeTruthy(); +}); + +Given('I have opened the image upload page', async ({ page, testContext }) => { + // An unhandled dialog blocks the page, so record every one and answer it. Dismissing a + // confirm answers "no", which keeps us on the page under test. + page.on('dialog', (dialog) => { + testContext.dialogs.push(dialog.message()); + void dialog.dismiss(); + }); + + // Accept the default of blurring graphic images up front. Kahuna otherwise shows a + // first-run explainer overlay that covers the top bar (see services/graphic-image-blur.js). + await page.context().addCookies([ + { + name: 'SHOULD_BLUR_GRAPHIC_IMAGES', + value: 'true', + domain: `.${new URL(KAHUNA_APP_URL).hostname}`, + path: '/', + }, + ]); + + await page.goto(`${KAHUNA_APP_URL}/upload`); +}); diff --git a/e2e-tests/steps/fixtures.ts b/e2e-tests/steps/fixtures.ts index 081db695d80..054a25c8655 100644 --- a/e2e-tests/steps/fixtures.ts +++ b/e2e-tests/steps/fixtures.ts @@ -1,5 +1,5 @@ import { expect } from '@playwright/test'; -import type { APIResponse, Response } from '@playwright/test'; +import type { APIResponse, FileChooser, Response } from '@playwright/test'; import { test as base, createBdd } from 'playwright-bdd'; import { DOMAIN } from '../setup/constants.ts'; @@ -12,6 +12,9 @@ export const KAHUNA_APP_URL = `https://media.${DOMAIN}`; interface TestContext { response?: APIResponse; mediaApiResponse?: Promise; + /** Messages from alerts and confirms, in the order the page raised them. */ + dialogs: string[]; + fileChooser?: Promise; } /** @@ -21,7 +24,7 @@ interface TestContext { */ export const test = base.extend<{ testContext: TestContext }>({ testContext: async ({}, use) => { - await use({}); + await use({ dialogs: [] }); }, }); diff --git a/e2e-tests/steps/support/upload-page.ts b/e2e-tests/steps/support/upload-page.ts new file mode 100644 index 00000000000..8ce5686b1b6 --- /dev/null +++ b/e2e-tests/steps/support/upload-page.ts @@ -0,0 +1,50 @@ +import { statSync } from 'node:fs'; +import * as path from 'node:path'; +import type { Page } from '@playwright/test'; + +declare global { + interface Window { + _clientConfig: { + systemName: string; + maybeUploadLimitInBytes: number; + }; + } +} + +const FIXTURE_IMAGES = path.join(import.meta.dirname, '..', '..', 'fixtures', 'images'); + +export interface TestImage { + fileName: string; + path: string; + bytes: number; +} + +const testImage = (fileName: string): TestImage => { + const filePath = path.join(FIXTURE_IMAGES, fileName); + return { fileName, path: filePath, bytes: statSync(filePath).size }; +}; + +/** Sizes are read from disk so the size-limit scenario can pick a threshold between them. */ +export const testImages = { + smaller: testImage('test-card-f.jpg'), + larger: testImage('test.jpg'), +}; + +export const uploadPage = (page: Page) => { + const prompt = page.getByRole('region', { name: 'File upload' }); + + return { + prompt, + main: page.getByRole('main', { name: 'Image uploads' }), + currentUploads: page.getByRole('region', { name: 'Your current uploads' }), + pastUploads: page.getByRole('region', { name: 'Your past 50 uploads' }), + dragAndDropUploader: page.getByRole('region', { name: 'Drag and drop uploader' }), + fileInput: prompt.locator('input[name="files"]'), + /* The upload and back-to-search controls carry aria-labels that override their visible + text, so filter on the text the feature file names rather than the accessible name. */ + uploadButton: (label: string) => prompt.getByRole('button').filter({ hasText: label }), + topBarLink: (label: string) => page.getByRole('banner').getByRole('link').filter({ hasText: label }), + /** A queued or in-flight upload, before it becomes an editable image. */ + job: (fileName: string) => page.getByRole('region', { name: `${fileName} upload` }), + }; +}; diff --git a/e2e-tests/steps/upload.steps.ts b/e2e-tests/steps/upload.steps.ts new file mode 100644 index 00000000000..bf433536c68 --- /dev/null +++ b/e2e-tests/steps/upload.steps.ts @@ -0,0 +1,141 @@ +import { Given, KAHUNA_APP_URL, Then, When, expect } from './fixtures.ts'; +import { testImages, uploadPage } from './support/upload-page.ts'; + +const FILES_TO_UPLOAD = [testImages.smaller, testImages.larger]; + +/** Long enough for a scenario's assertions, short enough not to drag out teardown. */ +const UPLOAD_HOLD_MS = 5_000; + +// --------------------------------------------------------------------------- +// Upload page shell +// --------------------------------------------------------------------------- + +Given('I am permitted to upload images', async ({ page }) => { + // Kahuna decides whether to show the upload tools by looking for a `loader` link on the + // media API root, which the API only emits for users holding the upload permission. + const mediaApiUri = await page.evaluate( + () => document.querySelector('link[rel="media-api-uri"]')?.getAttribute('href'), + ); + const response = await page.request.get(mediaApiUri!); + const { links } = (await response.json()) as { links: { rel: string }[] }; + + expect(links.map((link) => link.rel)).toContain('loader'); +}); + +When('the upload page loads', async ({ page }) => { + await expect(uploadPage(page).main).toBeVisible(); +}); + +Then('I should see the file upload prompt', async ({ page }) => { + await expect(uploadPage(page).prompt).toBeVisible(); +}); + +Then('I should see my past {int} uploads', async ({ page }, count: number) => { + await expect(page.getByRole('region', { name: `Your past ${count} uploads` })).toBeVisible(); +}); + +Then('the drag-and-drop uploader should be active', async ({ page }) => { + // The dropzone overlay only renders mid-drag, so "active" means mounted and listening. + await expect(uploadPage(page).dragAndDropUploader).toBeAttached(); +}); + +When('I choose {string} from the top bar', async ({ page }, label: string) => { + await uploadPage(page).topBarLink(label).click(); +}); + +Then('I should be taken to the image search page', async ({ page }) => { + await expect(page).toHaveURL((url) => url.origin === KAHUNA_APP_URL && url.pathname === '/search'); +}); + +Given('I have an upload in progress', async ({ page }) => { + // Hold the transfer to the ingest bucket open, otherwise the job reaches a terminal + // state within a second or so and is no longer "in progress" by the time we assert. + await page.route( + (url) => url.hostname.startsWith('localstack.'), + async (route) => { + if (route.request().method() !== 'PUT') return route.fallback(); + await new Promise((resolve) => setTimeout(resolve, UPLOAD_HOLD_MS)); + await route.abort(); + }, + ); + + await uploadPage(page).fileInput.setInputFiles(testImages.smaller.path); + await expect(uploadPage(page).job(testImages.smaller.fileName)).toBeVisible(); +}); + +Then('I should see my current uploads section', async ({ page }) => { + await expect(uploadPage(page).currentUploads).toBeVisible(); +}); + +When('I choose {string}', async ({ page }, label: string) => { + await uploadPage(page).pastUploads.getByRole('link', { name: label }).click(); +}); + +Then('I should be taken to a search filtered to images I uploaded', async ({ page }) => { + await expect(page).toHaveURL((url) => Boolean(url.searchParams.get('uploadedBy'))); +}); + +// --------------------------------------------------------------------------- +// File upload prompt +// --------------------------------------------------------------------------- + +Then('I should see a message telling me to drag and drop or click to upload to the system', async ({ page }) => { + const systemName = await page.evaluate(() => window._clientConfig.systemName); + + await expect(uploadPage(page).prompt).toContainText( + `Either drag 'n drop images onto this screen or click`, + ); + await expect(uploadPage(page).prompt).toContainText(`to get your images on ${systemName}.`); +}); + +// --------------------------------------------------------------------------- +// Select-files uploader +// --------------------------------------------------------------------------- + +When('I click the {string} button', async ({ page, testContext }, label: string) => { + testContext.fileChooser = page.waitForEvent('filechooser'); + await uploadPage(page).uploadButton(label).click(); +}); + +Then('the system file picker should open', async ({ testContext }) => { + const fileChooser = await testContext.fileChooser; + + expect(fileChooser?.isMultiple()).toBe(true); +}); + +When('I select one or more image files to upload', async ({ page }) => { + await uploadPage(page).fileInput.setInputFiles(FILES_TO_UPLOAD.map(({ path }) => path)); +}); + +Then('those files should be queued for upload', async ({ page }) => { + for (const { fileName } of FILES_TO_UPLOAD) { + await expect(uploadPage(page).job(fileName)).toBeVisible(); + } +}); + +Then('I should be taken to the upload progress view', async ({ page }) => { + await expect(uploadPage(page).currentUploads).toBeVisible(); +}); + +Given('an upload size limit is configured', async ({ page }) => { + // upload/manager.js reads the limit when files are selected, so setting it now is enough. + const limit = Math.round((testImages.smaller.bytes + testImages.larger.bytes) / 2); + + await page.evaluate((bytes) => { + window._clientConfig.maybeUploadLimitInBytes = bytes; + }, limit); +}); + +When('I select a file that is larger than the size limit', async ({ page }) => { + await uploadPage(page).fileInput.setInputFiles(FILES_TO_UPLOAD.map(({ path }) => path)); +}); + +Then('I should be warned that the oversized file will be skipped', async ({ testContext }) => { + await expect.poll(() => testContext.dialogs.join('\n')).toContain('are above the size limit'); + expect(testContext.dialogs.join('\n')).toContain(testImages.larger.fileName); +}); + +Then('only the files within the limit should be queued for upload', async ({ page }) => { + await expect(uploadPage(page).job(testImages.smaller.fileName)).toBeVisible(); + await expect(uploadPage(page).job(testImages.larger.fileName)).toBeHidden(); +}); diff --git a/kahuna/public/js/upload/view.html b/kahuna/public/js/upload/view.html index db0798bf6bb..dbe9cef750b 100644 --- a/kahuna/public/js/upload/view.html +++ b/kahuna/public/js/upload/view.html @@ -28,4 +28,4 @@

      Your past 50 uploads

      You are not authorised to upload images. Send an email to support if you think this is incorrect.
      - + From 7b371f0be3799fba91817cef0a2fe665513f5cb1 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 10 Sep 2026 12:15:06 +0000 Subject: [PATCH 358/373] Restore the uploads-in-progress leave warning scenario The warning does fire, but only for history navigations. On a ui-sref click ui-router destroys the UploadCtrl scope before broadcasting $locationChangeStart, so the listener registered on that scope never runs; pressing back reaches it. The step now goes back rather than clicking the top bar link. For back to have somewhere to return to within the same document, the background arrives at the upload page from search the way a user does, instead of deep-linking to /upload. --- e2e-tests/features/upload.feature | 3 --- e2e-tests/steps/common.steps.ts | 6 +++++- e2e-tests/steps/upload.steps.ts | 13 +++++++++++++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index e87e4b9d141..3ba0f25238c 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -52,9 +52,6 @@ Feature: Uploading images to the Grid Then I should be taken to a search filtered to images I uploaded # Evidence: kahuna/public/js/upload/view.html lines 22-23 - # Currently unreachable: ui-router destroys the UploadCtrl scope before broadcasting - # $locationChangeStart, so the controller's warning listener never runs on a ui-sref click. - @todo Scenario: Warning before leaving the page with uploads in progress Given I have an upload in progress When I try to navigate away from the upload page via the following buttons: diff --git a/e2e-tests/steps/common.steps.ts b/e2e-tests/steps/common.steps.ts index 271f1711e30..4a664e28f37 100644 --- a/e2e-tests/steps/common.steps.ts +++ b/e2e-tests/steps/common.steps.ts @@ -24,5 +24,9 @@ Given('I have opened the image upload page', async ({ page, testContext }) => { }, ]); - await page.goto(`${KAHUNA_APP_URL}/upload`); + // Arrive from search rather than deep-linking, so the upload page has a same-document + // history entry behind it and back-navigation behaves as it does for a real user. + await page.goto(KAHUNA_APP_URL); + await page.getByRole('banner').getByRole('link', { name: 'My recent uploads' }).click(); + await page.waitForURL('**/upload'); }); diff --git a/e2e-tests/steps/upload.steps.ts b/e2e-tests/steps/upload.steps.ts index bf433536c68..2e2d1696787 100644 --- a/e2e-tests/steps/upload.steps.ts +++ b/e2e-tests/steps/upload.steps.ts @@ -75,6 +75,19 @@ Then('I should be taken to a search filtered to images I uploaded', async ({ pag await expect(page).toHaveURL((url) => Boolean(url.searchParams.get('uploadedBy'))); }); +When('I try to navigate away from the upload page', async ({ page }) => { + // Going back is what a user does, and it is also the only navigation the controller's + // guard sees: on a ui-sref click ui-router destroys its scope before broadcasting + // $locationChangeStart, so the listener registered there never runs. + await page.goBack(); +}); + +Then('I should be warned that uploads are in progress and asked to confirm', async ({ testContext }) => { + await expect + .poll(() => testContext.dialogs) + .toContain('You have uploads in progress. Are you sure you want to leave this page?'); +}); + // --------------------------------------------------------------------------- // File upload prompt // --------------------------------------------------------------------------- From 8b5af0b6ef48bc5b76baa1af2568a7f57b7d7a0c Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Thu, 10 Sep 2026 12:28:03 +0000 Subject: [PATCH 359/373] Show the upload prompt's example label when no labels are applied prompt.html guarded the hint with `ctrl.presetLabels` and `ctrl.active`, neither of which filePrompt provided. That name was not simply unset: the nested has no scope of its own, so its `controllerAs: 'ctrl'` publishes FileUploaderCtrl onto the prompt's scope, and the guard read properties off that. Expose the check on the prompt's own scope instead, sourced from presetLabelService so it follows labels being added and removed in gr-preset-labels' isolated scope. The `!active` part of the guard is dropped: while the add-label form is open its input already shows the same example as a placeholder. --- e2e-tests/features/upload.feature | 3 --- e2e-tests/steps/upload.steps.ts | 12 ++++++++++++ kahuna/public/js/upload/prompt/prompt.html | 2 +- kahuna/public/js/upload/prompt/prompt.js | 9 +++++---- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index 3ba0f25238c..9d3c2ad112b 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -71,9 +71,6 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 1-3 # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 20 - # Currently unreachable: prompt.html guards the example label with `ctrl.presetLabels`, - # but filePrompt has an isolated scope and no controller, so `ctrl` is always undefined. - @todo Scenario: The prompt suggests an example label when no labels are applied Given I have not applied any preset labels When the upload page loads diff --git a/e2e-tests/steps/upload.steps.ts b/e2e-tests/steps/upload.steps.ts index 2e2d1696787..85e0f11bfd6 100644 --- a/e2e-tests/steps/upload.steps.ts +++ b/e2e-tests/steps/upload.steps.ts @@ -1,6 +1,9 @@ import { Given, KAHUNA_APP_URL, Then, When, expect } from './fixtures.ts'; import { testImages, uploadPage } from './support/upload-page.ts'; +/** Example label offered by the prompt, from kahuna/public/js/strings.json. */ +const EXAMPLE_LABEL = 'Observer'; + const FILES_TO_UPLOAD = [testImages.smaller, testImages.larger]; /** Long enough for a scenario's assertions, short enough not to drag out teardown. */ @@ -101,6 +104,15 @@ Then('I should see a message telling me to drag and drop or click to upload to t await expect(uploadPage(page).prompt).toContainText(`to get your images on ${systemName}.`); }); +Given('I have not applied any preset labels', async ({ page }) => { + await page.evaluate(() => window.localStorage.removeItem('preset-labels')); + await page.reload(); +}); + +Then('I should see a suggested example label to apply to all uploads', async ({ page }) => { + await expect(uploadPage(page).prompt).toContainText(`label e.g. ${EXAMPLE_LABEL}`); +}); + // --------------------------------------------------------------------------- // Select-files uploader // --------------------------------------------------------------------------- diff --git a/kahuna/public/js/upload/prompt/prompt.html b/kahuna/public/js/upload/prompt/prompt.html index 4ce1492b971..98475b59334 100644 --- a/kahuna/public/js/upload/prompt/prompt.html +++ b/kahuna/public/js/upload/prompt/prompt.html @@ -4,7 +4,7 @@

      Apply - label e.g. {{exampleLabel}} + label e.g. {{exampleLabel}}

      to all uploads

      diff --git a/kahuna/public/js/upload/prompt/prompt.js b/kahuna/public/js/upload/prompt/prompt.js index 06f8fd7324e..4022704a3f1 100644 --- a/kahuna/public/js/upload/prompt/prompt.js +++ b/kahuna/public/js/upload/prompt/prompt.js @@ -1,16 +1,16 @@ import angular from 'angular'; import './prompt.css'; -import template from './prompt.html'; import '../../components/gr-preset-labels/gr-preset-labels'; - +import template from './prompt.html'; import strings from '../../strings.json'; export let prompt = angular.module('kahuna.upload.prompt', [ - 'gr.presetLabels' + 'gr.presetLabels', + 'kahuna.services.presetLabel' ]); -prompt.directive('filePrompt', [function () { +prompt.directive('filePrompt', ['presetLabelService', function (presetLabelService) { return { restrict: 'E', transclude: 'replace', @@ -19,6 +19,7 @@ prompt.directive('filePrompt', [function () { link: function($scope) { $scope.systemName = window._clientConfig.systemName; $scope.exampleLabel = strings.exampleLabel; + $scope.hasPresetLabels = () => presetLabelService.getLabels().length > 0; } }; }]); From 1eda37991453e7fc5818f524e4c7776f1295b3c8 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 14 Sep 2026 14:35:12 +0000 Subject: [PATCH 360/373] Minor scenario changes to accommodate testing requirements --- e2e-tests/features/upload.feature | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index 9d3c2ad112b..fecb582fb49 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -35,6 +35,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/controller.js lines 36-41 Scenario: Returning to search from the upload page + Given I had searched for "cats" before opening the upload page When I choose "Back to search" from the top bar Then I should be taken to the image search page And my previous search should be intact @@ -52,6 +53,11 @@ Feature: Uploading images to the Grid Then I should be taken to a search filtered to images I uploaded # Evidence: kahuna/public/js/upload/view.html lines 22-23 + # Currently unreachable: none of these controls reach the controller's own confirm. + # "Back to search" is a ui-sref, and ui-router destroys the UploadCtrl scope before + # broadcasting $locationChangeStart; "Home" and "View all your uploads" are plain hrefs, + # so the browser shows its own generic beforeunload prompt instead. + @todo Scenario: Warning before leaving the page with uploads in progress Given I have an upload in progress When I try to navigate away from the upload page via the following buttons: From f361d4f375e03a8e544bc300af9e0689f878a81c Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Mon, 14 Sep 2026 14:35:35 +0000 Subject: [PATCH 361/373] Adapt tests to feature changes --- e2e-tests/steps/fixtures.ts | 2 ++ e2e-tests/steps/support/upload-page.ts | 2 ++ e2e-tests/steps/upload.steps.ts | 37 +++++++++++++++++++++----- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/e2e-tests/steps/fixtures.ts b/e2e-tests/steps/fixtures.ts index 054a25c8655..15460542ceb 100644 --- a/e2e-tests/steps/fixtures.ts +++ b/e2e-tests/steps/fixtures.ts @@ -15,6 +15,8 @@ interface TestContext { /** Messages from alerts and confirms, in the order the page raised them. */ dialogs: string[]; fileChooser?: Promise; + /** The query run on the search page before navigating to the upload page. */ + previousSearchQuery?: string; } /** diff --git a/e2e-tests/steps/support/upload-page.ts b/e2e-tests/steps/support/upload-page.ts index 8ce5686b1b6..68759793500 100644 --- a/e2e-tests/steps/support/upload-page.ts +++ b/e2e-tests/steps/support/upload-page.ts @@ -44,6 +44,8 @@ export const uploadPage = (page: Page) => { text, so filter on the text the feature file names rather than the accessible name. */ uploadButton: (label: string) => prompt.getByRole('button').filter({ hasText: label }), topBarLink: (label: string) => page.getByRole('banner').getByRole('link').filter({ hasText: label }), + /** Any control that takes you off the upload page, wherever it sits on it. */ + leaveLink: (label: string) => page.getByRole('link').filter({ hasText: label }), /** A queued or in-flight upload, before it becomes an editable image. */ job: (fileName: string) => page.getByRole('region', { name: `${fileName} upload` }), }; diff --git a/e2e-tests/steps/upload.steps.ts b/e2e-tests/steps/upload.steps.ts index 85e0f11bfd6..ca3754ac982 100644 --- a/e2e-tests/steps/upload.steps.ts +++ b/e2e-tests/steps/upload.steps.ts @@ -1,3 +1,4 @@ +import type { DataTable } from 'playwright-bdd'; import { Given, KAHUNA_APP_URL, Then, When, expect } from './fixtures.ts'; import { testImages, uploadPage } from './support/upload-page.ts'; @@ -42,6 +43,18 @@ Then('the drag-and-drop uploader should be active', async ({ page }) => { await expect(uploadPage(page).dragAndDropUploader).toBeAttached(); }); +Given( + 'I had searched for {string} before opening the upload page', + async ({ page, testContext }, query: string) => { + await page.goto(`${KAHUNA_APP_URL}/search?query=${encodeURIComponent(query)}`); + await expect(page.getByRole('main', { name: 'Image search results' })).toBeVisible(); + testContext.previousSearchQuery = query; + + await page.getByRole('banner').getByRole('link', { name: 'My recent uploads' }).click(); + await page.waitForURL('**/upload'); + }, +); + When('I choose {string} from the top bar', async ({ page }, label: string) => { await uploadPage(page).topBarLink(label).click(); }); @@ -50,6 +63,14 @@ Then('I should be taken to the image search page', async ({ page }) => { await expect(page).toHaveURL((url) => url.origin === KAHUNA_APP_URL && url.pathname === '/search'); }); +Then('my previous search should be intact', async ({ page, testContext }) => { + // The `search` state is a deep-state redirect, so entering it without params sends us + // back to the `search.results` params we left behind. + await expect(page).toHaveURL( + (url) => url.searchParams.get('query') === testContext.previousSearchQuery, + ); +}); + Given('I have an upload in progress', async ({ page }) => { // Hold the transfer to the ingest bucket open, otherwise the job reaches a terminal // state within a second or so and is no longer "in progress" by the time we assert. @@ -78,12 +99,16 @@ Then('I should be taken to a search filtered to images I uploaded', async ({ pag await expect(page).toHaveURL((url) => Boolean(url.searchParams.get('uploadedBy'))); }); -When('I try to navigate away from the upload page', async ({ page }) => { - // Going back is what a user does, and it is also the only navigation the controller's - // guard sees: on a ui-sref click ui-router destroys its scope before broadcasting - // $locationChangeStart, so the listener registered there never runs. - await page.goBack(); -}); +When( + 'I try to navigate away from the upload page via the following buttons:', + async ({ page }, buttons: DataTable) => { + for (const [label] of buttons.raw()) { + await uploadPage(page).leaveLink(label).click(); + // Dismissing the confirm answers "no", so each button leaves us here for the next one. + await expect(page).toHaveURL(/\/upload/); + } + }, +); Then('I should be warned that uploads are in progress and asked to confirm', async ({ testContext }) => { await expect From e1dd82d0e6a3a6865e78b738284db7bc4f6527ef Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 11:06:38 +0000 Subject: [PATCH 362/373] Move comments out of todo directives --- e2e-tests/features/upload.feature | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index fecb582fb49..b3bc089b71f 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -84,6 +84,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 5-11 # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 21 + @todo Scenario: Preset labels are applied to all uploads When I add a preset label via the 'apply label to all uploads' button Then that label should be applied to all my uploads @@ -155,7 +156,8 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 64-66, 219-220 # Evidence: kahuna/public/js/upload/manager.js lines 86-96 - @todo can remove this functionality in another PR + # can remove this functionality in another PR + @todo Scenario: Dropping a Witness contribution imports it When I drop a Witness contribution URL onto the page Then the importing overlay should be shown @@ -164,14 +166,16 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/dnd-uploader.html lines 8-12 # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 40-62, 69-78, 206-217 - @todo can remove this functionality in another PR + # can remove this functionality in another PR + @todo Scenario: A failed Witness import is reported Given I drop a Witness contribution URL onto the page When the Witness import fails Then I should see an alert that importing the Witness contribution failed # Evidence: kahuna/public/js/upload/dnd-uploader.js lines 79-85 - @todo not implemented afaics + # not implemented afaics + @todo Scenario: Dropping invalid content is rejected When I drop something that is not a valid file or URL Then I should see an alert that I must drop valid files or URLs From e94a8773daa48f11205ed15687655f634962288e Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 11:28:08 +0000 Subject: [PATCH 363/373] Move uploads steps to own folder --- e2e-tests/steps/global-setup.ts | 15 --------------- .../{support/upload-page.ts => upload/setup.ts} | 0 e2e-tests/steps/{ => upload}/upload.steps.ts | 4 ++-- 3 files changed, 2 insertions(+), 17 deletions(-) delete mode 100644 e2e-tests/steps/global-setup.ts rename e2e-tests/steps/{support/upload-page.ts => upload/setup.ts} (100%) rename e2e-tests/steps/{ => upload}/upload.steps.ts (98%) diff --git a/e2e-tests/steps/global-setup.ts b/e2e-tests/steps/global-setup.ts deleted file mode 100644 index 7f3724875f2..00000000000 --- a/e2e-tests/steps/global-setup.ts +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Playwright global setup. Attaches to a running Grid stack if there is one, otherwise - * boots a fresh one (see `setup/stack.ts`), stashing it for `global-teardown.ts` - * and exposing the Kahuna base URL via `GRID_BASE_URL`. - */ -import { ensureStack } from '../setup/stack.ts'; -import { setEnvironment } from '../setup/state.ts'; - -async function globalSetup(): Promise { - const environment = await ensureStack(); - process.env.GRID_BASE_URL = environment.baseUrl; - setEnvironment(environment); -} - -export default globalSetup; diff --git a/e2e-tests/steps/support/upload-page.ts b/e2e-tests/steps/upload/setup.ts similarity index 100% rename from e2e-tests/steps/support/upload-page.ts rename to e2e-tests/steps/upload/setup.ts diff --git a/e2e-tests/steps/upload.steps.ts b/e2e-tests/steps/upload/upload.steps.ts similarity index 98% rename from e2e-tests/steps/upload.steps.ts rename to e2e-tests/steps/upload/upload.steps.ts index ca3754ac982..2a87ca289e7 100644 --- a/e2e-tests/steps/upload.steps.ts +++ b/e2e-tests/steps/upload/upload.steps.ts @@ -1,6 +1,6 @@ import type { DataTable } from 'playwright-bdd'; -import { Given, KAHUNA_APP_URL, Then, When, expect } from './fixtures.ts'; -import { testImages, uploadPage } from './support/upload-page.ts'; +import { Given, KAHUNA_APP_URL, Then, When, expect } from '../fixtures.ts'; +import { testImages, uploadPage } from './setup.ts'; /** Example label offered by the prompt, from kahuna/public/js/strings.json. */ const EXAMPLE_LABEL = 'Observer'; From 208587b6868dc00f8221901f2faf35fd06696e99 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 12:09:33 +0000 Subject: [PATCH 364/373] Add detail to metadata w/o edito permission scenario --- e2e-tests/features/upload.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index b3bc089b71f..299c08a9414 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -367,7 +367,7 @@ Feature: Uploading images to the Grid @todo Scenario: Metadata editing is disabled without edit permission - Given I am not permitted to edit the image + Given I am not permitted to edit the image, as it has been uploaded by another user and I do not have edit_metadata permission When I view the metadata editor for an image I did not upload Then the metadata fields should be disabled # Evidence: kahuna/public/js/upload/jobs/required-metadata-editor.html lines 14, 44, 71, 101, 124, 152, 173 From fb208b24234c0bbe74dbd871d07649106ff5fafd Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 12:13:41 +0000 Subject: [PATCH 365/373] Colocate variable definitions with usage, and use lower camel case --- e2e-tests/steps/upload/upload.steps.ts | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/e2e-tests/steps/upload/upload.steps.ts b/e2e-tests/steps/upload/upload.steps.ts index 2a87ca289e7..00bc19a7a75 100644 --- a/e2e-tests/steps/upload/upload.steps.ts +++ b/e2e-tests/steps/upload/upload.steps.ts @@ -2,13 +2,7 @@ import type { DataTable } from 'playwright-bdd'; import { Given, KAHUNA_APP_URL, Then, When, expect } from '../fixtures.ts'; import { testImages, uploadPage } from './setup.ts'; -/** Example label offered by the prompt, from kahuna/public/js/strings.json. */ -const EXAMPLE_LABEL = 'Observer'; - -const FILES_TO_UPLOAD = [testImages.smaller, testImages.larger]; - -/** Long enough for a scenario's assertions, short enough not to drag out teardown. */ -const UPLOAD_HOLD_MS = 5_000; +const filesToUpload = [testImages.smaller, testImages.larger]; // --------------------------------------------------------------------------- // Upload page shell @@ -74,11 +68,13 @@ Then('my previous search should be intact', async ({ page, testContext }) => { Given('I have an upload in progress', async ({ page }) => { // Hold the transfer to the ingest bucket open, otherwise the job reaches a terminal // state within a second or so and is no longer "in progress" by the time we assert. + // This is long enough for a scenario's assertions, short enough not to drag out teardown. + const uploadHoldMs = 5_000; await page.route( (url) => url.hostname.startsWith('localstack.'), async (route) => { if (route.request().method() !== 'PUT') return route.fallback(); - await new Promise((resolve) => setTimeout(resolve, UPLOAD_HOLD_MS)); + await new Promise((resolve) => setTimeout(resolve, uploadHoldMs)); await route.abort(); }, ); @@ -135,7 +131,8 @@ Given('I have not applied any preset labels', async ({ page }) => { }); Then('I should see a suggested example label to apply to all uploads', async ({ page }) => { - await expect(uploadPage(page).prompt).toContainText(`label e.g. ${EXAMPLE_LABEL}`); + // The example label offered by the prompt comes from kahuna/public/js/strings.json. + await expect(uploadPage(page).prompt).toContainText(`label e.g. Observer`); }); // --------------------------------------------------------------------------- @@ -154,11 +151,11 @@ Then('the system file picker should open', async ({ testContext }) => { }); When('I select one or more image files to upload', async ({ page }) => { - await uploadPage(page).fileInput.setInputFiles(FILES_TO_UPLOAD.map(({ path }) => path)); + await uploadPage(page).fileInput.setInputFiles(filesToUpload.map(({ path }) => path)); }); Then('those files should be queued for upload', async ({ page }) => { - for (const { fileName } of FILES_TO_UPLOAD) { + for (const { fileName } of filesToUpload) { await expect(uploadPage(page).job(fileName)).toBeVisible(); } }); @@ -177,7 +174,7 @@ Given('an upload size limit is configured', async ({ page }) => { }); When('I select a file that is larger than the size limit', async ({ page }) => { - await uploadPage(page).fileInput.setInputFiles(FILES_TO_UPLOAD.map(({ path }) => path)); + await uploadPage(page).fileInput.setInputFiles(filesToUpload.map(({ path }) => path)); }); Then('I should be warned that the oversized file will be skipped', async ({ testContext }) => { From 7aa175cf2074b775c0e4856f23b57e4a7effe5ca Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 12:14:41 +0000 Subject: [PATCH 366/373] Shorter comments --- e2e-tests/steps/upload/upload.steps.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/e2e-tests/steps/upload/upload.steps.ts b/e2e-tests/steps/upload/upload.steps.ts index 00bc19a7a75..9760880e374 100644 --- a/e2e-tests/steps/upload/upload.steps.ts +++ b/e2e-tests/steps/upload/upload.steps.ts @@ -4,9 +4,9 @@ import { testImages, uploadPage } from './setup.ts'; const filesToUpload = [testImages.smaller, testImages.larger]; -// --------------------------------------------------------------------------- -// Upload page shell -// --------------------------------------------------------------------------- +/** + * Upload page shell + */ Given('I am permitted to upload images', async ({ page }) => { // Kahuna decides whether to show the upload tools by looking for a `loader` link on the @@ -112,9 +112,9 @@ Then('I should be warned that uploads are in progress and asked to confirm', asy .toContain('You have uploads in progress. Are you sure you want to leave this page?'); }); -// --------------------------------------------------------------------------- -// File upload prompt -// --------------------------------------------------------------------------- +/** + * File upload prompt + */ Then('I should see a message telling me to drag and drop or click to upload to the system', async ({ page }) => { const systemName = await page.evaluate(() => window._clientConfig.systemName); @@ -135,9 +135,9 @@ Then('I should see a suggested example label to apply to all uploads', async ({ await expect(uploadPage(page).prompt).toContainText(`label e.g. Observer`); }); -// --------------------------------------------------------------------------- -// Select-files uploader -// --------------------------------------------------------------------------- +/** + * Select-files uploader + */ When('I click the {string} button', async ({ page, testContext }, label: string) => { testContext.fileChooser = page.waitForEvent('filechooser'); From 67f7d42ed550a49a99c825456e7d9ecf23926044 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 12:17:52 +0000 Subject: [PATCH 367/373] Remove the prompt test and changes, which can be another PR --- e2e-tests/features/upload.feature | 1 + e2e-tests/steps/upload/upload.steps.ts | 10 ---------- kahuna/public/js/upload/prompt/prompt.html | 2 +- kahuna/public/js/upload/prompt/prompt.js | 9 ++++----- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index 299c08a9414..28c102b3441 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -77,6 +77,7 @@ Feature: Uploading images to the Grid # Evidence: kahuna/public/js/upload/prompt/prompt.html lines 1-3 # Evidence: kahuna/public/js/upload/prompt/prompt.js lines 20 + @todo Scenario: The prompt suggests an example label when no labels are applied Given I have not applied any preset labels When the upload page loads diff --git a/e2e-tests/steps/upload/upload.steps.ts b/e2e-tests/steps/upload/upload.steps.ts index 9760880e374..8b7ae917ac4 100644 --- a/e2e-tests/steps/upload/upload.steps.ts +++ b/e2e-tests/steps/upload/upload.steps.ts @@ -125,16 +125,6 @@ Then('I should see a message telling me to drag and drop or click to upload to t await expect(uploadPage(page).prompt).toContainText(`to get your images on ${systemName}.`); }); -Given('I have not applied any preset labels', async ({ page }) => { - await page.evaluate(() => window.localStorage.removeItem('preset-labels')); - await page.reload(); -}); - -Then('I should see a suggested example label to apply to all uploads', async ({ page }) => { - // The example label offered by the prompt comes from kahuna/public/js/strings.json. - await expect(uploadPage(page).prompt).toContainText(`label e.g. Observer`); -}); - /** * Select-files uploader */ diff --git a/kahuna/public/js/upload/prompt/prompt.html b/kahuna/public/js/upload/prompt/prompt.html index 98475b59334..4ce1492b971 100644 --- a/kahuna/public/js/upload/prompt/prompt.html +++ b/kahuna/public/js/upload/prompt/prompt.html @@ -4,7 +4,7 @@

      Apply - label e.g. {{exampleLabel}} + label e.g. {{exampleLabel}}

      to all uploads

      diff --git a/kahuna/public/js/upload/prompt/prompt.js b/kahuna/public/js/upload/prompt/prompt.js index 4022704a3f1..06f8fd7324e 100644 --- a/kahuna/public/js/upload/prompt/prompt.js +++ b/kahuna/public/js/upload/prompt/prompt.js @@ -1,16 +1,16 @@ import angular from 'angular'; import './prompt.css'; -import '../../components/gr-preset-labels/gr-preset-labels'; import template from './prompt.html'; +import '../../components/gr-preset-labels/gr-preset-labels'; + import strings from '../../strings.json'; export let prompt = angular.module('kahuna.upload.prompt', [ - 'gr.presetLabels', - 'kahuna.services.presetLabel' + 'gr.presetLabels' ]); -prompt.directive('filePrompt', ['presetLabelService', function (presetLabelService) { +prompt.directive('filePrompt', [function () { return { restrict: 'E', transclude: 'replace', @@ -19,7 +19,6 @@ prompt.directive('filePrompt', ['presetLabelService', function (presetLabelServi link: function($scope) { $scope.systemName = window._clientConfig.systemName; $scope.exampleLabel = strings.exampleLabel; - $scope.hasPresetLabels = () => presetLabelService.getLabels().length > 0; } }; }]); From 606337c12c811f89c471afdf9d12433b81601d06 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 12:30:58 +0000 Subject: [PATCH 368/373] Use list reporter to view timings in CI --- e2e-tests/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 4887d15bb6f..9e48019693d 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -6,8 +6,8 @@ "type": "module", "scripts": { "dev:e2e": "node --experimental-strip-types --disable-warning=ExperimentalWarning setup/dev.ts", - "test": "bddgen && playwright test", - "test:headed": "bddgen && playwright test --headed", + "test": "bddgen && playwright test --reporter list", + "test:headed": "bddgen && playwright test --headed --reporter list", "test:report": "playwright show-report", "test:ui": "bddgen && playwright test --ui-port 6080", "test:log": "bddgen && DEBUG=testcontainers* playwright test --ui-port 6080" From 26c803b3a20371392f5d60afb493c5300b700f80 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 13:44:55 +0100 Subject: [PATCH 369/373] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- e2e-tests/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e-tests/package.json b/e2e-tests/package.json index 9e48019693d..439d0420e30 100644 --- a/e2e-tests/package.json +++ b/e2e-tests/package.json @@ -6,8 +6,8 @@ "type": "module", "scripts": { "dev:e2e": "node --experimental-strip-types --disable-warning=ExperimentalWarning setup/dev.ts", - "test": "bddgen && playwright test --reporter list", - "test:headed": "bddgen && playwright test --headed --reporter list", + "test": "bddgen && playwright test --reporter=list,html", + "test:headed": "bddgen && playwright test --headed --reporter=list,html", "test:report": "playwright show-report", "test:ui": "bddgen && playwright test --ui-port 6080", "test:log": "bddgen && DEBUG=testcontainers* playwright test --ui-port 6080" From 7617a0638f62470ebf1c509b0019c39b6ec326dc Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 12:51:18 +0000 Subject: [PATCH 370/373] Add assertion that the current uploads UI is not visible if no uploads are active to the 'An authorised user sees the upload tools' scenario --- e2e-tests/features/upload.feature | 3 ++- e2e-tests/steps/upload/upload.steps.ts | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/e2e-tests/features/upload.feature b/e2e-tests/features/upload.feature index 28c102b3441..c9be5cf33a4 100644 --- a/e2e-tests/features/upload.feature +++ b/e2e-tests/features/upload.feature @@ -19,6 +19,7 @@ Feature: Uploading images to the Grid Then I should see the file upload prompt And I should see my past 50 uploads And the drag-and-drop uploader should be active + And I should not see the current uploads section # Evidence: kahuna/public/js/upload/view.html lines 11-12, 19-27, 31 # Evidence: kahuna/public/js/upload/controller.js lines 40-45 @@ -41,7 +42,7 @@ Feature: Uploading images to the Grid And my previous search should be intact # Evidence: kahuna/public/js/upload/view.html lines 3-6 - Scenario: The current uploads section only appears while an upload is running + Scenario: The current uploads section appears while an upload is running Given I have an upload in progress When the upload page loads Then I should see my current uploads section diff --git a/e2e-tests/steps/upload/upload.steps.ts b/e2e-tests/steps/upload/upload.steps.ts index 8b7ae917ac4..edf68cbcd9f 100644 --- a/e2e-tests/steps/upload/upload.steps.ts +++ b/e2e-tests/steps/upload/upload.steps.ts @@ -37,6 +37,10 @@ Then('the drag-and-drop uploader should be active', async ({ page }) => { await expect(uploadPage(page).dragAndDropUploader).toBeAttached(); }); +Then('I should not see the current uploads section', async ({ page }) => { + await expect(uploadPage(page).currentUploads).not.toBeVisible(); +}); + Given( 'I had searched for {string} before opening the upload page', async ({ page, testContext }, query: string) => { From 4c3c1cc685e2d19b20bad5df833a2be8fafdf1ca Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 12:52:29 +0000 Subject: [PATCH 371/373] Test for specific user in query params --- e2e-tests/steps/upload/upload.steps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/steps/upload/upload.steps.ts b/e2e-tests/steps/upload/upload.steps.ts index edf68cbcd9f..bd9665f13cb 100644 --- a/e2e-tests/steps/upload/upload.steps.ts +++ b/e2e-tests/steps/upload/upload.steps.ts @@ -96,7 +96,7 @@ When('I choose {string}', async ({ page }, label: string) => { }); Then('I should be taken to a search filtered to images I uploaded', async ({ page }) => { - await expect(page).toHaveURL((url) => Boolean(url.searchParams.get('uploadedBy'))); + await expect(page).toHaveURL((url) => url.searchParams.get('uploadedBy') === 'johndoe@example.com'); }); When( From acaaad67243e97a84c459ef70c7f557ae4cd875d Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 15:23:25 +0000 Subject: [PATCH 372/373] Always start caddy, which is necessary for Playwright to run tests locally --- e2e-tests/setup/stack.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e-tests/setup/stack.ts b/e2e-tests/setup/stack.ts index c7f3512d903..f56e5328934 100644 --- a/e2e-tests/setup/stack.ts +++ b/e2e-tests/setup/stack.ts @@ -508,7 +508,6 @@ export async function startStack(options: StartStackOptions = {}): Promise !proxy && 'using dev-nginx', task: async (ctx) => { const caddy = await proxyContainer(ctx.network!, buildCaddyfile(ctx.coreStackProps!)).start(); ctx.containers.push(caddy); From 66d181a09d00806f17d1091784563ed67dc77193 Mon Sep 17 00:00:00 2001 From: Jonathon Herbert Date: Tue, 15 Sep 2026 16:26:47 +0100 Subject: [PATCH 373/373] Remove metadata logging --- kahuna/public/js/upload/jobs/required-metadata-editor.js | 1 - 1 file changed, 1 deletion(-) diff --git a/kahuna/public/js/upload/jobs/required-metadata-editor.js b/kahuna/public/js/upload/jobs/required-metadata-editor.js index 7e609c21a5c..b0fb1ea12f9 100644 --- a/kahuna/public/js/upload/jobs/required-metadata-editor.js +++ b/kahuna/public/js/upload/jobs/required-metadata-editor.js @@ -59,7 +59,6 @@ jobs.controller('RequiredMetadataEditorCtrl', update(ctrl.resource, cleanMetadata, ctrl.image). then(resource => { ctrl.resource = resource; - setInterval(() => {console.log(ctrl.metadata);}, 5000); }). finally(() => ctrl.saving = false); };