From 5d09d6e43bfff14b97d10bc07f0506962c6c7ca7 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sat, 7 Nov 2026 07:00:59 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20fix:=20recognize=20OpenSearc?= =?UTF-8?q?h=20in=20Elasticsearch=20compatibility=20mode?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With compatibility.override_main_response_version, which AWS OpenSearch Service domains often enable, GET / reports Elasticsearch 7.10.2 with no distribution or build flavor, so Weir refused to start. The OpenSearch tagline still identifies it. CI now also runs the integration and process suites against OpenSearch 2.17.1 in that mode. --- .github/workflows/ci.yml | 4 +- README.md | 2 +- docs/configuration.md | 6 ++ docs/development.md | 13 ++- .../app/cli_container_integration_test.go | 6 +- internal/backend/search/adapter.go | 8 ++ internal/backend/search/product_test.go | 7 ++ .../backend/search/secure_integration_test.go | 4 +- .../search/security_integration_test.go | 3 + internal/testutil/testsearch/backend.go | 98 +++++++++++++------ internal/testutil/testsearch/secure.go | 32 +++--- scripts/search-local.sh | 19 +++- 12 files changed, 134 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8d7d553..a1740c5f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: strategy: fail-fast: false matrix: - product: [elasticsearch, opensearch] + product: [elasticsearch, opensearch, opensearch-compat] runs-on: ubuntu-24.04 timeout-minutes: 50 env: @@ -82,7 +82,7 @@ jobs: strategy: fail-fast: false matrix: - product: [elasticsearch, opensearch] + product: [elasticsearch, opensearch, opensearch-compat] runs-on: ubuntu-24.04 timeout-minutes: 25 env: diff --git a/README.md b/README.md index 37f51763..cbe97bb1 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ See [Performance](docs/performance.md) for the full results. | Go | 1.27.3, to build from source | | Operating system | Linux or macOS, amd64 or arm64 | | MongoDB | A nonsharded replica set with the client `bulkWrite` command (MongoDB 8.0 or later). Tested with 8.0.32. Lua transforms also require transactions. | -| Search | Tested with Elasticsearch 8.19.22 and OpenSearch 3.3.0 | +| Search | Tested with Elasticsearch 8.19.22, OpenSearch 3.3.0, and OpenSearch 2.17.1 with `compatibility.override_main_response_version` | | Collections and indices | Must exist before typed reads and mutations. Native commands can create them. | | Network | The application listener uses plaintext gRPC. Run it on an isolated network. Backend authentication stays local. | diff --git a/docs/configuration.md b/docs/configuration.md index e83861b7..589eeca3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -108,6 +108,12 @@ userinfo, path, query or fragment. Authentication or a custom CA requires HTTPS. Idle backend connections close after 30s. Each adapter invocation qualifies its distinct indexes without retaining metadata between invocations. +At startup Weir identifies the product from `GET /`: Elasticsearch by build +flavor `default`, and OpenSearch by distribution `opensearch`. OpenSearch with +`compatibility.override_main_response_version`, which AWS OpenSearch Service +domains often enable, reports Elasticsearch 7.10.2 without either field; Weir +recognizes it by the OpenSearch tagline. Any other product stops startup. + Both adapters make up to five attempts after a confirmed conflict or abort. Possible writes and ambiguous commits are never replayed. MongoDB may retry confirmation of the same commit separately. diff --git a/docs/development.md b/docs/development.md index d54887c5..06fa5935 100644 --- a/docs/development.md +++ b/docs/development.md @@ -24,9 +24,12 @@ fixtures: `mongod` on `127.0.0.1:27028`. `WEIR_MONGOD` selects another binary and `WEIR_MONGO_BIND` adds comma-separated literal, non-wildcard IP bind addresses. A running fixture is reused only when its bind matches. -- `scripts/search-local.sh start elasticsearch|opensearch normal|overload` runs - Elasticsearch 8.19.22 on `127.0.0.1:19200` or OpenSearch 3.3.0 on - `127.0.0.1:19202` in Docker. Use `normal` capacity for ordinary runs; the +- `scripts/search-local.sh start elasticsearch|opensearch|opensearch-compat normal|overload` + runs Elasticsearch 8.19.22 on `127.0.0.1:19200`, OpenSearch 3.3.0 on + `127.0.0.1:19202`, or OpenSearch 2.17.1 with + `compatibility.override_main_response_version=true` on `127.0.0.1:19203` in + Docker. The last matches AWS OpenSearch Service domains that report + Elasticsearch 7.10.2 to older clients. Use `normal` capacity for ordinary runs; the default `overload` profile shrinks the write thread pool for the Search capacity-rejection tests. `start` waits until the cluster is green and the write thread pool has stayed idle for 5 seconds, so startup system-index @@ -37,8 +40,8 @@ fixtures: The TLS and replica-set tests start their own `mongod` processes from the same binary. The HTTPS Search tests start their own 1.5 GiB container from the image `scripts/search-local.sh` pulled, so the whole suite needs about 3.5 GiB of -Docker memory. `WEIR_SEARCH_PRODUCT` selects `elasticsearch` (the default) or -`opensearch`. +Docker memory. `WEIR_SEARCH_PRODUCT` selects `elasticsearch` (the default), +`opensearch` or `opensearch-compat`. Run packages one at a time, because MongoDB fault injection uses a server-global failpoint: diff --git a/internal/app/cli_container_integration_test.go b/internal/app/cli_container_integration_test.go index 0824ffdf..af7c7941 100644 --- a/internal/app/cli_container_integration_test.go +++ b/internal/app/cli_container_integration_test.go @@ -74,11 +74,7 @@ func containerHostAddress(t *testing.T, address string) string { // container so a CLI container reaches Search without a host round trip. func searchContainerURL(t *testing.T, backend *testsearch.Backend) string { t.Helper() - name := "weir-local-elasticsearch" - if backend.Product == "opensearch" { - name = "weir-local-opensearch-3.3" - } - raw := runDocker(t, "inspect", "--type", "container", "--format", `{{index .Config.Labels "weir.owner"}} {{(index .NetworkSettings.Networks "bridge").IPAddress}} {{.State.Running}}`, name) + raw := runDocker(t, "inspect", "--type", "container", "--format", `{{index .Config.Labels "weir.owner"}} {{(index .NetworkSettings.Networks "bridge").IPAddress}} {{.State.Running}}`, backend.Container) fields := strings.Fields(raw) if len(fields) != 3 || fields[0] != "weir-search-local" || fields[2] != "true" { t.Fatal("local Search fixture container is not running", raw) diff --git a/internal/backend/search/adapter.go b/internal/backend/search/adapter.go index b0cdf841..d9f38aaa 100644 --- a/internal/backend/search/adapter.go +++ b/internal/backend/search/adapter.go @@ -23,6 +23,8 @@ import ( const ElasticsearchProduct = "elasticsearch" const OpenSearchProduct = "opensearch" +const openSearchTagline = "The OpenSearch Project: https://opensearch.org/" + // Config is static backend client configuration; credentials are never caller // identity. HTTPS without a CAFile uses the system trust roots. type Config struct { @@ -126,6 +128,7 @@ func (a *Adapter) qualify(ctx context.Context) error { call := exchange{path: "/"} status, raw, err := a.request(ctx, call) var info struct { + Tagline string Version struct { Distribution string BuildFlavor string `json:"build_flavor"` @@ -139,6 +142,11 @@ func (a *Adapter) qualify(ctx context.Context) error { a.dialect = ElasticsearchProduct case info.Version.Distribution == "opensearch": a.dialect = OpenSearchProduct + // compatibility.override_main_response_version, which AWS OpenSearch + // Service domains often enable, makes OpenSearch report Elasticsearch + // 7.10.2 without a distribution. Its tagline still names OpenSearch. + case info.Version.Distribution == "" && info.Version.BuildFlavor == "" && info.Tagline == openSearchTagline: + a.dialect = OpenSearchProduct default: return errors.New("unsupported Search server product") } diff --git a/internal/backend/search/product_test.go b/internal/backend/search/product_test.go index 8d835302..6d307402 100644 --- a/internal/backend/search/product_test.go +++ b/internal/backend/search/product_test.go @@ -33,6 +33,13 @@ func TestSearchProductDetection(t *testing.T) { {name: "wrong-flavor-type", metadata: `{"version":{"build_flavor":123}}`, failure: "search product identification failed"}, {name: "wrong-distribution-type", metadata: `{"version":{"distribution":{}}}`, failure: "search product identification failed"}, {name: "wrong-version-type", metadata: `{"version":[]}`, failure: "search product identification failed"}, + {name: "os-compatibility-mode", metadata: `{"name":"node","cluster_name":"123456789012:domain","version":{"number":"7.10.2","build_type":"tar","build_hash":"unknown","build_snapshot":false,"lucene_version":"9.11.1","minimum_wire_compatibility_version":"7.10.0","minimum_index_compatibility_version":"7.0.0"},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, product: OpenSearchProduct}, + {name: "es-oss-tagline", metadata: `{"version":{"number":"7.10.2","build_flavor":"oss"},"tagline":"You Know, for Search"}`, failure: "unsupported Search server product"}, + {name: "es-tagline-without-flavor", metadata: `{"version":{"number":"7.10.2"},"tagline":"You Know, for Search"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-with-flavor", metadata: `{"version":{"build_flavor":"oss"},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-unknown-distribution", metadata: `{"version":{"distribution":"other"},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-variant", metadata: `{"version":{},"tagline":"The OpenSearch Project"}`, failure: "unsupported Search server product"}, + {name: "wrong-tagline-type", metadata: `{"version":{},"tagline":1}`, failure: "search product identification failed"}, {name: "malformed", metadata: `{"version":`, failure: "search product identification failed"}, {name: "unauthorized", metadata: `{"error":"metadata-secret-sentinel"}`, status: http.StatusUnauthorized, failure: "search product identification failed"}, {name: "unavailable", metadata: `{"error":"metadata-secret-sentinel"}`, status: http.StatusServiceUnavailable, failure: "search product identification failed"}, diff --git a/internal/backend/search/secure_integration_test.go b/internal/backend/search/secure_integration_test.go index 2870cfb2..188930bc 100644 --- a/internal/backend/search/secure_integration_test.go +++ b/internal/backend/search/secure_integration_test.go @@ -235,8 +235,8 @@ func secureReplyFault(t *testing.T, fixture *testsearch.SecureFixture, operation switch b.Profile { case "elasticsearch-8.19.22": wantOutcome, wantCode = pb.MutationOutcome_NOT_APPLIED, pb.FailureCode_UNAUTHENTICATED - case "opensearch-3.3.0": - // This fixture returns an unstructured 401, so the adapter cannot prove non-application. + case "opensearch-3.3.0", "opensearch-2.17.1-compat": + // These fixtures return an unstructured 401, so the adapter cannot prove non-application. wantOutcome, wantCode = pb.MutationOutcome_UNKNOWN, pb.FailureCode_UNAVAILABLE default: t.Fatal("unqualified native authentication response profile", b.Profile) diff --git a/internal/backend/search/security_integration_test.go b/internal/backend/search/security_integration_test.go index 6c489097..c600ebe1 100644 --- a/internal/backend/search/security_integration_test.go +++ b/internal/backend/search/security_integration_test.go @@ -21,6 +21,9 @@ func TestSearchSecurityInputBoundary(t *testing.T) { t.Log("new native ES min_hash cap rejects 10001 before filter construction; no OOM input") return } + if b.Profile == "opensearch-2.17.1-compat" { + t.Skip("OpenSearch 2.17 has no search.query.max_query_string_length; this probes the backend, not Weir") + } // This disposable cluster has no external users. Restore the temporary limit // independently of the test result; production Weir never changes settings. t.Cleanup(func() { diff --git a/internal/testutil/testsearch/backend.go b/internal/testutil/testsearch/backend.go index 8b402671..7e463b73 100644 --- a/internal/testutil/testsearch/backend.go +++ b/internal/testutil/testsearch/backend.go @@ -21,35 +21,67 @@ var sequence atomic.Uint64 type Backend struct { URL, Product, Index string // Profile names the fixture product and version. - Profile string + Profile string + // Container names the scripts/search-local.sh container. + Container string Username, Password, CAFile string Client *http.Client } -// searchProduct returns WEIR_SEARCH_PRODUCT, which defaults to elasticsearch. -func searchProduct(t *testing.T) string { +// fixture describes one backend that scripts/search-local.sh runs. +type fixture struct { + // product is the Search dialect Weir must detect. + product, profile, container, url string + // images are the pinned images by GOARCH. They match the images + // scripts/search-local.sh pulls; the secure fixture runs them unpulled. + images map[string]string + // compatibility enables OpenSearch's Elasticsearch 7.10 main response. + compatibility bool +} + +var fixtures = map[string]fixture{ + "elasticsearch": { + product: "elasticsearch", profile: "elasticsearch-8.19.22", container: "weir-local-elasticsearch", url: "http://127.0.0.1:19200", + images: map[string]string{ + "amd64": "docker.elastic.co/elasticsearch/elasticsearch@sha256:f1e3d88299cb0db2131fc2ab9060faefccd88fd684d11a49256ae00d7dc3c857", + "arm64": "docker.elastic.co/elasticsearch/elasticsearch@sha256:c2a3ed5f968be6d59c960aa0c60cfdaee667b6bc8211142021a41d0e85b43237", + }, + }, + "opensearch": { + product: "opensearch", profile: "opensearch-3.3.0", container: "weir-local-opensearch-3.3", url: "http://127.0.0.1:19202", + images: map[string]string{ + "amd64": "opensearchproject/opensearch@sha256:59e9e49ac00f19c2af9094e367c8e6586f7d07e6c06465c3f4c43b40934309f2", + "arm64": "opensearchproject/opensearch@sha256:d96afaf6cbd2a6a3695aeb2f1d48c9a16ad5c8918eb849e5cbf43475f0f8e146", + }, + }, + "opensearch-compat": { + product: "opensearch", profile: "opensearch-2.17.1-compat", container: "weir-local-opensearch-2.17-compat", url: "http://127.0.0.1:19203", + images: map[string]string{ + "amd64": "opensearchproject/opensearch@sha256:ab6db8c14062060cda5f0428364e76ffe2a6f03ad265589b70182c0cecb7f9d7", + "arm64": "opensearchproject/opensearch@sha256:7d961ff222c267093b7b95fc2e397d31a06a42b6f3c42ee67fc5788417a274bf", + }, + compatibility: true, + }, +} + +// selectedFixture returns the fixture WEIR_SEARCH_PRODUCT names, which +// defaults to elasticsearch. +func selectedFixture(t *testing.T) fixture { t.Helper() - product := os.Getenv("WEIR_SEARCH_PRODUCT") - switch product { - case "": - return "elasticsearch" - case "elasticsearch", "opensearch": - return product + name := os.Getenv("WEIR_SEARCH_PRODUCT") + if name == "" { + name = "elasticsearch" } - t.Fatalf("WEIR_SEARCH_PRODUCT=%q; want elasticsearch or opensearch", product) - return "" + selected, ok := fixtures[name] + if !ok { + t.Fatalf("WEIR_SEARCH_PRODUCT=%q; want elasticsearch, opensearch or opensearch-compat", name) + } + return selected } func Open(t *testing.T) *Backend { t.Helper() - var endpoint, profile string - product := searchProduct(t) - switch product { - case "elasticsearch": - endpoint, profile = "http://127.0.0.1:19200", "elasticsearch-8.19.22" - case "opensearch": - endpoint, profile = "http://127.0.0.1:19202", "opensearch-3.3.0" - } + selected := selectedFixture(t) transport := &http.Transport{Proxy: nil, MaxConnsPerHost: 8, MaxIdleConnsPerHost: 8, DisableCompression: true} client := &http.Client{ Transport: transport, @@ -58,21 +90,18 @@ func Open(t *testing.T) *Backend { } index := fmt.Sprintf("weir_search_%d_%d", os.Getpid(), sequence.Add(1)) b := &Backend{ - URL: endpoint, - Product: product, - Profile: profile, - Index: index, - Client: client, + URL: selected.url, + Product: selected.product, + Profile: selected.profile, + Container: selected.container, + Index: index, + Client: client, } status, raw := b.Do(t, "GET", "/", "") if status != 200 { t.Fatal("wrong isolated backend", status) } - cluster := "weir-local-" + product - if profile == "opensearch-3.3.0" { - cluster = "weir-local-opensearch-3.3" - } - verifyVersion(t, raw, profile, cluster) + verifyVersion(t, raw, selected.profile, selected.container) t.Cleanup(transport.CloseIdleConnections) b.Create(t, index, `{"settings":{"number_of_shards":1,"number_of_replicas":0},"mappings":{"properties":{"n":{"type":"long"}}}}`) return b @@ -82,18 +111,23 @@ func verifyVersion(t *testing.T, raw []byte, profile, cluster string) { t.Helper() var root struct { ClusterName string `json:"cluster_name"` + Tagline string Version struct { Number, Distribution string - BuildFlavor string `json:"build_flavor"` - BuildHash string `json:"build_hash"` + BuildFlavor *string `json:"build_flavor"` + BuildHash string `json:"build_hash"` } } if json.Unmarshal(raw, &root) != nil || root.ClusterName != cluster || root.Version.BuildHash == "" { t.Fatal("wrong isolated backend identity") } v := root.Version - valid := profile == "elasticsearch-8.19.22" && v.Number == "8.19.22" && v.Distribution == "" && v.BuildFlavor == "default" + valid := profile == "elasticsearch-8.19.22" && v.Number == "8.19.22" && v.Distribution == "" && v.BuildFlavor != nil && *v.BuildFlavor == "default" valid = valid || profile == "opensearch-3.3.0" && v.Number == "3.3.0" && v.Distribution == "opensearch" + // Compatibility mode reports the Elasticsearch 7.10.2 shape; only the + // tagline still names OpenSearch. + valid = valid || profile == "opensearch-2.17.1-compat" && v.Number == "7.10.2" && v.Distribution == "" && v.BuildFlavor == nil && + root.Tagline == "The OpenSearch Project: https://opensearch.org/" if !valid { t.Fatal("wrong isolated backend version/distribution") } diff --git a/internal/testutil/testsearch/secure.go b/internal/testutil/testsearch/secure.go index 5e997747..f7dce1ff 100644 --- a/internal/testutil/testsearch/secure.go +++ b/internal/testutil/testsearch/secure.go @@ -30,15 +30,6 @@ import ( const secureOwner = "weir-mongo-local1" -// searchImages are the platform images that scripts/search-local.sh pulls. The -// secure fixture runs them without pulling. -var searchImages = map[string]string{ - "elasticsearch/amd64": "docker.elastic.co/elasticsearch/elasticsearch@sha256:f1e3d88299cb0db2131fc2ab9060faefccd88fd684d11a49256ae00d7dc3c857", - "elasticsearch/arm64": "docker.elastic.co/elasticsearch/elasticsearch@sha256:c2a3ed5f968be6d59c960aa0c60cfdaee667b6bc8211142021a41d0e85b43237", - "opensearch/amd64": "opensearchproject/opensearch@sha256:59e9e49ac00f19c2af9094e367c8e6586f7d07e6c06465c3f4c43b40934309f2", - "opensearch/arm64": "opensearchproject/opensearch@sha256:d96afaf6cbd2a6a3695aeb2f1d48c9a16ad5c8918eb849e5cbf43475f0f8e146", -} - // SecureFixture owns one native TLS database. Admin is used only for bootstrap // and independent observation. Backend contains the restricted application pair. // No process-global lookup by index, URL or database exists. @@ -48,19 +39,21 @@ type SecureFixture struct { Denied *Backend Root string name, image, product string + compatibility bool materials string created bool } func OpenSecure(t *testing.T) *SecureFixture { t.Helper() - product := searchProduct(t) - image, ok := searchImages[product+"/"+runtime.GOARCH] + selected := selectedFixture(t) + product, profile := selected.product, selected.profile + image, ok := selected.images[runtime.GOARCH] if !ok { - t.Fatalf("no pinned %s image for %s", product, runtime.GOARCH) + t.Fatalf("no pinned %s image for %s", profile, runtime.GOARCH) } suffix := fmt.Sprintf("%d-%d-%s", os.Getpid(), sequence.Add(1), randomPassword(t)[:12]) - f := &SecureFixture{product: product, name: "weir-search-secure-" + product + "-" + suffix, image: image} + f := &SecureFixture{product: product, compatibility: selected.compatibility, name: "weir-search-secure-" + product + "-" + suffix, image: image} parent := filepath.Join(testutil.Root(t), ".testdata") if err := os.MkdirAll(parent, 0700); err != nil { t.Fatal("fixture parent unavailable") @@ -134,10 +127,6 @@ func OpenSecure(t *testing.T) *SecureFixture { CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, } t.Cleanup(transport.CloseIdleConnections) - profile := "elasticsearch-8.19.22" - if product == "opensearch" { - profile = "opensearch-3.3.0" - } index := "weir_search_secure_" + strings.ReplaceAll(suffix, "-", "_") f.Admin = &Backend{ URL: "https://" + port, @@ -374,7 +363,7 @@ plugins.security.ssl.http.clientauth_mode: OPTIONAL plugins.security.nodes_dn: ['CN=weir-node'] plugins.security.authcz.admin_dn: ['CN=weir-admin'] plugins.security.allow_default_init_securityindex: false -`, 0644) +`+compatibilitySetting(f.compatibility), 0644) if err := os.Mkdir(filepath.Join(f.materials, "security"), 0755); err != nil { t.Fatal("fixture security directory") } @@ -486,3 +475,10 @@ func (f *SecureFixture) cleanup(t *testing.T) { } t.Log("fixture stopped; owned container/data/materials removed; log:", f.Root) } + +func compatibilitySetting(enabled bool) string { + if !enabled { + return "" + } + return "compatibility.override_main_response_version: true\n" +} diff --git a/scripts/search-local.sh b/scripts/search-local.sh index bbed1750..7de73aef 100755 --- a/scripts/search-local.sh +++ b/scripts/search-local.sh @@ -37,7 +37,20 @@ case "$product" in image=opensearchproject/opensearch@sha256:59e9e49ac00f19c2af9094e367c8e6586f7d07e6c06465c3f4c43b40934309f2 fi ;; - *) echo 'product must be elasticsearch or opensearch' >&2; exit 1;; + opensearch-compat) + # AWS OpenSearch Service domains often enable this mode for Elasticsearch + # 7.10 clients: GET / then reports 7.10.2 without a distribution. + name=weir-local-opensearch-2.17-compat + port=19203 + version=7.10.2 + if [ "$platform" = linux/arm64 ]; then + image=opensearchproject/opensearch@sha256:7d961ff222c267093b7b95fc2e397d31a06a42b6f3c42ee67fc5788417a274bf + else + image=opensearchproject/opensearch@sha256:ab6db8c14062060cda5f0428364e76ffe2a6f03ad265589b70182c0cecb7f9d7 + fi + set -- "$@" -e compatibility.override_main_response_version=true + ;; + *) echo 'product must be elasticsearch, opensearch or opensearch-compat' >&2; exit 1;; esac if docker container inspect "$name" >/dev/null 2>&1; then test "$(docker inspect -f '{{index .Config.Labels "weir.owner"}}' "$name")" = weir-search-local @@ -77,7 +90,7 @@ case "$action" in ready=false for attempt in $(seq 1 90); do if curl -fsS --max-time 2 "http://127.0.0.1:$port/" > /dev/null 2>&1; then - curl -fsS --max-time 2 "http://127.0.0.1:$port/" | python3 -c 'import json,sys; x=json.load(sys.stdin); v=x["version"]; assert x["cluster_name"]==sys.argv[1] and v["number"]==sys.argv[2]; assert (v.get("distribution")=="opensearch") if sys.argv[3].startswith("opensearch") else (v.get("distribution", "")=="" and v.get("build_flavor")=="default"); print(json.dumps(x,sort_keys=True))' "$name" "$version" "$product" + curl -fsS --max-time 2 "http://127.0.0.1:$port/" | python3 -c 'import json,sys; x=json.load(sys.stdin); v=x["version"]; assert x["cluster_name"]==sys.argv[1] and v["number"]==sys.argv[2]; assert (v.get("distribution")=="opensearch") if sys.argv[3]=="opensearch" else (v.get("distribution", "")=="" and "build_flavor" not in v and x["tagline"]=="The OpenSearch Project: https://opensearch.org/") if sys.argv[3]=="opensearch-compat" else (v.get("distribution", "")=="" and v.get("build_flavor")=="default"); print(json.dumps(x,sort_keys=True))' "$name" "$version" "$product" ready=true break fi @@ -128,5 +141,5 @@ case "$action" in docker logs "$name" > ".testdata/$name-$id.log" 2>&1 docker rm "$name" ;; - *) echo 'usage: scripts/search-local.sh start|stop elasticsearch|opensearch [normal|overload]' >&2;exit 1;; + *) echo 'usage: scripts/search-local.sh start|stop elasticsearch|opensearch|opensearch-compat [normal|overload]' >&2;exit 1;; esac From 640c7c5904f90080f11585880e8fa07efc7aabb9 Mon Sep 17 00:00:00 2001 From: Randy <857271370@qq.com> Date: Sat, 7 Nov 2026 07:19:29 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9C=85=20test:=20match=20compatibility?= =?UTF-8?q?=20mode=20by=20version=20and=20confirm=20the=202.17.1=20node?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - qualify() also requires version 7.10.2, and the unit table adds the near misses - the compat fixtures check the node release, which compatibility mode does not rewrite - the 2.17 skip names CVE-2025-9624; docs describe detection exactly --- docs/configuration.md | 9 ++++--- docs/development.md | 6 ++--- internal/backend/search/adapter.go | 5 +++- internal/backend/search/product_test.go | 9 ++++--- .../search/security_integration_test.go | 5 +++- internal/testutil/testsearch/backend.go | 25 +++++++++++++++++++ internal/testutil/testsearch/secure.go | 3 +++ scripts/search-local.sh | 5 ++++ 8 files changed, 55 insertions(+), 12 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 589eeca3..62577df5 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -109,10 +109,11 @@ Idle backend connections close after 30s. Each adapter invocation qualifies its distinct indexes without retaining metadata between invocations. At startup Weir identifies the product from `GET /`: Elasticsearch by build -flavor `default`, and OpenSearch by distribution `opensearch`. OpenSearch with -`compatibility.override_main_response_version`, which AWS OpenSearch Service -domains often enable, reports Elasticsearch 7.10.2 without either field; Weir -recognizes it by the OpenSearch tagline. Any other product stops startup. +flavor `default` with no distribution, and OpenSearch by distribution +`opensearch`. OpenSearch with `compatibility.override_main_response_version`, +which AWS OpenSearch Service domains often enable, reports version 7.10.2 +without either field; Weir recognizes it by that version and the OpenSearch +tagline. Any other product stops startup. Both adapters make up to five attempts after a confirmed conflict or abort. Possible writes and ambiguous commits are never replayed. MongoDB may retry diff --git a/docs/development.md b/docs/development.md index 06fa5935..60d30ff3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -28,9 +28,9 @@ fixtures: runs Elasticsearch 8.19.22 on `127.0.0.1:19200`, OpenSearch 3.3.0 on `127.0.0.1:19202`, or OpenSearch 2.17.1 with `compatibility.override_main_response_version=true` on `127.0.0.1:19203` in - Docker. The last matches AWS OpenSearch Service domains that report - Elasticsearch 7.10.2 to older clients. Use `normal` capacity for ordinary runs; the - default `overload` profile shrinks the write thread pool for the Search + Docker. The last matches AWS OpenSearch Service domains whose root reports + Elasticsearch 7.10.2 to every client. Use `normal` capacity for ordinary + runs; the default `overload` profile shrinks the write thread pool for the Search capacity-rejection tests. `start` waits until the cluster is green and the write thread pool has stayed idle for 5 seconds, so startup system-index writes do not compete with tests. Each fixture container runs a 1 GiB heap diff --git a/internal/backend/search/adapter.go b/internal/backend/search/adapter.go index d9f38aaa..30ad322f 100644 --- a/internal/backend/search/adapter.go +++ b/internal/backend/search/adapter.go @@ -130,6 +130,9 @@ func (a *Adapter) qualify(ctx context.Context) error { var info struct { Tagline string Version struct { + // Only compatibility mode is identified by its number, so other + // products may report any type. + Number any Distribution string BuildFlavor string `json:"build_flavor"` } @@ -145,7 +148,7 @@ func (a *Adapter) qualify(ctx context.Context) error { // compatibility.override_main_response_version, which AWS OpenSearch // Service domains often enable, makes OpenSearch report Elasticsearch // 7.10.2 without a distribution. Its tagline still names OpenSearch. - case info.Version.Distribution == "" && info.Version.BuildFlavor == "" && info.Tagline == openSearchTagline: + case info.Version.Number == "7.10.2" && info.Version.Distribution == "" && info.Version.BuildFlavor == "" && info.Tagline == openSearchTagline: a.dialect = OpenSearchProduct default: return errors.New("unsupported Search server product") diff --git a/internal/backend/search/product_test.go b/internal/backend/search/product_test.go index 6d307402..850c1bc8 100644 --- a/internal/backend/search/product_test.go +++ b/internal/backend/search/product_test.go @@ -36,9 +36,12 @@ func TestSearchProductDetection(t *testing.T) { {name: "os-compatibility-mode", metadata: `{"name":"node","cluster_name":"123456789012:domain","version":{"number":"7.10.2","build_type":"tar","build_hash":"unknown","build_snapshot":false,"lucene_version":"9.11.1","minimum_wire_compatibility_version":"7.10.0","minimum_index_compatibility_version":"7.0.0"},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, product: OpenSearchProduct}, {name: "es-oss-tagline", metadata: `{"version":{"number":"7.10.2","build_flavor":"oss"},"tagline":"You Know, for Search"}`, failure: "unsupported Search server product"}, {name: "es-tagline-without-flavor", metadata: `{"version":{"number":"7.10.2"},"tagline":"You Know, for Search"}`, failure: "unsupported Search server product"}, - {name: "os-tagline-with-flavor", metadata: `{"version":{"build_flavor":"oss"},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, - {name: "os-tagline-unknown-distribution", metadata: `{"version":{"distribution":"other"},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, - {name: "os-tagline-variant", metadata: `{"version":{},"tagline":"The OpenSearch Project"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-with-flavor", metadata: `{"version":{"number":"7.10.2","build_flavor":"oss"},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-unknown-distribution", metadata: `{"version":{"number":"7.10.2","distribution":"other"},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-variant", metadata: `{"version":{"number":"7.10.2"},"tagline":"The OpenSearch Project"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-without-version", metadata: `{"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-null-version", metadata: `{"version":null,"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, + {name: "os-tagline-other-number", metadata: `{"version":{"number":"8.19.22","build_flavor":""},"tagline":"The OpenSearch Project: https://opensearch.org/"}`, failure: "unsupported Search server product"}, {name: "wrong-tagline-type", metadata: `{"version":{},"tagline":1}`, failure: "search product identification failed"}, {name: "malformed", metadata: `{"version":`, failure: "search product identification failed"}, {name: "unauthorized", metadata: `{"error":"metadata-secret-sentinel"}`, status: http.StatusUnauthorized, failure: "search product identification failed"}, diff --git a/internal/backend/search/security_integration_test.go b/internal/backend/search/security_integration_test.go index c600ebe1..e49b19fa 100644 --- a/internal/backend/search/security_integration_test.go +++ b/internal/backend/search/security_integration_test.go @@ -22,7 +22,10 @@ func TestSearchSecurityInputBoundary(t *testing.T) { return } if b.Profile == "opensearch-2.17.1-compat" { - t.Skip("OpenSearch 2.17 has no search.query.max_query_string_length; this probes the backend, not Weir") + // The limit fixes CVE-2025-9624, a query_string denial of service, + // and first shipped in OpenSearch 2.19.4. The profile matches the + // 2.17 production domain, so it is a known-unpatched build. + t.Skip("OpenSearch 2.17 predates search.query.max_query_string_length (CVE-2025-9624); this probes the backend, not Weir") } // This disposable cluster has no external users. Restore the temporary limit // independently of the test result; production Weir never changes settings. diff --git a/internal/testutil/testsearch/backend.go b/internal/testutil/testsearch/backend.go index 7e463b73..f0fc48d8 100644 --- a/internal/testutil/testsearch/backend.go +++ b/internal/testutil/testsearch/backend.go @@ -102,11 +102,36 @@ func Open(t *testing.T) *Backend { t.Fatal("wrong isolated backend", status) } verifyVersion(t, raw, selected.profile, selected.container) + if selected.compatibility { + verifyNodeVersion(t, b, compatibilityVersion) + } t.Cleanup(transport.CloseIdleConnections) b.Create(t, index, `{"settings":{"number_of_shards":1,"number_of_replicas":0},"mappings":{"properties":{"n":{"type":"long"}}}}`) return b } +// compatibilityVersion is the OpenSearch release behind the compatibility +// profile, whose root reports 7.10.2 instead. +const compatibilityVersion = "2.17.1" + +// verifyNodeVersion checks the release the node itself reports, which +// compatibility mode does not rewrite. +func verifyNodeVersion(t *testing.T, b *Backend, want string) { + t.Helper() + status, raw := b.Do(t, "GET", "/_nodes/_local?filter_path=nodes.*.version", "") + var nodes struct { + Nodes map[string]struct{ Version string } + } + if status != 200 || json.Unmarshal(raw, &nodes) != nil || len(nodes.Nodes) != 1 { + t.Fatal("isolated backend node version unavailable", status) + } + for _, node := range nodes.Nodes { + if node.Version != want { + t.Fatalf("isolated backend runs OpenSearch %q, want %s", node.Version, want) + } + } +} + func verifyVersion(t *testing.T, raw []byte, profile, cluster string) { t.Helper() var root struct { diff --git a/internal/testutil/testsearch/secure.go b/internal/testutil/testsearch/secure.go index f7dce1ff..a8ac0dae 100644 --- a/internal/testutil/testsearch/secure.go +++ b/internal/testutil/testsearch/secure.go @@ -188,6 +188,9 @@ func OpenSecure(t *testing.T) *SecureFixture { t.Fatal("fixture Basic administrator bootstrap failed", status) } verifyVersion(t, raw, profile, f.name) + if f.compatibility { + verifyNodeVersion(t, f.Admin, compatibilityVersion) + } if err := os.WriteFile(filepath.Join(f.Root, "version.json"), raw, 0600); err != nil { t.Fatal("cannot retain backend build fingerprint") } diff --git a/scripts/search-local.sh b/scripts/search-local.sh index 7de73aef..b49b7024 100755 --- a/scripts/search-local.sh +++ b/scripts/search-local.sh @@ -43,6 +43,8 @@ case "$product" in name=weir-local-opensearch-2.17-compat port=19203 version=7.10.2 + # The root reports 7.10.2, so the node itself confirms the release. + node_version=2.17.1 if [ "$platform" = linux/arm64 ]; then image=opensearchproject/opensearch@sha256:7d961ff222c267093b7b95fc2e397d31a06a42b6f3c42ee67fc5788417a274bf else @@ -91,6 +93,9 @@ case "$action" in for attempt in $(seq 1 90); do if curl -fsS --max-time 2 "http://127.0.0.1:$port/" > /dev/null 2>&1; then curl -fsS --max-time 2 "http://127.0.0.1:$port/" | python3 -c 'import json,sys; x=json.load(sys.stdin); v=x["version"]; assert x["cluster_name"]==sys.argv[1] and v["number"]==sys.argv[2]; assert (v.get("distribution")=="opensearch") if sys.argv[3]=="opensearch" else (v.get("distribution", "")=="" and "build_flavor" not in v and x["tagline"]=="The OpenSearch Project: https://opensearch.org/") if sys.argv[3]=="opensearch-compat" else (v.get("distribution", "")=="" and v.get("build_flavor")=="default"); print(json.dumps(x,sort_keys=True))' "$name" "$version" "$product" + if [ -n "${node_version:-}" ]; then + curl -fsS --max-time 2 "http://127.0.0.1:$port/_nodes/_local?filter_path=nodes.*.version" | python3 -c 'import json,sys; n=list(json.load(sys.stdin)["nodes"].values()); assert len(n)==1 and n[0]["version"]==sys.argv[1], n' "$node_version" + fi ready=true break fi