Skip to content

feat(search): geohash aggregation (bleve + opensearch) #3272

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
dschmidt wants to merge 2 commits into from
Draft

feat(search): geohash aggregation (bleve + opensearch) #3272

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d3cdc11
feat(search): geohash_precision aggregation with opensearch geohash_grid
dschmidt Aug 8, 2026
59aacc8
feat(search): support geohash aggregation on the bleve backend
dschmidt Aug 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 125 additions & 111 deletions protogen/gen/opencloud/services/search/v0/search.pb.go
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions protogen/gen/opencloud/services/search/v0/search.swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,11 @@
"metricKind": {
"$ref": "#/definitions/v0MetricKind",
"description": "Optional. When set, this aggregation is a scalar metric over `field`\nrather than a bucket aggregation; the corresponding AggregationResult\ncarries `value` instead of `buckets`."
},
"geohashPrecision": {
"type": "integer",
"format": "int32",
"description": "Optional. When \u003e 0, this is a geohash-grid aggregation over `field` (which\nmust resolve to a geo-point field) at the given precision (1-12). Buckets\ncarry the geohash cell as key and its doc count."
}
}
},
Expand Down
4 changes: 4 additions & 0 deletions protogen/proto/opencloud/services/search/v0/search.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,10 @@ message AggregationOption {
// rather than a bucket aggregation; the corresponding AggregationResult
// carries `value` instead of `buckets`.
MetricKind metric_kind = 5 [(google.api.field_behavior) = OPTIONAL];
// Optional. When > 0, this is a geohash-grid aggregation over `field` (which
// must resolve to a geo-point field) at the given precision (1-12). Buckets
// carry the geohash cell as key and its doc count.
int32 geohash_precision = 6 [(google.api.field_behavior) = OPTIONAL];
}

enum MetricKind {
Expand Down
3 changes: 3 additions & 0 deletions services/graph/pkg/service/v0/searchquery.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,9 @@ func libregraphAggregationsToSearch(in []libregraph.AggregationOption) []*search
if a.MetricKind != nil {
agg.MetricKind = metricKindFromLibregraph(*a.MetricKind)
}
if a.GeohashPrecision != nil {
agg.GeohashPrecision = *a.GeohashPrecision
}
out = append(out, agg)
}
return out
Expand Down
20 changes: 17 additions & 3 deletions services/search/pkg/bleve/backend.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package bleve

import (
"context"
"fmt"
"math"
"strconv"
"strings"
Expand Down Expand Up @@ -92,7 +93,11 @@ func (b *Backend) Search(_ context.Context, sir *searchService.SearchIndexReques
}

for _, agg := range sir.GetAggregations() {
bleveReq.AddFacet(agg.GetField(), newBleveFacetRequest(agg))
fr, err := newBleveFacetRequest(agg)
if err != nil {
return nil, err
}
bleveReq.AddFacet(agg.GetField(), fr)
}

// Sub-aggregations need the matched hit set, not just count facets: widen
Expand Down Expand Up @@ -187,18 +192,27 @@ func needsSubAggScan(aggs []*searchService.AggregationOption) bool {
// after cross-space merge.
const defaultFacetSize = 1000

func newBleveFacetRequest(agg *searchService.AggregationOption) *bleve.FacetRequest {
func newBleveFacetRequest(agg *searchService.AggregationOption) (*bleve.FacetRequest, error) {
size := int(agg.GetSize())
if size <= 0 {
size = defaultFacetSize
}
// Geohash: terms facet on the precomputed geohash-prefix sibling field of
// the requested precision (bleve has no native geohash-grid aggregation).
if p := int(agg.GetGeohashPrecision()); p > 0 {
field, ok := searchQuery.ResolveGeohashField(agg.GetField(), p)
if !ok {
return nil, fmt.Errorf("geohash aggregation on non-geo field %q", agg.GetField())
}
return bleve.NewFacetRequest(field, size), nil
}
fr := bleve.NewFacetRequest(agg.GetField(), size)
for _, r := range aggregationRanges(agg) {
minP := parseFloatPtr(r.GetFrom())
maxP := parseFloatPtr(r.GetTo())
fr.AddNumericRange(rangeBucketKey(r), minP, maxP)
}
return fr
return fr, nil
}

func extractBleveAggregations(res *bleve.SearchResult, aggs []*searchService.AggregationOption) []*searchService.AggregationResult {
Expand Down
58 changes: 58 additions & 0 deletions services/search/pkg/bleve/backend_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,64 @@ var _ = Describe("Bleve", func() {
Expect(fields).To(ConsistOf("audio.artist", "audio.album"))
})

Describe("geohash aggregations", func() {
upsertGeo := func(id, name string, lat, lon float64) {
la, lo := lat, lon
r := search.Resource{
ID: id,
ParentID: rootResource.ID,
RootID: rootResource.ID,
Path: "./" + name,
Type: uint64(sprovider.ResourceType_RESOURCE_TYPE_FILE),
Document: content.Document{
Name: name,
MimeType: "image/jpeg",
Location: &libregraph.GeoCoordinates{Latitude: &la, Longitude: &lo},
},
}
Expect(eng.Upsert(r.ID, r)).To(Succeed())
}

BeforeEach(func() {
upsertGeo("1$2!2001", "a.jpg", 52.5100, 13.3800) // Berlin
upsertGeo("1$2!2002", "b.jpg", 52.5105, 13.3805) // ~60m away -> same geohash-5 cell
upsertGeo("1$2!2003", "c.jpg", 48.1372, 11.5756) // Munich -> different cell
})

It("buckets geo points into geohash cells by density", func() {
res := searchWithAggs("mediatype:image", &searchsvc.AggregationOption{
Field: "location",
GeohashPrecision: 5,
})
Expect(res.Aggregations).To(HaveLen(1))
agg := res.Aggregations[0]
Expect(agg.Field).To(Equal("location"))

counts := []int64{}
for _, b := range agg.Buckets {
Expect(b.Key).To(HaveLen(5), "expected a geohash-5 key, got %q", b.Key)
counts = append(counts, b.Count)
}
// Berlin pair shares a cell (2), Munich alone (1).
Expect(counts).To(ConsistOf(int64(2), int64(1)))
})

It("rejects a geohash aggregation on a non-geo field", func() {
rID, err := storagespace.ParseID(rootResource.ID)
Expect(err).ToNot(HaveOccurred())
_, err = eng.Search(context.Background(), &searchsvc.SearchIndexRequest{
Query: "mediatype:image",
Ref: &searchmsg.Reference{ResourceId: &searchmsg.ResourceID{
StorageId: rID.StorageId, SpaceId: rID.SpaceId, OpaqueId: rID.OpaqueId,
}},
Aggregations: []*searchsvc.AggregationOption{
{Field: "audio.artist", GeohashPrecision: 5},
},
})
Expect(err).To(HaveOccurred())
})
})

Describe("numeric range aggregations", func() {
upsertWithYear := func(id, name string, year int32) {
r := search.Resource{
Expand Down
168 changes: 168 additions & 0 deletions services/search/pkg/bleve/testdata/mapping.golden.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -507,6 +507,174 @@
}
}
},
"location_geohash_1": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_10": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_11": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_12": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_2": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_3": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_4": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_5": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_6": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_7": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_8": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geohash_9": {
"enabled": true,
"dynamic": true,
"fields": [
{
"type": "text",
"analyzer": "keyword",
"store": true,
"index": true,
"include_term_vectors": true,
"docvalues": true
}
]
},
"location_geopoint": {
"enabled": true,
"dynamic": true,
Expand Down
7 changes: 7 additions & 0 deletions services/search/pkg/mapping/bleve.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,13 @@ func buildBleveDocMapping(t reflect.Type, overrides map[string]FieldOpts, prefix
}
doc.AddSubDocumentMapping(fi.Name, subDoc)
doc.AddFieldMappingsAt(fi.Name+GeopointSuffix, bleve.NewGeoPointFieldMapping())
// Geohash prefix siblings power the terms-based geohash aggregation
// (bleve has no native geohash-grid).
for p := 1; p <= MaxGeohashPrecision; p++ {
gh := bleve.NewKeywordFieldMapping()
gh.IncludeInAll = false
doc.AddFieldMappingsAt(GeohashField(fi.Name, p), gh)
}
return nil
}

Expand Down
Loading
Loading