From 02b3dc049a9d40976b3846d3406247aa705d6522 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Thu, 6 Aug 2026 02:57:59 +1000 Subject: [PATCH] feat: add generated search_vector tsvector columns for PG full-text search (PPT-2644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a generated, stored tsvector column + GIN index to every table whose rest-api index route is searchable, replacing the Elasticsearch write path. 'simple' text-search config on both sides (no stemming — entity names, identifiers, emails; matches the guests.tsv_search precedent). Secrets and encrypted content are deliberately excluded from every vector. Co-Authored-By: Claude Fable 5 --- ...806100500000_add_search_vector_columns.sql | 275 ++++++++++++++++++ spec/search_vector_spec.cr | 101 +++++++ 2 files changed, 376 insertions(+) create mode 100644 migration/db/migrations/20260806100500000_add_search_vector_columns.sql create mode 100644 spec/search_vector_spec.cr diff --git a/migration/db/migrations/20260806100500000_add_search_vector_columns.sql b/migration/db/migrations/20260806100500000_add_search_vector_columns.sql new file mode 100644 index 00000000..e9bd3ffd --- /dev/null +++ b/migration/db/migrations/20260806100500000_add_search_vector_columns.sql @@ -0,0 +1,275 @@ +-- +micrate Up +-- SQL in section 'Up' is executed when this migration is applied + +-- PPT-2644: PostgreSQL full-text search columns replacing Elasticsearch. +-- Every searchable table gets a generated, stored tsvector maintained by +-- PostgreSQL itself and a GIN index for fast matching. The 'simple' +-- configuration is used on both write and query side (see guests.tsv_search +-- precedent): entity names, identifiers and emails must not be stemmed. +-- Secrets and encrypted content (password digests, tokens, client secrets, +-- IdP certificates, settings_string) are deliberately excluded from vectors. + +-- array_to_string is only STABLE, so generated columns can't call it +-- directly; for text[] input it is in fact immutable, hence this wrapper. +-- +micrate StatementBegin +CREATE OR REPLACE FUNCTION placeos_fts_join(arr text[]) RETURNS text +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT COALESCE(array_to_string(arr, ' '), '') +$$; +-- +micrate StatementEnd + +-- Emails are additionally split on [@._] so partial-token search works +-- (e.g. "reeves" or "place" matches "cam.reeves@place.tech"), while the +-- raw address is kept for whole-address matching. +-- +micrate StatementBegin +CREATE OR REPLACE FUNCTION placeos_fts_email(addr text) RETURNS text +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT regexp_replace(COALESCE(addr, ''), '[@._]', ' ', 'g') || ' ' || COALESCE(addr, '') +$$; +-- +micrate StatementEnd + +ALTER TABLE "sys" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(display_name, '') || ' ' || COALESCE(code, '') || ' ' || + COALESCE(type, '') || ' ' || COALESCE(description, '') || ' ' || + placeos_fts_join(features) || ' ' || placeos_fts_email(email) || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_sys_search_vector ON "sys" USING GIN (search_vector); + +ALTER TABLE "mod" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(custom_name, '') || ' ' || COALESCE(name, '') || ' ' || COALESCE(ip, '') || ' ' || + COALESCE(uri, '') || ' ' || COALESCE(notes, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_mod_search_vector ON "mod" USING GIN (search_vector); + +ALTER TABLE "driver" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(module_name, '') || ' ' || COALESCE(file_name, '') || ' ' || + COALESCE(description, '') || ' ' || COALESCE(default_uri, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_driver_search_vector ON "driver" USING GIN (search_vector); + +ALTER TABLE "zone" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(display_name, '') || ' ' || COALESCE(code, '') || ' ' || + COALESCE(type, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(location, '') || ' ' || + placeos_fts_join(tags) || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_zone_search_vector ON "zone" USING GIN (search_vector); + +ALTER TABLE "user" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(nickname, '') || ' ' || placeos_fts_email(email) || ' ' || + COALESCE(login_name, '') || ' ' || COALESCE(staff_id, '') || ' ' || + COALESCE(first_name, '') || ' ' || COALESCE(last_name, '') || ' ' || + COALESCE(department, '') || ' ' || COALESCE(building, '') || ' ' || + COALESCE(phone, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_user_search_vector ON "user" USING GIN (search_vector); + +ALTER TABLE "repo" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(folder_name, '') || ' ' || COALESCE(uri, '') || ' ' || + COALESCE(description, '') || ' ' || COALESCE(branch, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_repo_search_vector ON "repo" USING GIN (search_vector); + +ALTER TABLE "trigger" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_trigger_search_vector ON "trigger" USING GIN (search_vector); + +ALTER TABLE "authority" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(domain, '') || ' ' || COALESCE(description, '') || ' ' || + COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_authority_search_vector ON "authority" USING GIN (search_vector); + +ALTER TABLE "ldap_strat" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', COALESCE(name, '') || ' ' || COALESCE(id::text, '')) +) STORED; +CREATE INDEX idx_ldap_strat_search_vector ON "ldap_strat" USING GIN (search_vector); + +ALTER TABLE "oauth_strat" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', COALESCE(name, '') || ' ' || COALESCE(id::text, '')) +) STORED; +CREATE INDEX idx_oauth_strat_search_vector ON "oauth_strat" USING GIN (search_vector); + +ALTER TABLE "adfs_strat" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', COALESCE(name, '') || ' ' || COALESCE(id::text, '')) +) STORED; +CREATE INDEX idx_adfs_strat_search_vector ON "adfs_strat" USING GIN (search_vector); + +ALTER TABLE "edge" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_edge_search_vector ON "edge" USING GIN (search_vector); + +ALTER TABLE "api_key" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_api_key_search_vector ON "api_key" USING GIN (search_vector); + +ALTER TABLE "oauth_applications" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(uid, '') || ' ' || COALESCE(redirect_uri, '') || ' ' || + COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_oauth_applications_search_vector ON "oauth_applications" USING GIN (search_vector); + +ALTER TABLE "sets" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', placeos_fts_join(keys) || ' ' || COALESCE(id::text, '')) +) STORED; +CREATE INDEX idx_sets_search_vector ON "sets" USING GIN (search_vector); + +ALTER TABLE "json_schema" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_json_schema_search_vector ON "json_schema" USING GIN (search_vector); + +ALTER TABLE "asset" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(identifier, '') || ' ' || COALESCE(serial_number, '') || ' ' || + COALESCE(barcode, '') || ' ' || COALESCE(assigned_name, '') || ' ' || COALESCE(assigned_to, '') || ' ' || + COALESCE(notes, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_asset_search_vector ON "asset" USING GIN (search_vector); + +ALTER TABLE "asset_type" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(brand, '') || ' ' || COALESCE(model_number, '') || ' ' || + COALESCE(description, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_asset_type_search_vector ON "asset_type" USING GIN (search_vector); + +ALTER TABLE "asset_category" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_asset_category_search_vector ON "asset_category" USING GIN (search_vector); + +ALTER TABLE "asset_purchase_order" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(purchase_order_number, '') || ' ' || COALESCE(invoice_number, '') || ' ' || + COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_asset_purchase_order_search_vector ON "asset_purchase_order" USING GIN (search_vector); + +ALTER TABLE "alert" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_alert_search_vector ON "alert" USING GIN (search_vector); + +ALTER TABLE "alert_dashboard" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_alert_dashboard_search_vector ON "alert_dashboard" USING GIN (search_vector); + +ALTER TABLE "shortener" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(uri, '') || ' ' || COALESCE(description, '') || ' ' || + COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_shortener_search_vector ON "shortener" USING GIN (search_vector); + +ALTER TABLE "signage_plugin" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + COALESCE(name, '') || ' ' || COALESCE(description, '') || ' ' || COALESCE(uri, '') || ' ' || + COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_signage_plugin_search_vector ON "signage_plugin" USING GIN (search_vector); + +ALTER TABLE "pending_mail" ADD COLUMN search_vector tsvector GENERATED ALWAYS AS ( + to_tsvector('simple', + placeos_fts_join(template) || ' ' || COALESCE(rejected_reason, '') || ' ' || + placeos_fts_email(placeos_fts_join(send_to)) || ' ' || placeos_fts_email(send_from) || ' ' || + COALESCE(id::text, '') + ) +) STORED; +CREATE INDEX idx_pending_mail_search_vector ON "pending_mail" USING GIN (search_vector); + +-- +micrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +DROP INDEX IF EXISTS idx_sys_search_vector; +DROP INDEX IF EXISTS idx_mod_search_vector; +DROP INDEX IF EXISTS idx_driver_search_vector; +DROP INDEX IF EXISTS idx_zone_search_vector; +DROP INDEX IF EXISTS idx_user_search_vector; +DROP INDEX IF EXISTS idx_repo_search_vector; +DROP INDEX IF EXISTS idx_trigger_search_vector; +DROP INDEX IF EXISTS idx_authority_search_vector; +DROP INDEX IF EXISTS idx_ldap_strat_search_vector; +DROP INDEX IF EXISTS idx_oauth_strat_search_vector; +DROP INDEX IF EXISTS idx_adfs_strat_search_vector; +DROP INDEX IF EXISTS idx_edge_search_vector; +DROP INDEX IF EXISTS idx_api_key_search_vector; +DROP INDEX IF EXISTS idx_oauth_applications_search_vector; +DROP INDEX IF EXISTS idx_sets_search_vector; +DROP INDEX IF EXISTS idx_json_schema_search_vector; +DROP INDEX IF EXISTS idx_asset_search_vector; +DROP INDEX IF EXISTS idx_asset_type_search_vector; +DROP INDEX IF EXISTS idx_asset_category_search_vector; +DROP INDEX IF EXISTS idx_asset_purchase_order_search_vector; +DROP INDEX IF EXISTS idx_alert_search_vector; +DROP INDEX IF EXISTS idx_alert_dashboard_search_vector; +DROP INDEX IF EXISTS idx_shortener_search_vector; +DROP INDEX IF EXISTS idx_signage_plugin_search_vector; +DROP INDEX IF EXISTS idx_pending_mail_search_vector; + +ALTER TABLE "sys" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "mod" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "driver" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "zone" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "user" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "repo" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "trigger" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "authority" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "ldap_strat" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "oauth_strat" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "adfs_strat" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "edge" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "api_key" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "oauth_applications" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "sets" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "json_schema" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "asset" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "asset_type" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "asset_category" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "asset_purchase_order" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "alert" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "alert_dashboard" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "shortener" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "signage_plugin" DROP COLUMN IF EXISTS search_vector; +ALTER TABLE "pending_mail" DROP COLUMN IF EXISTS search_vector; + +DROP FUNCTION IF EXISTS placeos_fts_email(text); +DROP FUNCTION IF EXISTS placeos_fts_join(text[]); diff --git a/spec/search_vector_spec.cr b/spec/search_vector_spec.cr new file mode 100644 index 00000000..f673a306 --- /dev/null +++ b/spec/search_vector_spec.cr @@ -0,0 +1,101 @@ +require "./helper" + +# PPT-2644: generated `search_vector` tsvector columns replacing Elasticsearch. +# The query pattern pinned here (parameterized `to_tsquery('simple', ?)` with +# `:*` prefix tokens joined by `&`) is what rest-api's index routes use. +def vector_search(model, query : String) + model.where("search_vector @@ to_tsquery('simple', ?)", query).to_a +end + +module PlaceOS::Model + describe "search_vector columns" do + Spec.after_each do + Module.clear + Driver.clear + ControlSystem.clear + Zone.clear + User.clear + Settings.clear + end + + it "matches control systems on name, description and features with prefix tokens" do + cs = Generator.control_system + cs.name = "Video Conference Alpha" + cs.description = "boardroom telepresence" + cs.features = Set{"projector", "whiteboard"} + cs.save! + + other = Generator.control_system + other.name = "Lobby Display" + other.save! + + vector_search(ControlSystem, "video:* & conf:*").map(&.id).should eq [cs.id] + vector_search(ControlSystem, "telepres:*").map(&.id).should eq [cs.id] + # array columns (features) are searchable + vector_search(ControlSystem, "projector:*").map(&.id).should eq [cs.id] + # ids are searchable (Backoffice sends `id` in every fields list); the + # "sys" token comes only from the id prefix, so it matches both rows + vector_search(ControlSystem, "sys:*").size.should eq 2 + vector_search(ControlSystem, "nonexistenttoken:*").should be_empty + end + + it "matches users on tokenized email but never on secrets" do + user = Generator.user + user.email = Email.new("john.doe@example.com") + user.password_digest = "supersecrethash" + user.save! + + # split tokens of the email address match… + vector_search(User, "john:* & doe:*").map(&.id).should eq [user.id] + vector_search(User, "example:*").map(&.id).should eq [user.id] + # …and the raw address is kept as a single lexeme too + vector_search(User, "john.doe@example.com").map(&.id).should eq [user.id] + # secrets are deliberately not indexed + vector_search(User, "supersecrethash:*").should be_empty + end + + it "matches zones on tags" do + zone = Generator.zone + zone.name = "Level Three" + zone.tags = Set{"level", "building-a"} + zone.save! + + vector_search(Zone, "level:* & building:*").map(&.id).should eq [zone.id] + end + + it "matches modules on custom name, and drivers on name for join-based search" do + driver = Generator.driver + driver.name = "Cisco Video Switcher" + driver.save! + + mod = Generator.module(driver: driver) + mod.custom_name = "Projector Left" + mod.save! + + vector_search(Module, "projector:*").map(&.id).should eq [mod.id] + # module search by driver name is a query-time join (rest-api concern); + # the driver side of that join matches here + vector_search(Driver, "cisco:* & switch:*").map(&.id).should eq [driver.id] + + # the EXISTS pattern rest-api uses for parent-child parity + Module + .where( + "(search_vector @@ to_tsquery('simple', ?) OR EXISTS (SELECT 1 FROM driver d WHERE d.id = driver_id AND d.search_vector @@ to_tsquery('simple', ?)))", + "cisco:*", "cisco:*" + ).to_a.map(&.id).should eq [mod.id] + end + + it "matches settings on keys but not on the settings body" do + settings = Generator.settings( + settings_string: %({"api_secret_key": "hunter2"}), + encryption_level: Encryption::Level::None, + ) + settings.save! + + # NOTE: Settings versioning writes a history row on save, so match on + # inclusion rather than equality + vector_search(Settings, "api_secret_key:*").map(&.id).should contain(settings.id) + vector_search(Settings, "hunter2:*").should be_empty + end + end +end