From 186047ddc9e4a605acf8356496f55a7fe05c0c3c Mon Sep 17 00:00:00 2001 From: Adam Wright Date: Tue, 15 Sep 2026 16:28:32 +0000 Subject: [PATCH] Give each Chroma store its own Settings; a shared one crossed the bundles Reactome questions are broken on beta right now. Asked "What does CDK5 phosphorylate in Alzheimer disease?", the deployed container raises: PermissionError: [Errno 13] Permission denied: '/app/embeddings/.../userguide/Release95/sections/9c574827-...' A reactome question, failing inside the user guide bundle -- Chroma trying to create a segment folder for a collection that is not there, in a bundle it should never have opened. `chroma_settings` was one module-level Settings object shared by every store, and `langchain_chroma` mutates what it is handed: client_settings.persist_directory = ( persist_directory or client_settings.persist_directory ) So the last store constructed rewrote the persist directory for every earlier one, and chromadb returned a client keyed on those settings. With only the reactome bundle installed this was invisible: every store pointed into the same tree, so crossing them changed nothing. Installing the user guide bundle yesterday gave them different trees and turned a latent bug into an outage. `chroma_settings()` is now a factory returning a fresh object per store. Three tests, all of which fail against the shared constant. Verified in the order that broke beta -- user guide first, then reactome, then user guide again: 6 hits, 40 hits, 6 hits. **I saw this yesterday and dismissed it.** The same PermissionError appeared while I was reviewing #216, and I wrote it off as "a local artifact of the root-owned bundle" after a check that built the two retrievers and queried each once. That check did not reproduce the ordering, so it proved nothing, and I reported the system healthy. The bug was in production for a day. Co-Authored-By: Claude Opus 5 --- src/retrievers/csv_chroma.py | 29 +++++++++++++- src/retrievers/userguide/retriever.py | 2 +- tests/retrievers/test_chroma_settings.py | 48 ++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 tests/retrievers/test_chroma_settings.py diff --git a/src/retrievers/csv_chroma.py b/src/retrievers/csv_chroma.py index 69f8496..e114d34 100644 --- a/src/retrievers/csv_chroma.py +++ b/src/retrievers/csv_chroma.py @@ -21,7 +21,32 @@ from nltk.tokenize import word_tokenize from pydantic import ConfigDict -chroma_settings = chromadb.config.Settings(anonymized_telemetry=False) + +def chroma_settings() -> chromadb.config.Settings: + """A *fresh* Settings object for every Chroma store. + + Not a shared constant, which is what this was. `langchain_chroma` mutates + whatever it is handed: + + client_settings.persist_directory = ( + persist_directory or client_settings.persist_directory + ) + + so one shared instance means the last store constructed rewrites the + persist directory for every earlier one, and chromadb hands back a client + keyed on those settings. With only the reactome bundle installed nothing + showed, because every store pointed at the same tree. Installing the user + guide bundle gave them different trees, and a reactome question started + looking for its collection inside the user guide directory: + + PermissionError: [Errno 13] Permission denied: + '/app/embeddings/.../userguide/Release95/sections/9c574827-...' + + -- Chroma trying to create a segment folder for a collection that is not + there, in a bundle it should never have opened. + """ + return chromadb.config.Settings(anonymized_telemetry=False) + multi_query_prompt = PromptTemplate( input_variables=["question"], @@ -274,7 +299,7 @@ def from_subdirectory( vectordb = Chroma( persist_directory=str(embeddings_directory / subdirectory), embedding_function=embedding, - client_settings=chroma_settings, + client_settings=chroma_settings(), ) vector_retriever = vectordb.as_retriever( search_kwargs={"k": RESULTS_PER_RETRIEVER * VECTOR_OVERFETCH} diff --git a/src/retrievers/userguide/retriever.py b/src/retrievers/userguide/retriever.py index 8500cb8..a65160a 100644 --- a/src/retrievers/userguide/retriever.py +++ b/src/retrievers/userguide/retriever.py @@ -32,6 +32,6 @@ def create_userguide_retriever( vectordb = Chroma( persist_directory=str(chroma_path), embedding_function=embedding, - client_settings=chroma_settings, + client_settings=chroma_settings(), ) return vectordb.as_retriever(search_kwargs={"k": k}) diff --git a/tests/retrievers/test_chroma_settings.py b/tests/retrievers/test_chroma_settings.py new file mode 100644 index 0000000..73efabe --- /dev/null +++ b/tests/retrievers/test_chroma_settings.py @@ -0,0 +1,48 @@ +"""Each Chroma store needs its own Settings object. + +`langchain_chroma` mutates whatever settings it is handed: + + client_settings.persist_directory = ( + persist_directory or client_settings.persist_directory + ) + +so a shared instance means the last store constructed rewrites the persist +directory for every earlier one, and chromadb returns a client keyed on those +settings. + +With only the reactome bundle installed this was invisible -- every store +pointed into the same tree. Installing the user guide bundle gave them +different trees, and a *reactome* question went looking for its collection +inside the *user guide* directory: + + PermissionError: [Errno 13] Permission denied: + '/app/embeddings/.../userguide/Release95/sections/9c574827-...' + +Found on beta, in the deployed container, by asking "What does CDK5 +phosphorylate in Alzheimer disease?" after a user guide question. +""" + +from retrievers.csv_chroma import chroma_settings + + +def test_each_call_returns_a_distinct_object() -> None: + first, second = chroma_settings(), chroma_settings() + assert first is not second, ( + "a shared Settings object is mutated by langchain_chroma, so two stores " + "would end up pointing at one directory" + ) + + +def test_mutating_one_does_not_reach_another() -> None: + """This is the exact mutation langchain_chroma performs.""" + first = chroma_settings() + first.persist_directory = "/bundles/userguide/sections" + + second = chroma_settings() + assert second.persist_directory != "/bundles/userguide/sections" + + +def test_telemetry_stays_off() -> None: + # The reason this object existed in the first place; do not lose it while + # fixing the sharing. + assert chroma_settings().anonymized_telemetry is False