From 328693779c3e33cef0883d3df616c0cdb6c0cc05 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Fri, 25 Sep 2026 12:53:58 +0200 Subject: [PATCH 1/5] v7.0.0 dyanamic entities --- .../ResourceDocsAPIMethods.scala | 10 +- .../entity/helper/DynamicEntityHelper.scala | 126 +++++++++++------- .../main/scala/code/api/util/APIUtil.scala | 10 ++ .../main/scala/code/api/util/Glossary.scala | 2 +- .../scala/code/api/v6_0_0/Http4s600.scala | 10 +- .../v7_0_0/DynamicEntityDefinitionTest.scala | 72 ++++++++++ 6 files changed, 177 insertions(+), 53 deletions(-) diff --git a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala index f19c5ee61d..d72df6a3ec 100644 --- a/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala +++ b/obp-api/src/main/scala/code/api/ResourceDocs1_4_0/ResourceDocsAPIMethods.scala @@ -449,10 +449,16 @@ trait ResourceDocsAPIMethods extends MdcLoggable { // So they must keep their dynamic-* prefix here, matching getResourceDocsObpDynamicCached. // (Commit efb97531e over-corrected this branch to always use the requested version, // which made API Explorer render dynamic-entity CRUD URLs as /obp/v7.0.0/ — a 404.) - val dynamicDocs = allDynamicResourceDocs + // + // The exception is v7.0.0, which serves Dynamic Entity records at its own URLs, + // /obp/v7.0.0/banks/BANK_ID/dynamic-entities/... (BANK_ID a bank's id or SYS). Its listing + // documents those, with v7.0.0 operation ids, in place of the unversioned ones. + val dynamicDocs = allDynamicResourceDocsIn(requestedApiVersion) .map { it => it.specifiedUrl = - if (it.partialFunctionName.startsWith("dynamicEntity")) + if (it.partialFunctionName.startsWith("dynamicEntity") && it.implementedInApiVersion == ApiVersion.v7_0_0) + Some(s"/${it.implementedInApiVersion.urlPrefix}/${ApiVersion.v7_0_0}${it.requestUrl}") + else if (it.partialFunctionName.startsWith("dynamicEntity")) Some(s"/${it.implementedInApiVersion.urlPrefix}/${ApiVersion.`dynamic-entity`}${it.requestUrl}") else Some(s"/${it.implementedInApiVersion.urlPrefix}/${ApiVersion.`dynamic-endpoint`}${it.requestUrl}") diff --git a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala index c660d97b5c..6a73865884 100644 --- a/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala +++ b/obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala @@ -33,7 +33,7 @@ import code.api.util.ApiTag._ import code.api.util.ErrorMessages.{InvalidJsonFormat, UnknownError, UserHasMissingRoles, AuthenticatedUserIsRequired, ConsentMyResourcesMissing} import code.api.util._ import com.openbankproject.commons.model.enums.{DynamicEntityFieldType, DynamicEntityOperation} -import com.openbankproject.commons.util.ApiVersion +import com.openbankproject.commons.util.{ApiVersion, ScannedApiVersion} import org.json4s.JsonDSL._ import org.json4s._ import com.openbankproject.commons.util.JsonAliases._ @@ -221,6 +221,25 @@ object DynamicEntityHelper { collection.mutable.ArrayBuffer(docs:_*) } + /** + * The v7.0.0 docs of every entity's endpoints, at `/banks/BANK_ID/dynamic-entities/...` with the + * entity's own bank id, SYS for the system space. The same operations as [[doc]], with ids of the + * form `OBPv7.0.0-dynamicEntity_create_`. + * + * These are kept out of [[doc]] on purpose. [[doc]] is what the rest of OBP resolves operation ids + * and partial function names against (request dispatch, interceptors, metrics, validations), and a + * v7.0.0 doc shares its partial function name with the v4.0.0 one, so mixing them in would change + * which id those lookups return. Only the v7.0.0 resource-docs listing serves these. + */ + def v700Doc: List[ResourceDoc] = docsIn(ApiVersion.v7_0_0).values.toList + + /** + * The v7.0.0 URL of an entity's endpoints as a doc template: `/banks/BANK_ID/dynamic-entities` + * followed by what follows `/obp/dynamic-entity/[banks/BANK_ID/]` in the unversioned URL. + */ + def v700UrlPrefix(bankId: Option[String]): String = + s"/banks/${DynamicEntitySpace.bankIdOrSystem(bankId)}/dynamic-entities" + def createEntityId(entityName: String) = { // (?<=[a-z0-9])(?=[A-Z]) --> mean `Positive Lookbehind (?<=[a-z0-9])` && Positive Lookahead (?=[A-Z]) --> So we can find the space to replace to `_` val regexPattern = "(?<=[a-z0-9])(?=[A-Z])|-" @@ -228,7 +247,10 @@ object DynamicEntityHelper { s"${entityName}_Id".replaceAll(regexPattern, "_").toLowerCase } - def operationToResourceDoc: Map[(DynamicEntityOperation, String), ResourceDoc] = { + def operationToResourceDoc: Map[(DynamicEntityOperation, String), ResourceDoc] = docsIn(implementedInApiVersion) + + /** Every entity's docs in one API version: v4.0.0 for the unversioned URLs, v7.0.0 for the v7.0.0 ones. */ + private def docsIn(apiVersion: ScannedApiVersion): Map[(DynamicEntityOperation, String), ResourceDoc] = { val addPrefix = APIUtil.getPropsAsBoolValue("dynamic_entities_have_prefix", true) // record exists tag names, to avoid duplicated dynamic tag name. @@ -268,7 +290,7 @@ object DynamicEntityHelper { existsTagNames += tagName ApiTag(tagName) } - val fun: DynamicEntityInfo => mutable.Map[(DynamicEntityOperation, String), ResourceDoc] = createDocs(apiTag) + val fun: DynamicEntityInfo => mutable.Map[(DynamicEntityOperation, String), ResourceDoc] = createDocs(apiTag, apiVersion) val docs: Iterable[((DynamicEntityOperation, String), ResourceDoc)] = definitionsMap.values.flatMap(fun) docs.toMap } @@ -280,7 +302,7 @@ object DynamicEntityHelper { * @param dynamicEntityInfo dynamicEntityInfo * @return all ResourceDoc of given dynamicEntity */ - private def createDocs(fun: (String, String) => ResourceDocTag) + private def createDocs(fun: (String, String) => ResourceDocTag, apiVersion: ScannedApiVersion) (dynamicEntityInfo: DynamicEntityInfo): mutable.Map[(DynamicEntityOperation, String), ResourceDoc] = { val entityName = dynamicEntityInfo.entityName val hasPersonalEntity = dynamicEntityInfo.hasPersonalEntity @@ -296,8 +318,21 @@ object DynamicEntityHelper { val idNameInUrl = StringHelpers.snakify(dynamicEntityInfo.idName).toUpperCase() val listName = dynamicEntityInfo.listName val bankId = dynamicEntityInfo.bankId - val resourceDocUrl = if(bankId.isDefined) s"/banks/${bankId.getOrElse("")}/$entityName" else s"/$entityName" - val myResourceDocUrl = if(bankId.isDefined) s"/banks/${bankId.getOrElse("")}/my/$entityName" else s"/my/$entityName" + // What comes before the entity name: in v7.0.0 the space, always named (SYS included); in the + // unversioned URLs a bank when there is one, and nothing for the system space. + val urlPrefix = + if (apiVersion == ApiVersion.v7_0_0) v700UrlPrefix(bankId) + else bankId.map(b => s"/banks/$b").getOrElse("") + val resourceDocUrl = s"$urlPrefix/$entityName" + // Response examples. A v7.0.0 response always names its space, `"bank_id": "SYS"` included, where + // the unversioned URLs name a bank only for a bank level entity. + def inSpace(example: JObject): JObject = + if (apiVersion == ApiVersion.v7_0_0) + (("bank_id" -> DynamicEntitySpace.bankIdOrSystem(bankId)): JObject) merge JObject(example.obj.filterNot(_.name == "bank_id")) + else example + val singleExample = inSpace(dynamicEntityInfo.getSingleExample) + val listExample = inSpace(dynamicEntityInfo.getExampleList) + val myResourceDocUrl = s"$urlPrefix/my/$entityName" // (operationType, entityName) -> ResourceDoc @@ -305,7 +340,7 @@ object DynamicEntityHelper { val apiTag: ResourceDocTag = fun(entityName,splitNameWithBankId) resourceDocs += (DynamicEntityOperation.GET_ALL, splitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetAllFunctionName(bankId, entityName), "GET", s"$resourceDocUrl", @@ -322,7 +357,7 @@ object DynamicEntityHelper { |${dynamicEntityInfo.listQueryDoc(joinsSupported = true)} |""".stripMargin, EmptyBody, - dynamicEntityInfo.getExampleList, + listExample, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -335,7 +370,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.GET_ONE, splitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetOneFunctionName(bankId, entityName), "GET", s"$resourceDocUrl/$idNameInUrl", @@ -350,7 +385,7 @@ object DynamicEntityHelper { |${userAuthenticationMessage(true)} |""".stripMargin, EmptyBody, - dynamicEntityInfo.getSingleExample, + singleExample, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -363,7 +398,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.CREATE, splitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildCreateFunctionName(bankId, entityName), "POST", s"$resourceDocUrl", @@ -379,7 +414,7 @@ object DynamicEntityHelper { | |""", dynamicEntityInfo.getSingleExampleWithoutIdWritable, - dynamicEntityInfo.getSingleExample, + singleExample, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -393,7 +428,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.UPDATE, splitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildUpdateFunctionName(bankId, entityName), "PUT", s"$resourceDocUrl/$idNameInUrl", @@ -409,7 +444,7 @@ object DynamicEntityHelper { | |""", dynamicEntityInfo.getSingleExampleWithoutIdWritable, - dynamicEntityInfo.getSingleExample, + singleExample, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -423,7 +458,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.PATCH, splitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildPatchFunctionName(bankId, entityName), "PATCH", s"$resourceDocUrl/$idNameInUrl", @@ -446,7 +481,7 @@ object DynamicEntityHelper { | |""", dynamicEntityInfo.getSingleExampleWithoutId, - dynamicEntityInfo.getSingleExample, + singleExample, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -460,7 +495,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.DELETE, splitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildDeleteFunctionName(bankId, entityName), "DELETE", s"$resourceDocUrl/$idNameInUrl", @@ -473,7 +508,7 @@ object DynamicEntityHelper { | |""", dynamicEntityInfo.getSingleExampleWithoutIdWritable, - dynamicEntityInfo.getSingleExample, + singleExample, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -495,7 +530,7 @@ object DynamicEntityHelper { (if (personalRequiresRole) " The role is required in addition." else "") resourceDocs += (DynamicEntityOperation.GET_ALL, mySplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetAllFunctionName(bankId, s"My$entityName"), "GET", s"$myResourceDocUrl", @@ -514,7 +549,7 @@ object DynamicEntityHelper { |${dynamicEntityInfo.listQueryDoc(joinsSupported = true)} |""".stripMargin, EmptyBody, - dynamicEntityInfo.getExampleList, + listExample, myErrorMessages, List(apiTag, apiTagDynamicEntity, apiTagDynamic), if(personalRequiresRole) Some(List(dynamicEntityInfo.canGetRole)) else None, @@ -522,7 +557,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.GET_ONE, mySplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetOneFunctionName(bankId, s"My$entityName"), "GET", s"$myResourceDocUrl/$idNameInUrl", @@ -539,7 +574,7 @@ object DynamicEntityHelper { |$myConsentUserNote |""".stripMargin, EmptyBody, - dynamicEntityInfo.getSingleExample, + singleExample, myErrorMessages, List(apiTag, apiTagDynamicEntity, apiTagDynamic), if(personalRequiresRole) Some(List(dynamicEntityInfo.canGetRole)) else None, @@ -547,7 +582,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.CREATE, mySplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildCreateFunctionName(bankId, s"My$entityName"), "POST", s"$myResourceDocUrl", @@ -565,7 +600,7 @@ object DynamicEntityHelper { | |""", dynamicEntityInfo.getSingleExampleWithoutIdWritable, - dynamicEntityInfo.getSingleExample, + singleExample, myErrorMessagesWithJson, List(apiTag, apiTagDynamicEntity, apiTagDynamic), if(personalRequiresRole) Some(List(dynamicEntityInfo.canCreateRole)) else None, @@ -573,7 +608,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.UPDATE, mySplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildUpdateFunctionName(bankId, s"My$entityName"), "PUT", s"$myResourceDocUrl/$idNameInUrl", @@ -591,7 +626,7 @@ object DynamicEntityHelper { | |""", dynamicEntityInfo.getSingleExampleWithoutIdWritable, - dynamicEntityInfo.getSingleExample, + singleExample, myErrorMessagesWithJson, List(apiTag, apiTagDynamicEntity, apiTagDynamic), if(personalRequiresRole) Some(List(dynamicEntityInfo.canUpdateRole)) else Some(List(dynamicEntityInfo.canUpdateRole)), @@ -599,7 +634,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.PATCH, mySplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildPatchFunctionName(bankId, s"My$entityName"), "PATCH", s"$myResourceDocUrl/$idNameInUrl", @@ -620,7 +655,7 @@ object DynamicEntityHelper { | |""", dynamicEntityInfo.getSingleExampleWithoutId, - dynamicEntityInfo.getSingleExample, + singleExample, myErrorMessagesWithJson, List(apiTag, apiTagDynamicEntity, apiTagDynamic), if(personalRequiresRole) Some(List(dynamicEntityInfo.canUpdateRole)) else Some(List(dynamicEntityInfo.canUpdateRole)), @@ -628,7 +663,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.DELETE, mySplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildDeleteFunctionName(bankId, s"My$entityName"), "DELETE", s"$myResourceDocUrl/$idNameInUrl", @@ -643,7 +678,7 @@ object DynamicEntityHelper { | |""", dynamicEntityInfo.getSingleExampleWithoutIdWritable, - dynamicEntityInfo.getSingleExample, + singleExample, myErrorMessages, List(apiTag, apiTagDynamicEntity, apiTagDynamic), if(personalRequiresRole) Some(List(dynamicEntityInfo.canDeleteRole)) else None, @@ -653,11 +688,11 @@ object DynamicEntityHelper { val hasPublicAccess = dynamicEntityInfo.hasPublicAccess if(hasPublicAccess) { - val publicResourceDocUrl = if(bankId.isDefined) s"/banks/${bankId.getOrElse("")}/public/$entityName" else s"/public/$entityName" + val publicResourceDocUrl = s"$urlPrefix/public/$entityName" val publicSplitNameWithBankId = s"Public$splitNameWithBankId" resourceDocs += (DynamicEntityOperation.GET_ALL, publicSplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetAllFunctionName(bankId, s"Public$entityName"), "GET", s"$publicResourceDocUrl", @@ -674,7 +709,7 @@ object DynamicEntityHelper { |${dynamicEntityInfo.listQueryDoc(joinsSupported = false)} |""".stripMargin, EmptyBody, - dynamicEntityInfo.getExampleList, + listExample, List( UnknownError ), @@ -683,7 +718,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.GET_ONE, publicSplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetOneFunctionName(bankId, s"Public$entityName"), "GET", s"$publicResourceDocUrl/$idNameInUrl", @@ -698,7 +733,7 @@ object DynamicEntityHelper { |Authentication is Optional |""".stripMargin, EmptyBody, - dynamicEntityInfo.getSingleExample, + singleExample, List( UnknownError ), @@ -709,11 +744,11 @@ object DynamicEntityHelper { val hasCommunityAccess = dynamicEntityInfo.hasCommunityAccess if(hasCommunityAccess) { - val communityResourceDocUrl = if(bankId.isDefined) s"/banks/${bankId.getOrElse("")}/community/$entityName" else s"/community/$entityName" + val communityResourceDocUrl = s"$urlPrefix/community/$entityName" val communitySplitNameWithBankId = s"Community$splitNameWithBankId" resourceDocs += (DynamicEntityOperation.GET_ALL, communitySplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetAllFunctionName(bankId, s"Community$entityName"), "GET", s"$communityResourceDocUrl", @@ -730,7 +765,7 @@ object DynamicEntityHelper { |${dynamicEntityInfo.listQueryDoc(joinsSupported = false)} |""".stripMargin, EmptyBody, - dynamicEntityInfo.getExampleList, + listExample, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -742,7 +777,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.GET_ONE, communitySplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetOneFunctionName(bankId, s"Community$entityName"), "GET", s"$communityResourceDocUrl/$idNameInUrl", @@ -757,7 +792,7 @@ object DynamicEntityHelper { |Authentication is Required |""".stripMargin, EmptyBody, - dynamicEntityInfo.getSingleExample, + singleExample, List( AuthenticatedUserIsRequired, UserHasMissingRoles, @@ -793,7 +828,7 @@ object DynamicEntityHelper { |""".stripMargin resourceDocs += (DynamicEntityOperation.GET_ALL, accessSplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGetRowAccessFunctionName(bankId, entityName), "GET", s"$accessResourceDocUrl", @@ -817,9 +852,9 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.UPDATE, accessSplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildGrantRowAccessFunctionName(bankId, entityName), - "POST", + "PUT", s"$accessResourceDocUrl", s"Grant Access to a $splitName Record", s"""Grant (or update) another User's access to one $splitName record. @@ -846,7 +881,7 @@ object DynamicEntityHelper { ) resourceDocs += (DynamicEntityOperation.DELETE, accessSplitNameWithBankId) -> ResourceDoc( - implementedInApiVersion, + apiVersion, buildRevokeRowAccessFunctionName(bankId, entityName), "DELETE", s"$accessResourceDocUrl/USER_ID", @@ -1019,7 +1054,8 @@ case class DynamicEntityInfo(definition: String, entityName: String, bankId: Opt if (restricted.isEmpty) getSingleExampleWithoutId else JObject(getSingleExampleWithoutId.obj.filterNot(f => restricted.contains(f.name))) } - val bankIdJObject: JObject = ("bank-id" -> ExampleValue.bankIdExample.value) + // A bank level entity's responses name its bank, as `bank_id`; see Http4sDynamicEntity.wrapBankId. + val bankIdJObject: JObject = ("bank_id" -> bankId.getOrElse(ExampleValue.bankIdExample.value)) def getSingleExample: JObject = if (bankId.isDefined){ val SingleObject: JObject = (singleName -> (JObject(JField(idName, JString(ExampleValue.idExample.value)) :: getSingleExampleWithoutId.obj))) diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index b2d79e2b0c..794df2db25 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -5004,6 +5004,16 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ lazy val allStaticResourceDocs: List[ResourceDoc] = ResourceDocRegistry.allStaticResourceDocs def allDynamicResourceDocs= (DynamicEntityHelper.doc ++ DynamicEndpointHelper.doc ++ DynamicEndpoints.dynamicResourceDocs).toList + + /** + * The dynamic docs a versioned resource-docs listing shows. v7.0.0 documents Dynamic Entity records at + * their v7.0.0 URLs (/obp/v7.0.0/banks/BANK_ID/dynamic-entities/...); every other version at the + * unversioned /obp/dynamic-entity/... URLs, as [[allDynamicResourceDocs]] does. + */ + def allDynamicResourceDocsIn(requestedApiVersion: ScannedApiVersion): List[ResourceDoc] = + if (requestedApiVersion == ApiVersion.v7_0_0) + (DynamicEntityHelper.v700Doc ++ DynamicEndpointHelper.doc ++ DynamicEndpoints.dynamicResourceDocs).toList + else allDynamicResourceDocs def getAllResourceDocs = allStaticResourceDocs ++ allDynamicResourceDocs diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 1687c912a4..42b73a4582 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -3848,7 +3848,7 @@ object Glossary extends MdcLoggable { || Call | Does | ||---|---| || `GET /obp/dynamic-entity/ENTITY/RECORD_ID/access` | lists who may read, update, delete and grant this record, and who granted them | -|| `POST /obp/dynamic-entity/ENTITY/RECORD_ID/access` | grants or replaces one entry, or an array of them: `user_id` is required, `can_read`, `can_update` and `can_delete` default to `false`, `can_grant` defaults to `true` | +|| `PUT /obp/dynamic-entity/ENTITY/RECORD_ID/access` | grants or replaces one entry, or an array of them: `user_id` is required, `can_read`, `can_update` and `can_delete` default to `false`, `can_grant` defaults to `true` | || `DELETE /obp/dynamic-entity/ENTITY/RECORD_ID/access/USER_ID` | revokes that User, cascading to every grant they passed on | | |Bank level entities take the same paths under `/banks/BANK_ID/`. The caller needs `can_grant` on the record — the User who created it has it — or `CanGrantDynamicEntityRowAccess_SystemENTITY` (`CanGrantDynamicEntityRowAccess_ENTITY` at a bank), which administers any record. The calls return 400 on an entity that is not row level. Creating a record still takes the entity's Create role, and `useRowLevelAccess` is only supported for locally-backed entities. diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 447a40277b..7aa0ad2868 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -7302,7 +7302,7 @@ object Http4s600 { |* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`. |* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users). |* Set `personal_requires_role` to `true` to require the corresponding role (e.g. CanCreateDynamicEntity_, CanGetDynamicEntity_) for `/my/` personal entity endpoints. Default is `false` (any authenticated user can use `/my/` endpoints). - |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/POST /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. + |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/PUT /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. | |For more information see ${Glossary.getGlossaryItemLink("Dynamic-Entities")} and ${Glossary.getGlossaryItemLink("Dynamic-Entity-Access-Model")}""", CreateDynamicEntityRequestJsonV600( @@ -7375,7 +7375,7 @@ object Http4s600 { |* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`. |* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users). |* Set `personal_requires_role` to `true` to require the corresponding role (e.g. CanCreateDynamicEntity_, CanGetDynamicEntity_) for `/my/` personal entity endpoints. Default is `false` (any authenticated user can use `/my/` endpoints). - |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/POST /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. + |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/PUT /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. | |For more information see ${Glossary.getGlossaryItemLink("Dynamic-Entities")} and ${Glossary.getGlossaryItemLink("Dynamic-Entity-Access-Model")}""", CreateDynamicEntityRequestJsonV600( @@ -7450,7 +7450,7 @@ object Http4s600 { |* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`. |* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users). |* Set `personal_requires_role` to `true` to require the corresponding role (e.g. CanCreateDynamicEntity_, CanGetDynamicEntity_) for `/my/` personal entity endpoints. Default is `false` (any authenticated user can use `/my/` endpoints). - |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/POST /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. + |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/PUT /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. | |For more information see ${Glossary.getGlossaryItemLink("Dynamic-Entities")} and ${Glossary.getGlossaryItemLink("Dynamic-Entity-Access-Model")}""", UpdateDynamicEntityRequestJsonV600( @@ -7514,7 +7514,7 @@ object Http4s600 { |* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`. |* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users). |* Set `personal_requires_role` to `true` to require the corresponding role (e.g. CanCreateDynamicEntity_, CanGetDynamicEntity_) for `/my/` personal entity endpoints. Default is `false` (any authenticated user can use `/my/` endpoints). - |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/POST /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. + |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/PUT /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. | |For more information see ${Glossary.getGlossaryItemLink("Dynamic-Entities")} and ${Glossary.getGlossaryItemLink("Dynamic-Entity-Access-Model")}""", UpdateDynamicEntityRequestJsonV600( @@ -7584,7 +7584,7 @@ object Http4s600 { |* Set `auth_mode` to say who may hold the roles that guard the entity's data endpoints: `UserOnly` (default, the User's Entitlements), `ApplicationOnly` (the Consumer's Scopes), `UserOrApplication` (either) or `UserAndApplication` (both). Personal (`/my/`) endpoints always require a User. An entity with `has_personal_entity` cannot be `ApplicationOnly`. |* Set `has_community_access` to `true` to generate read-only community endpoints (GET only, authentication required + CanGet role) under `/community/`. Community endpoints return ALL records (personal + non-personal from all users). |* Set `personal_requires_role` to `true` to require the corresponding role (e.g. CanCreateDynamicEntity_, CanGetDynamicEntity_) for `/my/` personal entity endpoints. Default is `false` (any authenticated user can use `/my/` endpoints). - |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/POST /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. + |* Set `use_row_level_access` to `true` to decide read, update, delete and grant **per record** with an access list, in place of the entity's Get, Update and Delete roles. The User who creates a record holds all four permissions on it and shares it with `GET/PUT /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access` (a body of `user_id`, `can_read`, `can_update`, `can_delete`, `can_grant`) and `DELETE /obp/dynamic-entity/ENTITY_NAME/RECORD_ID/access/USER_ID`; revoking cascades to the grants that user passed on. Records the caller may not read are omitted from list responses and return 404 individually. Creating a record still needs the entity's Create role, `CanGrantDynamicEntityRowAccess_` administers the access list of any record, and field-level read and write roles still apply on top. It cannot be combined with `has_public_access` or `has_community_access`, and is only supported for locally-backed entities. | |For more information see ${Glossary.getGlossaryItemLink("My-Dynamic-Entities")} and ${Glossary.getGlossaryItemLink("Dynamic-Entity-Access-Model")}""", UpdateDynamicEntityRequestJsonV600( diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityDefinitionTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityDefinitionTest.scala index 2e9fea1ea4..4fdb077f46 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityDefinitionTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityDefinitionTest.scala @@ -352,4 +352,76 @@ class DynamicEntityDefinitionTest extends ServerSetupWithTestData { errorOf(response) should include(BankNotFound) } } + + feature("v7.0.0 resource docs of a Dynamic Entity's endpoints") { + + /** The OBPv7.0.0 listing as the API Explorer asks for it, narrowed to the given partial functions. */ + def v7Docs(functions: List[String]): List[JValue] = { + val response = makeGetRequest((v7 / "resource-docs" / "OBPv7.0.0" / "obp") < (d \ "operation_id").extract[String]) + + scenario("the OBPv7.0.0 listing documents records at their v7.0.0 URLs, in the system space and at a bank", VersionOfApi) { + val entityName = newEntityName() + val bankId = testBankId1.value + val systemEntityId = createdWithFlags(SYS, entityName, "use_row_level_access" -> true) + val bankEntityId = createdWithFlags(bankId, entityName) + try { + val functions = List( + s"dynamicEntity_create${entityName}_", + s"dynamicEntity_get${entityName}List_$bankId", + s"dynamicEntity_grant${entityName}RowAccess_") + val docs = v7Docs(functions) + def docWithId(id: String): JValue = + docs.find(d => (d \ "operation_id").extract[String] == id).getOrElse(fail(s"no resource doc $id in ${operationIds(docs)}")) + + Then("a system entity's docs have v7.0.0 ids and name SYS in the URL and in the example response") + val create = docWithId(s"OBPv7.0.0-dynamicEntity_create${entityName}_") + (create \ "request_verb").extract[String] should equal("POST") + (create \ "request_url").extract[String] should equal(s"/banks/$SYS/dynamic-entities/$entityName") + (create \ "specified_url").extract[String] should equal(s"/obp/v7.0.0/banks/$SYS/dynamic-entities/$entityName") + (create \ "implemented_by" \ "version").extract[String] should equal("OBPv7.0.0") + (create \ "success_response_body" \ "bank_id").extract[String] should equal(SYS) + + And("the row-level access grant is documented as the PUT it is served as") + val grantAccess = docWithId(s"OBPv7.0.0-dynamicEntity_grant${entityName}RowAccess_") + (grantAccess \ "request_verb").extract[String] should equal("PUT") + (grantAccess \ "request_url").extract[String] should equal(s"/banks/$SYS/dynamic-entities/$entityName/${entityName.toUpperCase}_ID/access") + + And("a bank level entity's docs name its bank") + val list = docWithId(s"OBPv7.0.0-dynamicEntity_get${entityName}List_$bankId") + (list \ "request_url").extract[String] should equal(s"/banks/$bankId/dynamic-entities/$entityName") + (list \ "success_response_body" \ "bank_id").extract[String] should equal(bankId) + + And("the v7.0.0 listing does not also carry the unversioned docs") + operationIds(docs).filter(_.startsWith("OBPv4.0.0-dynamicEntity_")) shouldBe empty + + Then("an older version's listing keeps the unversioned docs and their v4.0.0 ids") + val olderDocs = v4Docs(functions) + operationIds(olderDocs) should contain(s"OBPv4.0.0-dynamicEntity_create${entityName}_") + operationIds(olderDocs).filter(_.startsWith("OBPv7.0.0-")) shouldBe empty + val olderCreate = olderDocs.find(d => (d \ "operation_id").extract[String] == s"OBPv4.0.0-dynamicEntity_create${entityName}_").get + (olderCreate \ "specified_url").extract[String] should equal(s"/obp/dynamic-entity/$entityName") + (olderCreate \ "success_response_body" \ "bank_id") should equal(JNothing) + + And("an older listing's bank level example names the entity's bank as bank_id, as the response does") + val olderList = olderDocs.find(d => (d \ "operation_id").extract[String] == s"OBPv4.0.0-dynamicEntity_get${entityName}List_$bankId") + .getOrElse(fail(s"no v4.0.0 list doc in ${operationIds(olderDocs)}")) + (olderList \ "success_response_body" \ "bank_id").extract[String] should equal(bankId) + (olderList \ "success_response_body" \ "bank-id") should equal(JNothing) + } finally { + cascadeDelete(SYS, systemEntityId) + cascadeDelete(bankId, bankEntityId) + } + } + } } From 785f1a4b74778a1c86a2c8a2b9647daa25fa5891 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 27 Sep 2026 03:57:57 +0200 Subject: [PATCH 2/5] Fix lost Roles on connector traces, config props and updateAtm; Telemetry conventions and Glossary entries; JSON Schema cache keyed by connector name --- docs/telemetry_conventions.md | 264 ++++++++++++++++++ .../main/scala/code/api/util/Glossary.scala | 129 +++++++++ .../code/api/util/JsonSchemaGenerator.scala | 40 ++- .../api/v2_2_0/MessageDocsJsonCache.scala | 26 +- .../scala/code/api/v4_0_0/Http4s400.scala | 2 +- .../scala/code/api/v6_0_0/Http4s600.scala | 4 +- .../scala/code/metrics/ConnectorMetrics.scala | 2 +- obp-api/src/main/scala/code/util/Helper.scala | 12 +- .../main/scala/code/util/SecureLogging.scala | 6 + .../util/JsonSchemaGeneratorCacheTest.scala | 15 + .../test/scala/code/api/v4_0_0/AtmsTest.scala | 13 +- ...ConnectorTraceAndConfigPropsRoleTest.scala | 107 +++++++ .../api/v6_0_0/PerformanceBudgetTest.scala | 127 +++++++++ .../code/util/LoggingCostBudgetTest.scala | 74 +++++ .../parity_allowlist.json | 20 +- 15 files changed, 803 insertions(+), 38 deletions(-) create mode 100644 docs/telemetry_conventions.md create mode 100644 obp-api/src/test/scala/code/api/v6_0_0/ConnectorTraceAndConfigPropsRoleTest.scala create mode 100644 obp-api/src/test/scala/code/api/v6_0_0/PerformanceBudgetTest.scala create mode 100644 obp-api/src/test/scala/code/util/LoggingCostBudgetTest.scala diff --git a/docs/telemetry_conventions.md b/docs/telemetry_conventions.md new file mode 100644 index 0000000000..4ce042d7dd --- /dev/null +++ b/docs/telemetry_conventions.md @@ -0,0 +1,264 @@ +# Telemetry conventions + +This document sets the rules for Telemetry in OBP-API: the aggregated numbers (counts, rates, +durations, sizes, current levels) that tell an operator how the running service is behaving. Apply +it whenever you add a counter, a timer or a cache, or change code on a request path. + +It exists because, before it, each area of the code measured itself in its own way. The NMB sandbox +outage of 2026-09-23 (CPU above 600%, a full heap, hours of garbage-collection pauses) was found by +an external Nagios check, which could say that the service had failed but not why. The numbers that +would have explained it (live thread count, heap left after garbage collection, cache hit ratios, +log queue depth) were either not collected at all or were held in hand-written counters that nothing +exported. + +Status, 2026-09-27: the conventions are agreed; the code is not built yet. Section 11 lists what +exists today and section 12 the assumptions made while the DevOps answers are pending. + +## 1. The name: Telemetry, not metrics + +"Metrics" already has two fixed meanings in OBP, both documented, both exposed through endpoints, +and neither can be renamed: + +| | What it is | Where it lives | Carries identities? | +|---|---|---|---| +| **API Metrics** | one record per API call: who called what, when, how long it took | OBP database, read through the metrics endpoints | yes: consumer, user | +| **Connector Metrics** | one record per connector call | OBP database, read through the connector metrics endpoint | per call | +| **Telemetry** | aggregated numbers about the running instance | served for Prometheus to collect, viewed in Grafana; never stored in the OBP database | never | + +API Metrics and Connector Metrics are records you can query per consumer or per user, close to an +audit log. Telemetry only answers "how is the system behaving", and it is cheap because it never +writes to the database. + +So new code in this area uses the word Telemetry: the package `code.telemetry`, the object +`Telemetry`, the props `telemetry.*`, the Role `CanGetTelemetry`. New classes, objects and props do +not put "Metric" in their own names. Library class names such as Micrometer's `GuavaCacheMetrics` +are fine; we did not name them. Inside Prometheus and Grafana every series is called a "metric"; +that is those tools' vocabulary, not OBP's. + +## 2. Library + +OBP-API records Telemetry through **Micrometer**, which does for measurements what SLF4J does for +logging: one API in the code, with the backend chosen at start-up. + +- For Prometheus, the backend is `micrometer-registry-prometheus`, which renders every series in the + Prometheus text format. +- For OpenTelemetry, the backend is `micrometer-registry-otlp`, which pushes the same series to an + OpenTelemetry Collector. Both can run at once through a `CompositeMeterRegistry`. + +Code outside `code.telemetry` registers meters through the `Telemetry` object, not by calling +Micrometer's global registry directly. That keeps the prefix rule (section 5) and the tag rules +(section 6) in one place. + +## 3. The four types of measurement + +| Type | Question it answers | Examples | +|---|---|---| +| **Counter** (only goes up) | How often? | cache hits, cache misses, times a schema was generated, log entries dropped | +| **Gauge** (a current value) | How much, right now? | queue depth, live thread count, entries in a cache | +| **Distribution summary** (a histogram) | How big, and how spread out? | response body size in bytes, number of items in a returned list, size of a Redis value | +| **Timer** (a histogram of durations) | How long? | endpoint duration, connector call duration, time to build a missing cache entry | + +"The size of a returned object" and "the count in a list" are distribution summaries, not counters: +what matters is the spread per endpoint (the median, the 99th percentile), not a total. + +**Never store a ratio.** Record hits and misses as one counter with a `result` tag (`hit` or `miss`) +and compute the ratio when you query. A stored ratio cannot be added up across instances or across +time windows. + +## 4. What to measure + +Three standard checklists, from the Google Site Reliability Engineering book and the practice that +grew out of it: + +- **RED**, for anything that serves requests (endpoints, connector methods): **R**ate, + **E**rrors, **D**uration. +- **USE**, for anything that is a resource (the database pool, the log dispatch queue, thread + pools, Redis connections): **U**tilisation, **S**aturation (queue length, waiters), + **E**rrors. +- **Caches**: hits, misses, evictions, current size, and the time taken to build a missing entry. + Guava's `recordStats()` records all of these. + +Add the standard JVM figures before anything else: heap used and heap left after garbage +collection, garbage-collection pause time, live thread count, loaded classes. The 2026-09-23 outage +was a saturation failure, and these are the numbers that show saturation. + +## 5. Naming + +- **OBP-API's own series start with `obp_api_`.** The other products keep their own prefixes, + `obp_portal_` and `obp_mcp_`, so their series never collide with ours. +- **Standard library series stay as Micrometer names them**: `jvm_memory_used_bytes`, + `jvm_gc_pause_seconds`, `jvm_threads_live_threads`, `hikaricp_connections_active`, + `http_server_requests_seconds`. The ready-made community Grafana dashboards look for exactly + these names, and would find nothing under a prefix. Which product a series came from is recorded + anyway: Prometheus adds a `job` label per scrape target, and OpenTelemetry adds `service.name`. +- In Micrometer, write names in lowercase with dots: `obp.api.cache.gets`. Micrometer converts them + for Prometheus to `obp_api_cache_gets_total`, adding `_total` to counters itself. Do not write + `_total` into the name. +- Use base units, named in the metric: seconds and bytes, never milliseconds or kilobytes. + Micrometer's timers already report seconds. +- Name the thing measured, not the code that measures it: `obp.api.cache.gets`, not + `obp.api.json_schema_generator.cache_counter`. Put the specific cache in a tag. + +## 6. Tags, and why their values must be few + +Every distinct combination of tag values is stored by Prometheus as a separate series. A tag whose +value can be anything (a user id, a consent id, a raw URL) creates series without limit, and +Prometheus runs out of memory. This is the most common way a metrics system falls over. + +| Allowed tag | Values come from | Example | +|---|---|---| +| `operation` | the ResourceDoc's operation id (a fixed list) | `OBPv7.0.0-getBanks` | +| `api_version` | the fixed list of API versions | `v7.0.0` | +| `status` | the HTTP status class | `2xx`, `4xx`, `5xx` | +| `connector_method` | the connector's method names | `getBankAccount` | +| `cache` | a name given in code | `json_schema`, `message_docs` | +| `result` | a fixed pair or small set | `hit`, `miss` | +| `level` | log levels | `WARN` | + +Never use as a tag: user id, consumer id, consent id, account id, transaction id, customer id, +`api_instance_id` (see section 8), a raw URL or path, a free-text error message, a request header. +Bank id is also not a tag by default: the list of banks grows without an upper limit that the code +controls. Add a bank tag to one specific series only after deciding that its number of banks is +small and fixed. + +## 7. Cost on the request path + +- Counters and timers are cheap (an atomic add). Use them freely. +- Only measure a size where the value already exists. Record response bytes where the body has + already been rendered; never render or serialise something a second time just to measure it. +- Never build a tag value by formatting or serialising anything on the request path. +- Percentile histograms cost memory per series. Turn them on for timers and distribution summaries + at the shared points in section 9, not for every meter. + +## 8. Which instance is reporting + +Each running OBP-API process has its own id, `code.api.Constant.ApiInstanceId` +(`obp-api/src/main/scala/code/api/constant/constant.scala:102`). It is built from the +`api_instance_id` prop: used as it is when the prop ends in `final`, otherwise the prop plus a new +UUID at each start-up, or just a new UUID when the prop is not set. The same id is written to every +API Metrics row (`obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala:230`) and every Redis +log entry (`obp-api/src/main/scala/code/api/cache/RedisLogger.scala:355`), so it ties Telemetry to +those records. + +- **Do not put it on every series as a tag.** Unless the prop ends in `final` it changes on every + restart, so every restart would start a fresh set of series. Prometheus already identifies each + node with its own `instance` label (host and port). +- **Publish it once, in an info series**: `obp_api_instance_info{api_instance_id="…", + git_commit="…"} 1`. The value is always 1; the labels carry the descriptive values, and Grafana + can join on them. +- **Use `Constant.ApiInstanceId`, never the raw prop.** The cache configuration reads the raw + `api_instance_id` prop (`obp-api/src/main/scala/code/api/v6_0_0/JSONFactory6.0.0.scala:2388`) to + build the Redis key namespace, which instances share, so it deliberately has no per-process part. + It names the deployment, not the process. + +## 9. Where to measure: at the shared points, not in each function + +Most of the codebase gets Telemetry without any per-function code, because every request passes +through a few shared points. Instrument those, and a developer adding an endpoint or a connector +method gets RED measurements without doing anything. + +| Shared point | What it records | +|---|---| +| Endpoint middleware: `recordMetric` (`obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala:197`) and `ResourceDocMiddleware` | RED for every endpoint, tagged by `operation`, `api_version` and `status`; response bytes; item count for list responses | +| The connector proxy that intercepts every connector call (`obp-api/src/main/scala/code/bankconnectors/package.scala`) | RED for every connector method, tagged by `connector_method` | +| `Caching.memoize*` (`obp-api/src/main/scala/code/api/cache/Caching.scala:40`, `:52`, `:64`, `:76`) | hits, misses and build time for every memoised function, tagged by `cache` | +| `Redis.use` (`obp-api/src/main/scala/code/api/cache/Redis.scala:203`) | operations, errors, duration and value size, tagged by the Redis command | +| The batch writers for API Metrics and Connector Metrics (`MetricBatchWriter.scala`, `ConnectorMetricBatchWriter.scala`) | queue depth, rows written, rows dropped: Telemetry watching the recording of API Metrics | +| Micrometer's ready-made binders | JVM memory, garbage collection, threads and classes; HikariCP; executor pools (including the log dispatch pool); every Guava `CacheBuilder` cache | + +Hand-written meters are for business logic that no shared point can see, such as "times the JSON +Schema was actually generated". Register them through the `Telemetry` object. + +Any new Guava cache is built with `recordStats()` and registered with `Telemetry`, so that it shows +up without anyone remembering to add it later. + +## 10. How Telemetry is exposed + +**For Prometheus: a separate port on each instance.** + +- Props `telemetry.enabled` (default `false`) and `telemetry.port`, documented in + `sample.props.template`. The default is off because, on a bare host, any open port may be + reachable; each deployment switches it on deliberately. `9464` is the conventional port (it is + the OpenTelemetry Prometheus exporter's default). +- The path is `/telemetry`, not Prometheus's default `/metrics`, to keep the word "metrics" to its + OBP meaning. The scrape configuration sets `metrics_path: /telemetry`. +- The port is served by a small server on its own thread, not by the http4s server, so it keeps + answering while the main request pool is saturated: which is exactly when it is needed. +- The port is never published outside the host or cluster. It needs no credentials because the + network keeps it private. + +Why Prometheus must not scrape through the API itself: behind a load balancer each scrape would +reach a different node, so counters would jump between nodes' values; the request would pass +through authentication, the Role lookup and the request transaction, and so fail under the same +load it is meant to reveal; and every scrape would write an API Metrics row. + +**For people: a Role-gated endpoint.** + +- `GET /obp/v7.0.0/management/telemetry`, Role `CanGetTelemetry` with `requiresBankId = false`. + Telemetry is about the instance, which belongs to no bank, so the Role is held at the empty bank + id, like `CanReadMetrics` and `CanGetConfig`. It is a separate Role from `CanReadMetrics` because + JVM and cache figures are different information from API usage records. +- The response names the instance that answered (`api_instance_id`, section 8) and the build + commit, then gives the figures grouped by area. A reader behind a load balancer can then tell + which node the figures describe. +- It reads the same registry as the port, so the two views cannot disagree. +- Its ResourceDoc description states this instance's actual settings, taken from the props at + start-up (for example, the port Prometheus should scrape and the path, or that the port is off), + and why Prometheus should use the port rather than this endpoint. It gives no instructions on + which props to set; those belong in `sample.props.template`. + +## 11. Tests + +- **Budget tests** make real requests and check what they cost (generator runs, cache hits, log + lines), not only what they return. `PerformanceBudgetTest` and `LoggingCostBudgetTest` are the + first. Once the registry exists they read from it instead of from bespoke getters, so tests and + production dashboards look at the same numbers. +- **A structural test**, in the style of `MappedClassNameTest`, fails when: + - a new `AtomicLong` counter, or a Guava cache not registered with `Telemetry`, appears outside + `code.telemetry`; + - after a suite has run, a registered name breaks section 5, or a tag has more distinct values + than a small fixed limit (section 6). + +Hand-written counters that exist today, to move onto the registry (each keeps its getter until its +callers read the registry): + +| Site | Counter | Meaning | +|---|---|---| +| `obp-api/src/main/scala/code/util/Helper.scala:330` | `mdcLogDropped` | log entries dropped because the dispatch queue was full | +| `obp-api/src/main/scala/code/util/Helper.scala:331` | `mdcLogDispatched` | log entries accepted by the dispatch pool | +| `obp-api/src/main/scala/code/util/Helper.scala:332` | `mdcLogInline` | WARN or ERROR entries written on the caller because the queue was full | +| `obp-api/src/main/scala/code/util/SecureLogging.scala:169` | `maskCallsCounter` | times log masking ran | +| `obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala:82` | `generatorCallsCounter` | times the connector JSON Schema was generated | +| `obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala:62` | `generatorCallsCounter` | times the message-docs response was generated | +| `obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala:63`–`:65` | `sharedGets`, `sharedHits`, `sharedSets` | the shared (Redis) level of the message-docs cache | +| `obp-api/src/main/scala/code/metricsstream/MetricsEventBus.scala:167` | `dropped` | API Metrics stream events dropped | +| `obp-api/src/main/scala/code/logcache/LogCacheEventBus.scala:179` | `dropped` | log cache stream events dropped | +| `obp-api/src/main/scala/code/api/cache/RedisLogger.scala:111` | `consecutiveFailures` | Redis log shipping failures in a row; a gauge, not a counter | + +## 12. Assumptions pending the DevOps answers + +These are working assumptions, to be confirmed or replaced. The code does not depend on any of +them except the fifth, because Micrometer keeps the backend a start-up choice. + +1. Prometheus collects and Grafana displays. OpenTelemetry export and tracing come later. +2. Telemetry is served on a separate port, off by default, never published outside the host or + cluster. +3. Nothing depends on Kubernetes. Outside Kubernetes, Prometheus lists each instance in + `static_configs`; inside it, a `ServiceMonitor` or `PodMonitor` finds them. Only that discovery + setting changes. +4. Nagios stays as the external check of what users experience (up, fast, answering correctly). + Telemetry is the view from inside the service. Page people on symptoms; use Telemetry for + dashboards, diagnosis, and a few alerts on saturation that predicts an outage (heap after + garbage collection staying high, thread count climbing). +5. OBP-API's own series are prefixed `obp_api_` (decided, not an assumption; listed here because + the other products' prefixes depend on it). + +Open questions for DevOps: whether the Prometheus Operator is in use (so `ServiceMonitor` +resources); whether an OpenTelemetry Collector exists or is planned, and when tracing is wanted; +where dashboards and alerts live; and whether a separate port suits, or a path on the main port is +preferred. + +Before OBP-API itself carries Telemetry, the JVM figures can be collected from any running instance +with no code change: the Prometheus JMX exporter is a Java agent added to the JVM start command +(`-javaagent:jmx_prometheus_javaagent.jar=9404:config.yaml`), which serves heap, garbage-collection, +thread and class figures on its own port. Remove it once the Micrometer JVM binders are in. diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 42b73a4582..22ee2358a1 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -6773,6 +6773,135 @@ object Glossary extends MdcLoggable { """) + glossaryItems += GlossaryItem( + title = "API Metrics", + description = + s""" + |# API Metrics + | + |**API Metrics** are OBP-API's record of the calls made to it: one record for every API call, saying who made the call, which endpoint it reached, when, how long it took and what status it returned. They are used to see how the API is being used (which endpoints, which Consumers, which Users), to follow up a particular call, and to review the calls made by a Consumer or by an agent acting under a Consent. + | + |On this instance, API Metrics are ${if (code.metrics.MetricsProps.writeMetrics) "being recorded" else "not being recorded"}. + | + |## What each record contains + | + |- the date and time of the call, and its duration in milliseconds + |- the URL, the HTTP verb and the HTTP status code returned + |- the endpoint that handled the call and the API version it is implemented in + |- the User (user id and username) and the Consumer (consumer id, application name and developer email) + |- how the caller authenticated (for example DirectLogin, OAuth2, Consent or Anonymous) and, for a call made under a Consent, the Consent reference id + |- the correlation id, which is also returned to the caller in the `Correlation-Id` response header and is shared by every Connector call made while serving the call (see [Connector Metrics](/glossary#Connector-Metrics)) + |- the source and target addresses, taken from the `X-Forwarded-For` and `X-Forwarded-Host` request headers + |- the `api_instance_id` of the OBP-API instance that served the call + |- the response body, for selected endpoints only + | + |## How long records are kept + | + |Records are written to the database in batches. ${if (code.metrics.MetricsProps.enableMetricsScheduler) s"On this instance, records stay in the live metrics table for ${code.metrics.MetricsProps.retainMetricsDays} days, are then moved to the metrics archive, and are deleted from the archive after ${code.metrics.MetricsProps.retainArchiveMetricsDays} days." else "On this instance, records are not moved to the archive or deleted automatically."} + | + |## Reading API Metrics + | + |- `GET /management/metrics`: search the records, filtered by date, User, Consumer, endpoint, verb, status and more. Requires the Role CanReadMetrics. + |- `GET /management/aggregate-metrics`: counts and durations over a filtered set of records. Requires the Role CanReadAggregateMetrics. + |- `GET /management/metrics/top-apis`, `/top-consumers` and `/top-users`: the most used endpoints, the most active Consumers and the most active Users. Require the Role CanReadMetrics. + |- `GET /management/metrics/banks/BANK_ID`: the records for calls about one bank. Requires the Role CanGetMetricsAtOneBank at that bank. + |- `GET /my/metrics`: the calling User's own calls, together with the calls made by agents under Consents that User granted. No Role is required. + |- `GET /management/system/diagnostics/metrics`: the state of the metrics table and its archive. Requires the Role CanGetMetricsDiagnostics. + | + |## API Metrics are not Telemetry + | + |API Metrics record individual calls and who made them. [Telemetry](/glossary#Telemetry) is aggregated numbers about how an instance is behaving (request rates, durations, cache hit ratios, memory, threads) and never records who made a call. Use API Metrics to answer "who called what"; use Telemetry to answer "is this instance healthy". + | + |See also: [Connector Metrics](/glossary#Connector-Metrics), [Telemetry](/glossary#Telemetry), [Rate Limiting](/glossary#Rate-Limiting), [Consent](/glossary#Consent). + | +""") + + + glossaryItems += GlossaryItem( + title = "Connector Metrics", + description = + s""" + |# Connector Metrics + | + |**Connector Metrics** are OBP-API's record of the calls it makes to the [Connector](/glossary#Connector), the component that talks to the bank's systems: one record for every Connector call. A single API call can lead to several Connector calls, so Connector Metrics show where the time of an API call went and which calls to the bank's systems failed. + | + |On this instance, Connector Metrics are ${if (APIUtil.getPropsAsBoolValue("write_connector_metrics", false)) "being recorded" else "not being recorded"}. + | + |## What each record contains + | + |- the Connector name and the Connector method called (for example `getBankAccount`) + |- the date and time of the call, and its duration in milliseconds + |- whether the call succeeded + |- the key request parameters of the call + |- the correlation id of the API call it was made for, so the Connector calls behind one API call can be found from its [API Metrics](/glossary#API-Metrics) record + |- the `api_instance_id` of the OBP-API instance that made the call + | + |Records are written to the database in batches. + | + |## Reading Connector Metrics + | + |- `GET /management/connector/metrics`: search the records, filtered by date, Connector name, method and correlation id. Requires the Role CanGetConnectorMetrics. + | + |## Related records + | + |- **Connector call counts** are per-hour counters of Connector calls made and of successful and failed responses, per Connector method, held in Redis rather than in the database. On this instance they are ${if (code.metrics.ConnectorCountsRedis.isEnabled) "being counted" else "not being counted"}. Read them with `GET /management/connector/metrics/counts` (Role CanReadMetrics). + |- **Connector Traces** hold the complete messages sent to and received from the Connector for each call, for debugging. They are much larger than Connector Metrics and can contain customer and account data. On this instance they are ${if (APIUtil.getPropsAsBoolValue("write_connector_trace", false)) "being recorded" else "not being recorded"}. Read them with `GET /management/connector/traces`. + | + |## Connector Metrics are not Telemetry + | + |Connector Metrics record individual Connector calls. [Telemetry](/glossary#Telemetry) reports aggregated numbers about how an instance is behaving, including the rate, errors and duration of Connector calls per method, without keeping a record of each call. + | + |See also: [API Metrics](/glossary#API-Metrics), [Connector](/glossary#Connector), [Connector Method](/glossary#Connector-Method), [Telemetry](/glossary#Telemetry). + | +""") + + + glossaryItems += GlossaryItem( + title = "Telemetry", + description = + s""" + |# Telemetry + | + |**Telemetry** is the set of aggregated numbers that describe how a running OBP-API instance is behaving: how many requests it serves and how long they take, how often its caches answer without recomputing, how full its queues and connection pools are, how much memory it uses and how many threads it runs. Operators use it to see trouble building up (a heap filling, a queue backing up, a cache that has stopped hitting) and to find its cause once something has gone wrong. + | + |## Telemetry is not API Metrics + | + |OBP already uses the word "metrics" for two kinds of per-call record. Telemetry is a different thing and deliberately has a different name: + | + || | What it is | Where it lives | Carries identities? | + ||---|---|---|---| + || **API Metrics** | one record per API call: who called what, when, and how long it took | the OBP database, read through the metrics endpoints | yes: consumer and user | + || **Connector Metrics** | one record per call from OBP-API to the Connector | the OBP database, read through the connector metrics endpoint | per call | + || **Telemetry** | aggregated numbers about the running instance: counts, rates, durations, sizes, current levels | collected from each instance by a monitoring system such as Prometheus and viewed in a tool such as Grafana; never stored in the OBP database | never | + | + |API Metrics answer "who used the API, and how". Telemetry answers "is this instance healthy, and if not, why not". Because Telemetry never records who made a call, it can be collected on every request without writing to the database. + | + |## What Telemetry measures + | + |Telemetry uses four types of measurement: + | + |- **Counters** count how often something happened, for example cache hits and cache misses. + |- **Gauges** report a current level, for example the depth of a queue or the number of live threads. + |- **Distribution summaries** describe how large something is and how widely that varies, for example the size of response bodies or the number of items in a returned list. + |- **Timers** describe how long something takes, for example the duration of an endpoint or a Connector call. + | + |It covers each endpoint (request rate, errors and duration), each Connector method, the caches, Redis, the database connection pool, the log dispatch queue, and the Java virtual machine itself (memory, garbage collection, threads). + | + |## Naming + | + |OBP-API's own Telemetry series start with `obp_api_`, for example `obp_api_cache_gets_total` with a `cache` label naming the cache and a `result` label of `hit` or `miss`. Other Open Bank Project products use their own prefixes. Standard series from the Java virtual machine and the database connection pool keep their usual names (`jvm_*`, `hikaricp_*`), so that standard dashboards work with them. + | + |Telemetry labels only ever take values from small, fixed sets (an operation id, an API version, a status class, a cache name). They never contain a user id, consumer id, consent id, account id or any other identifier of a person or a record. + | + |## Which instance is reporting + | + |Each running OBP-API process has its own `api_instance_id`, the same id that appears on every API Metrics record it writes. Telemetry reports it alongside the build commit, so figures from one instance can be matched with that instance's API Metrics. + | + |See also: [API Metrics](/glossary#API-Metrics), [Connector Metrics](/glossary#Connector-Metrics), [Rate Limiting](/glossary#Rate-Limiting), [Connector](/glossary#Connector), [Resource Doc](/glossary#Resource-Doc). + | +""") + + /////////////////////////////////////////////////////////////////// // NOTE! Some glossary items are generated in ExampleValue.scala ////////////////////////////////////////////////////////////////// diff --git a/obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala b/obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala index 043a6eee50..6fe5b0e382 100644 --- a/obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala +++ b/obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala @@ -30,10 +30,12 @@ package code.api.util import org.json4s._ import code.api.util.APIUtil.MessageDoc import com.openbankproject.commons.util.ReflectUtils -import com.tesobe.CacheKeyFromArguments +import com.google.common.cache.{Cache, CacheBuilder, CacheStats} import org.json4s.JsonDSL._ -import scala.concurrent.duration._ +import java.util.concurrent.Callable +import java.util.concurrent.atomic.AtomicLong + import scala.reflect.runtime.universe._ /** @@ -56,20 +58,34 @@ object JsonSchemaGenerator { * Redis-backed cache in front of this, but that one silently falls through to a full * recompute if Redis is unreachable or slow -- this in-memory layer doesn't depend on * Redis at all, so it stays a working safety net even when Redis is the one struggling. + * + * The key is the connector name only. It must not be derived from `messageDocs`: turning the + * whole list (with every example message) into a key string costs megabytes per call. + * Callers always pass the named connector's own message docs. */ - def messageDocsToJsonSchema(messageDocs: List[MessageDoc], connectorName: String): JObject = { - // This 3-tuple of random UUIDs is a placeholder only -- CacheKeyFromArguments is a macro - // that replaces it at compile time with a real key derived from this method's owner, - // name and arguments (regardless of this method's own arity; the convention throughout - // this codebase is always a 3-tuple here). See: - // https://github.com/OpenBankProject/scala-macros/blob/master/macros/src/main/scala/com/tesobe/CacheKeyFromArgumentsMacro.scala#L49 - var cacheKey = (java.util.UUID.randomUUID().toString, java.util.UUID.randomUUID().toString, java.util.UUID.randomUUID().toString) - CacheKeyFromArguments.buildCacheKey { - code.api.cache.Caching.memoizeSyncWithImMemory(Some(cacheKey.toString()))(100000.days) { + def messageDocsToJsonSchema(messageDocs: List[MessageDoc], connectorName: String): JObject = + try schemaCache.get(connectorName, new Callable[JObject] { + def call(): JObject = { + generatorCallsCounter.incrementAndGet() messageDocsToJsonSchemaUncached(messageDocs, connectorName) } + }) + catch { + // Surface the generator's own exception, not Guava's wrapper. + case e: java.util.concurrent.ExecutionException if e.getCause != null => throw e.getCause + case e: com.google.common.util.concurrent.UncheckedExecutionException if e.getCause != null => throw e.getCause } - } + + private val schemaCache: Cache[String, JObject] = + CacheBuilder.newBuilder().maximumSize(64L).recordStats().build[String, JObject]() + + private val generatorCallsCounter = new AtomicLong(0) + + /** Cache hits and misses, for tests and monitoring. */ + def cacheStats: CacheStats = schemaCache.stats() + + /** Times a schema was actually built (cache misses that reached the reflection walk). */ + def generatorCalls: Long = generatorCallsCounter.get() private def messageDocsToJsonSchemaUncached(messageDocs: List[MessageDoc], connectorName: String): JObject = { val allDefinitions = scala.collection.mutable.Map[String, JObject]() diff --git a/obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala b/obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala index 0ab393ddc3..2fd9bf5976 100644 --- a/obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala +++ b/obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala @@ -1,9 +1,10 @@ package code.api.v2_2_0 import java.util.concurrent.Callable +import java.util.concurrent.atomic.AtomicLong import code.api.cache.Caching -import com.google.common.cache.{Cache, CacheBuilder} +import com.google.common.cache.{Cache, CacheBuilder, CacheStats} import com.openbankproject.commons.util.JsonAliases.{compactRender, parse} import net.liftweb.common.Loggable import org.json4s.JValue @@ -55,22 +56,32 @@ object MessageDocsJsonCache extends Loggable { private def sharedKey(connectorName: String) = s"message-docs-v2.2.0-$connectorName" private val cache: Cache[String, JValue] = - CacheBuilder.newBuilder().maximumSize(MaxEntries).build[String, JValue]() + CacheBuilder.newBuilder().maximumSize(MaxEntries).recordStats().build[String, JValue]() + + // Counters for tests and monitoring. They only ever go up; compare before and after values. + private val generatorCallsCounter = new AtomicLong(0) + private val sharedGetsCounter = new AtomicLong(0) + private val sharedHitsCounter = new AtomicLong(0) + private val sharedSetsCounter = new AtomicLong(0) def getOrCompute(connectorName: String, store: SharedStore = RedisStore)(generate: => JValue): JValue = try cache.get(connectorName, new Callable[JValue] { def call(): JValue = { val key = sharedKey(connectorName) + sharedGetsCounter.incrementAndGet() val fromShared = store.get(key).flatMap { s => try Some(parse(s)) catch { case e: Exception => logger.warn(s"Ignoring unparsable shared message-docs entry $key: ${e.getMessage}"); None } } + if (fromShared.isDefined) sharedHitsCounter.incrementAndGet() fromShared.getOrElse { // Serve the round-tripped form even on the instance that generated it. Otherwise this // instance would return the JValue it built while every other replica (and this one // after a restart) returns parse(compactRender(...)), and number formatting could differ // between them. + generatorCallsCounter.incrementAndGet() val rendered = compactRender(generate) + sharedSetsCounter.incrementAndGet() store.set(key, rendered) parse(rendered) } @@ -85,4 +96,15 @@ object MessageDocsJsonCache extends Loggable { def invalidateAll(): Unit = cache.invalidateAll() def size: Long = cache.size() + + /** In-process level hits and misses (a miss goes on to the shared level). */ + def stats: CacheStats = cache.stats() + + /** Times the response was actually built, i.e. both levels missed. */ + def generatorCalls: Long = generatorCallsCounter.get() + + /** Shared level reads, reads that found a usable entry, and writes. */ + def sharedGets: Long = sharedGetsCounter.get() + def sharedHits: Long = sharedHitsCounter.get() + def sharedSets: Long = sharedSetsCounter.get() } diff --git a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala index 8532d4d0e9..0f8bbb0edb 100644 --- a/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala +++ b/obp-api/src/main/scala/code/api/v4_0_0/Http4s400.scala @@ -6331,7 +6331,7 @@ object Http4s400 { atmJsonV400, List($AuthenticatedUserIsRequired, InvalidJsonFormat, UnknownError), List(apiTagATM), - Some(List(canUpdateAtm, canCreateAtmAtAnyBank)), + Some(List(canUpdateAtm)), http4sPartialFunction = Some(updateAtm) ) } diff --git a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala index 7aa0ad2868..24c61d1efe 100644 --- a/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala +++ b/obp-api/src/main/scala/code/api/v6_0_0/Http4s600.scala @@ -9164,7 +9164,7 @@ object Http4s600 { UnknownError ), List(apiTagMetric, apiTagApi), - None, + Some(List(canGetConnectorTrace)), http4sPartialFunction = Some(getConnectorTraces) ) resourceDocs += ResourceDoc( @@ -13774,7 +13774,7 @@ object Http4s600 { UnknownError ), apiTagApi :: Nil, - None, + Some(List(canGetConfigProps)), http4sPartialFunction = Some(getConfigProps) ) // Intentional drift from Lift's APIMethods600.scala source-of-truth: diff --git a/obp-api/src/main/scala/code/metrics/ConnectorMetrics.scala b/obp-api/src/main/scala/code/metrics/ConnectorMetrics.scala index f5e7dfcc0c..e31537ca6c 100644 --- a/obp-api/src/main/scala/code/metrics/ConnectorMetrics.scala +++ b/obp-api/src/main/scala/code/metrics/ConnectorMetrics.scala @@ -91,7 +91,7 @@ object ConnectorMetrics extends ConnectorMetricsProvider { } override def bulkDeleteConnectorMetrics(): Boolean = { - MappedMetric.bulkDelete_!!() + MappedConnectorMetric.bulkDelete_!!() } } diff --git a/obp-api/src/main/scala/code/util/Helper.scala b/obp-api/src/main/scala/code/util/Helper.scala index 0d23280b29..f1f93ee181 100644 --- a/obp-api/src/main/scala/code/util/Helper.scala +++ b/obp-api/src/main/scala/code/util/Helper.scala @@ -328,6 +328,8 @@ object Helper extends Loggable { // `MdcLogDropReportEvery`, so a sustained overload cannot turn into a stderr flood either. private val MdcLogDropReportEvery = 10000L private val mdcLogDropped = new java.util.concurrent.atomic.AtomicLong(0) + private val mdcLogDispatched = new java.util.concurrent.atomic.AtomicLong(0) + private val mdcLogInline = new java.util.concurrent.atomic.AtomicLong(0) /** * ThreadPoolExecutor and ArrayBlockingQueue reject a size below 1 with IllegalArgumentException. @@ -368,6 +370,12 @@ object Helper extends Loggable { /** Entries dropped because the dispatch queue was full since start-up. */ def mdcLogDroppedCount: Long = mdcLogDropped.get() + /** Entries accepted by the dispatch pool since start-up. */ + def mdcLogDispatchedCount: Long = mdcLogDispatched.get() + + /** WARN/ERROR entries written on the calling thread because the dispatch queue was full. */ + def mdcLogInlineCount: Long = mdcLogInline.get() + /** Entries currently waiting for a dispatch thread. */ def mdcLogQueueDepth: Int = mdcLoggingExecutor.getQueue.size() @@ -397,14 +405,14 @@ object Helper extends Loggable { if (previous == null) org.slf4j.MDC.remove(MdcCallerThreadKey) else org.slf4j.MDC.put(MdcCallerThreadKey, previous) } } - try executor.execute(task) + try { executor.execute(task); mdcLogDispatched.incrementAndGet() } catch { case _: java.util.concurrent.RejectedExecutionException => executor match { // The pool has been shut down (JVM exit): nothing is overloaded, so write the entry // on the caller instead of losing what other shutdown hooks log. case s: java.util.concurrent.ExecutorService if s.isShutdown => task.run() - case _ if critical => task.run() + case _ if critical => mdcLogInline.incrementAndGet(); task.run() case _ => val dropped = mdcLogDropped.incrementAndGet() if (dropped == 1L || dropped % MdcLogDropReportEvery == 0L) diff --git a/obp-api/src/main/scala/code/util/SecureLogging.scala b/obp-api/src/main/scala/code/util/SecureLogging.scala index 2f391369d7..84693787ea 100644 --- a/obp-api/src/main/scala/code/util/SecureLogging.scala +++ b/obp-api/src/main/scala/code/util/SecureLogging.scala @@ -166,7 +166,13 @@ object SecureLogging { customPatternCache.getOrElseUpdate(regex, Pattern.compile(regex, Pattern.CASE_INSENSITIVE)) // ===== Masking Logic ===== + private val maskCallsCounter = new java.util.concurrent.atomic.AtomicLong(0) + + /** Times maskSensitive has run since start-up. Lets tests check that skipped log levels cost nothing. */ + def maskCalls: Long = maskCallsCounter.get() + def maskSensitive(msg: AnyRef): String = { + maskCallsCounter.incrementAndGet() val msgString = Option(msg).map(_.toString).getOrElse("") if (msgString.isEmpty) return msgString diff --git a/obp-api/src/test/scala/code/api/util/JsonSchemaGeneratorCacheTest.scala b/obp-api/src/test/scala/code/api/util/JsonSchemaGeneratorCacheTest.scala index eef09f6d1e..c99b456737 100644 --- a/obp-api/src/test/scala/code/api/util/JsonSchemaGeneratorCacheTest.scala +++ b/obp-api/src/test/scala/code/api/util/JsonSchemaGeneratorCacheTest.scala @@ -103,6 +103,21 @@ class JsonSchemaGeneratorCacheTest extends FlatSpec with Matchers { } } + it should "build the schema once for repeated calls, keyed by connector name only" in { + val connectorName = freshConnectorName("counted") + val generatorBefore = JsonSchemaGenerator.generatorCalls + val hitsBefore = JsonSchemaGenerator.cacheStats.hitCount + + val first = JsonSchemaGenerator.messageDocsToJsonSchema(sampleMessageDocs(connectorName), connectorName) + (1 to 9).foreach(_ => JsonSchemaGenerator.messageDocsToJsonSchema(sampleMessageDocs(connectorName), connectorName)) + // The key must not be derived from the docs list (building that key costs megabytes per + // call on a real connector), so a different list under the same name is still a hit. + JsonSchemaGenerator.messageDocsToJsonSchema(Nil, connectorName) should equal(first) + + JsonSchemaGenerator.generatorCalls - generatorBefore shouldBe 1 + JsonSchemaGenerator.cacheStats.hitCount - hitsBefore shouldBe 10 + } + it should "isolate different connector names as independent cache entries" in { val connectorA = freshConnectorName("connector-a") val connectorB = freshConnectorName("connector-b") diff --git a/obp-api/src/test/scala/code/api/v4_0_0/AtmsTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/AtmsTest.scala index bcc28f2a14..942839eaf0 100644 --- a/obp-api/src/test/scala/code/api/v4_0_0/AtmsTest.scala +++ b/obp-api/src/test/scala/code/api/v4_0_0/AtmsTest.scala @@ -31,7 +31,7 @@ import org.json4s._ import code.api.ResourceDocs1_4_0.SwaggerDefinitionsJSON import code.api.util.APIUtil.OAuth._ import code.api.util.ApiRole -import code.api.util.ApiRole.{canUpdateAtm, canUpdateAtmAtAnyBank} +import code.api.util.ApiRole.{canCreateAtm, canUpdateAtm} import code.api.util.ErrorMessages.{$AuthenticatedUserIsRequired, UserHasMissingRoles} import code.api.v4_0_0.Http4s400.Implementations4_0_0 import code.entitlement.Entitlement @@ -78,9 +78,8 @@ class AtmsTest extends V400ServerSetup { val requestCreateAtmNoRole = (v4_0_0_Request / "banks" /bankId.value / "atms").POST <@ (user1) val responseCreateAtmNoRole = makePostRequest(requestCreateAtmNoRole, write(postAtmJson)) responseCreateAtmNoRole.code should be (403) - responseCreateAtmNoRole.body.extract[ErrorMessage].message.contains(UserHasMissingRoles) - responseCreateAtmNoRole.body.extract[ErrorMessage].message.contains(canUpdateAtm) - responseCreateAtmNoRole.body.extract[ErrorMessage].message.contains(canUpdateAtmAtAnyBank) + responseCreateAtmNoRole.body.extract[ErrorMessage].message should include (UserHasMissingRoles) + responseCreateAtmNoRole.body.extract[ErrorMessage].message should include (canCreateAtm.toString) } scenario("Put - error cases", ApiEndpoint1,ApiEndpoint8, VersionOfApi) { @@ -94,9 +93,7 @@ class AtmsTest extends V400ServerSetup { val requestUpdateAtmNoRole = (v4_0_0_Request / "banks" /bankId.value / "atms"/ "xxx").PUT <@ (user1) val responseUpdateAtmNoRole = makePutRequest(requestUpdateAtmNoRole, write(postAtmJson)) responseUpdateAtmNoRole.code should be (403) - responseUpdateAtmNoRole.body.extract[ErrorMessage].message.contains(UserHasMissingRoles) - responseUpdateAtmNoRole.body.extract[ErrorMessage].message.contains(canUpdateAtm) - responseUpdateAtmNoRole.body.extract[ErrorMessage].message.contains(canUpdateAtmAtAnyBank) + responseUpdateAtmNoRole.body.extract[ErrorMessage].message should equal (UserHasMissingRoles + canUpdateAtm) } } @@ -115,7 +112,7 @@ class AtmsTest extends V400ServerSetup { val atmId = responseBodyCreateAtm.id.getOrElse("") Then("We test the Update Atm") - Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, ApiRole.CanUpdateAtmAtAnyBank.toString) + Entitlement.entitlement.vend.addEntitlement(bankId.value, resourceUser1.userId, ApiRole.CanUpdateAtm.toString) val update = (v4_0_0_Request / "banks" /bankId.value / "atms" / atmId ).PUT <@ (user1) val postAtmJsonUpdate = SwaggerDefinitionsJSON.atmJsonV400.copy(bank_id= testBankId1.value, name="TestATM") diff --git a/obp-api/src/test/scala/code/api/v6_0_0/ConnectorTraceAndConfigPropsRoleTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/ConnectorTraceAndConfigPropsRoleTest.scala new file mode 100644 index 0000000000..5985d94d72 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/ConnectorTraceAndConfigPropsRoleTest.scala @@ -0,0 +1,107 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.api.v6_0_0 + +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole.{CanGetConfigProps, CanGetConnectorTrace} +import code.api.util.ErrorMessages +import code.api.util.ErrorMessages.UserHasMissingRoles +import code.api.v6_0_0.Http4s600.Implementations6_0_0 +import code.entitlement.Entitlement +import code.setup.DefaultUsers +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.ApiVersion +import org.json4s._ +import org.scalatest.Tag + +/** + * This suite checks that the Connector Traces and Config Props endpoints require their Roles. + * + * Both Roles were lost when the endpoints moved from Lift to http4s, which let any logged-in + * user read every connector message (with customer and account data) and every configuration + * value. Each endpoint gets the standard three scenarios: no login, login without the Role, + * and login with the Role. + */ +class ConnectorTraceAndConfigPropsRoleTest extends V600ServerSetup with DefaultUsers { + + object VersionOfApi extends Tag(ApiVersion.v6_0_0.toString) + object GetConnectorTracesEndpoint extends Tag(nameOf(Implementations6_0_0.getConnectorTraces)) + object GetConfigPropsEndpoint extends Tag(nameOf(Implementations6_0_0.getConfigProps)) + + private def connectorTracesRequest = v6_0_0_Request / "management" / "connector" / "traces" + private def configPropsRequest = v6_0_0_Request / "management" / "config-props" + + feature(s"Get Connector Traces - GET /obp/v6.0.0/management/connector/traces - $VersionOfApi") { + + scenario("Anonymous access fails with 401", GetConnectorTracesEndpoint, VersionOfApi) { + val response = makeGetRequest(connectorTracesRequest.GET) + response.code should equal(401) + response.body.extract[ErrorMessage].message should equal(ErrorMessages.AuthenticatedUserIsRequired) + } + + scenario("A logged-in user without CanGetConnectorTrace gets 403", GetConnectorTracesEndpoint, VersionOfApi) { + val response = makeGetRequest(connectorTracesRequest.GET <@ (user1)) + response.code should equal(403) + response.body.extract[ErrorMessage].message should equal(UserHasMissingRoles + CanGetConnectorTrace) + } + + scenario("A user with CanGetConnectorTrace gets 200", GetConnectorTracesEndpoint, VersionOfApi) { + val addedEntitlement = Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanGetConnectorTrace.toString) + val response = try makeGetRequest(connectorTracesRequest.GET <@ (user1)) + finally Entitlement.entitlement.vend.deleteEntitlement(addedEntitlement) + + response.code should equal(200) + response.body \ "connector_traces" shouldBe a[JArray] + } + } + + feature(s"Get Config Props - GET /obp/v6.0.0/management/config-props - $VersionOfApi") { + + scenario("Anonymous access fails with 401", GetConfigPropsEndpoint, VersionOfApi) { + val response = makeGetRequest(configPropsRequest.GET) + response.code should equal(401) + response.body.extract[ErrorMessage].message should equal(ErrorMessages.AuthenticatedUserIsRequired) + } + + scenario("A logged-in user without CanGetConfigProps gets 403", GetConfigPropsEndpoint, VersionOfApi) { + val response = makeGetRequest(configPropsRequest.GET <@ (user1)) + response.code should equal(403) + response.body.extract[ErrorMessage].message should equal(UserHasMissingRoles + CanGetConfigProps) + } + + scenario("A user with CanGetConfigProps gets 200", GetConfigPropsEndpoint, VersionOfApi) { + val addedEntitlement = Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanGetConfigProps.toString) + val response = try makeGetRequest(configPropsRequest.GET <@ (user1)) + finally Entitlement.entitlement.vend.deleteEntitlement(addedEntitlement) + + response.code should equal(200) + response.body \ "config_props" shouldBe a[JArray] + } + } +} diff --git a/obp-api/src/test/scala/code/api/v6_0_0/PerformanceBudgetTest.scala b/obp-api/src/test/scala/code/api/v6_0_0/PerformanceBudgetTest.scala new file mode 100644 index 0000000000..2f591bce17 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v6_0_0/PerformanceBudgetTest.scala @@ -0,0 +1,127 @@ +package code.api.v6_0_0 + +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.classic.{Level, Logger => LogbackLogger} +import ch.qos.logback.core.AppenderBase +import code.api.util.JsonSchemaGenerator +import code.api.v2_2_0.MessageDocsJsonCache +import code.setup.DefaultUsers +import code.util.Helper +import org.slf4j.LoggerFactory + +import java.util.concurrent.{CopyOnWriteArrayList, TimeUnit} +import scala.collection.JavaConverters._ + +/** + * Budget tests: make real HTTP requests and check what they cost (generator runs, cache hits, + * log lines), not only what they return. A change that breaks a cache or adds a log line to a + * hot path fails here instead of showing up as CPU and heap growth in production. + * + * Counters only ever go up and are shared by the whole JVM, so every check compares before + * and after values. Background jobs may log while a scenario runs, which is why the log + * budgets have some slack. + */ +class PerformanceBudgetTest extends V600ServerSetup with DefaultUsers { + + private val Requests = 20 + private val Connector = "rest_vMar2019" + + // Measured, then rounded up. Raise it only with a reason; a jump usually means a new log line + // on the request path. + private val MaxLogLinesPerCachedRequest = 5 + + private def v2_2_0_Request = baseRequest / "obp" / "v2.2.0" + + /** Captures every event reaching the root logger while `body` runs. */ + private def captureLogs(body: => Unit): List[ILoggingEvent] = { + val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[LogbackLogger] + val captured = new CopyOnWriteArrayList[ILoggingEvent]() + val appender = new AppenderBase[ILoggingEvent] { + override def append(event: ILoggingEvent): Unit = captured.add(event) + } + appender.setContext(root.getLoggerContext) + appender.start() + root.addAppender(appender) + try { + body + awaitLogsWritten(captured) + } finally root.detachAppender(appender) + captured.asScala.toList + } + + /** Log writes are asynchronous: wait for the dispatch queue to empty and output to settle. */ + private def awaitLogsWritten(captured: CopyOnWriteArrayList[ILoggingEvent]): Unit = { + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5) + var lastSize = -1 + while ((Helper.mdcLogQueueDepth > 0 || captured.size != lastSize) && System.nanoTime() < deadline) { + lastSize = captured.size + Thread.sleep(100) + } + } + + private def describe(events: List[ILoggingEvent]): String = + events.map(e => s"${e.getLevel} ${e.getLoggerName}: ${e.getFormattedMessage.take(160)}").mkString("\n") + + feature("GET /obp/v2.2.0/message-docs/CONNECTOR stays within its cost budget") { + + scenario("repeated requests build the response once and are then served from memory") { + MessageDocsJsonCache.invalidateAll() + val hitsBefore = MessageDocsJsonCache.stats.hitCount + val missesBefore = MessageDocsJsonCache.stats.missCount + val generatorBefore = MessageDocsJsonCache.generatorCalls + val sharedGetsBefore = MessageDocsJsonCache.sharedGets + + (1 to Requests).foreach { _ => + makeGetRequest((v2_2_0_Request / "message-docs" / Connector).GET).code should equal(200) + } + + MessageDocsJsonCache.stats.missCount - missesBefore shouldBe 1 + MessageDocsJsonCache.stats.hitCount - hitsBefore shouldBe (Requests - 1) + // 0 when an earlier run left the response in Redis, otherwise 1. + (MessageDocsJsonCache.generatorCalls - generatorBefore) should be <= 1L + MessageDocsJsonCache.sharedGets - sharedGetsBefore shouldBe 1 + } + + scenario("unknown connector names are rejected before they reach the cache") { + makeGetRequest((v2_2_0_Request / "message-docs" / Connector).GET).code should equal(200) + val sizeBefore = MessageDocsJsonCache.size + val generatorBefore = MessageDocsJsonCache.generatorCalls + + (1 to Requests).foreach { i => + makeGetRequest((v2_2_0_Request / "message-docs" / s"no_such_connector_$i").GET).code should not equal 200 + } + + MessageDocsJsonCache.size shouldBe sizeBefore + MessageDocsJsonCache.generatorCalls shouldBe generatorBefore + } + + scenario("a cached request writes no warnings or errors and few log lines") { + makeGetRequest((v2_2_0_Request / "message-docs" / Connector).GET).code should equal(200) // warm up + + val events = captureLogs { + (1 to Requests).foreach { _ => + makeGetRequest((v2_2_0_Request / "message-docs" / Connector).GET).code should equal(200) + } + } + + withClue(s"captured ${events.size} log events:\n${describe(events)}\n") { + events.filter(_.getLevel.isGreaterOrEqual(Level.WARN)) shouldBe empty + events.size should be <= (Requests * MaxLogLinesPerCachedRequest) + } + } + } + + feature("GET /obp/v6.0.0/message-docs/CONNECTOR/json-schema stays within its cost budget") { + + scenario("repeated requests build the schema at most once") { + val generatorBefore = JsonSchemaGenerator.generatorCalls + + (1 to Requests).foreach { _ => + makeGetRequest((v6_0_0_Request / "message-docs" / Connector / "json-schema").GET).code should equal(200) + } + + // Redis sits in front of the in-process cache; either way the schema is built at most once. + (JsonSchemaGenerator.generatorCalls - generatorBefore) should be <= 1L + } + } +} diff --git a/obp-api/src/test/scala/code/util/LoggingCostBudgetTest.scala b/obp-api/src/test/scala/code/util/LoggingCostBudgetTest.scala new file mode 100644 index 0000000000..90157e712a --- /dev/null +++ b/obp-api/src/test/scala/code/util/LoggingCostBudgetTest.scala @@ -0,0 +1,74 @@ +package code.util + +import ch.qos.logback.classic.spi.ILoggingEvent +import ch.qos.logback.classic.{Level, Logger => LogbackLogger} +import ch.qos.logback.core.AppenderBase +import org.scalatest.{FlatSpec, Matchers} +import org.slf4j.LoggerFactory + +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.{CopyOnWriteArrayList, TimeUnit} + +/** + * Budget tests for MdcLoggable: a log call below the enabled level must cost nothing (the + * message is never built and never masked), and an enabled one must be built, masked and + * written exactly once. + * + * SecureLogging.maskCalls counts for the whole JVM, and other threads may log while this runs, + * so the skipped-level check allows a small amount of unrelated masking. + */ +class LoggingCostBudgetTest extends FlatSpec with Matchers { + + private object Probe extends Helper.MdcLoggable { + def debug(msg: => AnyRef): Unit = logger.debug(msg) + } + + private val Calls = 1000 + + private def probeLogger = LoggerFactory.getLogger(Probe.getClass.getName).asInstanceOf[LogbackLogger] + + private def withLevel[T](level: Level)(body: => T): T = { + val original = probeLogger.getLevel + probeLogger.setLevel(level) + try body finally probeLogger.setLevel(original) + } + + "A DEBUG call on an INFO logger" should "never build or mask its message" in withLevel(Level.INFO) { + val built = new AtomicInteger(0) + val maskBefore = SecureLogging.maskCalls + val dispatchedBefore = Helper.mdcLogDispatchedCount + + (1 to Calls).foreach(i => Probe.debug { built.incrementAndGet(); s"skipped $i" }) + + built.get shouldBe 0 + withClue("unrelated threads may log meanwhile, but not once per skipped call: ") { + (SecureLogging.maskCalls - maskBefore) should be < (Calls / 10).toLong + (Helper.mdcLogDispatchedCount - dispatchedBefore) should be < (Calls / 10).toLong + } + } + + "A DEBUG call on a DEBUG logger" should "build, mask and write its message exactly once" in withLevel(Level.DEBUG) { + val built = new AtomicInteger(0) + val captured = new CopyOnWriteArrayList[ILoggingEvent]() + val appender = new AppenderBase[ILoggingEvent] { + override def append(event: ILoggingEvent): Unit = captured.add(event) + } + appender.setContext(probeLogger.getLoggerContext) + appender.start() + probeLogger.addAppender(appender) + val maskBefore = SecureLogging.maskCalls + val droppedBefore = Helper.mdcLogDroppedCount + + try { + (1 to Calls).foreach(i => Probe.debug { built.incrementAndGet(); s"written $i" }) + + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10) + while (captured.size < Calls && System.nanoTime() < deadline) Thread.sleep(20) + + built.get shouldBe Calls + captured.size shouldBe Calls + (SecureLogging.maskCalls - maskBefore) should be >= Calls.toLong + (Helper.mdcLogDroppedCount - droppedBefore) shouldBe 0L + } finally probeLogger.detachAppender(appender) + } +} diff --git a/scripts/resource_doc_baseline/parity_allowlist.json b/scripts/resource_doc_baseline/parity_allowlist.json index 402ecf6be8..f4a8981973 100644 --- a/scripts/resource_doc_baseline/parity_allowlist.json +++ b/scripts/resource_doc_baseline/parity_allowlist.json @@ -1183,41 +1183,41 @@ "version": "v6_0_0", "endpoint": "createBankLevelDynamicEntity", "field": "description", - "reason": "Digest refresh: description further extended with write_role_required/read_role_required/write_role/read_role field-level access control docs (real feature, per DynamicEntityAuthModeTest.scala/DynamicEntityProvider.scala) on top of the indexed/index docs already reviewed in Card 2.", + "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.", "lift_digest": "2e7c649041362776f7eec66ccbb9f025f3edfead992108d0fd5864af76e93961", - "http4s_digest": "a520ce22176587451f8d8d6bf6886bff796789885d37954bcf3229e1d8eb3f07" + "http4s_digest": "2177cf2f3d19eed21d73bacb62cf340e195298eb334857a67e24b4ccdaab61ce" }, { "version": "v6_0_0", "endpoint": "createSystemDynamicEntity", "field": "description", - "reason": "Digest refresh: description further extended with write_role_required/read_role_required/write_role/read_role field-level access control docs (real feature, per DynamicEntityAuthModeTest.scala/DynamicEntityProvider.scala) on top of the indexed/index docs already reviewed in Card 2.", + "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.", "lift_digest": "81310cdd2be1c2b33d465cabd1604cedee38bfa1ecccae4b016d0f99aa443ab6", - "http4s_digest": "da38d765f6c3d2d20e3ee1926b694379992f3516bbd216eb577ea87a31f05ff1" + "http4s_digest": "a216f7033fb45112130ff7ae7acee56b02ee0443d428e60fcaee1415ba4083b4" }, { "version": "v6_0_0", "endpoint": "updateBankLevelDynamicEntity", "field": "description", - "reason": "Digest refresh: description further extended with write_role_required/read_role_required/write_role/read_role field-level access control docs (real feature, per DynamicEntityAuthModeTest.scala/DynamicEntityProvider.scala) on top of the indexed/index docs already reviewed in Card 2.", + "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.", "lift_digest": "2d18665f9db52506a1b043ad7575b16ac52cedf0d11f036afd4a96b353eb8ddd", - "http4s_digest": "deb202032cdb70e656bba7d2b1ccce8e684040028cfa9a8dca5af32899d15919" + "http4s_digest": "0853c5554b070120aa477631bb4ce97f36b60e09acaad61f64199d407fd3d1da" }, { "version": "v6_0_0", "endpoint": "updateMyDynamicEntity", "field": "description", - "reason": "Digest refresh: description further extended with write_role_required/read_role_required/write_role/read_role field-level access control docs (real feature, per DynamicEntityAuthModeTest.scala/DynamicEntityProvider.scala) on top of the indexed/index docs already reviewed in Card 2.", + "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.", "lift_digest": "97f0c97cc592f89f4dba8a6641ad4b41f6ffa6f4f6e78bf3d3cc34a6a11f76e7", - "http4s_digest": "c416426234c796ac9b38889b4672ba90e20f956e3fc0bf198353d423cc9ece23" + "http4s_digest": "c8b602bc69dc86b8eb92e52f18e0a2714ef0d14d5de560186da8a372c6af2ecf" }, { "version": "v6_0_0", "endpoint": "updateSystemDynamicEntity", "field": "description", - "reason": "Digest refresh: description further extended with write_role_required/read_role_required/write_role/read_role field-level access control docs (real feature, per DynamicEntityAuthModeTest.scala/DynamicEntityProvider.scala) on top of the indexed/index docs already reviewed in Card 2.", + "reason": "Digest refresh: row-level access grant endpoint corrected from GET/POST to GET/PUT (matches Method.PUT route in Http4sDynamicEntity.scala), on top of the field-level access control and indexed/index docs already reviewed.", "lift_digest": "e28c2996f3fd54842534fd54d45d7ba49df8b346e65f541353be4fb5311b550c", - "http4s_digest": "5412357d39eff3acd773dcccb1d2675253d1261424c3d9aa37bf28fd99a37a10" + "http4s_digest": "787e40986d922b9f0d28ce3f1b3f5110f8521bc2a26fab8b3923e6851c41bfc4" }, { "version": "v3_0_0", From af35a71df62c0e882dae9a3e64d9dab082a80b8b Mon Sep 17 00:00:00 2001 From: simonredfern Date: Sun, 27 Sep 2026 09:32:23 +0200 Subject: [PATCH 3/5] Telemetry: Micrometer registry, separate /telemetry port for Prometheus, v7.0.0 GET /management/telemetry with CanGetTelemetry, guard tests + release --- docs/telemetry_conventions.md | 90 +++++--- obp-api/pom.xml | 12 + .../resources/props/sample.props.template | 12 + .../main/scala/bootstrap/liftweb/Boot.scala | 5 + .../bootstrap/liftweb/CustomDBVendor.scala | 3 + .../main/scala/code/api/cache/InMemory.scala | 4 +- .../src/main/scala/code/api/cache/Redis.scala | 9 +- .../scala/code/api/cache/RedisLogger.scala | 3 + .../main/scala/code/api/util/ApiRole.scala | 5 + .../main/scala/code/api/util/Glossary.scala | 5 + .../code/api/util/JsonSchemaGenerator.scala | 4 +- .../util/http4s/ResourceDocMiddleware.scala | 19 +- .../api/v2_2_0/MessageDocsJsonCache.scala | 4 +- .../scala/code/api/v7_0_0/Http4s700.scala | 3 + .../code/api/v7_0_0/Http4s700Telemetry.scala | 121 ++++++++++ .../code/api/v7_0_0/JSONFactory7.0.0.scala | 60 +++++ .../scala/code/bankconnectors/package.scala | 1 + .../code/logcache/LogCacheEventBus.scala | 4 + .../code/metricsstream/MetricsEventBus.scala | 4 + .../main/scala/code/telemetry/Telemetry.scala | 214 ++++++++++++++++++ .../code/telemetry/TelemetryBindings.scala | 74 ++++++ .../src/main/scala/code/users/LiftUsers.scala | 5 +- .../api/v7_0_0/TelemetryEndpointTest.scala | 112 +++++++++ .../telemetry/TelemetryConventionsTest.scala | 116 ++++++++++ .../scala/code/telemetry/TelemetryTest.scala | 77 +++++++ release_notes.md | 23 ++ 26 files changed, 953 insertions(+), 36 deletions(-) create mode 100644 obp-api/src/main/scala/code/api/v7_0_0/Http4s700Telemetry.scala create mode 100644 obp-api/src/main/scala/code/telemetry/Telemetry.scala create mode 100644 obp-api/src/main/scala/code/telemetry/TelemetryBindings.scala create mode 100644 obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala create mode 100644 obp-api/src/test/scala/code/telemetry/TelemetryConventionsTest.scala create mode 100644 obp-api/src/test/scala/code/telemetry/TelemetryTest.scala diff --git a/docs/telemetry_conventions.md b/docs/telemetry_conventions.md index 4ce042d7dd..ec51a53280 100644 --- a/docs/telemetry_conventions.md +++ b/docs/telemetry_conventions.md @@ -11,8 +11,9 @@ would have explained it (live thread count, heap left after garbage collection, log queue depth) were either not collected at all or were held in hand-written counters that nothing exported. -Status, 2026-09-27: the conventions are agreed; the code is not built yet. Section 11 lists what -exists today and section 12 the assumptions made while the DevOps answers are pending. +Status, 2026-09-27: the first part is built (`code.telemetry`, the separate port, the v7.0.0 +endpoint and the guard tests). Section 13 lists what is recorded today and what is not yet, and +section 12 the assumptions made while the DevOps answers are pending. ## 1. The name: Telemetry, not metrics @@ -176,8 +177,10 @@ up without anyone remembering to add it later. **For Prometheus: a separate port on each instance.** -- Props `telemetry.enabled` (default `false`) and `telemetry.port`, documented in - `sample.props.template`. The default is off because, on a bare host, any open port may be +- Props `telemetry.port.enabled` (default `false`), `telemetry.host` (default `0.0.0.0`) and + `telemetry.port` (default `9464`), documented in `sample.props.template`. They control the port + only: Telemetry is always recorded, because recording costs an atomic add and the endpoint below + reads the same registry. The port is off by default because, on a bare host, any open port may be reachable; each deployment switches it on deliberately. `9464` is the conventional port (it is the OpenTelemetry Prometheus exporter's default). - The path is `/telemetry`, not Prometheus's default `/metrics`, to keep the word "metrics" to its @@ -199,8 +202,9 @@ load it is meant to reveal; and every scrape would write an API Metrics row. id, like `CanReadMetrics` and `CanGetConfig`. It is a separate Role from `CanReadMetrics` because JVM and cache figures are different information from API usage records. - The response names the instance that answered (`api_instance_id`, section 8) and the build - commit, then gives the figures grouped by area. A reader behind a load balancer can then tell - which node the figures describe. + commit, then lists every meter with its tags and current values, sorted by name. The query + parameter `name_prefix` narrows the list to one area, for example `?name_prefix=obp.api.endpoint`. + A reader behind a load balancer can tell which node the figures describe. - It reads the same registry as the port, so the two views cannot disagree. - Its ResourceDoc description states this instance's actual settings, taken from the props at start-up (for example, the port Prometheus should scrape and the path, or that the port is off), @@ -213,27 +217,36 @@ load it is meant to reveal; and every scrape would write an API Metrics row. lines), not only what they return. `PerformanceBudgetTest` and `LoggingCostBudgetTest` are the first. Once the registry exists they read from it instead of from bespoke getters, so tests and production dashboards look at the same numbers. -- **A structural test**, in the style of `MappedClassNameTest`, fails when: - - a new `AtomicLong` counter, or a Guava cache not registered with `Telemetry`, appears outside - `code.telemetry`; - - after a suite has run, a registered name breaks section 5, or a tag has more distinct values - than a small fixed limit (section 6). - -Hand-written counters that exist today, to move onto the registry (each keeps its getter until its -callers read the registry): - -| Site | Counter | Meaning | +- **`TelemetryConventionsTest`** (`obp-api/src/test/scala/code/telemetry/`) holds the code to this + document. It fails when: + - a file outside `code.telemetry` holds more `AtomicLong(` counters than its allowlist entry + (the allowlist only shrinks; a lower count than the entry also fails, so the entry is lowered); + - a file builds a Guava cache (`CacheBuilder.newBuilder`) without wrapping it in + `Telemetry.monitorCache`; + - the running server registered a meter that is neither `obp.api.*` nor a standard family + (`jvm.`, `process.`, `system.`, `hikaricp.`, `cache.`); + - a series carries a tag key that names an identifier (user, consumer, consent, account, + transaction, customer, bank, URL, path, correlation id, `api_instance_id` outside the info + series). + A limit on the number of distinct values per tag is not checked yet. +- **`TelemetryTest`** checks the naming rule, the Prometheus form of counters, endpoint and Connector + timers (including the fixed buckets), and the separate port (served at `/telemetry`, 404 + elsewhere), without a server. +- **`TelemetryEndpointTest`** checks the v7.0.0 endpoint: 401, 403 without the Role, the instance + id, the standard and start-up meters, the middleware's count of the endpoint's own requests, and + `name_prefix`. + +The hand-written counters that existed when Telemetry arrived are now exported through +`TelemetryBindings`, which reads their getters, so the code that counts is unchanged: + +| Site | Counter | Exported as | |---|---|---| -| `obp-api/src/main/scala/code/util/Helper.scala:330` | `mdcLogDropped` | log entries dropped because the dispatch queue was full | -| `obp-api/src/main/scala/code/util/Helper.scala:331` | `mdcLogDispatched` | log entries accepted by the dispatch pool | -| `obp-api/src/main/scala/code/util/Helper.scala:332` | `mdcLogInline` | WARN or ERROR entries written on the caller because the queue was full | -| `obp-api/src/main/scala/code/util/SecureLogging.scala:169` | `maskCallsCounter` | times log masking ran | -| `obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala:82` | `generatorCallsCounter` | times the connector JSON Schema was generated | -| `obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala:62` | `generatorCallsCounter` | times the message-docs response was generated | -| `obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala:63`–`:65` | `sharedGets`, `sharedHits`, `sharedSets` | the shared (Redis) level of the message-docs cache | -| `obp-api/src/main/scala/code/metricsstream/MetricsEventBus.scala:167` | `dropped` | API Metrics stream events dropped | -| `obp-api/src/main/scala/code/logcache/LogCacheEventBus.scala:179` | `dropped` | log cache stream events dropped | -| `obp-api/src/main/scala/code/api/cache/RedisLogger.scala:111` | `consecutiveFailures` | Redis log shipping failures in a row; a gauge, not a counter | +| `obp-api/src/main/scala/code/util/Helper.scala:330`–`:332` | `mdcLogDropped`, `mdcLogDispatched`, `mdcLogInline` | `obp.api.log.dispatch.entries{result=dropped/dispatched/inline}`; queue depth as `obp.api.log.dispatch.queue.depth` | +| `obp-api/src/main/scala/code/util/SecureLogging.scala:169` | `maskCallsCounter` | `obp.api.log.masking.calls` | +| `obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala:82` | `generatorCallsCounter` | `obp.api.json_schema.generations` | +| `obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala:62`–`:65` | `generatorCallsCounter`, `sharedGets`, `sharedHits`, `sharedSets` | `obp.api.message_docs.generations`, `obp.api.message_docs.shared.gets{result=hit/miss}`, `obp.api.message_docs.shared.sets` | +| `obp-api/src/main/scala/code/metricsstream/MetricsEventBus.scala` and `code/logcache/LogCacheEventBus.scala` | per-subscriber `dropped` (logged only) | `obp.api.stream.messages.dropped{stream=metrics/log_cache}`, counted at the drop site across all subscribers | +| `obp-api/src/main/scala/code/api/cache/RedisLogger.scala:111` | `consecutiveFailures` | `obp.api.redis_logger.consecutive_failures` (a gauge) | ## 12. Assumptions pending the DevOps answers @@ -262,3 +275,28 @@ Before OBP-API itself carries Telemetry, the JVM figures can be collected from a with no code change: the Prometheus JMX exporter is a Java agent added to the JVM start command (`-javaagent:jmx_prometheus_javaagent.jar=9404:config.yaml`), which serves heap, garbage-collection, thread and class figures on its own port. Remove it once the Micrometer JVM binders are in. + +## 13. What is recorded today + +| Meter | Type | Tags | Recorded at | +|---|---|---|---| +| `obp.api.endpoint.requests` | timer, fixed buckets | `operation`, `api_version`, `status` | `ResourceDocMiddleware`, once per request, by the hop that matched a ResourceDoc | +| `obp.api.endpoint.response.size` | distribution summary, bytes | `operation` | the same, when the response states its length | +| `obp.api.connector.calls` | timer, fixed buckets | `connector`, `connector_method`, `result` | the Connector proxy (`code/bankconnectors/package.scala`) | +| `obp.api.redis.commands` | timer | `command`, `result` | `Redis.use` | +| `cache.gets`, `cache.puts`, `cache.evictions`, `cache.size` | standard | `cache` = `in_memory`, `json_schema`, `message_docs`, `on_behalf_of` | every Guava cache, through `Telemetry.monitorCache` | +| `hikaricp.*` | standard | `pool` | the database pool (`CustomDBVendor`) | +| `jvm.*`, `process.*`, `system.*` | standard | | JVM memory, heap after garbage collection (`jvm.memory.usage.after.gc`), garbage collection, threads, classes, CPU, uptime, open files | +| `obp.api.instance.info` | gauge, always 1 | `api_instance_id`, `git_commit` | start-up | +| the counters in section 11 | | | `TelemetryBindings` | + +Not recorded yet: +- hits and misses of `Caching.memoize*` with the Redis provider (the in-memory provider is covered + through the `in_memory` cache); +- the batch writers for API Metrics and Connector Metrics: their queues are + `ConcurrentLinkedQueue`s, whose size costs a walk of the whole queue, so they need their own + counters rather than a gauge on `size()`; +- item counts of list responses; +- endpoints without a ResourceDoc (Dynamic Entity records, Dynamic Endpoints, the unversioned + routes), which do not pass through the matching branch of `ResourceDocMiddleware`. + diff --git a/obp-api/pom.xml b/obp-api/pom.xml index 7ee2c04c29..2d5203ced5 100644 --- a/obp-api/pom.xml +++ b/obp-api/pom.xml @@ -344,6 +344,18 @@ HikariCP 4.0.3 + + + io.micrometer + micrometer-core + 1.17.1 + + + io.micrometer + micrometer-registry-prometheus + 1.17.1 + diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index a4e70dd12b..de3976c209 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -257,6 +257,18 @@ write_connector_metrics=false ## Enable writing connector traces (full outbound/inbound message payloads per call) to RDBMS table `connector_trace`. Verbose — keep off in prod unless debugging. write_connector_trace=false +## Telemetry: aggregated numbers about this instance (request rates and durations per endpoint, +## Connector calls, caches, the database pool, memory, garbage collection, threads) for Prometheus. +## Not API Metrics: see docs/telemetry_conventions.md. Telemetry is always recorded; these props +## only decide whether a separate port serves it. Prometheus scrapes each instance's port at the +## path /telemetry (set `metrics_path: /telemetry` in the scrape configuration). Never publish the +## port outside the host or cluster: it has no authentication. People can also read Telemetry +## through GET /obp/v7.0.0/management/telemetry with the Role CanGetTelemetry. +## Defaults live in code.telemetry.Telemetry. +# telemetry.port.enabled=false +# telemetry.host=0.0.0.0 +# telemetry.port=9464 + ## ElasticSearch #allow_elasticsearch=true #allow_elasticsearch_warehouse=true diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index 7ce66a3e92..714cca0964 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -578,6 +578,11 @@ class Boot extends MdcLoggable { // Sandbox account creation menu removed - API-only mode, no portal pages + // Telemetry (aggregated numbers about this instance, for Prometheus). Not API Metrics: see + // docs/telemetry_conventions.md. Recording is always on; the prop telemetry.port.enabled + // decides whether the separate port is opened. + code.telemetry.Telemetry.start() + // API Metrics (logs of API calls) // If set to true we will write each URL with params to a datastore / log file if (code.metrics.MetricsProps.writeMetrics) { diff --git a/obp-api/src/main/scala/bootstrap/liftweb/CustomDBVendor.scala b/obp-api/src/main/scala/bootstrap/liftweb/CustomDBVendor.scala index 94573ccf83..fe5370c93e 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/CustomDBVendor.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/CustomDBVendor.scala @@ -87,6 +87,9 @@ class CustomDBVendor(driverName: String, config.addDataSourceProperty("prepStmtCacheSize", "250") config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048") + // Telemetry: the pool's connections, waits and timeouts, as the standard hikaricp_* series. + config.setMetricRegistry(code.telemetry.Telemetry.registry) + val ds: HikariDataSource = new HikariDataSource(config) } diff --git a/obp-api/src/main/scala/code/api/cache/InMemory.scala b/obp-api/src/main/scala/code/api/cache/InMemory.scala index fddd50e1d4..5773c18504 100644 --- a/obp-api/src/main/scala/code/api/cache/InMemory.scala +++ b/obp-api/src/main/scala/code/api/cache/InMemory.scala @@ -44,8 +44,8 @@ object InMemory extends MdcLoggable { // a single Guava instance has to serve every one of them. The underlying store is declared at // Entry[Any] and narrowed per call: the cast is erased at run time, and a given key always holds // the type its own call site wrote, which is the same assumption the untyped ScalaCache made. - val underlyingGuavaCache: GuavaUnderlying[String, Entry[Any]] = - CacheBuilder.newBuilder().maximumSize(100000L).build[String, Entry[Any]]() + val underlyingGuavaCache: GuavaUnderlying[String, Entry[Any]] = code.telemetry.Telemetry.monitorCache( + CacheBuilder.newBuilder().maximumSize(100000L).recordStats().build[String, Entry[Any]](), "in_memory") // Built once, for the same reason as Redis's: the wrapper holds no per-type state and the cast is // erased, so one instance serves every A instead of one allocation per cache read. diff --git a/obp-api/src/main/scala/code/api/cache/Redis.scala b/obp-api/src/main/scala/code/api/cache/Redis.scala index 2f981566ab..bf869183d2 100644 --- a/obp-api/src/main/scala/code/api/cache/Redis.scala +++ b/obp-api/src/main/scala/code/api/cache/Redis.scala @@ -208,6 +208,8 @@ object Redis extends MdcLoggable { if(ttlSeconds.equals(Some(0))){ // set ttl = 0, we will totally turn off the cache None }else{ + val startNanos = System.nanoTime() + var succeeded = false try { jedisConnection = Some(jedisPool.getResource()) @@ -236,13 +238,18 @@ object Redis extends MdcLoggable { throw new RuntimeException("Please check the Redis.use parameters, if the method == set, the value can not be None !!!") } //change the null to Option - APIUtil.stringOrNone(redisResult) + val result = APIUtil.stringOrNone(redisResult) + succeeded = true + result } catch { case e: Throwable => throw new RuntimeException(e) } finally { if (jedisConnection.isDefined && jedisConnection.get != null) jedisConnection.map(_.close()) + code.telemetry.Telemetry.timer("obp.api.redis.commands", + "command" -> method.toString, "result" -> (if (succeeded) "success" else "error")) + .record(System.nanoTime() - startNanos, java.util.concurrent.TimeUnit.NANOSECONDS) } } } diff --git a/obp-api/src/main/scala/code/api/cache/RedisLogger.scala b/obp-api/src/main/scala/code/api/cache/RedisLogger.scala index 723cc58c5c..0002b3e8c4 100644 --- a/obp-api/src/main/scala/code/api/cache/RedisLogger.scala +++ b/obp-api/src/main/scala/code/api/cache/RedisLogger.scala @@ -109,6 +109,9 @@ object RedisLogger { // Circuit breaker state private val consecutiveFailures = new AtomicLong(0) + + /** Failed shipments in a row since the last success. Telemetry reports it as a gauge. */ + def consecutiveFailureCount: Long = consecutiveFailures.get() private val circuitBreakerOpen = new AtomicBoolean(false) private var lastFailureTime = 0L diff --git a/obp-api/src/main/scala/code/api/util/ApiRole.scala b/obp-api/src/main/scala/code/api/util/ApiRole.scala index aaa383e3c7..27236f080d 100644 --- a/obp-api/src/main/scala/code/api/util/ApiRole.scala +++ b/obp-api/src/main/scala/code/api/util/ApiRole.scala @@ -564,6 +564,11 @@ object ApiRole extends MdcLoggable{ case class CanGetConfigProps(requiresBankId: Boolean = false) extends ApiRole lazy val canGetConfigProps = CanGetConfigProps() + // Telemetry is about the running instance, which belongs to no bank, so the Role is held at the + // empty bank id. It is separate from CanReadMetrics: JVM and cache figures are not API usage records. + case class CanGetTelemetry(requiresBankId: Boolean = false) extends ApiRole + lazy val canGetTelemetry = CanGetTelemetry() + case class CanGetSignalStats(requiresBankId: Boolean = false) extends ApiRole lazy val canGetSignalStats = CanGetSignalStats() diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 22ee2358a1..a140ab3bc0 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -6897,6 +6897,11 @@ object Glossary extends MdcLoggable { | |Each running OBP-API process has its own `api_instance_id`, the same id that appears on every API Metrics record it writes. Telemetry reports it alongside the build commit, so figures from one instance can be matched with that instance's API Metrics. | + |## Reading Telemetry + | + |- **Prometheus** collects Telemetry from a separate port of each OBP-API instance, at the path `/telemetry`, in the Prometheus text format. Reading each instance directly keeps figures from different instances apart, and the port keeps answering when the API itself is overloaded. ${if (code.telemetry.Telemetry.portSettings.enabled) s"On this instance the port is open, on port ${code.telemetry.Telemetry.portSettings.port}." else "On this instance the port is not open."} + |- **People** can read the same figures with `GET /obp/v7.0.0/management/telemetry`, which requires the Role CanGetTelemetry. Its response names the instance that answered. + | |See also: [API Metrics](/glossary#API-Metrics), [Connector Metrics](/glossary#Connector-Metrics), [Rate Limiting](/glossary#Rate-Limiting), [Connector](/glossary#Connector), [Resource Doc](/glossary#Resource-Doc). | """) diff --git a/obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala b/obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala index 6fe5b0e382..222033f13f 100644 --- a/obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala +++ b/obp-api/src/main/scala/code/api/util/JsonSchemaGenerator.scala @@ -76,8 +76,8 @@ object JsonSchemaGenerator { case e: com.google.common.util.concurrent.UncheckedExecutionException if e.getCause != null => throw e.getCause } - private val schemaCache: Cache[String, JObject] = - CacheBuilder.newBuilder().maximumSize(64L).recordStats().build[String, JObject]() + private val schemaCache: Cache[String, JObject] = code.telemetry.Telemetry.monitorCache( + CacheBuilder.newBuilder().maximumSize(64L).recordStats().build[String, JObject](), "json_schema") private val generatorCallsCounter = new AtomicLong(0) diff --git a/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala index 62e89a37ab..b710cded52 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala @@ -197,7 +197,8 @@ object ResourceDocMiddleware extends MdcLoggable { else RequestScopeConnection.withBusinessDBTransaction(routeIO) executed.map(Option(_)) } - OptionT(work.timeoutTo(endpointTimeoutMs.millis, endpointTimeoutResponse(req))) + val startNanos = System.nanoTime() + OptionT(work.timeoutTo(endpointTimeoutMs.millis, endpointTimeoutResponse(req)).flatTap(recordTelemetry(resourceDoc, startNanos))) case None => // This group has no ResourceDoc for the request. Almost always the request is simply @@ -222,6 +223,22 @@ object ResourceDocMiddleware extends MdcLoggable { } } + /** + * Records Telemetry for a request this group served: its duration, status class and, when the + * response states its length, its size. Only the hop that matched a ResourceDoc records, so a + * request that crossed several version hops is counted once. A request that fell through (a + * disabled endpoint) returned no response here and is not recorded. + */ + private def recordTelemetry(resourceDoc: ResourceDoc, startNanos: Long)(response: Option[Response[IO]]): IO[Unit] = + response match { + case Some(served) => IO { + code.telemetry.Telemetry.recordEndpoint( + resourceDoc.operationId, resourceDoc.implementedInApiVersion.apiShortVersion, + served.status.code, System.nanoTime() - startNanos, served.contentLength) + } + case None => IO.unit + } + /** * Resolve the caller for a hop that has no ResourceDoc for the request, once per request. * diff --git a/obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala b/obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala index 2fd9bf5976..f150b916f1 100644 --- a/obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala +++ b/obp-api/src/main/scala/code/api/v2_2_0/MessageDocsJsonCache.scala @@ -55,8 +55,8 @@ object MessageDocsJsonCache extends Loggable { private def sharedKey(connectorName: String) = s"message-docs-v2.2.0-$connectorName" - private val cache: Cache[String, JValue] = - CacheBuilder.newBuilder().maximumSize(MaxEntries).recordStats().build[String, JValue]() + private val cache: Cache[String, JValue] = code.telemetry.Telemetry.monitorCache( + CacheBuilder.newBuilder().maximumSize(MaxEntries).recordStats().build[String, JValue](), "message_docs") // Counters for tests and monitoring. They only ever go up; compare before and after values. private val generatorCallsCounter = new AtomicLong(0) diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index 00327c5042..c97fe5f129 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -7243,6 +7243,9 @@ object Http4s700 { // object to keep this initialiser under the JVM's 64KB method limit. resourceDocs ++= Http4s700DynamicEntityDefinitions.resourceDocs + // Telemetry, for people; Prometheus reads the separate Telemetry port instead. + resourceDocs ++= Http4s700Telemetry.resourceDocs + val allRoutes: HttpRoutes[IO] = { val sorted = resourceDocs .sortBy(rd => -rd.requestUrl.split("/").count(_.nonEmpty)) diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Telemetry.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Telemetry.scala new file mode 100644 index 0000000000..ef831b0142 --- /dev/null +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700Telemetry.scala @@ -0,0 +1,121 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import cats.effect.IO +import code.api.Constant.ApiPathZero +import code.api.util.APIUtil.{EmptyBody, ResourceDoc} +import code.api.util.ApiRole._ +import code.api.util.ApiTag._ +import code.api.util.ErrorMessages._ +import code.api.util.http4s.Http4sRequestAttributes.EndpointHelpers +import code.api.util.{CustomJsonFormats, Glossary} +import code.telemetry.Telemetry +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.ExecutionContext.Implicits.global +import com.openbankproject.commons.util.ApiVersion +import org.http4s._ +import org.http4s.dsl.io._ +import org.json4s.Formats + +import scala.collection.mutable.ArrayBuffer +import scala.concurrent.Future + +/** + * This object holds the v7.0.0 Telemetry endpoint, which shows people (through API Explorer or API + * Manager) the same Telemetry that Prometheus collects from the separate Telemetry port. + * + * It is declared in its own object, like the Dynamic Entity definitions, to keep Http4s700's + * initialiser under the JVM's 64KB method limit. + */ +object Http4s700Telemetry { + + implicit val formats: Formats = CustomJsonFormats.formats + + private val implementedInApiVersion = ApiVersion.v7_0_0 + private val prefixPath = Root / ApiPathZero.toString / implementedInApiVersion.toString + + val resourceDocs = ArrayBuffer[ResourceDoc]() + + // Route: GET /obp/v7.0.0/management/telemetry + lazy val getTelemetry: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "telemetry" => + EndpointHelpers.withUser(req) { (_, _) => + val namePrefix = req.uri.query.params.get("name_prefix").filter(_.nonEmpty) + Future(JSONFactory700.createTelemetryJson(namePrefix)) + } + } + + /** How this instance's separate port is set, for the description. Read once, when the docs are built. */ + private val portDescription: String = { + val settings = Telemetry.portSettings + if (settings.enabled) + s"On this instance the Telemetry port is open: Prometheus collects Telemetry from port ${settings.port} " + + s"of each OBP-API instance, at the path `${Telemetry.ScrapePath}`." + else + s"On this instance the Telemetry port is not open, so Telemetry can be read only through this endpoint. " + + s"When it is open, Prometheus collects Telemetry from a separate port of each OBP-API instance, at the path `${Telemetry.ScrapePath}`." + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getTelemetry), + "GET", + "/management/telemetry", + "Get Telemetry", + s"""Get this OBP-API instance's Telemetry: aggregated numbers about how it is running, such as the + |number and duration of requests per endpoint, Connector calls per method, cache hits and misses, + |the database connection pool, the log dispatch queue, memory, garbage collection and threads. + | + |Telemetry is not API Metrics. It never records who made a call; see ${Glossary.getGlossaryItemLink("Telemetry")}. + | + |**Which instance answered.** Behind a load balancer, each call can reach a different OBP-API + |instance, and each instance reports only its own figures. The response names the instance that + |answered with `api_instance_id` (the same id its API Metrics records carry) and `git_commit`. + | + |**The separate port.** $portDescription + |Prometheus should use that port, not this endpoint: the port is read from each instance directly, + |so figures from different instances are never mixed; it keeps answering when the API is overloaded; + |and reading it writes no API Metrics record. + | + |**Names.** Meters are listed under their Micrometer names, with dots. Prometheus shows the same + |meters with underscores, `_total` added to counters and `_seconds` to timers, so + |`obp.api.endpoint.requests` appears there as `obp_api_endpoint_requests_seconds`. + |OBP-API's own meters start with `obp.api.`; the others (`jvm.`, `hikaricp.`, `cache.`, `process.`, + |`system.`) are standard. + | + |**Filter.** `name_prefix` limits the list to meters whose name starts with it, + |for example `?name_prefix=obp.api.endpoint`. + |""".stripMargin, + EmptyBody, + JSONFactory700.telemetryJsonV700Example, + List($AuthenticatedUserIsRequired, UserHasMissingRoles, UnknownError), + List(apiTagApi, apiTagSystem), + Some(List(canGetTelemetry)), + http4sPartialFunction = Some(getTelemetry) + ) +} diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index f9174dd4d9..0d505c9341 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -2762,4 +2762,64 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { attributes = Some(List(apiProductSubscriptionAttributeResponseJsonV700Example)) ) lazy val apiProductSubscriptionsJsonV700Example = ApiProductSubscriptionsJsonV700(List(apiProductSubscriptionJsonV700Example)) + + // ===== Telemetry ===== + + /** One meter: its Micrometer name, type, unit, tags and current values (count, total_time, max, value ...). */ + case class TelemetryMeterJsonV700( + name: String, + `type`: String, + base_unit: Option[String], + tags: Map[String, String], + measurements: Map[String, Double] + ) + + /** The separate port Prometheus scrapes, as configured on this instance. */ + case class TelemetryPortJsonV700(enabled: Boolean, port: Int, path: String) + + case class TelemetryJsonV700( + api_instance_id: String, + git_commit: String, + port: TelemetryPortJsonV700, + meters: List[TelemetryMeterJsonV700] + ) + + /** This instance's Telemetry, from the same registry the separate port serves, optionally limited to names starting with `namePrefix`. */ + def createTelemetryJson(namePrefix: Option[String]): TelemetryJsonV700 = { + import scala.jdk.CollectionConverters._ + val settings = code.telemetry.Telemetry.portSettings + val meters = code.telemetry.Telemetry.registry.getMeters.asScala.toList + .filter(meter => namePrefix.forall(prefix => meter.getId.getName.startsWith(prefix))) + .map { meter => + val id = meter.getId + TelemetryMeterJsonV700( + name = id.getName, + `type` = id.getType.name.toLowerCase, + base_unit = Option(id.getBaseUnit), + tags = id.getTags.asScala.map(tag => tag.getKey -> tag.getValue).toMap, + // A gauge whose source has gone reads NaN, which is not valid JSON. + measurements = meter.measure().asScala + .filter(measurement => java.lang.Double.isFinite(measurement.getValue)) + .map(measurement => measurement.getStatistic.name.toLowerCase -> measurement.getValue).toMap) + } + .sortBy(meter => (meter.name, meter.tags.toList.sorted.mkString(","))) + TelemetryJsonV700( + api_instance_id = Constant.ApiInstanceId, + git_commit = APIUtil.gitCommit, + port = TelemetryPortJsonV700(settings.enabled, settings.port, code.telemetry.Telemetry.ScrapePath), + meters = meters) + } + + lazy val telemetryJsonV700Example = TelemetryJsonV700( + api_instance_id = "obp_4f6b3c2a-9d1e-4b7a-8c5f-2e1d0a9b8c7d", + git_commit = "3286937795b4d0c2e1f6a8b9c0d1e2f3a4b5c6d7", + port = TelemetryPortJsonV700(enabled = true, port = code.telemetry.Telemetry.DefaultPort, path = code.telemetry.Telemetry.ScrapePath), + meters = List( + TelemetryMeterJsonV700("cache.gets", "counter", None, Map("cache" -> "json_schema", "result" -> "hit"), Map("count" -> 118.0)), + TelemetryMeterJsonV700("jvm.threads.live", "gauge", Some("threads"), Map.empty, Map("value" -> 64.0)), + TelemetryMeterJsonV700("obp.api.endpoint.requests", "timer", Some("seconds"), + Map("operation" -> "OBPv7.0.0-getBanks", "api_version" -> "v7.0.0", "status" -> "2xx"), + Map("count" -> 42.0, "total_time" -> 1.26, "max" -> 0.081)) + ) + ) } diff --git a/obp-api/src/main/scala/code/bankconnectors/package.scala b/obp-api/src/main/scala/code/bankconnectors/package.scala index 3057501ef3..1316618ccd 100644 --- a/obp-api/src/main/scala/code/bankconnectors/package.scala +++ b/obp-api/src/main/scala/code/bankconnectors/package.scala @@ -76,6 +76,7 @@ package object bankconnectors extends MdcLoggable { // Record the outcome of a connector call: counters, plus optional detailed metric/trace persistence. def recordConnectorInboundMetrics(connectorName: String, methodName: String, correlationId: String, duration: Long, isSuccess: Boolean, args: Array[AnyRef]): Unit = { + code.telemetry.Telemetry.recordConnectorCall(connectorName, methodName, duration, isSuccess) ConnectorCountsRedis.incrementInbound(connectorName, methodName, isSuccess) if (getPropsAsBoolValue("write_connector_metrics", false)) { val params = extractKeyParams(args) diff --git a/obp-api/src/main/scala/code/logcache/LogCacheEventBus.scala b/obp-api/src/main/scala/code/logcache/LogCacheEventBus.scala index 4fdb382782..5f3bb6287d 100644 --- a/obp-api/src/main/scala/code/logcache/LogCacheEventBus.scala +++ b/obp-api/src/main/scala/code/logcache/LogCacheEventBus.scala @@ -170,6 +170,9 @@ object LogCacheEventBus extends MdcLoggable { * On queue-full: drop oldest. On delivery error: stop the thread and let * the gRPC cancel handler clean up the subscription. */ + // Drops across every subscriber, for Telemetry. The per-subscriber count below is only logged. + private lazy val StreamDropsCounter = code.telemetry.Telemetry.counter("obp.api.stream.messages.dropped", "stream" -> "log_cache") + private class BufferedObserver( inner: StreamObserver[String], channelKey: String, @@ -186,6 +189,7 @@ object LogCacheEventBus extends MdcLoggable { if (!queue.offer(msg)) { queue.poll() // drop oldest queue.offer(msg) + StreamDropsCounter.increment() val d = dropped.incrementAndGet() if (d == 1L || d % 1000L == 0L) { logger.warn(s"LogCacheEventBus says: Dropping messages on $channelKey (slow consumer); dropped so far: $d") diff --git a/obp-api/src/main/scala/code/metricsstream/MetricsEventBus.scala b/obp-api/src/main/scala/code/metricsstream/MetricsEventBus.scala index e59c71a805..0b83706302 100644 --- a/obp-api/src/main/scala/code/metricsstream/MetricsEventBus.scala +++ b/obp-api/src/main/scala/code/metricsstream/MetricsEventBus.scala @@ -159,6 +159,9 @@ object MetricsEventBus extends MdcLoggable { // --- Buffered delivery (per-subscriber backpressure) --- + // Drops across every subscriber, for Telemetry. The per-subscriber count below is only logged. + private lazy val StreamDropsCounter = code.telemetry.Telemetry.counter("obp.api.stream.messages.dropped", "stream" -> "metrics") + private class BufferedObserver( inner: StreamObserver[String], queueSize: Int @@ -174,6 +177,7 @@ object MetricsEventBus extends MdcLoggable { if (!queue.offer(msg)) { queue.poll() // drop oldest queue.offer(msg) + StreamDropsCounter.increment() val d = dropped.incrementAndGet() if (d == 1L || d % 1000L == 0L) { logger.warn(s"MetricsEventBus says: Dropping messages (slow consumer); dropped so far: $d") diff --git a/obp-api/src/main/scala/code/telemetry/Telemetry.scala b/obp-api/src/main/scala/code/telemetry/Telemetry.scala new file mode 100644 index 0000000000..2cc91c791b --- /dev/null +++ b/obp-api/src/main/scala/code/telemetry/Telemetry.scala @@ -0,0 +1,214 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.telemetry + +import java.net.InetSocketAddress +import java.nio.charset.StandardCharsets +import java.time.Duration +import java.util.concurrent.{Executors, ThreadFactory} +import java.util.concurrent.atomic.AtomicBoolean + +import com.google.common.cache.Cache +import com.sun.net.httpserver.{HttpExchange, HttpServer} +import io.micrometer.core.instrument._ +import io.micrometer.core.instrument.binder.cache.GuavaCacheMetrics +import io.micrometer.core.instrument.binder.jvm._ +import io.micrometer.core.instrument.binder.system.{FileDescriptorMetrics, ProcessorMetrics, UptimeMetrics} +import io.micrometer.prometheusmetrics.{PrometheusConfig, PrometheusMeterRegistry} +import org.slf4j.LoggerFactory + +import scala.jdk.CollectionConverters._ + +/** + * This object is OBP-API's single entry point for Telemetry: the aggregated numbers (counts, + * durations, sizes, current levels) that describe how the running instance is behaving. + * + * Code elsewhere records Telemetry through the methods here rather than through Micrometer + * directly, so that the rules in docs/telemetry_conventions.md hold in one place: OBP-API's own + * meters are named `obp.api.*` (served to Prometheus as `obp_api_*`), and a tag never carries an + * identifier of a person or a record. + * + * Recording always happens, because it costs no more than an atomic add and the Role-gated + * Telemetry endpoint reads the same registry. The `telemetry.port.enabled` prop only decides + * whether the separate port that Prometheus scrapes is opened. + * + * This object deliberately does not extend MdcLoggable: Helper registers its log counters with + * Telemetry, and Telemetry must not need Helper to be initialised first. + */ +object Telemetry { + + private val logger = LoggerFactory.getLogger(getClass) + + /** Every meter OBP-API invents starts with this. Standard library meters (jvm.*, hikaricp.*, cache.*) keep their own names. */ + val OwnPrefix = "obp.api." + + /** The path the separate port serves. Not `/metrics`, which in OBP means API Metrics. */ + val ScrapePath = "/telemetry" + + val DefaultPort = 9464 + + /** + * Bucket limits for request-style timers (endpoints, Connector calls). A fixed, short list keeps + * the number of Prometheus series per operation small, where Micrometer's default percentile + * histogram would add about seventy buckets to every operation. + */ + val RequestDurationBuckets: Seq[Duration] = + Seq(10L, 50L, 100L, 250L, 500L, 1000L, 2500L, 5000L, 10000L, 30000L).map(Duration.ofMillis) + + lazy val registry: PrometheusMeterRegistry = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT) + + private def checkedName(name: String): String = { + require(name.startsWith(OwnPrefix), s"Telemetry meter '$name' must start with '$OwnPrefix'. See docs/telemetry_conventions.md.") + name + } + + private def tagsOf(tags: Seq[(String, String)]): java.lang.Iterable[Tag] = + tags.map { case (key, value) => Tag.of(key, Option(value).getOrElse("")) }.asJava + + // ===== Recording ===== + + def counter(name: String, tags: (String, String)*): Counter = + registry.counter(checkedName(name), tagsOf(tags)) + + def timer(name: String, tags: (String, String)*): Timer = + Timer.builder(checkedName(name)).tags(tagsOf(tags)).register(registry) + + /** A timer that also publishes the fixed [[RequestDurationBuckets]], for request-style work only. */ + def requestTimer(name: String, tags: (String, String)*): Timer = + Timer.builder(checkedName(name)).tags(tagsOf(tags)).serviceLevelObjectives(RequestDurationBuckets: _*).register(registry) + + def summary(name: String, baseUnit: String, tags: (String, String)*): DistributionSummary = + DistributionSummary.builder(checkedName(name)).baseUnit(baseUnit).tags(tagsOf(tags)).register(registry) + + /** A gauge read from `value` whenever Telemetry is collected. */ + def gauge(name: String, tags: (String, String)*)(value: => Double): Gauge = + Gauge.builder(checkedName(name), () => java.lang.Double.valueOf(value)).tags(tagsOf(tags)).strongReference(true).register(registry) + + /** A counter whose value lives elsewhere (an existing AtomicLong getter) and only ever goes up. */ + def functionCounter(name: String, tags: (String, String)*)(value: => Long): FunctionCounter = + FunctionCounter.builder(checkedName(name), this, (_: Telemetry.type) => value.toDouble).tags(tagsOf(tags)).register(registry) + + /** Records hits, misses, evictions and size of a Guava cache built with `recordStats()`, under the standard `cache.*` names. */ + def monitorCache[K, V](cache: Cache[K, V], cacheName: String): Cache[K, V] = + GuavaCacheMetrics.monitor[K, V, Cache[K, V]](registry, cache, cacheName) + + /** "2xx", "4xx", "5xx": the status class, which keeps the tag's values few. */ + def statusClass(code: Int): String = s"${code / 100}xx" + + /** Records one request served by an endpoint that has a ResourceDoc. */ + def recordEndpoint(operationId: String, apiVersion: String, statusCode: Int, nanos: Long, responseBytes: Option[Long]): Unit = { + requestTimer("obp.api.endpoint.requests", + "operation" -> operationId, "api_version" -> apiVersion, "status" -> statusClass(statusCode)) + .record(nanos, java.util.concurrent.TimeUnit.NANOSECONDS) + responseBytes.foreach(bytes => summary("obp.api.endpoint.response.size", "bytes", "operation" -> operationId).record(bytes.toDouble)) + } + + /** Records one call from OBP-API to a Connector method. */ + def recordConnectorCall(connectorName: String, methodName: String, millis: Long, isSuccess: Boolean): Unit = + requestTimer("obp.api.connector.calls", + "connector" -> connectorName, "connector_method" -> methodName, "result" -> (if (isSuccess) "success" else "failure")) + .record(millis, java.util.concurrent.TimeUnit.MILLISECONDS) + + /** Prometheus text format of everything recorded. */ + def scrape(): String = registry.scrape() + + // ===== Start-up ===== + + private val started = new AtomicBoolean(false) + @volatile private var server: Option[HttpServer] = None + + /** The prop values in force, read once at start-up. */ + case class PortSettings(enabled: Boolean, host: String, port: Int) + + def portSettings: PortSettings = { + import code.api.util.APIUtil + PortSettings( + enabled = APIUtil.getPropsAsBoolValue("telemetry.port.enabled", false), + host = APIUtil.getPropsValue("telemetry.host", "0.0.0.0"), + port = APIUtil.getPropsAsIntValue("telemetry.port", DefaultPort)) + } + + /** The port actually bound, when the separate port is open. */ + def boundPort: Option[Int] = server.map(_.getAddress.getPort) + + /** + * Registers the standard and OBP-API meters and, when `telemetry.port.enabled` is true, opens the + * separate port. Called once from Boot; later calls do nothing. + */ + def start(): Unit = if (started.compareAndSet(false, true)) { + bindStandardMeters() + TelemetryBindings.bindAll() + val settings = portSettings + if (settings.enabled) { + server = Some(startServer(settings.host, settings.port)) + logger.info(s"Telemetry.start says: serving Telemetry at http://${settings.host}:${boundPort.getOrElse(settings.port)}$ScrapePath") + } else { + logger.info("Telemetry.start says: telemetry.port.enabled is false, so the Telemetry port is not opened") + } + } + + private def bindStandardMeters(): Unit = { + List( + new JvmMemoryMetrics(), new JvmGcMetrics(), new JvmHeapPressureMetrics(), new JvmThreadMetrics(), + new ClassLoaderMetrics(), new JvmInfoMetrics(), new ProcessorMetrics(), new UptimeMetrics(), + new FileDescriptorMetrics() + ).foreach(_.bindTo(registry)) + gauge("obp.api.instance.info", + "api_instance_id" -> code.api.Constant.ApiInstanceId, + "git_commit" -> code.api.util.APIUtil.gitCommit)(1.0) + } + + /** + * Opens a small HTTP server with its own single thread, so it keeps answering while the main + * request pool is saturated. It serves only [[ScrapePath]]. Port 0 picks a free port (tests). + */ + def startServer(host: String, port: Int): HttpServer = { + val httpServer = HttpServer.create(new InetSocketAddress(host, port), 0) + httpServer.createContext("/", (exchange: HttpExchange) => { + try { + val (status, body, contentType) = + if (exchange.getRequestURI.getPath == ScrapePath && exchange.getRequestMethod == "GET") + (200, scrape(), "text/plain; version=0.0.4; charset=utf-8") + else + (404, s"Not found. Telemetry is served at $ScrapePath\n", "text/plain; charset=utf-8") + val bytes = body.getBytes(StandardCharsets.UTF_8) + exchange.getResponseHeaders.set("Content-Type", contentType) + exchange.sendResponseHeaders(status, bytes.length.toLong) + exchange.getResponseBody.write(bytes) + } finally exchange.close() + }) + httpServer.setExecutor(Executors.newSingleThreadExecutor(new ThreadFactory { + def newThread(runnable: Runnable): Thread = { + val thread = new Thread(runnable, "telemetry-http") + thread.setDaemon(true) + thread + } + })) + httpServer.start() + httpServer + } +} diff --git a/obp-api/src/main/scala/code/telemetry/TelemetryBindings.scala b/obp-api/src/main/scala/code/telemetry/TelemetryBindings.scala new file mode 100644 index 0000000000..49f1394ccb --- /dev/null +++ b/obp-api/src/main/scala/code/telemetry/TelemetryBindings.scala @@ -0,0 +1,74 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.telemetry + +import code.api.cache.RedisLogger +import code.api.util.JsonSchemaGenerator +import code.api.v2_2_0.MessageDocsJsonCache +import code.util.{Helper, SecureLogging} + +/** + * This object registers, with Telemetry, counters and levels that other parts of OBP-API already + * keep for themselves (mostly `AtomicLong` counters with public getters, written before Telemetry + * existed). + * + * It reads them through their getters when Telemetry is collected, so the code that counts does + * not change and pays nothing extra. Registering them here, in one list, rather than in each + * object, also keeps Telemetry's start-up from initialising those objects in an awkward order: + * Helper, in particular, is initialised by almost everything. + */ +object TelemetryBindings { + + def bindAll(): Unit = { + bindLogging() + bindMessageDocs() + bindRedisLogger() + } + + /** The log dispatch pool (Helper.MdcLoggable) and log masking. */ + private def bindLogging(): Unit = { + Telemetry.gauge("obp.api.log.dispatch.queue.depth")(Helper.mdcLogQueueDepth.toDouble) + Telemetry.functionCounter("obp.api.log.dispatch.entries", "result" -> "dispatched")(Helper.mdcLogDispatchedCount) + Telemetry.functionCounter("obp.api.log.dispatch.entries", "result" -> "dropped")(Helper.mdcLogDroppedCount) + Telemetry.functionCounter("obp.api.log.dispatch.entries", "result" -> "inline")(Helper.mdcLogInlineCount) + Telemetry.functionCounter("obp.api.log.masking.calls")(SecureLogging.maskCalls) + } + + /** The connector JSON Schema and the v2.2.0 message-docs response, which are built once and cached. */ + private def bindMessageDocs(): Unit = { + Telemetry.functionCounter("obp.api.json_schema.generations")(JsonSchemaGenerator.generatorCalls) + Telemetry.functionCounter("obp.api.message_docs.generations")(MessageDocsJsonCache.generatorCalls) + Telemetry.functionCounter("obp.api.message_docs.shared.gets", "result" -> "hit")(MessageDocsJsonCache.sharedHits) + Telemetry.functionCounter("obp.api.message_docs.shared.gets", "result" -> "miss")( + MessageDocsJsonCache.sharedGets - MessageDocsJsonCache.sharedHits) + Telemetry.functionCounter("obp.api.message_docs.shared.sets")(MessageDocsJsonCache.sharedSets) + } + + /** Shipping of log entries to Redis. */ + private def bindRedisLogger(): Unit = + Telemetry.gauge("obp.api.redis_logger.consecutive_failures")(RedisLogger.consecutiveFailureCount.toDouble) +} diff --git a/obp-api/src/main/scala/code/users/LiftUsers.scala b/obp-api/src/main/scala/code/users/LiftUsers.scala index 7185e79e88..8c32e7a698 100644 --- a/obp-api/src/main/scala/code/users/LiftUsers.scala +++ b/obp-api/src/main/scala/code/users/LiftUsers.scala @@ -61,11 +61,12 @@ object LiftUsers extends Users with MdcLoggable{ * until the entry expired. */ private lazy val onBehalfOfCacheTtlSeconds: Long = APIUtil.getPropsAsLongValue("on_behalf_of_user_id.cache_ttl_seconds", 600L) - private lazy val onBehalfOfCache: com.google.common.cache.Cache[String, Resolved] = + private lazy val onBehalfOfCache: com.google.common.cache.Cache[String, Resolved] = code.telemetry.Telemetry.monitorCache( com.google.common.cache.CacheBuilder.newBuilder() .expireAfterWrite(onBehalfOfCacheTtlSeconds, java.util.concurrent.TimeUnit.SECONDS) .maximumSize(100000) - .build[String, Resolved]() + .recordStats() + .build[String, Resolved](), "on_behalf_of") private def nonBlank(s: String): Boolean = s != null && s.nonEmpty diff --git a/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala new file mode 100644 index 0000000000..be44d36d08 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala @@ -0,0 +1,112 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.api.v7_0_0 + +import code.api.Constant +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole.CanGetTelemetry +import code.api.util.ErrorMessages.{AuthenticatedUserIsRequired, UserHasMissingRoles} +import code.api.v6_0_0.V600ServerSetup +import code.api.v7_0_0.JSONFactory700.TelemetryJsonV700 +import code.entitlement.Entitlement +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.ApiVersion +import org.scalatest.Tag + +/** + * This suite checks GET /obp/v7.0.0/management/telemetry: that it needs the CanGetTelemetry Role, + * that it names the instance that answered, and that it shows what the rest of OBP-API records + * (the endpoint requests counted by ResourceDocMiddleware, the standard JVM meters and the log + * dispatch counters registered at start-up). + */ +class TelemetryEndpointTest extends V600ServerSetup { + + def v7_0_0_Request = baseRequest / "obp" / "v7.0.0" + + object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString) + object ApiEndpoint extends Tag("getTelemetry") + + private def telemetryRequest = v7_0_0_Request / "management" / "telemetry" + + private def withTelemetryRole[T](body: => T): T = { + val entitlement = Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, CanGetTelemetry.toString) + try body finally Entitlement.entitlement.vend.deleteEntitlement(entitlement) + } + + feature(s"Get Telemetry - GET /obp/v7.0.0/management/telemetry - $VersionOfApi") { + + scenario("Anonymous access fails with 401", ApiEndpoint, VersionOfApi) { + val response = makeGetRequest(telemetryRequest.GET) + response.code should equal(401) + response.body.extract[ErrorMessage].message should equal(AuthenticatedUserIsRequired) + } + + scenario("A logged-in user without CanGetTelemetry gets 403", ApiEndpoint, VersionOfApi) { + val response = makeGetRequest(telemetryRequest.GET <@ (user1)) + response.code should equal(403) + response.body.extract[ErrorMessage].message should equal(UserHasMissingRoles + CanGetTelemetry) + } + + scenario("A user with CanGetTelemetry sees this instance's Telemetry", ApiEndpoint, VersionOfApi) { + val (first, second) = withTelemetryRole { + (makeGetRequest(telemetryRequest.GET <@ (user1)), makeGetRequest(telemetryRequest.GET <@ (user1))) + } + first.code should equal(200) + second.code should equal(200) + val telemetry = second.body.extract[TelemetryJsonV700] + + Then("it names the instance that answered") + telemetry.api_instance_id should equal(Constant.ApiInstanceId) + telemetry.port.path should equal("/telemetry") + + And("the standard JVM meters and the log dispatch counters were registered at start-up") + val names = telemetry.meters.map(_.name).toSet + names should contain("jvm.threads.live") + names should contain("obp.api.log.dispatch.entries") + names should contain("obp.api.instance.info") + + And("the first request was counted by the middleware, under its operation id") + val requestsToThisEndpoint = telemetry.meters.filter(meter => + meter.name == "obp.api.endpoint.requests" && + meter.tags.get("operation").contains("OBPv7.0.0-getTelemetry") && + meter.tags.get("status").contains("2xx")) + requestsToThisEndpoint should have size 1 + requestsToThisEndpoint.head.measurements("count") should be >= 1.0 + } + + scenario("name_prefix limits the list", ApiEndpoint, VersionOfApi) { + val response = withTelemetryRole { + makeGetRequest(telemetryRequest.GET <@ (user1) < "jvm.memory")) + } + response.code should equal(200) + val names = response.body.extract[TelemetryJsonV700].meters.map(_.name) + names should not be empty + all(names) should startWith("jvm.memory") + } + } +} diff --git a/obp-api/src/test/scala/code/telemetry/TelemetryConventionsTest.scala b/obp-api/src/test/scala/code/telemetry/TelemetryConventionsTest.scala new file mode 100644 index 0000000000..cab0b40ffb --- /dev/null +++ b/obp-api/src/test/scala/code/telemetry/TelemetryConventionsTest.scala @@ -0,0 +1,116 @@ +package code.telemetry + +import java.io.File + +import code.setup.ServerSetupWithTestData +import org.scalatest.Tag + +import scala.io.Source +import scala.jdk.CollectionConverters._ + +/** + * This suite holds OBP-API to the Telemetry conventions in docs/telemetry_conventions.md. + * + * Two guards read the source: new hand-written `AtomicLong` counters, and new Guava caches that are + * not registered with Telemetry, each have an allowlist of what exists today. **An allowlist only + * ever shrinks**, as in AnyBankScopeSweepTest: a new counter belongs in Telemetry, and a new cache + * is registered with `Telemetry.monitorCache`. + * + * Two guards read what the running server registered: meter names, and tag keys that would carry + * an identifier of a person or a record. + */ +class TelemetryConventionsTest extends ServerSetupWithTestData { + + object TelemetryConventions extends Tag("TelemetryConventions") + + // Maven runs the suite from obp-api/, an IDE often from the repository root. + private val sourceRoot: File = + List(new File("src/main/scala"), new File("obp-api/src/main/scala")).find(_.isDirectory) + .getOrElse(throw new IllegalStateException("TelemetryConventionsTest cannot find obp-api/src/main/scala")) + + private def scalaFilesOutsideTelemetry: List[File] = { + def walk(dir: File): List[File] = + Option(dir.listFiles).toList.flatten.flatMap(f => if (f.isDirectory) walk(f) else List(f)) + walk(sourceRoot).filter(_.getName.endsWith(".scala")).filterNot(_.getPath.contains("/code/telemetry/")) + } + + private def relative(file: File): String = sourceRoot.toPath.relativize(file.toPath).toString + + private def read(file: File): String = { + val source = Source.fromFile(file, "UTF-8") + try source.mkString finally source.close() + } + + private def occurrences(text: String, needle: String): Int = text.sliding(needle.length).count(_ == needle) + + /** Hand-written `AtomicLong` counters that existed when Telemetry arrived, per file. */ + private val atomicLongAllowlist: Map[String, Int] = Map( + "code/metricsstream/MetricsEventBus.scala" -> 1, + "code/util/Helper.scala" -> 3, + "code/util/SecureLogging.scala" -> 1, + "code/api/cache/RedisLogger.scala" -> 1, + "code/api/util/JsonSchemaGenerator.scala" -> 1, + "code/api/v2_2_0/MessageDocsJsonCache.scala" -> 4, + "code/logcache/LogCacheEventBus.scala" -> 1 + ) + + /** Standard meter families OBP-API registers through Micrometer's own binders. */ + private val standardPrefixes = List("jvm.", "process.", "system.", "hikaricp.", "cache.", "disk.") + + /** Tag keys that would put an identifier of a person or a record on a series. */ + private val forbiddenTagKeys = Set( + "user_id", "username", "consumer_id", "consent_id", "consent_reference_id", "account_id", + "transaction_id", "customer_id", "bank_id", "url", "path", "correlation_id", "api_instance_id") + + feature("Hand-written counters and unregistered caches do not grow") { + + scenario("No new AtomicLong counter outside code.telemetry", TelemetryConventions) { + val found = scalaFilesOutsideTelemetry + .map(file => relative(file) -> occurrences(read(file), "AtomicLong(")) + .filter(_._2 > 0).toMap + val grown = found.filter { case (file, count) => count > atomicLongAllowlist.getOrElse(file, 0) } + withClue("These files hold more AtomicLong counters than when Telemetry arrived. Record the count with " + + "Telemetry.counter instead (docs/telemetry_conventions.md, section 9).\n" + grown.mkString("\n") + "\n") { + grown shouldBe empty + } + val shrunk = atomicLongAllowlist.filter { case (file, count) => found.getOrElse(file, 0) < count } + withClue("These files hold fewer AtomicLong counters than the allowlist says. Lower or delete their lines.\n" + + shrunk.mkString("\n") + "\n") { + shrunk shouldBe empty + } + } + + scenario("Every Guava cache is registered with Telemetry", TelemetryConventions) { + val unregistered = scalaFilesOutsideTelemetry.map(file => relative(file) -> read(file)).filter { case (_, text) => + occurrences(text, "CacheBuilder.newBuilder") > occurrences(text, "Telemetry.monitorCache(") + }.map(_._1) + withClue("These files build a Guava cache without registering it. Build it with recordStats() and wrap it in " + + "Telemetry.monitorCache(cache, \"name\").\n" + unregistered.mkString("\n") + "\n") { + unregistered shouldBe empty + } + } + } + + feature("What the running server registered follows the naming and tag rules") { + + scenario("Every meter is either OBP-API's own (obp.api.) or a standard family", TelemetryConventions) { + val offenders = Telemetry.registry.getMeters.asScala.map(_.getId.getName).toSet + .filterNot(name => name.startsWith(Telemetry.OwnPrefix) || standardPrefixes.exists(p => name.startsWith(p))) + withClue(s"These meters are neither obp.api.* nor standard (${standardPrefixes.mkString(", ")}).\n" + + offenders.mkString("\n") + "\n") { + offenders shouldBe empty + } + } + + scenario("No series carries an identifier of a person or a record", TelemetryConventions) { + val offenders = Telemetry.registry.getMeters.asScala.map(_.getId) + .filterNot(_.getName == "obp.api.instance.info") // names the instance once, by design + .flatMap(id => id.getTags.asScala.map(_.getKey).filter(forbiddenTagKeys.contains).map(key => s"${id.getName} [$key]")) + .toSet + withClue("These series carry a tag whose values are unbounded identifiers (docs/telemetry_conventions.md, section 6).\n" + + offenders.mkString("\n") + "\n") { + offenders shouldBe empty + } + } + } +} diff --git a/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala b/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala new file mode 100644 index 0000000000..d4100ecf85 --- /dev/null +++ b/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala @@ -0,0 +1,77 @@ +package code.telemetry + +import java.net.{HttpURLConnection, URI} +import java.nio.charset.StandardCharsets +import java.util.UUID + +import org.scalatest.{FlatSpec, Matchers} + +/** + * This suite checks the Telemetry object on its own, without an OBP-API server: the naming rule, + * what a recorded endpoint request looks like to Prometheus, and the separate port. + * + * The registry is shared by the whole JVM, so every check uses a meter name or tag value unique + * to the check. + */ +class TelemetryTest extends FlatSpec with Matchers { + + private def unique(label: String) = s"${label}_${UUID.randomUUID().toString.replace("-", "").take(8)}" + + private def httpGet(port: Int, path: String): (Int, String) = { + val connection = URI.create(s"http://127.0.0.1:$port$path").toURL.openConnection().asInstanceOf[HttpURLConnection] + try { + val status = connection.getResponseCode + val stream = if (status < 400) connection.getInputStream else connection.getErrorStream + (status, new String(stream.readAllBytes(), StandardCharsets.UTF_8)) + } finally connection.disconnect() + } + + "Telemetry" should "refuse an OBP-API meter whose name does not start with obp.api." in { + an[IllegalArgumentException] should be thrownBy Telemetry.counter("cache.gets") + an[IllegalArgumentException] should be thrownBy Telemetry.timer("obp.apisomething") + } + + it should "serve a counter to Prometheus with underscores and _total" in { + val name = unique("probe") + Telemetry.counter(s"obp.api.test.$name", "result" -> "hit").increment(3) + Telemetry.scrape() should include(s"""obp_api_test_${name}_total{result="hit"} 3.0""") + } + + it should "record an endpoint request with its status class and the fixed duration buckets" in { + val operation = unique("OBPv7.0.0-probe") + Telemetry.recordEndpoint(operation, "v7.0.0", 404, 30L * 1000 * 1000, Some(512L)) + val scraped = Telemetry.scrape() + scraped should include(s"""obp_api_endpoint_requests_seconds_count{api_version="v7.0.0",operation="$operation",status="4xx"} 1""") + // 30 ms falls in the 50 ms bucket and every larger one, not in the 10 ms bucket. + scraped should include(s"""obp_api_endpoint_requests_seconds_bucket{api_version="v7.0.0",operation="$operation",status="4xx",le="0.01"} 0""") + scraped should include(s"""obp_api_endpoint_requests_seconds_bucket{api_version="v7.0.0",operation="$operation",status="4xx",le="0.05"} 1""") + scraped should include(s"""obp_api_endpoint_response_size_bytes_sum{operation="$operation"} 512.0""") + } + + it should "record a Connector call as success or failure" in { + val method = unique("getProbe") + Telemetry.recordConnectorCall("star", method, 5L, isSuccess = false) + Telemetry.scrape() should include(s"""obp_api_connector_calls_seconds_count{connector="star",connector_method="$method",result="failure"} 1""") + } + + it should "report a status class, never a status code" in { + Telemetry.statusClass(200) shouldBe "2xx" + Telemetry.statusClass(403) shouldBe "4xx" + Telemetry.statusClass(503) shouldBe "5xx" + } + + "The Telemetry port" should "serve Telemetry at /telemetry and nothing else" in { + val name = unique("port_probe") + Telemetry.counter(s"obp.api.test.$name").increment() + val server = Telemetry.startServer("127.0.0.1", 0) + try { + val port = server.getAddress.getPort + val (status, body) = httpGet(port, "/telemetry") + status shouldBe 200 + body should include(s"obp_api_test_${name}_total 1.0") + + httpGet(port, "/metrics")._1 shouldBe 404 + httpGet(port, "/")._1 shouldBe 404 + } finally server.stop(0) + } +} diff --git a/release_notes.md b/release_notes.md index 6ce3de4e3d..5514bd4eff 100644 --- a/release_notes.md +++ b/release_notes.md @@ -3,6 +3,29 @@ ### Most recent changes at top of file ``` Date Commit Action +27/09/2026 TBD NEW: Telemetry, aggregated numbers about each running instance for + Prometheus and Grafana (not API Metrics; see the Glossary entry + "Telemetry" and docs/telemetry_conventions.md). Recorded always: + requests per endpoint (by operation id, API version and status class), + Connector calls per method, Redis commands, cache hits and misses, the + database pool, the log dispatch queue, memory, garbage collection and + threads. + NEW props: telemetry.port.enabled (default false), telemetry.host + (default 0.0.0.0), telemetry.port (default 9464). When enabled, a + separate port serves Telemetry at /telemetry in the Prometheus text + format. It has no authentication: never publish it outside the host or + cluster. + NEW in v7.0.0: GET /management/telemetry, the same figures as JSON, with + the new Role CanGetTelemetry (instance-wide, empty bank id). + NEW dependency: Micrometer 1.17.1 (micrometer-core, + micrometer-registry-prometheus). +27/09/2026 785f1a4b7 FIXED: three endpoints had lost their Role in the move to http4s and + now require it again, so callers without it get 403: + GET /obp/v6.0.0/management/connector/traces CanGetConnectorTrace + GET /obp/v6.0.0/management/config-props CanGetConfigProps + PUT /obp/v4.0.0/banks/BANK_ID/atms/ATM_ID CanUpdateAtm (at BANK_ID) + updateAtm had accepted CanCreateAtmAtAnyBank by mistake; it now accepts + only CanUpdateAtm at the bank, in line with retiring any-bank Roles. 24/09/2026 TBD RENAMED and RE-SCOPED: the Roles that gate a Dynamic Entity's DEFINITION, completing the change below. Each System and BankLevel pair is now one Role, granted at a bank's id or at SYS for the system space: From 274d892e7972bd0e68c63a376c653e42459174cb Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 28 Sep 2026 06:22:04 +0200 Subject: [PATCH 4/5] Telemetry: memoize hits and misses per cached method (Redis and in-memory), rows queued/written/lost and queue depth for the API Metrics and Connector Metrics batch writers, item counts of list responses. --- docs/telemetry_conventions.md | 26 +++++--- .../main/scala/code/api/cache/Caching.scala | 44 +++++++++++-- .../code/api/util/http4s/Http4sSupport.scala | 34 +++++++--- .../metrics/ConnectorMetricBatchWriter.scala | 10 +++ .../code/metrics/MetricBatchWriter.scala | 10 +++ .../code/telemetry/BatchWriterTelemetry.scala | 64 +++++++++++++++++++ .../main/scala/code/telemetry/Telemetry.scala | 20 ++++++ .../code/api/cache/MemoizeTelemetryTest.scala | 44 +++++++++++++ .../api/v7_0_0/TelemetryEndpointTest.scala | 28 ++++++++ .../scala/code/telemetry/TelemetryTest.scala | 21 ++++++ 10 files changed, 278 insertions(+), 23 deletions(-) create mode 100644 obp-api/src/main/scala/code/telemetry/BatchWriterTelemetry.scala create mode 100644 obp-api/src/test/scala/code/api/cache/MemoizeTelemetryTest.scala diff --git a/docs/telemetry_conventions.md b/docs/telemetry_conventions.md index ec51a53280..aa81641403 100644 --- a/docs/telemetry_conventions.md +++ b/docs/telemetry_conventions.md @@ -282,6 +282,11 @@ thread and class figures on its own port. Remove it once the Micrometer JVM bind |---|---|---|---| | `obp.api.endpoint.requests` | timer, fixed buckets | `operation`, `api_version`, `status` | `ResourceDocMiddleware`, once per request, by the hop that matched a ResourceDoc | | `obp.api.endpoint.response.size` | distribution summary, bytes | `operation` | the same, when the response states its length | +| `obp.api.endpoint.response.items` | distribution summary, items | `operation` | `EndpointHelpers`, for a response that is a JSON array or an object wrapping exactly one array; counted from the JSON the helper decomposes anyway | +| `obp.api.memoize.gets` | counter | `provider` (`redis`, `in_memory`), `cache`, `result` (`hit`, `miss`) | `Caching.memoize*`. `cache` is `Owner.method` for a key built by `CacheKeyFromArguments`, and `other` for a key a caller wrote itself, which may hold an identifier | +| `obp.api.batch_writer.rows` | counter | `writer` (`api_metrics`, `connector_metrics`), `result` (`queued`, `written`, `lost`) | the batch writers; `lost` is a batch dropped by a failed flush | +| `obp.api.batch_writer.queue.depth` | gauge | `writer` | rows queued minus rows written or lost (the queue's own `size()` walks it) | +| `obp.api.batch_writer.flushes` | timer | `writer`, `result` | each flush that had rows | | `obp.api.connector.calls` | timer, fixed buckets | `connector`, `connector_method`, `result` | the Connector proxy (`code/bankconnectors/package.scala`) | | `obp.api.redis.commands` | timer | `command`, `result` | `Redis.use` | | `cache.gets`, `cache.puts`, `cache.evictions`, `cache.size` | standard | `cache` = `in_memory`, `json_schema`, `message_docs`, `on_behalf_of` | every Guava cache, through `Telemetry.monitorCache` | @@ -290,13 +295,16 @@ thread and class figures on its own port. Remove it once the Micrometer JVM bind | `obp.api.instance.info` | gauge, always 1 | `api_instance_id`, `git_commit` | start-up | | the counters in section 11 | | | `TelemetryBindings` | -Not recorded yet: -- hits and misses of `Caching.memoize*` with the Redis provider (the in-memory provider is covered - through the `in_memory` cache); -- the batch writers for API Metrics and Connector Metrics: their queues are - `ConcurrentLinkedQueue`s, whose size costs a walk of the whole queue, so they need their own - counters rather than a gauge on `size()`; -- item counts of list responses; -- endpoints without a ResourceDoc (Dynamic Entity records, Dynamic Endpoints, the unversioned - routes), which do not pass through the matching branch of `ResourceDocMiddleware`. +Not recorded yet: requests served outside `ResourceDocMiddleware`. Every endpoint should have a +ResourceDoc; these either have one but are served by routes the middleware does not wrap, or have +none: +- documented, served outside the middleware: the resource-docs, Swagger and OpenAPI routes + (`Http4sResourceDocs`), the unversioned `POST /my/logins/direct` (documented at v6.0.0), Dynamic + Entity records (unversioned and v7.0.0) and Dynamic Endpoints; +- no ResourceDoc: `POST /my/logins/siwe/challenge`, `POST /my/logins/siwe`, and + `GET /obp/PREFIX/resource-docs/API_VERSION/openapi.yaml`; +- server pages, also without a ResourceDoc: `/`, `/apps`, `/status`, `/health`, `/alive`. + +The full picture, and the decision to leave these routes as they are for now, is in +`docs/resource_doc_and_endpoint_consistency_status.md`. diff --git a/obp-api/src/main/scala/code/api/cache/Caching.scala b/obp-api/src/main/scala/code/api/cache/Caching.scala index 69d6db1aed..1678e13f48 100644 --- a/obp-api/src/main/scala/code/api/cache/Caching.scala +++ b/obp-api/src/main/scala/code/api/cache/Caching.scala @@ -37,12 +37,48 @@ import scala.concurrent.duration.Duration import scala.language.postfixOps object Caching extends MdcLoggable { + // ===== Telemetry ===== + + // A key built by CacheKeyFromArguments renders as "(Owner,method,arguments...)". Its first two + // fields name the cached code, from a list the code fixes; the arguments do not. + private val MacroKeyShape = """^\(([A-Za-z_][\w.$]*),([A-Za-z_][\w$]*),""".r.unanchored + + /** + * The Telemetry label for a cache key: "Owner.method" for a key built by CacheKeyFromArguments, + * and "other" for a key a caller wrote itself, which may contain an identifier (a consumer id, + * a date) and so must never become a tag value. + */ + private[cache] def telemetryLabel(cacheKey: Option[String]): String = cacheKey match { + case Some(MacroKeyShape(owner, method)) => s"$owner.$method" + case _ => "other" + } + + /** Counts one memoised call as a hit, or as a miss when the cached function had to run. */ + private def recordMemoizeGet(provider: String, cacheKey: Option[String], computed: Boolean): Unit = + code.telemetry.Telemetry.counter("obp.api.memoize.gets", + "provider" -> provider, "cache" -> telemetryLabel(cacheKey), "result" -> (if (computed) "miss" else "hit")) + .increment() + + private def recordedSync[A](provider: String, cacheKey: Option[String])(memoize: (=> A) => A)(f: => A): A = { + var computed = false + val result = memoize { computed = true; f } + recordMemoizeGet(provider, cacheKey, computed) + result + } + + private def recordedAsync[A](provider: String, cacheKey: Option[String])(memoize: (=> Future[A]) => Future[A])(f: => Future[A]): Future[A] = { + val computed = new java.util.concurrent.atomic.AtomicBoolean(false) + val result = memoize { computed.set(true); f } + result.onComplete(_ => recordMemoizeGet(provider, cacheKey, computed.get))(scala.concurrent.ExecutionContext.parasitic) + result + } + def memoizeSyncWithProvider[A](cacheKey: Option[String])(ttl: Duration)(f: => A)(implicit m: Manifest[A]): A = { (cacheKey, ttl) match { case (_, t) if t == Duration.Zero => // Just forwarding a call f case (Some(_), _) => // Caching a call - Redis.memoizeSyncWithRedis(cacheKey)(ttl)(f) + recordedSync[A]("redis", cacheKey)(g => Redis.memoizeSyncWithRedis(cacheKey)(ttl)(g))(f) case _ => // Just forwarding a call f } @@ -54,7 +90,7 @@ object Caching extends MdcLoggable { case (_, t) if t == Duration.Zero => // Just forwarding a call f case (Some(_), _) => // Caching a call - Redis.memoizeWithRedis(cacheKey)(ttl)(f) + recordedAsync[A]("redis", cacheKey)(g => Redis.memoizeWithRedis(cacheKey)(ttl)(g))(f) case _ => // Just forwarding a call f } @@ -66,7 +102,7 @@ object Caching extends MdcLoggable { case (_, t) if t == Duration.Zero => // Just forwarding a call f case (Some(_), _) => // Caching a call - InMemory.memoizeSyncWithInMemory(cacheKey)(ttl)(f) + recordedSync[A]("in_memory", cacheKey)(g => InMemory.memoizeSyncWithInMemory(cacheKey)(ttl)(g))(f) case _ => // Just forwarding a call f } @@ -78,7 +114,7 @@ object Caching extends MdcLoggable { case (_, t) if t == Duration.Zero => // Just forwarding a call f case (Some(_), _) => // Caching a call - InMemory.memoizeWithInMemory(cacheKey)(ttl)(f) + recordedAsync[A]("in_memory", cacheKey)(g => InMemory.memoizeWithInMemory(cacheKey)(ttl)(g))(f) case _ => // Just forwarding a call f } diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala index fe36cfc42a..2d9aebbaa7 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sSupport.scala @@ -181,8 +181,22 @@ object Http4sRequestAttributes { case (r, (name, value)) => r.putHeaders(Header.Raw(CIString(name), value)) } + /** + * Renders a handler's result as JSON. On the way it records, for Telemetry, how many items a + * list response carried. The count is taken from the JSON this method builds for rendering, so + * the result is not decomposed a second time. What counting does cost, on a list response, is + * one walk of the list (json4s holds array elements in a linked List, whose size is counted), + * a registry lookup of the operation's meter, and an atomic update: small next to decomposing + * and rendering the same list, but not nothing. + */ + private def renderJson[A](result: A)(implicit formats: Formats, cc: CallContext): String = { + val json = Extraction.decompose(result) + cc.operationId.foreach(operationId => code.telemetry.Telemetry.recordResponseItems(operationId, json)) + prettyRender(json) + } + private def toJsonOk[A](result: A)(implicit formats: Formats, cc: CallContext): IO[Response[IO]] = { - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) Ok(jsonString, jsonContentType).map(withCallContextHeaders) } @@ -278,7 +292,7 @@ object Http4sRequestAttributes { } yield result io.attempt.flatMap { case Right(result) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } @@ -322,7 +336,7 @@ object Http4sRequestAttributes { case Right(body) => RequestScopeConnection.fromFuture(f(body, cc)).attempt.flatMap { case Right(result) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } @@ -364,7 +378,7 @@ object Http4sRequestAttributes { } yield result io.attempt.flatMap { case Right(result) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } @@ -408,7 +422,7 @@ object Http4sRequestAttributes { } yield result io.attempt.flatMap { case Right(result) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } @@ -446,7 +460,7 @@ object Http4sRequestAttributes { } yield result io.attempt.flatMap { case Right(result) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } @@ -469,7 +483,7 @@ object Http4sRequestAttributes { } yield result io.attempt.flatMap { case Right(result) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } @@ -534,7 +548,7 @@ object Http4sRequestAttributes { implicit val cc: CallContext = req.callContext RequestScopeConnection.fromFuture(f).attempt.flatMap { case Right(result) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) Created(jsonString, jsonContentType).map(withCallContextHeaders).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) } @@ -554,7 +568,7 @@ object Http4sRequestAttributes { implicit val cc: CallContext = req.callContext RequestScopeConnection.fromFuture(f).attempt.flatMap { case Right((result, code)) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) val status = Status.fromInt(code).getOrElse(Status.Ok) IO.pure(withCallContextHeaders(Response[IO](status).withEntity(jsonString).withContentType(jsonContentType))).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) @@ -576,7 +590,7 @@ object Http4sRequestAttributes { io.attempt.flatMap { case Right((_, 204)) => NoContent().map(withCallContextHeaders).flatTap(recordMetric("", _)) case Right((result, code)) => - val jsonString = prettyRender(Extraction.decompose(result)) + val jsonString = renderJson(result) val status = Status.fromInt(code).getOrElse(Status.Ok) IO.pure(withCallContextHeaders(Response[IO](status).withEntity(jsonString).withContentType(jsonContentType))).flatTap(recordMetric(result, _)) case Left(err) => ErrorResponseConverter.toHttp4sResponse(err, cc).flatTap(recordMetric(err.getMessage, _)) diff --git a/obp-api/src/main/scala/code/metrics/ConnectorMetricBatchWriter.scala b/obp-api/src/main/scala/code/metrics/ConnectorMetricBatchWriter.scala index 9fc8655eb7..43f778eb17 100644 --- a/obp-api/src/main/scala/code/metrics/ConnectorMetricBatchWriter.scala +++ b/obp-api/src/main/scala/code/metrics/ConnectorMetricBatchWriter.scala @@ -82,11 +82,18 @@ object ConnectorMetricBatchWriter extends MdcLoggable { } } + // Rows queued, written and lost, and the queue depth, for Telemetry. + private lazy val telemetry = new code.telemetry.BatchWriterTelemetry("connector_metrics") + def enqueue(row: ConnectorMetricRow): Unit = { queue.add(row) + telemetry.queued() } private[code] def flush(): Unit = { + val flushStart = System.nanoTime() + // Rows taken off the queue by this flush. If the write fails they are lost, so Telemetry counts them. + var drainedRows = 0 try { val batch = new java.util.ArrayList[ConnectorMetricRow]() var item = queue.poll() @@ -94,6 +101,7 @@ object ConnectorMetricBatchWriter extends MdcLoggable { batch.add(item) item = queue.poll() } + drainedRows = batch.size() if (!batch.isEmpty) { val rows = { @@ -136,10 +144,12 @@ object ConnectorMetricBatchWriter extends MdcLoggable { } yield n val count = DoobieUtil.runQuery(program) logger.debug(s"ConnectorMetricBatchWriter says: flushed $count connector metrics via doobie-pool") + telemetry.written(drainedRows, System.nanoTime() - flushStart) } } catch { case e: Exception => logger.error(s"ConnectorMetricBatchWriter says: flush failed", e) + if (drainedRows > 0) telemetry.lost(drainedRows, System.nanoTime() - flushStart) } } } diff --git a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala index aa348db284..0f27ea4621 100644 --- a/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala +++ b/obp-api/src/main/scala/code/metrics/MetricBatchWriter.scala @@ -104,14 +104,21 @@ object MetricBatchWriter extends MdcLoggable { * Enqueue a metric for batched writing. Never blocks the calling thread. * The background scheduler handles all flushing. */ + // Rows queued, written and lost, and the queue depth, for Telemetry. + private lazy val telemetry = new code.telemetry.BatchWriterTelemetry("api_metrics") + def enqueue(row: MetricRow): Unit = { queue.add(row) + telemetry.queued() } /** * Drain the queue and batch-insert all pending metrics via Doobie. */ private[code] def flush(): Unit = { + val flushStart = System.nanoTime() + // Rows taken off the queue by this flush. If the write fails they are lost, so Telemetry counts them. + var drainedRows = 0 try { val batch = new java.util.ArrayList[MetricRow]() var item = queue.poll() @@ -119,6 +126,7 @@ object MetricBatchWriter extends MdcLoggable { batch.add(item) item = queue.poll() } + drainedRows = batch.size() if (!batch.isEmpty) { val rows = { @@ -170,6 +178,7 @@ object MetricBatchWriter extends MdcLoggable { } yield n val count = DoobieUtil.runQuery(program) logger.debug(s"MetricBatchWriter says: flushed $count metrics via doobie-pool") + telemetry.written(drainedRows, System.nanoTime() - flushStart) } } catch { case e: Exception => @@ -179,6 +188,7 @@ object MetricBatchWriter extends MdcLoggable { // (e.g. "value too long for type character varying(N)") is lost and metrics are // silently dropped. Walk the chain so the root cause is always logged. logger.error(s"MetricBatchWriter says: flush failed${sqlChainDetail(e)}", e) + if (drainedRows > 0) telemetry.lost(drainedRows, System.nanoTime() - flushStart) } } diff --git a/obp-api/src/main/scala/code/telemetry/BatchWriterTelemetry.scala b/obp-api/src/main/scala/code/telemetry/BatchWriterTelemetry.scala new file mode 100644 index 0000000000..33ecfd5d1f --- /dev/null +++ b/obp-api/src/main/scala/code/telemetry/BatchWriterTelemetry.scala @@ -0,0 +1,64 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.telemetry + +import java.util.concurrent.TimeUnit + +/** + * This class records Telemetry for a writer that queues records in memory and writes them to the + * database in batches (the API Metrics and Connector Metrics writers). + * + * Such a writer can fall behind (its queue grows) or lose records (a failed flush drops the whole + * batch), and before this neither showed anywhere but the log. The queue is a + * ConcurrentLinkedQueue, whose size() walks the whole queue, so the depth is not read from it: it + * is the rows queued minus the rows taken off the queue by a flush, whether written or lost. + * + * @param writer the value of the `writer` tag, for example "api_metrics" + */ +class BatchWriterTelemetry(writer: String) { + + private val queuedRows = Telemetry.counter("obp.api.batch_writer.rows", "writer" -> writer, "result" -> "queued") + private val writtenRows = Telemetry.counter("obp.api.batch_writer.rows", "writer" -> writer, "result" -> "written") + private val lostRows = Telemetry.counter("obp.api.batch_writer.rows", "writer" -> writer, "result" -> "lost") + + Telemetry.gauge("obp.api.batch_writer.queue.depth", "writer" -> writer)( + queuedRows.count() - writtenRows.count() - lostRows.count()) + + def queued(): Unit = queuedRows.increment() + + /** A flush wrote `rows` rows, taking `nanos`. */ + def written(rows: Int, nanos: Long): Unit = { + writtenRows.increment(rows.toDouble) + Telemetry.timer("obp.api.batch_writer.flushes", "writer" -> writer, "result" -> "success").record(nanos, TimeUnit.NANOSECONDS) + } + + /** A flush failed after taking `rows` rows off the queue, so they are gone. */ + def lost(rows: Int, nanos: Long): Unit = { + lostRows.increment(rows.toDouble) + Telemetry.timer("obp.api.batch_writer.flushes", "writer" -> writer, "result" -> "failure").record(nanos, TimeUnit.NANOSECONDS) + } +} diff --git a/obp-api/src/main/scala/code/telemetry/Telemetry.scala b/obp-api/src/main/scala/code/telemetry/Telemetry.scala index 2cc91c791b..25a8e7584b 100644 --- a/obp-api/src/main/scala/code/telemetry/Telemetry.scala +++ b/obp-api/src/main/scala/code/telemetry/Telemetry.scala @@ -127,6 +127,26 @@ object Telemetry { responseBytes.foreach(bytes => summary("obp.api.endpoint.response.size", "bytes", "operation" -> operationId).record(bytes.toDouble)) } + /** + * The number of items in a list response: the length of the response itself when it is a JSON + * array, or of its only array field when it is an object wrapping one list (`{"banks": [...]}`, + * the usual OBP shape). None for anything else, which is then not recorded. + */ + def listItemCount(json: org.json4s.JValue): Option[Int] = json match { + case org.json4s.JArray(items) => Some(items.size) + case org.json4s.JObject(fields) => + fields.collect { case (_, org.json4s.JArray(items)) => items.size } match { + case List(size) => Some(size) + case _ => None + } + case _ => None + } + + /** Records how many items a list response carried, when it is one. */ + def recordResponseItems(operationId: String, json: org.json4s.JValue): Unit = + listItemCount(json).foreach(size => + summary("obp.api.endpoint.response.items", "items", "operation" -> operationId).record(size.toDouble)) + /** Records one call from OBP-API to a Connector method. */ def recordConnectorCall(connectorName: String, methodName: String, millis: Long, isSuccess: Boolean): Unit = requestTimer("obp.api.connector.calls", diff --git a/obp-api/src/test/scala/code/api/cache/MemoizeTelemetryTest.scala b/obp-api/src/test/scala/code/api/cache/MemoizeTelemetryTest.scala new file mode 100644 index 0000000000..8bce1335e5 --- /dev/null +++ b/obp-api/src/test/scala/code/api/cache/MemoizeTelemetryTest.scala @@ -0,0 +1,44 @@ +package code.api.cache + +import java.util.UUID + +import code.telemetry.Telemetry +import org.scalatest.{FlatSpec, Matchers} + +import scala.concurrent.duration._ + +/** + * This suite checks the Telemetry recorded by Caching's memoize functions: a call that had to run + * the cached function is a miss, a call answered from the cache is a hit, and the `cache` tag names + * the cached code only when the key was built by CacheKeyFromArguments. A key a caller wrote itself + * may contain an identifier, so it is labelled "other". + */ +class MemoizeTelemetryTest extends FlatSpec with Matchers { + + private def gets(cache: String, result: String): Double = + Option(Telemetry.registry.find("obp.api.memoize.gets") + .tags("provider", "in_memory", "cache", cache, "result", result).counter()) + .map(_.count()).getOrElse(0.0) + + "Caching.telemetryLabel" should "name Owner.method for a key built by CacheKeyFromArguments" in { + Caching.telemetryLabel(Some("(code.bankconnectors.LocalMappedConnector$,getBanks,List())")) shouldBe + "code.bankconnectors.LocalMappedConnector$.getBanks" + } + + it should "say other for a key a caller wrote itself" in { + Caching.telemetryLabel(Some("rl_active_2f6c1f0e-consumer-id_2026-09-27-10")) shouldBe "other" + Caching.telemetryLabel(None) shouldBe "other" + } + + "memoizeSyncWithImMemory" should "count the first call as a miss and the second as a hit" in { + val method = s"probe${UUID.randomUUID().toString.replace("-", "").take(8)}" + val key = s"(MemoizeTelemetryTest,$method,1)" + val label = s"MemoizeTelemetryTest.$method" + var runs = 0 + Caching.memoizeSyncWithImMemory(Some(key))(60.seconds) { runs += 1; "value" } shouldBe "value" + Caching.memoizeSyncWithImMemory(Some(key))(60.seconds) { runs += 1; "value" } shouldBe "value" + runs shouldBe 1 + gets(label, "miss") shouldBe 1.0 + gets(label, "hit") shouldBe 1.0 + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala index be44d36d08..4c73ec9865 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/TelemetryEndpointTest.scala @@ -90,6 +90,18 @@ class TelemetryEndpointTest extends V600ServerSetup { names should contain("obp.api.log.dispatch.entries") names should contain("obp.api.instance.info") + And("every meter the API Manager Telemetry page reads is present under the name it expects") + // OBP-Frontend apps/api-manager/src/lib/telemetry/telemetry.ts reads these by name. + List( + "jvm.memory.used", "jvm.memory.max", "jvm.memory.usage.after.gc", "jvm.gc.overhead", + "jvm.threads.live", "process.cpu.usage", "process.uptime", + "hikaricp.connections", "hikaricp.connections.active", "hikaricp.connections.idle", + "hikaricp.connections.pending", "hikaricp.connections.max", "hikaricp.connections.timeout", + "cache.gets", "cache.size", "cache.evictions", + "obp.api.log.dispatch.queue.depth", "obp.api.log.masking.calls" + ).foreach(expected => names should contain(expected)) + telemetry.meters.filter(_.name == "cache.gets").flatMap(_.tags.get("result")).toSet should equal(Set("hit", "miss")) + And("the first request was counted by the middleware, under its operation id") val requestsToThisEndpoint = telemetry.meters.filter(meter => meter.name == "obp.api.endpoint.requests" && @@ -99,6 +111,22 @@ class TelemetryEndpointTest extends V600ServerSetup { requestsToThisEndpoint.head.measurements("count") should be >= 1.0 } + scenario("List responses record their item counts, and memoised Connector calls are labelled by method", ApiEndpoint, VersionOfApi) { + Given("a list endpoint and a Connector call that is memoised with a key built by CacheKeyFromArguments") + makeGetRequest((v7_0_0_Request / "banks").GET).code should equal(200) + val response = withTelemetryRole { makeGetRequest(telemetryRequest.GET <@ (user1)) } + val meters = response.body.extract[TelemetryJsonV700].meters + + Then("the list response's item count was recorded under its operation id") + meters.exists(meter => meter.name == "obp.api.endpoint.response.items" && meter.tags.get("operation").exists(_.endsWith("-getBanks"))) shouldBe true + + And("memoised calls whose key names the cached method are labelled Owner.method, not other") + val memoizeLabels = meters.filter(_.name == "obp.api.memoize.gets").flatMap(_.tags.get("cache")).toSet + withClue(s"memoize labels seen: ${memoizeLabels.mkString(", ")} ") { + memoizeLabels.exists(label => label != "other") shouldBe true + } + } + scenario("name_prefix limits the list", ApiEndpoint, VersionOfApi) { val response = withTelemetryRole { makeGetRequest(telemetryRequest.GET <@ (user1) < "jvm.memory")) diff --git a/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala b/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala index d4100ecf85..1a616a6bcf 100644 --- a/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala +++ b/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala @@ -60,6 +60,27 @@ class TelemetryTest extends FlatSpec with Matchers { Telemetry.statusClass(503) shouldBe "5xx" } + it should "count the items of a list response, whether a bare array or an object wrapping one list" in { + import org.json4s._ + Telemetry.listItemCount(JArray(List(JInt(1), JInt(2)))) shouldBe Some(2) + Telemetry.listItemCount(JObject(List("banks" -> JArray(List(JInt(1), JInt(2), JInt(3)))))) shouldBe Some(3) + // An object with no list, or with two lists, is not a list response. + Telemetry.listItemCount(JObject(List("bank_id" -> JString("x")))) shouldBe None + Telemetry.listItemCount(JObject(List("a" -> JArray(Nil), "b" -> JArray(Nil)))) shouldBe None + } + + "BatchWriterTelemetry" should "derive the queue depth from rows queued, written and lost" in { + val writer = unique("writer") + val batchTelemetry = new BatchWriterTelemetry(writer) + (1 to 5).foreach(_ => batchTelemetry.queued()) + batchTelemetry.written(2, 1000L) + batchTelemetry.lost(1, 1000L) + val scraped = Telemetry.scrape() + scraped should include(s"""obp_api_batch_writer_queue_depth{writer="$writer"} 2.0""") + scraped should include(s"""obp_api_batch_writer_rows_total{result="lost",writer="$writer"} 1.0""") + scraped should include(s"""obp_api_batch_writer_flushes_seconds_count{result="failure",writer="$writer"} 1""") + } + "The Telemetry port" should "serve Telemetry at /telemetry and nothing else" in { val name = unique("port_probe") Telemetry.counter(s"obp.api.test.$name").increment() From 222ce6189eba1c9e202df49e622acddc3c4b1d57 Mon Sep 17 00:00:00 2001 From: simonredfern Date: Mon, 28 Sep 2026 09:44:02 +0200 Subject: [PATCH 5/5] Rate limit scope "documentation" (per IP, shadow by default) for all public docs incl. glossary, message-docs, api/tags; per-scope mode shown in rate-limiter-config. ResourceDocFilters + ResourceDocVocabulary (all static and dynamic tags/functions): cache keys only accept sorted, known filter values in order to limit cache size / cache misses. Telemetry: resource-docs timing, Redis cache hits/misses/errors, limiter outcomes. Add resource_doc_and_endpoint_consistency_status.md. --- ...rce_doc_and_endpoint_consistency_status.md | 161 ++++++++++++++++++ docs/telemetry_conventions.md | 2 + .../resources/props/sample.props.template | 8 + .../main/scala/code/api/cache/Caching.scala | 54 ++++-- .../main/scala/code/api/util/APIUtil.scala | 11 +- .../code/api/util/ResourceDocFilters.scala | 141 +++++++++++++++ .../api/util/SelfServiceRateLimiter.scala | 39 ++++- .../api/util/http4s/Http4sResourceDocs.scala | 84 +++++---- .../SelfServiceRateLimitMiddleware.scala | 27 ++- .../scala/code/api/v7_0_0/Http4s700.scala | 2 +- .../code/api/v7_0_0/JSONFactory7.0.0.scala | 17 +- .../main/scala/code/telemetry/Telemetry.scala | 16 ++ .../ResourceDocsFilterNormalisationTest.scala | 68 ++++++++ .../code/api/cache/MemoizeTelemetryTest.scala | 24 +++ .../api/util/ResourceDocFiltersTest.scala | 81 +++++++++ .../api/util/SelfServiceRateLimiterTest.scala | 65 +++++++ .../code/api/v7_0_0/RateLimitersTest.scala | 11 ++ .../scala/code/telemetry/TelemetryTest.scala | 13 ++ release_notes.md | 19 +++ 19 files changed, 778 insertions(+), 65 deletions(-) create mode 100644 docs/resource_doc_and_endpoint_consistency_status.md create mode 100644 obp-api/src/main/scala/code/api/util/ResourceDocFilters.scala create mode 100644 obp-api/src/test/scala/code/api/ResourceDocs1_4_0/ResourceDocsFilterNormalisationTest.scala create mode 100644 obp-api/src/test/scala/code/api/util/ResourceDocFiltersTest.scala diff --git a/docs/resource_doc_and_endpoint_consistency_status.md b/docs/resource_doc_and_endpoint_consistency_status.md new file mode 100644 index 0000000000..79623c3978 --- /dev/null +++ b/docs/resource_doc_and_endpoint_consistency_status.md @@ -0,0 +1,161 @@ +# ResourceDoc and endpoint consistency: status + +This document records how OBP-API uses ResourceDocs, which endpoints are served without going +through them, and what that costs. It is a status record, not a work plan. + +## 1. What a ResourceDoc is + +A ResourceDoc describes one endpoint: its verb and URL template, summary and description, example +request and response bodies, possible errors, tags, the Roles it requires, and (for the http4s +routes) the handler that serves it (`http4sPartialFunction`). OBP's principle is that every +endpoint has one. + +## 2. What ResourceDocs are used for + +**Documentation.** The resource-docs, Swagger and OpenAPI endpoints, API Explorer, OBP-MCP's +endpoint discovery and the Glossary's endpoint links are all generated from ResourceDocs. + +**Building the routes.** In every versioned API group, the routes are built from the ResourceDocs +themselves: `allRoutes` is the list of each doc's `http4sPartialFunction`, ordered by the number of +path segments (for example `Http4s700.scala`, `val allRoutes`). So in those groups a route cannot +exist without a ResourceDoc. The same construction is used in 11 route groups. + +**Checking every request, in `ResourceDocMiddleware`.** Each versioned group is wrapped in +`ResourceDocMiddleware.apply(resourceDocs)` (`obp-api/src/main/scala/code/api/util/http4s/ResourceDocMiddleware.scala`). +For each request it finds the matching ResourceDoc and then, in this order +(`validateOnly`, which follows the order Lift used): + +1. rejects duplicated query parameters; +2. authenticates the caller, if the doc requires it (the doc's error list contains + `AuthenticatedUserIsRequired`, or it declares Roles); +3. refuses an unresolved UK Open Banking consent; +4. resolves `BANK_ID` (404 when unknown); +5. checks the doc's Roles (403); +6. resolves `ACCOUNT_ID`, `VIEW_ID` (and the caller's access to the view) and `COUNTERPARTY_ID`; +7. applies Force-Error, the authentication-type validation and the JSON Schema validation + configured for the endpoint's operation id. + +It then runs the handler: inside a database transaction for POST, PUT, DELETE and PATCH, and +on auto-commit for GET and HEAD. It also applies the endpoint timeout, honours the +enable/disable props by operation id, puts the doc and its operation id on the `CallContext`, and +records Telemetry for the request. `IdempotencyMiddleware` is nested inside it. + +**Other uses.** The JSON baseline under `scripts/resource_doc_baseline/` and the parity audit +compare ResourceDocs with their Lift originals. Guard tests such as `AnyBankScopeSweepTest` read +the ResourceDoc catalogue to check Role scoping. API Metrics records carry the endpoint's name and +version taken from the doc. + +## 3. Where the pairing holds + +Every group below builds its routes from its ResourceDocs and is wrapped by +`ResourceDocMiddleware`: + +- OBP API v1.2.1, v1.3.0, v1.4.0, v2.0.0, v2.1.0, v2.2.0, v3.0.0, v3.1.0, v4.0.0, v5.0.0, v5.1.0, + v6.0.0, v7.0.0, including the bridges that pass a request from one version to the one below; +- Berlin Group v1.3 (and its alias path) and v2; +- UK Open Banking v2.0, v3.1 and v4.0.1. + +One narrow exception inside these groups: when no ResourceDoc matches (for example a URL with an +empty segment such as `/banks//accounts`), the middleware still lets the group's routes try the +request, after resolving the caller but without the doc-based checks. That branch exists so a +malformed URL gets 403 or 404 rather than a misleading 401. It is documented in `CLAUDE.md` +("Empty path segments"). + +## 4. Where it does not hold + +These routes are served from `Http4sApp`'s chain outside any `ResourceDocMiddleware` +(`obp-api/src/main/scala/code/api/util/http4s/Http4sApp.scala`, `baseServices`). + +| Route | Paths | ResourceDoc? | How it checks requests instead | Stated reason | +|---|---|---|---|---| +| Dynamic Entity records | `/obp/dynamic-entity/...`, and the v7.0.0 form `/obp/v7.0.0/banks/BANK_ID/dynamic-entities/...` (served by `wrappedRoutesDynamicEntityV700`, `Http4s700.scala:7294`) | Generated at runtime, one set per entity; they show only the unversioned URLs so far | Inline authentication, Role and bank checks, ported from the Lift handlers, plus the before/after interceptors called inline (`Http4sDynamicEntity.scala`, header comment) | Entities are created at runtime, and the middleware builds its ResourceDoc index once, at start-up | +| Dynamic Endpoints | `/obp/dynamic-endpoint/...` | Generated at runtime | Proxy endpoints: `APIMethodsDynamicEndpoint.proxyHandle`. Compiled endpoints: `ResourceDoc.authCheckIO` (`APIUtil.scala:1835`), a copy of the doc-driven checks | Not stated; the same runtime-definition problem applies | +| Resource docs, Swagger, OpenAPI | `/obp/PREFIX/resource-docs/API_VERSION/{obp,swagger,openapi}` and the bank-scoped form | Yes (`ResourceDocs1_4_0/ResourceDocsAPIMethods.scala`) | Inline; `resource_docs_requires_role` is checked in the route (`Http4sResourceDocs.scala:108`) | Not stated | +| OpenAPI as YAML | `/obp/PREFIX/resource-docs/API_VERSION/openapi.yaml` | **No** | As above | Not stated | +| DirectLogin, unversioned | `POST /my/logins/direct` | The same endpoint is documented and served at `/obp/v6.0.0/my/logins/direct`, behind the middleware (`Http4s600.scala:13117`) | `DirectLoginRoutes` | Clients use the unversioned path | +| SIWE login | `POST /my/logins/siwe/challenge`, `POST /my/logins/siwe` | **No** | `SIWERoutes` | Not stated | +| Server pages | `/`, `/apps`, `/status`, `/health`, `/alive` | **No** | None needed: no authentication | Not API endpoints in the usual sense | +| CORS preflight and the JSON 404 | any `OPTIONS`; any unmatched path | Not applicable | `corsHandler`, `notFoundCatchAll` | Infrastructure | + +What these routes do not get from the middleware (checked by reading the code, 2026-09-28): + +| Route | API Metrics | Request transaction | Endpoint timeout | Idempotency | Telemetry timing | +|---|---|---|---|---|---| +| Dynamic Entity records | yes (through `EndpointHelpers`) | yes (inline) | no | no | no | +| Dynamic Endpoints | yes (through `EndpointHelpers`) | compiled: yes; proxy: auto-commit, as before the migration | no | no | no | +| Resource docs, Swagger, OpenAPI | no | not needed (read only) | no | not applicable | no | +| DirectLogin, SIWE | yes (through `EndpointHelpers`) | no | no | no | no | + +Whether the inline checks give exactly the same answers as the middleware (the same status codes, +the same error messages, the same check order) has not been verified route by route. The Dynamic +Entity checks were ported to match the Lift handlers, and its test suites pin that behaviour. + +## 5. What the inconsistency costs, and when to act + +**Costs.** +- The checks exist in more than one place and can drift apart. A drift of this kind, between a + doc's Roles and what the handler enforced, is what let three endpoints lose their Role in the + migration (fixed in `785f1a4b7`). Those were inside the middleware groups, but the risk is the + same wherever a check is copied. +- A feature added to the middleware (the endpoint timeout, idempotency, Telemetry timing) does not + reach these routes unless someone adds it a second time. +- Three endpoints have no ResourceDoc at all, so they appear in no documentation, no API Explorer + and no MCP listing. + +**Triggers for changing a route listed in section 4.** Change one when: +- a defect is traced to its checks differing from the middleware's; +- it needs something only the middleware gives (for example Dynamic Entity requests need the + endpoint timeout, or Telemetry timing, to diagnose an incident); +- it is being changed for another reason anyway, and the change can be tested with the route's + existing suites. + +**Changes that are safe now because they change no behaviour** (candidates, not scheduled): +- ResourceDocs for SIWE and `openapi.yaml`. Care is needed: in a version file, a ResourceDoc + with an `http4sPartialFunction` also adds a route under that version's prefix, which is a + behaviour change. A doc that only documents the existing unversioned path must not register a + new route by accident. +- A guard test that lists the route groups in `Http4sApp.baseServices` and fails when a new group + is added outside `ResourceDocMiddleware` without being added to an allowlist. The allowlist + starts as section 4 and only shrinks. It changes no runtime behaviour and stops the list from + growing while it stays as it is. + +**Changes not to make now:** moving Dynamic Entities or Dynamic Endpoints behind the middleware. +It needs the middleware to accept ResourceDocs that change at runtime, and Dynamic Entities have +deliberate behaviour (anonymous access where the entity allows it, personal entities, access +through Consents) that a move would put at risk. There is no reported defect that it would fix. + +**The resource-docs routes belong behind the middleware in principle, and stay outside it for +now.** They were built when ResourceDocs were documentation served beside the API. Now that +ResourceDocs drive routing, authentication, Roles, validation and Telemetry, the endpoints that +publish them are the exception. Moving them (decided against, 2026-09-28) meets four obstacles, +each of which changes what callers receive: + +1. The middleware sets `Content-Type: application/json` on every response not already marked as + JSON (`ensureJsonContentType` in `ResourceDocMiddleware.scala`). `openapi.yaml`, and the + routes' plain-text error responses, would be relabelled. +2. The docs are declared once, at v1.4.0, but the routes answer under every version prefix, and + the output depends on the prefix: v4.0.0 and later get the newer shape, v6.0.0 also adds the + technology field (`Http4sResourceDocs.scala`, `includeTechnologyForPrefix` and the + `isVersion4OrHigher` choice in `routes`). The middleware matches a doc by the version in the + path, so it would need a copy of each doc in every version group, or a matcher that ignores + versions, and either change reaches beyond these routes. +3. They are open by default and need a Role only when `resource_docs_requires_role=true`, checked + inline (`withOptionalRoleCheck`). A doc can declare a Role conditionally, but the choice is + fixed when the docs are built at start-up (see the shard 10 note in `CLAUDE.md`). +4. The middleware's endpoint timeout could turn a slow first render of the whole API's + documentation, with a cold cache, into a 504 where today it eventually succeeds. API Explorer + and the Portal load these documents on start-up. + +What the move would bring (Telemetry timing, API Metrics records, the shared Role check, +enable/disable by operation id) can mostly be added without moving them. Telemetry timing and +counters for their Redis caches were added that way (see `docs/telemetry_conventions.md`, +section 13). + +## 6. Related + +- `docs/telemetry_conventions.md`, section 13: which of these routes Telemetry does not time. +- `CLAUDE.md`: the migration rules (ResourceDoc registration order, the middleware's handling of + `BANK_ID`, `ACCOUNT_ID`, `VIEW_ID`, `COUNTERPARTY_ID`, and the gotchas). +- Stale comment: `Http4sDynamicEndpoint.scala`'s header still says an unmatched request falls + through to "the Lift bridge"; since the bridge was removed it reaches `notFoundCatchAll` (JSON + 404). diff --git a/docs/telemetry_conventions.md b/docs/telemetry_conventions.md index aa81641403..777dd7033c 100644 --- a/docs/telemetry_conventions.md +++ b/docs/telemetry_conventions.md @@ -287,6 +287,8 @@ thread and class figures on its own port. Remove it once the Micrometer JVM bind | `obp.api.batch_writer.rows` | counter | `writer` (`api_metrics`, `connector_metrics`), `result` (`queued`, `written`, `lost`) | the batch writers; `lost` is a batch dropped by a failed flush | | `obp.api.batch_writer.queue.depth` | gauge | `writer` | rows queued minus rows written or lost (the queue's own `size()` walks it) | | `obp.api.batch_writer.flushes` | timer | `writer`, `result` | each flush that had rows | +| `obp.api.redis_cache.gets`, `obp.api.redis_cache.sets` | counter | `cache` (`static_resource_docs`, `dynamic_resource_docs`, `all_resource_docs`, `static_swagger`, `financial_products`, `api_products`), `result` (`hit`, `miss`, `error` for gets; `success`, `error` for sets) | `Caching.tryGet` / `trySet`; `error` means Redis was unreachable | +| `obp.api.self_service_rate_limit.checks` | counter | `scope`, `outcome` (`allowed`, `warned`, `blocked`, `skipped`) | `SelfServiceRateLimiter.check`; in shadow mode, `warned` shows how often a scope would refuse real traffic | | `obp.api.connector.calls` | timer, fixed buckets | `connector`, `connector_method`, `result` | the Connector proxy (`code/bankconnectors/package.scala`) | | `obp.api.redis.commands` | timer | `command`, `result` | `Redis.use` | | `cache.gets`, `cache.puts`, `cache.evictions`, `cache.size` | standard | `cache` = `in_memory`, `json_schema`, `message_docs`, `on_behalf_of` | every Guava cache, through `Telemetry.monitorCache` | diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index de3976c209..2433cd22b5 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -1242,15 +1242,23 @@ featured_apis=elasticSearchWarehouseV300 # consumer_registration POST /dynamic-registration/consumers # lookup POST /account/check/scheme/iban # signal_channel_create POST /signal-channels/CHANNEL_NAME/messages when the channel does not exist yet +# documentation GET of the public documentation, any version prefix: resource-docs (obp, swagger, +# openapi, openapi.yaml, bank level), message-docs (plain, json-schema, swagger2.0), +# api/glossary, api/tags, api/versions, api/error-messages, api/popular-endpoints, +# endpoints/json-schema-validations, endpoints/authentication-type-validations # Enabled by default in shadow mode: a trip is logged (event=self_service_rate_limit_shadow_trip) # and reported to the caller in the X-Rate-Limit-Warning header (OBP-10059), but the request is # allowed. Set mode to enforce to answer 429 OBP-10060 instead. Counters live in Redis and fail open. # self_service.rate_limit.enabled = true # self_service.rate_limit.mode = shadow +# A scope can have its own mode, which overrides the one above. The documentation scope stays in +# shadow mode unless its own mode is set, even when the mode above is enforce: +# self_service.rate_limit.documentation.mode = shadow # Generic per-IP limits; -1 switches a window off, 0 blocks every call in it. # Built-in per-scope defaults (applied when neither a scope prop nor a generic prop is set): # signup 3/5/10 | password_reset 3/5/10 | consent_request 10/30/100 # consumer_registration 5/10/20 | lookup 20/60/200 | signal_channel_create 5/20/50 +# documentation 60/1000/10000 # self_service.rate_limit.per_ip.per_minute = 10 # self_service.rate_limit.per_ip.per_hour = 60 # self_service.rate_limit.per_ip.per_day = 200 diff --git a/obp-api/src/main/scala/code/api/cache/Caching.scala b/obp-api/src/main/scala/code/api/cache/Caching.scala index 1678e13f48..39ec880b42 100644 --- a/obp-api/src/main/scala/code/api/cache/Caching.scala +++ b/obp-api/src/main/scala/code/api/cache/Caching.scala @@ -126,51 +126,69 @@ object Caching extends MdcLoggable { // into a 500. These are the documents API Explorer and Portal load on startup, so a Redis blip // used to take the whole surface down. def getDynamicResourceDocCache(key: String): Option[String] = - tryGet(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_DYNAMIC_RESOURCE_DOCS_TTL) + tryGet("dynamic_resource_docs", DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_DYNAMIC_RESOURCE_DOCS_TTL) def setDynamicResourceDocCache(key: String, value: String): Unit = - trySet(DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_DYNAMIC_RESOURCE_DOCS_TTL, value) + trySet("dynamic_resource_docs", DYNAMIC_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_DYNAMIC_RESOURCE_DOCS_TTL, value) def getStaticResourceDocCache(key: String): Option[String] = - tryGet(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_STATIC_RESOURCE_DOCS_TTL) + tryGet("static_resource_docs", STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_STATIC_RESOURCE_DOCS_TTL) def setStaticResourceDocCache(key: String, value: String): Unit = - trySet(STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_STATIC_RESOURCE_DOCS_TTL, value) + trySet("static_resource_docs", STATIC_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_STATIC_RESOURCE_DOCS_TTL, value) def getAllResourceDocCache(key: String): Option[String] = - tryGet(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_DYNAMIC_RESOURCE_DOCS_TTL) + tryGet("all_resource_docs", ALL_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_DYNAMIC_RESOURCE_DOCS_TTL) def setAllResourceDocCache(key: String, value: String): Unit = - trySet(ALL_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_DYNAMIC_RESOURCE_DOCS_TTL, value) + trySet("all_resource_docs", ALL_RESOURCE_DOC_CACHE_KEY_PREFIX, key, GET_DYNAMIC_RESOURCE_DOCS_TTL, value) + // Also holds the connector JSON Schemas served by v6.0.0 message-docs/CONNECTOR/json-schema. def getStaticSwaggerDocCache(key: String): Option[String] = - tryGet(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX, key, GET_STATIC_RESOURCE_DOCS_TTL) + tryGet("static_swagger", STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX, key, GET_STATIC_RESOURCE_DOCS_TTL) def setStaticSwaggerDocCache(key: String, value: String): Unit = - trySet(STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX, key, GET_STATIC_RESOURCE_DOCS_TTL, value) + trySet("static_swagger", STATIC_SWAGGER_DOC_CACHE_KEY_PREFIX, key, GET_STATIC_RESOURCE_DOCS_TTL, value) // Fail-safe wrappers around Redis.use. If Redis is unreachable (dev without a // running Redis, transient failure, etc.) we treat it as a miss and recompute instead of failing // the whole request. - private def tryGet(prefix: String, key: String, ttlSeconds: Int): Option[String] = - try use(JedisMethod.GET, (prefix + key).intern(), Some(ttlSeconds)) - catch { case e: Throwable => logger.debug(s"Cache GET failed for $prefix$key: ${e.getMessage}"); None } + // + // Each read and write is counted for Telemetry under `cacheName`, a fixed name chosen at the call + // site (the key prefixes carry the instance and version namespace and are not meant for people). + // A read that failed because Redis was unreachable counts as "error", not "miss", so a Redis + // outage does not look like a cold cache. + private def tryGet(cacheName: String, prefix: String, key: String, ttlSeconds: Int): Option[String] = { + val outcome: Either[Throwable, Option[String]] = + try Right(use(JedisMethod.GET, (prefix + key).intern(), Some(ttlSeconds))) + catch { case e: Throwable => logger.debug(s"Cache GET failed for $prefix$key: ${e.getMessage}"); Left(e) } + val result = outcome match { + case Right(Some(_)) => "hit" + case Right(None) => "miss" + case Left(_) => "error" + } + code.telemetry.Telemetry.counter("obp.api.redis_cache.gets", "cache" -> cacheName, "result" -> result).increment() + outcome.toOption.flatten + } - private def trySet(prefix: String, key: String, ttlSeconds: Int, value: String): Unit = - try { use(JedisMethod.SET, (prefix + key).intern(), Some(ttlSeconds), Some(value)); () } - catch { case e: Throwable => logger.debug(s"Cache SET failed for $prefix$key: ${e.getMessage}") } + private def trySet(cacheName: String, prefix: String, key: String, ttlSeconds: Int, value: String): Unit = { + val result = + try { use(JedisMethod.SET, (prefix + key).intern(), Some(ttlSeconds), Some(value)); "success" } + catch { case e: Throwable => logger.debug(s"Cache SET failed for $prefix$key: ${e.getMessage}"); "error" } + code.telemetry.Telemetry.counter("obp.api.redis_cache.sets", "cache" -> cacheName, "result" -> result).increment() + } def getFinancialProductsCache(key: String, ttlSeconds: Int): Option[String] = - tryGet(FINANCIAL_PRODUCTS_PREFIX, key, ttlSeconds) + tryGet("financial_products", FINANCIAL_PRODUCTS_PREFIX, key, ttlSeconds) def setFinancialProductsCache(key: String, value: String, ttlSeconds: Int): Unit = - trySet(FINANCIAL_PRODUCTS_PREFIX, key, ttlSeconds, value) + trySet("financial_products", FINANCIAL_PRODUCTS_PREFIX, key, ttlSeconds, value) def getApiProductsCache(key: String, ttlSeconds: Int): Option[String] = - tryGet(API_PRODUCTS_PREFIX, key, ttlSeconds) + tryGet("api_products", API_PRODUCTS_PREFIX, key, ttlSeconds) def setApiProductsCache(key: String, value: String, ttlSeconds: Int): Unit = - trySet(API_PRODUCTS_PREFIX, key, ttlSeconds, value) + trySet("api_products", API_PRODUCTS_PREFIX, key, ttlSeconds, value) /** * Invalidate all rate limit cache entries for a specific consumer. diff --git a/obp-api/src/main/scala/code/api/util/APIUtil.scala b/obp-api/src/main/scala/code/api/util/APIUtil.scala index 794df2db25..751523505f 100644 --- a/obp-api/src/main/scala/code/api/util/APIUtil.scala +++ b/obp-api/src/main/scala/code/api/util/APIUtil.scala @@ -5155,16 +5155,21 @@ object APIUtil extends MdcLoggable with CustomJsonFormats{ ) } + /** + * The cache key of a rendered documentation response. The filters are a [[ResourceDocFilters]], + * never raw request values: that type can only be built sorted, de-duplicated and (for ResourceDoc + * listings) limited to values some ResourceDoc carries, so a caller cannot multiply cache entries + * by varying the filters. The key text keeps its earlier shape. + */ def createResourceDocCacheKey( bankId : Option[String], requestedApiVersionString: String, - tags: Option[List[ResourceDocTag]], - partialFunctions: Option[List[String]], + filters: ResourceDocFilters, locale: Option[String], contentParam: Option[ContentParam], apiCollectionIdParam: Option[String], isVersion4OrHigher: Option[Boolean] - ) = s"requestedApiVersionString:$requestedApiVersionString-bankId:$bankId-tags:$tags-partialFunctions:$partialFunctions-locale:${locale.toString}" + + ) = s"requestedApiVersionString:$requestedApiVersionString-bankId:$bankId-tags:${filters.tags}-partialFunctions:${filters.functions}-locale:${locale.toString}" + // The Glossary version belongs in the key: endpoint descriptions embed Glossary text, so a // Dynamic Glossary Item that overrides a static one must not stay masked by a cached document // for the rest of the resource-doc / swagger TTL. Reading it is an in-memory lookup that diff --git a/obp-api/src/main/scala/code/api/util/ResourceDocFilters.scala b/obp-api/src/main/scala/code/api/util/ResourceDocFilters.scala new file mode 100644 index 0000000000..7f2d943ead --- /dev/null +++ b/obp-api/src/main/scala/code/api/util/ResourceDocFilters.scala @@ -0,0 +1,141 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.util + +import java.util.concurrent.atomic.AtomicReference + +import code.api.Constant +import code.api.util.ApiTag.ResourceDocTag +import code.api.util.APIUtil.ResourceDoc + +/** + * This class holds the `tags` and `functions` filters of a documentation request in the form that + * may go into a cache key: sorted, without repeats, and (for ResourceDoc listings) limited to values + * that some ResourceDoc actually carries. + * + * The problem it solves: the resource-docs routes cache each rendered document under a key built + * from the filters. Built from the raw request values, the key had as many variations as a caller + * cared to invent (`?tags=Account,Bank`, `?tags=Bank,Account`, `?tags=Account,junk123`), and each + * variation forced a fresh render and a new cache entry, which is how a scanner can defeat the + * cache. The constructor is private, so the only way to get a value of this type is through one of + * the builders below, and `APIUtil.createResourceDocCacheKey` accepts only this type: a raw request + * value cannot reach a cache key. + * + * None of this changes a response. The filters match by membership (a document is kept when it + * carries any listed tag, or is any listed function), so the order and repeats of the values do + * not matter, and a value that no document carries matches nothing. When every value given is + * unknown the filter becomes an empty list, which matches no document, exactly as the unknown + * values did. + */ +final case class ResourceDocFilters private ( + tags: Option[List[ResourceDocTag]], + functions: Option[List[String]] +) + +object ResourceDocFilters { + + /** + * Filters for a ResourceDoc listing (resource-docs, Swagger, OpenAPI, bank level): sorted, + * de-duplicated, and limited to [[ResourceDocVocabulary]]. + */ + def forResourceDocs(tagValues: Option[List[String]], functionValues: Option[List[String]]): ResourceDocFilters = { + val vocabulary = ResourceDocVocabulary.current() + ResourceDocFilters( + tags = tagValues.map(values => canonical(values).filter(vocabulary.tags.contains).map(ResourceDocTag(_))), + functions = functionValues.map(values => canonical(values).filter(vocabulary.functions.contains)) + ) + } + + /** + * Filters for a listing of something other than ResourceDocs (message docs), whose values + * [[ResourceDocVocabulary]] does not describe: sorted and de-duplicated only. + */ + def normalisedOnly(tagValues: Option[List[String]], functionValues: Option[List[String]]): ResourceDocFilters = + ResourceDocFilters( + tags = tagValues.map(values => canonical(values).map(ResourceDocTag(_))), + functions = functionValues.map(canonical) + ) + + private def canonical(values: List[String]): List[String] = values.distinct.sorted +} + +/** + * This object is the list of every tag and every function name that a ResourceDoc on this instance + * carries, static and dynamic together. It is what [[ResourceDocFilters.forResourceDocs]] checks + * request values against. + * + * The static part comes from `APIUtil.allStaticResourceDocs`, every static ResourceDoc of every + * version, which does not change after start-up. The dynamic part (Dynamic Entities, Dynamic + * Endpoints, dynamic resource docs, and the v7.0.0 form of Dynamic Entity docs) changes at runtime, + * so it is rebuilt when the dynamic resource-docs cache namespace version changes (every Dynamic + * Entity write bumps it) and at most [[DynamicMaxAgeMillis]] after it was built, because a Dynamic + * Endpoint change does not bump that version. Cached dynamic documents can already be up to their + * cache TTL old, so this bound is well inside existing behaviour. + * + * The list is a superset for any one request, whose documents are a subset of all of these, so + * leaving out a value that is not in it can never leave out a document. + */ +object ResourceDocVocabulary { + + final case class Vocabulary(tags: Set[String], functions: Set[String]) + + val DynamicMaxAgeMillis: Long = 30000L + + private def vocabularyOf(docs: Iterable[ResourceDoc]): Vocabulary = + Vocabulary(docs.flatMap(_.tags.map(_.tag)).toSet, docs.map(_.partialFunctionName).toSet) + + private lazy val staticVocabulary: Vocabulary = vocabularyOf(APIUtil.allStaticResourceDocs) + + private final case class DynamicSnapshot(namespaceVersion: Long, builtAtMillis: Long, vocabulary: Vocabulary) + + private val dynamicSnapshot = new AtomicReference[Option[DynamicSnapshot]](None) + + private def dynamicDocs: List[ResourceDoc] = + APIUtil.allDynamicResourceDocs ++ code.api.dynamic.entity.helper.DynamicEntityHelper.v700Doc + + private def dynamicVocabulary(): Vocabulary = { + val namespaceVersion = Constant.getCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) + val now = System.currentTimeMillis() + dynamicSnapshot.get() match { + case Some(snapshot) if snapshot.namespaceVersion == namespaceVersion && now - snapshot.builtAtMillis < DynamicMaxAgeMillis => + snapshot.vocabulary + case _ => + val rebuilt = DynamicSnapshot(namespaceVersion, now, vocabularyOf(dynamicDocs)) + dynamicSnapshot.set(Some(rebuilt)) + rebuilt.vocabulary + } + } + + /** Every tag and function name a ResourceDoc on this instance carries now. */ + def current(): Vocabulary = { + val dynamic = dynamicVocabulary() + Vocabulary(staticVocabulary.tags ++ dynamic.tags, staticVocabulary.functions ++ dynamic.functions) + } + + /** Forget the dynamic part, so the next call rebuilds it (tests, and after bulk dynamic changes). */ + def refreshDynamic(): Unit = dynamicSnapshot.set(None) +} diff --git a/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala b/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala index 878d13229a..4d2e038275 100644 --- a/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala +++ b/obp-api/src/main/scala/code/api/util/SelfServiceRateLimiter.scala @@ -87,9 +87,21 @@ object SelfServiceRateLimiter extends MdcLoggable { "consent_request" -> ScopeDefaults(perMinute = 10, perHour = 30, perDay = 100, globalPerHour = -1), "consumer_registration" -> ScopeDefaults(perMinute = 5, perHour = 10, perDay = 20, globalPerHour = 500), "lookup" -> ScopeDefaults(perMinute = 20, perHour = 60, perDay = 200, globalPerHour = -1), - "signal_channel_create" -> ScopeDefaults(perMinute = 5, perHour = 20, perDay = 50, globalPerHour = -1) + "signal_channel_create" -> ScopeDefaults(perMinute = 5, perHour = 20, perDay = 50, globalPerHour = -1), + // Public documentation (resource docs, Swagger, OpenAPI). Generous, because one API Explorer + // session loads several documents and many users may reach OBP-API from one proxy address; + // the NMB scan of 2026-09-23 ran at about 13,000 requests an hour. + "documentation" -> ScopeDefaults(perMinute = 60, perHour = 1000, perDay = 10000, globalPerHour = -1) ) + /** + * Scopes that stay in shadow mode unless their own mode prop says otherwise, even when + * `self_service.rate_limit.mode` is enforce. Documentation is here so that switching on + * enforcement for sign-ups and password resets cannot start refusing documentation to API + * Explorer before the documentation limits have been checked against real traffic. + */ + val shadowUnlessSetScopes: Set[String] = Set("documentation") + /** One counter window after this request was counted. */ final case class Window(name: String, period: LimitCallPeriod, limit: Long, current: Long, resetSeconds: Long) { def remaining: Long = math.max(0L, limit - current) @@ -128,6 +140,18 @@ object SelfServiceRateLimiter extends MdcLoggable { def isEnforcing: Boolean = mode == ModeEnforce + /** + * The mode for one scope: `self_service.rate_limit..mode` when set, otherwise shadow + * for the scopes in [[shadowUnlessSetScopes]], otherwise the general mode. + */ + def modeFor(scope: String): String = + APIUtil.getPropsValue(s"$PropsPrefix.$scope.mode").toOption.map(_.trim.toLowerCase).filter(_.nonEmpty) match { + case Some(ModeEnforce) => ModeEnforce + case Some(_) => ModeShadow + case None if shadowUnlessSetScopes.contains(scope) => ModeShadow + case None => mode + } + /** Optional operator text naming when enforcement is planned, e.g. "2026-10-01". Only * appended to the warning when set; nothing about timing is claimed otherwise. */ def enforceAnnouncedFrom: Option[String] = @@ -192,12 +216,21 @@ object SelfServiceRateLimiter extends MdcLoggable { } } - if (windows.isEmpty) return Skipped(safeScope) + val outcome: Outcome = + if (windows.isEmpty) Skipped(safeScope) + else decide(safeScope, keyKind, safeKey, windows) + // Telemetry: how often each scope is allowed, warned (shadow trip) or blocked. In shadow mode + // this is what shows whether a scope's limits would hurt real traffic before enforcing them. + code.telemetry.Telemetry.counter("obp.api.self_service_rate_limit.checks", + "scope" -> safeScope, "outcome" -> outcome.getClass.getSimpleName.toLowerCase).increment() + outcome + } + private def decide(safeScope: String, keyKind: String, safeKey: String, windows: List[Window]): Outcome = { // Report the shortest exceeded window first, matching the consumer limiter's precedence. windows.find(_.exceeded) match { case None => Allowed(safeScope, windows) - case Some(exceeded) if isEnforcing => + case Some(exceeded) if modeFor(safeScope) == ModeEnforce => logger.warn(logLine("trip", safeScope, keyKind, safeKey, exceeded)) Blocked(safeScope, windows, exceeded) case Some(exceeded) => diff --git a/obp-api/src/main/scala/code/api/util/http4s/Http4sResourceDocs.scala b/obp-api/src/main/scala/code/api/util/http4s/Http4sResourceDocs.scala index 1e74bff282..a97eb4ee72 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/Http4sResourceDocs.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/Http4sResourceDocs.scala @@ -35,7 +35,7 @@ import code.api.ResponseHeader import code.api.cache.Caching import code.api.util.ApiRole.{canReadDynamicResourceDocsAtOneBank, canReadResourceDoc} import code.api.util.ErrorMessages._ -import code.api.util.{APIUtil, ApiRole, ApiVersionUtils, CustomJsonFormats, YAMLUtils} +import code.api.util.{APIUtil, ApiRole, ApiVersionUtils, CustomJsonFormats, ResourceDocFilters, YAMLUtils} import code.api.v1_4_0.JSONFactory1_4_0 import code.apicollectionendpoint.MappedApiCollectionEndpointsProvider import code.bankconnectors.rest.RestConnector_vMar2019 @@ -112,8 +112,10 @@ object Http4sResourceDocs extends MdcLoggable { // `req.uri.query.params` instead of Lift's `ObpS.param` / `S.request`. private final case class ParsedParams( - tags: Option[List[ResourceDocTag]], - partialFunctions: Option[List[String]], + // The raw filter values. They never reach a cache key or a filter directly: each handler turns + // them into a ResourceDocFilters first (see ResourceDocFilters for why). + tagValues: Option[List[String]], + functionValues: Option[List[String]], locale: Option[String], contentParam: Option[ContentParam], apiCollectionId: Option[String], @@ -133,7 +135,7 @@ object Http4sResourceDocs extends MdcLoggable { val tags = rawTags match { case None | Some("") => None case Some(s) => - val list = s.trim.split(",").toList.map(_.trim).filter(_.nonEmpty).map(ResourceDocTag(_)) + val list = s.trim.split(",").toList.map(_.trim).filter(_.nonEmpty) if (list.nonEmpty) Some(list) else None } val partialFunctions = rawFunctions match { @@ -311,13 +313,13 @@ object Http4sResourceDocs extends MdcLoggable { isVersion4OrHigher: Boolean ): Either[(Status, String), JValue] = { try { + val filters = ResourceDocFilters.forResourceDocs(params.tagValues, params.functionValues) val impl = implForPrefix(prefix) val includeTech = includeTechnologyForPrefix(prefix) val cacheKey = APIUtil.createResourceDocCacheKey( None, requestedApiVersionString, - params.tags, - params.partialFunctions, + filters, params.locale, params.contentParam, params.apiCollectionId, @@ -334,7 +336,7 @@ object Http4sResourceDocs extends MdcLoggable { val cached = Caching.getDynamicResourceDocCache(cacheKey) if (cached.isDefined) json.parse(cached.get) else { - val rdJson = impl.getResourceDocsObpDynamicCached(params.tags, params.partialFunctions, params.locale, None, isVersion4OrHigher = false).head + val rdJson = impl.getResourceDocsObpDynamicCached(filters.tags, filters.functions, params.locale, None, isVersion4OrHigher = false).head val jv = resourceDocsJsonToJsonResponse(rdJson) Caching.setDynamicResourceDocCache(cacheKey, json.compactRender(jv)) jv @@ -343,7 +345,7 @@ object Http4sResourceDocs extends MdcLoggable { val cached = Caching.getStaticResourceDocCache(cacheKey) if (cached.isDefined) json.parse(cached.get) else { - val rdJson = impl.getStaticResourceDocsObpCached(requestedApiVersionString, params.tags, params.partialFunctions, params.locale, isVersion4OrHigher).head + val rdJson = impl.getStaticResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, isVersion4OrHigher).head val jv = resourceDocsJsonToJsonResponse(rdJson) Caching.setStaticResourceDocCache(cacheKey, json.compactRender(jv)) jv @@ -352,7 +354,7 @@ object Http4sResourceDocs extends MdcLoggable { val cached = Caching.getAllResourceDocCache(cacheKey) if (cached.isDefined) json.parse(cached.get) else { - val rdJson = impl.getAllResourceDocsObpCached(requestedApiVersionString, params.tags, params.partialFunctions, params.locale, params.contentParam, isVersion4OrHigher).head + val rdJson = impl.getAllResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, params.contentParam, isVersion4OrHigher).head val jv = resourceDocsJsonToJsonResponse(rdJson) Caching.setAllResourceDocCache(cacheKey, json.compactRender(jv)) jv @@ -393,13 +395,13 @@ object Http4sResourceDocs extends MdcLoggable { requestedApiVersionString: String ): Either[(Status, String), JValue] = { try { + val filters = ResourceDocFilters.forResourceDocs(params.tagValues, params.functionValues) val impl = implForPrefix(prefix) val isVersion4OrHigher = true val cacheKey = APIUtil.createResourceDocCacheKey( None, requestedApiVersionString, - params.tags, - params.partialFunctions, + filters, params.locale, params.contentParam, params.apiCollectionId, @@ -418,11 +420,11 @@ object Http4sResourceDocs extends MdcLoggable { case None => params.contentParam match { case Some(DYNAMIC) => - impl.getResourceDocsObpDynamicCached(params.tags, params.partialFunctions, params.locale, None, isVersion4OrHigher).head.resource_docs + impl.getResourceDocsObpDynamicCached(filters.tags, filters.functions, params.locale, None, isVersion4OrHigher).head.resource_docs case Some(STATIC) => - impl.getStaticResourceDocsObpCached(requestedApiVersionString, params.tags, params.partialFunctions, params.locale, isVersion4OrHigher).head.resource_docs + impl.getStaticResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, isVersion4OrHigher).head.resource_docs case _ => - impl.getAllResourceDocsObpCached(requestedApiVersionString, params.tags, params.partialFunctions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs + impl.getAllResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs } } impl.convertResourceDocsToSwaggerJvalueAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered) @@ -466,13 +468,13 @@ object Http4sResourceDocs extends MdcLoggable { requestedApiVersionString: String ): Either[(Status, String), JValue] = { try { + val filters = ResourceDocFilters.forResourceDocs(params.tagValues, params.functionValues) val impl = implForPrefix(prefix) val isVersion4OrHigher = true val cacheKey = APIUtil.createResourceDocCacheKey( Some("openapi31"), requestedApiVersionString, - params.tags, - params.partialFunctions, + filters, params.locale, params.contentParam, params.apiCollectionId, @@ -491,11 +493,11 @@ object Http4sResourceDocs extends MdcLoggable { case None => params.contentParam match { case Some(DYNAMIC) => - impl.getResourceDocsObpDynamicCached(params.tags, params.partialFunctions, params.locale, None, isVersion4OrHigher).head.resource_docs + impl.getResourceDocsObpDynamicCached(filters.tags, filters.functions, params.locale, None, isVersion4OrHigher).head.resource_docs case Some(STATIC) => - impl.getStaticResourceDocsObpCached(requestedApiVersionString, params.tags, params.partialFunctions, params.locale, isVersion4OrHigher).head.resource_docs + impl.getStaticResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, isVersion4OrHigher).head.resource_docs case _ => - impl.getAllResourceDocsObpCached(requestedApiVersionString, params.tags, params.partialFunctions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs + impl.getAllResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs } } impl.convertResourceDocsToOpenAPI31JvalueAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered) @@ -536,13 +538,13 @@ object Http4sResourceDocs extends MdcLoggable { requestedApiVersionString: String ): Either[(Status, String), String] = { try { + val filters = ResourceDocFilters.forResourceDocs(params.tagValues, params.functionValues) val impl = implForPrefix(prefix) val isVersion4OrHigher = true val cacheKey = APIUtil.createResourceDocCacheKey( Some("openapi31yaml"), requestedApiVersionString, - params.tags, - params.partialFunctions, + filters, params.locale, params.contentParam, params.apiCollectionId, @@ -561,11 +563,11 @@ object Http4sResourceDocs extends MdcLoggable { case None => params.contentParam match { case Some(DYNAMIC) => - impl.getResourceDocsObpDynamicCached(params.tags, params.partialFunctions, params.locale, None, isVersion4OrHigher).head.resource_docs + impl.getResourceDocsObpDynamicCached(filters.tags, filters.functions, params.locale, None, isVersion4OrHigher).head.resource_docs case Some(STATIC) => - impl.getStaticResourceDocsObpCached(requestedApiVersionString, params.tags, params.partialFunctions, params.locale, isVersion4OrHigher).head.resource_docs + impl.getStaticResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, isVersion4OrHigher).head.resource_docs case _ => - impl.getAllResourceDocsObpCached(requestedApiVersionString, params.tags, params.partialFunctions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs + impl.getAllResourceDocsObpCached(requestedApiVersionString, filters.tags, filters.functions, params.locale, params.contentParam, isVersion4OrHigher).head.resource_docs } } impl.convertResourceDocsToOpenAPI31YAMLAndSetCache(cacheKey, requestedApiVersionString, resourceDocsJsonFiltered) @@ -622,12 +624,12 @@ object Http4sResourceDocs extends MdcLoggable { requestedApiVersionString: String ): Either[(Status, String), JValue] = { try { + val filters = ResourceDocFilters.forResourceDocs(params.tagValues, params.functionValues) val impl = implForPrefix(prefix) val cacheKey = APIUtil.createResourceDocCacheKey( Some(bankIdStr), requestedApiVersionString, - params.tags, - params.partialFunctions, + filters, params.locale, params.contentParam, params.apiCollectionId, @@ -637,7 +639,7 @@ object Http4sResourceDocs extends MdcLoggable { val jv: JValue = if (cached.isDefined) json.parse(cached.get) else { - val rdJson = impl.getResourceDocsObpDynamicCached(params.tags, params.partialFunctions, params.locale, None, isVersion4OrHigher = false).head + val rdJson = impl.getResourceDocsObpDynamicCached(filters.tags, filters.functions, params.locale, None, isVersion4OrHigher = false).head val response = resourceDocsJsonToJsonResponse(rdJson) Caching.setDynamicResourceDocCache(cacheKey, json.compactRender(response)) response @@ -665,11 +667,11 @@ object Http4sResourceDocs extends MdcLoggable { private def buildMessageDocsSwagger(params: ParsedParams, connector: String): Either[(Status, String), JValue] = { try { + val filters = ResourceDocFilters.normalisedOnly(params.tagValues, params.functionValues) val cacheKey = APIUtil.createResourceDocCacheKey( None, connector, - params.tags, - params.partialFunctions, + filters, params.locale, params.contentParam, params.apiCollectionId, @@ -680,7 +682,7 @@ object Http4sResourceDocs extends MdcLoggable { if (cached.isDefined) json.parse(cached.get) else { val convertedToResourceDocs = RestConnector_vMar2019.messageDocs.map(APIUtil.toResourceDoc).toList - val resourceDocListFiltered = ResourceDocsAPIMethodsUtil.filterResourceDocs(convertedToResourceDocs, params.tags, params.partialFunctions) + val resourceDocListFiltered = ResourceDocsAPIMethodsUtil.filterResourceDocs(convertedToResourceDocs, filters.tags, filters.functions) val resourceDocJsonList = JSONFactory1_4_0.createResourceDocsJson(resourceDocListFiltered, isVersion4OrHigher = true, None).resource_docs val swaggerResourceDoc = code.api.ResourceDocs1_4_0.SwaggerJSONFactory.createSwaggerResourceDoc(resourceDocJsonList, ApiVersion.v3_1_0) val allSwaggerDefinitionCaseClasses = @@ -705,6 +707,15 @@ object Http4sResourceDocs extends MdcLoggable { // along the path. The Lift dispatch did the same thing (one dispatcher per // version prefix, all calling the same handlers). + /** + * Records Telemetry for one of these routes under its ResourceDoc's operation id. These routes + * answer outside ResourceDocMiddleware, which is where every other endpoint is timed; their docs + * are declared in ResourceDocs1_4_0, at v1.4.0, whatever version prefix the request used. + */ + private def timed(handlerName: String)(response: IO[Response[IO]]): IO[Response[IO]] = + code.telemetry.Telemetry.timeEndpoint( + APIUtil.buildOperationId(ApiVersion.v1_4_0, handlerName), ApiVersion.v1_4_0.apiShortVersion)(response) + val routes: HttpRoutes[IO] = HttpRoutes.of[IO] { case req @ GET -> Root / "obp" / prefix / "resource-docs" / requestedApiVersionString / "obp" => // Match the Lift dispatchers' `isVersion4OrHigher` setting per prefix — @@ -714,10 +725,10 @@ object Http4sResourceDocs extends MdcLoggable { case "v4.0.0" | "v5.0.0" | "v5.1.0" | "v6.0.0" => true case _ => false } - handleGetResourceDocsObp(req, prefix, requestedApiVersionString, isVersion4OrHigher = isV4OrHigher) + timed("getResourceDocsObp")(handleGetResourceDocsObp(req, prefix, requestedApiVersionString, isVersion4OrHigher = isV4OrHigher)) case req @ GET -> Root / "obp" / prefix / "resource-docs" / requestedApiVersionString / "swagger" => - handleGetResourceDocsSwagger(req, prefix, requestedApiVersionString) + timed("getResourceDocsSwagger")(handleGetResourceDocsSwagger(req, prefix, requestedApiVersionString)) // OpenAPI 3.1 JSON and YAML — served for every URL prefix. // @@ -731,15 +742,18 @@ object Http4sResourceDocs extends MdcLoggable { // for non-v6 prefixes; `isVersion4OrHigher` is hardcoded `true` inside the // handlers because the OpenAPI converter always consumes the v4-shape input. case req @ GET -> Root / "obp" / prefix / "resource-docs" / requestedApiVersionString / "openapi" => - handleGetResourceDocsOpenAPI31(req, prefix, requestedApiVersionString) + timed("getResourceDocsOpenAPI31")(handleGetResourceDocsOpenAPI31(req, prefix, requestedApiVersionString)) + // No ResourceDoc describes the YAML form, so it has no operation id and is not timed. case req @ GET -> Root / "obp" / prefix / "resource-docs" / requestedApiVersionString / "openapi.yaml" => handleGetResourceDocsOpenAPI31Yaml(req, prefix, requestedApiVersionString) case req @ GET -> Root / "obp" / prefix / "banks" / bankIdStr / "resource-docs" / requestedApiVersionString / "obp" => - handleGetBankLevelDynamicResourceDocsObp(req, prefix, bankIdStr, requestedApiVersionString) + timed("getBankLevelDynamicResourceDocsObp")(handleGetBankLevelDynamicResourceDocsObp(req, prefix, bankIdStr, requestedApiVersionString)) case req @ GET -> Root / "obp" / _ / "message-docs" / connector / "swagger2.0" => - handleGetMessageDocsSwagger(req, connector) + code.telemetry.Telemetry.timeEndpoint( + APIUtil.buildOperationId(ApiVersion.v3_1_0, "getMessageDocsSwagger"), ApiVersion.v3_1_0.apiShortVersion + )(handleGetMessageDocsSwagger(req, connector)) } } diff --git a/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala b/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala index b8e76ea822..be15d3eaf3 100644 --- a/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala +++ b/obp-api/src/main/scala/code/api/util/http4s/SelfServiceRateLimitMiddleware.scala @@ -85,7 +85,32 @@ object SelfServiceRateLimitMiddleware extends MdcLoggable { // signal_channel_create: the one unbounded write into Redis. Counted only when the // channel named in the path does not exist yet, so ordinary publishing is untouched. Entry("signal_channel_create", Method.POST, s"^$V/signal-channels/([^/]+)/messages$$".r, - (_, m) => code.api.cache.RedisMessaging.channelInfo(m.group(1)).isEmpty) + (_, m) => code.api.cache.RedisMessaging.channelInfo(m.group(1)).isEmpty), + // documentation: every public documentation read, for any version prefix. Public and + // anonymous by design, and some are expensive when not cached (rendering the whole API, + // or working out popular endpoints from usage records). The resource-docs routes are served + // outside ResourceDocMiddleware, so the per-IP limit for anonymous calls never reaches them; + // for them this entry is the only per-IP limit. `/root` is left out: it is cheap, and + // monitoring polls it. Shadow mode unless the scope's own mode prop is set + // (SelfServiceRateLimiter.shadowUnlessSetScopes). + Entry("documentation", Method.GET, "^/obp/[^/]+/resource-docs/[^/]+/(obp|swagger|openapi|openapi\\.yaml)$".r), + Entry("documentation", Method.GET, "^/obp/[^/]+/banks/[^/]+/resource-docs/[^/]+/obp$".r), + Entry("documentation", Method.GET, "^/obp/[^/]+/message-docs/[^/]+(/json-schema|/swagger2\\.0)?$".r), + Entry("documentation", Method.GET, "^/obp/[^/]+/api/(glossary(/[^/]+)?|tags|versions|error-messages|popular-endpoints)$".r), + Entry("documentation", Method.GET, "^/obp/[^/]+/endpoints/(json-schema-validations|authentication-type-validations)$".r) + ) + + /** What each scope counts, in words, for the rate limiter configuration endpoint and API Manager. */ + val scopeDescriptions: Map[String, String] = Map( + "signup" -> "POST /users, /users/email-validation, /banks/BANK_ID/user-invitations", + "password_reset" -> "POST /users/password-reset-url, /users/password", + "consent_request" -> "POST /consumer/consent-requests, /consumer/vrp-consent-requests", + "consumer_registration" -> "POST /dynamic-registration/consumers", + "lookup" -> "POST /account/check/scheme/iban", + "signal_channel_create" -> "POST /signal-channels/CHANNEL_NAME/messages, when the channel does not exist yet", + "documentation" -> ("GET of the public documentation: resource-docs (obp, swagger, openapi, openapi.yaml, bank level), " + + "message-docs (plain, json-schema, swagger2.0), api/glossary, api/tags, api/versions, api/error-messages, " + + "api/popular-endpoints, endpoints/json-schema-validations, endpoints/authentication-type-validations") ) def scopeFor(req: Request[IO]): Option[String] = { diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index c97fe5f129..a24cf0a720 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -7110,7 +7110,7 @@ object Http4s700 { "Get Rate Limiter Config", s"""Returns the live configuration of the three rate limiters on this instance, in the order they are checked: | - |1. **self_service** runs before routing and authentication, keyed by client IP address, on the endpoints anyone can call before the bank has granted them anything. A trip answers 429 `OBP-10060`. + |1. **self_service** runs before routing and authentication, keyed by client IP address, on the endpoints anyone can call before the bank has granted them anything, and on the public documentation. A trip answers 429 `OBP-10060`. Each of its `limits` rows is a scope, with the endpoints it `covers` and its own `mode`: a scope can stay in shadow mode while the limiter's `mode` is enforce (the `documentation` scope does, unless its own mode is set). |2. **authentication** runs inside the credential check, keyed by IP address and account. A trip answers 429 `OBP-10061`. |3. **consumer** runs after authentication, keyed by Consumer, or by IP address for anonymous calls. A trip answers 429 `OBP-10018`. | diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala index 0d505c9341..2505309962 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory7.0.0.scala @@ -235,7 +235,10 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { per_day: Option[Long] = None, per_week: Option[Long] = None, per_month: Option[Long] = None, - global_per_hour: Option[Long] = None + global_per_hour: Option[Long] = None, + // Self-service scopes only: the scope's own mode (it can differ from the limiter's), and what it counts. + mode: Option[String] = None, + covers: Option[String] = None ) /** One of the three rate limiters, in the order they are checked. `mode` is shadow or enforce. */ case class RateLimiterJsonV700( @@ -254,7 +257,11 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { val rateLimitersJsonV700Example: RateLimitersJsonV700 = RateLimitersJsonV700(List( RateLimiterJsonV700("self_service", 1, "OBP-10060", enabled = true, "shadow", "client IP address", "before routing and before authentication, on the self-service endpoints", "self_service.rate_limit", - List(RateLimiterLimitJsonV700("signup", per_minute = Some(3), per_hour = Some(5), per_day = Some(10), global_per_hour = Some(500)))), + List( + RateLimiterLimitJsonV700("documentation", per_minute = Some(60), per_hour = Some(1000), per_day = Some(10000), global_per_hour = Some(-1), + mode = Some("shadow"), covers = Some("GET of the public documentation: resource-docs, message-docs, api/glossary, api/tags, api/versions, ...")), + RateLimiterLimitJsonV700("signup", per_minute = Some(3), per_hour = Some(5), per_day = Some(10), global_per_hour = Some(500), + mode = Some("shadow"), covers = Some("POST /users, /users/email-validation, /banks/BANK_ID/user-invitations")))), RateLimiterJsonV700("authentication", 2, "OBP-10061", enabled = false, "shadow", "client IP address and account", "inside the credential check of Direct Login, DAuth, Gateway Login and SIWE", "auth.rate_limit", List(RateLimiterLimitJsonV700("ip", per_minute = Some(10), per_hour = Some(100)), RateLimiterLimitJsonV700("account", per_minute = Some(6)))), @@ -273,14 +280,16 @@ object JSONFactory700 extends MdcLoggable with code.api.util.CustomJsonFormats { name = "self_service", order = 1, error_code = errorCode(ErrorMessages.TooManyRequestsSelfService), enabled = SelfServiceRateLimiter.enabled, mode = SelfServiceRateLimiter.mode, keyed_by = "client IP address", - runs = "before routing and before authentication, on the self-service endpoints", + runs = "before routing and before authentication, on the self-service endpoints and the public documentation", props_prefix = SelfServiceRateLimiter.PropsPrefix, limits = SelfServiceRateLimiter.scopeDefaults.keys.toList.sorted.map { scope => RateLimiterLimitJsonV700(scope, per_minute = opt(SelfServiceRateLimiter.perKeyLimit(scope, "per_minute")), per_hour = opt(SelfServiceRateLimiter.perKeyLimit(scope, "per_hour")), per_day = opt(SelfServiceRateLimiter.perKeyLimit(scope, "per_day")), - global_per_hour = opt(SelfServiceRateLimiter.globalPerHourLimit(scope))) + global_per_hour = opt(SelfServiceRateLimiter.globalPerHourLimit(scope)), + mode = Some(SelfServiceRateLimiter.modeFor(scope)), + covers = code.api.util.http4s.SelfServiceRateLimitMiddleware.scopeDescriptions.get(scope)) } ) val authentication = RateLimiterJsonV700( diff --git a/obp-api/src/main/scala/code/telemetry/Telemetry.scala b/obp-api/src/main/scala/code/telemetry/Telemetry.scala index 25a8e7584b..ce741038c1 100644 --- a/obp-api/src/main/scala/code/telemetry/Telemetry.scala +++ b/obp-api/src/main/scala/code/telemetry/Telemetry.scala @@ -127,6 +127,22 @@ object Telemetry { responseBytes.foreach(bytes => summary("obp.api.endpoint.response.size", "bytes", "operation" -> operationId).record(bytes.toDouble)) } + /** + * Times a route that answers outside ResourceDocMiddleware and records it as that middleware + * would: under the operation id of the route's own ResourceDoc and that doc's API version. Used + * where a route cannot sit behind the middleware (see + * docs/resource_doc_and_endpoint_consistency_status.md). A response that fails with an exception + * is recorded as 5xx, then the exception carries on. + */ + def timeEndpoint(operationId: String, apiVersion: String)( + response: cats.effect.IO[org.http4s.Response[cats.effect.IO]] + ): cats.effect.IO[org.http4s.Response[cats.effect.IO]] = + cats.effect.IO(System.nanoTime()).flatMap { startNanos => + response + .flatTap(served => cats.effect.IO(recordEndpoint(operationId, apiVersion, served.status.code, System.nanoTime() - startNanos, served.contentLength))) + .onError { case _ => cats.effect.IO(recordEndpoint(operationId, apiVersion, 500, System.nanoTime() - startNanos, None)) } + } + /** * The number of items in a list response: the length of the response itself when it is a JSON * array, or of its only array field when it is an object wrapping one list (`{"banks": [...]}`, diff --git a/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/ResourceDocsFilterNormalisationTest.scala b/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/ResourceDocsFilterNormalisationTest.scala new file mode 100644 index 0000000000..879fb1eb9e --- /dev/null +++ b/obp-api/src/test/scala/code/api/ResourceDocs1_4_0/ResourceDocsFilterNormalisationTest.scala @@ -0,0 +1,68 @@ +package code.api.ResourceDocs1_4_0 + +import code.telemetry.Telemetry +import com.openbankproject.commons.util.JsonAliases.compactRender +import org.scalatest.Tag + +/** + * This suite checks that the resource-docs `tags` and `functions` filters are normalised: the + * order of the values and repeated values do not change the response, and every spelling of the + * same filter shares one cached document. Before, each spelling built its own cache key, so a + * scanner varying the order or repeating values forced a fresh render every time. + */ +class ResourceDocsFilterNormalisationTest extends ResourceDocsV140ServerSetup { + + object FilterNormalisation extends Tag("ResourceDocsFilterNormalisation") + + private def staticDocsReads(result: String): Double = + Option(Telemetry.registry.find("obp.api.redis_cache.gets").tags("cache", "static_resource_docs", "result", result).counter()) + .map(_.count()).getOrElse(0.0) + + private def docs(tags: String, functions: Option[String] = None) = { + val params = List("content" -> "static", "tags" -> tags) ++ functions.map("functions" -> _) + makeGetRequest((ResourceDocsV5_1Request / "resource-docs" / "v5.1.0" / "obp").GET < errorsBefore) { + second shouldBe None + reads("error") - errorsBefore shouldBe 2.0 + } else { + second shouldBe Some("stored") + reads("miss") - missesBefore shouldBe 1.0 + reads("hit") - hitsBefore shouldBe 1.0 + } + } } diff --git a/obp-api/src/test/scala/code/api/util/ResourceDocFiltersTest.scala b/obp-api/src/test/scala/code/api/util/ResourceDocFiltersTest.scala new file mode 100644 index 0000000000..496af886dc --- /dev/null +++ b/obp-api/src/test/scala/code/api/util/ResourceDocFiltersTest.scala @@ -0,0 +1,81 @@ +package code.api.util + +import code.api.Constant +import code.api.util.ApiTag.ResourceDocTag +import code.dynamicEntity.{DynamicEntityCommons, DynamicEntityProvider} +import code.setup.ServerSetupWithTestData +import org.scalatest.Tag + +/** + * This suite checks ResourceDocFilters and ResourceDocVocabulary: filters are sorted, without + * repeats and limited to values a ResourceDoc carries, and the vocabulary covers the dynamic docs, + * including those of a Dynamic Entity registered after start-up. + */ +class ResourceDocFiltersTest extends ServerSetupWithTestData { + + object Filters extends Tag("ResourceDocFilters") + + feature("ResourceDocFilters.forResourceDocs") { + + scenario("sorts, removes repeats and leaves out values no ResourceDoc carries", Filters) { + val filters = ResourceDocFilters.forResourceDocs( + Some(List("Bank", "not-a-tag-7f3e", "Account", "Bank")), + Some(List("getBanks", "notAFunction7f3e", "getBank", "getBanks"))) + filters.tags shouldBe Some(List(ResourceDocTag("Account"), ResourceDocTag("Bank"))) + filters.functions shouldBe Some(List("getBank", "getBanks")) + } + + scenario("a filter whose values are all unknown stays a filter, and matches nothing", Filters) { + val filters = ResourceDocFilters.forResourceDocs(Some(List("junk1", "junk2")), Some(List("noSuchFunction"))) + filters.tags shouldBe Some(Nil) + filters.functions shouldBe Some(Nil) + code.api.ResourceDocs1_4_0.ResourceDocsAPIMethodsUtil.filterResourceDocs(APIUtil.allStaticResourceDocs, filters.tags, None) shouldBe empty + } + + scenario("no filter stays no filter", Filters) { + val filters = ResourceDocFilters.forResourceDocs(None, None) + filters.tags shouldBe None + filters.functions shouldBe None + } + + scenario("normalisedOnly sorts and removes repeats but keeps every value", Filters) { + val filters = ResourceDocFilters.normalisedOnly(Some(List("b", "a", "b")), Some(List("z", "y"))) + filters.tags shouldBe Some(List(ResourceDocTag("a"), ResourceDocTag("b"))) + filters.functions shouldBe Some(List("y", "z")) + } + } + + feature("ResourceDocVocabulary") { + + scenario("it holds every tag and function of the static ResourceDocs", Filters) { + val vocabulary = ResourceDocVocabulary.current() + APIUtil.allStaticResourceDocs.flatMap(_.tags.map(_.tag)).toSet.subsetOf(vocabulary.tags) shouldBe true + APIUtil.allStaticResourceDocs.map(_.partialFunctionName).toSet.subsetOf(vocabulary.functions) shouldBe true + } + + scenario("a Dynamic Entity registered after start-up enters it when the dynamic docs version changes", Filters) { + ResourceDocVocabulary.current() // build a snapshot before the entity exists + val entity = s"vocabulary_probe_${java.util.UUID.randomUUID().toString.replace("-", "").take(8)}" + DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate( + DynamicEntityCommons( + entityName = entity, + metadataJson = s"""{"$entity":{"description":"d","required":[],"properties":{"name":{"type":"string","example":"Alice","description":"a name"}}}}""", + dynamicEntityId = None, + userId = resourceUser1.userId, + bankId = None, + hasPersonalEntity = false, + hasCommunityAccess = true + ) + ).openOrThrowException(s"could not register $entity") + // What NewStyle does after every Dynamic Entity write. + Constant.incrementCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE) + + val dynamicDocs = APIUtil.allDynamicResourceDocs ++ code.api.dynamic.entity.helper.DynamicEntityHelper.v700Doc + withClue(s"no dynamic ResourceDoc mentions $entity: ") { dynamicDocs.exists(_.requestUrl.contains(entity)) shouldBe true } + // No refreshDynamic(): the version bump alone must make the snapshot rebuild. + val vocabulary = ResourceDocVocabulary.current() + dynamicDocs.flatMap(_.tags.map(_.tag)).toSet.subsetOf(vocabulary.tags) shouldBe true + dynamicDocs.map(_.partialFunctionName).toSet.subsetOf(vocabulary.functions) shouldBe true + } + } +} diff --git a/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala b/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala index 529d72af94..7ab645902b 100644 --- a/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala +++ b/obp-api/src/test/scala/code/api/util/SelfServiceRateLimiterTest.scala @@ -185,6 +185,38 @@ class SelfServiceRateLimiterTest extends ServerSetup { } } + feature("The documentation scope") { + + scenario("it has generous built-in limits and no global cap") { + setPropsValues(s"$P.per_ip.per_minute" -> "", s"$P.per_ip.per_hour" -> "", s"$P.per_ip.per_day" -> "", + s"$P.documentation.per_ip.per_minute" -> "", s"$P.documentation.per_ip.per_hour" -> "", + s"$P.documentation.per_ip.per_day" -> "", s"$P.documentation.global.per_hour" -> "") + SelfServiceRateLimiter.perKeyLimit("documentation", "per_minute") shouldBe 60L + SelfServiceRateLimiter.perKeyLimit("documentation", "per_hour") shouldBe 1000L + SelfServiceRateLimiter.perKeyLimit("documentation", "per_day") shouldBe 10000L + SelfServiceRateLimiter.globalPerHourLimit("documentation") shouldBe -1L + } + + scenario("it stays in shadow mode when the other scopes are enforced, until its own mode is set") { + setPropsValues(s"$P.mode" -> "enforce", s"$P.documentation.mode" -> "") + SelfServiceRateLimiter.modeFor("documentation") shouldBe SelfServiceRateLimiter.ModeShadow + SelfServiceRateLimiter.modeFor("signup") shouldBe SelfServiceRateLimiter.ModeEnforce + + setPropsValues(s"$P.documentation.mode" -> "enforce") + SelfServiceRateLimiter.modeFor("documentation") shouldBe SelfServiceRateLimiter.ModeEnforce + setPropsValues(s"$P.mode" -> "", s"$P.documentation.mode" -> "") + } + + scenario("a trip in the documentation scope is only a warning under a general enforce mode") { + setPropsValues(s"$P.enabled" -> "true", s"$P.mode" -> "enforce", s"$P.documentation.mode" -> "", + s"$P.documentation.per_ip.per_minute" -> "1") + val ip = freshIp() + SelfServiceRateLimiter.check("documentation", ip) shouldBe a[Allowed] + SelfServiceRateLimiter.check("documentation", ip) shouldBe a[Warned] + setPropsValues(s"$P.mode" -> "", s"$P.documentation.per_ip.per_minute" -> "") + } + } + feature("SelfServiceRateLimitMiddleware scope table") { def post(path: String): Request[IO] = Request[IO](Method.POST, Uri.unsafeFromString(path)) @@ -207,6 +239,39 @@ class SelfServiceRateLimiterTest extends ServerSetup { SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v4.0.0/account/check/scheme/iban")) shouldBe Some("lookup") } + scenario("the documentation routes map to the documentation scope, for any version prefix") { + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/resource-docs/v7.0.0/obp")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v3.1.0/resource-docs/v5.1.0/swagger")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/resource-docs/v6.0.0/openapi")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/resource-docs/v6.0.0/openapi.yaml")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v4.0.0/banks/gh.29.uk/resource-docs/v4.0.0/obp")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v5.1.0/message-docs/rest_vMar2019/swagger2.0")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v2.2.0/message-docs/rest_vMar2019")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/message-docs/rest_vMar2019/json-schema")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/api/glossary")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/api/glossary/Consent")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/api/tags")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/api/versions")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/api/error-messages")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/api/popular-endpoints")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v4.0.0/endpoints/json-schema-validations")) shouldBe Some("documentation") + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v4.0.0/endpoints/authentication-type-validations")) shouldBe Some("documentation") + // Only reads; and the cheap /root that monitoring polls is not counted. + SelfServiceRateLimitMiddleware.scopeFor(post("/obp/v7.0.0/resource-docs/v7.0.0/obp")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/root")) shouldBe None + SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v7.0.0/management/dynamic-message-docs")) shouldBe None + } + + scenario("every scope in the table has built-in limits and a description") { + val scopes = SelfServiceRateLimitMiddleware.entries.map(_.scope).toSet + scopes.foreach { scope => + withClue(s"scope $scope: ") { + SelfServiceRateLimiter.scopeDefaults.keySet should contain(scope) + SelfServiceRateLimitMiddleware.scopeDescriptions.keySet should contain(scope) + } + } + } + scenario("everything else is left alone") { SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/users")) shouldBe None SelfServiceRateLimitMiddleware.scopeFor(get("/obp/v6.0.0/users/current")) shouldBe None diff --git a/obp-api/src/test/scala/code/api/v7_0_0/RateLimitersTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/RateLimitersTest.scala index 5a9a44c2f0..a46e176179 100644 --- a/obp-api/src/test/scala/code/api/v7_0_0/RateLimitersTest.scala +++ b/obp-api/src/test/scala/code/api/v7_0_0/RateLimitersTest.scala @@ -80,6 +80,17 @@ class RateLimitersTest extends ServerSetupWithTestData { selfServiceScopes should contain allOf ("signup", "password_reset", "consumer_registration") selfServiceScopes should not contain "login" + Then("every self-service scope says what it covers and its own mode; documentation is shadow unless set") + val selfServiceRows = (limiters.head \ "limits").asInstanceOf[JArray].arr + selfServiceRows.foreach { row => + str(row, "covers") should not be empty + List("shadow", "enforce") should contain(str(row, "mode")) + } + val documentation = selfServiceRows.find(row => str(row, "scope") == "documentation") + documentation should not be empty + str(documentation.get, "mode") should equal("shadow") + str(documentation.get, "covers") should include("api/glossary") + Then("the authentication limiter reports an ip and an account window") val authScopes = (limiters(1) \ "limits").asInstanceOf[JArray].arr.map(l => str(l, "scope")) authScopes should equal(List("ip", "account")) diff --git a/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala b/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala index 1a616a6bcf..99d33da907 100644 --- a/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala +++ b/obp-api/src/test/scala/code/telemetry/TelemetryTest.scala @@ -69,6 +69,19 @@ class TelemetryTest extends FlatSpec with Matchers { Telemetry.listItemCount(JObject(List("a" -> JArray(Nil), "b" -> JArray(Nil)))) shouldBe None } + it should "time a route outside the middleware under its operation id, and count an exception as 5xx" in { + import cats.effect.IO + import cats.effect.unsafe.implicits.global + import org.http4s.{Response, Status} + val operation = unique("OBPv1.4.0-probeDocs") + Telemetry.timeEndpoint(operation, "v1.4.0")(IO.pure(Response[IO](Status.Ok))).unsafeRunSync().status shouldBe Status.Ok + an[RuntimeException] should be thrownBy + Telemetry.timeEndpoint(operation, "v1.4.0")(IO.raiseError[Response[IO]](new RuntimeException("boom"))).unsafeRunSync() + val scraped = Telemetry.scrape() + scraped should include(s"""obp_api_endpoint_requests_seconds_count{api_version="v1.4.0",operation="$operation",status="2xx"} 1""") + scraped should include(s"""obp_api_endpoint_requests_seconds_count{api_version="v1.4.0",operation="$operation",status="5xx"} 1""") + } + "BatchWriterTelemetry" should "derive the queue depth from rows queued, written and lost" in { val writer = unique("writer") val batchTelemetry = new BatchWriterTelemetry(writer) diff --git a/release_notes.md b/release_notes.md index 5514bd4eff..de3ff124e7 100644 --- a/release_notes.md +++ b/release_notes.md @@ -3,6 +3,25 @@ ### Most recent changes at top of file ``` Date Commit Action +28/09/2026 TBD NEW rate-limit scope "documentation" in the self-service (per client IP) + limiter, covering every public documentation read under any version prefix: + resource-docs, message-docs, api/glossary, api/tags, api/versions, + api/error-messages, api/popular-endpoints and the endpoints/*-validations + lists. Built-in limits 60 a minute, 1000 an hour, 10000 a day per IP. It + stays in shadow mode (X-Rate-Limit-Warning, no 429) even when + self_service.rate_limit.mode is enforce, until + self_service.rate_limit.documentation.mode = enforce is set. + NEW prop: self_service.rate_limit..mode, a mode per scope. + CHANGED: the resource-docs tags and functions filters are sorted, + de-duplicated and limited to values some ResourceDoc carries (static or + dynamic, see ResourceDocVocabulary), so every spelling of the same filter, + with or without made-up values, shares one cached document. Responses are + unchanged: the filters match by membership, and a filter of only unknown + values still returns no documents. Cache keys now accept only a + ResourceDocFilters, which can only be built this way. + NEW Telemetry: resource-docs, Swagger, OpenAPI and message-docs Swagger + requests are timed; hits, misses and errors of the Redis-backed document + and product caches; outcomes of the self-service limiter per scope. 27/09/2026 TBD NEW: Telemetry, aggregated numbers about each running instance for Prometheus and Grafana (not API Metrics; see the Glossary entry "Telemetry" and docs/telemetry_conventions.md). Recorded always: