From 5a58ad497d50a40b2e92483ee164d17ea9ee76b7 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Fri, 23 Oct 2026 15:44:34 -0700 Subject: [PATCH 1/5] Describe every usage metric and simple metric feature area Every usage metrics provider now reports a description with each value, and simple metric feature areas get descriptions too, including the client-side areas counted by @labkey/components. --- .../announcements/AnnouncementModule.java | 13 +- .../api/assay/actions/DesignerAction.java | 3 +- .../assay/transform/DataTransformService.java | 5 +- .../labkey/api/data/RequestCancellation.java | 5 +- .../org/labkey/api/module/DefaultModule.java | 3 + .../api/query/AbstractQueryImportAction.java | 17 ++- .../report/ExternalScriptEngineReport.java | 5 +- .../report/InternalScriptEngineReport.java | 5 +- .../reports/report/ScriptEngineReport.java | 1 - .../reports/report/python/IpynbReport.java | 4 +- .../api/security/AuthenticationManager.java | 4 +- .../labkey/api/security/SecurityManager.java | 4 +- .../api/usageMetrics/ApiMetricArea.java | 35 +++++ .../api/usageMetrics/AssayMetricArea.java | 37 +++++ .../api/usageMetrics/CoreMetricArea.java | 37 +++++ .../usageMetrics/ExperimentMetricArea.java | 38 +++++ ...ricsProvider.java => QueryMetricArea.java} | 21 ++- .../api/usageMetrics/SimpleMetricArea.java | 27 ++++ .../usageMetrics/SimpleMetricsService.java | 31 ++++ .../api/usageMetrics/UsageMetricGroup.java | 3 +- .../labkey/api/usageMetrics/UsageMetrics.java | 143 ++++++++++++++++-- .../usageMetrics/UsageMetricsProvider.java | 16 +- .../api/usageMetrics/UsageMetricsService.java | 29 +--- api/src/org/labkey/api/util/HttpUtil.java | 4 +- api/src/org/labkey/api/view/FileServlet.java | 3 +- api/src/org/labkey/api/view/Portal.java | 4 +- .../filters/ContentSecurityPolicyFilter.java | 5 +- assay/src/org/labkey/assay/AssayModule.java | 5 + .../src/org/labkey/assay/PlateController.java | 9 +- .../PipelineDataCollectorRedirectAction.java | 4 +- .../assay/plate/PlateMetricsProvider.java | 20 +-- core/src/org/labkey/core/CoreController.java | 13 +- core/src/org/labkey/core/CoreModule.java | 92 +++++------ .../core/admin/DisplayFormatAnalyzer.java | 5 +- .../usageMetrics/UsageMetricsServiceImpl.java | 6 +- .../org/labkey/core/login/DbLoginManager.java | 2 +- .../metrics/ClientSideMetricDescriptions.java | 52 +++++++ .../metrics/SimpleMetricsServiceImpl.java | 50 +++++- .../metrics/WebSocketConnectionManager.java | 15 +- .../labkey/experiment/ExperimentModule.java | 36 +++-- .../experiment/FileLinkMetricsProvider.java | 4 +- .../SpecialCharacterMetricsProvider.java | 4 +- .../api/ExpDataClassDataTableImpl.java | 3 +- .../experiment/api/ExpRunTableImpl.java | 3 +- .../api/SampleTypeUpdateServiceDI.java | 3 +- .../api/property/PropertyServiceImpl.java | 4 +- .../controllers/exp/ExperimentController.java | 11 +- .../filecontent/FileContentMetricArea.java | 36 +++++ .../labkey/filecontent/FileContentModule.java | 29 ++-- issues/src/org/labkey/issue/IssuesModule.java | 12 +- list/src/org/labkey/list/ListModule.java | 13 +- .../list/model/ListQueryUpdateService.java | 4 +- .../org/labkey/pipeline/PipelineModule.java | 15 +- query/src/org/labkey/query/QueryModule.java | 12 ++ .../labkey/query/persist/QueryManager.java | 15 +- .../query/reports/ReportServiceImpl.java | 13 +- .../org/labkey/search/SearchMetricArea.java | 36 +++++ .../src/org/labkey/search/SearchModule.java | 9 +- .../model/WritableIndexManagerImpl.java | 4 +- study/src/org/labkey/study/StudyModule.java | 55 ++++--- .../src/org/labkey/survey/SurveyModule.java | 10 +- 61 files changed, 801 insertions(+), 305 deletions(-) create mode 100644 api/src/org/labkey/api/usageMetrics/ApiMetricArea.java create mode 100644 api/src/org/labkey/api/usageMetrics/AssayMetricArea.java create mode 100644 api/src/org/labkey/api/usageMetrics/CoreMetricArea.java create mode 100644 api/src/org/labkey/api/usageMetrics/ExperimentMetricArea.java rename api/src/org/labkey/api/usageMetrics/{DescribedUsageMetricsProvider.java => QueryMetricArea.java} (56%) create mode 100644 api/src/org/labkey/api/usageMetrics/SimpleMetricArea.java create mode 100644 core/src/org/labkey/core/metrics/ClientSideMetricDescriptions.java create mode 100644 filecontent/src/org/labkey/filecontent/FileContentMetricArea.java create mode 100644 search/src/org/labkey/search/SearchMetricArea.java diff --git a/announcements/src/org/labkey/announcements/AnnouncementModule.java b/announcements/src/org/labkey/announcements/AnnouncementModule.java index 05329cbfbe4..c10df200cf5 100644 --- a/announcements/src/org/labkey/announcements/AnnouncementModule.java +++ b/announcements/src/org/labkey/announcements/AnnouncementModule.java @@ -190,7 +190,7 @@ public void doStartup(ModuleContext moduleContext) UsageMetricsService svc = UsageMetricsService.get(); if (svc != null) { - svc.registerUsageMetrics(NAME, () -> { + svc.registerUsageMetrics(NAME, metrics -> { MultiSet secure = new HashMultiSet<>(); MultiSet moderator = new HashMultiSet<>(); CommSchema comm = CommSchema.getInstance(); @@ -208,14 +208,13 @@ public void doStartup(ModuleContext moduleContext) boards.getAndIncrement(); }); } - return Map.of( - "messageBoards", boards.get(), - "messages", new TableSelector(comm.getTableInfoAnnouncements()).getRowCount(), - "settings", Map.of( + return metrics + .put("messageBoards", boards.get(), "Containers with at least one message") + .put("messages", new TableSelector(comm.getTableInfoAnnouncements()).getRowCount(), "Messages across all containers, including responses") + .put("settings", Map.of( "secure", MultiSetUtils.getOccurrenceMap(secure), "moderatorReview", MultiSetUtils.getOccurrenceMap(moderator) - ) - ); + ), "Message boards counted in messageBoards, by their secure setting value and by their moderator review setting value"); }); } } diff --git a/api/src/org/labkey/api/assay/actions/DesignerAction.java b/api/src/org/labkey/api/assay/actions/DesignerAction.java index f450a3c985b..d477285efef 100644 --- a/api/src/org/labkey/api/assay/actions/DesignerAction.java +++ b/api/src/org/labkey/api/assay/actions/DesignerAction.java @@ -25,6 +25,7 @@ import org.labkey.api.module.ModuleHtmlView; import org.labkey.api.module.ModuleLoader; import org.labkey.api.security.RequiresPermission; +import org.labkey.api.usageMetrics.AssayMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.view.ActionURL; @@ -71,7 +72,7 @@ public ModelAndView getView(DesignerForm form, BindException errors) String[] files = request.getParameterValues("file"); if (request.getParameter("isRedirect") == null && path != null && files != null && files.length == 1) { - SimpleMetricsService.get().increment("Assay", "AssayDesigner", "InferDesignFromFileAndImport"); + SimpleMetricsService.get().increment(AssayMetricArea.ASSAY_DESIGNER, "InferDesignFromFileAndImport"); ActionURL url = getViewContext().cloneActionURL(); url.addParameter("file", files[0]); url.addParameter("isRedirect", true); diff --git a/api/src/org/labkey/api/assay/transform/DataTransformService.java b/api/src/org/labkey/api/assay/transform/DataTransformService.java index 45cc73f46c3..1256a97c76e 100644 --- a/api/src/org/labkey/api/assay/transform/DataTransformService.java +++ b/api/src/org/labkey/api/assay/transform/DataTransformService.java @@ -35,6 +35,7 @@ import org.labkey.api.reports.LabKeyScriptEngineManager; import org.labkey.api.security.SecurityManager; import org.labkey.api.settings.AppProps; +import org.labkey.api.usageMetrics.AssayMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.FileUtil; import org.labkey.api.util.PageFlowUtil; @@ -157,9 +158,9 @@ public TransformResult transformAndValidate( logIfInTransaction(context, operation); if (operation == TransformOperation.UPDATE) - SimpleMetricsService.get().increment("Assay", "transformScript", "executionCountForUpdate"); + SimpleMetricsService.get().increment(AssayMetricArea.TRANSFORM_SCRIPT, "executionCountForUpdate"); if (operation == TransformOperation.INSERT) - SimpleMetricsService.get().increment("Assay", "transformScript", "executionCountForInsert"); + SimpleMetricsService.get().increment(AssayMetricArea.TRANSFORM_SCRIPT, "executionCountForInsert"); // TODO: don't bother slurping the contents of binary scripts StringBuilder sb = new StringBuilder(); diff --git a/api/src/org/labkey/api/data/RequestCancellation.java b/api/src/org/labkey/api/data/RequestCancellation.java index ab0b1c5ab98..17ed7eeb72b 100644 --- a/api/src/org/labkey/api/data/RequestCancellation.java +++ b/api/src/org/labkey/api/data/RequestCancellation.java @@ -26,8 +26,8 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.module.DefaultModule; import org.labkey.api.security.User; +import org.labkey.api.usageMetrics.CoreMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.TestContext; import org.labkey.api.util.logging.LogHelper; @@ -57,7 +57,6 @@ public final class RequestCancellation { private static final Logger LOG = LogHelper.getLogger(RequestCancellation.class, "Cancellation of in-flight requests and their queries"); // statementCancelled and backendCancelled overlap when a timeout cancels an async query, so they don't add up - private static final String METRIC_AREA = "requestCancel"; private static final long STUCK_AFTER_MS = 10_000; /** Tests set this to make the driver's cancel signal nothing, forcing the pg_cancel_backend fallback */ @@ -312,7 +311,7 @@ private static void increment(String metricName) { try { - SimpleMetricsService.get().increment(DefaultModule.CORE_MODULE_NAME, METRIC_AREA, metricName); + SimpleMetricsService.get().increment(CoreMetricArea.REQUEST_CANCEL, metricName); } catch (RuntimeException e) { diff --git a/api/src/org/labkey/api/module/DefaultModule.java b/api/src/org/labkey/api/module/DefaultModule.java index acc7a23ef9c..e396bb120bf 100644 --- a/api/src/org/labkey/api/module/DefaultModule.java +++ b/api/src/org/labkey/api/module/DefaultModule.java @@ -236,6 +236,9 @@ public void unregister() final public void startup(ModuleContext moduleContext) { doStartup(moduleContext); + // After doStartup() because Core registers SimpleMetricsService there + if (!_controllerNameToClass.isEmpty()) + SimpleMetricsService.get().describeFeatureArea(getName(), "controllerHits", "Requests handled by this module's controllers, by controller name"); } protected abstract void doStartup(ModuleContext moduleContext); diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index 5f0b7e51899..f712c212449 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -54,6 +54,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.usageMetrics.QueryMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.CPUTimer; import org.labkey.api.util.FileStream; @@ -608,7 +609,7 @@ else if (!dataFileDir.exists()) PipelineService.get().queueJob(job, getQueryImportJobNotificationProviderName()); if (_target != null) - SimpleMetricsService.get().increment("query", "fileBackgroundImports", getMetricPrefix(_target)); + SimpleMetricsService.get().increment(QueryMetricArea.FILE_BACKGROUND_IMPORTS, getMetricPrefix(_target)); JSONObject response = new JSONObject(); response.put("success", true); response.put("jobId", PipelineService.get().getJobId(user, getContainer(), job.getJobGUID())); @@ -1001,7 +1002,7 @@ else if (!context.isCrossTypeImport()) public static void addImportValidationErrorMetric(QueryUpdateService.InsertOption insertOption, @Nullable TableInfo target, @Nullable String queryName) { - String featureArea = insertOption.toString().toLowerCase() + "FailWithValidationError"; + String featureArea = getValidationErrorFeatureArea(insertOption); String targetType = null; if (target != null) targetType = getMetricPrefix(target); @@ -1011,6 +1012,16 @@ else if (queryName != null) SimpleMetricsService.get().increment("query", featureArea, targetType); } + public static String getValidationErrorFeatureArea(QueryUpdateService.InsertOption insertOption) + { + return insertOption.toString().toLowerCase() + "FailWithValidationError"; + } + + public static String getRowCountFeatureArea(QueryUpdateService.InsertOption insertOption) + { + return "file" + StringUtils.capitalize(insertOption.toString().toLowerCase()) + "Counts"; + } + private static String getMetricPrefix(@NotNull TableInfo target) { String metricPrefix = target.getUserSchema() == null ? target.getSchema().getName() : target.getUserSchema().getSchemaName(); @@ -1019,7 +1030,7 @@ private static String getMetricPrefix(@NotNull TableInfo target) public static void incrementRowCountMetric(int count, QueryUpdateService.InsertOption insertOption, String prefix) { - String featureArea = "file" + StringUtils.capitalize(insertOption.toString().toLowerCase()) + "Counts"; + String featureArea = getRowCountFeatureArea(insertOption); String lcPrefix = prefix.toLowerCase(); if (count <= 1000) SimpleMetricsService.get().increment("query", featureArea, lcPrefix + "1to1000"); diff --git a/api/src/org/labkey/api/reports/report/ExternalScriptEngineReport.java b/api/src/org/labkey/api/reports/report/ExternalScriptEngineReport.java index 6c1639d0d7c..79058ff7505 100644 --- a/api/src/org/labkey/api/reports/report/ExternalScriptEngineReport.java +++ b/api/src/org/labkey/api/reports/report/ExternalScriptEngineReport.java @@ -18,11 +18,9 @@ import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; -import org.labkey.api.ApiModule; import org.labkey.api.assay.transform.DataTransformService; import org.labkey.api.attachments.AttachmentParent; import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.module.ModuleLoader; import org.labkey.api.pipeline.PipeRoot; import org.labkey.api.pipeline.PipelineService; import org.labkey.api.query.QueryService.NamedParameterNotProvided; @@ -40,6 +38,7 @@ import org.labkey.api.security.SecurityManager; import org.labkey.api.security.SecurityManager.TransformSession; import org.labkey.api.thumbnail.Thumbnail; +import org.labkey.api.usageMetrics.ApiMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.DateUtil; import org.labkey.api.util.ExceptionUtil; @@ -418,7 +417,7 @@ protected Object runScript(ScriptEngine engine, ViewContext context, List outputSubst, bindings.put(ExternalScriptEngine.WORKING_DIRECTORY, getReportDirFileLike(context.getContainer().getId()).toNioPathForWrite().toString()); - SimpleMetricsService.get().increment(ModuleLoader.getInstance().getModule(ApiModule.class).getName(), METRIC_FEATURE_AREA, "Internal-" + engine.getFactory().getEngineName()); + SimpleMetricsService.get().increment(ApiMetricArea.REPORT_INVOCATION, "Internal-" + engine.getFactory().getEngineName()); Object output = engine.eval(createScript(engine, context, outputSubst, inputDataTsv, inputParameters)); consolePw.flush(); diff --git a/api/src/org/labkey/api/reports/report/ScriptEngineReport.java b/api/src/org/labkey/api/reports/report/ScriptEngineReport.java index 936a7448d5f..6856c68192a 100644 --- a/api/src/org/labkey/api/reports/report/ScriptEngineReport.java +++ b/api/src/org/labkey/api/reports/report/ScriptEngineReport.java @@ -102,7 +102,6 @@ public abstract class ScriptEngineReport extends ScriptReport implements Report. public static final String SUBSTITUTION_MAP = "substitutionMap.txt"; public static final String CONSOLE_OUTPUT = "console.txt"; - public static final String METRIC_FEATURE_AREA = "ReportInvocation"; private static final Logger LOG = LogManager.getLogger(ScriptEngineReport.class); diff --git a/api/src/org/labkey/api/reports/report/python/IpynbReport.java b/api/src/org/labkey/api/reports/report/python/IpynbReport.java index be938644e0e..bf494e9e878 100644 --- a/api/src/org/labkey/api/reports/report/python/IpynbReport.java +++ b/api/src/org/labkey/api/reports/report/python/IpynbReport.java @@ -33,7 +33,6 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.json.JSONObject; -import org.labkey.api.ApiModule; import org.labkey.api.data.Container; import org.labkey.api.data.DbScope; import org.labkey.api.module.ModuleLoader; @@ -47,6 +46,7 @@ import org.labkey.api.reports.report.r.view.ConsoleOutput; import org.labkey.api.reports.report.r.view.IpynbOutput; import org.labkey.api.security.SessionApiKeyManager; +import org.labkey.api.usageMetrics.ApiMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.CSRFUtil; import org.labkey.api.util.ConfigurationException; @@ -380,7 +380,7 @@ public int execute(ViewContext context, String apiKey, FileLike working, FileLik tryPing(service); - SimpleMetricsService.get().increment(ModuleLoader.getInstance().getModule(ApiModule.class).getName(), ScriptEngineReport.METRIC_FEATURE_AREA, getClass().getSimpleName()); + SimpleMetricsService.get().increment(ApiMetricArea.REPORT_INVOCATION, getClass().getSimpleName()); try (CloseableHttpClient client = HttpClients.createDefault()) { diff --git a/api/src/org/labkey/api/security/AuthenticationManager.java b/api/src/org/labkey/api/security/AuthenticationManager.java index 8b42f774126..ba047c7ff6e 100644 --- a/api/src/org/labkey/api/security/AuthenticationManager.java +++ b/api/src/org/labkey/api/security/AuthenticationManager.java @@ -1890,11 +1890,11 @@ public static URLHelper getAfterLoginURL(Container current, @Nullable LoginRetur public static void registerMetricsProvider() { - UsageMetricsService.get().registerUsageMetrics(ModuleLoader.getInstance().getCoreModule().getName(), () -> { + UsageMetricsService.get().registerUsageMetrics(ModuleLoader.getInstance().getCoreModule().getName(), metrics -> { Map map = AuthenticationConfigurationCache.getActive(AuthenticationConfiguration.class).stream() .collect(Collectors.groupingBy(config->config.getAuthenticationProvider().getName(), Collectors.counting())); - return Collections.singletonMap("authenticationConfigurations", map); + return metrics.put("authenticationConfigurations", map, "Enabled authentication configurations by provider (Database, LDAP, SAML, CAS, etc.)"); }); } diff --git a/api/src/org/labkey/api/security/SecurityManager.java b/api/src/org/labkey/api/security/SecurityManager.java index c601cd3ec09..3baa56a6588 100644 --- a/api/src/org/labkey/api/security/SecurityManager.java +++ b/api/src/org/labkey/api/security/SecurityManager.java @@ -209,7 +209,7 @@ private static class Authentication {} public static UsageMetricsProvider getMetricsProvider() { - return () -> { + return metrics -> { MultiValuedMap mvm = new HashSetValuedHashMap<>(); Set siteRoles = RoleManager.getSiteRoles(); for (RoleAssignment assignment : ContainerManager.getRoot().getPolicy().getAssignments()) @@ -231,7 +231,7 @@ public static UsageMetricsProvider getMetricsProvider() Map roleCounts = mvm.keySet().stream() .collect(Collectors.toMap(Role::getName, key -> mvm.get(key).size())); - return Map.of("SiteRoleUserCounts", roleCounts); + return metrics.put("SiteRoleUserCounts", roleCounts, "Active users holding each site-level role, directly or through group membership"); }; } diff --git a/api/src/org/labkey/api/usageMetrics/ApiMetricArea.java b/api/src/org/labkey/api/usageMetrics/ApiMetricArea.java new file mode 100644 index 00000000000..d46040961d3 --- /dev/null +++ b/api/src/org/labkey/api/usageMetrics/ApiMetricArea.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.usageMetrics; + +public enum ApiMetricArea implements SimpleMetricArea +{ + FILE_SERVLET("FileServlet", "/files/ URLs forwarded to the FileContent module's sendFile action (urlsDispatched)"), + REPORT_INVOCATION("ReportInvocation", "Script report executions by report and engine: - for external engines, Internal- for in-process engines, and the report class for Jupyter notebook reports"); + + private final Definition _definition; + + ApiMetricArea(String areaName, String description) + { + _definition = new Definition("API", areaName, description); + } + + @Override + public Definition getDefinition() + { + return _definition; + } +} diff --git a/api/src/org/labkey/api/usageMetrics/AssayMetricArea.java b/api/src/org/labkey/api/usageMetrics/AssayMetricArea.java new file mode 100644 index 00000000000..dbc42ce5e13 --- /dev/null +++ b/api/src/org/labkey/api/usageMetrics/AssayMetricArea.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.usageMetrics; + +public enum AssayMetricArea implements SimpleMetricArea +{ + ASSAY_DESIGNER("AssayDesigner", "Assay imports started from files selected in the file browser: InferDesignFromFileAndImport creates a design inferred from one file, ImportFromFile imports into an existing design"), + TRANSFORM_SCRIPT("transformScript", "Assay transform script executions, by operation (executionCountForInsert, executionCountForUpdate)"), + PLATE("plate", "Plate export requests, by format (exportCSV, exportTSV, exportMAP)"), + PLATE_SET("plateSet", "Plate set worklist exports (exportWorklist)"); + + private final Definition _definition; + + AssayMetricArea(String areaName, String description) + { + _definition = new Definition("Assay", areaName, description); + } + + @Override + public Definition getDefinition() + { + return _definition; + } +} diff --git a/api/src/org/labkey/api/usageMetrics/CoreMetricArea.java b/api/src/org/labkey/api/usageMetrics/CoreMetricArea.java new file mode 100644 index 00000000000..9d4e889cbd9 --- /dev/null +++ b/api/src/org/labkey/api/usageMetrics/CoreMetricArea.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.usageMetrics; + +import org.labkey.api.module.DefaultModule; + +public enum CoreMetricArea implements SimpleMetricArea +{ + CLIENT_API_REQUESTS("clientApiRequests", "Requests from LabKey client API libraries, by library (Java, Python, R, etc.), identified by user agent"), + REQUEST_CANCEL("requestCancel", "Cancelled requests by source (timeout, clientDisconnect; shutdown counts database backends instead), plus statementCancelled: running statements cancelled, and backendCancelled: database backends signalled"); + + private final Definition _definition; + + CoreMetricArea(String areaName, String description) + { + _definition = new Definition(DefaultModule.CORE_MODULE_NAME, areaName, description); + } + + @Override + public Definition getDefinition() + { + return _definition; + } +} diff --git a/api/src/org/labkey/api/usageMetrics/ExperimentMetricArea.java b/api/src/org/labkey/api/usageMetrics/ExperimentMetricArea.java new file mode 100644 index 00000000000..b66bc6d6f36 --- /dev/null +++ b/api/src/org/labkey/api/usageMetrics/ExperimentMetricArea.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.usageMetrics; + +import org.labkey.api.exp.api.ExperimentService; + +public enum ExperimentMetricArea implements SimpleMetricArea +{ + MOVE_ENTITIES("moveEntities", "Requests that moved entities to another folder, by entity type (samples, sources for data class rows, assayRuns, list, jobs, notebooks)"), + CONVERT_TABLE("convertTable", "Client-supplied grid data converted to a download, by format (asExcel, asDelimited); only counted when the request includes query info for auditing"), + SAMPLE_IMPORT("sampleImport", "Non-background sample imports spanning multiple sample types, including ones that failed validation (crossTypeImport), plus the subset using the update (crossTypeUpdate) or merge (crossTypeMerge) option"); + + private final Definition _definition; + + ExperimentMetricArea(String areaName, String description) + { + _definition = new Definition(ExperimentService.MODULE_NAME, areaName, description); + } + + @Override + public Definition getDefinition() + { + return _definition; + } +} diff --git a/api/src/org/labkey/api/usageMetrics/DescribedUsageMetricsProvider.java b/api/src/org/labkey/api/usageMetrics/QueryMetricArea.java similarity index 56% rename from api/src/org/labkey/api/usageMetrics/DescribedUsageMetricsProvider.java rename to api/src/org/labkey/api/usageMetrics/QueryMetricArea.java index 3375f58bf74..bf8d9b3b495 100644 --- a/api/src/org/labkey/api/usageMetrics/DescribedUsageMetricsProvider.java +++ b/api/src/org/labkey/api/usageMetrics/QueryMetricArea.java @@ -15,23 +15,20 @@ */ package org.labkey.api.usageMetrics; -import java.util.Map; - -/** Reports metric values together with their descriptions. Prefer this over a plain {@link UsageMetricsProvider}. */ -@FunctionalInterface -public interface DescribedUsageMetricsProvider extends UsageMetricsProvider +public enum QueryMetricArea implements SimpleMetricArea { - UsageMetrics addUsageMetrics(UsageMetrics metrics); + FILE_BACKGROUND_IMPORTS("fileBackgroundImports", "Query imports queued as background pipeline jobs, by target schema name"); - @Override - default Map getUsageMetrics() + private final Definition _definition; + + QueryMetricArea(String areaName, String description) { - return collectUsageMetrics().getValues(); + _definition = new Definition("query", areaName, description); } - default UsageMetrics collectUsageMetrics() + @Override + public Definition getDefinition() { - UsageMetrics metrics = new UsageMetrics(); - return addUsageMetrics(metrics); + return _definition; } } diff --git a/api/src/org/labkey/api/usageMetrics/SimpleMetricArea.java b/api/src/org/labkey/api/usageMetrics/SimpleMetricArea.java new file mode 100644 index 00000000000..c6d9843ecb2 --- /dev/null +++ b/api/src/org/labkey/api/usageMetrics/SimpleMetricArea.java @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.usageMetrics; + +/** + * A {@link SimpleMetricsService} feature area with a fixed name, typically an enum value. + * The area name is persisted with its counts, so it must never change. + */ +public interface SimpleMetricArea +{ + record Definition(String moduleName, String areaName, String description) {} + + Definition getDefinition(); +} diff --git a/api/src/org/labkey/api/usageMetrics/SimpleMetricsService.java b/api/src/org/labkey/api/usageMetrics/SimpleMetricsService.java index 18e5febbdae..1124c0a3a10 100644 --- a/api/src/org/labkey/api/usageMetrics/SimpleMetricsService.java +++ b/api/src/org/labkey/api/usageMetrics/SimpleMetricsService.java @@ -15,6 +15,7 @@ */ package org.labkey.api.usageMetrics; +import org.labkey.api.module.DefaultModule; import org.labkey.api.services.ServiceRegistry; public interface SimpleMetricsService @@ -36,6 +37,7 @@ static void setInstance(SimpleMetricsService impl) /** * Increment the persistent counter associated with the given module, feature area, and metric name combination. + * Prefer {@link #increment(SimpleMetricArea, String)} unless the feature area's name is computed. * The total will be reported to mothership in this module's "simpleMetricCounts" node. Isn't that simple? * @param moduleName Module name. Must match a currently deployed module's name, though we'll grudgingly accept * casing differences vs. the module's canonical name. But just use the module's name constant so @@ -48,4 +50,33 @@ static void setInstance(SimpleMetricsService impl) * @return new value for this counter */ long increment(String moduleName, String featureArea, String metricName); + + default long increment(SimpleMetricArea area, String metricName) + { + SimpleMetricArea.Definition definition = area.getDefinition(); + return increment(definition.moduleName(), definition.areaName(), metricName); + } + + /** + * Describe what a feature area's metrics count, for the usage metric descriptions. Its metric names share the + * description, so this works for names only known at runtime. Call from module startup. + */ + default void describeFeatureArea(String moduleName, String featureArea, String description) + { + } + + /** Describes a feature area counted by core-incrementClientSideMetricCount, which records it under Core */ + default void describeClientSideFeatureArea(String featureArea, String description) + { + describeFeatureArea(DefaultModule.CORE_MODULE_NAME, featureArea, description); + } + + default void describeFeatureAreas(SimpleMetricArea... areas) + { + for (SimpleMetricArea area : areas) + { + SimpleMetricArea.Definition definition = area.getDefinition(); + describeFeatureArea(definition.moduleName(), definition.areaName(), definition.description()); + } + } } diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java b/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java index c86c4b05171..a5b9adc80ac 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java @@ -18,13 +18,12 @@ import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.util.List; import java.util.Map; /** A described nested object; its children may omit descriptions because they inherit the group's */ public class UsageMetricGroup extends UsageMetrics { - UsageMetricGroup(@NotNull String path, Map values, List descriptions) + UsageMetricGroup(@NotNull String path, @Nullable Map values, Map descriptions) { super(path, values, descriptions); } diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetrics.java b/api/src/org/labkey/api/usageMetrics/UsageMetrics.java index 1d7209e71df..ef8d5787195 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetrics.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetrics.java @@ -20,11 +20,11 @@ import org.junit.Assert; import org.junit.Test; -import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.function.Supplier; /** * Collects a provider's metric values together with their descriptions. Only the values are sent to mothership; @@ -46,14 +46,15 @@ public record Description(@NotNull String path, @Nullable String description, @N private final String _path; private final Map _values; - private final List _descriptions; + private final Map _descriptions; public UsageMetrics() { - this(null, new LinkedHashMap<>(), new ArrayList<>()); + this(null, new LinkedHashMap<>(), new LinkedHashMap<>()); } - UsageMetrics(@Nullable String path, Map values, List descriptions) + /** @param values null for a group that's only described */ + UsageMetrics(@Nullable String path, @Nullable Map values, Map descriptions) { _path = path; _values = values; @@ -67,8 +68,43 @@ public UsageMetrics put(@NotNull String key, @Nullable Object value, @NotNull St public UsageMetrics put(@NotNull String key, @Nullable Object value, @NotNull String description, @NotNull Kind kind) { - _descriptions.add(new Description(describedPath(key), description, kind)); - _values.put(key, value); + describe(key, description, kind); + getMutableValues().put(key, value); + return this; + } + + /** Calls the supplier only when values are reported, so a {@code reportedIf(false)} view skips the work */ + public UsageMetrics put(@NotNull String key, @NotNull Supplier value, @NotNull String description) + { + return put(key, value, description, Kind.CURRENT); + } + + public UsageMetrics put(@NotNull String key, @NotNull Supplier value, @NotNull String description, @NotNull Kind kind) + { + describe(key, description, kind); + if (_values != null) + _values.put(key, value.get()); + return this; + } + + /** @return this, or if not reported, a view at the same path that only describes what's put into it, groups included */ + public UsageMetrics reportedIf(boolean reported) + { + return reported ? this : new UsageMetrics(_path, null, _descriptions); + } + + /** Describes a key without reporting it, for keys this server omits, such as ones only reported under some condition */ + public UsageMetrics describe(@NotNull String key, @NotNull String description) + { + return describe(key, description, Kind.CURRENT); + } + + public UsageMetrics describe(@NotNull String key, @NotNull String description, @NotNull Kind kind) + { + Description added = new Description(describedPath(key), description, kind); + Description existing = _descriptions.putIfAbsent(added.path(), added); + if (existing != null && !existing.equals(added)) + throw new IllegalStateException("Usage metric " + added.path() + " is described inconsistently: " + existing + " vs. " + added); return this; } @@ -80,14 +116,28 @@ public UsageMetricGroup group(@NotNull String key, @NotNull String description) public UsageMetricGroup group(@NotNull String key, @NotNull String description, @NotNull Kind kind) { - _descriptions.add(new Description(describedPath(key), description, kind)); + describe(key, description, kind); return addGroup(key); } + /** Describes a nested object without reporting it. Its children can be described, but not given values. */ + public UsageMetricGroup describeGroup(@NotNull String key, @NotNull String description) + { + return describeGroup(key, description, Kind.CURRENT); + } + + public UsageMetricGroup describeGroup(@NotNull String key, @NotNull String description, @NotNull Kind kind) + { + describe(key, description, kind); + return new UsageMetricGroup(childPath(key), null, _descriptions); + } + UsageMetricGroup addGroup(@NotNull String key) { + if (_values == null) + return new UsageMetricGroup(childPath(key), null, _descriptions); Map child = new LinkedHashMap<>(); - _values.put(key, child); + getMutableValues().put(key, child); return new UsageMetricGroup(childPath(key), child, _descriptions); } @@ -106,17 +156,19 @@ String childPath(String key) Map getMutableValues() { + if (_values == null) + throw new IllegalStateException("Usage metric group " + _path + " is only described, so it can't hold values"); return _values; } public Map getValues() { - return Collections.unmodifiableMap(_values); + return _values == null ? Map.of() : Collections.unmodifiableMap(_values); } public List getDescriptions() { - return Collections.unmodifiableList(_descriptions); + return List.copyOf(_descriptions.values()); } public static class TestCase extends Assert @@ -155,7 +207,78 @@ public void testRejectsDottedKeys() UsageMetrics metrics = new UsageMetrics(); assertThrows(IllegalArgumentException.class, () -> metrics.put("a.b", 1L, "Dotted")); assertThrows(IllegalArgumentException.class, () -> metrics.group("a.b", "Dotted")); + assertThrows(IllegalArgumentException.class, () -> metrics.describe("a.b", "Dotted")); metrics.group("files", "Files by name").put("report.xlsx", 1L); } + + @Test + public void testDescribeWithoutReporting() + { + UsageMetrics metrics = new UsageMetrics(); + metrics.describe("optional", "Only reported sometimes"); + metrics.group("byType", "Counts by type").describe("rare", "Rarely present type", Kind.CUMULATIVE); + + assertEquals(Map.of("byType", Map.of()), metrics.getValues()); + assertEquals(List.of( + new Description("optional", "Only reported sometimes", Kind.CURRENT), + new Description("byType", "Counts by type", Kind.CURRENT), + new Description("byType.rare", "Rarely present type", Kind.CUMULATIVE)), + metrics.getDescriptions()); + } + + @Test + public void testRepeatedDescriptions() + { + UsageMetrics metrics = new UsageMetrics(); + metrics.describe("count", "Things"); + metrics.put("count", 1L, "Things"); + assertEquals(List.of(new Description("count", "Things", Kind.CURRENT)), metrics.getDescriptions()); + + assertThrows(IllegalStateException.class, () -> metrics.put("count", 1L, "Other things")); + assertThrows(IllegalStateException.class, () -> metrics.describe("count", "Things", Kind.CUMULATIVE)); + } + + @Test + public void testDescribeGroupWithoutReporting() + { + UsageMetrics metrics = new UsageMetrics(); + UsageMetricGroup absent = metrics.describeGroup("perFolder", "Folder counts", Kind.CURRENT); + absent.describe("runs", "Runs"); + + assertEquals(Map.of(), metrics.getValues()); + assertEquals(List.of( + new Description("perFolder", "Folder counts", Kind.CURRENT), + new Description("perFolder.runs", "Runs", Kind.CURRENT)), + metrics.getDescriptions()); + assertThrows(IllegalStateException.class, () -> absent.put("runs", 1L)); + assertThrows(IllegalStateException.class, () -> absent.put("runs", 1L, "Runs")); + } + + @Test + public void testReportedIf() + { + UsageMetrics metrics = new UsageMetrics(); + assertSame(metrics, metrics.reportedIf(true)); + metrics.reportedIf(true).put("shown", () -> 1L, "Shown"); + metrics.reportedIf(false).put("hidden", () -> { throw new AssertionError("Supplier called for unreported key"); }, "Hidden", Kind.CUMULATIVE); + UsageMetricGroup colors = metrics.group("colors", "Colors"); + colors.put("count", 0L, "Colors defined"); + colors.reportedIf(false).put("archived", () -> 2L, "Archived colors"); + UsageMetricGroup absent = metrics.reportedIf(false).group("absent", "Absent group"); + absent.group("child", "Absent child").describe("leaf", "Leaf"); + assertThrows(IllegalStateException.class, () -> absent.put("x", 1L)); + + assertEquals(Map.of("shown", 1L, "colors", Map.of("count", 0L)), metrics.getValues()); + assertEquals(List.of( + new Description("shown", "Shown", Kind.CURRENT), + new Description("hidden", "Hidden", Kind.CUMULATIVE), + new Description("colors", "Colors", Kind.CURRENT), + new Description("colors.count", "Colors defined", Kind.CURRENT), + new Description("colors.archived", "Archived colors", Kind.CURRENT), + new Description("absent", "Absent group", Kind.CURRENT), + new Description("absent.child", "Absent child", Kind.CURRENT), + new Description("absent.child.leaf", "Leaf", Kind.CURRENT)), + metrics.getDescriptions()); + } } } diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetricsProvider.java b/api/src/org/labkey/api/usageMetrics/UsageMetricsProvider.java index 06873917662..fc889498a7b 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetricsProvider.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetricsProvider.java @@ -18,10 +18,22 @@ import java.util.Map; /** - * Functional interface for modules to report their own usage metrics. + * Functional interface for modules to report their own usage metrics, together with their descriptions. * Register an implementation with UsageMetricsService at module startup. */ +@FunctionalInterface public interface UsageMetricsProvider { - Map getUsageMetrics(); + UsageMetrics addUsageMetrics(UsageMetrics metrics); + + default Map getUsageMetrics() + { + return collectUsageMetrics().getValues(); + } + + default UsageMetrics collectUsageMetrics() + { + UsageMetrics metrics = new UsageMetrics(); + return addUsageMetrics(metrics); + } } diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetricsService.java b/api/src/org/labkey/api/usageMetrics/UsageMetricsService.java index f4e471874cb..3058ac1cee4 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetricsService.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetricsService.java @@ -27,11 +27,9 @@ * UsageMetricsService svc = UsageMetricsService.get(); * if (null != svc) * { - * svc.registerUsageMetrics(DataIntegrationModule.NAME, () -> { - * Map metric = new HashMap<>(); - * metric.put("etlRunCount", new SqlSelector(DbSchema.get("dataintegration", DbSchemaType.Module), "SELECT COUNT(*) FROM dataintegration.TransformRun").getObject(Long.class)); - * return metric; - * }); + * svc.registerUsageMetrics(DataIntegrationModule.NAME, metrics -> metrics.put("etlRunCount", + * new SqlSelector(DbSchema.get("dataintegration", DbSchemaType.Module), "SELECT COUNT(*) FROM dataintegration.TransformRun").getObject(Long.class), + * "ETL runs")); * } */ public interface UsageMetricsService @@ -52,36 +50,17 @@ static void setInstance(UsageMetricsService impl) @Nullable Map> getModuleUsageMetrics(); /** - * Switch to using the DescribedUsageMetricsProvider variant. - * * Method to register a module's metrics. Call from doStartup() / startupAfterSpringConfiguration() * Metrics can be included at UsageReportingLevel OFF or ON. Usage metrics will only be sent for ON. * * @param moduleName The name of the module - * @param metrics Implementation of the functional interface UsageMetricsProvider.getUsageMetrics() method. Typically, - * this will return a map of metric name -> value pairs. * @throws IllegalArgumentException if moduleName doesn't correspond to an existing module */ - @Deprecated void registerUsageMetrics(String moduleName, UsageMetricsProvider metrics); - /** - * - * Method to register a module's metrics. Call from doStartup() / startupAfterSpringConfiguration() - * Metrics can be included at UsageReportingLevel OFF or ON. Usage metrics will only be sent for ON. - * - * @param moduleName The name of the module - * @throws IllegalArgumentException if moduleName doesn't correspond to an existing module - */ - default void registerUsageMetrics(String moduleName, DescribedUsageMetricsProvider metrics) - { - registerUsageMetrics(moduleName, (UsageMetricsProvider) metrics); - } - /** * Not included in the mothership report. Runs every provider, so it costs as much as generating a report. - * @return map of module name to its metric descriptions; top-level keys from providers that aren't - * {@link DescribedUsageMetricsProvider}s are included with a null description + * @return map of module name to its metric descriptions */ Map> getModuleUsageMetricDescriptions(); } diff --git a/api/src/org/labkey/api/util/HttpUtil.java b/api/src/org/labkey/api/util/HttpUtil.java index 18c5882f559..cb08b1701b8 100644 --- a/api/src/org/labkey/api/util/HttpUtil.java +++ b/api/src/org/labkey/api/util/HttpUtil.java @@ -33,7 +33,7 @@ import org.labkey.api.action.BaseApiAction; import org.labkey.api.miniprofiler.CustomTiming; import org.labkey.api.miniprofiler.MiniProfiler; -import org.labkey.api.module.DefaultModule; +import org.labkey.api.usageMetrics.CoreMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.logging.LogHelper; import org.labkey.api.view.ActionURL; @@ -255,7 +255,7 @@ public static void trackClientApiRequests(HttpServletRequest request) { String clientLibrary = getClientLibrary(request); if (null != clientLibrary) - SimpleMetricsService.get().increment(DefaultModule.CORE_MODULE_NAME, "clientApiRequests", clientLibrary); + SimpleMetricsService.get().increment(CoreMetricArea.CLIENT_API_REQUESTS, clientLibrary); } public static @Nullable String getClientLibrary(HttpServletRequest request) diff --git a/api/src/org/labkey/api/view/FileServlet.java b/api/src/org/labkey/api/view/FileServlet.java index a6a819d39b0..b9036469e80 100644 --- a/api/src/org/labkey/api/view/FileServlet.java +++ b/api/src/org/labkey/api/view/FileServlet.java @@ -20,6 +20,7 @@ import org.apache.logging.log4j.Logger; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.usageMetrics.ApiMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.URLHelper; @@ -74,7 +75,7 @@ protected void service(HttpServletRequest request, HttpServletResponse response) request.setAttribute(ViewServlet.ORIGINAL_URL_URLHELPER, helper); } - SimpleMetricsService.get().increment("API", "FileServlet", "urlsDispatched"); + SimpleMetricsService.get().increment(ApiMetricArea.FILE_SERVLET, "urlsDispatched"); String dispatchUrl = containerPath + "/filecontent-sendFile.view?" + (null == fileNameParam ? "fileName=" + PageFlowUtil.encodeURIComponent(fileName) : ""); _log.info("FileServlet dispatching {} to {}", request.getRequestURL(), dispatchUrl); // NOTE other parameters seem to get magically propagated... diff --git a/api/src/org/labkey/api/view/Portal.java b/api/src/org/labkey/api/view/Portal.java index a528abbb391..bcb0adcac44 100644 --- a/api/src/org/labkey/api/view/Portal.java +++ b/api/src/org/labkey/api/view/Portal.java @@ -205,13 +205,13 @@ public static void clearWebPartProperties(String nameSearchText, String properti /** Issue 51727 - metrics to track web part usage */ public static UsageMetricsProvider getMetricsProvider() { - return () -> + return metrics -> { SQLFragment sql = new SQLFragment("SELECT Name, COUNT(*) AS C FROM "). append(getTableInfoPortalWebParts(), "pwp"). unsafeAppend(" GROUP BY Name"); - return Map.of("webPartCounts", new SqlSelector(getSchema(), sql).getValueMap(String.class)); + return metrics.put("webPartCounts", new SqlSelector(getSchema(), sql).getValueMap(String.class), "Web parts on portal pages across the server, by web part name"); }; } diff --git a/api/src/org/labkey/filters/ContentSecurityPolicyFilter.java b/api/src/org/labkey/filters/ContentSecurityPolicyFilter.java index 3dc9fa8fd34..2c3df6d96d9 100644 --- a/api/src/org/labkey/filters/ContentSecurityPolicyFilter.java +++ b/api/src/org/labkey/filters/ContentSecurityPolicyFilter.java @@ -424,7 +424,7 @@ public static List getMissingSubstitutions(ContentSecurityPolicyType typ public static void registerMetricsProvider() { - UsageMetricsService.get().registerUsageMetrics("API", () -> Map.of("cspFilters", CSP_FILTERS.values().stream() + UsageMetricsService.get().registerUsageMetrics("API", metrics -> metrics.put("cspFilters", CSP_FILTERS.values().stream() .collect(Collectors.toMap(ContentSecurityPolicyFilter::getType, filter -> { StringExpression expression = filter.ensurePolicyExpression(); @@ -433,7 +433,8 @@ public static void registerMetricsProvider() "csp", filter.getStashedTemplate(), "cspSubstituted", expression.getSource() ); - })) + })), + "Content Security Policy filters by type (Enforce, Report): config version, the policy template, and the policy after substitutions" ) ); } diff --git a/assay/src/org/labkey/assay/AssayModule.java b/assay/src/org/labkey/assay/AssayModule.java index bf087ee5e64..268b607e7d8 100644 --- a/assay/src/org/labkey/assay/AssayModule.java +++ b/assay/src/org/labkey/assay/AssayModule.java @@ -71,6 +71,8 @@ import org.labkey.api.security.roles.RoleManager; import org.labkey.api.settings.OptionalFeatureFlag; import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.AssayMetricArea; +import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.util.ContextListener; import org.labkey.api.util.GUID; @@ -217,6 +219,9 @@ protected void init() @Override protected void startupAfterSpringConfig(ModuleContext moduleContext) { + SimpleMetricsService simpleMetrics = SimpleMetricsService.get(); + simpleMetrics.describeFeatureAreas(AssayMetricArea.values()); + FolderTypeManager.get().registerFolderType(this, new AssayFolderType(this)); PipelineJobService.get().registerTaskFactoryFactory(AssayImportRunTaskType.type, new AssayImportRunTask.FactoryFactory()); diff --git a/assay/src/org/labkey/assay/PlateController.java b/assay/src/org/labkey/assay/PlateController.java index 4ac4da4101e..f76aa411286 100644 --- a/assay/src/org/labkey/assay/PlateController.java +++ b/assay/src/org/labkey/assay/PlateController.java @@ -57,6 +57,7 @@ import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.usageMetrics.AssayMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.ContainerTree; import org.labkey.api.util.FileUtil; @@ -1646,7 +1647,7 @@ public Object execute(WorklistForm form, BindException errors) throws Exception String fullFileName = plateSetSource.getName() + " - " + plateSetDestination.getName(); PlateManager.get().getPlateSetExportFile(fullFileName, exportColumns, plateDataRows, form.getFileType(), getViewContext().getResponse()); - SimpleMetricsService.get().increment(AssayModule.NAME, "plateSet", "exportWorklist"); + SimpleMetricsService.get().increment(AssayMetricArea.PLATE_SET, "exportWorklist"); } catch (Exception e) { @@ -1816,19 +1817,19 @@ public Object execute(PlateExportForm form, BindException errors) throws Excepti if (form.getExportType() == PlateExportType.CSV) { - metricsService.increment(AssayModule.NAME, "plate", "exportCSV"); + metricsService.increment(AssayMetricArea.PLATE, "exportCSV"); fileBytes = PlateManager.get().exportPlateData(getContainer(), getUser(), cf, form.getPlateIds(), TSVWriter.DELIM.COMMA); fileExtension = TSVWriter.DELIM.COMMA.extension; } else if (form.getExportType() == PlateExportType.TSV) { - metricsService.increment(AssayModule.NAME, "plate", "exportTSV"); + metricsService.increment(AssayMetricArea.PLATE, "exportTSV"); fileBytes = PlateManager.get().exportPlateData(getContainer(), getUser(), cf, form.getPlateIds(), TSVWriter.DELIM.TAB); fileExtension = TSVWriter.DELIM.TAB.extension; } else { - metricsService.increment(AssayModule.NAME, "plate", "exportMAP"); + metricsService.increment(AssayMetricArea.PLATE, "exportMAP"); fileBytes = PlateManager.get().exportPlateMaps(getContainer(), getUser(), cf, form.getPlateIds()); fileExtension = "xlsx"; } diff --git a/assay/src/org/labkey/assay/actions/PipelineDataCollectorRedirectAction.java b/assay/src/org/labkey/assay/actions/PipelineDataCollectorRedirectAction.java index f1bb13412c3..10e4a661c8b 100644 --- a/assay/src/org/labkey/assay/actions/PipelineDataCollectorRedirectAction.java +++ b/assay/src/org/labkey/assay/actions/PipelineDataCollectorRedirectAction.java @@ -34,6 +34,7 @@ import org.labkey.api.pipeline.PipelineService; import org.labkey.api.security.RequiresPermission; import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.usageMetrics.AssayMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.DOM; import org.labkey.api.util.NetworkDrive; @@ -46,7 +47,6 @@ import org.labkey.api.view.NotFoundException; import org.labkey.api.view.RedirectException; import org.labkey.api.view.template.PageConfig; -import org.labkey.assay.AssayModule; import org.labkey.vfs.FileLike; import org.labkey.vfs.FileSystemLike; import org.springframework.beans.MutablePropertyValues; @@ -182,7 +182,7 @@ public ModelAndView getView(UploadRedirectForm form, BindException errors) if (form.isAllowCrossRunFileInputs()) url.addParameter("allowCrossRunFileInputs", "true"); - SimpleMetricsService.get().increment(AssayModule.NAME, "AssayDesigner", "ImportFromFile"); + SimpleMetricsService.get().increment(AssayMetricArea.ASSAY_DESIGNER, "ImportFromFile"); throw new RedirectException(url); } diff --git a/assay/src/org/labkey/assay/plate/PlateMetricsProvider.java b/assay/src/org/labkey/assay/plate/PlateMetricsProvider.java index 40577ca2ee9..4db2c869f3c 100644 --- a/assay/src/org/labkey/assay/plate/PlateMetricsProvider.java +++ b/assay/src/org/labkey/assay/plate/PlateMetricsProvider.java @@ -30,6 +30,8 @@ import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.security.User; +import org.labkey.api.usageMetrics.UsageMetricGroup; +import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsProvider; import org.labkey.assay.TsvAssayProvider; import org.labkey.assay.query.AssayDbSchema; @@ -204,9 +206,9 @@ private Long getMetadataFieldsCount() } @Override - public Map getUsageMetrics() + public UsageMetrics addUsageMetrics(UsageMetrics metrics) { - var plateMetrics = new HashMap(); + UsageMetricGroup plateMetrics = metrics.group("plates", "Plate feature usage across all containers"); var schema = AssayDbSchema.getInstance(); var dbSchema = schema.getSchema(); TableInfo plateSetTable = schema.getTableInfoPlateSet(); @@ -228,7 +230,7 @@ public Map getUsageMetrics() plateSets.put("plateSetsWith1To10PlatesCount", plateSetPlatesCountBetween(dbSchema, plateSetTable, plateTable, 0, 11)); plateSets.put("plateSetsWith11to30PlatesCount", plateSetPlatesCountBetween(dbSchema, plateSetTable, plateTable, 10, 31)); plateSets.put("plateSetsWith31to60PlatesCount", plateSetPlatesCountBetween(dbSchema, plateSetTable, plateTable, 30, 61)); - plateMetrics.put("plateSets", plateSets); + plateMetrics.put("plateSets", plateSets, "Plate sets: plateSetCount and archivedPlateSetCount exclude templates; type counts (primary, assay, stand-alone assay) include archived and templates; plate-count stats and buckets cover non-archived, non-template plate sets and count all their plates; maximumPlatesInPlateSet is null if there are none"); } // plates @@ -246,7 +248,7 @@ public Map getUsageMetrics() plates.put("platesWithNegativeControlGroups", plateWellGroupCount(dbSchema, plateTable, wellGroupTable, false, WellGroup.Type.NEGATIVE_CONTROL)); plates.put("platesWithPositiveControlGroups", plateWellGroupCount(dbSchema, plateTable, wellGroupTable, false, WellGroup.Type.POSITIVE_CONTROL)); plates.put("platesWithReplicateGroups", plateWellGroupCount(dbSchema, plateTable, wellGroupTable, false, WellGroup.Type.REPLICATE)); - plateMetrics.put("plates", plates); + plateMetrics.put("plates", plates, "Non-template plates: active and archived counts, then active plates by size (12 to 384 wells) and with each control or replicate well group type; distinctPlatedSamples counts distinct samples in any well, including templates and archived plates"); } // templates @@ -262,7 +264,7 @@ public Map getUsageMetrics() templates.put("templatesWithNegativeControlGroups", plateWellGroupCount(dbSchema, plateTable, wellGroupTable, true, WellGroup.Type.NEGATIVE_CONTROL)); templates.put("templatesWithPositiveControlGroups", plateWellGroupCount(dbSchema, plateTable, wellGroupTable, true, WellGroup.Type.POSITIVE_CONTROL)); templates.put("templatesWithReplicateGroups", plateWellGroupCount(dbSchema, plateTable, wellGroupTable, true, WellGroup.Type.REPLICATE)); - plateMetrics.put("templates", templates); + plateMetrics.put("templates", templates, "Plate templates: active and archived counts, then active templates by size (12 to 384 wells) and with each control or replicate well group type"); } // assays @@ -278,7 +280,7 @@ public Map getUsageMetrics() "assayResultsCount", getPlateBasedAssayResultsCount(plateEnabledProtocols), "domainsWithFilterCriteriaConfigured", new SqlSelector(dbSchema, new SQLFragment("SELECT COUNT(DISTINCT domainId) FROM ").append(filterCriteriaTable, "fc")).getObject(Long.class), "columnsWithFilterCriteria", new SqlSelector(dbSchema, new SQLFragment("SELECT COUNT(DISTINCT propertyId) FROM ").append(filterCriteriaTable, "fc")).getObject(Long.class) - )); + ), "Plate-based assay usage: hits and distinct plate sets with hits; Standard assay designs with plate metadata enabled visible in Biologics projects (Shared designs counted once per project) and their runs and results within those projects; distinct domains and columns with hit selection filter criteria"); } // wells @@ -287,13 +289,13 @@ public Map getUsageMetrics() Map wells = new HashMap<>(); wells.put("wellCount", new SqlSelector(dbSchema, new SQLFragment("SELECT COUNT(*) FROM ").append(wellTable, "w")).getObject(Long.class)); wells.put("wellsWithSamples", new SqlSelector(dbSchema, new SQLFragment("SELECT COUNT(*) FROM ").append(wellTable, "w").unsafeAppend(" WHERE sampleId IS NOT NULL")).getObject(Long.class)); - plateMetrics.put("wells", wells); + plateMetrics.put("wells", wells, "All wells, including those on templates and archived plates, and those with a sample"); } plateMetrics.put("metadata", Map.of( "fieldsCount", getMetadataFieldsCount() - )); + ), "Custom plate metadata fields, summed across Biologics projects"); - return Map.of("plates", plateMetrics); + return metrics; } } diff --git a/core/src/org/labkey/core/CoreController.java b/core/src/org/labkey/core/CoreController.java index d31bfa7a2a9..5c5ce45a773 100644 --- a/core/src/org/labkey/core/CoreController.java +++ b/core/src/org/labkey/core/CoreController.java @@ -2796,14 +2796,13 @@ public ApiResponse execute(ClientSideMetricForm form, BindException errors) String metricName = form.getMetricName(); response.put("featureArea", featureArea); response.put("metricName", metricName); - response.put("count", SimpleMetricsService.get().increment(form.getModuleName(), featureArea, metricName)); + response.put("count", SimpleMetricsService.get().increment(CoreModule.CORE_MODULE_NAME, featureArea, metricName)); return response; } } public static class ClientSideMetricForm { - private String _moduleName = CoreModule.CORE_MODULE_NAME; private String _featureArea; private String _metricName; @@ -2826,16 +2825,6 @@ public void setMetricName(String metricName) { _metricName = metricName; } - - public String getModuleName() - { - return _moduleName; - } - - public void setModuleName(String moduleName) - { - _moduleName = moduleName; - } } @RequiresPermission(ReadPermission.class) diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index 0d35505458f..a29fe8a697f 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -172,7 +172,11 @@ import org.labkey.api.study.Study; import org.labkey.api.study.StudyService; import org.labkey.api.thumbnail.ThumbnailService; +import org.labkey.api.usageMetrics.ApiMetricArea; +import org.labkey.api.usageMetrics.CoreMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.usageMetrics.UsageMetricGroup; +import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.util.ContextListener; import org.labkey.api.util.ExceptionUtil; @@ -261,6 +265,7 @@ import org.labkey.core.login.DbLoginManager; import org.labkey.core.login.LoginAttemptDisableLoginProvider; import org.labkey.core.login.LoginController; +import org.labkey.core.metrics.ClientSideMetricDescriptions; import org.labkey.core.metrics.SimpleMetricsServiceImpl; import org.labkey.core.metrics.WebSocketConnectionManager; import org.labkey.core.metrics.WebSocketMXBean; @@ -1058,6 +1063,12 @@ public void startupAfterSpringConfig(ModuleContext moduleContext) ContextListener.addShutdownListener(ShutdownListener.of("Temp file cleanup", null, this::deleteTempFiles)); SimpleMetricsService.setInstance(new SimpleMetricsServiceImpl()); + ClientSideMetricDescriptions.register(); + SimpleMetricsService.get().describeFeatureAreas(CoreMetricArea.values()); + // API module counters, declared here because the API module starts before this service exists + SimpleMetricsService.get().describeFeatureAreas(ApiMetricArea.values()); + SimpleMetricsService.get().describeFeatureArea("API", "rPackageUsage", "R packages loaded (including dependencies) by server-run R scripts that completed, by package name, once per run; base packages excluded, knitr reports count only knitr and rmarkdown, and ~~packageLimitReached~~ counts runs that loaded over 250 packages"); + SimpleMetricsService.get().describeFeatureArea("API", "pythonPackageUsage", "Python packages imported directly by server-run Python scripts that completed, by package name, once per run; standard library excluded, and ~~packageLimitReached~~ counts runs that imported over 250 packages"); // Export action stats on graceful shutdown ContextListener.addShutdownListener(new ShutdownListener() { @@ -1241,62 +1252,59 @@ public void moduleStartupComplete(ServletContext servletContext) AuthenticationManager.populateSettingsWithStartupProps(); AnalyticsServiceImpl.populateSettingsWithStartupProps(); - UsageMetricsService.get().registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - Map javaInfo = new HashMap<>(); - javaInfo.put("java.vendor", System.getProperty("java.vendor")); - javaInfo.put("java.vm.name", System.getProperty("java.vm.name")); - results.put("javaRuntime", javaInfo); - results.put("distributionFilename", AppProps.getInstance().getDistributionFilename()); - results.put("applicationMenuDisplayMode", LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getApplicationMenuDisplayMode()); - results.put("optionalFeatures", OptionalFeatureService.get().getOptionalFeatureFlags().stream() + UsageMetricsService.get().registerUsageMetrics(getName(), metrics -> { + metrics.group("javaRuntime", "JVM vendor and VM name, keyed by Java system property") + .put("java.vendor", System.getProperty("java.vendor")) + .put("java.vm.name", System.getProperty("java.vm.name")); + metrics.put("distributionFilename", AppProps.getInstance().getDistributionFilename(), "Filename of the distribution the server was installed from; localBuild for developer builds"); + metrics.put("applicationMenuDisplayMode", LookAndFeelProperties.getInstance(ContainerManager.getRoot()).getApplicationMenuDisplayMode(), "Site-level setting for when the application menu is shown"); + metrics.put("optionalFeatures", OptionalFeatureService.get().getOptionalFeatureFlags().stream() .collect(Collectors.groupingBy(optionalFeatureFlag -> optionalFeatureFlag.getType().name().toLowerCase(), Collectors.mapping(flag -> flag, Collectors.toMap(OptionalFeatureFlag::getFlag, OptionalFeatureFlag::isEnabled)) - )) + )), + "Optional feature flags by type (experimental, deprecated, etc.), each mapping flag name to whether it's enabled" ); - results.put("productFeaturesEnabled", ProductRegistry.getProductFeatureSet()); - results.put("analyticsTrackingStatus", AnalyticsServiceImpl.get().getTrackingStatus().toString()); + metrics.put("productFeaturesEnabled", ProductRegistry.getProductFeatureSet(), "Product features enabled by the server's license and installed modules"); + metrics.put("analyticsTrackingStatus", AnalyticsServiceImpl.get().getTrackingStatus().toString(), "Web analytics tracking modes enabled; [] if none"); String labkeyContextPath = AppProps.getInstance().getContextPath(); - results.put("webappContextPath", labkeyContextPath); - results.put("embeddedTomcat", true); + metrics.put("webappContextPath", labkeyContextPath, "Context path LabKey is deployed under; empty when deployed at the root"); + metrics.put("embeddedTomcat", true, "Always true; the server runs in embedded Tomcat"); boolean customLog4JConfig = false; if (ModuleLoader.getServletContext() != null) { customLog4JConfig = Boolean.parseBoolean(ModuleLoader.getServletContext().getInitParameter("org.labkey.customLog4JConfig")); } - results.put("customLog4JConfig", customLog4JConfig); - results.put("runtimeMode", AppProps.getInstance().isDevMode() ? "development" : "production"); + metrics.put("customLog4JConfig", customLog4JConfig, "Whether the server is configured with a custom Log4J configuration"); + metrics.put("runtimeMode", AppProps.getInstance().isDevMode() ? "development" : "production", "development or production mode"); Set deployedApps = new HashSet<>(CoreWarningProvider.collectAllDeployedApps()); deployedApps.remove(labkeyContextPath); if (labkeyContextPath.startsWith("/")) { deployedApps.remove(labkeyContextPath.substring(1)); } - results.put("otherDeployedWebapps", StringUtils.join(deployedApps, ",")); - - // Report the total number of login entries in the audit log - results.put("totalLogins", UserManager.getAuthCount(null, false, false, false)); - results.put("apiKeyLogins", UserManager.getAuthCount(null, false, true, false)); - results.put("sessionTimeout", ModuleLoader.getServletContext().getSessionTimeout()); - results.put("userLimits", new LimitActiveUsersSettings().getMetricsMap()); - results.put("systemUserCount", UserManager.getSystemUserCount()); + metrics.put("otherDeployedWebapps", StringUtils.join(deployedApps, ","), "Comma-separated names of other webapps deployed alongside LabKey"); + + metrics.put("totalLogins", UserManager.getAuthCount(null, false, false, false), "Login events in the audit log; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE); + metrics.put("apiKeyLogins", UserManager.getAuthCount(null, false, true, false), "Login events in the audit log that used an API key; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE); + metrics.put("sessionTimeout", ModuleLoader.getServletContext().getSessionTimeout(), "HTTP session timeout in minutes"); + metrics.put("userLimits", new LimitActiveUsersSettings().getMetricsMap(), "Active user warning and limit settings: whether each is on, its user count threshold, and its message"); + metrics.put("systemUserCount", UserManager.getSystemUserCount(), "Active users marked as system users, which don't count toward user limits"); Calendar cal = new GregorianCalendar(); cal.add(Calendar.DATE, -30); - results.put("uniqueRecentUserCount", UserManager.getAuthCount(cal.getTime(), false, false, true)); - results.put("uniqueRecentNonSystemUserCount", UserManager.getAuthCount(cal.getTime(), true, false, true)); - if (OptionalFeatureService.get().isFeatureEnabled(FEATURE_FLAG_EXTENDED_METRICS)) - { - // Optionally include a list of active users, Issue #53050 - results.put("activeUsers", UserManager.getActiveUsers().stream() + metrics.put("uniqueRecentUserCount", UserManager.getAuthCount(cal.getTime(), false, false, true), "Distinct users who logged in during the last 30 days"); + metrics.put("uniqueRecentNonSystemUserCount", UserManager.getAuthCount(cal.getTime(), true, false, true), "Distinct non-system users who logged in during the last 30 days"); + // Optionally include a list of active users, Issue #53050 + metrics.reportedIf(OptionalFeatureService.get().isFeatureEnabled(FEATURE_FLAG_EXTENDED_METRICS)) + .put("activeUsers", () -> UserManager.getActiveUsers().stream() .filter(u -> !u.isSystem()) .map(User::getEmail) - .toList() + .toList(), + "Emails of active non-system users; only reported when the extended metrics feature flag is on" ); - } - results.put("workbookCount", ContainerManager.getWorkbookCount()); - results.put("archivedFolderCount", ContainerManager.getArchivedContainerCount()); - results.put("databaseSize", CoreSchema.getInstance().getSchema().getScope().getDatabaseSize()); + metrics.put("workbookCount", ContainerManager.getWorkbookCount(), "Workbooks across the server"); + metrics.put("archivedFolderCount", ContainerManager.getArchivedContainerCount(), "Archived folders across the server"); + metrics.put("databaseSize", CoreSchema.getInstance().getSchema().getScope().getDatabaseSize(), "Size of the LabKey database in bytes"); // Exclude temp schema to avoid PG exceptions when tables are appearing/disappearing during execution // Note that they can be non-trivial in size. @@ -1305,25 +1313,23 @@ public void moduleStartupComplete(ServletContext servletContext) sql.unsafeAppend(" WHERE table_schema != 'temp' GROUP BY table_schema"); var schemaSizes = new SqlSelector(CoreSchema.getInstance().getSchema(), sql).getValueMap(); - results.put("databaseSchemaSize", schemaSizes); + metrics.put("databaseSchemaSize", schemaSizes, "Size in bytes of each database schema, including indexes; temp excluded"); - results.put("scriptEngines", LabKeyScriptEngineManager.get().getScriptEngineMetrics()); - results.put("customLabels", CustomLabelService.get().getCustomLabelMetrics()); - Map roleAssignments = new HashMap<>(); + metrics.put("scriptEngines", LabKeyScriptEngineManager.get().getScriptEngineMetrics(), "Configured script engines (R, Python, etc.) keyed by name, with factory and engine class, file extensions, and whether each is external and enabled"); + metrics.put("customLabels", CustomLabelService.get().getCustomLabelMetrics(), "Custom noun labels by label provider: count of relabeled nouns and of projects that override labels"); + UsageMetricGroup roleAssignments = metrics.group("roleAssignments", "Direct assignments of designer roles to users or groups, by role, across all containers"); final String roleCountSql = "SELECT COUNT(*) FROM core.RoleAssignments WHERE userid > 0 AND role = ?"; roleAssignments.put("assayDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.assay.security.AssayDesignerRole").getObject(Long.class)); roleAssignments.put("dataClassDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.DataClassDesignerRole").getObject(Long.class)); roleAssignments.put("sampleTypeDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.SampleTypeDesignerRole").getObject(Long.class)); - results.put("roleAssignments", roleAssignments); - Map allowListCounts = new HashMap<>(); + UsageMetricGroup allowListCounts = metrics.group("allowListCounts", "Entries in each site allow list, such as external redirect hosts and file extensions"); for (AllowListType type : AllowListType.values()) { allowListCounts.put(type.name(), type.getValues().size()); } - results.put("allowListCounts", allowListCounts); - return results; + return metrics; }); UsageMetricsService.get().registerUsageMetrics(getName(), WebSocketConnectionManager.getInstance()); diff --git a/core/src/org/labkey/core/admin/DisplayFormatAnalyzer.java b/core/src/org/labkey/core/admin/DisplayFormatAnalyzer.java index 0b8cc7ee0b7..2201a411e34 100644 --- a/core/src/org/labkey/core/admin/DisplayFormatAnalyzer.java +++ b/core/src/org/labkey/core/admin/DisplayFormatAnalyzer.java @@ -45,7 +45,6 @@ import java.util.Arrays; import java.util.HashSet; -import java.util.Map; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; @@ -221,10 +220,10 @@ public static UsageMetricsProvider getMetricsProvider() { // Collect the unique set of non-standard date display formats across the entire site Set badFormats = new HashSet<>(); - return () -> { + return metrics -> { DisplayFormatAnalyzer analyzer = new DisplayFormatAnalyzer(); analyzer.handleAll(User.getAdminServiceUser(), (c, type, format, contextProvider) -> badFormats.add(format)); - return Map.of("nonStandardDateDisplayFormats", badFormats); + return metrics.put("nonStandardDateDisplayFormats", badFormats, "Distinct non-standard date display formats used in folder defaults, query metadata, or field properties anywhere on the server"); }; } diff --git a/core/src/org/labkey/core/admin/usageMetrics/UsageMetricsServiceImpl.java b/core/src/org/labkey/core/admin/usageMetrics/UsageMetricsServiceImpl.java index 1e726cf7a8d..b98891e1286 100644 --- a/core/src/org/labkey/core/admin/usageMetrics/UsageMetricsServiceImpl.java +++ b/core/src/org/labkey/core/admin/usageMetrics/UsageMetricsServiceImpl.java @@ -21,7 +21,6 @@ import org.labkey.api.collections.CopyOnWriteHashMap; import org.labkey.api.module.Module; import org.labkey.api.module.ModuleLoader; -import org.labkey.api.usageMetrics.DescribedUsageMetricsProvider; import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsProvider; import org.labkey.api.usageMetrics.UsageMetricsService; @@ -102,10 +101,7 @@ public Map> getModuleUsageMetricDescripti { try { - if (provider instanceof DescribedUsageMetricsProvider described) - moduleDescriptions.addAll(described.collectUsageMetrics().getDescriptions()); - else - provider.getUsageMetrics().keySet().forEach(key -> moduleDescriptions.add(new UsageMetrics.Description(key, null, null))); + moduleDescriptions.addAll(provider.collectUsageMetrics().getDescriptions()); } catch (Exception e) { diff --git a/core/src/org/labkey/core/login/DbLoginManager.java b/core/src/org/labkey/core/login/DbLoginManager.java index c5cb453f599..29dca971ac4 100644 --- a/core/src/org/labkey/core/login/DbLoginManager.java +++ b/core/src/org/labkey/core/login/DbLoginManager.java @@ -222,6 +222,6 @@ public static void saveProperties(User user, SaveDbLoginPropertiesForm form) public static UsageMetricsProvider getMetricsProvider() { - return () -> Map.of("databaseAuthentication", getProperties()); + return metrics -> metrics.put("databaseAuthentication", getProperties(), "Database authentication settings, such as password strength and expiration; empty if never saved"); } } diff --git a/core/src/org/labkey/core/metrics/ClientSideMetricDescriptions.java b/core/src/org/labkey/core/metrics/ClientSideMetricDescriptions.java new file mode 100644 index 00000000000..3073f5958a4 --- /dev/null +++ b/core/src/org/labkey/core/metrics/ClientSideMetricDescriptions.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.core.metrics; + +import org.labkey.api.usageMetrics.SimpleMetricsService; + +/** Describes the client-side feature areas that @labkey/components counts */ +public final class ClientSideMetricDescriptions +{ + private ClientSideMetricDescriptions() + { + } + + public static void register() + { + describe("pagination", "Grid paging clicks: loadFirstPage, loadPreviousPage, loadNextPage, loadLastPage, setPageSize, showTotalRowCount"); + describe("grid", "Grid row selection using shift-click (shiftSelect)"); + describe("customViews", "Grid columns reordered by drag and drop (columnReorderDragNDrop)"); + describe("export", "Grid exports by format and header type: csv-caption, excel-caption, tsv-caption, csv-importfield, excel-importfield, tsv-importfield"); + describe("BarTender", "BarTender label template downloads (DownloadTemplateCount)"); + describe("QueryModel", "Grid loads recovered from a failure: ViewDoesNotExist (fell back to the default view), CalculatedFieldError (calculated fields omitted)"); + describe("expressionAssistant", "Calculated field expression assistant use: clickButton, submitPrompt, applyExpression, clickHelpWithValidate"); + describe("applicationNavigation", "Product menu navigation: toHomeProject, toServerContainer, toServerContainerTab, and toLimsSection, toSampleManagerSection, toBiologicsSection, toApplicationSection"); + describe("releaseNotes", "Release notes opened, by entry point: FromHelpMenu, FromBanner"); + describe("helpLink", "Help link clicks from the user menu (clickCount)"); + describe("SampleManagerSearch", "Sample Manager searches (count) and result clicks (resultClicked, plus firstResultClicked for the top result)"); + describe("BiologicsSearch", "Biologics searches (count) and result clicks (resultClicked, plus firstResultClicked for the top result)"); + describe("LIMSSearch", "LIMS searches (count) and result clicks (resultClicked, plus firstResultClicked for the top result)"); + describe("gridInsertCounts", "Rows inserted from editable grids and assay imports, by data type and row-count bucket: 1To50, 51To100, 101To250, GT250"); + describe("gridUpdateCounts", "Rows updated from editable grids, by data type and row-count bucket: 1To50, 51To100, 101To250, GT250"); + describe("findSamplesById", "Find Samples by ID or barcode: searches submitted (findCount), plus actions taken from its results grid (e.g. addSamplesToPicklist, sampleFinderFindDerivatives)"); + describe("sampleFinder", "Sample Finder use: headerMenuNavigation, dashboardButtonNavigation, filterModalApply, filterModalError, goToChooseValuesTab, hasWithoutAssayResultChecked, assay design counts (with AssayDesgin(s)), plus actions taken from its results grid (e.g. addSamplesToPicklist)"); + } + + private static void describe(String featureArea, String description) + { + SimpleMetricsService.get().describeClientSideFeatureArea(featureArea, description); + } +} diff --git a/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java b/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java index cc4e421ac39..5892adc1ee9 100644 --- a/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java +++ b/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java @@ -24,6 +24,8 @@ import org.labkey.api.module.ModuleLoader; import org.labkey.api.settings.AppProps; import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.usageMetrics.UsageMetricGroup; +import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsProvider; import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.util.ContextListener; @@ -31,7 +33,6 @@ import org.labkey.api.util.ShutdownListener; import org.labkey.api.util.logging.LogHelper; -import java.util.Collections; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.RejectedExecutionException; @@ -55,6 +56,9 @@ public class SimpleMetricsServiceImpl implements SimpleMetricsService */ private final Map>> _counts = new ConcurrentHashMap<>(); + /** Feature area descriptions by module name, then feature area */ + private final Map> _featureAreaDescriptions = new ConcurrentHashMap<>(); + /** Timestamp when we last saved the counts to the DB */ private Runnable _saver; private boolean _shutdownStarted = false; @@ -91,9 +95,17 @@ public SimpleMetricsProvider(String moduleName) } @Override - public Map getUsageMetrics() + public UsageMetrics addUsageMetrics(UsageMetrics metrics) { - return Collections.singletonMap("simpleMetricCounts", _counts.get(_moduleName)); + String groupDescription = "Counts of feature usage events by feature area, then by event name, as recorded by the module's code; persisted across restarts"; + Map> counts = _counts.get(_moduleName); + // A module whose feature areas were described but never counted reports nothing + UsageMetricGroup group = counts.isEmpty() + ? metrics.describeGroup("simpleMetricCounts", groupDescription, UsageMetrics.Kind.CUMULATIVE) + : metrics.group("simpleMetricCounts", groupDescription, UsageMetrics.Kind.CUMULATIVE).putAll(counts); + _featureAreaDescriptions.getOrDefault(_moduleName, Map.of()) + .forEach((featureArea, description) -> group.describe(featureArea, description, UsageMetrics.Kind.CUMULATIVE)); + return metrics; } } @@ -154,6 +166,32 @@ private Map> getModuleMetrics(Module module) }); } + private @NotNull Module getModule(String requestedModuleName) + { + Module module = ModuleLoader.getInstance().getModule(requestedModuleName); + if (null == module) + { + throw new IllegalArgumentException("Unknown module: " + requestedModuleName); + } + return module; + } + + @Override + public void describeFeatureArea(@NotNull String requestedModuleName, @NotNull String featureArea, @NotNull String description) + { + // Checked here because the provider would only fail later, when reporting + if (featureArea.contains(".")) + { + throw new IllegalArgumentException("Described feature area names can't contain '.': " + featureArea); + } + + Module module = getModule(requestedModuleName); + getModuleMetrics(module); // Registers the provider so the description is reported before anything is counted + String existing = _featureAreaDescriptions.computeIfAbsent(module.getName(), k -> new ConcurrentHashMap<>()).putIfAbsent(featureArea, description); + if (existing != null && !existing.equals(description)) + throw new IllegalStateException("Feature area " + module.getName() + "." + featureArea + " is described inconsistently: \"" + existing + "\" vs. \"" + description + "\""); + } + @Override public long increment(@NotNull String requestedModuleName, @NotNull String featureArea, @NotNull String metricName) { @@ -162,11 +200,7 @@ public long increment(@NotNull String requestedModuleName, @NotNull String featu throw new IllegalArgumentException("Feature area names cannot contain commas"); } - Module module = ModuleLoader.getInstance().getModule(requestedModuleName); - if (null == module) - { - throw new IllegalArgumentException("Unknown module: " + requestedModuleName); - } + Module module = getModule(requestedModuleName); String scoping = getScoping(module, featureArea); Map> moduleMetrics = getModuleMetrics(module); diff --git a/core/src/org/labkey/core/metrics/WebSocketConnectionManager.java b/core/src/org/labkey/core/metrics/WebSocketConnectionManager.java index c79d2c26aba..744913663c7 100644 --- a/core/src/org/labkey/core/metrics/WebSocketConnectionManager.java +++ b/core/src/org/labkey/core/metrics/WebSocketConnectionManager.java @@ -15,6 +15,7 @@ */ package org.labkey.core.metrics; +import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsProvider; import org.labkey.api.websocket.WebSocketTracker; @@ -69,14 +70,14 @@ public boolean showWarning() } @Override - public Map getUsageMetrics() + public UsageMetrics addUsageMetrics(UsageMetrics metrics) { - return Map.of( - "webSocketConnections", Map.of( - "success", successCounter.get(), - "failure", failureCounter.get(), - "open", WebSocketTracker.getOpenCount() - ) + return metrics.put("webSocketConnections", Map.of( + "success", successCounter.get(), + "failure", failureCounter.get(), + "open", WebSocketTracker.getOpenCount() + ), + "WebSocket connections: successful and failed connection attempts since server startup, and connections currently open" ); } } \ No newline at end of file diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 7f62341226c..d4d076fdb71 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -94,6 +94,8 @@ import org.labkey.api.security.roles.RoleManager; import org.labkey.api.settings.AppProps; import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.ExperimentMetricArea; +import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.usageMetrics.UsageMetricGroup; import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsService; @@ -198,6 +200,8 @@ public class ExperimentModule extends SpringModule private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; + // Usage metric descriptions shared by the branches that report a key and the ones that only describe it + public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; @@ -506,6 +510,9 @@ public Map> getCustomSearchJsonMap(User user, @NotNu @Override protected void startupAfterSpringConfig(ModuleContext moduleContext) { + SimpleMetricsService simpleMetrics = SimpleMetricsService.get(); + simpleMetrics.describeFeatureAreas(ExperimentMetricArea.values()); + SearchService ss = SearchService.get(); // ss.addSearchCategory(OntologyManager.conceptCategory); ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); @@ -764,22 +771,19 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM UsageMetricGroup sampleColorMetrics = metrics.group("sampleColors", "Sample color usage; only colorCount is reported when no colors exist"); Long colorCount = new SqlSelector(schema, "SELECT COUNT(*) FROM exp.datacolors").getObject(Long.class); sampleColorMetrics.put("colorCount", colorCount, "Sample colors defined, archived included"); - if (colorCount > 0) - { - Long archivedColorCount = new SqlSelector(schema, new SQLFragment("SELECT COUNT(*) FROM exp.datacolors WHERE archived = ?", true)).getObject(Long.class); - sampleColorMetrics.put("archivedColorCount", archivedColorCount, "Archived sample colors"); - sampleColorMetrics.put("samplesWithColorCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE expmaterialcolor IS NOT NULL").getObject(Long.class), "Samples with a color assigned"); - sampleColorMetrics.put("samplesWithArchivedColorCount", new SqlSelector(schema, new SQLFragment("SELECT COUNT(*) FROM exp.material m JOIN exp.datacolors dc ON m.expmaterialcolor = dc.rowid WHERE dc.archived = ?", true)).getObject(Long.class), "Samples assigned a color that is now archived"); - - sampleColorMetrics.put("sampleTypesWithColorsEnabledCount", new SqlSelector(schema, new SQLFragment( - "SELECT COUNT(*) FROM exp.materialsource ms WHERE EXISTS (" + - "SELECT 1 FROM exp.datacolors dc WHERE dc.container = ms.container AND dc.archived = ? AND NOT EXISTS (" + - "SELECT 1 FROM exp.datatypecolorexclusion e WHERE e.datatype = ? AND e.datatyperowid = ms.rowid AND e.colorrowid = dc.rowid))", - false, ExperimentService.DataTypeForExclusion.SampleType.name())).getObject(Long.class), "Sample types whose folder has at least one active color not excluded for the type"); - sampleColorMetrics.put("sampleTypesWithColorsDisabledCount", new SqlSelector(schema, new SQLFragment( - "SELECT COUNT(DISTINCT datatyperowid) FROM exp.datatypecolorexclusion WHERE datatype = ?") - .add(ExperimentService.DataTypeForExclusion.SampleType.name())).getObject(Long.class), "Sample types with at least one color excluded, not necessarily all"); - } + UsageMetrics colorDetailMetrics = sampleColorMetrics.reportedIf(colorCount > 0); + colorDetailMetrics.put("archivedColorCount", () -> new SqlSelector(schema, new SQLFragment("SELECT COUNT(*) FROM exp.datacolors WHERE archived = ?", true)).getObject(Long.class), "Archived sample colors"); + colorDetailMetrics.put("samplesWithColorCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE expmaterialcolor IS NOT NULL").getObject(Long.class), "Samples with a color assigned"); + colorDetailMetrics.put("samplesWithArchivedColorCount", () -> new SqlSelector(schema, new SQLFragment("SELECT COUNT(*) FROM exp.material m JOIN exp.datacolors dc ON m.expmaterialcolor = dc.rowid WHERE dc.archived = ?", true)).getObject(Long.class), "Samples assigned a color that is now archived"); + + colorDetailMetrics.put("sampleTypesWithColorsEnabledCount", () -> new SqlSelector(schema, new SQLFragment( + "SELECT COUNT(*) FROM exp.materialsource ms WHERE EXISTS (" + + "SELECT 1 FROM exp.datacolors dc WHERE dc.container = ms.container AND dc.archived = ? AND NOT EXISTS (" + + "SELECT 1 FROM exp.datatypecolorexclusion e WHERE e.datatype = ? AND e.datatyperowid = ms.rowid AND e.colorrowid = dc.rowid))", + false, ExperimentService.DataTypeForExclusion.SampleType.name())).getObject(Long.class), "Sample types whose folder has at least one active color not excluded for the type"); + colorDetailMetrics.put("sampleTypesWithColorsDisabledCount", () -> new SqlSelector(schema, new SQLFragment( + "SELECT COUNT(DISTINCT datatyperowid) FROM exp.datatypecolorexclusion WHERE datatype = ?") + .add(ExperimentService.DataTypeForExclusion.SampleType.name())).getObject(Long.class), "Sample types with at least one color excluded, not necessarily all"); metrics.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class), "Names used by more than one sample within a sample type; counts names, not samples"); diff --git a/experiment/src/org/labkey/experiment/FileLinkMetricsProvider.java b/experiment/src/org/labkey/experiment/FileLinkMetricsProvider.java index 1c15e22a795..bb51953c5c8 100644 --- a/experiment/src/org/labkey/experiment/FileLinkMetricsProvider.java +++ b/experiment/src/org/labkey/experiment/FileLinkMetricsProvider.java @@ -15,12 +15,12 @@ */ package org.labkey.experiment; -import org.labkey.api.usageMetrics.DescribedUsageMetricsProvider; import org.labkey.api.usageMetrics.UsageMetrics; +import org.labkey.api.usageMetrics.UsageMetricsProvider; import java.util.Map; -public class FileLinkMetricsProvider implements DescribedUsageMetricsProvider +public class FileLinkMetricsProvider implements UsageMetricsProvider { private final static FileLinkMetricsProvider _instance = new FileLinkMetricsProvider(); private volatile Map _metrics = Map.of("Run time", "Not run yet."); diff --git a/experiment/src/org/labkey/experiment/SpecialCharacterMetricsProvider.java b/experiment/src/org/labkey/experiment/SpecialCharacterMetricsProvider.java index fb3f1af9ada..13fe84b9441 100644 --- a/experiment/src/org/labkey/experiment/SpecialCharacterMetricsProvider.java +++ b/experiment/src/org/labkey/experiment/SpecialCharacterMetricsProvider.java @@ -15,12 +15,12 @@ */ package org.labkey.experiment; -import org.labkey.api.usageMetrics.DescribedUsageMetricsProvider; import org.labkey.api.usageMetrics.UsageMetrics; +import org.labkey.api.usageMetrics.UsageMetricsProvider; import java.util.Map; -public class SpecialCharacterMetricsProvider implements DescribedUsageMetricsProvider +public class SpecialCharacterMetricsProvider implements UsageMetricsProvider { public static final String METRIC_KEY = "specialCharacterFields"; diff --git a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java index 293c459df08..f6c19969997 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataClassDataTableImpl.java @@ -114,6 +114,7 @@ import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.study.assay.FileLinkDisplayColumn; +import org.labkey.api.usageMetrics.ExperimentMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.CachingSupplier; import org.labkey.api.util.PageFlowUtil; @@ -1275,7 +1276,7 @@ public Map moveRows(User user, Container container, Container ta } } - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "sources"); + SimpleMetricsService.get().increment(ExperimentMetricArea.MOVE_ENTITIES, "sources"); } return new HashMap<>(allContainerResponse); } diff --git a/experiment/src/org/labkey/experiment/api/ExpRunTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpRunTableImpl.java index 527d9380549..55b8adc73ea 100644 --- a/experiment/src/org/labkey/experiment/api/ExpRunTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpRunTableImpl.java @@ -92,6 +92,7 @@ import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.settings.AppProps; import org.labkey.api.settings.ProductFeature; +import org.labkey.api.usageMetrics.ExperimentMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.HtmlString; import org.labkey.api.util.LinkBuilder; @@ -1180,7 +1181,7 @@ public Map moveRows(User user, Container container, Container ta throw new BatchValidationException(new ValidationException(e.getMessage())); } } - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "assayRuns"); + SimpleMetricsService.get().increment(ExperimentMetricArea.MOVE_ENTITIES, "assayRuns"); } return new HashMap<>(allContainerResponse); diff --git a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java index c0b2131323c..6e56ac2592c 100644 --- a/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java +++ b/experiment/src/org/labkey/experiment/api/SampleTypeUpdateServiceDI.java @@ -107,6 +107,7 @@ import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.roles.ReaderRole; import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.usageMetrics.ExperimentMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.GUID; import org.labkey.api.util.JobRunner; @@ -635,7 +636,7 @@ public Map moveRows(User user, Container container, Container ta } } - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "samples"); + SimpleMetricsService.get().increment(ExperimentMetricArea.MOVE_ENTITIES, "samples"); } return new HashMap<>(allContainerResponse); } diff --git a/experiment/src/org/labkey/experiment/api/property/PropertyServiceImpl.java b/experiment/src/org/labkey/experiment/api/property/PropertyServiceImpl.java index 69b587065f5..2b50f3484be 100644 --- a/experiment/src/org/labkey/experiment/api/property/PropertyServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/PropertyServiceImpl.java @@ -79,8 +79,8 @@ import org.labkey.api.query.QueryService; import org.labkey.api.query.UserSchema; import org.labkey.api.security.User; -import org.labkey.api.usageMetrics.DescribedUsageMetricsProvider; import org.labkey.api.usageMetrics.UsageMetrics; +import org.labkey.api.usageMetrics.UsageMetricsProvider; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Pair; import org.labkey.api.util.URIUtil; @@ -106,7 +106,7 @@ import static org.labkey.api.exp.property.DefaultPropertyValidator.createValidatorURI; -public class PropertyServiceImpl implements PropertyService, DescribedUsageMetricsProvider +public class PropertyServiceImpl implements PropertyService, UsageMetricsProvider { private final List> _domainTypes = new CopyOnWriteArrayList<>(); private final Map _validatorTypes = new ConcurrentHashMap<>(); diff --git a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java index 10bc1c7caeb..0ad11b2e8bb 100644 --- a/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java +++ b/experiment/src/org/labkey/experiment/controllers/exp/ExperimentController.java @@ -214,6 +214,7 @@ import org.labkey.api.study.StudyService; import org.labkey.api.study.StudyUrls; import org.labkey.api.study.publish.StudyPublishService; +import org.labkey.api.usageMetrics.ExperimentMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.CsrfInput; import org.labkey.api.util.DOM; @@ -2748,7 +2749,7 @@ public void export(ConvertArraysToExcelForm form, HttpServletResponse response, qInfo.getString("query"), getViewContext().getActionURL(), rootObject.getString("auditMessage") + filename, null); - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "convertTable", "asExcel"); + SimpleMetricsService.get().increment(ExperimentMetricArea.CONVERT_TABLE, "asExcel"); } } } @@ -2814,7 +2815,7 @@ public void export(ConvertArraysToExcelForm form, HttpServletResponse response, getViewContext().getActionURL(), rootObject.getString("auditMessage") + filename, rowsArray.length()); - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "convertTable", "asDelimited"); + SimpleMetricsService.get().increment(ExperimentMetricArea.CONVERT_TABLE, "asDelimited"); } } catch (JSONException e) @@ -4447,11 +4448,11 @@ protected int importData( if (getOptionParamValue(Params.crossTypeImport)) { - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeImport"); + SimpleMetricsService.get().increment(ExperimentMetricArea.SAMPLE_IMPORT, "crossTypeImport"); if (_context.getInsertOption() == QueryUpdateService.InsertOption.UPDATE) - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeUpdate"); + SimpleMetricsService.get().increment(ExperimentMetricArea.SAMPLE_IMPORT, "crossTypeUpdate"); else if (_context.getInsertOption() == QueryUpdateService.InsertOption.MERGE) - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "sampleImport", "crossTypeMerge"); + SimpleMetricsService.get().increment(ExperimentMetricArea.SAMPLE_IMPORT, "crossTypeMerge"); } return count; diff --git a/filecontent/src/org/labkey/filecontent/FileContentMetricArea.java b/filecontent/src/org/labkey/filecontent/FileContentMetricArea.java new file mode 100644 index 00000000000..bdbb9855272 --- /dev/null +++ b/filecontent/src/org/labkey/filecontent/FileContentMetricArea.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.filecontent; + +import org.labkey.api.usageMetrics.SimpleMetricArea; + +public enum FileContentMetricArea implements SimpleMetricArea +{ + NAMED_FILE_SETS("namedFileSets", "Uses of named file sets: legacySendFile for downloads via the sendFile action, webdavResolve for user WebDAV lookups of a file set (search crawler excluded)"); + + private final Definition _definition; + + FileContentMetricArea(String areaName, String description) + { + _definition = new Definition(FileContentModule.NAME, areaName, description); + } + + @Override + public Definition getDefinition() + { + return _definition; + } +} diff --git a/filecontent/src/org/labkey/filecontent/FileContentModule.java b/filecontent/src/org/labkey/filecontent/FileContentModule.java index 045f635b4b8..43f39513c08 100644 --- a/filecontent/src/org/labkey/filecontent/FileContentModule.java +++ b/filecontent/src/org/labkey/filecontent/FileContentModule.java @@ -58,7 +58,6 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -77,7 +76,7 @@ public String getName() static void incrementNamedFileSetMetric(String metricName) { - SimpleMetricsService.get().increment(NAME, "namedFileSets", metricName); + SimpleMetricsService.get().increment(FileContentMetricArea.NAMED_FILE_SETS, metricName); } private static boolean usesNamedFileSet(@Nullable String webPartProperties) @@ -190,6 +189,8 @@ public Collection getSummary(Container c) @Override public void doStartup(ModuleContext moduleContext) { + SimpleMetricsService.get().describeFeatureAreas(FileContentMetricArea.values()); + WebdavService.get().addProvider(new FileWebdavProvider()); // initialize message digests @@ -211,19 +212,17 @@ public void doStartup(ModuleContext moduleContext) AuditLogService.get().registerAuditType(new FileSystemBatchAuditProvider()); - UsageMetricsService.get().registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - + UsageMetricsService.get().registerUsageMetrics(getName(), metrics -> { // During system maintenance, FileRootMaintenanceTask populates FileRootSize and FileRootLastCrawled for every container (subject to a timeout) TableInfo containers = CoreSchema.getInstance().getTableInfoContainers(); String select = "SELECT SUM(FileRootSize) AS TotalSize, MIN(FileRootLastCrawled) EarliestCrawl, COUNT(*) AS AllFileRoots, COUNT(FileRootLastCrawled) AS FileRootsCrawled FROM "; Map map = new SqlSelector(containers.getSchema(), select + containers.getSelectName()) .getMap(); - results.put("fileRootsTotalSize", map.get("TotalSize")); - results.put("fileRootsEarliestCrawlTime", map.get("EarliestCrawl")); + metrics.put("fileRootsTotalSize", map.get("TotalSize"), "Sum of all containers' file root sizes in bytes, as of each one's last system maintenance crawl; null if none crawled"); + metrics.put("fileRootsEarliestCrawlTime", map.get("EarliestCrawl"), "Oldest last-crawled time across containers' file roots, showing how stale fileRootsTotalSize may be; null if none crawled"); long crawled = ((Number)map.get("FileRootsCrawled")).longValue(); - results.put("fileRootsCrawled", crawled); - results.put("fileRootsNotYetCrawled", ((Number)map.get("AllFileRoots")).longValue() - crawled); + metrics.put("fileRootsCrawled", crawled, "Containers whose file root has been crawled for size at least once, including invalid or missing roots"); + metrics.put("fileRootsNotYetCrawled", ((Number)map.get("AllFileRoots")).longValue() - crawled, "Containers whose file root has never been crawled for size"); // findPipelineRoot() is non-null only for a valid pipeline override on the folder or an ancestor, since a disabled file root has no default Map disabledFileRoots = new TableSelector(FileRootManager.getTinfoFileRoots(), PageFlowUtil.set("Container"),new SimpleFilter(FieldKey.fromParts("Enabled"), false), null) @@ -231,23 +230,23 @@ public void doStartup(ModuleContext moduleContext) .map(ContainerManager::getForId) .filter(c -> c != null && !c.isRoot()) .collect(Collectors.partitioningBy(c -> PipelineService.get().findPipelineRoot(c) != null, Collectors.counting())); - results.put("disabledFileRootsWithPipelineOverride", disabledFileRoots.get(true)); - results.put("disabledFileRootsWithoutPipelineOverride", disabledFileRoots.get(false)); + metrics.put("disabledFileRootsWithPipelineOverride", disabledFileRoots.get(true), "Folders with a disabled file root that still have a valid pipeline root override on the folder or an ancestor"); + metrics.put("disabledFileRootsWithoutPipelineOverride", disabledFileRoots.get(false), "Folders with a disabled file root and no valid pipeline root override"); TableInfo mappedDirectories = CoreSchema.getInstance().getMappedDirectories(); Map fileSets = new SqlSelector(mappedDirectories.getSchema(), new SQLFragment("SELECT COUNT(*) AS NamedFileSets, COUNT(DISTINCT Container) AS NamedFileSetContainers FROM ").append(mappedDirectories, "md")) .getMap(); - results.put("namedFileSets", fileSets.get("NamedFileSets")); - results.put("namedFileSetContainers", fileSets.get("NamedFileSetContainers")); + metrics.put("namedFileSets", fileSets.get("NamedFileSets"), "Named file sets across all containers"); + metrics.put("namedFileSetContainers", fileSets.get("NamedFileSetContainers"), "Distinct containers that define at least one named file set"); try (Stream webPartProperties = new TableSelector(Portal.getTableInfoPortalWebParts(), PageFlowUtil.set("Properties"), new SimpleFilter(FieldKey.fromParts("Name"), FilesWebPart.PART_NAME), null) .uncachedStream(String.class)) { - results.put("filesWebPartsUsingFileSet", webPartProperties.filter(FileContentModule::usesNamedFileSet).count()); + metrics.put("filesWebPartsUsingFileSet", webPartProperties.filter(FileContentModule::usesNamedFileSet).count(), "Files web parts across the server configured to show a named file set"); } - return results; + return metrics; }); SystemMaintenance.addTask(new FileRootMaintenanceTask()); diff --git a/issues/src/org/labkey/issue/IssuesModule.java b/issues/src/org/labkey/issue/IssuesModule.java index 10f8622699a..f7aad8c302a 100644 --- a/issues/src/org/labkey/issue/IssuesModule.java +++ b/issues/src/org/labkey/issue/IssuesModule.java @@ -63,10 +63,8 @@ import java.util.Collection; import java.util.Collections; import java.util.Date; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.Set; import static org.labkey.api.issues.IssuesSchema.ISSUE_DEF_SCHEMA_NAME; @@ -153,13 +151,11 @@ public void doStartup(ModuleContext moduleContext) UsageMetricsService svc = UsageMetricsService.get(); if (svc != null) { - svc.registerUsageMetrics(getName(), () -> { - Map metric = new HashMap<>(); + svc.registerUsageMetrics(getName(), metrics -> { + metrics.put("issueDefCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issueListDef").getObject(Long.class), "Issue list definitions across all containers"); + metrics.put("issuesCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issues").getObject(Long.class), "Issues across all containers and issue lists, open and closed"); - metric.put("issueDefCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issueListDef").getObject(Long.class)); - metric.put("issuesCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issues").getObject(Long.class)); - - return metric; + return metrics; }); } } diff --git a/list/src/org/labkey/list/ListModule.java b/list/src/org/labkey/list/ListModule.java index 30394938f95..c7ae728cecb 100644 --- a/list/src/org/labkey/list/ListModule.java +++ b/list/src/org/labkey/list/ListModule.java @@ -70,9 +70,7 @@ import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Set; public class ListModule extends SpringModule @@ -155,13 +153,12 @@ public void startupAfterSpringConfig(ModuleContext moduleContext) UsageMetricsService svc = UsageMetricsService.get(); if (null != svc) { - svc.registerUsageMetrics(getName(), () -> { - Map metric = new HashMap<>(); + svc.registerUsageMetrics(getName(), metrics -> { DbSchema dbSchema = DbSchema.get("exp", DbSchemaType.Module); - metric.put("listCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list").getObject(Long.class)); - metric.put("publicPicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PublicPicklist + "'").getObject(Long.class)); - metric.put("privatePicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PrivatePicklist + "'").getObject(Long.class)); - return metric; + metrics.put("listCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list").getObject(Long.class), "List definitions across all containers, including picklists"); + metrics.put("publicPicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PublicPicklist + "'").getObject(Long.class), "Shared (public) picklists across all containers"); + metrics.put("privatePicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PrivatePicklist + "'").getObject(Long.class), "Private picklists across all containers"); + return metrics; }); } } diff --git a/list/src/org/labkey/list/model/ListQueryUpdateService.java b/list/src/org/labkey/list/model/ListQueryUpdateService.java index 0dbdbfab634..026865dc0f6 100644 --- a/list/src/org/labkey/list/model/ListQueryUpdateService.java +++ b/list/src/org/labkey/list/model/ListQueryUpdateService.java @@ -51,7 +51,6 @@ import org.labkey.api.dataiterator.ImportProgress; import org.labkey.api.exp.ObjectProperty; import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.exp.list.ListDefinition; import org.labkey.api.exp.list.ListItem; import org.labkey.api.exp.list.ListService; @@ -81,6 +80,7 @@ import org.labkey.api.security.permissions.UpdatePermission; import org.labkey.api.security.roles.EditorRole; import org.labkey.api.test.TestTimeout; +import org.labkey.api.usageMetrics.ExperimentMetricArea; import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.GUID; import org.labkey.api.util.PageFlowUtil; @@ -760,7 +760,7 @@ public Map moveRows( tx.commit(); - SimpleMetricsService.get().increment(ExperimentService.MODULE_NAME, "moveEntities", "list"); + SimpleMetricsService.get().increment(ExperimentMetricArea.MOVE_ENTITIES, "list"); } return moveRowsCounts(fileAttachmentsMovedCount, listAuditEventsCreatedCount, listAuditEventsMovedCount, listRecordsCount, queryAuditEventsMovedCount); diff --git a/pipeline/src/org/labkey/pipeline/PipelineModule.java b/pipeline/src/org/labkey/pipeline/PipelineModule.java index 63968a631d2..cba565e5278 100644 --- a/pipeline/src/org/labkey/pipeline/PipelineModule.java +++ b/pipeline/src/org/labkey/pipeline/PipelineModule.java @@ -225,10 +225,9 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) AuditLogService.get().registerAuditType(new ProtocolManagementAuditProvider()); - UsageMetricsService.get().registerUsageMetrics(getName(), () -> { + UsageMetricsService.get().registerUsageMetrics(getName(), metrics -> { DbSchema pipelineSchema = PipelineSchema.getInstance().getSchema(); SqlDialect dialect = PipelineSchema.getInstance().getSchema().getSqlDialect(); - Map result = new HashMap<>(); Map jobCounts = new HashMap<>(); SQLFragment jobSQL = new SQLFragment("SELECT COUNT(*) AS JobCount, COALESCE(Provider, 'Unknown') AS Provider FROM "); @@ -237,7 +236,7 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) new SqlSelector(pipelineSchema, jobSQL).forEach(rs -> jobCounts.put(rs.getString("Provider"), rs.getLong("JobCount"))); - result.put("jobCounts", jobCounts); + metrics.put("jobCounts", jobCounts, "Pipeline jobs in the job status table across all containers, by provider; Unknown if no provider; jobs whose status rows were deleted aren't counted"); Map> triggerCounts = new HashMap<>(); SQLFragment triggerSQL = new SQLFragment("SELECT SUM(CASE WHEN Enabled = ").appendValue(true,dialect).unsafeAppend(" THEN 1 ELSE 0 END) AS Enabled, "); @@ -261,13 +260,13 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) m.put("Disabled", rs.getLong("Disabled")); m.put("Jobs", rs.getLong("Jobs")); }); - result.put("triggerCounts", triggerCounts); - result.put("jmsType", PipelineService.get().getJmsType().toString()); + metrics.put("triggerCounts", triggerCounts, "Pipeline trigger configurations across all containers, by task pipeline: Enabled, Disabled, and Jobs run by that pipeline; computed over config-job join rows, so Enabled/Disabled are inflated when jobs exist"); + metrics.put("jmsType", PipelineService.get().getJmsType().toString(), "JMS queue type for pipeline jobs: none, inProcess, external, or unknown"); - result.put("pipelineRootCount", PipelineService.get().getAllPipelineRoots().size()); - result.put("supplementalDirectories", new TableSelector(PipelineSchema.getInstance().getTableInfoPipelineRoots(), new SimpleFilter("SupplementalPath", null, CompareType.NONBLANK), null).getRowCount()); + metrics.put("pipelineRootCount", PipelineService.get().getAllPipelineRoots().size(), "Containers with an explicitly configured primary pipeline root"); + metrics.put("supplementalDirectories", new TableSelector(PipelineSchema.getInstance().getTableInfoPipelineRoots(), new SimpleFilter("SupplementalPath", null, CompareType.NONBLANK), null).getRowCount(), "Configured pipeline roots that have a supplemental directory"); - return result; + return metrics; }); } diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index f13ae98d526..8aa00025eef 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -42,6 +42,7 @@ import org.labkey.api.module.Module; import org.labkey.api.module.ModuleContext; import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.AbstractQueryImportAction; import org.labkey.api.query.DefaultSchema; import org.labkey.api.query.JavaExportScriptFactory; import org.labkey.api.query.JavaScriptExportScriptFactory; @@ -49,6 +50,7 @@ import org.labkey.api.query.PythonExportScriptFactory; import org.labkey.api.query.QuerySchema; import org.labkey.api.query.QueryService; +import org.labkey.api.query.QueryUpdateService; import org.labkey.api.query.QueryView; import org.labkey.api.query.RExportScriptFactory; import org.labkey.api.query.SasExportScriptFactory; @@ -81,6 +83,8 @@ import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.stats.AnalyticsProviderRegistry; import org.labkey.api.stats.SummaryStatisticRegistry; +import org.labkey.api.usageMetrics.QueryMetricArea; +import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.JspTestCase; import org.labkey.api.util.JunitUtil; import org.labkey.api.util.PageFlowUtil; @@ -272,6 +276,14 @@ public boolean hasScripts() @Override public void doStartup(ModuleContext moduleContext) { + SimpleMetricsService simpleMetrics = SimpleMetricsService.get(); + simpleMetrics.describeFeatureAreas(QueryMetricArea.values()); + for (QueryUpdateService.InsertOption option : QueryUpdateService.InsertOption.values()) + { + simpleMetrics.describeFeatureArea(getName(), AbstractQueryImportAction.getRowCountFeatureArea(option), "Completed query imports (file, pasted text, background job, or other server code) using this insert option, by lowercase target schema name plus row-count bucket (1to1000, 1001to5000, 5001to10000, GT10000)"); + simpleMetrics.describeFeatureArea(getName(), AbstractQueryImportAction.getValidationErrorFeatureArea(option), "Query imports using this insert option that failed with validation errors, by target schema name, or query name when the table is unresolved (materials as samples)"); + } + ContainerManager.addContainerListener(QueryManager.CONTAINER_LISTENER, ContainerManager.ContainerListener.Order.Last); if (null != PipelineService.get()) diff --git a/query/src/org/labkey/query/persist/QueryManager.java b/query/src/org/labkey/query/persist/QueryManager.java index 4d7efcde706..3d0f96cf920 100644 --- a/query/src/org/labkey/query/persist/QueryManager.java +++ b/query/src/org/labkey/query/persist/QueryManager.java @@ -63,6 +63,7 @@ import org.labkey.api.query.UserSchema; import org.labkey.api.search.SearchService; import org.labkey.api.security.User; +import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.view.NotFoundException; import org.labkey.query.ExternalSchema; @@ -1054,7 +1055,7 @@ public static void registerUsageMetrics(String moduleName) UsageMetricsService svc = UsageMetricsService.get(); if (null != svc) { - svc.registerUsageMetrics(moduleName, () -> { + svc.registerUsageMetrics(moduleName, metrics -> { MultiSet multiSet = DbScope.getDbScopes().stream() .filter(scope -> !scope.isLabKeyScope()).map(DbScope::getDatabaseProductName) .collect(Collectors.toCollection(HashMultiSet::new)); @@ -1062,8 +1063,7 @@ public static void registerUsageMetrics(String moduleName) Map statsMap = multiSet.uniqueSet().stream() .collect(Collectors.toMap(Function.identity(), multiSet::getCount)); - Map metrics = new HashMap<>(); - metrics.put("externalDatasources", statsMap); + metrics.put("externalDatasources", statsMap, "External (non-LabKey) data sources by database product name"); metrics.put( "customViewCounts", Map.of( @@ -1071,14 +1071,15 @@ public static void registerUsageMetrics(String moduleName) "SampleTypes", getSchemaCustomViewCounts("samples"), "Assays", getSchemaCustomViewCounts("assay"), "Inventory", getSchemaCustomViewCounts("inventory") - ) + ), + "Saved custom views across all containers by schema area (DataClasses, SampleTypes, Assays, Inventory): defaultOverrides (unnamed) and namedViews exclude hidden and snapshot views, inheritable likewise, shared (no owner) and identifyingFieldsViews count all" ); - metrics.put("customViewWithLineageColumn", getLineageCustomViewMetrics()); - metrics.put("queryDefWithCalculatedFieldsCounts", getCalculatedFieldsCountsMetric()); + metrics.put("customViewWithLineageColumn", getLineageCustomViewMetrics(), "Custom views with lineage (input, output, ancestor) columns, hidden and snapshot views excluded: view counts, plus total and min/percentile/max/avg columns per view when any exist"); + metrics.put("queryDefWithCalculatedFieldsCounts", getCalculatedFieldsCountsMetric(), "Query definitions whose metadata defines a calculated field, by schema (assay.* schemas grouped as assay)"); Map exportTypeCounts = getExportTypeCountsMetric(); if (exportTypeCounts != null) - metrics.put("exportTypeCounts", exportTypeCounts); + metrics.put("exportTypeCounts", exportTypeCounts, "Query export audit events by export type, from the audit log across all folders; \"Excel multi-tab\" approximates multi-tab Excel exports", UsageMetrics.Kind.CUMULATIVE); return metrics; }); } diff --git a/query/src/org/labkey/query/reports/ReportServiceImpl.java b/query/src/org/labkey/query/reports/ReportServiceImpl.java index 93e006be423..939dd60b76a 100644 --- a/query/src/org/labkey/query/reports/ReportServiceImpl.java +++ b/query/src/org/labkey/query/reports/ReportServiceImpl.java @@ -1090,7 +1090,7 @@ public static void registerUsageMetrics(String moduleName) UsageMetricsService svc = UsageMetricsService.get(); if (null != svc) { - svc.registerUsageMetrics(moduleName, () -> { + svc.registerUsageMetrics(moduleName, metrics -> { // Iterate all the database reports once and produce two occurrence maps: all reports by type and just the charts by render type MultiSet chartCountsByRenderType = new HashMultiSet<>(); AtomicInteger genericChartWithTrendlineTypeCount = new AtomicInteger(); @@ -1123,12 +1123,11 @@ public static void registerUsageMetrics(String moduleName) .collect(Collectors.groupingBy(Report::getType, Collectors.counting())); } - return Map.of( - "reportCountsByType", countsByType, - "genericChartCountsByRenderType", MultiSetUtils.getOccurrenceMap(chartCountsByRenderType), - "genericChartWithTrendlineTypeCount", genericChartWithTrendlineTypeCount, - "genericChartWithErrorBarsCount", genericChartWithErrorBarsCount - ); + return metrics + .put("reportCountsByType", countsByType, "Saved reports in existing containers by report type") + .put("genericChartCountsByRenderType", MultiSetUtils.getOccurrenceMap(chartCountsByRenderType), "Saved charts (GenericChartReport) by render type (BAR_PLOT, BOX_PLOT, etc.)") + .put("genericChartWithTrendlineTypeCount", genericChartWithTrendlineTypeCount, "Saved charts with a trendline type set") + .put("genericChartWithErrorBarsCount", genericChartWithErrorBarsCount, "Saved charts with SD or SEM error bars"); }); } } diff --git a/search/src/org/labkey/search/SearchMetricArea.java b/search/src/org/labkey/search/SearchMetricArea.java new file mode 100644 index 00000000000..92c7cbc63ef --- /dev/null +++ b/search/src/org/labkey/search/SearchMetricArea.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.search; + +import org.labkey.api.usageMetrics.SimpleMetricArea; + +public enum SearchMetricArea implements SimpleMetricArea +{ + DELETE("Delete", "Search index document deletes that failed because the index was already closed, other than for a full disk (IndexAlreadyClosed)"); + + private final Definition _definition; + + SearchMetricArea(String areaName, String description) + { + _definition = new Definition(SearchModule.NAME, areaName, description); + } + + @Override + public Definition getDefinition() + { + return _definition; + } +} diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index 902bef6009a..db7d675515b 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -42,6 +42,8 @@ import org.labkey.api.settings.AdminConsole; import org.labkey.api.settings.StandardStartupPropertyHandler; import org.labkey.api.settings.StartupPropertyEntry; +import org.labkey.api.usageMetrics.SimpleMetricsService; +import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.util.JobRunner; import org.labkey.api.util.PageFlowUtil; @@ -63,7 +65,6 @@ import javax.management.StandardMBean; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Queue; @@ -136,6 +137,8 @@ public WebdavResource resolve(@NotNull String path) @Override public void doStartup(ModuleContext moduleContext) { + SimpleMetricsService.get().describeFeatureAreas(SearchMetricArea.values()); + ModuleLoader.getInstance().handleStartupProperties( new StandardStartupPropertyHandler<>("SearchSettings", SearchStartupProperties.class) { @@ -177,7 +180,7 @@ public void handle(Map properties FolderManagement.addTab(FolderManagement.TYPE.FolderManagement, "Search", "fullTextSearch", FolderManagement.NOT_ROOT, SearchController.SearchSettingsAction.class); - UsageMetricsService.get().registerUsageMetrics(getName(), () -> + UsageMetricsService.get().registerUsageMetrics(getName(), metrics -> { // Report the total number of search entries in the audit log User user = new LimitedUser(User.getSearchUser(), CanSeeAuditLogRole.class); @@ -185,7 +188,7 @@ public void handle(Map properties TableInfo auditTable = auditSchema.getTableOrThrow(SearchAuditProvider.EVENT_TYPE, ContainerFilter.getUnsafeEverythingFilter()); long count = new TableSelector(auditTable).getRowCount(); - return Collections.singletonMap("fullTextSearches", count); + return metrics.put("fullTextSearches", count, "Full-text search events in the audit log across all containers; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE); }); } diff --git a/search/src/org/labkey/search/model/WritableIndexManagerImpl.java b/search/src/org/labkey/search/model/WritableIndexManagerImpl.java index 48277ff9b73..e7b58fe2652 100644 --- a/search/src/org/labkey/search/model/WritableIndexManagerImpl.java +++ b/search/src/org/labkey/search/model/WritableIndexManagerImpl.java @@ -36,7 +36,7 @@ import org.labkey.api.usageMetrics.SimpleMetricsService; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.ExceptionUtil; -import org.labkey.search.SearchModule; +import org.labkey.search.SearchMetricArea; import org.quartz.DateBuilder; import org.quartz.Job; import org.quartz.JobBuilder; @@ -207,7 +207,7 @@ public void deleteDocuments(Collection ids) else { _log.error("Indexing error deleting {} indexer is already closed", StringUtils.trimToEmpty(currentId), e); - SimpleMetricsService.get().increment(SearchModule.NAME,"Delete", "IndexAlreadyClosed"); + SimpleMetricsService.get().increment(SearchMetricArea.DELETE, "IndexAlreadyClosed"); } } catch (Throwable e) diff --git a/study/src/org/labkey/study/StudyModule.java b/study/src/org/labkey/study/StudyModule.java index 50392f984ba..fab9f49c309 100644 --- a/study/src/org/labkey/study/StudyModule.java +++ b/study/src/org/labkey/study/StudyModule.java @@ -425,45 +425,44 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) UsageMetricsService svc = UsageMetricsService.get(); if (null != svc) { - svc.registerUsageMetrics(MODULE_NAME, () -> { - Map metric = new HashMap<>(); - metric.put("studyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.study").getObject(Long.class)); - metric.put("datasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.dataset").getObject(Long.class)); + svc.registerUsageMetrics(MODULE_NAME, metrics -> { + metrics.put("studyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.study").getObject(Long.class), "Studies across the server"); + metrics.put("datasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.dataset").getObject(Long.class), "Datasets across all studies, all dataset types included"); - metric.put("securityType", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT SecurityType, COUNT(*) FROM study.Study GROUP BY SecurityType").getValueMap()); - metric.put("timepointType", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT TimepointType, COUNT(*) FROM study.Study GROUP BY TimepointType").getValueMap()); + metrics.put("securityType", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT SecurityType, COUNT(*) FROM study.Study GROUP BY SecurityType").getValueMap(), "Studies by security type (BASIC_READ, BASIC_WRITE, ADVANCED_READ, ADVANCED_WRITE)"); + metrics.put("timepointType", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT TimepointType, COUNT(*) FROM study.Study GROUP BY TimepointType").getValueMap(), "Studies by timepoint type (VISIT, DATE, CONTINUOUS)"); - metric.put("linkedSampleTypeDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(PublishSourceType) FROM study.dataset WHERE PublishSourceType = 'SampleType'").getObject(Long.class)); - metric.put("linkedAssayDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(PublishSourceType) FROM study.dataset WHERE PublishSourceType = 'Assay'").getObject(Long.class)); + metrics.put("linkedSampleTypeDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(PublishSourceType) FROM study.dataset WHERE PublishSourceType = 'SampleType'").getObject(Long.class), "Datasets linked from sample types; same value as datasetsLinkedFromSamples"); + metrics.put("linkedAssayDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(PublishSourceType) FROM study.dataset WHERE PublishSourceType = 'Assay'").getObject(Long.class), "Datasets linked from assay results; same value as datasetsLinkedFromAssays"); - metric.put("redcapCount", new SqlSelector(PropertySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM prop.PropertySets WHERE Category = 'RedcapConfigurationSettings'").getObject(Long.class)); + metrics.put("redcapCount", new SqlSelector(PropertySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM prop.PropertySets WHERE Category = 'RedcapConfigurationSettings'").getObject(Long.class), "Saved REDCap configuration property sets across all containers"); // Note: study.StudySnapshot maintains a rows for deleted child studies (allowing re-creation of those child studies). // In that case, Destination is set to null. COUNT(DISTINCT) skips these null values, so this returns the count of existing child studies. - metric.put("publishStudyCount", new SqlSelector(PropertySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT destination) FROM study.StudySnapshot WHERE Type = 'publish'").getObject(Long.class)); + metrics.put("publishStudyCount", new SqlSelector(PropertySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT destination) FROM study.StudySnapshot WHERE Type = 'publish'").getObject(Long.class), "Existing child studies created by publishing a study; deleted children excluded"); SqlDialect dialect = StudySchema.getInstance().getSqlDialect(); - metric.put("demographicsDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE DemographicData = " + dialect.getBooleanTRUE()).getObject(Long.class)); - metric.put("standardDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE Type = 'Standard'").getObject(Long.class)); + metrics.put("demographicsDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE DemographicData = " + dialect.getBooleanTRUE()).getObject(Long.class), "Demographics datasets across all studies"); + metrics.put("standardDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE Type = 'Standard'").getObject(Long.class), "Datasets of type Standard across all studies; placeholder datasets excluded"); - metric.put("managedThirdKeyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE KeyManagementType <> 'None'").getObject(Long.class)); - metric.put("thirdKeyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE (KeyManagementType = 'None' AND KeyPropertyName IS NOT NULL) OR (UseTimeKeyField = " + dialect.getBooleanTRUE() + ")").getObject(Long.class)); + metrics.put("managedThirdKeyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE KeyManagementType <> 'None'").getObject(Long.class), "Datasets whose additional key is server-managed (RowId or GUID)"); + metrics.put("thirdKeyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE (KeyManagementType = 'None' AND KeyPropertyName IS NOT NULL) OR (UseTimeKeyField = " + dialect.getBooleanTRUE() + ")").getObject(Long.class), "Datasets with a user-managed additional key field or using time as the additional key"); - metric.put("datasetsLinkedFromAssays", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE PublishSourceType = 'Assay'").getObject(Long.class)); - metric.put("datasetsLinkedFromSamples", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE PublishSourceType = 'SampleType'").getObject(Long.class)); + metrics.put("datasetsLinkedFromAssays", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE PublishSourceType = 'Assay'").getObject(Long.class), "Datasets linked from assay results; same value as linkedAssayDatasetCount"); + metrics.put("datasetsLinkedFromSamples", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE PublishSourceType = 'SampleType'").getObject(Long.class), "Datasets linked from sample types; same value as linkedSampleTypeDatasetCount"); - metric.put("assayScheduleCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT container) FROM study.AssaySpecimen").getObject(Long.class)); + metrics.put("assayScheduleCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT container) FROM study.AssaySpecimen").getObject(Long.class), "Containers with at least one assay schedule entry"); - metric.put("alternateParticipantIdCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Study WHERE AlternateIdPrefix IS NOT NULL").getObject(Long.class)); - metric.put("participantIdMappingCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT container) FROM study.Participant WHERE AlternateId IS NOT NULL").getObject(Long.class)); - metric.put("participantAliasCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Study WHERE ParticipantAliasDatasetId IS NOT NULL").getObject(Long.class)); + metrics.put("alternateParticipantIdCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Study WHERE AlternateIdPrefix IS NOT NULL").getObject(Long.class), "Studies with an alternate participant ID prefix configured"); + metrics.put("participantIdMappingCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT container) FROM study.Participant WHERE AlternateId IS NOT NULL").getObject(Long.class), "Containers with at least one participant assigned an alternate ID"); + metrics.put("participantAliasCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Study WHERE ParticipantAliasDatasetId IS NOT NULL").getObject(Long.class), "Studies with a participant alias dataset configured"); - metric.put("datasetsWithFileColumnCount", new SqlSelector(StudySchema.getInstance().getSchema(), """ + metrics.put("datasetsWithFileColumnCount", new SqlSelector(StudySchema.getInstance().getSchema(), """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", StudySchema.getInstance().getDatasetSchemaName(), PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", StudySchema.getInstance().getDatasetSchemaName(), PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class), "Datasets with at least one File column"); // grab the counts of report and dataset notification settings (by notification option) Collection allStudies = StudyManager.getInstance().getAllStudies(); @@ -478,7 +477,7 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM }); Map notificationMap = notifyOptionCounts.entrySet().stream() .collect(Collectors.toMap(e -> e.getKey().name(), Map.Entry::getValue)); - metric.put("reportAndDatasetNotificationOptions", notificationMap); + metrics.put("reportAndDatasetNotificationOptions", notificationMap, "Users subscribed to report and dataset change notifications by option (NONE, ALL, CATEGORY, DATASET); a user counts once per study they subscribe in"); long cloudBackedStudies = allStudies.stream() .filter(s -> { @@ -486,7 +485,7 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM return pr != null && pr.isCloudRoot(); }) .count(); - metric.put("cloudBackedStudies", cloudBackedStudies); + metrics.put("cloudBackedStudies", cloudBackedStudies, "Studies whose pipeline root is in cloud storage"); // Count the studies that have configured at least one group for per-dataset access control long studiesWithAnyPerDatasetGroup = allStudies.stream() @@ -494,7 +493,7 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM .filter(study -> SecurityManager.getGroups(study.getContainer().getProject(), true).stream() .anyMatch(group -> GroupSecurityType.getTypeForGroup(group, study) == GroupSecurityType.PER_DATASET)) .count(); - metric.put("perDatasetSecurityStudyCount", studiesWithAnyPerDatasetGroup); + metrics.put("perDatasetSecurityStudyCount", studiesWithAnyPerDatasetGroup, "Studies with a security type supporting per-dataset permissions and at least one project group set to per-dataset access"); // Count the studies that use products and treatments MutableInt hasProducts = new MutableInt(0); @@ -515,10 +514,10 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM } }); - metric.put("studyProducts", hasProducts.intValue()); - metric.put("studyTreatments", hasTreatments.intValue()); + metrics.put("studyProducts", hasProducts.intValue(), "Studies with at least one study product, counted only in folders with the StudyDesign module active"); + metrics.put("studyTreatments", hasTreatments.intValue(), "Studies with at least one treatment, counted only in folders with the StudyDesign module active"); - return metric; + return metrics; }); } diff --git a/survey/src/org/labkey/survey/SurveyModule.java b/survey/src/org/labkey/survey/SurveyModule.java index d72fe84786c..a8a53f54634 100644 --- a/survey/src/org/labkey/survey/SurveyModule.java +++ b/survey/src/org/labkey/survey/SurveyModule.java @@ -65,7 +65,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -131,11 +130,10 @@ public void doStartup(ModuleContext moduleContext) UsageMetricsService svc = UsageMetricsService.get(); if (null != svc) { - svc.registerUsageMetrics(NAME, () -> { - Map metric = new HashMap<>(); - metric.put("surveyDesigns", new SqlSelector(DbSchema.get("survey", DbSchemaType.Module), "SELECT COUNT(*) FROM survey.SurveyDesigns").getObject(Long.class)); - metric.put("surveys", new SqlSelector(DbSchema.get("survey", DbSchemaType.Module), "SELECT COUNT(*) FROM survey.Surveys").getObject(Long.class)); - return metric; + svc.registerUsageMetrics(NAME, metrics -> { + metrics.put("surveyDesigns", new SqlSelector(DbSchema.get("survey", DbSchemaType.Module), "SELECT COUNT(*) FROM survey.SurveyDesigns").getObject(Long.class), "Survey designs across all containers"); + metrics.put("surveys", new SqlSelector(DbSchema.get("survey", DbSchemaType.Module), "SELECT COUNT(*) FROM survey.Surveys").getObject(Long.class), "Survey responses across all containers, submitted or in progress"); + return metrics; }); } } From 1f5d68717d3a52c2bb8e7a01b459e3fe06694fa2 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Fri, 23 Oct 2026 18:22:30 -0700 Subject: [PATCH 2/5] Claude review --- .../api/query/AbstractQueryImportAction.java | 24 +++----- .../api/usageMetrics/QueryMetricArea.java | 35 ++++++++++- .../api/usageMetrics/UsageMetricGroup.java | 8 ++- .../labkey/api/usageMetrics/UsageMetrics.java | 58 +++++++++++++------ .../metrics/SimpleMetricsServiceImpl.java | 2 +- .../labkey/experiment/ExperimentModule.java | 2 - query/src/org/labkey/query/QueryModule.java | 8 +-- 7 files changed, 89 insertions(+), 48 deletions(-) diff --git a/api/src/org/labkey/api/query/AbstractQueryImportAction.java b/api/src/org/labkey/api/query/AbstractQueryImportAction.java index f712c212449..85eee29be9e 100644 --- a/api/src/org/labkey/api/query/AbstractQueryImportAction.java +++ b/api/src/org/labkey/api/query/AbstractQueryImportAction.java @@ -1002,24 +1002,14 @@ else if (!context.isCrossTypeImport()) public static void addImportValidationErrorMetric(QueryUpdateService.InsertOption insertOption, @Nullable TableInfo target, @Nullable String queryName) { - String featureArea = getValidationErrorFeatureArea(insertOption); + QueryMetricArea featureArea = QueryMetricArea.validationErrors(insertOption); String targetType = null; if (target != null) targetType = getMetricPrefix(target); else if (queryName != null) targetType = queryName.equalsIgnoreCase("materials") ? "samples" : queryName; if (targetType != null) - SimpleMetricsService.get().increment("query", featureArea, targetType); - } - - public static String getValidationErrorFeatureArea(QueryUpdateService.InsertOption insertOption) - { - return insertOption.toString().toLowerCase() + "FailWithValidationError"; - } - - public static String getRowCountFeatureArea(QueryUpdateService.InsertOption insertOption) - { - return "file" + StringUtils.capitalize(insertOption.toString().toLowerCase()) + "Counts"; + SimpleMetricsService.get().increment(featureArea, targetType); } private static String getMetricPrefix(@NotNull TableInfo target) @@ -1030,16 +1020,16 @@ private static String getMetricPrefix(@NotNull TableInfo target) public static void incrementRowCountMetric(int count, QueryUpdateService.InsertOption insertOption, String prefix) { - String featureArea = getRowCountFeatureArea(insertOption); + QueryMetricArea featureArea = QueryMetricArea.rowCounts(insertOption); String lcPrefix = prefix.toLowerCase(); if (count <= 1000) - SimpleMetricsService.get().increment("query", featureArea, lcPrefix + "1to1000"); + SimpleMetricsService.get().increment(featureArea, lcPrefix + "1to1000"); else if (count <= 5000) - SimpleMetricsService.get().increment("query", featureArea, lcPrefix + "1001to5000"); + SimpleMetricsService.get().increment(featureArea, lcPrefix + "1001to5000"); else if (count <= 10000) - SimpleMetricsService.get().increment("query", featureArea, lcPrefix + "5001to10000"); + SimpleMetricsService.get().increment(featureArea, lcPrefix + "5001to10000"); else - SimpleMetricsService.get().increment("query", featureArea, lcPrefix + "GT10000"); + SimpleMetricsService.get().increment(featureArea, lcPrefix + "GT10000"); } diff --git a/api/src/org/labkey/api/usageMetrics/QueryMetricArea.java b/api/src/org/labkey/api/usageMetrics/QueryMetricArea.java index bf8d9b3b495..c026aad1e33 100644 --- a/api/src/org/labkey/api/usageMetrics/QueryMetricArea.java +++ b/api/src/org/labkey/api/usageMetrics/QueryMetricArea.java @@ -15,9 +15,32 @@ */ package org.labkey.api.usageMetrics; +import org.labkey.api.query.QueryUpdateService.InsertOption; + public enum QueryMetricArea implements SimpleMetricArea { - FILE_BACKGROUND_IMPORTS("fileBackgroundImports", "Query imports queued as background pipeline jobs, by target schema name"); + FILE_BACKGROUND_IMPORTS("fileBackgroundImports", "Query imports queued as background pipeline jobs, by target schema name"), + FILE_INSERT_COUNTS("fileInsertCounts", Descriptions.ROW_COUNTS), + FILE_UPSERT_COUNTS("fileUpsertCounts", Descriptions.ROW_COUNTS), + FILE_IMPORT_COUNTS("fileImportCounts", Descriptions.ROW_COUNTS), + FILE_MERGE_COUNTS("fileMergeCounts", Descriptions.ROW_COUNTS), + FILE_REPLACE_COUNTS("fileReplaceCounts", Descriptions.ROW_COUNTS), + FILE_IMPORT_IDENTITY_COUNTS("fileImport_identityCounts", Descriptions.ROW_COUNTS), + FILE_UPDATE_COUNTS("fileUpdateCounts", Descriptions.ROW_COUNTS), + INSERT_FAIL_WITH_VALIDATION_ERROR("insertFailWithValidationError", Descriptions.VALIDATION_ERRORS), + UPSERT_FAIL_WITH_VALIDATION_ERROR("upsertFailWithValidationError", Descriptions.VALIDATION_ERRORS), + IMPORT_FAIL_WITH_VALIDATION_ERROR("importFailWithValidationError", Descriptions.VALIDATION_ERRORS), + MERGE_FAIL_WITH_VALIDATION_ERROR("mergeFailWithValidationError", Descriptions.VALIDATION_ERRORS), + REPLACE_FAIL_WITH_VALIDATION_ERROR("replaceFailWithValidationError", Descriptions.VALIDATION_ERRORS), + IMPORT_IDENTITY_FAIL_WITH_VALIDATION_ERROR("import_identityFailWithValidationError", Descriptions.VALIDATION_ERRORS), + UPDATE_FAIL_WITH_VALIDATION_ERROR("updateFailWithValidationError", Descriptions.VALIDATION_ERRORS); + + // Enum constants can't refer to the enum's own static fields + private static class Descriptions + { + static final String ROW_COUNTS = "Completed query imports (file, pasted text, background job, or other server code) using this insert option, by lowercase target schema name plus row-count bucket (1to1000, 1001to5000, 5001to10000, GT10000)"; + static final String VALIDATION_ERRORS = "Query imports using this insert option that failed with validation errors, by target schema name, or query name when the table is unresolved (materials as samples)"; + } private final Definition _definition; @@ -31,4 +54,14 @@ public Definition getDefinition() { return _definition; } + + public static QueryMetricArea rowCounts(InsertOption option) + { + return valueOf("FILE_" + option.name() + "_COUNTS"); + } + + public static QueryMetricArea validationErrors(InsertOption option) + { + return valueOf(option.name() + "_FAIL_WITH_VALIDATION_ERROR"); + } } diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java b/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java index a5b9adc80ac..f671498a4c6 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java @@ -30,13 +30,17 @@ public class UsageMetricGroup extends UsageMetrics public UsageMetricGroup put(@NotNull String key, @Nullable Object value) { - getMutableValues().put(key, value); + Map values = getMutableValues(); + if (values != null) + values.put(key, value); return this; } public UsageMetricGroup putAll(@NotNull Map values) { - getMutableValues().putAll(values); + Map mutable = getMutableValues(); + if (mutable != null) + mutable.putAll(values); return this; } diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetrics.java b/api/src/org/labkey/api/usageMetrics/UsageMetrics.java index ef8d5787195..3a582b5cce8 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetrics.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetrics.java @@ -15,10 +15,13 @@ */ package org.labkey.api.usageMetrics; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Test; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.logging.LogHelper; import java.util.Collections; import java.util.LinkedHashMap; @@ -33,6 +36,8 @@ */ public class UsageMetrics { + private static final Logger LOG = LogHelper.getLogger(UsageMetrics.class, "Usage metric descriptions"); + public enum Kind { /** Recomputed from current data at each report */ @@ -69,25 +74,26 @@ public UsageMetrics put(@NotNull String key, @Nullable Object value, @NotNull St public UsageMetrics put(@NotNull String key, @Nullable Object value, @NotNull String description, @NotNull Kind kind) { describe(key, description, kind); - getMutableValues().put(key, value); + if (_values != null) + _values.put(key, value); return this; } - /** Calls the supplier only when values are reported, so a {@code reportedIf(false)} view skips the work */ - public UsageMetrics put(@NotNull String key, @NotNull Supplier value, @NotNull String description) + /** Calls the supplier only when values are reported, so a {@code reportedIf(false)} view skips the work. A literal null value resolves here. */ + public UsageMetrics put(@NotNull String key, @Nullable Supplier value, @NotNull String description) { return put(key, value, description, Kind.CURRENT); } - public UsageMetrics put(@NotNull String key, @NotNull Supplier value, @NotNull String description, @NotNull Kind kind) + public UsageMetrics put(@NotNull String key, @Nullable Supplier value, @NotNull String description, @NotNull Kind kind) { describe(key, description, kind); if (_values != null) - _values.put(key, value.get()); + _values.put(key, value == null ? null : value.get()); return this; } - /** @return this, or if not reported, a view at the same path that only describes what's put into it, groups included */ + /** @return this, or if not reported, a view at the same path that describes what's put into it and drops the values, groups included */ public UsageMetrics reportedIf(boolean reported) { return reported ? this : new UsageMetrics(_path, null, _descriptions); @@ -104,10 +110,18 @@ public UsageMetrics describe(@NotNull String key, @NotNull String description, @ Description added = new Description(describedPath(key), description, kind); Description existing = _descriptions.putIfAbsent(added.path(), added); if (existing != null && !existing.equals(added)) - throw new IllegalStateException("Usage metric " + added.path() + " is described inconsistently: " + existing + " vs. " + added); + reportInconsistentDescription("Usage metric " + added.path() + " is described inconsistently: " + existing + " vs. " + added); return this; } + /** Throws in dev mode; otherwise warns and keeps the first description, so a wording mismatch can't break startup or a report */ + public static void reportInconsistentDescription(@NotNull String message) + { + if (AppProps.getInstance().isDevMode()) + throw new IllegalStateException(message); + LOG.warn(message); + } + /** Adds a nested object whose children, including keys only known at runtime, share this description */ public UsageMetricGroup group(@NotNull String key, @NotNull String description) { @@ -120,7 +134,7 @@ public UsageMetricGroup group(@NotNull String key, @NotNull String description, return addGroup(key); } - /** Describes a nested object without reporting it. Its children can be described, but not given values. */ + /** Describes a nested object without reporting it. Its children can be described; values put into it are dropped. */ public UsageMetricGroup describeGroup(@NotNull String key, @NotNull String description) { return describeGroup(key, description, Kind.CURRENT); @@ -137,7 +151,7 @@ UsageMetricGroup addGroup(@NotNull String key) if (_values == null) return new UsageMetricGroup(childPath(key), null, _descriptions); Map child = new LinkedHashMap<>(); - getMutableValues().put(key, child); + _values.put(key, child); return new UsageMetricGroup(childPath(key), child, _descriptions); } @@ -154,10 +168,9 @@ String childPath(String key) return _path == null ? key : _path + "." + key; } - Map getMutableValues() + /** @return null for a view that's only described */ + @Nullable Map getMutableValues() { - if (_values == null) - throw new IllegalStateException("Usage metric group " + _path + " is only described, so it can't hold values"); return _values; } @@ -250,8 +263,9 @@ public void testDescribeGroupWithoutReporting() new Description("perFolder", "Folder counts", Kind.CURRENT), new Description("perFolder.runs", "Runs", Kind.CURRENT)), metrics.getDescriptions()); - assertThrows(IllegalStateException.class, () -> absent.put("runs", 1L)); - assertThrows(IllegalStateException.class, () -> absent.put("runs", 1L, "Runs")); + absent.put("runs", 1L); + absent.put("runs", 1L, "Runs"); + assertEquals(Map.of(), metrics.getValues()); } @Test @@ -266,9 +280,15 @@ public void testReportedIf() colors.reportedIf(false).put("archived", () -> 2L, "Archived colors"); UsageMetricGroup absent = metrics.reportedIf(false).group("absent", "Absent group"); absent.group("child", "Absent child").describe("leaf", "Leaf"); - assertThrows(IllegalStateException.class, () -> absent.put("x", 1L)); - - assertEquals(Map.of("shown", 1L, "colors", Map.of("count", 0L)), metrics.getValues()); + absent.put("x", 1L); + metrics.reportedIf(false).put("plain", 3L, "Plain value"); + metrics.put("none", null, "Null value"); + + Map expected = new LinkedHashMap<>(); + expected.put("shown", 1L); + expected.put("colors", Map.of("count", 0L)); + expected.put("none", null); + assertEquals(expected, metrics.getValues()); assertEquals(List.of( new Description("shown", "Shown", Kind.CURRENT), new Description("hidden", "Hidden", Kind.CUMULATIVE), @@ -277,7 +297,9 @@ public void testReportedIf() new Description("colors.archived", "Archived colors", Kind.CURRENT), new Description("absent", "Absent group", Kind.CURRENT), new Description("absent.child", "Absent child", Kind.CURRENT), - new Description("absent.child.leaf", "Leaf", Kind.CURRENT)), + new Description("absent.child.leaf", "Leaf", Kind.CURRENT), + new Description("plain", "Plain value", Kind.CURRENT), + new Description("none", "Null value", Kind.CURRENT)), metrics.getDescriptions()); } } diff --git a/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java b/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java index 5892adc1ee9..3fd38955da3 100644 --- a/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java +++ b/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java @@ -189,7 +189,7 @@ public void describeFeatureArea(@NotNull String requestedModuleName, @NotNull St getModuleMetrics(module); // Registers the provider so the description is reported before anything is counted String existing = _featureAreaDescriptions.computeIfAbsent(module.getName(), k -> new ConcurrentHashMap<>()).putIfAbsent(featureArea, description); if (existing != null && !existing.equals(description)) - throw new IllegalStateException("Feature area " + module.getName() + "." + featureArea + " is described inconsistently: \"" + existing + "\" vs. \"" + description + "\""); + UsageMetrics.reportInconsistentDescription("Feature area " + module.getName() + "." + featureArea + " is described inconsistently: \"" + existing + "\" vs. \"" + description + "\""); } @Override diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index f7215e29bb9..dcee33be74d 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -200,8 +200,6 @@ public class ExperimentModule extends SpringModule private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; - // Usage metric descriptions shared by the branches that report a key and the ones that only describe it - public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index 8aa00025eef..5ceba12e811 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -276,13 +276,7 @@ public boolean hasScripts() @Override public void doStartup(ModuleContext moduleContext) { - SimpleMetricsService simpleMetrics = SimpleMetricsService.get(); - simpleMetrics.describeFeatureAreas(QueryMetricArea.values()); - for (QueryUpdateService.InsertOption option : QueryUpdateService.InsertOption.values()) - { - simpleMetrics.describeFeatureArea(getName(), AbstractQueryImportAction.getRowCountFeatureArea(option), "Completed query imports (file, pasted text, background job, or other server code) using this insert option, by lowercase target schema name plus row-count bucket (1to1000, 1001to5000, 5001to10000, GT10000)"); - simpleMetrics.describeFeatureArea(getName(), AbstractQueryImportAction.getValidationErrorFeatureArea(option), "Query imports using this insert option that failed with validation errors, by target schema name, or query name when the table is unresolved (materials as samples)"); - } + SimpleMetricsService.get().describeFeatureAreas(QueryMetricArea.values()); ContainerManager.addContainerListener(QueryManager.CONTAINER_LISTENER, ContainerManager.ContainerListener.Order.Last); From 24471336530c345e3c19e8918c1ef6285d0e63e7 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Mon, 26 Oct 2026 18:49:27 -0700 Subject: [PATCH 3/5] Skip usage metric queries when only collecting descriptions Also flag simple metric feature areas that lack a description and clarify describe-only terminology, per review feedback. --- .../announcements/AnnouncementModule.java | 2 +- .../api/usageMetrics/UsageMetricGroup.java | 10 ++ .../labkey/api/usageMetrics/UsageMetrics.java | 29 +++- .../usageMetrics/UsageMetricsProvider.java | 7 + api/src/org/labkey/api/view/Portal.java | 2 +- .../assay/plate/PlateMetricsProvider.java | 4 +- core/src/org/labkey/core/CoreModule.java | 22 +-- .../usageMetrics/UsageMetricsServiceImpl.java | 4 +- .../metrics/SimpleMetricsServiceImpl.java | 8 ++ .../labkey/experiment/ExperimentModule.java | 128 +++++++++--------- .../api/property/PropertyServiceImpl.java | 10 +- issues/src/org/labkey/issue/IssuesModule.java | 4 +- list/src/org/labkey/list/ListModule.java | 6 +- .../org/labkey/pipeline/PipelineModule.java | 4 +- .../labkey/query/persist/QueryManager.java | 6 +- study/src/org/labkey/study/StudyModule.java | 38 +++--- .../src/org/labkey/survey/SurveyModule.java | 4 +- 17 files changed, 168 insertions(+), 120 deletions(-) diff --git a/announcements/src/org/labkey/announcements/AnnouncementModule.java b/announcements/src/org/labkey/announcements/AnnouncementModule.java index c10df200cf5..cfd94667cbc 100644 --- a/announcements/src/org/labkey/announcements/AnnouncementModule.java +++ b/announcements/src/org/labkey/announcements/AnnouncementModule.java @@ -210,7 +210,7 @@ public void doStartup(ModuleContext moduleContext) } return metrics .put("messageBoards", boards.get(), "Containers with at least one message") - .put("messages", new TableSelector(comm.getTableInfoAnnouncements()).getRowCount(), "Messages across all containers, including responses") + .put("messages", () -> new TableSelector(comm.getTableInfoAnnouncements()).getRowCount(), "Messages across all containers, including responses") .put("settings", Map.of( "secure", MultiSetUtils.getOccurrenceMap(secure), "moderatorReview", MultiSetUtils.getOccurrenceMap(moderator) diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java b/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java index f671498a4c6..1c622b70ea7 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.Nullable; import java.util.Map; +import java.util.function.Supplier; /** A described nested object; its children may omit descriptions because they inherit the group's */ public class UsageMetricGroup extends UsageMetrics @@ -36,6 +37,15 @@ public UsageMetricGroup put(@NotNull String key, @Nullable Object value) return this; } + /** Skips the supplier on a describe-only group. Java binds {@code put(key, null)} to this overload. */ + public UsageMetricGroup put(@NotNull String key, @Nullable Supplier value) + { + Map values = getMutableValues(); + if (values != null) + values.put(key, value == null ? null : value.get()); + return this; + } + public UsageMetricGroup putAll(@NotNull Map values) { Map mutable = getMutableValues(); diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetrics.java b/api/src/org/labkey/api/usageMetrics/UsageMetrics.java index 3a582b5cce8..933966f6267 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetrics.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetrics.java @@ -58,6 +58,12 @@ public UsageMetrics() this(null, new LinkedHashMap<>(), new LinkedHashMap<>()); } + /** @return an instance that records descriptions and drops values without calling their suppliers */ + public static UsageMetrics describeOnly() + { + return new UsageMetrics(null, null, new LinkedHashMap<>()); + } + /** @param values null for a group that's only described */ UsageMetrics(@Nullable String path, @Nullable Map values, Map descriptions) { @@ -79,7 +85,7 @@ public UsageMetrics put(@NotNull String key, @Nullable Object value, @NotNull St return this; } - /** Calls the supplier only when values are reported, so a {@code reportedIf(false)} view skips the work. A literal null value resolves here. */ + /** Skips the supplier on a describe-only instance from {@code reportedIf(false)}. Java binds {@code put(key, null, description)} to this overload. */ public UsageMetrics put(@NotNull String key, @Nullable Supplier value, @NotNull String description) { return put(key, value, description, Kind.CURRENT); @@ -93,7 +99,7 @@ public UsageMetrics put(@NotNull String key, @Nullable Supplier value, @NotNu return this; } - /** @return this, or if not reported, a view at the same path that describes what's put into it and drops the values, groups included */ + /** @return this, or if not reported, a describe-only instance at the same path: it records descriptions of what's put into it, groups included, but drops the values */ public UsageMetrics reportedIf(boolean reported) { return reported ? this : new UsageMetrics(_path, null, _descriptions); @@ -168,7 +174,7 @@ String childPath(String key) return _path == null ? key : _path + "." + key; } - /** @return null for a view that's only described */ + /** @return null for a describe-only instance */ @Nullable Map getMutableValues() { return _values; @@ -268,6 +274,23 @@ public void testDescribeGroupWithoutReporting() assertEquals(Map.of(), metrics.getValues()); } + @Test + public void testDescribeOnly() + { + UsageMetrics metrics = describeOnly(); + metrics.put("count", () -> { throw new AssertionError("Supplier called while only describing"); }, "Count"); + UsageMetricGroup group = metrics.group("byType", "Counts by type"); + group.put("lazy", () -> { throw new AssertionError("Supplier called while only describing"); }); + group.put("eager", 1L, "Eager value"); + + assertEquals(Map.of(), metrics.getValues()); + assertEquals(List.of( + new Description("count", "Count", Kind.CURRENT), + new Description("byType", "Counts by type", Kind.CURRENT), + new Description("byType.eager", "Eager value", Kind.CURRENT)), + metrics.getDescriptions()); + } + @Test public void testReportedIf() { diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetricsProvider.java b/api/src/org/labkey/api/usageMetrics/UsageMetricsProvider.java index fc889498a7b..69d0911ebeb 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetricsProvider.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetricsProvider.java @@ -15,6 +15,7 @@ */ package org.labkey.api.usageMetrics; +import java.util.List; import java.util.Map; /** @@ -31,6 +32,12 @@ default Map getUsageMetrics() return collectUsageMetrics().getValues(); } + /** Skips values passed as suppliers; work done outside a supplier still runs */ + default List getUsageMetricDescriptions() + { + return addUsageMetrics(UsageMetrics.describeOnly()).getDescriptions(); + } + default UsageMetrics collectUsageMetrics() { UsageMetrics metrics = new UsageMetrics(); diff --git a/api/src/org/labkey/api/view/Portal.java b/api/src/org/labkey/api/view/Portal.java index bcb0adcac44..127daa1e125 100644 --- a/api/src/org/labkey/api/view/Portal.java +++ b/api/src/org/labkey/api/view/Portal.java @@ -211,7 +211,7 @@ public static UsageMetricsProvider getMetricsProvider() append(getTableInfoPortalWebParts(), "pwp"). unsafeAppend(" GROUP BY Name"); - return metrics.put("webPartCounts", new SqlSelector(getSchema(), sql).getValueMap(String.class), "Web parts on portal pages across the server, by web part name"); + return metrics.put("webPartCounts", () -> new SqlSelector(getSchema(), sql).getValueMap(String.class), "Web parts on portal pages across the server, by web part name"); }; } diff --git a/assay/src/org/labkey/assay/plate/PlateMetricsProvider.java b/assay/src/org/labkey/assay/plate/PlateMetricsProvider.java index 4db2c869f3c..d462b3401ec 100644 --- a/assay/src/org/labkey/assay/plate/PlateMetricsProvider.java +++ b/assay/src/org/labkey/assay/plate/PlateMetricsProvider.java @@ -272,7 +272,7 @@ public UsageMetrics addUsageMetrics(UsageMetrics metrics) TableInfo hitTable = schema.getTableInfoHit(); TableInfo filterCriteriaTable = schema.getTableInfoFilterCriteria(); List plateEnabledProtocols = getPlateEnabledAssayProtocols(); - plateMetrics.put("assays", Map.of( + plateMetrics.put("assays", () -> Map.of( "hitCount", new SqlSelector(dbSchema, new SQLFragment("SELECT COUNT(*) FROM ").append(hitTable, "h")).getObject(Long.class), "plateSetsWithHits", new SqlSelector(dbSchema, new SQLFragment("SELECT COUNT(DISTINCT plateSetPath) FROM ").append(hitTable, "h")).getObject(Long.class), "assaysWithPlateMetadataEnabled", plateEnabledProtocols.size(), @@ -292,7 +292,7 @@ public UsageMetrics addUsageMetrics(UsageMetrics metrics) plateMetrics.put("wells", wells, "All wells, including those on templates and archived plates, and those with a sample"); } - plateMetrics.put("metadata", Map.of( + plateMetrics.put("metadata", () -> Map.of( "fieldsCount", getMetadataFieldsCount() ), "Custom plate metadata fields, summed across Biologics projects"); diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index 343e3329a27..fb0072c171f 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -1285,15 +1285,15 @@ public void moduleStartupComplete(ServletContext servletContext) } metrics.put("otherDeployedWebapps", StringUtils.join(deployedApps, ","), "Comma-separated names of other webapps deployed alongside LabKey"); - metrics.put("totalLogins", UserManager.getAuthCount(null, false, false, false), "Login events in the audit log; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE); - metrics.put("apiKeyLogins", UserManager.getAuthCount(null, false, true, false), "Login events in the audit log that used an API key; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE); + metrics.put("totalLogins", () -> UserManager.getAuthCount(null, false, false, false), "Login events in the audit log; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE); + metrics.put("apiKeyLogins", () -> UserManager.getAuthCount(null, false, true, false), "Login events in the audit log that used an API key; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE); metrics.put("sessionTimeout", ModuleLoader.getServletContext().getSessionTimeout(), "HTTP session timeout in minutes"); metrics.put("userLimits", new LimitActiveUsersSettings().getMetricsMap(), "Active user warning and limit settings: whether each is on, its user count threshold, and its message"); metrics.put("systemUserCount", UserManager.getSystemUserCount(), "Active users marked as system users, which don't count toward user limits"); Calendar cal = new GregorianCalendar(); cal.add(Calendar.DATE, -30); - metrics.put("uniqueRecentUserCount", UserManager.getAuthCount(cal.getTime(), false, false, true), "Distinct users who logged in during the last 30 days"); - metrics.put("uniqueRecentNonSystemUserCount", UserManager.getAuthCount(cal.getTime(), true, false, true), "Distinct non-system users who logged in during the last 30 days"); + metrics.put("uniqueRecentUserCount", () -> UserManager.getAuthCount(cal.getTime(), false, false, true), "Distinct users who logged in during the last 30 days"); + metrics.put("uniqueRecentNonSystemUserCount", () -> UserManager.getAuthCount(cal.getTime(), true, false, true), "Distinct non-system users who logged in during the last 30 days"); // Optionally include a list of active users, Issue #53050 metrics.reportedIf(OptionalFeatureService.get().isFeatureEnabled(FEATURE_FLAG_EXTENDED_METRICS)) .put("activeUsers", () -> UserManager.getActiveUsers().stream() @@ -1303,9 +1303,9 @@ public void moduleStartupComplete(ServletContext servletContext) "Emails of active non-system users; only reported when the extended metrics feature flag is on" ); - metrics.put("workbookCount", ContainerManager.getWorkbookCount(), "Workbooks across the server"); - metrics.put("archivedFolderCount", ContainerManager.getArchivedContainerCount(), "Archived folders across the server"); - metrics.put("databaseSize", CoreSchema.getInstance().getSchema().getScope().getDatabaseSize(), "Size of the LabKey database in bytes"); + metrics.put("workbookCount", () -> ContainerManager.getWorkbookCount(), "Workbooks across the server"); + metrics.put("archivedFolderCount", () -> ContainerManager.getArchivedContainerCount(), "Archived folders across the server"); + metrics.put("databaseSize", () -> CoreSchema.getInstance().getSchema().getScope().getDatabaseSize(), "Size of the LabKey database in bytes"); // Exclude temp schema to avoid PG exceptions when tables are appearing/disappearing during execution // Note that they can be non-trivial in size. @@ -1317,12 +1317,12 @@ public void moduleStartupComplete(ServletContext servletContext) metrics.put("databaseSchemaSize", schemaSizes, "Size in bytes of each database schema, including indexes; temp excluded"); metrics.put("scriptEngines", LabKeyScriptEngineManager.get().getScriptEngineMetrics(), "Configured script engines (R, Python, etc.) keyed by name, with factory and engine class, file extensions, and whether each is external and enabled"); - metrics.put("customLabels", CustomLabelService.get().getCustomLabelMetrics(), "Custom noun labels by label provider: count of relabeled nouns and of projects that override labels"); + metrics.put("customLabels", () -> CustomLabelService.get().getCustomLabelMetrics(), "Custom noun labels by label provider: count of relabeled nouns and of projects that override labels"); UsageMetricGroup roleAssignments = metrics.group("roleAssignments", "Direct assignments of designer roles to users or groups, by role, across all containers"); final String roleCountSql = "SELECT COUNT(*) FROM core.RoleAssignments WHERE userid > 0 AND role = ?"; - roleAssignments.put("assayDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.assay.security.AssayDesignerRole").getObject(Long.class)); - roleAssignments.put("dataClassDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.DataClassDesignerRole").getObject(Long.class)); - roleAssignments.put("sampleTypeDesignerCount", new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.SampleTypeDesignerRole").getObject(Long.class)); + roleAssignments.put("assayDesignerCount", () -> new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.assay.security.AssayDesignerRole").getObject(Long.class)); + roleAssignments.put("dataClassDesignerCount", () -> new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.DataClassDesignerRole").getObject(Long.class)); + roleAssignments.put("sampleTypeDesignerCount", () -> new SqlSelector(CoreSchema.getInstance().getSchema(), roleCountSql, "org.labkey.experiment.security.SampleTypeDesignerRole").getObject(Long.class)); UsageMetricGroup allowListCounts = metrics.group("allowListCounts", "Entries in each site allow list, such as external redirect hosts and file extensions"); for (AllowListType type : AllowListType.values()) diff --git a/core/src/org/labkey/core/admin/usageMetrics/UsageMetricsServiceImpl.java b/core/src/org/labkey/core/admin/usageMetrics/UsageMetricsServiceImpl.java index b98891e1286..2612440b14c 100644 --- a/core/src/org/labkey/core/admin/usageMetrics/UsageMetricsServiceImpl.java +++ b/core/src/org/labkey/core/admin/usageMetrics/UsageMetricsServiceImpl.java @@ -101,11 +101,11 @@ public Map> getModuleUsageMetricDescripti { try { - moduleDescriptions.addAll(provider.collectUsageMetrics().getDescriptions()); + moduleDescriptions.addAll(provider.getUsageMetricDescriptions()); } catch (Exception e) { - LOG.error("Failed to collect usage metric descriptions for module {}", moduleName, e); + LOG.error("A provider in module {} failed to collect usage metric descriptions", moduleName, e); } }); moduleDescriptions.sort(Comparator.comparing(UsageMetrics.Description::path, String.CASE_INSENSITIVE_ORDER)); diff --git a/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java b/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java index 3fd38955da3..c7929d8511d 100644 --- a/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java +++ b/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java @@ -15,6 +15,7 @@ */ package org.labkey.core.metrics; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.labkey.api.action.SpringActionController; @@ -34,6 +35,7 @@ import org.labkey.api.util.logging.LogHelper; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; @@ -59,6 +61,9 @@ public class SimpleMetricsServiceImpl implements SimpleMetricsService /** Feature area descriptions by module name, then feature area */ private final Map> _featureAreaDescriptions = new ConcurrentHashMap<>(); + /** "module.featureArea" keys already warned about lacking a description */ + private final Set _undescribedWarned = ConcurrentHashMap.newKeySet(); + /** Timestamp when we last saved the counts to the DB */ private Runnable _saver; private boolean _shutdownStarted = false; @@ -206,6 +211,9 @@ public long increment(@NotNull String requestedModuleName, @NotNull String featu Map> moduleMetrics = getModuleMetrics(module); String moduleName = module.getName(); // Use canonical name to ensure consistent casing + if (!_featureAreaDescriptions.getOrDefault(moduleName, Map.of()).containsKey(featureArea) && _undescribedWarned.add(moduleName + "." + featureArea)) + LOG.log(AppProps.getInstance().isDevMode() ? Level.ERROR : Level.WARN, "Simple metric feature area {}.{} has no description. Describe it via SimpleMetricsService.describeFeatureArea() or, for client-side metrics, ClientSideMetricDescriptions", moduleName, featureArea); + Map counts = moduleMetrics.computeIfAbsent(featureArea, k -> { // Queue an async, immediate save for the properties object. Doing it async helps avoid deadlocks diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index b0eec3686c3..3028b5d6349 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -615,7 +615,7 @@ public void containerDeleted(Container c, User user) // Run count across all assay designs of this type SQLFragment runSQL = new SQLFragment(baseRunSQL); runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); - protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); + protocolMetrics.put("runCount", () -> new SqlSelector(schema, runSQL).getObject(Long.class)); // Number of assay designs of this type SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); @@ -626,24 +626,24 @@ public void containerDeleted(Container c, User user) List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); - protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); + protocolMetrics.put("resultRowCount", () -> assayProvider.getResultRowCount(wrappedProtocols)); // Primary implementation class protocolMetrics.put("implementingClass", assayProvider.getClass()); } - assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class), "Assay designs with auto-link to a study configured; archived included"); - assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, + assayMetrics.put("autoLinkedAssayCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class), "Assay designs with auto-link to a study configured; archived included"); + assayMetrics.put("protocolsWithTransformScriptCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ?", AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, ExpProtocol.Status.Active.toString() ).getObject(Long.class), "Active assay designs with at least one transform script"); - assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, + assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, ExpProtocol.Status.Active.toString(), "%\"" + DataTransformService.TransformOperation.UPDATE + "\"%" ).getObject(Long.class), "Active assay designs with a transform script that runs on update"); - assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, + assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue LIKE ?", AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, ExpProtocol.Status.Active.toString(), @@ -655,14 +655,14 @@ public void containerDeleted(Container c, User user) // for cases that aren't counted: // 1. Any path containing "@scripts" matches, so a script in a sibling container's @scripts looks compliant here but is rejected on save. // 2. Undercounts designs with a MIX of compliant and non-compliant scripts - assayMetrics.put("protocolsWithTransformScriptNotInScriptsDirCount", new SqlSelector(schema, + assayMetrics.put("protocolsWithTransformScriptNotInScriptsDirCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = ? AND status = ? AND OP.stringvalue NOT LIKE ?", AbstractAssayProvider.TRANSFORM_SCRIPT_PROPERTY_NAME, ExpProtocol.Status.Active.toString(), "%" + FileContentService.SCRIPTS_LINK + "%" ).getObject(Long.class), "Active assay designs with no transform script path containing @scripts; approximate"); - assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class), "Assay designs with plate support enabled; archived included"); + assayMetrics.put("standardAssayWithPlateSupportCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class), "Assay designs with plate support enabled; archived included"); SQLFragment runsWithPlateSQL = new SQLFragment(""" SELECT COUNT(*) FROM exp.experimentrun r INNER JOIN exp.object o ON o.objectUri = r.lsid @@ -670,17 +670,17 @@ SELECT COUNT(*) FROM exp.experimentrun r WHERE op.propertyid IN ( SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? )"""); - assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class), "Assay runs with a plate template selected"); - assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class), "Assay runs with a plate set selected"); + assayMetrics.put("standardAssayRunsWithPlateTemplate", () -> new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class), "Assay runs with a plate template selected"); + assayMetrics.put("standardAssayRunsWithPlateSet", () -> new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class), "Assay runs with a plate set selected"); - assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ + assayMetrics.put("assayRunsFileColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class), "Assay designs whose run fields include a File field; counts designs, not fields"); - assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ + assayMetrics.put("assayResultsFileColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid @@ -688,7 +688,7 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class), "Assay designs whose result fields include a File field; counts designs, not fields"); // metric to count the number of Luminex and Standard assay runs that were imported with > 1 data file - assayMetrics.put("assayRunsWithMultipleInputFiles", new SqlSelector(schema, """ + assayMetrics.put("assayRunsWithMultipleInputFiles", () -> new SqlSelector(schema, """ SELECT COUNT(*) FROM ( SELECT sourceapplicationid, COUNT(*) AS count FROM exp.data WHERE lsid NOT LIKE '%:RelatedFile.%' AND sourceapplicationid IN ( @@ -703,15 +703,15 @@ SELECT sourceapplicationid, COUNT(*) AS count FROM exp.data SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); - sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); + sampleLookupCountMetrics.put("batchDomain", () -> new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); - sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); + sampleLookupCountMetrics.put("runDomain", () -> new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); + sampleLookupCountMetrics.put("resultDomain", () -> new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( """ @@ -725,18 +725,18 @@ SELECT PD.domainid, COUNT(*) AS PropCount ) X WHERE X.PropCount > 1""" ); resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); + sampleLookupCountMetrics.put("resultDomainWithMultiple", () -> new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) - metrics.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ + metrics.put("assayResultWithBooleanColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class), "Assay designs whose result fields include a Boolean field"); - metrics.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ + metrics.put("assayRunWithBooleanColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid @@ -744,8 +744,8 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class), "Assay designs whose run fields include a Boolean field"); } - metrics.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class), "Sample types with auto-link to a study configured"); - metrics.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class), "Sample types"); + metrics.put("autoLinkedSampleSetCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class), "Sample types with auto-link to a study configured"); + metrics.put("sampleSetCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class), "Sample types"); Collection> numSampleCounts = new SqlSelector(schema, """ SELECT totalCount, numberNameCount FROM @@ -758,15 +758,15 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM map -> (Long) map.get("totalCount") > 0 && map.get("totalCount").equals(map.get("numberNameCount")) ).count(), "Sample types whose sample names are all digits and periods"); - metrics.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class), "Samples, aliquots included, plus untyped materials"); - metrics.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class), "Aliquots"); - metrics.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class), "Samples and untyped materials with no stored amount; aliquots included"); - metrics.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class), "Samples and untyped materials with a negative stored amount"); - metrics.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class), "Samples whose units differ from their sample type's metric unit; null units on either side not counted"); - metrics.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class), "Uncategorized sample types with no metric unit; categorized types such as media excluded"); - metrics.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class), "Uncategorized sample types with a mass metric unit (kg, g, mg, ug, ng)"); - metrics.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class), "Uncategorized sample types with a volume metric unit (L, mL, uL)"); - metrics.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class), "Uncategorized sample types with the count metric unit 'unit'"); + metrics.put("sampleCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class), "Samples, aliquots included, plus untyped materials"); + metrics.put("aliquotCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class), "Aliquots"); + metrics.put("sampleNullAmountCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class), "Samples and untyped materials with no stored amount; aliquots included"); + metrics.put("sampleNegativeAmountCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class), "Samples and untyped materials with a negative stored amount"); + metrics.put("sampleUnitsDifferCount", () -> new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class), "Samples whose units differ from their sample type's metric unit; null units on either side not counted"); + metrics.put("sampleTypesWithoutUnitsCount", () -> new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class), "Uncategorized sample types with no metric unit; categorized types such as media excluded"); + metrics.put("sampleTypesWithMassTypeUnit", () -> new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class), "Uncategorized sample types with a mass metric unit (kg, g, mg, ug, ng)"); + metrics.put("sampleTypesWithVolumeTypeUnit", () -> new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class), "Uncategorized sample types with a volume metric unit (L, mL, uL)"); + metrics.put("sampleTypesWithCountTypeUnit", () -> new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class), "Uncategorized sample types with the count metric unit 'unit'"); UsageMetricGroup sampleColorMetrics = metrics.group("sampleColors", "Sample color usage; only colorCount is reported when no colors exist"); Long colorCount = new SqlSelector(schema, "SELECT COUNT(*) FROM exp.datacolors").getObject(Long.class); @@ -785,9 +785,9 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM "SELECT COUNT(DISTINCT datatyperowid) FROM exp.datatypecolorexclusion WHERE datatype = ?") .add(ExperimentService.DataTypeForExclusion.SampleType.name())).getObject(Long.class), "Sample types with at least one color excluded, not necessarily all"); - metrics.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + metrics.put("duplicateSampleMaterialNameCount", () -> new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class), "Names used by more than one sample within a sample type; counts names, not samples"); - metrics.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + metrics.put("duplicateSpecimenMaterialNameCount", () -> new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class), "Names used by more than one untyped material, such as specimens; counts names, not materials"); String duplicateCaseInsensitiveSampleNameCountSql = """ SELECT COUNT(*) FROM @@ -809,12 +809,12 @@ GROUP BY LOWER(name), classid HAVING COUNT(*) > 1 ) AS duplicates """; - metrics.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class), "Names used more than once within a sample type ignoring case, exact duplicates included; counts names, not samples"); - metrics.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class), "Names used more than once within a data class ignoring case, exact duplicates included; counts names, not rows"); + metrics.put("duplicateCaseInsensitiveSampleNameCount", () -> new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class), "Names used more than once within a sample type ignoring case, exact duplicates included; counts names, not samples"); + metrics.put("duplicateCaseInsensitiveDataNameCount", () -> new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class), "Names used more than once within a data class ignoring case, exact duplicates included; counts names, not rows"); - metrics.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class), "Data classes, including source types and Biologics registry and media types"); - metrics.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class), "Data class rows"); - metrics.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + + metrics.put("dataClassCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class), "Data classes, including source types and Biologics registry and media types"); + metrics.put("dataClassRowCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class), "Data class rows"); + metrics.put("dataWithDataParentsCount", () -> new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class), "Protocol steps that both produced data and consumed data inputs; approximates derivations from data parents, counting steps, not data"); Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ @@ -827,108 +827,108 @@ HAVING COUNT(*) > 1 metrics.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> (Long) map.get("totalCount") > 0 && map.get("totalCount").equals(map.get("numberNameCount"))).count(), "Data types whose names are all digits and periods; includes non-data-class data such as files"); - metrics.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class), "Fields with a principal ontology concept code"); - metrics.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class), "Ontology lookup fields"); - metrics.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class), "Fields restricted to an ontology concept subtree"); - metrics.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class), "Fields with an ontology concept import field set"); - metrics.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class), "Fields with an ontology concept label field set"); + metrics.put("ontologyPrincipalConceptCodeCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class), "Fields with a principal ontology concept code"); + metrics.put("ontologyLookupColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class), "Ontology lookup fields"); + metrics.put("ontologyConceptSubtreeCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class), "Fields restricted to an ontology concept subtree"); + metrics.put("ontologyConceptImportColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class), "Fields with an ontology concept import field set"); + metrics.put("ontologyConceptLabelColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class), "Fields with an ontology concept label field set"); - metrics.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class), "Fields marked scannable"); - metrics.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class), "Unique ID fields"); - metrics.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ + metrics.put("scannableColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class), "Fields marked scannable"); + metrics.put("uniqueIdColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class), "Unique ID fields"); + metrics.put("sampleTypeWithUniqueIdCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class), "Domains with a Unique ID field; in practice sample types"); - metrics.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class), "File fields across all domains"); - metrics.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ + metrics.put("fileColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class), "File fields across all domains"); + metrics.put("sampleTypeWithFileColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class), "Sample types with at least one File field"); - metrics.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ + metrics.put("sampleTypeWithBooleanColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class), "Sample types with at least one Boolean field"); - metrics.put("sampleTypeWithMultiValueColumnCount", new SqlSelector(schema, """ + metrics.put("sampleTypeWithMultiValueColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class), "Sample types with at least one multi-value Text Choice field"); - metrics.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ + metrics.put("sampleTypeAliquotSpecificField", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT D.PropertyURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class), "Sample type fields that apply only to aliquots"); - metrics.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ + metrics.put("sampleTypeParentOnlyField", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT D.PropertyURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class), "Sample type fields that apply only to parent samples, including fields with no aliquot setting"); - metrics.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ + metrics.put("sampleTypeParentAndAliquotField", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT D.PropertyURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class), "Sample type fields that apply to both parent samples and aliquots"); - metrics.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class), "Attachment fields across all domains"); - metrics.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ + metrics.put("attachmentColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class), "Attachment fields across all domains"); + metrics.put("dataClassWithAttachmentColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class), "Data classes with at least one Attachment field"); - metrics.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ + metrics.put("dataClassWithBooleanColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class), "Data classes with at least one Boolean field"); - metrics.put("dataClassWithMultiValueColumnCount", new SqlSelector(schema, """ + metrics.put("dataClassWithMultiValueColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class), "Data classes with at least one multi-value Text Choice field"); - metrics.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class), "Single-value Text Choice fields"); - metrics.put("multiValueTextChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class), "Multi-value Text Choice fields"); + metrics.put("textChoiceColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class), "Single-value Text Choice fields"); + metrics.put("multiValueTextChoiceColumnCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class), "Multi-value Text Choice fields"); - metrics.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ + metrics.put("domainsWithDateTimeColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class), "Domains of any kind with a DateTime field"); - metrics.put("domainsWithDateColumnCount", new SqlSelector(schema, """ + metrics.put("domainsWithDateColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class), "Domains of any kind with a Date field"); - metrics.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ + metrics.put("domainsWithTimeColumnCount", () -> new SqlSelector(schema, """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class), "Domains of any kind with a Time field"); - metrics.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class), "Highest exp.Object id; proxy for experiment objects ever created", UsageMetrics.Kind.CUMULATIVE); - metrics.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class), "Highest material row id; proxy for samples ever created", UsageMetrics.Kind.CUMULATIVE); + metrics.put("maxObjectObjectId", () -> new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class), "Highest exp.Object id; proxy for experiment objects ever created", UsageMetrics.Kind.CUMULATIVE); + metrics.put("maxMaterialRowId", () -> new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class), "Highest material row id; proxy for samples ever created", UsageMetrics.Kind.CUMULATIVE); - metrics.put("domainFieldsWithContainerAlias", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE LOWER(importaliases) = 'container' OR importaliases ILIKE '%, container' OR importaliases ILIKE 'container, %' OR importaliases ILIKE '%, container, %'").getObject(Long.class), "Fields with 'container' as an import alias"); + metrics.put("domainFieldsWithContainerAlias", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE LOWER(importaliases) = 'container' OR importaliases ILIKE '%, container' OR importaliases ILIKE 'container, %' OR importaliases ILIKE '%, container, %'").getObject(Long.class), "Fields with 'container' as an import alias"); return ExperimentService.get().addDomainMetrics(metrics); }); diff --git a/experiment/src/org/labkey/experiment/api/property/PropertyServiceImpl.java b/experiment/src/org/labkey/experiment/api/property/PropertyServiceImpl.java index 2b50f3484be..01cf5b66928 100644 --- a/experiment/src/org/labkey/experiment/api/property/PropertyServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/property/PropertyServiceImpl.java @@ -704,13 +704,13 @@ public UsageMetrics addUsageMetrics(UsageMetrics metrics) long storageColumnNameMismatches = new TableSelector(t, new SimpleFilter(FieldKey.fromParts("StorageColumnNameMatch"), false), null).getRowCount(); metrics.group("propertyValidators", "Field validators") - .put("byType", new SqlSelector(schema, + .put("byType", () -> new SqlSelector(schema, new SQLFragment("SELECT typeuri, COUNT(*) AS count FROM exp.propertyvalidator GROUP BY typeuri") ).getMapCollection().stream().reduce(new HashMap<>(), (x, m) -> { x.put(m.get("typeuri").toString(), m.get("count")); return x; }), "Validators by type URI; a Text Choice field's choices are one validator") - .put("textChoiceValues", new SqlSelector(schema, + .put("textChoiceValues", () -> new SqlSelector(schema, new SQLFragment("SELECT MIN(NumValues) AS min, MAX(NumValues) AS max, AVG(NumValues) AS avg FROM (") .append(String.format("SELECT (%s(expression) - %s(replace(expression, '|', '')) + 1) AS NumValues FROM exp.propertyvalidator WHERE typeuri = ?", lengthFn, lengthFn)) .unsafeAppend(") AS X") @@ -730,9 +730,9 @@ public UsageMetrics addUsageMetrics(UsageMetrics metrics) new SQLFragment("SELECT CASE WHEN ConceptURI IS NULL THEN 'null' ELSE ConceptURI END, COUNT(*) AS Count FROM exp.PropertyDescriptor GROUP BY ConceptURI") ).getValueMap(String.class))); metrics.group("defaultValuePropertyCounts", "Fields with a saved folder- or user-level default value, by default value type and by owning domain's LSID namespace; counts fields, not saved values") - .put("folder", savedDefaultValuePropertyCounts(schema, "%:" + DefaultValueService.DOMAIN_DEFAULT_VALUE_LSID_PREFIX + "%.Folder-%")) - .put("user", savedDefaultValuePropertyCounts(schema, "%:" + DefaultValueService.USER_DEFAULT_VALUE_LSID_PREFIX + ".Folder-%")); - metrics.put("conditionalFormattingFields", new SqlSelector(schema, new SQLFragment("SELECT COUNT (DISTINCT propertyid) from exp.conditionalformat")).getObject(Long.class), "Fields with conditional formatting"); + .put("folder", () -> savedDefaultValuePropertyCounts(schema, "%:" + DefaultValueService.DOMAIN_DEFAULT_VALUE_LSID_PREFIX + "%.Folder-%")) + .put("user", () -> savedDefaultValuePropertyCounts(schema, "%:" + DefaultValueService.USER_DEFAULT_VALUE_LSID_PREFIX + ".Folder-%")); + metrics.put("conditionalFormattingFields", () -> new SqlSelector(schema, new SQLFragment("SELECT COUNT (DISTINCT propertyid) from exp.conditionalformat")).getObject(Long.class), "Fields with conditional formatting"); metrics.put("storageColumnNameMismatches", storageColumnNameMismatches, "Fields in provisioned domains whose database column name differs from the field name"); return metrics; } diff --git a/issues/src/org/labkey/issue/IssuesModule.java b/issues/src/org/labkey/issue/IssuesModule.java index f7aad8c302a..7bcb07e0edb 100644 --- a/issues/src/org/labkey/issue/IssuesModule.java +++ b/issues/src/org/labkey/issue/IssuesModule.java @@ -152,8 +152,8 @@ public void doStartup(ModuleContext moduleContext) if (svc != null) { svc.registerUsageMetrics(getName(), metrics -> { - metrics.put("issueDefCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issueListDef").getObject(Long.class), "Issue list definitions across all containers"); - metrics.put("issuesCount", new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issues").getObject(Long.class), "Issues across all containers and issue lists, open and closed"); + metrics.put("issueDefCount", () -> new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issueListDef").getObject(Long.class), "Issue list definitions across all containers"); + metrics.put("issuesCount", () -> new SqlSelector(IssuesSchema.getInstance().getSchema(), "SELECT COUNT(*) FROM issues.issues").getObject(Long.class), "Issues across all containers and issue lists, open and closed"); return metrics; }); diff --git a/list/src/org/labkey/list/ListModule.java b/list/src/org/labkey/list/ListModule.java index c7ae728cecb..21e4d7d4b51 100644 --- a/list/src/org/labkey/list/ListModule.java +++ b/list/src/org/labkey/list/ListModule.java @@ -155,9 +155,9 @@ public void startupAfterSpringConfig(ModuleContext moduleContext) { svc.registerUsageMetrics(getName(), metrics -> { DbSchema dbSchema = DbSchema.get("exp", DbSchemaType.Module); - metrics.put("listCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list").getObject(Long.class), "List definitions across all containers, including picklists"); - metrics.put("publicPicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PublicPicklist + "'").getObject(Long.class), "Shared (public) picklists across all containers"); - metrics.put("privatePicklistCount", new SqlSelector(dbSchema, "SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PrivatePicklist + "'").getObject(Long.class), "Private picklists across all containers"); + metrics.put("listCount", () -> new SqlSelector(dbSchema,"SELECT COUNT(*) FROM exp.list").getObject(Long.class), "List definitions across all containers, including picklists"); + metrics.put("publicPicklistCount", () -> new SqlSelector(dbSchema,"SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PublicPicklist + "'").getObject(Long.class), "Shared (public) picklists across all containers"); + metrics.put("privatePicklistCount", () -> new SqlSelector(dbSchema,"SELECT COUNT(*) FROM exp.list WHERE category ='" + ListDefinition.Category.PrivatePicklist + "'").getObject(Long.class), "Private picklists across all containers"); return metrics; }); } diff --git a/pipeline/src/org/labkey/pipeline/PipelineModule.java b/pipeline/src/org/labkey/pipeline/PipelineModule.java index cba565e5278..26a3f3c6446 100644 --- a/pipeline/src/org/labkey/pipeline/PipelineModule.java +++ b/pipeline/src/org/labkey/pipeline/PipelineModule.java @@ -263,8 +263,8 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) metrics.put("triggerCounts", triggerCounts, "Pipeline trigger configurations across all containers, by task pipeline: Enabled, Disabled, and Jobs run by that pipeline; computed over config-job join rows, so Enabled/Disabled are inflated when jobs exist"); metrics.put("jmsType", PipelineService.get().getJmsType().toString(), "JMS queue type for pipeline jobs: none, inProcess, external, or unknown"); - metrics.put("pipelineRootCount", PipelineService.get().getAllPipelineRoots().size(), "Containers with an explicitly configured primary pipeline root"); - metrics.put("supplementalDirectories", new TableSelector(PipelineSchema.getInstance().getTableInfoPipelineRoots(), new SimpleFilter("SupplementalPath", null, CompareType.NONBLANK), null).getRowCount(), "Configured pipeline roots that have a supplemental directory"); + metrics.put("pipelineRootCount", () -> PipelineService.get().getAllPipelineRoots().size(), "Containers with an explicitly configured primary pipeline root"); + metrics.put("supplementalDirectories", () -> new TableSelector(PipelineSchema.getInstance().getTableInfoPipelineRoots(), new SimpleFilter("SupplementalPath", null, CompareType.NONBLANK), null).getRowCount(), "Configured pipeline roots that have a supplemental directory"); return metrics; }); diff --git a/query/src/org/labkey/query/persist/QueryManager.java b/query/src/org/labkey/query/persist/QueryManager.java index 3d0f96cf920..af683325a40 100644 --- a/query/src/org/labkey/query/persist/QueryManager.java +++ b/query/src/org/labkey/query/persist/QueryManager.java @@ -1066,7 +1066,7 @@ public static void registerUsageMetrics(String moduleName) metrics.put("externalDatasources", statsMap, "External (non-LabKey) data sources by database product name"); metrics.put( "customViewCounts", - Map.of( + () -> Map.of( "DataClasses", getSchemaCustomViewCounts("exp.data"), "SampleTypes", getSchemaCustomViewCounts("samples"), "Assays", getSchemaCustomViewCounts("assay"), @@ -1074,8 +1074,8 @@ public static void registerUsageMetrics(String moduleName) ), "Saved custom views across all containers by schema area (DataClasses, SampleTypes, Assays, Inventory): defaultOverrides (unnamed) and namedViews exclude hidden and snapshot views, inheritable likewise, shared (no owner) and identifyingFieldsViews count all" ); - metrics.put("customViewWithLineageColumn", getLineageCustomViewMetrics(), "Custom views with lineage (input, output, ancestor) columns, hidden and snapshot views excluded: view counts, plus total and min/percentile/max/avg columns per view when any exist"); - metrics.put("queryDefWithCalculatedFieldsCounts", getCalculatedFieldsCountsMetric(), "Query definitions whose metadata defines a calculated field, by schema (assay.* schemas grouped as assay)"); + metrics.put("customViewWithLineageColumn", () -> getLineageCustomViewMetrics(), "Custom views with lineage (input, output, ancestor) columns, hidden and snapshot views excluded: view counts, plus total and min/percentile/max/avg columns per view when any exist"); + metrics.put("queryDefWithCalculatedFieldsCounts", () -> getCalculatedFieldsCountsMetric(), "Query definitions whose metadata defines a calculated field, by schema (assay.* schemas grouped as assay)"); Map exportTypeCounts = getExportTypeCountsMetric(); if (exportTypeCounts != null) diff --git a/study/src/org/labkey/study/StudyModule.java b/study/src/org/labkey/study/StudyModule.java index fab9f49c309..8ab0f0eb5ce 100644 --- a/study/src/org/labkey/study/StudyModule.java +++ b/study/src/org/labkey/study/StudyModule.java @@ -426,38 +426,38 @@ protected void startupAfterSpringConfig(ModuleContext moduleContext) if (null != svc) { svc.registerUsageMetrics(MODULE_NAME, metrics -> { - metrics.put("studyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.study").getObject(Long.class), "Studies across the server"); - metrics.put("datasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.dataset").getObject(Long.class), "Datasets across all studies, all dataset types included"); + metrics.put("studyCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.study").getObject(Long.class), "Studies across the server"); + metrics.put("datasetCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.dataset").getObject(Long.class), "Datasets across all studies, all dataset types included"); - metrics.put("securityType", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT SecurityType, COUNT(*) FROM study.Study GROUP BY SecurityType").getValueMap(), "Studies by security type (BASIC_READ, BASIC_WRITE, ADVANCED_READ, ADVANCED_WRITE)"); - metrics.put("timepointType", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT TimepointType, COUNT(*) FROM study.Study GROUP BY TimepointType").getValueMap(), "Studies by timepoint type (VISIT, DATE, CONTINUOUS)"); + metrics.put("securityType", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT SecurityType, COUNT(*) FROM study.Study GROUP BY SecurityType").getValueMap(), "Studies by security type (BASIC_READ, BASIC_WRITE, ADVANCED_READ, ADVANCED_WRITE)"); + metrics.put("timepointType", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT TimepointType, COUNT(*) FROM study.Study GROUP BY TimepointType").getValueMap(), "Studies by timepoint type (VISIT, DATE, CONTINUOUS)"); - metrics.put("linkedSampleTypeDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(PublishSourceType) FROM study.dataset WHERE PublishSourceType = 'SampleType'").getObject(Long.class), "Datasets linked from sample types; same value as datasetsLinkedFromSamples"); - metrics.put("linkedAssayDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(PublishSourceType) FROM study.dataset WHERE PublishSourceType = 'Assay'").getObject(Long.class), "Datasets linked from assay results; same value as datasetsLinkedFromAssays"); + metrics.put("linkedSampleTypeDatasetCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(PublishSourceType) FROM study.dataset WHERE PublishSourceType = 'SampleType'").getObject(Long.class), "Datasets linked from sample types; same value as datasetsLinkedFromSamples"); + metrics.put("linkedAssayDatasetCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(PublishSourceType) FROM study.dataset WHERE PublishSourceType = 'Assay'").getObject(Long.class), "Datasets linked from assay results; same value as datasetsLinkedFromAssays"); - metrics.put("redcapCount", new SqlSelector(PropertySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM prop.PropertySets WHERE Category = 'RedcapConfigurationSettings'").getObject(Long.class), "Saved REDCap configuration property sets across all containers"); + metrics.put("redcapCount", () -> new SqlSelector(PropertySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM prop.PropertySets WHERE Category = 'RedcapConfigurationSettings'").getObject(Long.class), "Saved REDCap configuration property sets across all containers"); // Note: study.StudySnapshot maintains a rows for deleted child studies (allowing re-creation of those child studies). // In that case, Destination is set to null. COUNT(DISTINCT) skips these null values, so this returns the count of existing child studies. - metrics.put("publishStudyCount", new SqlSelector(PropertySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT destination) FROM study.StudySnapshot WHERE Type = 'publish'").getObject(Long.class), "Existing child studies created by publishing a study; deleted children excluded"); + metrics.put("publishStudyCount", () -> new SqlSelector(PropertySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT destination) FROM study.StudySnapshot WHERE Type = 'publish'").getObject(Long.class), "Existing child studies created by publishing a study; deleted children excluded"); SqlDialect dialect = StudySchema.getInstance().getSqlDialect(); - metrics.put("demographicsDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE DemographicData = " + dialect.getBooleanTRUE()).getObject(Long.class), "Demographics datasets across all studies"); - metrics.put("standardDatasetCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE Type = 'Standard'").getObject(Long.class), "Datasets of type Standard across all studies; placeholder datasets excluded"); + metrics.put("demographicsDatasetCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE DemographicData = " + dialect.getBooleanTRUE()).getObject(Long.class), "Demographics datasets across all studies"); + metrics.put("standardDatasetCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE Type = 'Standard'").getObject(Long.class), "Datasets of type Standard across all studies; placeholder datasets excluded"); - metrics.put("managedThirdKeyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE KeyManagementType <> 'None'").getObject(Long.class), "Datasets whose additional key is server-managed (RowId or GUID)"); - metrics.put("thirdKeyCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE (KeyManagementType = 'None' AND KeyPropertyName IS NOT NULL) OR (UseTimeKeyField = " + dialect.getBooleanTRUE() + ")").getObject(Long.class), "Datasets with a user-managed additional key field or using time as the additional key"); + metrics.put("managedThirdKeyCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE KeyManagementType <> 'None'").getObject(Long.class), "Datasets whose additional key is server-managed (RowId or GUID)"); + metrics.put("thirdKeyCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE (KeyManagementType = 'None' AND KeyPropertyName IS NOT NULL) OR (UseTimeKeyField = " + dialect.getBooleanTRUE() + ")").getObject(Long.class), "Datasets with a user-managed additional key field or using time as the additional key"); - metrics.put("datasetsLinkedFromAssays", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE PublishSourceType = 'Assay'").getObject(Long.class), "Datasets linked from assay results; same value as linkedAssayDatasetCount"); - metrics.put("datasetsLinkedFromSamples", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE PublishSourceType = 'SampleType'").getObject(Long.class), "Datasets linked from sample types; same value as linkedSampleTypeDatasetCount"); + metrics.put("datasetsLinkedFromAssays", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE PublishSourceType = 'Assay'").getObject(Long.class), "Datasets linked from assay results; same value as linkedAssayDatasetCount"); + metrics.put("datasetsLinkedFromSamples", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Dataset WHERE PublishSourceType = 'SampleType'").getObject(Long.class), "Datasets linked from sample types; same value as linkedSampleTypeDatasetCount"); - metrics.put("assayScheduleCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT container) FROM study.AssaySpecimen").getObject(Long.class), "Containers with at least one assay schedule entry"); + metrics.put("assayScheduleCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT container) FROM study.AssaySpecimen").getObject(Long.class), "Containers with at least one assay schedule entry"); - metrics.put("alternateParticipantIdCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Study WHERE AlternateIdPrefix IS NOT NULL").getObject(Long.class), "Studies with an alternate participant ID prefix configured"); - metrics.put("participantIdMappingCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT container) FROM study.Participant WHERE AlternateId IS NOT NULL").getObject(Long.class), "Containers with at least one participant assigned an alternate ID"); - metrics.put("participantAliasCount", new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Study WHERE ParticipantAliasDatasetId IS NOT NULL").getObject(Long.class), "Studies with a participant alias dataset configured"); + metrics.put("alternateParticipantIdCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Study WHERE AlternateIdPrefix IS NOT NULL").getObject(Long.class), "Studies with an alternate participant ID prefix configured"); + metrics.put("participantIdMappingCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(DISTINCT container) FROM study.Participant WHERE AlternateId IS NOT NULL").getObject(Long.class), "Containers with at least one participant assigned an alternate ID"); + metrics.put("participantAliasCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), "SELECT COUNT(*) FROM study.Study WHERE ParticipantAliasDatasetId IS NOT NULL").getObject(Long.class), "Studies with a participant alias dataset configured"); - metrics.put("datasetsWithFileColumnCount", new SqlSelector(StudySchema.getInstance().getSchema(), """ + metrics.put("datasetsWithFileColumnCount", () -> new SqlSelector(StudySchema.getInstance().getSchema(), """ SELECT COUNT(DISTINCT DD.DomainURI) FROM exp.PropertyDescriptor D\s JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid diff --git a/survey/src/org/labkey/survey/SurveyModule.java b/survey/src/org/labkey/survey/SurveyModule.java index a8a53f54634..6c7eb35693f 100644 --- a/survey/src/org/labkey/survey/SurveyModule.java +++ b/survey/src/org/labkey/survey/SurveyModule.java @@ -131,8 +131,8 @@ public void doStartup(ModuleContext moduleContext) if (null != svc) { svc.registerUsageMetrics(NAME, metrics -> { - metrics.put("surveyDesigns", new SqlSelector(DbSchema.get("survey", DbSchemaType.Module), "SELECT COUNT(*) FROM survey.SurveyDesigns").getObject(Long.class), "Survey designs across all containers"); - metrics.put("surveys", new SqlSelector(DbSchema.get("survey", DbSchemaType.Module), "SELECT COUNT(*) FROM survey.Surveys").getObject(Long.class), "Survey responses across all containers, submitted or in progress"); + metrics.put("surveyDesigns", () -> new SqlSelector(DbSchema.get("survey", DbSchemaType.Module), "SELECT COUNT(*) FROM survey.SurveyDesigns").getObject(Long.class), "Survey designs across all containers"); + metrics.put("surveys", () -> new SqlSelector(DbSchema.get("survey", DbSchemaType.Module), "SELECT COUNT(*) FROM survey.Surveys").getObject(Long.class), "Survey responses across all containers, submitted or in progress"); return metrics; }); } From 456a15998e0cbfeadd67c5a97a98beb3aab3ed85 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Tue, 27 Oct 2026 09:46:16 -0700 Subject: [PATCH 4/5] Register more descriptions; defer logging until all registrations are done --- .../metrics/ClientSideMetricDescriptions.java | 12 +---- .../metrics/SimpleMetricsServiceImpl.java | 50 +++++++++++++++++-- query/src/org/labkey/query/QueryModule.java | 1 + 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/core/src/org/labkey/core/metrics/ClientSideMetricDescriptions.java b/core/src/org/labkey/core/metrics/ClientSideMetricDescriptions.java index 3073f5958a4..6e8a3c56be9 100644 --- a/core/src/org/labkey/core/metrics/ClientSideMetricDescriptions.java +++ b/core/src/org/labkey/core/metrics/ClientSideMetricDescriptions.java @@ -17,7 +17,7 @@ import org.labkey.api.usageMetrics.SimpleMetricsService; -/** Describes the client-side feature areas that @labkey/components counts */ +/** Describes the generic client-side feature areas that @labkey/components counts; modules describe their own product-specific areas */ public final class ClientSideMetricDescriptions { private ClientSideMetricDescriptions() @@ -30,19 +30,11 @@ public static void register() describe("grid", "Grid row selection using shift-click (shiftSelect)"); describe("customViews", "Grid columns reordered by drag and drop (columnReorderDragNDrop)"); describe("export", "Grid exports by format and header type: csv-caption, excel-caption, tsv-caption, csv-importfield, excel-importfield, tsv-importfield"); - describe("BarTender", "BarTender label template downloads (DownloadTemplateCount)"); describe("QueryModel", "Grid loads recovered from a failure: ViewDoesNotExist (fell back to the default view), CalculatedFieldError (calculated fields omitted)"); - describe("expressionAssistant", "Calculated field expression assistant use: clickButton, submitPrompt, applyExpression, clickHelpWithValidate"); describe("applicationNavigation", "Product menu navigation: toHomeProject, toServerContainer, toServerContainerTab, and toLimsSection, toSampleManagerSection, toBiologicsSection, toApplicationSection"); - describe("releaseNotes", "Release notes opened, by entry point: FromHelpMenu, FromBanner"); - describe("helpLink", "Help link clicks from the user menu (clickCount)"); - describe("SampleManagerSearch", "Sample Manager searches (count) and result clicks (resultClicked, plus firstResultClicked for the top result)"); - describe("BiologicsSearch", "Biologics searches (count) and result clicks (resultClicked, plus firstResultClicked for the top result)"); - describe("LIMSSearch", "LIMS searches (count) and result clicks (resultClicked, plus firstResultClicked for the top result)"); + describe("gridSearch", "Grid search box use: engaged (first typing), suggestions chosen by kind (filterChosen, composeChosen, searchChosen), filter suggestions by source (exactValue, identity, identityContains, integerValue, key, partialMatch, partialValue, shape, shapeContains), and Name filters by data type (sampleNameChosen, dataClassNameChosen)"); describe("gridInsertCounts", "Rows inserted from editable grids and assay imports, by data type and row-count bucket: 1To50, 51To100, 101To250, GT250"); describe("gridUpdateCounts", "Rows updated from editable grids, by data type and row-count bucket: 1To50, 51To100, 101To250, GT250"); - describe("findSamplesById", "Find Samples by ID or barcode: searches submitted (findCount), plus actions taken from its results grid (e.g. addSamplesToPicklist, sampleFinderFindDerivatives)"); - describe("sampleFinder", "Sample Finder use: headerMenuNavigation, dashboardButtonNavigation, filterModalApply, filterModalError, goToChooseValuesTab, hasWithoutAssayResultChecked, assay design counts (with AssayDesgin(s)), plus actions taken from its results grid (e.g. addSamplesToPicklist)"); } private static void describe(String featureArea, String description) diff --git a/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java b/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java index c7929d8511d..20448056973 100644 --- a/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java +++ b/core/src/org/labkey/core/metrics/SimpleMetricsServiceImpl.java @@ -15,6 +15,7 @@ */ package org.labkey.core.metrics; +import jakarta.servlet.ServletContext; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; @@ -32,6 +33,7 @@ import org.labkey.api.util.ContextListener; import org.labkey.api.util.JobRunner; import org.labkey.api.util.ShutdownListener; +import org.labkey.api.util.StartupListener; import org.labkey.api.util.logging.LogHelper; import java.util.Map; @@ -61,8 +63,11 @@ public class SimpleMetricsServiceImpl implements SimpleMetricsService /** Feature area descriptions by module name, then feature area */ private final Map> _featureAreaDescriptions = new ConcurrentHashMap<>(); - /** "module.featureArea" keys already warned about lacking a description */ - private final Set _undescribedWarned = ConcurrentHashMap.newKeySet(); + /** Feature areas counted without a description, already warned about or held until startup completes; also guards _startupComplete */ + private final Set _undescribed = ConcurrentHashMap.newKeySet(); + private boolean _startupComplete = ModuleLoader.getInstance().isStartupComplete(); + + private record FeatureArea(String moduleName, String featureArea) {} /** Timestamp when we last saved the counts to the DB */ private Runnable _saver; @@ -88,6 +93,36 @@ public void shutdownStarted() save(); } }); + // Areas can be counted before their module's startup() describes them, so judge only once every module has started + ContextListener.addStartupListener(new StartupListener() + { + @Override + public String getName() + { + return "Simple Metrics Service"; + } + + @Override + public void moduleStartupComplete(ServletContext servletContext) + { + synchronized (_undescribed) + { + _startupComplete = true; + _undescribed.removeIf(area -> isDescribed(area.moduleName(), area.featureArea())); + _undescribed.forEach(SimpleMetricsServiceImpl::logUndescribed); + } + } + }); + } + + private boolean isDescribed(String moduleName, String featureArea) + { + return _featureAreaDescriptions.getOrDefault(moduleName, Map.of()).containsKey(featureArea); + } + + private static void logUndescribed(FeatureArea area) + { + LOG.log(AppProps.getInstance().isDevMode() ? Level.ERROR : Level.WARN, "Simple metric feature area {}.{} has no description. Describe it via SimpleMetricsService.describeFeatureArea() or, for client-side metrics, ClientSideMetricDescriptions", area.moduleName(), area.featureArea()); } private class SimpleMetricsProvider implements UsageMetricsProvider @@ -211,8 +246,15 @@ public long increment(@NotNull String requestedModuleName, @NotNull String featu Map> moduleMetrics = getModuleMetrics(module); String moduleName = module.getName(); // Use canonical name to ensure consistent casing - if (!_featureAreaDescriptions.getOrDefault(moduleName, Map.of()).containsKey(featureArea) && _undescribedWarned.add(moduleName + "." + featureArea)) - LOG.log(AppProps.getInstance().isDevMode() ? Level.ERROR : Level.WARN, "Simple metric feature area {}.{} has no description. Describe it via SimpleMetricsService.describeFeatureArea() or, for client-side metrics, ClientSideMetricDescriptions", moduleName, featureArea); + FeatureArea area = new FeatureArea(moduleName, featureArea); + if (!isDescribed(moduleName, featureArea) && !_undescribed.contains(area)) + { + synchronized (_undescribed) + { + if (_undescribed.add(area) && _startupComplete) + logUndescribed(area); + } + } Map counts = moduleMetrics.computeIfAbsent(featureArea, k -> { diff --git a/query/src/org/labkey/query/QueryModule.java b/query/src/org/labkey/query/QueryModule.java index 33f614af137..1644a2de546 100644 --- a/query/src/org/labkey/query/QueryModule.java +++ b/query/src/org/labkey/query/QueryModule.java @@ -280,6 +280,7 @@ public boolean hasScripts() public void doStartup(ModuleContext moduleContext) { SimpleMetricsService.get().describeFeatureAreas(QueryMetricArea.values()); + SimpleMetricsService.get().describeClientSideFeatureArea("expressionAssistant", "Calculated field expression assistant use: clickButton, submitPrompt, applyExpression, clickHelpWithValidate"); ContainerManager.addContainerListener(QueryManager.CONTAINER_LISTENER, ContainerManager.ContainerListener.Order.Last); From a1908c26fd366660e6a96e4805500bf2b64de481 Mon Sep 17 00:00:00 2001 From: labkey-jeckels Date: Tue, 27 Oct 2026 10:58:46 -0700 Subject: [PATCH 5/5] Claude review --- core/src/org/labkey/core/CoreModule.java | 3 +- .../labkey/experiment/ExperimentModule.java | 9 +- .../labkey/query/persist/QueryManager.java | 4 +- .../src/org/labkey/search/SearchModule.java | 8 +- study/src/org/labkey/study/StudyModule.java | 84 ++++++++++--------- 5 files changed, 54 insertions(+), 54 deletions(-) diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index fb0072c171f..d36729785a5 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -1313,8 +1313,7 @@ public void moduleStartupComplete(ServletContext servletContext) sql.append(new PostgresTableSizesTable(new PostgresUserSchema(User.getAdminServiceUser(), ContainerManager.getRoot())), "t"); sql.unsafeAppend(" WHERE table_schema != 'temp' GROUP BY table_schema"); - var schemaSizes = new SqlSelector(CoreSchema.getInstance().getSchema(), sql).getValueMap(); - metrics.put("databaseSchemaSize", schemaSizes, "Size in bytes of each database schema, including indexes; temp excluded"); + metrics.put("databaseSchemaSize", () -> new SqlSelector(CoreSchema.getInstance().getSchema(), sql).getValueMap(), "Size in bytes of each database schema, including indexes; temp excluded"); metrics.put("scriptEngines", LabKeyScriptEngineManager.get().getScriptEngineMetrics(), "Configured script engines (R, Python, etc.) keyed by name, with factory and engine class, file extensions, and whether each is external and enabled"); metrics.put("customLabels", () -> CustomLabelService.get().getCustomLabelMetrics(), "Custom noun labels by label provider: count of relabeled nouns and of projects that override labels"); diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 3028b5d6349..559b913b484 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -15,6 +15,7 @@ */ package org.labkey.experiment; +import com.google.common.base.Suppliers; import org.apache.commons.lang3.math.NumberUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -747,14 +748,14 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM metrics.put("autoLinkedSampleSetCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class), "Sample types with auto-link to a study configured"); metrics.put("sampleSetCount", () -> new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class), "Sample types"); - Collection> numSampleCounts = new SqlSelector(schema, """ + Supplier>> numSampleCounts = Suppliers.memoize(() -> new SqlSelector(schema, """ SELECT totalCount, numberNameCount FROM (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t JOIN (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - metrics.put("sampleSetWithNumberNamesCount", numSampleCounts.size(), "Sample types with at least one sample named only with digits and periods; untyped materials count as one more type"); - metrics.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( + ON t.cpastype = ns.cpastype""").getMapCollection()); + metrics.put("sampleSetWithNumberNamesCount", () -> numSampleCounts.get().size(), "Sample types with at least one sample named only with digits and periods; untyped materials count as one more type"); + metrics.put("sampleSetWithOnlyNumberNamesCount", () -> numSampleCounts.get().stream().filter( map -> (Long) map.get("totalCount") > 0 && map.get("totalCount").equals(map.get("numberNameCount")) ).count(), "Sample types whose sample names are all digits and periods"); diff --git a/query/src/org/labkey/query/persist/QueryManager.java b/query/src/org/labkey/query/persist/QueryManager.java index af683325a40..e077814e667 100644 --- a/query/src/org/labkey/query/persist/QueryManager.java +++ b/query/src/org/labkey/query/persist/QueryManager.java @@ -1077,9 +1077,7 @@ public static void registerUsageMetrics(String moduleName) metrics.put("customViewWithLineageColumn", () -> getLineageCustomViewMetrics(), "Custom views with lineage (input, output, ancestor) columns, hidden and snapshot views excluded: view counts, plus total and min/percentile/max/avg columns per view when any exist"); metrics.put("queryDefWithCalculatedFieldsCounts", () -> getCalculatedFieldsCountsMetric(), "Query definitions whose metadata defines a calculated field, by schema (assay.* schemas grouped as assay)"); - Map exportTypeCounts = getExportTypeCountsMetric(); - if (exportTypeCounts != null) - metrics.put("exportTypeCounts", exportTypeCounts, "Query export audit events by export type, from the audit log across all folders; \"Excel multi-tab\" approximates multi-tab Excel exports", UsageMetrics.Kind.CUMULATIVE); + metrics.put("exportTypeCounts", () -> getExportTypeCountsMetric(), "Query export audit events by export type, from the audit log across all folders; \"Excel multi-tab\" approximates multi-tab Excel exports", UsageMetrics.Kind.CUMULATIVE); return metrics; }); } diff --git a/search/src/org/labkey/search/SearchModule.java b/search/src/org/labkey/search/SearchModule.java index db7d675515b..2e0312c629b 100644 --- a/search/src/org/labkey/search/SearchModule.java +++ b/search/src/org/labkey/search/SearchModule.java @@ -180,16 +180,14 @@ public void handle(Map properties FolderManagement.addTab(FolderManagement.TYPE.FolderManagement, "Search", "fullTextSearch", FolderManagement.NOT_ROOT, SearchController.SearchSettingsAction.class); - UsageMetricsService.get().registerUsageMetrics(getName(), metrics -> + UsageMetricsService.get().registerUsageMetrics(getName(), metrics -> metrics.put("fullTextSearches", () -> { // Report the total number of search entries in the audit log User user = new LimitedUser(User.getSearchUser(), CanSeeAuditLogRole.class); UserSchema auditSchema = AuditLogService.get().createSchema(user, ContainerManager.getRoot()); TableInfo auditTable = auditSchema.getTableOrThrow(SearchAuditProvider.EVENT_TYPE, ContainerFilter.getUnsafeEverythingFilter()); - - long count = new TableSelector(auditTable).getRowCount(); - return metrics.put("fullTextSearches", count, "Full-text search events in the audit log across all containers; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE); - }); + return new TableSelector(auditTable).getRowCount(); + }, "Full-text search events in the audit log across all containers; drops if the audit log is purged", UsageMetrics.Kind.CUMULATIVE)); } @Override diff --git a/study/src/org/labkey/study/StudyModule.java b/study/src/org/labkey/study/StudyModule.java index 8ab0f0eb5ce..1d867ff70ad 100644 --- a/study/src/org/labkey/study/StudyModule.java +++ b/study/src/org/labkey/study/StudyModule.java @@ -16,6 +16,7 @@ package org.labkey.study; +import com.google.common.base.Suppliers; import org.apache.commons.lang3.mutable.MutableInt; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -465,57 +466,60 @@ SELECT COUNT(DISTINCT DD.DomainURI) FROM WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", StudySchema.getInstance().getDatasetSchemaName(), PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class), "Datasets with at least one File column"); // grab the counts of report and dataset notification settings (by notification option) - Collection allStudies = StudyManager.getInstance().getAllStudies(); - Map notifyOptionCounts = new HashMap<>(); - allStudies.forEach(study -> { - Map> settings = ReportContentEmailManager.getUserCategoryMap(study.getContainer()); - for (SortedSet config : settings.values()) - { - ReportContentEmailManager.NotifyOption option = ReportContentEmailManager.removeNotifyOption(config); - notifyOptionCounts.put(option, 1 + notifyOptionCounts.computeIfAbsent(option, (k) -> 0)); - } - }); - Map notificationMap = notifyOptionCounts.entrySet().stream() - .collect(Collectors.toMap(e -> e.getKey().name(), Map.Entry::getValue)); - metrics.put("reportAndDatasetNotificationOptions", notificationMap, "Users subscribed to report and dataset change notifications by option (NONE, ALL, CATEGORY, DATASET); a user counts once per study they subscribe in"); - - long cloudBackedStudies = allStudies.stream() + Supplier> allStudies = Suppliers.memoize(() -> StudyManager.getInstance().getAllStudies()); + metrics.put("reportAndDatasetNotificationOptions", () -> { + Map notifyOptionCounts = new HashMap<>(); + allStudies.get().forEach(study -> { + Map> settings = ReportContentEmailManager.getUserCategoryMap(study.getContainer()); + for (SortedSet config : settings.values()) + { + ReportContentEmailManager.NotifyOption option = ReportContentEmailManager.removeNotifyOption(config); + notifyOptionCounts.put(option, 1 + notifyOptionCounts.computeIfAbsent(option, (k) -> 0)); + } + }); + return notifyOptionCounts.entrySet().stream() + .collect(Collectors.toMap(e -> e.getKey().name(), Map.Entry::getValue)); + }, "Users subscribed to report and dataset change notifications by option (NONE, ALL, CATEGORY, DATASET); a user counts once per study they subscribe in"); + + metrics.put("cloudBackedStudies", () -> allStudies.get().stream() .filter(s -> { PipeRoot pr = PipelineService.get().findPipelineRoot(s.getContainer()); return pr != null && pr.isCloudRoot(); }) - .count(); - metrics.put("cloudBackedStudies", cloudBackedStudies, "Studies whose pipeline root is in cloud storage"); + .count(), "Studies whose pipeline root is in cloud storage"); // Count the studies that have configured at least one group for per-dataset access control - long studiesWithAnyPerDatasetGroup = allStudies.stream() + metrics.put("perDatasetSecurityStudyCount", () -> allStudies.get().stream() .filter(study -> study.getSecurityType().isSupportsPerDatasetPermissions()) .filter(study -> SecurityManager.getGroups(study.getContainer().getProject(), true).stream() .anyMatch(group -> GroupSecurityType.getTypeForGroup(group, study) == GroupSecurityType.PER_DATASET)) - .count(); - metrics.put("perDatasetSecurityStudyCount", studiesWithAnyPerDatasetGroup, "Studies with a security type supporting per-dataset permissions and at least one project group set to per-dataset access"); + .count(), "Studies with a security type supporting per-dataset permissions and at least one project group set to per-dataset access"); // Count the studies that use products and treatments - MutableInt hasProducts = new MutableInt(0); - MutableInt hasTreatments = new MutableInt(0); - - allStudies.stream() - .map(study->StudyQuerySchema.createSchema(study, User.getSearchUser(), RoleManager.getRole(ReaderRole.class))) - .forEach(schema->{ - if (StudyDesignManager.get().isModuleActive(schema.getContainer())) - { - TableInfo products = schema.getTable(StudyDesignQuerySchema.PRODUCT_TABLE_NAME); - if (new TableSelector(products).exists()) - hasProducts.increment(); - - TableInfo treatments = schema.getTable(StudyDesignQuerySchema.TREATMENT_TABLE_NAME); - if (new TableSelector(treatments).exists()) - hasTreatments.increment(); - } - }); - - metrics.put("studyProducts", hasProducts.intValue(), "Studies with at least one study product, counted only in folders with the StudyDesign module active"); - metrics.put("studyTreatments", hasTreatments.intValue(), "Studies with at least one treatment, counted only in folders with the StudyDesign module active"); + record StudyDesignCounts(int products, int treatments) {} + Supplier studyDesignCounts = Suppliers.memoize(() -> { + MutableInt hasProducts = new MutableInt(0); + MutableInt hasTreatments = new MutableInt(0); + + allStudies.get().stream() + .map(study->StudyQuerySchema.createSchema(study, User.getSearchUser(), RoleManager.getRole(ReaderRole.class))) + .forEach(schema->{ + if (StudyDesignManager.get().isModuleActive(schema.getContainer())) + { + TableInfo products = schema.getTable(StudyDesignQuerySchema.PRODUCT_TABLE_NAME); + if (new TableSelector(products).exists()) + hasProducts.increment(); + + TableInfo treatments = schema.getTable(StudyDesignQuerySchema.TREATMENT_TABLE_NAME); + if (new TableSelector(treatments).exists()) + hasTreatments.increment(); + } + }); + return new StudyDesignCounts(hasProducts.intValue(), hasTreatments.intValue()); + }); + + metrics.put("studyProducts", () -> studyDesignCounts.get().products(), "Studies with at least one study product, counted only in folders with the StudyDesign module active"); + metrics.put("studyTreatments", () -> studyDesignCounts.get().treatments(), "Studies with at least one treatment, counted only in folders with the StudyDesign module active"); return metrics; });