diff --git a/api/src/org/labkey/api/ApiModule.java b/api/src/org/labkey/api/ApiModule.java index 2de521aac88..823fe72c2e9 100644 --- a/api/src/org/labkey/api/ApiModule.java +++ b/api/src/org/labkey/api/ApiModule.java @@ -41,6 +41,7 @@ import org.labkey.api.cache.BlockingCache; import org.labkey.api.cache.CacheManager; import org.labkey.api.cache.ContainerKeySimpleCache; +import org.labkey.api.cache.NightlyRecalculatingSuppliers; import org.labkey.api.cache.SimpleKeyMappingCache; import org.labkey.api.cache.UserKeySimpleCache; import org.labkey.api.collections.ArrayListMap; @@ -300,6 +301,7 @@ protected Collection createWebPartFactories() protected void doStartup(ModuleContext moduleContext) { SystemMaintenance.addTask(new ApiKeyMaintenanceTask()); + SystemMaintenance.addTask(new NightlyRecalculatingSuppliers.RecalculationMaintenanceTask()); AuthenticationManager.registerMetricsProvider(); ContentSecurityPolicyFilter.registerMetricsProvider(); ApiKeyManager.get().handleStartupProperties(); diff --git a/api/src/org/labkey/api/cache/BlockingCache.java b/api/src/org/labkey/api/cache/BlockingCache.java index ed1a7d0712b..1224661c282 100644 --- a/api/src/org/labkey/api/cache/BlockingCache.java +++ b/api/src/org/labkey/api/cache/BlockingCache.java @@ -160,7 +160,7 @@ public V get(@NotNull K key, @Nullable Object argument, CacheLoader loader throw new IllegalStateException("cache loader was not provided"); V value = load(key, argument, loader); - CacheManager.validate("BlockingCache over \"" + _cache + "\" cache", value); + CacheManager.validate("BlockingCache over \"" + _cache + "\" cache", key, value); synchronized (w.getLockObject()) { diff --git a/api/src/org/labkey/api/cache/CacheManager.java b/api/src/org/labkey/api/cache/CacheManager.java index b834f7de775..59a631f26fa 100644 --- a/api/src/org/labkey/api/cache/CacheManager.java +++ b/api/src/org/labkey/api/cache/CacheManager.java @@ -182,16 +182,16 @@ public static void shutdown() private static final Set> CLASSES = new HashSet<>(); // Validate a cached value. Log errors for mutable collections/arrays and for values holding Container or User objects. - public static void validate(String debugName, @Nullable V value) + public static void validate(String debugName, K key, @Nullable V value) { if (value instanceof Wrapper) return; - String description = CollectionUtils.getModifiableCollectionMapOrArrayType(value); + String valueDescription = CollectionUtils.getModifiableCollectionMapOrArrayType(value); - if (null != description) + if (null != valueDescription) { - LOG.error("{} attempted to cache {}, which could be mutated by callers!", debugName, description); + LOG.error("{} attempted to cache {}, which could be mutated by callers, at key {}", debugName, valueDescription, key); } // Flag values that hold a Container or User object diff --git a/api/src/org/labkey/api/cache/CacheWrapper.java b/api/src/org/labkey/api/cache/CacheWrapper.java index 991665fcc6c..56cd3ce846d 100644 --- a/api/src/org/labkey/api/cache/CacheWrapper.java +++ b/api/src/org/labkey/api/cache/CacheWrapper.java @@ -66,7 +66,7 @@ public void put(@NotNull K key, V value) value = _nullMarker; _cache.put(key, value); - trackPut(value); + trackPut(key, value); } catch (IllegalStateException ise) { @@ -84,7 +84,7 @@ public void put(@NotNull K key, V value, long timeToLive) value = _nullMarker; _cache.put(key, value, timeToLive); - trackPut(value); + trackPut(key, value); } catch (IllegalStateException ise) { @@ -260,10 +260,10 @@ private V trackGet(V value) return value; } - private void trackPut(V value) + private void trackPut(K key, V value) { assert null != value : "Attempt to cache null into " + getDebugName() + "; must use marker for null instead."; - CacheManager.validate("\"" + getDebugName() + "\" cache", value); + CacheManager.validate("\"" + getDebugName() + "\" cache", key, value); _stats.puts.incrementAndGet(); diff --git a/api/src/org/labkey/api/cache/NightlyRecalculatingSuppliers.java b/api/src/org/labkey/api/cache/NightlyRecalculatingSuppliers.java new file mode 100644 index 00000000000..6951d12ad22 --- /dev/null +++ b/api/src/org/labkey/api/cache/NightlyRecalculatingSuppliers.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.cache; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.util.logging.LogHelper; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +public class NightlyRecalculatingSuppliers +{ + private static final Logger LOG = LogHelper.getLogger(NightlyRecalculatingSuppliers.class, "Nightly recalculated values"); + private static final Map> SUPPLIERS = new ConcurrentHashMap<>(); + private static final BlockingCache CACHE = + CacheManager.getBlockingCache(String.class, 100, CacheManager.WEEK, "Nightly recalculated values", null); + + private NightlyRecalculatingSuppliers() + { + } + + /** + * Returns a supplier that caches the value retrieved during the first call to {@code supplier.get()} and returns + * that value on subsequent calls to {@code supplier.get()}. Every call to {@code NightlyRecalculatingSuppliers.get()} + * with the same unique name and delegate supplier class returns the same caching supplier, so the efficiency + * continues across multiple calls, with no need to hold the supplier. All suppliers proactively recalculate during + * nightly maintenance, so their values are never more than 24 hours out-of-date. + * @param uniqueName unique across the server; include anything the delegate captures that varies between calls + * @param delegate a supplier for the value that's cached. All cached values, and therefore, anything a delegate + * supplier calculates, must be unmodifiable. + * @return the same instance for every call with this name; the first registered delegate is kept + */ + @SuppressWarnings("unchecked") + public static Supplier get(@NotNull String uniqueName, @NotNull Supplier delegate) + { + NightlyRecalculatingSupplier supplier = SUPPLIERS.computeIfAbsent(uniqueName, _ -> new NightlyRecalculatingSupplier<>(uniqueName, delegate)); + Class registered = supplier.delegate().getClass(); + if (registered != delegate.getClass()) + reportConflict("Nightly recalculating supplier \"" + uniqueName + "\" was registered by " + registered.getName() + " and again by " + delegate.getClass().getName()); + return (Supplier) supplier; + } + + /** Computes each new value before replacing the old one, so callers rarely block on a recalculation. A failing delegate keeps its old value. */ + private static void recalculateAll(Logger pipelineLog) + { + SUPPLIERS.values().forEach(supplier -> { + try + { + pipelineLog.info("Recalculating {}", supplier.name()); + CACHE.put(supplier.name(), supplier.calculate()); + } + catch (Exception e) + { + ExceptionUtil.decorateException(e, ExceptionUtil.ExceptionInfo.ExtraMessage, "Nightly recalculating supplier: " + supplier.name(), true); + ExceptionUtil.logExceptionToMothership(null, e); + } + }); + } + + public static class RecalculationMaintenanceTask implements SystemMaintenance.MaintenanceTask + { + @Override + public String getDescription() + { + return "Recalculate nightly recalculated values, such as slow usage metrics"; + } + + @Override + public String getName() + { + return "RecalculateNightlyValues"; + } + + @Override + public void run(Logger pipelineLog) + { + pipelineLog.info("Recalculating {} values", SUPPLIERS.size()); + recalculateAll(pipelineLog); + } + + // Disabling would leave every value frozen at its first calculation + @Override + public boolean canDisable() + { + return false; + } + } + + /** Throws in dev mode; otherwise warns and keeps the first delegate */ + private static void reportConflict(String message) + { + if (AppProps.getInstance().isDevMode()) + throw new IllegalStateException(message); + LOG.warn(message); + } + + private record NightlyRecalculatingSupplier(@NotNull String name, @NotNull Supplier delegate) implements Supplier + { + @Override + @SuppressWarnings("unchecked") + public T get() + { + return (T) CACHE.get(name, null, (_, _) -> calculate()); + } + + private T calculate() + { + LOG.debug("Calculating {}", name); + T value = delegate.get(); + LOG.debug("Calculated {}", name); + return value; + } + } +} diff --git a/api/src/org/labkey/api/reports/ReportService.java b/api/src/org/labkey/api/reports/ReportService.java index 76aac263862..ded70724df7 100644 --- a/api/src/org/labkey/api/reports/ReportService.java +++ b/api/src/org/labkey/api/reports/ReportService.java @@ -71,17 +71,17 @@ static void registerProvider(ReportService provider) */ void registerReport(Report report); - /** Sees every saved report during the single usage metrics pass, then adds its own metrics */ - interface ReportMetricsVisitor + /** Sees every saved report during the nightly pass, then adds its own metrics */ + interface ReportMetricsVisitor> { void visit(Report report); - /** Each supplier must call visitAll first; it runs the single pass on demand, so describing the metrics visits nothing */ - void addMetrics(UsageMetrics metrics, Runnable visitAll); + /** Suppliers must read results from visited.get(), the cached instance that saw every report, not from this one. Calling it runs the pass on demand, so describing the metrics visits nothing. */ + void addMetrics(UsageMetrics metrics, Supplier visited); } - /** The factory supplies a fresh visitor for each usage metrics collection */ - void registerReportMetricsVisitor(Supplier factory); + /** The factory supplies a fresh visitor for each pass, plus one per usage metrics collection to add the metrics */ + void registerReportMetricsVisitor(Supplier> factory); /** * A descriptor class must be registered with the service in order to be valid. diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java b/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java index cdc21e75386..2f5bb0acb43 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetricGroup.java @@ -19,6 +19,7 @@ import org.jetbrains.annotations.Nullable; import java.util.Map; +import java.util.Objects; import java.util.function.Supplier; /** A described nested object; its children may omit descriptions because they inherit the group's */ @@ -34,7 +35,7 @@ public UsageMetricGroup put(@NotNull String key, @Nullable Supplier supplier) { Map values = getMutableValues(); if (values != null) - values.put(key, supplier == null ? null : supplier.get()); + values.put(key, supplier == null ? null : supply(childPath(key), supplier)); return this; } @@ -43,7 +44,11 @@ public UsageMetricGroup putAll(@NotNull Supplier> mapSu { Map mutable = getMutableValues(); if (mutable != null) - mutable.putAll(mapSupplier.get()); + { + Map map = supply(getPath(), mapSupplier); + if (map != null) + mutable.putAll(map); + } return this; } @@ -51,4 +56,10 @@ public UsageMetricGroup group(@NotNull String key) { return addGroup(key); } + + @Override + @NotNull String getPath() + { + return Objects.requireNonNull(super.getPath()); + } } diff --git a/api/src/org/labkey/api/usageMetrics/UsageMetrics.java b/api/src/org/labkey/api/usageMetrics/UsageMetrics.java index bf78a139b01..036c05398f6 100644 --- a/api/src/org/labkey/api/usageMetrics/UsageMetrics.java +++ b/api/src/org/labkey/api/usageMetrics/UsageMetrics.java @@ -21,12 +21,15 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.settings.AppProps; +import org.labkey.api.util.ExceptionUtil; +import org.labkey.api.util.Formats; import org.labkey.api.util.logging.LogHelper; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.BooleanSupplier; import java.util.function.Supplier; @@ -37,7 +40,8 @@ */ public class UsageMetrics { - private static final Logger LOG = LogHelper.getLogger(UsageMetrics.class, "Usage metric descriptions"); + private static final Logger LOG = LogHelper.getLogger(UsageMetrics.class, "Usage metric collection and descriptions"); + private static final long SLOW_SUPPLIER_MS = 1000; public enum Kind { @@ -83,10 +87,39 @@ public UsageMetrics put(@NotNull String key, @Nullable Supplier supplier, @No { describe(key, description, kind); if (_values != null) - _values.put(key, supplier == null ? null : supplier.get()); + { + Object value = supplier == null ? null : supply(childPath(key), supplier); + _values.put(key, value); + } return this; } + /** Logs slow suppliers in dev mode. A throwing supplier reports null and goes to mothership, so one broken metric doesn't drop the rest. */ + static @Nullable T supply(@NotNull String path, @NotNull Supplier supplier) + { + boolean timed = AppProps.getInstance().isDevMode(); + long start = timed ? System.nanoTime() : 0; + try + { + return supplier.get(); + } + catch (Exception e) + { + ExceptionUtil.decorateException(e, ExceptionUtil.ExceptionInfo.ExtraMessage, "Usage metric: " + path, true); + ExceptionUtil.logExceptionToMothership(null, e); + return null; + } + finally + { + if (timed) + { + long elapsed = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start); + if (elapsed > SLOW_SUPPLIER_MS) + LOG.warn("Usage metric {} took {} ms", path, Formats.commaf0.format(elapsed)); + } + } + } + /** @return this, or if not reported, a describe-only instance at the same path: it records descriptions of what's put into it, groups included, but drops the values */ public UsageMetrics reportedIf(boolean reported) { @@ -168,6 +201,12 @@ String childPath(String key) return _path == null ? key : _path + "." + key; } + /** @return null for the top level */ + @Nullable String getPath() + { + return _path; + } + /** @return null for a describe-only instance */ @Nullable Map getMutableValues() { diff --git a/core/src/org/labkey/core/CoreModule.java b/core/src/org/labkey/core/CoreModule.java index 27c44ea8b2d..c045f8216c0 100644 --- a/core/src/org/labkey/core/CoreModule.java +++ b/core/src/org/labkey/core/CoreModule.java @@ -48,6 +48,7 @@ import org.labkey.api.audit.provider.GroupAuditProvider; import org.labkey.api.audit.provider.ModulePropertiesAuditProvider; import org.labkey.api.cache.CacheManager; +import org.labkey.api.cache.NightlyRecalculatingSuppliers; import org.labkey.api.data.ConnectionUsage; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; @@ -1302,17 +1303,20 @@ public void moduleStartupComplete(ServletContext servletContext) "Emails of active non-system users; only reported when the extended metrics feature flag is on" ); - metrics.put("workbookCount", () -> ContainerManager.getWorkbookCount(), "Workbooks across the server"); - metrics.put("archivedFolderCount", () -> ContainerManager.getArchivedContainerCount(), "Archived folders across the server"); - metrics.put("databaseSize", () -> CoreSchema.getInstance().getSchema().getScope().getDatabaseSize(), "Size of the LabKey database in bytes"); - - // Exclude temp schema to avoid PG exceptions when tables are appearing/disappearing during execution - // Note that they can be non-trivial in size. - SQLFragment sql = new SQLFragment("SELECT table_schema, SUM(total_size) FROM "); - sql.append(new PostgresTableSizesTable(new PostgresUserSchema(User.getAdminServiceUser(), ContainerManager.getRoot())), "t"); - sql.unsafeAppend(" WHERE table_schema != 'temp' GROUP BY table_schema"); - - metrics.put("databaseSchemaSize", () -> new SqlSelector(CoreSchema.getInstance().getSchema(), sql).getValueMap(), "Size in bytes of each database schema, including indexes; temp excluded"); + metrics.put("workbookCount", ContainerManager::getWorkbookCount, "Workbooks across the server"); + metrics.put("archivedFolderCount", ContainerManager::getArchivedContainerCount, "Archived folders across the server"); + String dbSize = "databaseSize"; + metrics.put(dbSize, NightlyRecalculatingSuppliers.get(dbSize, () -> CoreSchema.getInstance().getSchema().getScope().getDatabaseSize()), "Size of the LabKey database in bytes"); + + String schemaSize = "databaseSchemaSize"; + metrics.put(schemaSize, NightlyRecalculatingSuppliers.get(schemaSize, () -> { + // Exclude temp schema to avoid PG exceptions when tables are appearing/disappearing during execution + // Note that they can be non-trivial in size. + SQLFragment sql = SQLFragment.unsafe("SELECT table_schema, SUM(total_size) FROM "); + sql.append(new PostgresTableSizesTable(new PostgresUserSchema(User.getAdminServiceUser(), ContainerManager.getRoot())), "t"); + sql.unsafeAppend(" WHERE table_schema != 'temp' GROUP BY table_schema"); + return Collections.unmodifiableMap(new SqlSelector(CoreSchema.getInstance().getSchema(), sql).getValueMap()); + }), "Size in bytes of each database schema, including indexes; temp excluded"); metrics.put("scriptEngines", () -> LabKeyScriptEngineManager.get().getScriptEngineMetrics(),"Configured script engines (R, Python, etc.) keyed by name, with factory and engine class, file extensions, and whether each is external and enabled"); metrics.put("customLabels", () -> CustomLabelService.get().getCustomLabelMetrics(), "Custom noun labels by label provider: count of relabeled nouns and of projects that override labels"); diff --git a/core/src/org/labkey/core/admin/DisplayFormatAnalyzer.java b/core/src/org/labkey/core/admin/DisplayFormatAnalyzer.java index 42713c881f6..8cc598c9ab2 100644 --- a/core/src/org/labkey/core/admin/DisplayFormatAnalyzer.java +++ b/core/src/org/labkey/core/admin/DisplayFormatAnalyzer.java @@ -21,6 +21,7 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.action.UrlProvider; import org.labkey.api.admin.AdminUrls; +import org.labkey.api.cache.NightlyRecalculatingSuppliers; import org.labkey.api.data.Container; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.PropertyManager; @@ -44,6 +45,7 @@ import org.labkey.data.xml.TableType; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.Set; import java.util.function.Supplier; @@ -218,12 +220,13 @@ public void handleAll(User user, DisplayFormatHandler handler) public static UsageMetricsProvider getMetricsProvider() { - return metrics -> metrics.put("nonStandardDateDisplayFormats", () -> { + String name = "nonStandardDateDisplayFormats"; + return metrics -> metrics.put(name, NightlyRecalculatingSuppliers.get(name, () -> { // Collect the unique set of non-standard date display formats across the entire site Set badFormats = new HashSet<>(); new DisplayFormatAnalyzer().handleAll(User.getAdminServiceUser(), (c, type, format, contextProvider) -> badFormats.add(format)); - return badFormats; - }, "Distinct non-standard date display formats used in folder defaults, query metadata, or field properties anywhere on the server"); + return Collections.unmodifiableSet(badFormats); + }), "Distinct non-standard date display formats used in folder defaults, query metadata, or field properties anywhere on the server"); } private

@NotNull P urlProvider(Class

inter) diff --git a/query/src/org/labkey/query/persist/QueryManager.java b/query/src/org/labkey/query/persist/QueryManager.java index 23895e0b782..3e0127c82ba 100644 --- a/query/src/org/labkey/query/persist/QueryManager.java +++ b/query/src/org/labkey/query/persist/QueryManager.java @@ -26,6 +26,7 @@ import org.json.JSONObject; import org.labkey.api.audit.AbstractAuditTypeProvider; import org.labkey.api.audit.AuditLogService; +import org.labkey.api.cache.NightlyRecalculatingSuppliers; import org.labkey.api.collections.CaseInsensitiveHashSet; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.Container; @@ -1056,14 +1057,15 @@ public static void registerUsageMetrics(String moduleName) if (null != svc) { svc.registerUsageMetrics(moduleName, metrics -> { - metrics.put("externalDatasources", () -> { + String name = "externalDatasources"; + metrics.put(name, NightlyRecalculatingSuppliers.get(name, () -> { MultiSet multiSet = DbScope.getDbScopes().stream() .filter(scope -> !scope.isLabKeyScope()).map(DbScope::getDatabaseProductName) .collect(Collectors.toCollection(HashMultiSet::new)); - return multiSet.uniqueSet().stream() - .collect(Collectors.toMap(Function.identity(), multiSet::getCount)); - }, "External (non-LabKey) data sources by database product name"); + return Collections.unmodifiableMap(multiSet.uniqueSet().stream() + .collect(Collectors.toMap(Function.identity(), multiSet::getCount))); + }), "External (non-LabKey) data sources by database product name"); metrics.put( "customViewCounts", () -> Map.of( @@ -1074,10 +1076,10 @@ public static void registerUsageMetrics(String moduleName) ), "Saved custom views across all containers by schema area (DataClasses, SampleTypes, Assays, Inventory): defaultOverrides (unnamed) and namedViews exclude hidden and snapshot views, inheritable likewise, shared (no owner) and identifyingFieldsViews count all" ); - metrics.put("customViewWithLineageColumn", () -> getLineageCustomViewMetrics(), "Custom views with lineage (input, output, ancestor) columns, hidden and snapshot views excluded: view counts, plus total and min/percentile/max/avg columns per view when any exist"); - metrics.put("queryDefWithCalculatedFieldsCounts", () -> getCalculatedFieldsCountsMetric(), "Query definitions whose metadata defines a calculated field, by schema (assay.* schemas grouped as assay)"); + metrics.put("customViewWithLineageColumn", QueryManager::getLineageCustomViewMetrics, "Custom views with lineage (input, output, ancestor) columns, hidden and snapshot views excluded: view counts, plus total and min/percentile/max/avg columns per view when any exist"); + metrics.put("queryDefWithCalculatedFieldsCounts", QueryManager::getCalculatedFieldsCountsMetric, "Query definitions whose metadata defines a calculated field, by schema (assay.* schemas grouped as assay)"); - metrics.put("exportTypeCounts", () -> getExportTypeCountsMetric(), "Query export audit events by export type, from the audit log across all folders; \"Excel multi-tab\" approximates multi-tab Excel exports", UsageMetrics.Kind.CUMULATIVE); + metrics.put("exportTypeCounts", QueryManager::getExportTypeCountsMetric, "Query export audit events by export type, from the audit log across all folders; \"Excel multi-tab\" approximates multi-tab Excel exports", UsageMetrics.Kind.CUMULATIVE); return metrics; }); } diff --git a/query/src/org/labkey/query/reports/ReportServiceImpl.java b/query/src/org/labkey/query/reports/ReportServiceImpl.java index d9f4b556813..35e506066c7 100644 --- a/query/src/org/labkey/query/reports/ReportServiceImpl.java +++ b/query/src/org/labkey/query/reports/ReportServiceImpl.java @@ -16,7 +16,6 @@ package org.labkey.query.reports; -import com.google.common.base.Suppliers; import org.apache.commons.beanutils.BeanUtils; import org.apache.commons.beanutils.ConvertUtils; import org.apache.commons.io.FileUtils; @@ -31,6 +30,7 @@ import org.labkey.api.admin.FolderExportContext; import org.labkey.api.admin.FolderImportContext; import org.labkey.api.audit.AuditLogService; +import org.labkey.api.cache.NightlyRecalculatingSuppliers; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.CoreSchema; @@ -51,7 +51,6 @@ import org.labkey.api.query.ValidationException; import org.labkey.api.reports.Report; import org.labkey.api.reports.ReportService; -import org.labkey.api.reports.ReportService.ReportMetricsVisitor; import org.labkey.api.reports.model.ReportPropsManager; import org.labkey.api.reports.model.ViewCategory; import org.labkey.api.reports.model.ViewCategoryListener; @@ -77,6 +76,7 @@ import org.labkey.api.study.Dataset; import org.labkey.api.study.Study; import org.labkey.api.study.StudyService; +import org.labkey.api.usageMetrics.UsageMetrics; import org.labkey.api.usageMetrics.UsageMetricsService; import org.labkey.api.util.ContainerUtil; import org.labkey.api.util.GUID; @@ -124,7 +124,7 @@ public class ReportServiceImpl implements ContainerManager.ContainerListener, Re private static final List _uiProviders = new CopyOnWriteArrayList<>(); private static final Map _typeToProviderMap = new ConcurrentHashMap<>(); private static final List _globalItemFilterTypes = new CopyOnWriteArrayList<>(); - private static final List> _metricsVisitorFactories = new CopyOnWriteArrayList<>(); + private static final List>> _metricsVisitorFactories = new CopyOnWriteArrayList<>(); /** * maps descriptor types to providers @@ -226,7 +226,7 @@ public void registerReport(Report report) } @Override - public void registerReportMetricsVisitor(Supplier factory) + public void registerReportMetricsVisitor(Supplier> factory) { _metricsVisitorFactories.add(factory); } @@ -1099,18 +1099,29 @@ public static void registerUsageMetrics(String moduleName) if (null != svc) { svc.registerUsageMetrics(moduleName, metrics -> { - List visitors = _metricsVisitorFactories.stream().map(Supplier::get).toList(); - Supplier> countsByType = Suppliers.memoize(() -> visitReports(visitors)); - metrics.put("reportCountsByType", countsByType, "Saved reports in existing containers by report type"); - visitors.forEach(visitor -> visitor.addMetrics(metrics, countsByType::get)); + Supplier pass = NightlyRecalculatingSuppliers.get("reportMetricsPass", ReportServiceImpl::visitReports); + metrics.put("reportCountsByType", () -> pass.get().countsByType(), "Saved reports in existing containers by report type"); + for (int i = 0; i < _metricsVisitorFactories.size(); i++) + addVisitorMetrics(metrics, _metricsVisitorFactories.get(i).get(), pass, i); return metrics; }); } } - /** Visits every saved report in an existing container and counts them by type */ - private static Map visitReports(List visitors) + /** The visitors are in factory order */ + private record ReportMetricsPass(Map countsByType, List> visitors) {} + + @SuppressWarnings("unchecked") + private static > void addVisitorMetrics(UsageMetrics metrics, ReportMetricsVisitor visitor, Supplier pass, int index) { + visitor.addMetrics(metrics, () -> (V) pass.get().visitors().get(index)); + } + + /** Visits every saved report in an existing container with fresh visitors and counts them by type */ + private static ReportMetricsPass visitReports() + { + List> visitors = _metricsVisitorFactories.stream().>map(Supplier::get).toList(); + // Type comes from the descriptor XML, so instantiate each report, but stream rows rather than load every container's report cache TableInfo table = getTable(); SQLFragment sql = new SQLFragment("SELECT * FROM ").append(table, "r") @@ -1118,11 +1129,12 @@ private static Map visitReports(List visitor ReportServiceImpl reportService = getInstance(); try (Stream rows = new SqlSelector(table.getSchema(), sql).uncachedStream(ReportDB.class)) { - return rows + Map countsByType = rows .map(reportService::_getInstance) .filter(Objects::nonNull) .peek(report -> visitors.forEach(visitor -> visitor.visit(report))) .collect(Collectors.groupingBy(Report::getType, Collectors.counting())); + return new ReportMetricsPass(Collections.unmodifiableMap(countsByType), visitors); } } } diff --git a/visualization/src/org/labkey/visualization/report/ChartMetricsVisitor.java b/visualization/src/org/labkey/visualization/report/ChartMetricsVisitor.java index 7209cdb86b6..afea98d2953 100644 --- a/visualization/src/org/labkey/visualization/report/ChartMetricsVisitor.java +++ b/visualization/src/org/labkey/visualization/report/ChartMetricsVisitor.java @@ -24,7 +24,9 @@ import org.labkey.api.visualization.GenericChartReport; import org.labkey.api.visualization.VisualizationReportDescriptor; -public class ChartMetricsVisitor implements ReportMetricsVisitor +import java.util.function.Supplier; + +public class ChartMetricsVisitor implements ReportMetricsVisitor { private final MultiSet _chartCountsByRenderType = new HashMultiSet<>(); private int _trendlineTypeCount = 0; @@ -48,11 +50,11 @@ public void visit(Report report) } @Override - public void addMetrics(UsageMetrics metrics, Runnable visitAll) + public void addMetrics(UsageMetrics metrics, Supplier visited) { metrics - .put("genericChartCountsByRenderType", () -> { visitAll.run(); return MultiSetUtils.getOccurrenceMap(_chartCountsByRenderType); }, "Saved charts (GenericChartReport) by render type (BAR_PLOT, BOX_PLOT, etc.)") - .put("genericChartWithTrendlineTypeCount", () -> { visitAll.run(); return _trendlineTypeCount; }, "Saved charts with a trendline type set") - .put("genericChartWithErrorBarsCount", () -> { visitAll.run(); return _errorBarsCount; }, "Saved charts with SD or SEM error bars"); + .put("genericChartCountsByRenderType", () -> MultiSetUtils.getOccurrenceMap(visited.get()._chartCountsByRenderType), "Saved charts (GenericChartReport) by render type (BAR_PLOT, BOX_PLOT, etc.)") + .put("genericChartWithTrendlineTypeCount", () -> visited.get()._trendlineTypeCount, "Saved charts with a trendline type set") + .put("genericChartWithErrorBarsCount", () -> visited.get()._errorBarsCount, "Saved charts with SD or SEM error bars"); } }