GitHub Issue #1130: Metrics to track R & Python package usage

api/src/org/labkey/api/reports/ExternalScriptEngine.java

@@ -23,6 +23,7 @@
 import org.labkey.api.miniprofiler.MiniProfiler;
 import org.labkey.api.pipeline.PipelineJobService;
 import org.labkey.api.reader.Readers;
+import org.labkey.api.reports.report.ScriptPackageUsageTracker;
 import org.labkey.api.reports.report.r.ParamReplacementSvc;
 import org.labkey.api.util.ExceptionUtil;
 import org.labkey.api.util.LabKeyProcessBuilder;
@@ -83,6 +84,8 @@ public class ExternalScriptEngine extends AbstractScriptEngine implements LabKey
     public static final String DEFAULT_WORKING_DIRECTORY = "ExternalScript";
     private static final Pattern scriptCmdPattern = Pattern.compile("'([^']+)'|\\\"([^\\\"]+)\\\"|(^[^\\s]+)|(\\s[^\\s^'^\\\"]+)");
 
+    private static final int MAX_PACKAGES_PER_RUN = 250;
+
     private FileLike _workingDirectory;
 
     protected ExternalScriptEngineDefinition _def;
@@ -113,15 +116,124 @@ public boolean isBinary(FileLike file)
     public Object eval(String script, ScriptContext context) throws ScriptException
     {
         List<String> extensions = getFactory().getExtensions();
+        if (extensions.isEmpty())
+            throw new ScriptException("There are no file name extensions registered for this ScriptEngine : " + getFactory().getLanguageName());
+
+        FileLike scriptFile = prepareScriptFile(appendPackageCaptureEpilog(script, context), context, extensions);
+        Object result = eval(scriptFile, context);
+
+        // Only reached when the script succeeded; a failed run reports no package usage
+        recordPackageUsage(context);
+
+        return result;
+    }
+
+    /**
+     * Prepare the on-disk script file that will be executed. The default writes the script as-is; subclasses (e.g. the
+     * R engine's knitr handling) may wrap it in a different driver script.
+     */
+    protected FileLike prepareScriptFile(String script, ScriptContext context, List<String> extensions)
+    {
+        return writeScriptFile(script, context, extensions);
+    }
 
-        if (!extensions.isEmpty())
+    /**
+     * GitHub Issue #1130
+     * Script appended to the end of the user script (running in the same process) that captures the loaded
+     * packages/modules, writing them one per line to a sidecar file in the working directory for
+     * {@link #recordPackageUsage} to read back. The default returns null (no capture); language-specific engines
+     * (e.g. R, Python) override this.
+     */
+    protected @Nullable String getPackageCaptureEpilog(ScriptContext context)
+    {
+        return null;
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Append this engine's package capture epilog, if it has one, to the end of the given script.
+     * Never throws: package tracking must not affect script execution.
+     */
+    protected String appendPackageCaptureEpilog(String script, ScriptContext context)
+    {
+        try
         {
-            // write out the script file to disk using the first extension as the default
-            FileLike scriptFile = writeScriptFile(script, context, extensions);
-            return eval(scriptFile, context);
+            String epilog = getPackageCaptureEpilog(context);
+            if (epilog != null)
+                return script + "\n" + epilog;
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to build the script package capture epilog", e);
+        }
+
+        return script;
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Called after a script has run successfully (eval returned without throwing), to record the packages it loaded.
+     * The default does nothing; language-specific engines override this, typically delegating to
+     * {@link #readPackageSidecar}.
+     */
+    protected void recordPackageUsage(ScriptContext context)
+    {
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Read a sidecar file of package names (one per line) from the working directory and record each under the given
+     * language in {@link ScriptPackageUsageTracker}, up to {@link #MAX_PACKAGES_PER_RUN} per run. Never throws. A
+     * missing file means the capture code never ran (or was skipped) - nothing to do.
+     * The file is deleted after reading.
+     */
+    protected void readPackageSidecar(ScriptContext context, String fileName, String language)
+    {
+        FileLike packagesFile;
+        try
+        {
+            packagesFile = getWorkingDir(context).resolveChild(fileName);
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to locate " + language + " package usage sidecar", e);
+            return;
+        }
+
+        if (!packagesFile.exists())
+            return;
+
+        try (BufferedReader reader = Readers.getReader(packagesFile.openInputStream()))
+        {
+            String packageName;
+            int recorded = 0;
+            while ((packageName = reader.readLine()) != null)
+            {
+                if (recorded >= MAX_PACKAGES_PER_RUN)
+                {
+                    LOG.warn("Recorded the first {} {} packages for this script run and ignored the rest", MAX_PACKAGES_PER_RUN, language);
+                    ScriptPackageUsageTracker.record(language, "~~packageLimitReached~~");
+                    break;
+                }
+                ScriptPackageUsageTracker.record(language, packageName);
+                recorded++;
+            }
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to record " + language + " package usage", e);
+        }
+        finally
+        {
+            try
+            {
+                packagesFile.delete();
+            }
+            catch (Exception e)
+            {
+                LOG.warn("Failed to delete " + language + " package usage sidecar", e);
+            }
         }
-        else
-            throw new ScriptException("There are no file name extensions registered for this ScriptEngine : " + getFactory().getLanguageName());
     }
 
     protected Object eval(FileLike scriptFile, ScriptContext context) throws ScriptException

api/src/org/labkey/api/reports/report/ScriptPackageUsageTracker.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.reports.report;
+
+import org.apache.logging.log4j.Logger;
+import org.labkey.api.reports.ExternalScriptEngine;
+import org.labkey.api.usageMetrics.SimpleMetricsService;
+import org.labkey.api.util.logging.LogHelper;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Tracks which packages/modules are loaded by scripts run on this server (R reports, assay transform scripts, Python
+ * scripts, and anything else that runs through {@link ExternalScriptEngine}). Populated by a language-specific epilog
+ * appended to each script that writes the loaded packages to a sidecar file, which the engine reads back after the
+ * script has run successfully. Usage is tracked per language (e.g. "r", "python").
+ * <p>
+ * Note that this only sees scripts that ran to completion: a script that fails, or that exits early via q() or
+ * sys.exit(), never reaches the epilog and so reports nothing. Counts are a lower bound.
+ *
+ * Each load is recorded via {@link SimpleMetricsService}, which persists a cumulative per-package load count across
+ * restarts and reports it to mothership under "simpleMetricCounts". The package name is the metric name and the feature
+ * area is "&lt;language&gt;PackageUsage".
+ */
+public class ScriptPackageUsageTracker
+{
+    private static final Logger LOG = LogHelper.getLogger(ScriptPackageUsageTracker.class, "Tracks R & Python package usage by server-side scripts");
+
+    private static final String MODULE_NAME = "API";
+    private static final String FEATURE_AREA_SUFFIX = "PackageUsage";
+    private static final int MAX_METRIC_NAME_LENGTH = 255;
+
+    /**
+     * Packages that ship with a given language's runtime and are always present, so aren't interesting as "library
+     * usage". R's base packages are filtered here; Python needs no entry because its capture epilog already subtracts
+     * sys.stdlib_module_names.
+     */
+    private static final Map<String, Set<String>> BASE_PACKAGES = Map.of(
+        "r", Set.of("base", "compiler", "datasets", "graphics", "grDevices", "grid", "methods", "parallel", "splines", "stats", "stats4", "tcltk", "tools", "utils")
+    );
+
+    private ScriptPackageUsageTracker()
+    {
+    }
+
+    private static boolean isBasePackage(String language, String packageName)
+    {
+        return BASE_PACKAGES.getOrDefault(language, Set.of()).contains(packageName);
+    }
+
+    /**
+     * Record that the given package was loaded by a script run in the given language (e.g. "r", "python"). Safe to call
+     * repeatedly; base packages and blank names are ignored.
+     */
+    public static void record(String language, String packageName)
+    {
+        if (packageName == null || packageName.isBlank() || isBasePackage(language, packageName))
+            return;
+
+        try
+        {
+            SimpleMetricsService.get().increment(MODULE_NAME, language + FEATURE_AREA_SUFFIX, truncateMetricName(packageName));
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to record {} package usage for '{}'", language, packageName, e);
+        }
+    }
+
+    /**
+     * The package name is used as the metric name, and the names come from whatever the script actually loaded rather
+     * than from a fixed list, so cap the length at what the DB column holds.
+     */
+    private static String truncateMetricName(String packageName)
+    {
+        return packageName.length() <= MAX_METRIC_NAME_LENGTH ? packageName : packageName.substring(0, MAX_METRIC_NAME_LENGTH);
+    }
+}

api/src/org/labkey/api/reports/report/python/PythonScriptEngine.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.reports.report.python;
+
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.reports.ExternalScriptEngine;
+import org.labkey.api.reports.ExternalScriptEngineDefinition;
+import org.labkey.vfs.FileLike;
+
+import javax.script.ScriptContext;
+
+/**
+ * Script engine for locally-executed Python scripts (Python assay transform scripts configured as an
+ * external ".py" engine). Behaves like the base {@link ExternalScriptEngine} except that it appends a capture epilog to
+ * track which Python packages each script imports; see
+ * {@link org.labkey.api.reports.report.ScriptPackageUsageTracker}.
+ */
+public class PythonScriptEngine extends ExternalScriptEngine
+{
+    private static final String PACKAGES_FILE = "labkeyPythonPackages.txt";
+
+    // Python appended to a user script to capture the modules it imports. The sidecar file's absolute path is
+    // embedded (see getPackageCaptureEpilog) rather than derived from os.getcwd(), so that a script that changes the
+    // working directory still writes the sidecar where recordPackageUsage() looks for it.
+    // try/except means a capture failure can never break the script run.
+    // Note: we are tracking module names as written in the source, not installable distribution names - 'yaml', not 'PyYAML'.
+    private static final String PACKAGE_CAPTURE_EPILOG = """
+            try:
+                import ast as _lk_ast, sys as _lk_sys
+
+                def _lk_pkg(_lk_dotted):
+                    # Report the shortest prefix that is a real module
+                    _lk_parts = _lk_dotted.split('.')
+                    for _lk_i in range(1, len(_lk_parts)):
+                        _lk_prefix = '.'.join(_lk_parts[:_lk_i])
+                        if getattr(_lk_sys.modules.get(_lk_prefix), '__file__', None):
+                            return _lk_prefix
+                    # Nothing loaded under this name - an optional dependency this server lacks, or an import in a
+                    # branch that never ran - so record it as written.
+                    return _lk_dotted if _lk_parts[0] in _lk_sys.modules else _lk_parts[0]
+
+                with open(globals().get('__file__') or _lk_sys.argv[0], 'rb') as _lk_src:
+                    _lk_tree = _lk_ast.parse(_lk_src.read())
+                _lk_names = set()
+                for _lk_node in _lk_ast.walk(_lk_tree):
+                    if isinstance(_lk_node, _lk_ast.Import):
+                        _lk_names.update(_lk_pkg(_lk_a.name) for _lk_a in _lk_node.names)
+                    elif isinstance(_lk_node, _lk_ast.ImportFrom) and not _lk_node.level and _lk_node.module:
+                        _lk_names.update(_lk_pkg(_lk_node.module + '.' + _lk_a.name) for _lk_a in _lk_node.names)
+                _lk_std = set(getattr(_lk_sys, 'stdlib_module_names', ('ast', 'sys')))
+                _lk_names = sorted(n for n in _lk_names if n[:1] != '_' and n.split('.')[0] not in _lk_std)
+                with open('%s', 'w') as _lk_f:
+                    _lk_f.write('\\n'.join(_lk_names))
+            except Exception:
+                pass
+            """;
+
+    public PythonScriptEngine(ExternalScriptEngineDefinition def)
+    {
+        super(def);
+    }
+
+    @Override
+    protected @Nullable String getPackageCaptureEpilog(ScriptContext context)
+    {
+        return PACKAGE_CAPTURE_EPILOG.formatted(toPythonPath(getWorkingDir(context).resolveChild(PACKAGES_FILE)));
+    }
+
+    private static String toPythonPath(FileLike file)
+    {
+        return file.toNioPathForWrite().toFile().getAbsolutePath()
+                .replace('\\', '/')
+                .replace("'", "\\'");
+    }
+
+    @Override
+    protected void recordPackageUsage(ScriptContext context)
+    {
+        readPackageSidecar(context, PACKAGES_FILE, "python");
+    }
+}