GH Issue 1720: Run saveBatch and saveRuns transform scripts outside the save transaction

api/src/org/labkey/api/ApiModule.java

@@ -30,6 +30,7 @@
 import org.labkey.api.assay.AssayResultsFileWriter;
 import org.labkey.api.assay.ReplacedRunFilter;
 import org.labkey.api.assay.sample.MaterialInputRoleComparator;
+import org.labkey.api.assay.transform.PreparedTransform;
 import org.labkey.api.attachments.AttachmentService;
 import org.labkey.api.attachments.ImageServlet;
 import org.labkey.api.attachments.LookAndFeelResourceType;
@@ -589,6 +590,7 @@ public void registerServlets(ServletContext servletCtx)
             ParameterSubstitutionTest.class,
             Portal.TestCase.class,
             PostgresSnapshot.TestCase.class,
+            PreparedTransform.TestCase.class,
             PropertyManager.TestCase.class,
             QueryDataIteratorBuilder.TestCase.class,
             QueryProfiler.ExecutionPlanUnavailableTestCase.class,

api/src/org/labkey/api/assay/AssayFileWriter.java

@@ -20,6 +20,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jetbrains.annotations.NotNull;
+import org.labkey.api.assay.transform.PendingFileAudit;
 import org.labkey.api.audit.AuditLogService;
 import org.labkey.api.audit.provider.FileSystemAuditProvider;
 import org.labkey.api.collections.CollectionUtils;
@@ -262,7 +263,11 @@ public Map<String, FileLike> savePostedFiles(ContextType context, @NotNull Set<S
                                 event.setProvidedFileName(fileName);
                                 event.setFile(file.getName());
                                 event.setDirectory(dir.toURI().getPath());
-                                AuditLogService.get().addEvent(context.getUser(), event);
+                                List<PendingFileAudit> pending = context.getPendingFileAudits();
+                                if (pending != null)
+                                    pending.add(new PendingFileAudit(event, file));
+                                else
+                                    AuditLogService.get().addEvent(context.getUser(), event);
                             }
                             if (!isAfterFirstFile)  // first file gets stored with multipartFile's name
                             {

api/src/org/labkey/api/assay/AssayRunCreator.java

@@ -55,10 +55,10 @@ Pair<ExpExperiment, ExpRun> saveExperimentRun(AssayRunUploadContext<ProviderType
     /**
      * Runs the assay's transform scripts ahead of the import transaction, when the provider supports it, so that
      * saveExperimentRun only records their output. Call before opening a transaction around saveExperimentRun and
-     * close the result after that transaction ends.
+     * close the result after that transaction ends. Don't call it for an import that will run in a background job.
      * @return null when nothing ran
      */
-    default @Nullable PreparedTransform prepareTransform(AssayRunUploadContext<ProviderType> context, boolean forceAsync)
+    default @Nullable PreparedTransform prepareTransform(AssayRunUploadContext<ProviderType> context)
             throws ExperimentException, ValidationException
     {
         return null;

api/src/org/labkey/api/assay/AssayRunUploadContext.java

@@ -28,6 +28,7 @@
 import org.labkey.api.exp.api.ExpProtocol;
 import org.labkey.api.exp.api.ExpRun;
 import org.labkey.api.exp.property.DomainProperty;
+import org.labkey.api.assay.transform.PendingFileAudit;
 import org.labkey.api.assay.transform.PreparedTransform;
 import org.labkey.api.assay.transform.TransformResult;
 import org.labkey.api.security.User;
@@ -37,6 +38,7 @@
 import org.labkey.api.writer.ContainerUser;
 import org.labkey.vfs.FileLike;
 
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
@@ -188,6 +190,17 @@ default void setPreparedTransform(@Nullable PreparedTransform prepared)
         throw new UnsupportedOperationException(getClass().getName() + " can't carry a prepared transform");
     }
 
+    /** Audits of files saved while a transform is prepared, or null to add each event as its file is saved. */
+    default @Nullable List<PendingFileAudit> getPendingFileAudits()
+    {
+        return null;
+    }
+
+    /** Contexts that can't hold audits ignore this and add each event as its file is saved. */
+    default void setPendingFileAudits(@Nullable List<PendingFileAudit> audits)
+    {
+    }
+
     /** The RowId for the run that is being deleted and reuploaded, or null if this is a new run */
     Long getReRunId();
 

api/src/org/labkey/api/assay/AssayRunUploadContextImpl.java

@@ -32,6 +32,7 @@
 import org.labkey.api.exp.property.Domain;
 import org.labkey.api.exp.property.DomainProperty;
 import org.labkey.api.assay.transform.DefaultTransformResult;
+import org.labkey.api.assay.transform.PendingFileAudit;
 import org.labkey.api.assay.transform.PreparedTransform;
 import org.labkey.api.assay.transform.TransformResult;
 import org.labkey.api.security.User;
@@ -95,6 +96,7 @@ public class AssayRunUploadContextImpl<ProviderType extends AssayProvider> imple
     // Mutable fields
     private TransformResult _transformResult;
     private @Nullable PreparedTransform _preparedTransform;
+    private @Nullable List<PendingFileAudit> _pendingFileAudits;
 
     // async fields
     protected String _jobDescription;
@@ -453,6 +455,18 @@ public void setPreparedTransform(@Nullable PreparedTransform prepared)
         _preparedTransform = prepared;
     }
 
+    @Override
+    public @Nullable List<PendingFileAudit> getPendingFileAudits()
+    {
+        return _pendingFileAudits;
+    }
+
+    @Override
+    public void setPendingFileAudits(@Nullable List<PendingFileAudit> audits)
+    {
+        _pendingFileAudits = audits;
+    }
+
     @Override
     public void uploadComplete(ExpRun run)
     {

api/src/org/labkey/api/assay/AssaySaveHandler.java

@@ -16,7 +16,16 @@
 
 package org.labkey.api.assay;
 
+import org.jetbrains.annotations.NotNull;
+import org.json.JSONObject;
+import org.labkey.api.assay.transform.PreparedTransforms;
+import org.labkey.api.exp.ExperimentException;
+import org.labkey.api.exp.api.ExpProtocol;
 import org.labkey.api.exp.api.ExperimentSaveHandler;
+import org.labkey.api.query.ValidationException;
+import org.labkey.api.view.ViewContext;
+
+import java.util.List;
 
 /**
  * User: dax
@@ -64,4 +73,14 @@ public interface AssaySaveHandler extends ExperimentSaveHandler
      */
     void setProvider(AssayProvider provider);
     AssayProvider getProvider();
+
+    /**
+     * Runs the transform scripts of the given runs before the save's transaction opens, so a script's callbacks into
+     * the server can't deadlock with it. The handler records the outputs as it saves each run; runs it doesn't prepare
+     * run their scripts inside the transaction. Don't rely on beforeSave having run: saveAssayRuns never calls it.
+     */
+    default @NotNull PreparedTransforms prepareTransforms(ViewContext context, ExpProtocol protocol, List<JSONObject> runJsons) throws ExperimentException, ValidationException
+    {
+        return new PreparedTransforms();
+    }
 }

api/src/org/labkey/api/assay/DefaultAssayRunCreator.java

@@ -29,6 +29,7 @@
 import org.labkey.api.assay.pipeline.AssayUploadPipelineJob;
 import org.labkey.api.assay.sample.AssaySampleLookupContext;
 import org.labkey.api.assay.transform.DataTransformService;
+import org.labkey.api.assay.transform.PendingFileAudit;
 import org.labkey.api.assay.transform.PreparedTransform;
 import org.labkey.api.assay.transform.TransformDataHandler;
 import org.labkey.api.assay.transform.TransformResult;
@@ -134,9 +135,9 @@ public TransformResult transform(AssayRunUploadContext<ProviderType> context, Ex
     }
 
     @Override
-    public @Nullable PreparedTransform prepareTransform(AssayRunUploadContext<ProviderType> context, boolean forceAsync) throws ExperimentException, ValidationException
+    public @Nullable PreparedTransform prepareTransform(AssayRunUploadContext<ProviderType> context) throws ExperimentException, ValidationException
     {
-        if (isBackgroundImport(context.getProvider(), context.getProtocol(), forceAsync) || !hasTransformBeforeRun(context))
+        if (!hasTransformBeforeRun(context))
             return null;
 
         context.init();
@@ -148,11 +149,32 @@ private PreparedTransform prepareTransformAfterInit(AssayRunUploadContext<Provid
         Map<DomainProperty, String> allProperties = new HashMap<>(context.getRunProperties());
         allProperties.putAll(context.getBatchProperties());
 
-        Map<ExpData, String> inputDatas = new HashMap<>();
-        addRunDataInputs(context, null, findResolverType(allProperties), inputDatas);
-        PreparedTransform prepared = runTransformBeforeRun(context, inputDatas.keySet());
-        // Run creation reuses these rather than collecting again, since a thaw list resolver writes a new file each time
-        prepared.setRunDataInputs(inputDatas);
+        // Collecting the inputs saves posted files, whose audit events belong to the import transaction that isn't open yet
+        List<PendingFileAudit> fileAudits = new ArrayList<>();
+        context.setPendingFileAudits(fileAudits);
+        Map<ExpData, String> resolverInputs = new HashMap<>();
+        PreparedTransform prepared;
+        try
+        {
+            ParticipantVisitResolverType resolverType = findResolverType(allProperties);
+            if (resolverType != null)
+                resolverType.configureRun(context, null, resolverInputs);
+            Map<ExpData, String> inputDatas = new HashMap<>(resolverInputs);
+            addInputDatas(context, inputDatas);
+            prepared = runTransformBeforeRun(context, inputDatas.keySet());
+        }
+        catch (Exception e)
+        {
+            PreparedTransform.addFileAuditEventsForRemainingFiles(context.getUser(), fileAudits);
+            throw e;
+        }
+        finally
+        {
+            context.setPendingFileAudits(null);
+        }
+        prepared.holdFileAudits(context.getUser(), fileAudits);
+        // Run creation reuses these rather than resolving again, since a thaw list resolver writes a new file each time
+        prepared.setResolverDataInputs(resolverInputs);
         return prepared;
     }
 
@@ -165,18 +187,6 @@ private PreparedTransform prepareTransformAfterInit(AssayRunUploadContext<Provid
             .orElse(null);
     }
 
-    private void addRunDataInputs(
-        AssayRunUploadContext<ProviderType> context,
-        @Nullable ExpRun run,
-        @Nullable ParticipantVisitResolverType resolverType,
-        Map<ExpData, String> inputDatas
-    ) throws ExperimentException, ValidationException
-    {
-        if (resolverType != null)
-            resolverType.configureRun(context, run, inputDatas);
-        addInputDatas(context, inputDatas);
-    }
-
     private boolean hasTransformBeforeRun(AssayRunUploadContext<ProviderType> context)
     {
         return context.getProvider().isTransformBeforeRunSupported(context.getProtocol())
@@ -396,10 +406,11 @@ public ExpExperiment saveExperimentRun(
 
         ParticipantVisitResolverType resolverType = findResolverType(allProperties);
         PreparedTransform preparedBeforeRun = context.getPreparedTransform();
-        if (preparedBeforeRun != null && preparedBeforeRun.getRunDataInputs() != null)
-            inputDatas.putAll(preparedBeforeRun.getRunDataInputs());
-        else
-            addRunDataInputs(context, run, resolverType, inputDatas);
+        if (preparedBeforeRun != null && preparedBeforeRun.getResolverDataInputs() != null)
+            inputDatas.putAll(preparedBeforeRun.getResolverDataInputs());
+        else if (resolverType != null)
+            resolverType.configureRun(context, run, inputDatas);
+        addInputDatas(context, inputDatas);
 
         // TODO: Share these RemapCache and materialCache instances with AbstractAssayTsvDataHandler.checkData and ExpressionMatrixDataHandler.importFile
         // Cache of resolved alternate lookup keys -> rowId
@@ -433,6 +444,8 @@ public ExpExperiment saveExperimentRun(
                     AbstractQueryUpdateService.addTransactionAuditEvent(transaction, context.getUser(), auditEvent);
                 }
             }
+            if (preparedBeforeRun != null)
+                preparedBeforeRun.recordFileAuditEvents(transaction);
             boolean saveBatchProps = forceSaveBatchProps;
 
             // Add any material/data inputs related to the specimen IDs, etc in the incoming data.

api/src/org/labkey/api/assay/actions/AssayRunUploadForm.java

@@ -49,6 +49,7 @@
 import org.labkey.api.exp.property.DomainProperty;
 import org.labkey.api.pipeline.PipelineService;
 import org.labkey.api.assay.transform.DefaultTransformResult;
+import org.labkey.api.assay.transform.PendingFileAudit;
 import org.labkey.api.assay.transform.PreparedTransform;
 import org.labkey.api.assay.transform.TransformResult;
 import org.labkey.api.qc.TsvDataExchangeHandler;
@@ -108,6 +109,7 @@ public class AssayRunUploadForm<ProviderType extends AssayProvider> extends Prot
     private List<AssayDataCollector> _collectors;
     private TransformResult _transformResult = DefaultTransformResult.createEmptyResult();
     private @Nullable PreparedTransform _preparedTransform;
+    private @Nullable List<PendingFileAudit> _pendingFileAudits;
     private ExpRun _reRun;
     private boolean _allowCrossRunFileInputs;
 
@@ -727,6 +729,18 @@ public void setPreparedTransform(@Nullable PreparedTransform prepared)
         _preparedTransform = prepared;
     }
 
+    @Override
+    public @Nullable List<PendingFileAudit> getPendingFileAudits()
+    {
+        return _pendingFileAudits;
+    }
+
+    @Override
+    public void setPendingFileAudits(@Nullable List<PendingFileAudit> audits)
+    {
+        _pendingFileAudits = audits;
+    }
+
     @Override
     public Long getReRunId()
     {

api/src/org/labkey/api/assay/transform/DataTransformService.java

@@ -27,7 +27,6 @@
 import org.labkey.api.exp.api.ExpData;
 import org.labkey.api.exp.api.ExpProtocol;
 import org.labkey.api.exp.api.ExpRun;
-import org.labkey.api.exp.api.ExperimentService;
 import org.labkey.api.exp.property.DomainProperty;
 import org.labkey.api.query.ValidationException;
 import org.labkey.api.reader.Readers;
@@ -155,8 +154,6 @@ public TransformResult transformAndValidate(
                             "the configuration for this assay design.");
                 }
 
-                logIfInTransaction(context, operation);
-
                 if (operation == TransformOperation.UPDATE)
                     SimpleMetricsService.get().increment(AssayMetricArea.TRANSFORM_SCRIPT, "executionCountForUpdate");
                 if (operation == TransformOperation.INSERT)
@@ -298,17 +295,6 @@ public TransformResult recordTransform(AssayRunUploadContext<?> context, @Nullab
         return prepared.getResult();
     }
 
-    private void logIfInTransaction(AssayRunUploadContext<?> context, TransformOperation operation)
-    {
-        if (LOG.isDebugEnabled()
-                && operation == TransformOperation.INSERT
-                && context.getProvider().isTransformBeforeRunSupported(context.getProtocol())
-                && ExperimentService.get().getSchema().getScope().isTransactionActive())
-        {
-            LOG.debug("Transform script for assay '{}' is running inside a database transaction; callbacks from the script into the server can deadlock", context.getProtocol().getName(), new Throwable());
-        }
-    }
-
     public void addStandardParameters(@Nullable HttpServletRequest request, @Nullable Container container, @Nullable FileLike scriptFile, @Nullable String apiKey, @NotNull Map<String, String> paramMap)
     {
         if (scriptFile != null)

api/src/org/labkey/api/assay/transform/PendingFileAudit.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.assay.transform;
+
+import org.jetbrains.annotations.NotNull;
+import org.labkey.api.audit.provider.FileSystemAuditProvider;
+import org.labkey.vfs.FileLike;
+
+/** The audit event of a file saved while preparing a transform, held until the import transaction opens. */
+public record PendingFileAudit(@NotNull FileSystemAuditProvider.FileSystemAuditEvent event, @NotNull FileLike file) {}