BI-2973: Prevent Duplicate Variants Error from Blocking Subsequent Genotype Data Imports

pom.xml

@@ -464,6 +464,11 @@
             <artifactId>micronaut-amazon-awssdk-s3</artifactId>
             <version>2.0.5-micronaut-2.0</version>
         </dependency>
+        <dependency>
+            <groupId>com.github.samtools</groupId>
+            <artifactId>htsjdk</artifactId>
+            <version>2.24.1</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java

@@ -1,9 +1,30 @@
+/*
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 package org.breedinginsight.services.geno.impl;
 
 import com.agorapulse.micronaut.amazon.awssdk.s3.SimpleStorageService;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
 import com.google.gson.JsonObject;
+import htsjdk.samtools.util.CloseableIterator;
+import htsjdk.tribble.TribbleException;
+import htsjdk.variant.variantcontext.VariantContext;
+import htsjdk.variant.vcf.VCFFileReader;
 import io.micronaut.context.annotation.Property;
 import io.micronaut.http.HttpStatus;
 import io.micronaut.http.multipart.CompletedFileUpload;
@@ -65,6 +86,8 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.LocalDateTime;
 import java.time.OffsetDateTime;
 import java.util.*;
@@ -81,6 +104,8 @@ public class GigwaGenotypeServiceImpl implements GenotypeService {
     private static final String BEARER = "Bearer ";
     private static final String GIGWA_REST_BASE_PATH = "gigwa/rest";
     private static final String GIGWA_BRAPI_BASE_PATH = GIGWA_REST_BASE_PATH + BrapiVersion.BRAPI_V2;
+    private static final String INVALID_REF_ALT_MESSAGE = "The file is not a valid VCF or contains unsupported REF/ALT allele values.";
+    private static final String DUPLICATE_POSITIONAL_KEY_MESSAGE = "Duplicate chromosomal position(s) detected. CHROM:POS key must be unique for variant type.";
 
     private static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json");
 
@@ -211,7 +236,8 @@ public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID submi
 
         try {
             byte[] fileContents = uploadedFile.getBytes();
-            if(validateSamples(program, submissionId, fileContents, upload)) {
+            if (validateSamples(program, submissionId, fileContents, upload)
+                    && validateVariantRecords(fileContents, uploadedFile.getFilename(), upload)) {
                 executor.execute(() -> {
                     try {
                         processSubmission(gigwaAuthToken, program, submissionId, fileContents, uploadedFile.getFilename(), upload, progress);
@@ -391,6 +417,70 @@ private boolean validateSamples(Program program, UUID submissionId, byte[] fileC
         return true;
     }
 
+    private boolean validateVariantRecords(byte[] fileContents, String filename, ImportUpload upload) {
+        Set<String> positionalKeys = new HashSet<>();
+        Path tempVcfFile = null;
+        int parsedVariantCount = 0;
+
+        try {
+            tempVcfFile = Files.createTempFile("bi-vcf-validation-", ".vcf");
+            Files.write(tempVcfFile, fileContents);
+
+            try (VCFFileReader reader = new VCFFileReader(tempVcfFile.toFile(), false);
+                 CloseableIterator<VariantContext> variants = reader.iterator()) {
+
+                while (variants.hasNext()) {
+                    VariantContext variant = variants.next();
+                    parsedVariantCount++;
+
+                    String positionalKey =
+                            variant.getType() + ":" +
+                                    variant.getContig() + ":" +
+                                    variant.getStart();
+
+                    if (!positionalKeys.add(positionalKey)) {
+                        log.error("Duplicate Gigwa positional key detected during VCF validation for file '{}'. Parsed records: {}. Key: {}",
+                                filename, parsedVariantCount, positionalKey);
+
+                        upload.getProgress().setStatuscode((short) HttpStatus.BAD_REQUEST.getCode());
+                        upload.getProgress().setMessage(DUPLICATE_POSITIONAL_KEY_MESSAGE);
+                        importDAO.updateProgress(upload.getProgress());
+                        return false;
+                    }
+                }
+            }
+
+            log.info("Completed HTSJDK VCF validation for file '{}'. Parsed {} variant record(s) with no validation errors",
+                    filename, parsedVariantCount);
+        } catch (TribbleException | IllegalArgumentException e) {
+            log.error("HTSJDK VCF validation failed for file '{}'. Parsed {} variant record(s) before failure. Error: {}",
+                    filename, parsedVariantCount, e.getMessage(), e);
+
+            upload.getProgress().setStatuscode((short) HttpStatus.BAD_REQUEST.getCode());
+            upload.getProgress().setMessage(INVALID_REF_ALT_MESSAGE);
+            importDAO.updateProgress(upload.getProgress());
+            return false;
+        } catch (IOException e) {
+            log.error("I/O failure during VCF validation setup for file '{}'. Parsed {} variant record(s) before failure. Error: {}",
+                    filename, parsedVariantCount, e.getMessage(), e);
+
+            upload.getProgress().setStatuscode((short) HttpStatus.INTERNAL_SERVER_ERROR.getCode());
+            upload.getProgress().setMessage("An error occurred while trying to validate VCF variant information");
+            importDAO.updateProgress(upload.getProgress());
+            return false;
+        } finally {
+            if (tempVcfFile != null) {
+                try {
+                    Files.deleteIfExists(tempVcfFile);
+                } catch (IOException e) {
+                    log.warn("Unable to delete temporary VCF validation file {}", tempVcfFile, e);
+                }
+            }
+        }
+
+        return true;
+    }
+
     private boolean validateVcfHeader(String[] headerParts) {
         if(headerParts.length < 8) {
             return false;

src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java

@@ -579,6 +579,120 @@ public void testSubmitMissingSubmissionSamples() throws ApiException {
         assertEquals("There are samples that are not linked to the selected submission", response.getProgress().getMessage());
     }
 
+    @Test
+    public void testSubmitDuplicatePositionalKeysShowsSingleMessage() throws Exception {
+        UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+        String programKey = "TESTDUPKEY";
+        UUID submissionId = UUID.randomUUID();
+
+        setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_duplicate_positional_key.vcf"));
+
+        AtomicReference<ImportResponse> importResponse = new AtomicReference<>();
+        assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_duplicate_positional_key.vcf")), "Upload did not complete within the time period");
+
+        ImportResponse response = importResponse.get();
+        assertNotNull(response);
+        assertNotNull(response.getProgress());
+        assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
+        assertEquals("Duplicate chromosomal position(s) detected. CHROM:POS key must be unique for variant type.", response.getProgress().getMessage());
+    }
+
+    @Test
+    public void testSubmitSameChromPosDifferentVariantTypesAccepted() throws Exception {
+        UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+        String programKey = "TESTSAMEPOSDIFFTYPE";
+        UUID submissionId = UUID.randomUUID();
+
+        setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_same_pos_different_variant_type.vcf"));
+
+        AtomicReference<ImportResponse> importResponse = new AtomicReference<>();
+        assertTimeout(
+                Duration.of(2, ChronoUnit.MINUTES),
+                () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_same_pos_different_variant_type.vcf")),
+                "Upload did not complete within the time period"
+        );
+
+        ImportResponse response = importResponse.get();
+        assertNotNull(response);
+        assertNotNull(response.getProgress());
+        assertEquals(
+                (short) HttpStatus.ACCEPTED.getCode(),
+                response.getProgress().getStatuscode(),
+                "Error importing geno file: " + response.getProgress().getMessage()
+        );
+    }
+
+    @Test
+    public void testSubmitInvalidRefShowsSingleMessage() throws Exception {
+        UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+        String programKey = "TESTBADREF";
+        UUID submissionId = UUID.randomUUID();
+
+        setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_ref.vcf"));
+
+        AtomicReference<ImportResponse> importResponse = new AtomicReference<>();
+        assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_ref.vcf")), "Upload did not complete within the time period");
+
+        ImportResponse response = importResponse.get();
+        assertNotNull(response);
+        assertNotNull(response.getProgress());
+        assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
+        assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage());
+    }
+
+    @Test
+    public void testSubmitInvalidAltShowsSingleMessage() throws Exception {
+        UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+        String programKey = "TESTBADALT";
+        UUID submissionId = UUID.randomUUID();
+
+        setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_alt.vcf"));
+
+        AtomicReference<ImportResponse> importResponse = new AtomicReference<>();
+        assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_alt.vcf")), "Upload did not complete within the time period");
+
+        ImportResponse response = importResponse.get();
+        assertNotNull(response);
+        assertNotNull(response.getProgress());
+        assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
+        assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage());
+    }
+
+    @Test
+    public void testSubmitRefDotShowsSingleMessage() throws Exception {
+        UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+        String programKey = "TESTDOTREF";
+        UUID submissionId = UUID.randomUUID();
+
+        setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_ref_dot.vcf"));
+
+        AtomicReference<ImportResponse> importResponse = new AtomicReference<>();
+        assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_ref_dot.vcf")), "Upload did not complete within the time period");
+
+        ImportResponse response = importResponse.get();
+        assertNotNull(response);
+        assertNotNull(response.getProgress());
+        assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
+        assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage());
+    }
+
+    @Test
+    public void testSubmitMultiAllelicAltAccepted() throws Exception {
+        UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+        String programKey = "TESTMULTIALT";
+        UUID submissionId = UUID.randomUUID();
+
+        setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_valid_multi_alt.vcf"));
+
+        AtomicReference<ImportResponse> importResponse = new AtomicReference<>();
+        assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_valid_multi_alt.vcf")), "Upload did not complete within the time period");
+
+        ImportResponse response = importResponse.get();
+        assertNotNull(response);
+        assertNotNull(response.getProgress());
+        assertEquals((short) HttpStatus.ACCEPTED.getCode(), response.getProgress().getStatuscode(), "Error importing geno file: " + response.getProgress().getMessage());
+    }
+
     private void setupMocksForSubmitGenoData(UUID programId, UUID submissionId, List<BrAPISample> samples) throws ApiException {
         SampleSubmission submission = new SampleSubmission();
         submission.setId(submissionId);
@@ -685,7 +799,11 @@ private ImportResponse submitGenoData(UUID programId, String programKey, UUID su
     }
 
     private List<BrAPISample> buildSamplesFromValidVcf() throws IOException {
-        try (Scanner sc = new Scanner(new FileInputStream("src/test/resources/files/geno/sample.vcf"), "UTF-8")) {
+        return buildSamplesFromVcf("sample.vcf");
+    }
+
+    private List<BrAPISample> buildSamplesFromVcf(String fileName) throws IOException {
+        try (Scanner sc = new Scanner(new FileInputStream("src/test/resources/files/geno/" + fileName), "UTF-8")) {
             String[] headerParts = null;
             boolean foundHeader = false;
             while (sc.hasNextLine() && !foundHeader) {
@@ -696,7 +814,7 @@ private List<BrAPISample> buildSamplesFromValidVcf() throws IOException {
                 }
             }
 
-            assertTrue(foundHeader, "Could not find sample.vcf header file");
+            assertTrue(foundHeader, "Could not find " + fileName + " header file");
 
             List<BrAPISample> samples = new ArrayList<>();
             for (int i = 9; i < headerParts.length; i++) {

src/test/resources/files/geno/sample_duplicate_positional_key.vcf

@@ -0,0 +1,4 @@
+##fileformat=VCFv4.2
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	USDAMSP1_A01
+1	100	var1	A	T	.	PASS	.	GT	0/1
+1	100	var2	G	C	.	PASS	.	GT	0/1

src/test/resources/files/geno/sample_invalid_alt.vcf

@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	USDAMSP1_A01
+1	100	var1	A	-	.	PASS	.	GT	0/1

src/test/resources/files/geno/sample_invalid_ref.vcf

@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	USDAMSP1_A01
+1	100	var1	X	T	.	PASS	.	GT	0/1

src/test/resources/files/geno/sample_invalid_ref_dot.vcf

@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	USDAMSP1_A01
+1	100	var1	.	T	.	PASS	.	GT	0/1

src/test/resources/files/geno/sample_same_pos_different_variant_type.vcf

@@ -0,0 +1,4 @@
+##fileformat=VCFv4.2
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	USDAMSP1_A01
+1	100	var1	A	T	.	PASS	.	GT	0/1
+1	100	var2	A	AG	.	PASS	.	GT	0/1

src/test/resources/files/geno/sample_valid_missing_ref_alt.vcf

@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	USDAMSP1_A01
+1	100	var1	.	.	.	PASS	.	GT	0/0

src/test/resources/files/geno/sample_valid_multi_alt.vcf

@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	USDAMSP1_A01
+1	100	var1	C	CAG,T,CGG	.	PASS	.	GT	1/2