diff --git a/pom.xml b/pom.xml
index 674e7ffea..3afa66cab 100644
--- a/pom.xml
+++ b/pom.xml
@@ -464,6 +464,11 @@
micronaut-amazon-awssdk-s3
2.0.5-micronaut-2.0
+
+ com.github.samtools
+ htsjdk
+ 2.24.1
+
diff --git a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java
index 534043dc1..dab8cb7a2 100644
--- a/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java
+++ b/src/main/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImpl.java
@@ -1,9 +1,30 @@
+/*
+ * See the NOTICE file distributed with this work for additional information
+ * regarding copyright ownership.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.breedinginsight.services.geno.impl;
import com.agorapulse.micronaut.amazon.awssdk.s3.SimpleStorageService;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonObject;
+import htsjdk.samtools.util.CloseableIterator;
+import htsjdk.tribble.TribbleException;
+import htsjdk.variant.variantcontext.VariantContext;
+import htsjdk.variant.vcf.VCFFileReader;
import io.micronaut.context.annotation.Property;
import io.micronaut.http.HttpStatus;
import io.micronaut.http.multipart.CompletedFileUpload;
@@ -65,6 +86,8 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.time.LocalDateTime;
import java.time.OffsetDateTime;
import java.util.*;
@@ -81,6 +104,8 @@ public class GigwaGenotypeServiceImpl implements GenotypeService {
private static final String BEARER = "Bearer ";
private static final String GIGWA_REST_BASE_PATH = "gigwa/rest";
private static final String GIGWA_BRAPI_BASE_PATH = GIGWA_REST_BASE_PATH + BrapiVersion.BRAPI_V2;
+ private static final String INVALID_REF_ALT_MESSAGE = "The file is not a valid VCF or contains unsupported REF/ALT allele values.";
+ private static final String DUPLICATE_POSITIONAL_KEY_MESSAGE = "Duplicate chromosomal position(s) detected. CHROM:POS key must be unique for variant type.";
private static final MediaType MEDIA_TYPE_JSON = MediaType.parse("application/json");
@@ -211,7 +236,8 @@ public ImportResponse submitGenotypeData(UUID userId, UUID programId, UUID submi
try {
byte[] fileContents = uploadedFile.getBytes();
- if(validateSamples(program, submissionId, fileContents, upload)) {
+ if (validateSamples(program, submissionId, fileContents, upload)
+ && validateVariantRecords(fileContents, uploadedFile.getFilename(), upload)) {
executor.execute(() -> {
try {
processSubmission(gigwaAuthToken, program, submissionId, fileContents, uploadedFile.getFilename(), upload, progress);
@@ -391,6 +417,70 @@ private boolean validateSamples(Program program, UUID submissionId, byte[] fileC
return true;
}
+ private boolean validateVariantRecords(byte[] fileContents, String filename, ImportUpload upload) {
+ Set positionalKeys = new HashSet<>();
+ Path tempVcfFile = null;
+ int parsedVariantCount = 0;
+
+ try {
+ tempVcfFile = Files.createTempFile("bi-vcf-validation-", ".vcf");
+ Files.write(tempVcfFile, fileContents);
+
+ try (VCFFileReader reader = new VCFFileReader(tempVcfFile.toFile(), false);
+ CloseableIterator variants = reader.iterator()) {
+
+ while (variants.hasNext()) {
+ VariantContext variant = variants.next();
+ parsedVariantCount++;
+
+ String positionalKey =
+ variant.getType() + ":" +
+ variant.getContig() + ":" +
+ variant.getStart();
+
+ if (!positionalKeys.add(positionalKey)) {
+ log.error("Duplicate Gigwa positional key detected during VCF validation for file '{}'. Parsed records: {}. Key: {}",
+ filename, parsedVariantCount, positionalKey);
+
+ upload.getProgress().setStatuscode((short) HttpStatus.BAD_REQUEST.getCode());
+ upload.getProgress().setMessage(DUPLICATE_POSITIONAL_KEY_MESSAGE);
+ importDAO.updateProgress(upload.getProgress());
+ return false;
+ }
+ }
+ }
+
+ log.info("Completed HTSJDK VCF validation for file '{}'. Parsed {} variant record(s) with no validation errors",
+ filename, parsedVariantCount);
+ } catch (TribbleException | IllegalArgumentException e) {
+ log.error("HTSJDK VCF validation failed for file '{}'. Parsed {} variant record(s) before failure. Error: {}",
+ filename, parsedVariantCount, e.getMessage(), e);
+
+ upload.getProgress().setStatuscode((short) HttpStatus.BAD_REQUEST.getCode());
+ upload.getProgress().setMessage(INVALID_REF_ALT_MESSAGE);
+ importDAO.updateProgress(upload.getProgress());
+ return false;
+ } catch (IOException e) {
+ log.error("I/O failure during VCF validation setup for file '{}'. Parsed {} variant record(s) before failure. Error: {}",
+ filename, parsedVariantCount, e.getMessage(), e);
+
+ upload.getProgress().setStatuscode((short) HttpStatus.INTERNAL_SERVER_ERROR.getCode());
+ upload.getProgress().setMessage("An error occurred while trying to validate VCF variant information");
+ importDAO.updateProgress(upload.getProgress());
+ return false;
+ } finally {
+ if (tempVcfFile != null) {
+ try {
+ Files.deleteIfExists(tempVcfFile);
+ } catch (IOException e) {
+ log.warn("Unable to delete temporary VCF validation file {}", tempVcfFile, e);
+ }
+ }
+ }
+
+ return true;
+ }
+
private boolean validateVcfHeader(String[] headerParts) {
if(headerParts.length < 8) {
return false;
diff --git a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java
index d3c7c7415..e0cf269ac 100644
--- a/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java
+++ b/src/test/java/org/breedinginsight/services/geno/impl/GigwaGenotypeServiceImplIntegrationTest.java
@@ -579,6 +579,120 @@ public void testSubmitMissingSubmissionSamples() throws ApiException {
assertEquals("There are samples that are not linked to the selected submission", response.getProgress().getMessage());
}
+ @Test
+ public void testSubmitDuplicatePositionalKeysShowsSingleMessage() throws Exception {
+ UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+ String programKey = "TESTDUPKEY";
+ UUID submissionId = UUID.randomUUID();
+
+ setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_duplicate_positional_key.vcf"));
+
+ AtomicReference importResponse = new AtomicReference<>();
+ assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_duplicate_positional_key.vcf")), "Upload did not complete within the time period");
+
+ ImportResponse response = importResponse.get();
+ assertNotNull(response);
+ assertNotNull(response.getProgress());
+ assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
+ assertEquals("Duplicate chromosomal position(s) detected. CHROM:POS key must be unique for variant type.", response.getProgress().getMessage());
+ }
+
+ @Test
+ public void testSubmitSameChromPosDifferentVariantTypesAccepted() throws Exception {
+ UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+ String programKey = "TESTSAMEPOSDIFFTYPE";
+ UUID submissionId = UUID.randomUUID();
+
+ setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_same_pos_different_variant_type.vcf"));
+
+ AtomicReference importResponse = new AtomicReference<>();
+ assertTimeout(
+ Duration.of(2, ChronoUnit.MINUTES),
+ () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_same_pos_different_variant_type.vcf")),
+ "Upload did not complete within the time period"
+ );
+
+ ImportResponse response = importResponse.get();
+ assertNotNull(response);
+ assertNotNull(response.getProgress());
+ assertEquals(
+ (short) HttpStatus.ACCEPTED.getCode(),
+ response.getProgress().getStatuscode(),
+ "Error importing geno file: " + response.getProgress().getMessage()
+ );
+ }
+
+ @Test
+ public void testSubmitInvalidRefShowsSingleMessage() throws Exception {
+ UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+ String programKey = "TESTBADREF";
+ UUID submissionId = UUID.randomUUID();
+
+ setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_ref.vcf"));
+
+ AtomicReference importResponse = new AtomicReference<>();
+ assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_ref.vcf")), "Upload did not complete within the time period");
+
+ ImportResponse response = importResponse.get();
+ assertNotNull(response);
+ assertNotNull(response.getProgress());
+ assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
+ assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage());
+ }
+
+ @Test
+ public void testSubmitInvalidAltShowsSingleMessage() throws Exception {
+ UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+ String programKey = "TESTBADALT";
+ UUID submissionId = UUID.randomUUID();
+
+ setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_alt.vcf"));
+
+ AtomicReference importResponse = new AtomicReference<>();
+ assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_alt.vcf")), "Upload did not complete within the time period");
+
+ ImportResponse response = importResponse.get();
+ assertNotNull(response);
+ assertNotNull(response.getProgress());
+ assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
+ assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage());
+ }
+
+ @Test
+ public void testSubmitRefDotShowsSingleMessage() throws Exception {
+ UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+ String programKey = "TESTDOTREF";
+ UUID submissionId = UUID.randomUUID();
+
+ setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_invalid_ref_dot.vcf"));
+
+ AtomicReference importResponse = new AtomicReference<>();
+ assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_invalid_ref_dot.vcf")), "Upload did not complete within the time period");
+
+ ImportResponse response = importResponse.get();
+ assertNotNull(response);
+ assertNotNull(response.getProgress());
+ assertEquals((short) HttpStatus.BAD_REQUEST.getCode(), response.getProgress().getStatuscode());
+ assertEquals("The file is not a valid VCF or contains unsupported REF/ALT allele values.", response.getProgress().getMessage());
+ }
+
+ @Test
+ public void testSubmitMultiAllelicAltAccepted() throws Exception {
+ UUID programId = UUID.fromString("29162e85-e739-4f19-9fd0-0c377ed59956");
+ String programKey = "TESTMULTIALT";
+ UUID submissionId = UUID.randomUUID();
+
+ setupMocksForSubmitGenoData(programId, submissionId, buildSamplesFromVcf("sample_valid_multi_alt.vcf"));
+
+ AtomicReference importResponse = new AtomicReference<>();
+ assertTimeout(Duration.of(2, ChronoUnit.MINUTES), () -> importResponse.set(submitGenoData(programId, programKey, submissionId, "sample_valid_multi_alt.vcf")), "Upload did not complete within the time period");
+
+ ImportResponse response = importResponse.get();
+ assertNotNull(response);
+ assertNotNull(response.getProgress());
+ assertEquals((short) HttpStatus.ACCEPTED.getCode(), response.getProgress().getStatuscode(), "Error importing geno file: " + response.getProgress().getMessage());
+ }
+
private void setupMocksForSubmitGenoData(UUID programId, UUID submissionId, List samples) throws ApiException {
SampleSubmission submission = new SampleSubmission();
submission.setId(submissionId);
@@ -685,7 +799,11 @@ private ImportResponse submitGenoData(UUID programId, String programKey, UUID su
}
private List buildSamplesFromValidVcf() throws IOException {
- try (Scanner sc = new Scanner(new FileInputStream("src/test/resources/files/geno/sample.vcf"), "UTF-8")) {
+ return buildSamplesFromVcf("sample.vcf");
+ }
+
+ private List buildSamplesFromVcf(String fileName) throws IOException {
+ try (Scanner sc = new Scanner(new FileInputStream("src/test/resources/files/geno/" + fileName), "UTF-8")) {
String[] headerParts = null;
boolean foundHeader = false;
while (sc.hasNextLine() && !foundHeader) {
@@ -696,7 +814,7 @@ private List buildSamplesFromValidVcf() throws IOException {
}
}
- assertTrue(foundHeader, "Could not find sample.vcf header file");
+ assertTrue(foundHeader, "Could not find " + fileName + " header file");
List samples = new ArrayList<>();
for (int i = 9; i < headerParts.length; i++) {
diff --git a/src/test/resources/files/geno/sample_duplicate_positional_key.vcf b/src/test/resources/files/geno/sample_duplicate_positional_key.vcf
new file mode 100644
index 000000000..a538b8624
--- /dev/null
+++ b/src/test/resources/files/geno/sample_duplicate_positional_key.vcf
@@ -0,0 +1,4 @@
+##fileformat=VCFv4.2
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01
+1 100 var1 A T . PASS . GT 0/1
+1 100 var2 G C . PASS . GT 0/1
diff --git a/src/test/resources/files/geno/sample_invalid_alt.vcf b/src/test/resources/files/geno/sample_invalid_alt.vcf
new file mode 100644
index 000000000..3563d66c5
--- /dev/null
+++ b/src/test/resources/files/geno/sample_invalid_alt.vcf
@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01
+1 100 var1 A - . PASS . GT 0/1
diff --git a/src/test/resources/files/geno/sample_invalid_ref.vcf b/src/test/resources/files/geno/sample_invalid_ref.vcf
new file mode 100644
index 000000000..22e7fb0e1
--- /dev/null
+++ b/src/test/resources/files/geno/sample_invalid_ref.vcf
@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01
+1 100 var1 X T . PASS . GT 0/1
diff --git a/src/test/resources/files/geno/sample_invalid_ref_dot.vcf b/src/test/resources/files/geno/sample_invalid_ref_dot.vcf
new file mode 100644
index 000000000..97d4db7dc
--- /dev/null
+++ b/src/test/resources/files/geno/sample_invalid_ref_dot.vcf
@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01
+1 100 var1 . T . PASS . GT 0/1
diff --git a/src/test/resources/files/geno/sample_same_pos_different_variant_type.vcf b/src/test/resources/files/geno/sample_same_pos_different_variant_type.vcf
new file mode 100644
index 000000000..a3743e676
--- /dev/null
+++ b/src/test/resources/files/geno/sample_same_pos_different_variant_type.vcf
@@ -0,0 +1,4 @@
+##fileformat=VCFv4.2
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01
+1 100 var1 A T . PASS . GT 0/1
+1 100 var2 A AG . PASS . GT 0/1
\ No newline at end of file
diff --git a/src/test/resources/files/geno/sample_valid_missing_ref_alt.vcf b/src/test/resources/files/geno/sample_valid_missing_ref_alt.vcf
new file mode 100644
index 000000000..df0ebb00f
--- /dev/null
+++ b/src/test/resources/files/geno/sample_valid_missing_ref_alt.vcf
@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01
+1 100 var1 . . . PASS . GT 0/0
diff --git a/src/test/resources/files/geno/sample_valid_multi_alt.vcf b/src/test/resources/files/geno/sample_valid_multi_alt.vcf
new file mode 100644
index 000000000..2f0b66b04
--- /dev/null
+++ b/src/test/resources/files/geno/sample_valid_multi_alt.vcf
@@ -0,0 +1,3 @@
+##fileformat=VCFv4.2
+#CHROM POS ID REF ALT QUAL FILTER INFO FORMAT USDAMSP1_A01
+1 100 var1 C CAG,T,CGG . PASS . GT 1/2