getFieldKeys(TableInfo ti)
keys.add(10, FieldKey.fromString("Id/demographics/gender"));
keys.add(12, FieldKey.fromString("Id/demographics/geographic_origin"));
+ // the social code sits beside Initial Location, whose index the inserts above have shifted, so find it
+ keys.add(keys.indexOf(FieldKey.fromString("cage")) + 1, FieldKey.fromString("Id/demographics/socialCode"));
+
return keys;
}
}
diff --git a/nbri_ehr/src/org/labkey/nbri_ehr/dataentry/section/NBRIBirthFormSection.java b/nbri_ehr/src/org/labkey/nbri_ehr/dataentry/section/NBRIBirthFormSection.java
index ddecd53..326b540 100644
--- a/nbri_ehr/src/org/labkey/nbri_ehr/dataentry/section/NBRIBirthFormSection.java
+++ b/nbri_ehr/src/org/labkey/nbri_ehr/dataentry/section/NBRIBirthFormSection.java
@@ -37,8 +37,8 @@ public class NBRIBirthFormSection extends NewAnimalFormSection
FieldKey.fromString("Id/demographics/dam"),
FieldKey.fromString("Id/demographics/sire"),
FieldKey.fromString("cage"),
+ FieldKey.fromString("Id/demographics/socialCode"),
FieldKey.fromString("type"),
- FieldKey.fromString("cond"),
FieldKey.fromString("breedingType"),
FieldKey.fromString("remark"),
FieldKey.fromString("performedby")
diff --git a/nbri_ehr/src/org/labkey/nbri_ehr/query/NBRI_EHRTriggerHelper.java b/nbri_ehr/src/org/labkey/nbri_ehr/query/NBRI_EHRTriggerHelper.java
index 662ab54..893c53a 100644
--- a/nbri_ehr/src/org/labkey/nbri_ehr/query/NBRI_EHRTriggerHelper.java
+++ b/nbri_ehr/src/org/labkey/nbri_ehr/query/NBRI_EHRTriggerHelper.java
@@ -323,6 +323,103 @@ public boolean deathExists(String id)
return false;
}
+ /**
+ * Derives the denormalized birth/death values on study.demographics from the birth and deaths event records, which
+ * are authoritative, and returns only the animals whose stored values disagree. The result is intended to be handed
+ * straight to the shared trigger helper's updateDemographicsRecord(), so that lsid resolution and the demographics
+ * cache recache stay in the single place that already handles them.
+ *
+ * Only public (Completed) event records count, so a record still in data entry never overwrites a saved value.
+ *
+ * calculated_status is deliberately absent from the result. It belongs to the shared status recalc, which owns the
+ * death/departure/re-arrival precedence.
+ *
+ * Every lookup is set-based - one query per event dataset for the whole id list, not one per animal - because a
+ * bulk save can pass hundreds of ids and per-animal SQL in a trigger exhausts the script's wall-clock budget.
+ *
+ * @param ids animals touched by the current save
+ * @return rows ready for updateDemographicsRecord(); empty when nothing has drifted
+ */
+ public List> computeDemographicsSync(List ids)
+ {
+ if (ids == null || ids.isEmpty())
+ return Collections.emptyList();
+
+ Set idSet = new HashSet<>(ids);
+
+ Map births = getPublicEventDates("birth", idSet);
+ Map deaths = getPublicEventDates("deaths", idSet);
+
+ List> updates = new ArrayList<>();
+
+ TableInfo demographics = getTableInfo("study", "demographics");
+ SimpleFilter filter = new SimpleFilter(FieldKey.fromString("Id"), idSet, CompareType.IN);
+ TableSelector ts = new TableSelector(demographics, PageFlowUtil.set("Id", "birth", "death"), filter, null);
+
+ for (Map current : ts.getMapCollection())
+ {
+ String id = (String)current.get("Id");
+ Map update = new CaseInsensitiveHashMap<>();
+
+ // A public event record always wins - the same rule createDemographicsRecord() already applies to death on
+ // insert, extended to updates and to birth. The absence of an event record is NOT evidence the stored value
+ // is wrong: animals loaded by ETL, or acquired before these datasets were in use, legitimately carry a date
+ // with no event row, so a missing record leaves the value alone. Clearing a value is only ever driven by an
+ // explicit delete of the event record.
+ Date birth = births.get(id);
+ if (birth != null && differsByDay(birth, (Date)current.get("birth")))
+ update.put("birth", birth);
+
+ Date death = deaths.get(id);
+ if (death != null && differsByDay(death, (Date)current.get("death")))
+ update.put("death", death);
+
+ if (!update.isEmpty())
+ {
+ update.put("Id", id);
+ updates.add(update);
+ }
+ }
+
+ if (!updates.isEmpty())
+ _log.info("Demographics birth/death out of sync with event records for {} animal(s); updating", updates.size());
+
+ return updates;
+ }
+
+ /** Most recent public event date per animal for a demographic event dataset, in a single query. */
+ private Map getPublicEventDates(String queryName, Set ids)
+ {
+ SimpleFilter filter = new SimpleFilter(FieldKey.fromString("Id"), ids, CompareType.IN);
+ filter.addCondition(FieldKey.fromString("qcstate/publicdata"), true);
+
+ Map ret = new HashMap<>();
+ new TableSelector(getTableInfo("study", queryName), PageFlowUtil.set("Id", "date"), filter, null)
+ .forEachMap(row -> {
+ String id = (String)row.get("Id");
+ Date date = ConvertHelper.convert(row.get("date"), Date.class);
+ // birth and deaths are demographic datasets (one row per animal), but tolerate duplicates from a
+ // legacy load by keeping the latest rather than picking arbitrarily.
+ if (date != null && (ret.get(id) == null || date.after(ret.get(id))))
+ ret.put(id, date);
+ });
+
+ return ret;
+ }
+
+ /**
+ * Compares to day precision. Event dates are entered with the time stripped, but values that arrived by ETL or
+ * predate that behavior can carry a time component; treating those as drift would rewrite the whole colony on the
+ * first save.
+ */
+ private boolean differsByDay(Date a, Date b)
+ {
+ if (a == null || b == null)
+ return a != b;
+
+ return !DateUtils.isSameDay(a, b);
+ }
+
public boolean upsertWeightRecord(Map row) throws QueryUpdateServiceException, DuplicateKeyException, SQLException, BatchValidationException, InvalidKeyException
{
return upsertWeightRecord(row, true);
diff --git a/nbri_ehr/test/sampledata/nbri_ehr/study/study/datasets/datasetBirth.tsv b/nbri_ehr/test/sampledata/nbri_ehr/study/study/datasets/datasetBirth.tsv
index cecdcb7..6ca8a47 100644
--- a/nbri_ehr/test/sampledata/nbri_ehr/study/study/datasets/datasetBirth.tsv
+++ b/nbri_ehr/test/sampledata/nbri_ehr/study/study/datasets/datasetBirth.tsv
@@ -1,9 +1,9 @@
objectid Id date QCStateLabel performedby
-1 44444 -2895d Completed 1004
-2 44445 -2854d Completed 1004
-3 44446 -2703d Completed 1004
+1 44444 -1381d Completed 1004
+2 44445 -1414d Completed 1004
+3 44446 -1406d Completed 1004
4 44447 -2600d Completed 1004
-5 TEST6390238 -5601d Completed 1004
-6 TEST5904521 -5602d Completed 1004
-7 TEST3804589 -2603d Completed 1004
-8 TEST2312318 -3000d Completed 1004
+5 TEST6390238 -3923d Completed 1004
+6 TEST5904521 -5431d Completed 1004
+7 TEST3804589 -5806d Completed 1004
+8 TEST2312318 -8069d Completed 1004
diff --git a/nbri_ehr/test/sampledata/nbri_ehr/study/study/datasets/datasetDeaths.tsv b/nbri_ehr/test/sampledata/nbri_ehr/study/study/datasets/datasetDeaths.tsv
index 0c46745..0a481b8 100644
--- a/nbri_ehr/test/sampledata/nbri_ehr/study/study/datasets/datasetDeaths.tsv
+++ b/nbri_ehr/test/sampledata/nbri_ehr/study/study/datasets/datasetDeaths.tsv
@@ -1,3 +1,3 @@
objectid Id date QCStateLabel cause manner remark performedby
-1 TEST1993532 -2212d Completed 1 S sed nibh viverra 1004
-2 44445 -723d Completed 2 X labor omnia vincit 1004
+1 TEST1993532 -2259d Completed 1 S sed nibh viverra 1004
+2 44445 -726d Completed 2 X labor omnia vincit 1004
diff --git a/nbri_ehr/test/src/org.labkey.test/tests.nbri_ehr/NBRI_EHRTest.java b/nbri_ehr/test/src/org.labkey.test/tests.nbri_ehr/NBRI_EHRTest.java
index afaab10..73501bc 100644
--- a/nbri_ehr/test/src/org.labkey.test/tests.nbri_ehr/NBRI_EHRTest.java
+++ b/nbri_ehr/test/src/org.labkey.test/tests.nbri_ehr/NBRI_EHRTest.java
@@ -71,6 +71,7 @@
import java.io.File;
import java.io.IOException;
import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
@@ -632,9 +633,13 @@ public void testWeightValidation()
}
@Test
- public void testArrivalForm()
+ public void testArrivalForm() throws IOException, CommandException
{
String arrivedAnimal = "30905";
+ // demographics.socialCode holds an ehr_lookups.social_code code; the grids display its title
+ String socialCode = "Acquired";
+ // animal_group_members.groupId holds an ehr_lookups.breeding_type code; the grids display its title
+ String animalGroup = "Assigned Breeding Protocol";
LocalDateTime now = LocalDateTime.now();
gotoEnterData();
@@ -653,6 +658,12 @@ public void testArrivalForm()
arrivals.setGridCell(1, "Id/demographics/species", "Pig-Tailed Macaque");
arrivals.setGridCellJS(1, "Id/demographics/birth", now.minusDays(7).format(DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_STRING)));
arrivals.setGridCell(1, "sourceFacility", "Bioqual, Incorporated");
+ arrivals.setGridCell(1, "Id/demographics/socialCode", socialCode);
+
+ log("Verifying Social Code is required");
+ arrivals.setGridCellJS(1, "Id/demographics/socialCode", null);
+ waitForFormError("The field: Social Code is required");
+ arrivals.setGridCell(1, "Id/demographics/socialCode", socialCode);
Ext4GridRef protocolAssignments = _helper.getExt4GridForFormSection("Protocol Assignment");
_helper.addRecordToGrid(protocolAssignments);
@@ -666,6 +677,12 @@ public void testArrivalForm()
projectAssignments.setGridCellJS(1, "date", now.minusDays(1).format(DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_STRING)));
projectAssignments.setGridCell(1, "project", "640991");
+ Ext4GridRef groupAssignments = _helper.getExt4GridForFormSection("Group Assignments");
+ _helper.addRecordToGrid(groupAssignments);
+ groupAssignments.setGridCell(1, "Id", arrivedAnimal);
+ groupAssignments.setGridCellJS(1, "date", now.minusDays(1).format(DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_STRING)));
+ groupAssignments.setGridCell(1, "groupId", animalGroup);
+
submitForm("Submit Final", "Finalize");
goToSchemaBrowser();
@@ -687,11 +704,31 @@ public void testArrivalForm()
table.setFilter("Id", "Equals", arrivedAnimal);
Assert.assertEquals("Invalid protocol assignment", Arrays.asList("dummyprotocol"), table.getRowDataAsText(0, "protocol"));
+ goToSchemaBrowser();
+ table = viewQueryData("study", "animal_group_members");
+ table.setFilter("Id", "Equals", arrivedAnimal);
+ Assert.assertEquals("Invalid group assignment", Arrays.asList(animalGroup), table.getRowDataAsText(0, "groupId"));
+
verifyRowCreated("study", "birth", arrivedAnimal, 1);
verifyRowCreated("study", "assignment", arrivedAnimal, 1);
verifyRowCreated("study", "protocolAssignment", arrivedAnimal, 1);
+ verifyRowCreated("study", "animal_group_members", arrivedAnimal, 1);
verifyRowCreated("study", "demographics", arrivedAnimal, 1);
verifyRowCreated("study", "housing", arrivedAnimal, 1);
+
+ log("Verifying the social code reached demographics");
+ goToSchemaBrowser();
+ table = viewQueryData("study", "demographics");
+ table.setFilter("Id", "Equals", arrivedAnimal);
+ Assert.assertEquals("Social code entered on the arrival form did not reach demographics",
+ Arrays.asList(socialCode), table.getRowDataAsText(0, "socialCode"));
+
+ log("Verifying the birth date reached demographics and agrees with the birth record");
+ String arrivalBirthDay = now.minusDays(7).format(_dateFormat);
+ assertEquals("Birth record does not carry the birth date entered on the arrival form",
+ arrivalBirthDay, getDatasetDay("birth", arrivedAnimal, "date"));
+ assertEquals("Demographics birth date does not match the birth record",
+ arrivalBirthDay, getDatasetDay("demographics", arrivedAnimal, "birth"));
}
@Test
@@ -705,6 +742,10 @@ public void testBirthForm() throws Exception
String damSpecies = "Brown-Tufted Capuchin";
String conceptId = "TESTCONCEPT1";
String breedingType = "Time-Mated";
+ // demographics.socialCode holds an ehr_lookups.social_code code; the grids display its title
+ String socialCode = "Mother-rearing (for indoors)";
+ // animal_group_members.groupId holds an ehr_lookups.breeding_type code; the grids display its title
+ String animalGroup = "Project Breeding";
LocalDateTime now = LocalDateTime.now();
log("Creating the dam and sire of the conception");
@@ -748,6 +789,12 @@ public void testBirthForm() throws Exception
births.setGridCell(1, "cage", "C3");
births.setGridCell(1, "Id/demographics/gender", "Female");
births.setGridCell(1, "breedingType", breedingType);
+ births.setGridCell(1, "Id/demographics/socialCode", socialCode);
+
+ log("Verifying Social Code is required");
+ births.setGridCellJS(1, "Id/demographics/socialCode", null);
+ waitForFormError("The field: Social Code is required");
+ births.setGridCell(1, "Id/demographics/socialCode", socialCode);
Ext4GridRef protocolAssignments = _helper.getExt4GridForFormSection("Protocol Assignment");
_helper.addRecordToGrid(protocolAssignments);
@@ -761,6 +808,12 @@ public void testBirthForm() throws Exception
projectAssignments.setGridCellJS(1, "date", now.minusDays(1).format(DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_STRING)));
projectAssignments.setGridCell(1, "project", "795644");
+ Ext4GridRef groupAssignments = _helper.getExt4GridForFormSection("Group Assignments");
+ _helper.addRecordToGrid(groupAssignments);
+ groupAssignments.setGridCell(1, "Id", bornAnimal);
+ groupAssignments.setGridCellJS(1, "date", now.minusDays(1).format(DateTimeFormatter.ofPattern(DATE_TIME_FORMAT_STRING)));
+ groupAssignments.setGridCell(1, "groupId", animalGroup);
+
submitForm("Submit Final", "Finalize");
goToSchemaBrowser();
@@ -778,6 +831,8 @@ public void testBirthForm() throws Exception
Assert.assertEquals("Invalid demographics record", Arrays.asList(damId), table.getRowDataAsText(0, "dam"));
Assert.assertEquals("Invalid demographics record", Arrays.asList(sireId), table.getRowDataAsText(0, "sire"));
Assert.assertEquals("Invalid demographics record", Arrays.asList(damSpecies), table.getRowDataAsText(0, "species"));
+ Assert.assertEquals("Social code entered on the birth form did not reach demographics",
+ Arrays.asList(socialCode), table.getRowDataAsText(0, "socialCode"));
goToSchemaBrowser();
table = viewQueryData("study", "assignment");
@@ -789,11 +844,24 @@ public void testBirthForm() throws Exception
table.setFilter("Id", "Equals", bornAnimal);
Assert.assertEquals("Invalid protocol assignment", Arrays.asList("protocol101"), table.getRowDataAsText(0, "protocol"));
+ goToSchemaBrowser();
+ table = viewQueryData("study", "animal_group_members");
+ table.setFilter("Id", "Equals", bornAnimal);
+ Assert.assertEquals("Invalid group assignment", Arrays.asList(animalGroup), table.getRowDataAsText(0, "groupId"));
+
verifyRowCreated("study", "assignment", bornAnimal, 1);
verifyRowCreated("study", "protocolAssignment", bornAnimal, 1);
+ verifyRowCreated("study", "animal_group_members", bornAnimal, 1);
verifyRowCreated("study", "housing", bornAnimal, 1);
verifyRowCreated("study", "demographics", bornAnimal, 1);
+ log("Verifying the birth date reached demographics and agrees with the birth record");
+ String bornBirthDay = now.minusDays(1).format(_dateFormat);
+ assertEquals("Birth record does not carry the date entered on the birth form",
+ bornBirthDay, getDatasetDay("birth", bornAnimal, "date"));
+ assertEquals("Demographics birth date does not match the birth record",
+ bornBirthDay, getDatasetDay("demographics", bornAnimal, "birth"));
+
log("Verifying conception outcome and offspring in ConceptionsByDam");
goToSchemaBrowser();
DataRegionTable report = viewQueryData("nbri_ehr", "ConceptionsByDam");
@@ -863,7 +931,6 @@ public void testConceptionForm()
_helper.addRecordToGrid(conceptions);
conceptions.setGridCell(1, "ConceptId", conceptId);
conceptions.setGridCellJS(1, "ConceptDate", now.minusDays(30).format(_dateFormat));
- conceptions.setGridCellJS(1, "ConceptTermDate", now.plusDays(135).format(_dateFormat));
conceptions.setGridCellJS(1, "Estimated", true);
conceptions.setGridCell(1, "Dam", damId);
conceptions.setGridCell(1, "Sire", sireId);
@@ -1454,6 +1521,14 @@ public void testDeathNecropsyForm() throws IOException, CommandException
goToEHRFolder();
verifyRowCreated("study", "weight", aliveAnimalId, 1);
+ log("Verify the death date reached demographics and agrees with the death record");
+ String finalizedDeathDay = getDatasetDay("deaths", aliveAnimalId, "date");
+ Assert.assertNotNull("Death record has no date", finalizedDeathDay);
+ assertEquals("Demographics death date does not match the death record",
+ finalizedDeathDay, getDatasetDay("demographics", aliveAnimalId, "death"));
+ // the waitForText below is a weak check: the necropsy diagnosis on this page is also the text "Dead"
+ assertEquals("Animal should be Dead once the death is finalized", "Dead", getCalculatedStatus(aliveAnimalId));
+
log("Verify animal is marked as dead");
AnimalHistoryPage historyPage = AnimalHistoryPage.beginAt(this);
historyPage.searchSingleAnimal(aliveAnimalId);
@@ -1479,6 +1554,48 @@ public void testDeathNecropsyForm() throws IOException, CommandException
}
+ /**
+ * Deleting a death record has to hand the status back to the shared recalc rather than assume the animal is alive:
+ * an animal that also has a departure is Shipped, not Alive. Also covers demographics.death being cleared.
+ */
+ @Test
+ public void testDeathDeleteRestoresDepartedStatus() throws Exception
+ {
+ String animalId = "DD9001";
+ LocalDateTime now = LocalDateTime.now();
+
+ log("Creating an animal via a birth record");
+ getApiHelper().doSaveRows(DATA_ADMIN.getEmail(), getApiHelper().prepareInsertCommand("study", "birth", "lsid",
+ new String[]{"Id", "Date", "gender", "QCStateLabel", "performedby"},
+ new Object[][]{{animalId, now.minusDays(30), getMale(), "Completed", 1004}}
+ ), getExtraContext());
+
+ // the death has to be recorded before the departure: the deaths trigger rejects an animal that has shipped
+ log("Recording the death");
+ InsertRowsCommand deaths = new InsertRowsCommand("study", "deaths");
+ deaths.addRow(Map.of("Id", animalId, "date", now.minusDays(10), "reason", "4", "QCStateLabel", "Completed", "performedby", 1004));
+ deaths.execute(getApiHelper().getConnection(), getContainerPath());
+
+ assertEquals("Demographics death date does not match the death record",
+ now.minusDays(10).format(_dateFormat), getDatasetDay("demographics", animalId, "death"));
+ assertEquals("Animal should be Dead while the death record exists", "Dead", getCalculatedStatus(animalId));
+
+ log("Departing the animal, so the deleted death has a departure to fall back to");
+ InsertRowsCommand departure = new InsertRowsCommand("study", "departure");
+ departure.addRow(Map.of("Id", animalId, "date", now.minusDays(5), "destination", "ORPRC", "QCStateLabel", "Completed", "performedby", 1004));
+ departure.execute(getApiHelper().getConnection(), getContainerPath());
+
+ assertEquals("A death outranks a departure", "Dead", getCalculatedStatus(animalId));
+
+ log("Deleting the death record");
+ getApiHelper().deleteAllRecords("study", "deaths", new Filter("Id", animalId));
+
+ Assert.assertNull("Demographics death date should be cleared when the death record is deleted",
+ getDatasetDay("demographics", animalId, "death"));
+ assertEquals("Deleting the death should fall back to the departure, not to Alive",
+ "Shipped", getCalculatedStatus(animalId));
+ }
+
@Test
public void testClinicalCasesWorkflow()
{
@@ -1963,7 +2080,8 @@ private void createBreedingPair(String damId, String sireId, String species) thr
private void verifyBirthColumnOrder(Ext4GridRef births)
{
List expectedOrder = List.of("Id", "date", "conceptId", "Id/demographics/species", "Id/demographics/gender",
- "Id/demographics/dam", "Id/demographics/sire", "cage", "type", "cond", "breedingType", "remark", "performedby");
+ "Id/demographics/dam", "Id/demographics/sire", "cage", "Id/demographics/socialCode", "type",
+ "breedingType", "remark", "performedby");
int previousIdx = 0;
String previousCol = null;
@@ -1981,6 +2099,42 @@ private void waitForFormError(String message)
waitFor(() -> isTextPresent(message), "Form did not report: " + message, WAIT_FOR_JAVASCRIPT);
}
+ /**
+ * Reads a date field for one animal through the API rather than off a grid, so assertions compare stored values
+ * instead of formatted display text, and normalizes to the day: event dates are entered with the time stripped,
+ * but values reaching demographics by other paths can carry a time component.
+ *
+ * @return the date as yyyy-MM-dd, or null when the field is empty
+ */
+ private String getDatasetDay(String queryName, String animalId, String column) throws IOException, CommandException
+ {
+ Object value = getSingleRowForAnimal(queryName, animalId, List.of("Id", column)).get(column);
+ if (value == null)
+ return null;
+
+ if (value instanceof Date)
+ return _dateFormat.format(((Date)value).toInstant().atZone(ZoneId.systemDefault()).toLocalDate());
+
+ String text = String.valueOf(value);
+ return text.length() >= 10 ? text.substring(0, 10) : text;
+ }
+
+ private String getCalculatedStatus(String animalId) throws IOException, CommandException
+ {
+ return (String)getSingleRowForAnimal("demographics", animalId, List.of("Id", "calculated_status")).get("calculated_status");
+ }
+
+ private Map getSingleRowForAnimal(String queryName, String animalId, List columns) throws IOException, CommandException
+ {
+ SelectRowsCommand select = new SelectRowsCommand("study", queryName);
+ select.setColumns(columns);
+ select.addFilter(new Filter("Id", animalId));
+ SelectRowsResponse response = select.execute(getApiHelper().getConnection(), getContainerPath());
+
+ Assert.assertEquals("Expected exactly one study." + queryName + " row for " + animalId, 1, response.getRows().size());
+ return response.getRows().get(0);
+ }
+
private void verifyRowCreated(String schema, String query, String animalId, int rowCount)
{
goToSchemaBrowser();