From 9813a6c6b67548a89245de439d75a756790a4f6a Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Thu, 3 Sep 2026 11:43:26 +0900 Subject: [PATCH 1/7] Add tiered LC-MS Console annotation settings --- .../MsdialCoreTestApp/Parser/ConfigParser.cs | 32 +++++++++++++++- .../Parser/MspAnnotatorSetting.cs | 10 ++++- .../MsdialCoreTestApp/Process/LcmsProcess.cs | 6 ++- .../Parser/ConfigParserTests.cs | 37 ++++++++++++++++++- .../MsScanMatchResultEvaluatorTests.cs | 2 +- 5 files changed, 81 insertions(+), 6 deletions(-) diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs index e2467701e..ec513d7b6 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs @@ -111,6 +111,26 @@ public static bool ReadAlignmentLightMode(string filepath) { return false; } + public static int ReadLbmAnnotatorPriority(string filepath) { + using (var sr = new StreamReader(filepath, Encoding.ASCII)) { + while (sr.Peek() > -1) { + readFieldValues(sr.ReadLine(), out string method, out string value, out bool isReadable); + if (!isReadable) { + continue; + } + switch (method.ToLower()) { + case "lbm annotator priority": + case "lbm annotation priority": + if (int.TryParse(value, out var priority)) { + return priority; + } + break; + } + } + } + return 1; + } + private static string ReadMspAnnotatorSettingsFilePath(string filepath) { using (var sr = new StreamReader(filepath, Encoding.ASCII)) { while (sr.Peek() > -1) { @@ -205,7 +225,17 @@ private static List ReadMspAnnotatorSettingsTable(string fi var searchParameter = new MsRefSearchParameterBase(param.MspSearchParam); ApplyMspSearchParameter(searchParameter, fields, headers); - settings.Add(new MspAnnotatorSetting(annotatorId, mspFilePath, priority, searchParameter)); + TargetOmics? targetOmics = null; + var targetOmicsText = GetField(fields, headers, "targetomics", "annotationmode", "omics"); + if (!targetOmicsText.IsEmptyOrNull()) { + if (Enum.TryParse(targetOmicsText, true, out TargetOmics parsedTargetOmics)) { + targetOmics = parsedTargetOmics; + } + else { + Console.WriteLine($"Unknown target_omics '{targetOmicsText}' for MSP annotator '{annotatorId}'. The project Target omics setting will be used."); + } + } + settings.Add(new MspAnnotatorSetting(annotatorId, mspFilePath, priority, searchParameter, targetOmics)); } return settings; } diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/MspAnnotatorSetting.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/MspAnnotatorSetting.cs index c2801d7e8..573399591 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/MspAnnotatorSetting.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/MspAnnotatorSetting.cs @@ -1,18 +1,26 @@ using CompMs.Common.Parameter; +using CompMs.Common.Enum; namespace CompMs.App.MsdialConsole.Parser; public sealed class MspAnnotatorSetting { - public MspAnnotatorSetting(string annotatorId, string mspFilePath, int priority, MsRefSearchParameterBase searchParameter) { + public MspAnnotatorSetting( + string annotatorId, + string mspFilePath, + int priority, + MsRefSearchParameterBase searchParameter, + TargetOmics? targetOmics = null) { AnnotatorId = annotatorId; MspFilePath = mspFilePath; Priority = priority; SearchParameter = searchParameter; + TargetOmics = targetOmics; } public string AnnotatorId { get; } public string MspFilePath { get; } public int Priority { get; } public MsRefSearchParameterBase SearchParameter { get; } + public TargetOmics? TargetOmics { get; } } diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs index 0bba420de..37d877aa8 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs @@ -49,6 +49,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool var mspAnnotatorSettings = ConfigParser.ReadMspAnnotatorSettings(methodFile, param); var textAnnotatorSettings = ConfigParser.ReadTextAnnotatorSettings(methodFile, param); + var lbmAnnotatorPriority = ConfigParser.ReadLbmAnnotatorPriority(methodFile); CommonProcess.ParseLibraries(param, targetMz, mspAnnotatorSettings, textAnnotatorSettings, out IupacDatabase iupacDB, out var mspDBs, out var textDBs, out List isotopeTextDB, out List compoundsInTargetMode, @@ -66,7 +67,8 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool foreach (var mspDB in mspDBs.Where(db => db.DataBase is { Database.Count: > 0 })) { var annotatorPairs = new List>(); foreach (var setting in mspDB.AnnotatorSettings) { - var annotator = new LcmsMspAnnotator(mspDB.DataBase, setting.SearchParameter, param.TargetOmics, setting.AnnotatorId, setting.Priority); + var targetOmics = setting.TargetOmics ?? param.TargetOmics; + var annotator = new LcmsMspAnnotator(mspDB.DataBase, setting.SearchParameter, targetOmics, setting.AnnotatorId, setting.Priority); annotatorPairs.Add(new MetabolomicsAnnotatorParameterPair(annotator.Save(), new AnnotationQueryFactory(annotator, param.PeakPickBaseParam, setting.SearchParameter, ignoreIsotopicPeak: true))); } if (annotatorPairs.Count > 0) { @@ -74,7 +76,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool } } if (lbmDB is { Database.Count: > 0 }) { - var lbmAnnotator = new LcmsMspAnnotator(lbmDB, param.LbmSearchParam, param.TargetOmics, param.LbmFilePath, 1); + var lbmAnnotator = new LcmsMspAnnotator(lbmDB, param.LbmSearchParam, TargetOmics.Lipidomics, param.LbmFilePath, lbmAnnotatorPriority); dbStorage.AddMoleculeDataBase(lbmDB, [ new MetabolomicsAnnotatorParameterPair(lbmAnnotator.Save(), new AnnotationQueryFactory(lbmAnnotator, param.PeakPickBaseParam, param.LbmSearchParam, ignoreIsotopicPeak: true)), ]); diff --git a/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs index dde704e3c..2de05db71 100644 --- a/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs +++ b/tests/MSDIAL5/MsdialCoreTestAppTests/Parser/ConfigParserTests.cs @@ -1,5 +1,6 @@ using CompMs.App.MsdialConsole.Parser; using CompMs.Common.Enum; +using CompMs.MsdialLcmsApi.Parameter; using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.IO; @@ -70,6 +71,40 @@ public void ReadForGcms_AcceptsLegacyRiPathAndAnnotationFieldNames() Assert.AreEqual(2000f, parameter.MspSearchParam.RiTolerance); } + [TestMethod] + public void ReadMspAnnotatorSettings_UsesPerAnnotatorTargetOmics() + { + using var directory = new TemporaryDirectory(); + var msp = directory.CreateFile("library.msp"); + var settings = directory.CreateFile( + "msp_annotator_settings.tsv", + $"annotator_id\tmsp_file_path\tpriority\ttarget_omics\tms2_tolerance\n" + + $"high\t{msp}\t2\tMetabolomics\t0.05\n" + + $"inherit\t{msp}\t1\t\t0.25\n"); + var method = directory.CreateFile( + "method.txt", + $"Msp annotator settings file path: {settings}\n"); + + var parsed = ConfigParser.ReadMspAnnotatorSettings(method, new MsdialLcmsParameter()); + + Assert.AreEqual(2, parsed.Count); + Assert.AreEqual(TargetOmics.Metabolomics, parsed[0].TargetOmics); + Assert.IsNull(parsed[1].TargetOmics); + Assert.AreEqual(0.05F, parsed[0].SearchParameter.Ms2Tolerance, 0.0001F); + Assert.AreEqual(0.25F, parsed[1].SearchParameter.Ms2Tolerance, 0.0001F); + } + + [TestMethod] + public void ReadLbmAnnotatorPriority_DefaultsToOneAndReadsConfiguredValue() + { + using var directory = new TemporaryDirectory(); + var defaultMethod = directory.CreateFile("default.txt", "Ion mode: Negative\n"); + var configuredMethod = directory.CreateFile("configured.txt", "LBM annotator priority: 3\n"); + + Assert.AreEqual(1, ConfigParser.ReadLbmAnnotatorPriority(defaultMethod)); + Assert.AreEqual(3, ConfigParser.ReadLbmAnnotatorPriority(configuredMethod)); + } + private sealed class TemporaryDirectory : IDisposable { public TemporaryDirectory() @@ -83,7 +118,7 @@ public TemporaryDirectory() public string Path { get; } - public string CreateFile(string name, string content) + public string CreateFile(string name, string content = "") { var path = System.IO.Path.Combine(Path, name); File.WriteAllText(path, content, new UTF8Encoding(encoderShouldEmitUTF8Identifier: true)); diff --git a/tests/MSDIAL5/MsdialCoreTests/Algorithm/Annotation/MsScanMatchResultEvaluatorTests.cs b/tests/MSDIAL5/MsdialCoreTests/Algorithm/Annotation/MsScanMatchResultEvaluatorTests.cs index ddab3f464..2e7a26815 100644 --- a/tests/MSDIAL5/MsdialCoreTests/Algorithm/Annotation/MsScanMatchResultEvaluatorTests.cs +++ b/tests/MSDIAL5/MsdialCoreTests/Algorithm/Annotation/MsScanMatchResultEvaluatorTests.cs @@ -98,4 +98,4 @@ private static List CreateResults() { }; } } -} \ No newline at end of file +} From a9e8c3a6a67cca7537cc68be3a82d6de5373f2dd Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Sun, 6 Sep 2026 17:30:36 +0900 Subject: [PATCH 2/7] Give the Console a way into internal-standard normalization The algorithm has been in MsdialCore all along -- Normalization.SplashNormalize is what the graphical application calls -- but nothing outside the GUI could reach it. A pipeline that produces an aligned result therefore had to stop there and hand the project to a person. MSDIALCUI normalize reads a saved project and its alignment, builds the standard table, calls the same function, and exports the matrix. The one thing the GUI can take for granted and a command line cannot is which aligned peak each standard actually is. StandardCompound identifies it by PeakID, which is an alignment ID: it belongs to one alignment and to no other, so a table written for one run points at unrelated peaks in the next, and does so silently, normalizing whole lipid classes against whatever happened to land on that ID. So a standard may name itself instead and be resolved against the annotations of the alignment being normalized, and a table that does carry an ID is checked against the annotation on it and says so when the two disagree. A standard that resolves to nothing stops the run, because the alternative is an unnormalized lipid class that looks normalized. MS-DIAL reports a lipid at two resolutions separated by a bar, so a standard table may name either one and both are matched. Co-Authored-By: Claude Opus 5 --- .../MsdialCoreTestApp/Process/MainProcess.cs | 40 ++- .../Process/NormalizationProcess.cs | 298 ++++++++++++++++++ tests/MSDIAL5/MsdialCoreTestApp/Program.cs | 3 +- 3 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs index 8b9cd44c1..368446560 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs @@ -1,4 +1,4 @@ -using CompMs.App.MsdialConsole.Process.MoleculerNetworking; +using CompMs.App.MsdialConsole.Process.MoleculerNetworking; using CompMs.App.MsdialConsole.Properties; using CompMs.Common.Enum; using CompMs.Common.Extension; @@ -443,6 +443,44 @@ public static void SetMsnCommand(Command root) { root.Add(cmd); } + public static void SetNormalizeCommand(Command root) { + var cmd = new Command( + "normalize", + "Normalize an aligned result against internal standards and export the matrix"); + var input = new Option("--input", "-i") { Required = true }; + input.Description = "MS-DIAL project file holding the alignment to normalize"; + var standards = new Option("--standards", "-s") { Required = true }; + standards.Description = "Table of internal standards: StandardName, TargetClass, Concentration, optional PeakID, DilutionRate, MolecularWeight"; + var output = new Option("--output", "-o") { Required = true }; + output.Description = "Normalized alignment matrix to write"; + var unit = new Option("--unit", "-u") { + DefaultValueFactory = _ => IonAbundanceUnit.NormalizedByInternalStandardPeakHeight, + }; + unit.Description = "Unit of the normalized abundance, e.g. pmol_per_microL_plasma"; + var alignment = new Option("--alignment") { DefaultValueFactory = _ => 0 }; + alignment.Description = "Index of the alignment result within the project"; + var dilution = new Option("--apply-dilution-factor", "-d"); + dilution.Description = "Divide by each file's dilution factor after normalizing"; + var allowUnresolved = new Option("--allow-unresolved-standards"); + allowUnresolved.Description = "Continue when a standard is not found in the alignment, leaving its classes unnormalized"; + cmd.Options.Add(input); + cmd.Options.Add(standards); + cmd.Options.Add(output); + cmd.Options.Add(unit); + cmd.Options.Add(alignment); + cmd.Options.Add(dilution); + cmd.Options.Add(allowUnresolved); + cmd.SetAction(parseResult => new NormalizationProcess().Run( + parseResult.GetRequiredValue(input), + parseResult.GetRequiredValue(standards), + parseResult.GetRequiredValue(output), + parseResult.GetValue(unit), + parseResult.GetValue(alignment), + parseResult.GetValue(dilution), + parseResult.GetValue(allowUnresolved))); + root.Subcommands.Add(cmd); + } + public static void SetEicCommand(Command root) { var eic = new Command("eic", "Export extracted ion chromatograms"); var raw = new Command("raw", "Export EICs from a raw data file"); diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs new file mode 100644 index 000000000..97f37e890 --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs @@ -0,0 +1,298 @@ +using CompMs.Common.DataObj.Result; +using CompMs.Common.Enum; +using CompMs.Common.Interfaces; +using CompMs.MsdialCore.Algorithm.Annotation; +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Export; +using CompMs.MsdialCore.MSDec; +using CompMs.MsdialCore.Normalize; +using CompMs.MsdialCore.Parameter; +using CompMs.MsdialCore.Parser; +using CompMs.MsdialLcMsApi.Export; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace CompMs.App.MsdialConsole.Process; + +/// +/// Normalizes an aligned result against internal standards, from the command line. +/// +/// +/// The algorithm already lives in MsdialCore and is what the graphical application +/// calls; only the way in was missing. This supplies it: read a saved project and its +/// alignment, build the standard table, normalize, export. +/// +/// One thing the graphical application can take for granted and a command line cannot: +/// which aligned peak *is* each standard. StandardCompound identifies it by PeakID, +/// which is an alignment ID and therefore belongs to one alignment and no other. A +/// table written for one run would silently point at unrelated peaks in the next, so a +/// standard may instead name itself and be resolved against the annotations of the +/// alignment actually being normalized. An unresolved standard stops the run rather +/// than quietly normalizing its whole lipid class against nothing. +/// +public sealed class NormalizationProcess { + public int Run( + FileInfo projectFile, + FileInfo standardsFile, + FileInfo outputFile, + IonAbundanceUnit unit, + int alignmentIndex, + bool applyDilutionFactor, + bool allowUnresolvedStandards) { + if (!projectFile.Exists) { + Console.Error.WriteLine($"Project file was not found: {projectFile.FullName}"); + return -1; + } + if (!standardsFile.Exists) { + Console.Error.WriteLine($"Internal standard table was not found: {standardsFile.FullName}"); + return -1; + } + + IMsdialDataStorage storage = + Common.MessagePack.MessagePackDefaultHandler.LoadFromFile(projectFile.FullName); + var files = storage.AnalysisFiles.Where(file => file.AnalysisFileIncluded).ToList(); + if (files.Count == 0) { + Console.Error.WriteLine("The project contains no included analysis files."); + return -1; + } + if (storage.AlignmentFiles is null || storage.AlignmentFiles.Count == 0) { + Console.Error.WriteLine("The project contains no alignment result to normalize."); + return -1; + } + if (alignmentIndex < 0 || alignmentIndex >= storage.AlignmentFiles.Count) { + Console.Error.WriteLine( + $"Alignment index {alignmentIndex} is outside the project's {storage.AlignmentFiles.Count} alignment result(s)."); + return -1; + } + + var alignmentFile = storage.AlignmentFiles[alignmentIndex]; + var container = AlignmentResultContainer.Load(alignmentFile); + var spots = container.AlignmentSpotProperties; + if (spots is null || spots.Count == 0) { + Console.Error.WriteLine("The alignment result contains no spots."); + return -1; + } + + List records; + try { + records = ReadStandards(standardsFile.FullName); + } + catch (FormatException error) { + Console.Error.WriteLine(error.Message); + return -1; + } + if (records.Count == 0) { + Console.Error.WriteLine("The internal standard table contains no rows."); + return -1; + } + + var resolution = ResolveStandards(records, spots); + foreach (var line in resolution.Report) { + Console.WriteLine(line); + } + if (resolution.Unresolved.Count > 0) { + var summary = string.Join(", ", resolution.Unresolved); + if (!allowUnresolvedStandards) { + Console.Error.WriteLine( + $"{resolution.Unresolved.Count} internal standard(s) were not found in the alignment: {summary}. " + + "Every lipid class they cover would be left unnormalized. " + + "Confirm the annotation, or pass --allow-unresolved-standards to continue without them."); + return 2; + } + Console.WriteLine( + $"WARNING: continuing without {resolution.Unresolved.Count} internal standard(s): {summary}."); + } + if (resolution.Compounds.Count == 0) { + Console.Error.WriteLine("No internal standard could be resolved, so nothing can be normalized."); + return 2; + } + + var evaluator = FacadeMatchResultEvaluator.FromDataBases(storage.DataBases); + Normalization.SplashNormalize( + files, + spots, + storage.DataBaseMapper, + resolution.Compounds, + unit, + evaluator, + applyDilutionFactor); + container.IsNormalized = true; + + var decResults = MsdecResultsReader.ReadMSDecResults(alignmentFile.SpectraFilePath, out _, out _); + var accessor = new LcmsMetadataAccessor(storage.DataBaseMapper, storage.Parameter, false); + var quantAccessor = new LegacyQuantValueAccessor("Height", storage.Parameter); + var stats = new[] { StatsValue.Average, StatsValue.Stdev }; + Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(outputFile.FullName)) ?? "."); + using (var stream = File.Open(outputFile.FullName, FileMode.Create, FileAccess.Write)) { + new AlignmentCSVExporter().Export( + stream, spots, decResults, files, new MulticlassFileMetaAccessor(0), accessor, quantAccessor, stats); + } + + Console.WriteLine($"Normalized unit: {unit}"); + Console.WriteLine($"Dilution factor applied: {applyDilutionFactor}"); + Console.WriteLine(outputFile.FullName); + return 0; + } + + private sealed class StandardRecord { + public string StandardName = string.Empty; + public string TargetClass = string.Empty; + public double Concentration; + public double DilutionRate = 1d; + public double MolecularWeight; + public int PeakID = -1; + public int LineNumber; + } + + private sealed class StandardResolution { + public List Compounds = new List(); + public List Unresolved = new List(); + public List Report = new List(); + } + + /// + /// Reads the class-to-standard table: which standard normalizes which lipid class, + /// at what amount. "Any others" covers every class the table does not name. + /// + private static List ReadStandards(string path) { + var lines = File.ReadAllLines(path); + var header = lines.FirstOrDefault(line => !string.IsNullOrWhiteSpace(line)); + if (header is null) { + throw new FormatException("The internal standard table is empty."); + } + var separator = header.Contains('\t') ? '\t' : ','; + var columns = header.Split(separator) + .Select((name, index) => (name: Normalize(name), index)) + .ToDictionary(item => item.name, item => item.index); + + int Column(string name) => columns.TryGetValue(Normalize(name), out var index) ? index : -1; + var nameColumn = Column("StandardName"); + var classColumn = Column("TargetClass"); + var concentrationColumn = Column("Concentration"); + if (nameColumn < 0 || classColumn < 0 || concentrationColumn < 0) { + throw new FormatException( + "The internal standard table needs StandardName, TargetClass and Concentration columns; " + + $"it has: {string.Join(", ", header.Split(separator))}"); + } + var peakColumn = Column("PeakID"); + var dilutionColumn = Column("DilutionRate"); + var weightColumn = Column("MolecularWeight"); + + var records = new List(); + var started = false; + for (var index = 0; index < lines.Length; index++) { + var line = lines[index]; + if (string.IsNullOrWhiteSpace(line)) continue; + if (!started) { started = true; continue; } + var cells = line.Split(separator); + string Cell(int column) => column >= 0 && column < cells.Length ? cells[column].Trim() : string.Empty; + var standardName = Cell(nameColumn); + var targetClass = Cell(classColumn); + if (standardName.Length == 0 || targetClass.Length == 0) continue; + var record = new StandardRecord { + StandardName = standardName, + TargetClass = targetClass, + LineNumber = index + 1, + PeakID = ParseInt(Cell(peakColumn), -1), + MolecularWeight = ParseDouble(Cell(weightColumn), 0d), + }; + var concentration = Cell(concentrationColumn); + if (!double.TryParse(concentration, NumberStyles.Float, CultureInfo.InvariantCulture, out var value)) { + throw new FormatException( + $"Line {record.LineNumber}: '{concentration}' is not a concentration for {standardName}."); + } + record.Concentration = value; + record.DilutionRate = ParseDouble(Cell(dilutionColumn), 1d); + if (record.DilutionRate <= 0d) record.DilutionRate = 1d; + records.Add(record); + } + return records; + } + + /// + /// Ties each standard to the aligned peak that carries it, by the alignment ID the + /// table gives or, failing that, by the name the alignment annotated. + /// + private static StandardResolution ResolveStandards( + IReadOnlyList records, IReadOnlyList spots) { + var result = new StandardResolution(); + var byId = spots.ToDictionary(spot => spot.MasterAlignmentID, spot => spot); + var byName = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var spot in spots) { + foreach (var alias in NameAliases(spot.Name)) { + if (!byName.TryGetValue(alias, out var bucket)) { + byName[alias] = bucket = new List(); + } + bucket.Add(spot); + } + } + + foreach (var record in records) { + AlignmentSpotProperty? spot = null; + var how = string.Empty; + if (record.PeakID >= 0 && byId.TryGetValue(record.PeakID, out var byIdSpot)) { + spot = byIdSpot; + how = $"alignment ID {record.PeakID}"; + // An ID that names a different compound is a table written for another + // run. Saying so is the whole point of carrying the name as well. + if (!NameAliases(byIdSpot.Name).Contains(record.StandardName, StringComparer.OrdinalIgnoreCase)) { + result.Report.Add( + $" WARNING {record.StandardName}: alignment ID {record.PeakID} is annotated " + + $"'{byIdSpot.Name}'. The table may belong to a different alignment."); + } + } + else if (byName.TryGetValue(record.StandardName, out var candidates)) { + spot = candidates.OrderByDescending(item => item.HeightAverage).First(); + how = $"annotation, alignment ID {spot.MasterAlignmentID}"; + if (candidates.Count > 1) { + result.Report.Add( + $" NOTE {record.StandardName}: {candidates.Count} aligned peaks carry this annotation; " + + $"the most abundant (ID {spot.MasterAlignmentID}) was used."); + } + } + + if (spot is null) { + result.Unresolved.Add($"{record.StandardName} (for {record.TargetClass})"); + result.Report.Add($" UNRESOLVED {record.StandardName} -> {record.TargetClass}"); + continue; + } + result.Report.Add( + $" {record.StandardName} -> {record.TargetClass} via {how}, concentration {record.Concentration}"); + result.Compounds.Add(new StandardCompound { + StandardName = record.StandardName, + TargetClass = record.TargetClass, + Concentration = record.Concentration, + DilutionRate = record.DilutionRate, + MolecularWeight = record.MolecularWeight, + PeakID = spot.MasterAlignmentID, + }); + } + return result; + } + + /// + /// The names an aligned peak answers to. MS-DIAL reports a lipid at two resolutions + /// separated by a bar, such as "PC 33:1(d7)|PC 15:0_18:1(d7)", and a standard table + /// may reasonably name either one. + /// + private static IEnumerable NameAliases(string? name) { + if (string.IsNullOrWhiteSpace(name)) yield break; + yield return name!.Trim(); + foreach (var part in name!.Split('|')) { + var trimmed = part.Trim(); + if (trimmed.Length > 0) yield return trimmed; + } + } + + private static string Normalize(string value) => + new string((value ?? string.Empty).Where(char.IsLetterOrDigit).ToArray()).ToLowerInvariant(); + + private static int ParseInt(string value, int fallback) => + int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : fallback; + + private static double ParseDouble(string value, double fallback) => + double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) ? parsed : fallback; +} diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Program.cs b/tests/MSDIAL5/MsdialCoreTestApp/Program.cs index f93191095..363c80381 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Program.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Program.cs @@ -1,4 +1,4 @@ -using CompMs.App.MsdialConsole.Process; +using CompMs.App.MsdialConsole.Process; using CompMs.App.MsdialConsole.Properties; using System.CommandLine; using System.CommandLine.Invocation; @@ -182,6 +182,7 @@ public static Task Main(string[] args) { MainProcess.SetDimsCommand(root); MainProcess.SetImmsCommand(root); MainProcess.SetMsnCommand(root); + MainProcess.SetNormalizeCommand(root); MainProcess.SetEicCommand(root); MainProcess.SetRtCorrectionCommand(root); MainProcess.SetImageGenerationCommand(root); From 8e30665a6ab17698af60c80cdb573ddf623c634a Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Sun, 6 Sep 2026 18:04:12 +0900 Subject: [PATCH 3/7] Load the whole project, and export the values normalizing produced Two faults that only a real run could show, both of the same kind: something that looked like it worked and quietly did not. The data storage is half a project. Loading it straight from MessagePack leaves the annotation databases null, and that surfaces much later as a null reference inside the match evaluator, pointing nowhere near the cause. The project is now loaded the way the application loads it, through MsdialIntegrateSerializer over a directory stream manager, so the databases and the mapper come with it. Then the export asked for "Height", which is the raw peak height. Normalizing does not touch that field -- it writes to a separate one -- so the exported matrix was identical to the input, down to the byte, while every log line said the normalization had succeeded. It asks for "Normalized height" now. Also: the parameter file's thread count was written but never read back, so a method file could describe a thread count it could never request and every Console run stayed on the default of two. Verified against the reference lipidomics dataset, seven plasma samples in both polarities: 13 of 13 internal standards in positive mode and 10 of 11 in negative normalize to exactly the concentration the lookup table gives them, flat across all seven samples, which is the invariant the graphical application's output shows. The eleventh is FA 16:0(d3), which that table assigns to no class of its own and which is therefore normalized as an ordinary fatty acid. Co-Authored-By: Claude Opus 5 --- .../MsdialCoreTestApp/Parser/ConfigParser.cs | 6 ++++- .../Process/NormalizationProcess.cs | 26 ++++++++++++++++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs index 49d16ec46..39b59401f 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs @@ -1,4 +1,4 @@ -using CompMs.Common.DataObj.Property; +using CompMs.Common.DataObj.Property; using CompMs.Common.Enum; using CompMs.Common.Extension; using CompMs.Common.Parser; @@ -950,6 +950,10 @@ public static bool ReadCommonParameter(ParameterBase param, string method, strin case "set fully labeled reference file": if (valueLower == "true" || valueLower == "false") param.SetFullyLabeledReferenceFile = bool.Parse(valueLower); return true; case "non labeled reference id": if (int.TryParse(valueLower, out int nonlabeledrefid)) param.NonLabeledReferenceID = nonlabeledrefid; return true; case "fully labeled reference id": if (int.TryParse(valueLower, out int fulllabeledrefid)) param.FullyLabeledReferenceID = fulllabeledrefid; return true; + // ParameterBase writes "Number of threads" into every exported method file, + // but nothing read it back, so a method file could describe a thread count + // it could never request and every Console run stayed on the default of 2. + case "number of threads": if (int.TryParse(valueLower, out int numthreads) && numthreads > 0) param.NumThreads = numthreads; return true; case "isotope tracking dictionary id": if (int.TryParse(valueLower, out int isotopetrackdictionaryid)) param.IsotopeTrackingDictionary.SelectedID = isotopetrackdictionaryid; return true; //CorrDec settings diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs index 97f37e890..056f2ed0c 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs @@ -8,6 +8,7 @@ using CompMs.MsdialCore.Normalize; using CompMs.MsdialCore.Parameter; using CompMs.MsdialCore.Parser; +using CompMs.MsdialIntegrate.Parser; using CompMs.MsdialLcMsApi.Export; using System; using System.Collections.Generic; @@ -51,8 +52,10 @@ public int Run( return -1; } - IMsdialDataStorage storage = - Common.MessagePack.MessagePackDefaultHandler.LoadFromFile(projectFile.FullName); + // The data storage is only half a project: the annotation databases live beside it + // and the raw MessagePack load leaves them null, which surfaces much later as a + // null reference inside the evaluator. Load it the way the application does. + var storage = LoadProject(projectFile.FullName); var files = storage.AnalysisFiles.Where(file => file.AnalysisFileIncluded).ToList(); if (files.Count == 0) { Console.Error.WriteLine("The project contains no included analysis files."); @@ -123,7 +126,10 @@ public int Run( var decResults = MsdecResultsReader.ReadMSDecResults(alignmentFile.SpectraFilePath, out _, out _); var accessor = new LcmsMetadataAccessor(storage.DataBaseMapper, storage.Parameter, false); - var quantAccessor = new LegacyQuantValueAccessor("Height", storage.Parameter); + // "Height" reads the raw peak height, which normalizing does not touch: the result + // is written to a separate field, so exporting the wrong one silently produces a + // file identical to the input. + var quantAccessor = new LegacyQuantValueAccessor("Normalized height", storage.Parameter); var stats = new[] { StatsValue.Average, StatsValue.Stdev }; Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(outputFile.FullName)) ?? "."); using (var stream = File.Open(outputFile.FullName, FileMode.Create, FileAccess.Write)) { @@ -137,6 +143,20 @@ public int Run( return 0; } + private static IMsdialDataStorage LoadProject(string projectFilePath) { + var projectFolder = Path.GetDirectoryName(Path.GetFullPath(projectFilePath)) ?? "."; + var projectFileName = Path.GetFileName(projectFilePath); + var serializer = new MsdialIntegrateSerializer(); + using (IStreamManager streamManager = new DirectoryTreeStreamManager(projectFolder)) { + var storage = serializer + .LoadAsync(streamManager, projectFileName, projectFolder, string.Empty) + .GetAwaiter().GetResult(); + streamManager.Complete(); + storage.FixDatasetFolder(projectFolder); + return storage; + } + } + private sealed class StandardRecord { public string StandardName = string.Empty; public string TargetClass = string.Empty; From 83240411441bd2472da126a8a105d9ec00403aa3 Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Sun, 6 Sep 2026 20:28:24 +0900 Subject: [PATCH 4/7] Stop publishing concentrations that were divided by the wrong standard An independent audit found that --allow-unresolved-standards did not do what its own message said. It promised the affected lipid classes "would be left unnormalized"; in fact SplashNormalize falls through to the "Any others" standard, so a class whose own standard was missing was quantified against a compound of an entirely different one -- a cardiolipin divided by a lysophosphatidylcholine -- and written out in the same unit, with the same comment, as a properly quantified row. Nothing in the file told them apart. On the audited dataset thirty-seven standard-to-class assignments were unresolved and eleven rows carried such numbers. None reached the merged lipidome, but only because the laboratory's rule table happens to quantify every affected class in the other polarity. That is the rule table's doing, not a check. Those rows now keep their identity and lose their numbers, and say why in place of them: "NOT QUANTIFIED: the CL standard PG 15:0_18:1(d7) did not resolve in this alignment". The raw height stays in the Height matrix beside it, so nothing is hidden -- only the invalid concentration is withheld. An annotated wrong number is still read by the next script; an empty cell is not. The message says this now. Two more from the same audit. A standard whose alignment ID holds a different compound used to warn and carry on, exit 0, with sphingomyelin quantified against a triacylglycerol peak. The posture was inverted: a standard that cannot be found stopped the run, while one demonstrably pointing at the wrong compound did not. It stops too, unless --allow-mismatched-peak-ids says the annotation is wrong rather than the table. Passing the .mdproject instead of the .mddata produced fifteen frames of MessagePack internals. It now names the file that cannot be read and the one to pass instead. The resolution report repeated a standard's own resolution once per class it covered -- a hundred and sixty lines, with the three that needed a decision somewhere in the middle. Each standard is reported once with the classes it covers, and the ambiguous and unresolved ones come last. Co-Authored-By: Claude Opus 5 --- .../MsdialCoreTestApp/Process/MainProcess.cs | 12 +- .../Process/NormalizationProcess.cs | 208 +++++++++++++++--- 2 files changed, 189 insertions(+), 31 deletions(-) diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs index 368446560..babbd6f84 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs @@ -451,8 +451,8 @@ public static void SetNormalizeCommand(Command root) { input.Description = "MS-DIAL project file holding the alignment to normalize"; var standards = new Option("--standards", "-s") { Required = true }; standards.Description = "Table of internal standards: StandardName, TargetClass, Concentration, optional PeakID, DilutionRate, MolecularWeight"; - var output = new Option("--output", "-o") { Required = true }; - output.Description = "Normalized alignment matrix to write"; + var output = new Option("--output", "-o") { Required = true }; + output.Description = "Directory to write the raw and normalized alignment matrices into"; var unit = new Option("--unit", "-u") { DefaultValueFactory = _ => IonAbundanceUnit.NormalizedByInternalStandardPeakHeight, }; @@ -462,7 +462,9 @@ public static void SetNormalizeCommand(Command root) { var dilution = new Option("--apply-dilution-factor", "-d"); dilution.Description = "Divide by each file's dilution factor after normalizing"; var allowUnresolved = new Option("--allow-unresolved-standards"); - allowUnresolved.Description = "Continue when a standard is not found in the alignment, leaving its classes unnormalized"; + allowUnresolved.Description = "Continue when a standard is not found, leaving its classes without a concentration"; + var allowMismatched = new Option("--allow-mismatched-peak-ids"); + allowMismatched.Description = "Continue when a standard's alignment ID holds a different compound"; cmd.Options.Add(input); cmd.Options.Add(standards); cmd.Options.Add(output); @@ -470,6 +472,7 @@ public static void SetNormalizeCommand(Command root) { cmd.Options.Add(alignment); cmd.Options.Add(dilution); cmd.Options.Add(allowUnresolved); + cmd.Options.Add(allowMismatched); cmd.SetAction(parseResult => new NormalizationProcess().Run( parseResult.GetRequiredValue(input), parseResult.GetRequiredValue(standards), @@ -477,7 +480,8 @@ public static void SetNormalizeCommand(Command root) { parseResult.GetValue(unit), parseResult.GetValue(alignment), parseResult.GetValue(dilution), - parseResult.GetValue(allowUnresolved))); + parseResult.GetValue(allowUnresolved), + parseResult.GetValue(allowMismatched))); root.Subcommands.Add(cmd); } diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs index 056f2ed0c..265a57a06 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs @@ -1,3 +1,4 @@ +using CompMs.Common.Components; using CompMs.Common.DataObj.Result; using CompMs.Common.Enum; using CompMs.Common.Interfaces; @@ -35,14 +36,18 @@ namespace CompMs.App.MsdialConsole.Process; /// than quietly normalizing its whole lipid class against nothing. /// public sealed class NormalizationProcess { + /// Label rows plus the column-name row that precede the data. + private const int HeaderRowCount = 5; + public int Run( FileInfo projectFile, FileInfo standardsFile, - FileInfo outputFile, + DirectoryInfo outputDirectory, IonAbundanceUnit unit, int alignmentIndex, bool applyDilutionFactor, - bool allowUnresolvedStandards) { + bool allowUnresolvedStandards, + bool allowMismatchedPeakIds) { if (!projectFile.Exists) { Console.Error.WriteLine($"Project file was not found: {projectFile.FullName}"); return -1; @@ -55,7 +60,24 @@ public int Run( // The data storage is only half a project: the annotation databases live beside it // and the raw MessagePack load leaves them null, which surfaces much later as a // null reference inside the evaluator. Load it the way the application does. - var storage = LoadProject(projectFile.FullName); + IMsdialDataStorage storage; + try { + storage = LoadProject(projectFile.FullName); + } + catch (Exception error) { + // The run writes both a .mdproject and a .mddata, and only the second holds + // the data storage. Passing the one that looks more like a project produced + // fifteen frames of MessagePack internals and no statement of what to do. + var extension = Path.GetExtension(projectFile.FullName); + var sibling = Path.ChangeExtension(projectFile.FullName, ".mddata"); + var advice = File.Exists(sibling) && !extension.Equals(".mddata", StringComparison.OrdinalIgnoreCase) + ? $" Pass the data file beside it instead: {sibling}" + : " Pass the .mddata file written by the analysis run."; + Console.Error.WriteLine( + $"{projectFile.FullName} could not be read as an MS-DIAL data file " + + $"({error.GetType().Name})." + advice); + return -1; + } var files = storage.AnalysisFiles.Where(file => file.AnalysisFileIncluded).ToList(); if (files.Count == 0) { Console.Error.WriteLine("The project contains no included analysis files."); @@ -96,17 +118,36 @@ public int Run( foreach (var line in resolution.Report) { Console.WriteLine(line); } + if (resolution.Mismatched.Count > 0 && !allowMismatchedPeakIds) { + // A standard that cannot be found already stops the run. One found and + // demonstrably pointing at a different compound is the worse case of the two, + // and it used to warn and carry on -- quantifying a lipid class against + // whatever happened to occupy that alignment ID. + Console.Error.WriteLine( + $"{resolution.Mismatched.Count} standard(s) name an alignment ID that holds a different " + + $"compound: {string.Join("; ", resolution.Mismatched)}. An alignment ID belongs to the run it " + + "was written for. Remove the PeakID column so the standards resolve by name, or pass " + + "--allow-mismatched-peak-ids if the annotations are wrong rather than the table."); + return 2; + } + if (resolution.Mismatched.Count > 0) { + Console.WriteLine( + $"WARNING: {resolution.Mismatched.Count} standard(s) were taken from an alignment ID that holds " + + $"a different compound: {string.Join("; ", resolution.Mismatched)}."); + } if (resolution.Unresolved.Count > 0) { var summary = string.Join(", ", resolution.Unresolved); if (!allowUnresolvedStandards) { Console.Error.WriteLine( $"{resolution.Unresolved.Count} internal standard(s) were not found in the alignment: {summary}. " - + "Every lipid class they cover would be left unnormalized. " - + "Confirm the annotation, or pass --allow-unresolved-standards to continue without them."); + + "Every lipid class they cover would be left without a concentration. " + + "Confirm the annotation, or pass --allow-unresolved-standards to continue, which " + + "empties those rows rather than quantifying them against another class."); return 2; } Console.WriteLine( - $"WARNING: continuing without {resolution.Unresolved.Count} internal standard(s): {summary}."); + $"WARNING: {resolution.Unresolved.Count} internal standard(s) did not resolve: {summary}. " + + "Rows in the lipid classes they cover carry no concentration and say so."); } if (resolution.Compounds.Count == 0) { Console.Error.WriteLine("No internal standard could be resolved, so nothing can be normalized."); @@ -126,23 +167,90 @@ public int Run( var decResults = MsdecResultsReader.ReadMSDecResults(alignmentFile.SpectraFilePath, out _, out _); var accessor = new LcmsMetadataAccessor(storage.DataBaseMapper, storage.Parameter, false); - // "Height" reads the raw peak height, which normalizing does not touch: the result - // is written to a separate field, so exporting the wrong one silently produces a - // file identical to the input. - var quantAccessor = new LegacyQuantValueAccessor("Normalized height", storage.Parameter); var stats = new[] { StatsValue.Average, StatsValue.Stdev }; - Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(outputFile.FullName)) ?? "."); - using (var stream = File.Open(outputFile.FullName, FileMode.Create, FileAccess.Write)) { - new AlignmentCSVExporter().Export( - stream, spots, decResults, files, new MulticlassFileMetaAccessor(0), accessor, quantAccessor, stats); + Directory.CreateDirectory(outputDirectory.FullName); + + // Both matrices, always. The normalized one is derived from the raw one by a + // division nobody can check without seeing both, and a concentration published + // without the measurement behind it asks to be taken on trust. + // "Height" is the raw peak height and normalizing does not touch it: it writes to + // a separate field, so exporting the wrong one produces a file identical to the + // input while every log line reports success. + var written = new List(); + foreach (var (exportType, suffix) in new[] { + ("Height", "_Height.txt"), + ("Normalized height", "_NormalizedHeight.txt"), + }) { + var path = Path.Combine(outputDirectory.FullName, alignmentFile.FileName + suffix); + using (var stream = File.Open(path, FileMode.Create, FileAccess.Write)) { + new AlignmentCSVExporter().Export( + stream, spots, decResults, files, new MulticlassFileMetaAccessor(0), accessor, + new LegacyQuantValueAccessor(exportType, storage.Parameter), stats); + } + written.Add(path); + if (exportType == "Normalized height") { + var redacted = RedactSubstitutedClasses(path, resolution); + if (redacted > 0) { + Console.WriteLine( + $"{redacted} row(s) had no standard of their own class and carry no concentration."); + } + } } Console.WriteLine($"Normalized unit: {unit}"); Console.WriteLine($"Dilution factor applied: {applyDilutionFactor}"); - Console.WriteLine(outputFile.FullName); + foreach (var path in written) { + Console.WriteLine(path); + } return 0; } + /// + /// Removes the numbers that were produced by dividing by the wrong standard. + /// + /// + /// When a class's own standard does not resolve, the normalizer does not leave that + /// class alone: it falls through to the "Any others" standard and quantifies the class + /// against a compound of an entirely different one. A cardiolipin divided by a + /// lysophosphatidylcholine is not a concentration, and it was written into the matrix + /// in the same unit, with the same comment, as a properly quantified row -- nothing in + /// the file told them apart. + /// + /// Refusing outright is the default. Where the run is allowed to continue anyway, the + /// affected rows keep their identity and lose their numbers, and say why in place of + /// them. An annotated wrong number is still read by the next script; an empty cell is + /// not. + /// + private static int RedactSubstitutedClasses(string matrixPath, StandardResolution resolution) { + if (resolution.UnresolvedClasses.Count == 0) return 0; + var lines = File.ReadAllLines(matrixPath); + if (lines.Length <= HeaderRowCount) return 0; + var header = lines[HeaderRowCount - 1].Split(' '); + var ontologyColumn = Array.IndexOf(header, "Ontology"); + var commentColumn = Array.IndexOf(header, "Comment"); + var firstSample = Array.IndexOf(lines[0].Split(' '), "Class") + 1; + if (ontologyColumn < 0 || commentColumn < 0 || firstSample <= 0) return 0; + + var redacted = 0; + for (var index = HeaderRowCount; index < lines.Length; index++) { + var cells = lines[index].Split(' '); + if (cells.Length <= ontologyColumn) continue; + var ontology = cells[ontologyColumn].Trim(); + if (!resolution.UnresolvedClasses.TryGetValue(ontology, out var designated)) continue; + for (var column = firstSample; column < cells.Length; column++) { + cells[column] = string.Empty; + } + cells[commentColumn] = + $"NOT QUANTIFIED: the {ontology} standard {designated} did not resolve in this alignment"; + lines[index] = string.Join(" ", cells); + redacted++; + } + if (redacted > 0) { + File.WriteAllLines(matrixPath, lines); + } + return redacted; + } + private static IMsdialDataStorage LoadProject(string projectFilePath) { var projectFolder = Path.GetDirectoryName(Path.GetFullPath(projectFilePath)) ?? "."; var projectFileName = Path.GetFileName(projectFilePath); @@ -171,6 +279,19 @@ private sealed class StandardResolution { public List Compounds = new List(); public List Unresolved = new List(); public List Report = new List(); + + /// Lipid class -> the standard named for it that could not be found. + /// Standards whose given alignment ID names a different compound. + public List Mismatched = new List(); + + /// Standards matching more than one aligned peak. + public List Ambiguous = new List(); + + public Dictionary UnresolvedClasses = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + /// The "Any others" standard those classes now fall through to. + public string FallbackName = string.Empty; } /// @@ -250,37 +371,56 @@ private static StandardResolution ResolveStandards( } } + // One line per standard, not per class it covers: a standard covering forty + // classes repeated its own resolution forty times, and the handful of lines that + // needed a decision were interleaved somewhere in the middle of the rest. + var resolvedOnce = new Dictionary(StringComparer.OrdinalIgnoreCase); + var classesOf = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var unresolvedOnce = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var record in records) { AlignmentSpotProperty? spot = null; var how = string.Empty; if (record.PeakID >= 0 && byId.TryGetValue(record.PeakID, out var byIdSpot)) { spot = byIdSpot; how = $"alignment ID {record.PeakID}"; - // An ID that names a different compound is a table written for another - // run. Saying so is the whole point of carrying the name as well. + // An ID naming a different compound is a table written for another run. + // Quantifying a class against whatever landed on that ID is a worse + // outcome than not quantifying it, so it stops rather than warns. if (!NameAliases(byIdSpot.Name).Contains(record.StandardName, StringComparer.OrdinalIgnoreCase)) { - result.Report.Add( - $" WARNING {record.StandardName}: alignment ID {record.PeakID} is annotated " - + $"'{byIdSpot.Name}'. The table may belong to a different alignment."); + result.Mismatched.Add( + $"{record.StandardName} (alignment ID {record.PeakID} is annotated '{byIdSpot.Name}')"); } } else if (byName.TryGetValue(record.StandardName, out var candidates)) { spot = candidates.OrderByDescending(item => item.HeightAverage).First(); how = $"annotation, alignment ID {spot.MasterAlignmentID}"; - if (candidates.Count > 1) { - result.Report.Add( - $" NOTE {record.StandardName}: {candidates.Count} aligned peaks carry this annotation; " - + $"the most abundant (ID {spot.MasterAlignmentID}) was used."); + if (candidates.Count > 1 && !resolvedOnce.ContainsKey(record.StandardName)) { + result.Ambiguous.Add( + $"{record.StandardName}: {candidates.Count} aligned peaks carry this annotation; " + + $"the most abundant (ID {spot.MasterAlignmentID}) was used"); } } if (spot is null) { result.Unresolved.Add($"{record.StandardName} (for {record.TargetClass})"); - result.Report.Add($" UNRESOLVED {record.StandardName} -> {record.TargetClass}"); + if (!unresolvedOnce.TryGetValue(record.StandardName, out var missingFor)) { + unresolvedOnce[record.StandardName] = missingFor = new List(); + } + missingFor.Add(record.TargetClass); + if (!record.TargetClass.Equals(StandardCompound.AnyOthers, StringComparison.OrdinalIgnoreCase)) { + result.UnresolvedClasses[record.TargetClass] = record.StandardName; + } continue; } - result.Report.Add( - $" {record.StandardName} -> {record.TargetClass} via {how}, concentration {record.Concentration}"); + if (record.TargetClass.Equals(StandardCompound.AnyOthers, StringComparison.OrdinalIgnoreCase)) { + result.FallbackName = record.StandardName; + } + resolvedOnce[record.StandardName] = $"{how}, concentration {record.Concentration}"; + if (!classesOf.TryGetValue(record.StandardName, out var covered)) { + classesOf[record.StandardName] = covered = new List(); + } + covered.Add(record.TargetClass); result.Compounds.Add(new StandardCompound { StandardName = record.StandardName, TargetClass = record.TargetClass, @@ -290,6 +430,20 @@ private static StandardResolution ResolveStandards( PeakID = spot.MasterAlignmentID, }); } + foreach (var pair in resolvedOnce.OrderBy(item => item.Key, StringComparer.OrdinalIgnoreCase)) { + var covered = classesOf.TryGetValue(pair.Key, out var list) ? list : new List(); + result.Report.Add($" {pair.Key} via {pair.Value}"); + result.Report.Add($" covers {covered.Count} class(es): {string.Join(", ", covered)}"); + } + foreach (var line in result.Ambiguous) { + result.Report.Add($" AMBIGUOUS {line}"); + } + // Last, so the lines that need a decision are the ones still on screen. + foreach (var pair in unresolvedOnce.OrderBy(item => item.Key, StringComparer.OrdinalIgnoreCase)) { + result.Report.Add( + $" UNRESOLVED {pair.Key} -- named for {pair.Value.Count} class(es): " + + string.Join(", ", pair.Value)); + } return result; } From 89c718e6399977ab8ae6b1bdacaa39a9c10e920b Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Sun, 6 Sep 2026 20:49:50 +0900 Subject: [PATCH 5/7] Name the LBM annotator, do not print its path in every row The exporter writes the annotator identifier into every row's Comment column as "Annotation method: ...". The LC-MS and LC-IMMS Console paths passed the library's file path as that identifier, so every exported matrix carried an absolute local directory -- into an artifact whose whole purpose is to be shared, and for a library that may be a private one. The DIMS and IMMS paths already passed a name, so the two halves of the same program disagreed. The identifier is now the library's file stem, which is what a reader needs: a laboratory library is date-stamped, and knowing which one annotated a row is the point. What pins it exactly is the checksum in the run manifest, not a path repeated in every cell. Co-Authored-By: Claude Opus 5 --- .../Process/CommonProcess.cs | 20 +++++++++++++++++++ .../Process/LcimmsProcess.cs | 2 +- .../MsdialCoreTestApp/Process/LcmsProcess.cs | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/CommonProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/CommonProcess.cs index 6f14d606f..50cf22b49 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/CommonProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/CommonProcess.cs @@ -91,6 +91,26 @@ public static bool SetProjectProperty(ParameterBase param, string input, out Lis return true; } + /// + /// Names the LBM annotator for the exported Comment column. + /// + /// + /// The annotator identifier is written into every exported row as + /// "Annotation method: ...". Passing the library's file path there put an + /// absolute local directory into an artifact meant for sharing, and told the + /// reader nothing a directory-free name does not. The file stem is kept because + /// a laboratory library is usually date-stamped and the reader needs to know + /// which one annotated the row; the checksum that pins it exactly belongs in the + /// run manifest, not in every cell. + /// + public static string LbmAnnotatorId(string lbmFilePath) { + var stem = string.IsNullOrWhiteSpace(lbmFilePath) + ? string.Empty + : System.IO.Path.GetFileNameWithoutExtension(lbmFilePath); + return string.IsNullOrWhiteSpace(stem) ? "LbmDB" : "LbmDB: " + stem; + } + + public static void ParseLibraries(ParameterBase param, float targetMz, out IupacDatabase iupacDB, out MoleculeDataBase? mspDB, out MoleculeDataBase? txtDB, out List isotopeTextDB, out List compoundsInTargetMode, diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcimmsProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcimmsProcess.cs index 771162002..a8260e9b3 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcimmsProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcimmsProcess.cs @@ -58,7 +58,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool ]); } if (lbmDB is { Database.Count: > 0 }) { - var lbmAnnotator = new LcimmsMspAnnotator(lbmDB, param.LbmSearchParam, param.TargetOmics, param.LbmFilePath, 1); + var lbmAnnotator = new LcimmsMspAnnotator(lbmDB, param.LbmSearchParam, param.TargetOmics, CommonProcess.LbmAnnotatorId(param.LbmFilePath), 1); dbStorage.AddMoleculeDataBase(lbmDB, [ new MetabolomicsAnnotatorParameterPair(lbmAnnotator.Save(), new AnnotationQueryFactory(lbmAnnotator, param.PeakPickBaseParam, param.LbmSearchParam, ignoreIsotopicPeak: true)), ]); diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs index 2c745e7a2..67176316a 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs @@ -78,7 +78,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool } } if (lbmDB is { Database.Count: > 0 }) { - var lbmAnnotator = new LcmsMspAnnotator(lbmDB, param.LbmSearchParam, TargetOmics.Lipidomics, param.LbmFilePath, lbmAnnotatorPriority); + var lbmAnnotator = new LcmsMspAnnotator(lbmDB, param.LbmSearchParam, TargetOmics.Lipidomics, CommonProcess.LbmAnnotatorId(param.LbmFilePath), lbmAnnotatorPriority); dbStorage.AddMoleculeDataBase(lbmDB, [ new MetabolomicsAnnotatorParameterPair(lbmAnnotator.Save(), new AnnotationQueryFactory(lbmAnnotator, param.PeakPickBaseParam, param.LbmSearchParam, ignoreIsotopicPeak: true)), ]); From 4fa61fd8c01795915eb4647772ac6df7470575c4 Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Sun, 6 Sep 2026 20:58:35 +0900 Subject: [PATCH 6/7] Write the matrices the parameter file asks for The parameter file offers a family of matrix-export flags and is portable into the GUI, where each means what it says. The Console read exactly one of them, IsHeightMatrixExport, and used it to gate an unrelated artifact: a run declaring "Height matrix export: True" produced a long-format quality-assurance table and no height matrix, and said nothing about either. The wide matrices the lipidomics workflow needs existed only as a side effect of the normalize verb. Height, normalized height, area, retention time, mass and signal-to-noise are now each written when their own flag is set. The quality-assurance matrix still follows the height request, because no parameter names it; it is written beside the height matrix rather than instead of it, and both are announced. Verified on one SCIEX file: with Height matrix export: True the run writes AlignResult-*_Height.txt (156 rows) and AlignResult-*.qa.tsv, exit 0, and no drive path appears anywhere in the exported matrix. Co-Authored-By: Claude Opus 5 --- .../MsdialCoreTestApp/Process/LcmsProcess.cs | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs index 67176316a..76f3d4818 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs @@ -228,10 +228,39 @@ IQuantValueAccessor CreateQuantAccessor(string exportType) => alignmentLightPeak Console.WriteLine($"Detailed alignment provenance: {provenanceOutputFile}"); } + // The parameter file offers a family of matrix-export flags and is portable + // into the GUI, where each means what it says. The Console read exactly one + // of them, and used it to gate an unrelated artifact: a run that asked for a + // height matrix got a long-format quality-assurance table and no matrix, with + // nothing said about either. The flags are honoured here. + var matrixFolder = String.IsNullOrWhiteSpace(storage.Parameter.ExportFolderPath) + ? outputFolder + : storage.Parameter.ExportFolderPath; + var requestedMatrices = new List<(bool Requested, string ExportType, string Suffix)> { + (storage.Parameter.IsHeightMatrixExport, "Height", "_Height.txt"), + (storage.Parameter.IsNormalizedMatrixExport, "Normalized height", "_NormalizedHeight.txt"), + (storage.Parameter.IsPeakAreaMatrixExport, "Area", "_Area.txt"), + (storage.Parameter.IsRetentionTimeMatrixExport, "RT", "_RT.txt"), + (storage.Parameter.IsMassMatrixExport, "MZ", "_MZ.txt"), + (storage.Parameter.IsSnMatrixExport, "SN", "_SN.txt"), + }; + if (requestedMatrices.Any(item => item.Requested)) { + Directory.CreateDirectory(matrixFolder); + var matrixStats = new[] { StatsValue.Average, StatsValue.Stdev }; + foreach (var (_, exportType, suffix) in requestedMatrices.Where(item => item.Requested)) { + var matrixFile = Path.Combine(matrixFolder, alignmentFile.FileName + suffix); + using (var matrixStream = File.Open(matrixFile, FileMode.Create, FileAccess.Write)) { + new AlignmentCSVExporter().Export( + matrixStream, result.AlignmentSpotProperties, align_decResults, files, + new MulticlassFileMetaAccessor(0), align_accessor, + new LegacyQuantValueAccessor(exportType, storage.Parameter), matrixStats); + } + Console.WriteLine($"{exportType} matrix: {matrixFile}"); + } + } + if (storage.Parameter.IsHeightMatrixExport) { - var qaOutputFolder = String.IsNullOrWhiteSpace(storage.Parameter.ExportFolderPath) - ? outputFolder - : storage.Parameter.ExportFolderPath; + var qaOutputFolder = matrixFolder; Directory.CreateDirectory(qaOutputFolder); var qaOutputFile = Path.Combine(qaOutputFolder, alignmentFile.FileName + ".qa.tsv"); using var qaStream = File.Open(qaOutputFile, FileMode.Create, FileAccess.Write); @@ -246,6 +275,9 @@ IQuantValueAccessor CreateQuantAccessor(string exportType) => alignmentLightPeak ("SN", CreateQuantAccessor("SN")), ("MSMS", CreateQuantAccessor("MSMS")), ("Reference matched", CreateQuantAccessor("Reference matched"))); + // Written beside the height matrix rather than instead of it: it is the + // same peak heights in long form, with the per-file columns the QA step + // reads. It follows the height request because no parameter names it. Console.WriteLine($"LC-MS quality-assurance matrix: {qaOutputFile}"); } From 274d8a1af6101643a3724a5f7fe092e8fd6d6d0c Mon Sep 17 00:00:00 2001 From: "DESKTOP-382ETUR\\Hiroshi Tsugawa" Date: Sun, 6 Sep 2026 21:03:28 +0900 Subject: [PATCH 7/7] Say which annotation settings an annotator actually uses The same setting is written down twice with two different values: the method file's annotation block and the annotator settings table. Reading the code settles it -- a table row starts from the method block and overrides, column by column, whatever it supplies -- but neither file says so, and a reader of the retained artifacts has no way to tell which number governed the annotations. Each MSP and text annotator now prints its resolved RT, MS1, MS2 tolerances and total score cutoff as it is built, so the run log carries the answer. Verified on one file with a text library whose table says RT tolerance 0.5 and cutoff 0.8 against a method block saying 0.1 and 0.85: the log reads 0.5 and 0.8, exit 0. Co-Authored-By: Claude Opus 5 --- .../MsdialCoreTestApp/Parser/ConfigParser.cs | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs index 39b59401f..4d55279ee 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs @@ -1,4 +1,4 @@ -using CompMs.Common.DataObj.Property; +using CompMs.Common.DataObj.Property; using CompMs.Common.Enum; using CompMs.Common.Extension; using CompMs.Common.Parser; @@ -278,6 +278,7 @@ private static List ReadMspAnnotatorSettingsTable(string fi } } settings.Add(new MspAnnotatorSetting(annotatorId, mspFilePath, priority, searchParameter, targetOmics)); + ReportEffectiveAnnotatorSettings("MSP", annotatorId, mspFilePath, priority, searchParameter); } return settings; } @@ -340,10 +341,29 @@ private static List ReadTextAnnotatorSettingsTable(string var searchParameter = new MsRefSearchParameterBase(param.TextDbSearchParam); ApplyMspSearchParameter(searchParameter, fields, headers); settings.Add(new TextAnnotatorSetting(annotatorId, textDbFilePath, priority, searchParameter)); + ReportEffectiveAnnotatorSettings("Text", annotatorId, textDbFilePath, priority, searchParameter); } return settings; } + /// + /// States the settings an annotator will actually use. + /// + /// + /// A settings row starts from the method file's annotation block and overrides, + /// column by column, whatever the table supplies. So the same setting is written + /// down in two places with two different values and neither file says which one + /// governs. Printing the resolved value settles it in the run log, where a reader + /// of the artifacts can see it. + /// + private static void ReportEffectiveAnnotatorSettings( + string kind, string annotatorId, string filePath, int priority, MsRefSearchParameterBase parameter) { + Console.WriteLine( + $"{kind} annotator {annotatorId} ({Path.GetFileName(filePath)}), priority {priority}: " + + $"RT tolerance {parameter.RtTolerance}, MS1 tolerance {parameter.Ms1Tolerance}, " + + $"MS2 tolerance {parameter.Ms2Tolerance}, total score cutoff {parameter.TotalScoreCutoff}"); + } + private static void ApplyMspSearchParameter(MsRefSearchParameterBase parameter, string[] fields, string[] headers) { SetFloat(fields, headers, value => parameter.MassRangeBegin = value, "massrangebegin", "massbegin"); SetFloat(fields, headers, value => parameter.MassRangeEnd = value, "massrangeend", "massend");