diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs index 49d16ec46..4d55279ee 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Parser/ConfigParser.cs @@ -278,6 +278,7 @@ private static List ReadMspAnnotatorSettingsTable(string fi } } settings.Add(new MspAnnotatorSetting(annotatorId, mspFilePath, priority, searchParameter, targetOmics)); + ReportEffectiveAnnotatorSettings("MSP", annotatorId, mspFilePath, priority, searchParameter); } return settings; } @@ -340,10 +341,29 @@ private static List ReadTextAnnotatorSettingsTable(string var searchParameter = new MsRefSearchParameterBase(param.TextDbSearchParam); ApplyMspSearchParameter(searchParameter, fields, headers); settings.Add(new TextAnnotatorSetting(annotatorId, textDbFilePath, priority, searchParameter)); + ReportEffectiveAnnotatorSettings("Text", annotatorId, textDbFilePath, priority, searchParameter); } return settings; } + /// + /// States the settings an annotator will actually use. + /// + /// + /// A settings row starts from the method file's annotation block and overrides, + /// column by column, whatever the table supplies. So the same setting is written + /// down in two places with two different values and neither file says which one + /// governs. Printing the resolved value settles it in the run log, where a reader + /// of the artifacts can see it. + /// + private static void ReportEffectiveAnnotatorSettings( + string kind, string annotatorId, string filePath, int priority, MsRefSearchParameterBase parameter) { + Console.WriteLine( + $"{kind} annotator {annotatorId} ({Path.GetFileName(filePath)}), priority {priority}: " + + $"RT tolerance {parameter.RtTolerance}, MS1 tolerance {parameter.Ms1Tolerance}, " + + $"MS2 tolerance {parameter.Ms2Tolerance}, total score cutoff {parameter.TotalScoreCutoff}"); + } + private static void ApplyMspSearchParameter(MsRefSearchParameterBase parameter, string[] fields, string[] headers) { SetFloat(fields, headers, value => parameter.MassRangeBegin = value, "massrangebegin", "massbegin"); SetFloat(fields, headers, value => parameter.MassRangeEnd = value, "massrangeend", "massend"); @@ -950,6 +970,10 @@ public static bool ReadCommonParameter(ParameterBase param, string method, strin case "set fully labeled reference file": if (valueLower == "true" || valueLower == "false") param.SetFullyLabeledReferenceFile = bool.Parse(valueLower); return true; case "non labeled reference id": if (int.TryParse(valueLower, out int nonlabeledrefid)) param.NonLabeledReferenceID = nonlabeledrefid; return true; case "fully labeled reference id": if (int.TryParse(valueLower, out int fulllabeledrefid)) param.FullyLabeledReferenceID = fulllabeledrefid; return true; + // ParameterBase writes "Number of threads" into every exported method file, + // but nothing read it back, so a method file could describe a thread count + // it could never request and every Console run stayed on the default of 2. + case "number of threads": if (int.TryParse(valueLower, out int numthreads) && numthreads > 0) param.NumThreads = numthreads; return true; case "isotope tracking dictionary id": if (int.TryParse(valueLower, out int isotopetrackdictionaryid)) param.IsotopeTrackingDictionary.SelectedID = isotopetrackdictionaryid; return true; //CorrDec settings diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/CommonProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/CommonProcess.cs index 6f14d606f..50cf22b49 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/CommonProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/CommonProcess.cs @@ -91,6 +91,26 @@ public static bool SetProjectProperty(ParameterBase param, string input, out Lis return true; } + /// + /// Names the LBM annotator for the exported Comment column. + /// + /// + /// The annotator identifier is written into every exported row as + /// "Annotation method: ...". Passing the library's file path there put an + /// absolute local directory into an artifact meant for sharing, and told the + /// reader nothing a directory-free name does not. The file stem is kept because + /// a laboratory library is usually date-stamped and the reader needs to know + /// which one annotated the row; the checksum that pins it exactly belongs in the + /// run manifest, not in every cell. + /// + public static string LbmAnnotatorId(string lbmFilePath) { + var stem = string.IsNullOrWhiteSpace(lbmFilePath) + ? string.Empty + : System.IO.Path.GetFileNameWithoutExtension(lbmFilePath); + return string.IsNullOrWhiteSpace(stem) ? "LbmDB" : "LbmDB: " + stem; + } + + public static void ParseLibraries(ParameterBase param, float targetMz, out IupacDatabase iupacDB, out MoleculeDataBase? mspDB, out MoleculeDataBase? txtDB, out List isotopeTextDB, out List compoundsInTargetMode, diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcimmsProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcimmsProcess.cs index 771162002..a8260e9b3 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcimmsProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcimmsProcess.cs @@ -58,7 +58,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool ]); } if (lbmDB is { Database.Count: > 0 }) { - var lbmAnnotator = new LcimmsMspAnnotator(lbmDB, param.LbmSearchParam, param.TargetOmics, param.LbmFilePath, 1); + var lbmAnnotator = new LcimmsMspAnnotator(lbmDB, param.LbmSearchParam, param.TargetOmics, CommonProcess.LbmAnnotatorId(param.LbmFilePath), 1); dbStorage.AddMoleculeDataBase(lbmDB, [ new MetabolomicsAnnotatorParameterPair(lbmAnnotator.Save(), new AnnotationQueryFactory(lbmAnnotator, param.PeakPickBaseParam, param.LbmSearchParam, ignoreIsotopicPeak: true)), ]); diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs index 2c745e7a2..76f3d4818 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/LcmsProcess.cs @@ -78,7 +78,7 @@ public int Run(string inputFolder, string outputFolder, string methodFile, bool } } if (lbmDB is { Database.Count: > 0 }) { - var lbmAnnotator = new LcmsMspAnnotator(lbmDB, param.LbmSearchParam, TargetOmics.Lipidomics, param.LbmFilePath, lbmAnnotatorPriority); + var lbmAnnotator = new LcmsMspAnnotator(lbmDB, param.LbmSearchParam, TargetOmics.Lipidomics, CommonProcess.LbmAnnotatorId(param.LbmFilePath), lbmAnnotatorPriority); dbStorage.AddMoleculeDataBase(lbmDB, [ new MetabolomicsAnnotatorParameterPair(lbmAnnotator.Save(), new AnnotationQueryFactory(lbmAnnotator, param.PeakPickBaseParam, param.LbmSearchParam, ignoreIsotopicPeak: true)), ]); @@ -228,10 +228,39 @@ IQuantValueAccessor CreateQuantAccessor(string exportType) => alignmentLightPeak Console.WriteLine($"Detailed alignment provenance: {provenanceOutputFile}"); } + // The parameter file offers a family of matrix-export flags and is portable + // into the GUI, where each means what it says. The Console read exactly one + // of them, and used it to gate an unrelated artifact: a run that asked for a + // height matrix got a long-format quality-assurance table and no matrix, with + // nothing said about either. The flags are honoured here. + var matrixFolder = String.IsNullOrWhiteSpace(storage.Parameter.ExportFolderPath) + ? outputFolder + : storage.Parameter.ExportFolderPath; + var requestedMatrices = new List<(bool Requested, string ExportType, string Suffix)> { + (storage.Parameter.IsHeightMatrixExport, "Height", "_Height.txt"), + (storage.Parameter.IsNormalizedMatrixExport, "Normalized height", "_NormalizedHeight.txt"), + (storage.Parameter.IsPeakAreaMatrixExport, "Area", "_Area.txt"), + (storage.Parameter.IsRetentionTimeMatrixExport, "RT", "_RT.txt"), + (storage.Parameter.IsMassMatrixExport, "MZ", "_MZ.txt"), + (storage.Parameter.IsSnMatrixExport, "SN", "_SN.txt"), + }; + if (requestedMatrices.Any(item => item.Requested)) { + Directory.CreateDirectory(matrixFolder); + var matrixStats = new[] { StatsValue.Average, StatsValue.Stdev }; + foreach (var (_, exportType, suffix) in requestedMatrices.Where(item => item.Requested)) { + var matrixFile = Path.Combine(matrixFolder, alignmentFile.FileName + suffix); + using (var matrixStream = File.Open(matrixFile, FileMode.Create, FileAccess.Write)) { + new AlignmentCSVExporter().Export( + matrixStream, result.AlignmentSpotProperties, align_decResults, files, + new MulticlassFileMetaAccessor(0), align_accessor, + new LegacyQuantValueAccessor(exportType, storage.Parameter), matrixStats); + } + Console.WriteLine($"{exportType} matrix: {matrixFile}"); + } + } + if (storage.Parameter.IsHeightMatrixExport) { - var qaOutputFolder = String.IsNullOrWhiteSpace(storage.Parameter.ExportFolderPath) - ? outputFolder - : storage.Parameter.ExportFolderPath; + var qaOutputFolder = matrixFolder; Directory.CreateDirectory(qaOutputFolder); var qaOutputFile = Path.Combine(qaOutputFolder, alignmentFile.FileName + ".qa.tsv"); using var qaStream = File.Open(qaOutputFile, FileMode.Create, FileAccess.Write); @@ -246,6 +275,9 @@ IQuantValueAccessor CreateQuantAccessor(string exportType) => alignmentLightPeak ("SN", CreateQuantAccessor("SN")), ("MSMS", CreateQuantAccessor("MSMS")), ("Reference matched", CreateQuantAccessor("Reference matched"))); + // Written beside the height matrix rather than instead of it: it is the + // same peak heights in long form, with the per-file columns the QA step + // reads. It follows the height request because no parameter names it. Console.WriteLine($"LC-MS quality-assurance matrix: {qaOutputFile}"); } diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs index 8b9cd44c1..babbd6f84 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/MainProcess.cs @@ -1,4 +1,4 @@ -using CompMs.App.MsdialConsole.Process.MoleculerNetworking; +using CompMs.App.MsdialConsole.Process.MoleculerNetworking; using CompMs.App.MsdialConsole.Properties; using CompMs.Common.Enum; using CompMs.Common.Extension; @@ -443,6 +443,48 @@ public static void SetMsnCommand(Command root) { root.Add(cmd); } + public static void SetNormalizeCommand(Command root) { + var cmd = new Command( + "normalize", + "Normalize an aligned result against internal standards and export the matrix"); + var input = new Option("--input", "-i") { Required = true }; + input.Description = "MS-DIAL project file holding the alignment to normalize"; + var standards = new Option("--standards", "-s") { Required = true }; + standards.Description = "Table of internal standards: StandardName, TargetClass, Concentration, optional PeakID, DilutionRate, MolecularWeight"; + var output = new Option("--output", "-o") { Required = true }; + output.Description = "Directory to write the raw and normalized alignment matrices into"; + var unit = new Option("--unit", "-u") { + DefaultValueFactory = _ => IonAbundanceUnit.NormalizedByInternalStandardPeakHeight, + }; + unit.Description = "Unit of the normalized abundance, e.g. pmol_per_microL_plasma"; + var alignment = new Option("--alignment") { DefaultValueFactory = _ => 0 }; + alignment.Description = "Index of the alignment result within the project"; + var dilution = new Option("--apply-dilution-factor", "-d"); + dilution.Description = "Divide by each file's dilution factor after normalizing"; + var allowUnresolved = new Option("--allow-unresolved-standards"); + allowUnresolved.Description = "Continue when a standard is not found, leaving its classes without a concentration"; + var allowMismatched = new Option("--allow-mismatched-peak-ids"); + allowMismatched.Description = "Continue when a standard's alignment ID holds a different compound"; + cmd.Options.Add(input); + cmd.Options.Add(standards); + cmd.Options.Add(output); + cmd.Options.Add(unit); + cmd.Options.Add(alignment); + cmd.Options.Add(dilution); + cmd.Options.Add(allowUnresolved); + cmd.Options.Add(allowMismatched); + cmd.SetAction(parseResult => new NormalizationProcess().Run( + parseResult.GetRequiredValue(input), + parseResult.GetRequiredValue(standards), + parseResult.GetRequiredValue(output), + parseResult.GetValue(unit), + parseResult.GetValue(alignment), + parseResult.GetValue(dilution), + parseResult.GetValue(allowUnresolved), + parseResult.GetValue(allowMismatched))); + root.Subcommands.Add(cmd); + } + public static void SetEicCommand(Command root) { var eic = new Command("eic", "Export extracted ion chromatograms"); var raw = new Command("raw", "Export EICs from a raw data file"); diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs b/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs new file mode 100644 index 000000000..265a57a06 --- /dev/null +++ b/tests/MSDIAL5/MsdialCoreTestApp/Process/NormalizationProcess.cs @@ -0,0 +1,472 @@ +using CompMs.Common.Components; +using CompMs.Common.DataObj.Result; +using CompMs.Common.Enum; +using CompMs.Common.Interfaces; +using CompMs.MsdialCore.Algorithm.Annotation; +using CompMs.MsdialCore.DataObj; +using CompMs.MsdialCore.Export; +using CompMs.MsdialCore.MSDec; +using CompMs.MsdialCore.Normalize; +using CompMs.MsdialCore.Parameter; +using CompMs.MsdialCore.Parser; +using CompMs.MsdialIntegrate.Parser; +using CompMs.MsdialLcMsApi.Export; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; + +namespace CompMs.App.MsdialConsole.Process; + +/// +/// Normalizes an aligned result against internal standards, from the command line. +/// +/// +/// The algorithm already lives in MsdialCore and is what the graphical application +/// calls; only the way in was missing. This supplies it: read a saved project and its +/// alignment, build the standard table, normalize, export. +/// +/// One thing the graphical application can take for granted and a command line cannot: +/// which aligned peak *is* each standard. StandardCompound identifies it by PeakID, +/// which is an alignment ID and therefore belongs to one alignment and no other. A +/// table written for one run would silently point at unrelated peaks in the next, so a +/// standard may instead name itself and be resolved against the annotations of the +/// alignment actually being normalized. An unresolved standard stops the run rather +/// than quietly normalizing its whole lipid class against nothing. +/// +public sealed class NormalizationProcess { + /// Label rows plus the column-name row that precede the data. + private const int HeaderRowCount = 5; + + public int Run( + FileInfo projectFile, + FileInfo standardsFile, + DirectoryInfo outputDirectory, + IonAbundanceUnit unit, + int alignmentIndex, + bool applyDilutionFactor, + bool allowUnresolvedStandards, + bool allowMismatchedPeakIds) { + if (!projectFile.Exists) { + Console.Error.WriteLine($"Project file was not found: {projectFile.FullName}"); + return -1; + } + if (!standardsFile.Exists) { + Console.Error.WriteLine($"Internal standard table was not found: {standardsFile.FullName}"); + return -1; + } + + // The data storage is only half a project: the annotation databases live beside it + // and the raw MessagePack load leaves them null, which surfaces much later as a + // null reference inside the evaluator. Load it the way the application does. + IMsdialDataStorage storage; + try { + storage = LoadProject(projectFile.FullName); + } + catch (Exception error) { + // The run writes both a .mdproject and a .mddata, and only the second holds + // the data storage. Passing the one that looks more like a project produced + // fifteen frames of MessagePack internals and no statement of what to do. + var extension = Path.GetExtension(projectFile.FullName); + var sibling = Path.ChangeExtension(projectFile.FullName, ".mddata"); + var advice = File.Exists(sibling) && !extension.Equals(".mddata", StringComparison.OrdinalIgnoreCase) + ? $" Pass the data file beside it instead: {sibling}" + : " Pass the .mddata file written by the analysis run."; + Console.Error.WriteLine( + $"{projectFile.FullName} could not be read as an MS-DIAL data file " + + $"({error.GetType().Name})." + advice); + return -1; + } + var files = storage.AnalysisFiles.Where(file => file.AnalysisFileIncluded).ToList(); + if (files.Count == 0) { + Console.Error.WriteLine("The project contains no included analysis files."); + return -1; + } + if (storage.AlignmentFiles is null || storage.AlignmentFiles.Count == 0) { + Console.Error.WriteLine("The project contains no alignment result to normalize."); + return -1; + } + if (alignmentIndex < 0 || alignmentIndex >= storage.AlignmentFiles.Count) { + Console.Error.WriteLine( + $"Alignment index {alignmentIndex} is outside the project's {storage.AlignmentFiles.Count} alignment result(s)."); + return -1; + } + + var alignmentFile = storage.AlignmentFiles[alignmentIndex]; + var container = AlignmentResultContainer.Load(alignmentFile); + var spots = container.AlignmentSpotProperties; + if (spots is null || spots.Count == 0) { + Console.Error.WriteLine("The alignment result contains no spots."); + return -1; + } + + List records; + try { + records = ReadStandards(standardsFile.FullName); + } + catch (FormatException error) { + Console.Error.WriteLine(error.Message); + return -1; + } + if (records.Count == 0) { + Console.Error.WriteLine("The internal standard table contains no rows."); + return -1; + } + + var resolution = ResolveStandards(records, spots); + foreach (var line in resolution.Report) { + Console.WriteLine(line); + } + if (resolution.Mismatched.Count > 0 && !allowMismatchedPeakIds) { + // A standard that cannot be found already stops the run. One found and + // demonstrably pointing at a different compound is the worse case of the two, + // and it used to warn and carry on -- quantifying a lipid class against + // whatever happened to occupy that alignment ID. + Console.Error.WriteLine( + $"{resolution.Mismatched.Count} standard(s) name an alignment ID that holds a different " + + $"compound: {string.Join("; ", resolution.Mismatched)}. An alignment ID belongs to the run it " + + "was written for. Remove the PeakID column so the standards resolve by name, or pass " + + "--allow-mismatched-peak-ids if the annotations are wrong rather than the table."); + return 2; + } + if (resolution.Mismatched.Count > 0) { + Console.WriteLine( + $"WARNING: {resolution.Mismatched.Count} standard(s) were taken from an alignment ID that holds " + + $"a different compound: {string.Join("; ", resolution.Mismatched)}."); + } + if (resolution.Unresolved.Count > 0) { + var summary = string.Join(", ", resolution.Unresolved); + if (!allowUnresolvedStandards) { + Console.Error.WriteLine( + $"{resolution.Unresolved.Count} internal standard(s) were not found in the alignment: {summary}. " + + "Every lipid class they cover would be left without a concentration. " + + "Confirm the annotation, or pass --allow-unresolved-standards to continue, which " + + "empties those rows rather than quantifying them against another class."); + return 2; + } + Console.WriteLine( + $"WARNING: {resolution.Unresolved.Count} internal standard(s) did not resolve: {summary}. " + + "Rows in the lipid classes they cover carry no concentration and say so."); + } + if (resolution.Compounds.Count == 0) { + Console.Error.WriteLine("No internal standard could be resolved, so nothing can be normalized."); + return 2; + } + + var evaluator = FacadeMatchResultEvaluator.FromDataBases(storage.DataBases); + Normalization.SplashNormalize( + files, + spots, + storage.DataBaseMapper, + resolution.Compounds, + unit, + evaluator, + applyDilutionFactor); + container.IsNormalized = true; + + var decResults = MsdecResultsReader.ReadMSDecResults(alignmentFile.SpectraFilePath, out _, out _); + var accessor = new LcmsMetadataAccessor(storage.DataBaseMapper, storage.Parameter, false); + var stats = new[] { StatsValue.Average, StatsValue.Stdev }; + Directory.CreateDirectory(outputDirectory.FullName); + + // Both matrices, always. The normalized one is derived from the raw one by a + // division nobody can check without seeing both, and a concentration published + // without the measurement behind it asks to be taken on trust. + // "Height" is the raw peak height and normalizing does not touch it: it writes to + // a separate field, so exporting the wrong one produces a file identical to the + // input while every log line reports success. + var written = new List(); + foreach (var (exportType, suffix) in new[] { + ("Height", "_Height.txt"), + ("Normalized height", "_NormalizedHeight.txt"), + }) { + var path = Path.Combine(outputDirectory.FullName, alignmentFile.FileName + suffix); + using (var stream = File.Open(path, FileMode.Create, FileAccess.Write)) { + new AlignmentCSVExporter().Export( + stream, spots, decResults, files, new MulticlassFileMetaAccessor(0), accessor, + new LegacyQuantValueAccessor(exportType, storage.Parameter), stats); + } + written.Add(path); + if (exportType == "Normalized height") { + var redacted = RedactSubstitutedClasses(path, resolution); + if (redacted > 0) { + Console.WriteLine( + $"{redacted} row(s) had no standard of their own class and carry no concentration."); + } + } + } + + Console.WriteLine($"Normalized unit: {unit}"); + Console.WriteLine($"Dilution factor applied: {applyDilutionFactor}"); + foreach (var path in written) { + Console.WriteLine(path); + } + return 0; + } + + /// + /// Removes the numbers that were produced by dividing by the wrong standard. + /// + /// + /// When a class's own standard does not resolve, the normalizer does not leave that + /// class alone: it falls through to the "Any others" standard and quantifies the class + /// against a compound of an entirely different one. A cardiolipin divided by a + /// lysophosphatidylcholine is not a concentration, and it was written into the matrix + /// in the same unit, with the same comment, as a properly quantified row -- nothing in + /// the file told them apart. + /// + /// Refusing outright is the default. Where the run is allowed to continue anyway, the + /// affected rows keep their identity and lose their numbers, and say why in place of + /// them. An annotated wrong number is still read by the next script; an empty cell is + /// not. + /// + private static int RedactSubstitutedClasses(string matrixPath, StandardResolution resolution) { + if (resolution.UnresolvedClasses.Count == 0) return 0; + var lines = File.ReadAllLines(matrixPath); + if (lines.Length <= HeaderRowCount) return 0; + var header = lines[HeaderRowCount - 1].Split(' '); + var ontologyColumn = Array.IndexOf(header, "Ontology"); + var commentColumn = Array.IndexOf(header, "Comment"); + var firstSample = Array.IndexOf(lines[0].Split(' '), "Class") + 1; + if (ontologyColumn < 0 || commentColumn < 0 || firstSample <= 0) return 0; + + var redacted = 0; + for (var index = HeaderRowCount; index < lines.Length; index++) { + var cells = lines[index].Split(' '); + if (cells.Length <= ontologyColumn) continue; + var ontology = cells[ontologyColumn].Trim(); + if (!resolution.UnresolvedClasses.TryGetValue(ontology, out var designated)) continue; + for (var column = firstSample; column < cells.Length; column++) { + cells[column] = string.Empty; + } + cells[commentColumn] = + $"NOT QUANTIFIED: the {ontology} standard {designated} did not resolve in this alignment"; + lines[index] = string.Join(" ", cells); + redacted++; + } + if (redacted > 0) { + File.WriteAllLines(matrixPath, lines); + } + return redacted; + } + + private static IMsdialDataStorage LoadProject(string projectFilePath) { + var projectFolder = Path.GetDirectoryName(Path.GetFullPath(projectFilePath)) ?? "."; + var projectFileName = Path.GetFileName(projectFilePath); + var serializer = new MsdialIntegrateSerializer(); + using (IStreamManager streamManager = new DirectoryTreeStreamManager(projectFolder)) { + var storage = serializer + .LoadAsync(streamManager, projectFileName, projectFolder, string.Empty) + .GetAwaiter().GetResult(); + streamManager.Complete(); + storage.FixDatasetFolder(projectFolder); + return storage; + } + } + + private sealed class StandardRecord { + public string StandardName = string.Empty; + public string TargetClass = string.Empty; + public double Concentration; + public double DilutionRate = 1d; + public double MolecularWeight; + public int PeakID = -1; + public int LineNumber; + } + + private sealed class StandardResolution { + public List Compounds = new List(); + public List Unresolved = new List(); + public List Report = new List(); + + /// Lipid class -> the standard named for it that could not be found. + /// Standards whose given alignment ID names a different compound. + public List Mismatched = new List(); + + /// Standards matching more than one aligned peak. + public List Ambiguous = new List(); + + public Dictionary UnresolvedClasses = + new Dictionary(StringComparer.OrdinalIgnoreCase); + + /// The "Any others" standard those classes now fall through to. + public string FallbackName = string.Empty; + } + + /// + /// Reads the class-to-standard table: which standard normalizes which lipid class, + /// at what amount. "Any others" covers every class the table does not name. + /// + private static List ReadStandards(string path) { + var lines = File.ReadAllLines(path); + var header = lines.FirstOrDefault(line => !string.IsNullOrWhiteSpace(line)); + if (header is null) { + throw new FormatException("The internal standard table is empty."); + } + var separator = header.Contains('\t') ? '\t' : ','; + var columns = header.Split(separator) + .Select((name, index) => (name: Normalize(name), index)) + .ToDictionary(item => item.name, item => item.index); + + int Column(string name) => columns.TryGetValue(Normalize(name), out var index) ? index : -1; + var nameColumn = Column("StandardName"); + var classColumn = Column("TargetClass"); + var concentrationColumn = Column("Concentration"); + if (nameColumn < 0 || classColumn < 0 || concentrationColumn < 0) { + throw new FormatException( + "The internal standard table needs StandardName, TargetClass and Concentration columns; " + + $"it has: {string.Join(", ", header.Split(separator))}"); + } + var peakColumn = Column("PeakID"); + var dilutionColumn = Column("DilutionRate"); + var weightColumn = Column("MolecularWeight"); + + var records = new List(); + var started = false; + for (var index = 0; index < lines.Length; index++) { + var line = lines[index]; + if (string.IsNullOrWhiteSpace(line)) continue; + if (!started) { started = true; continue; } + var cells = line.Split(separator); + string Cell(int column) => column >= 0 && column < cells.Length ? cells[column].Trim() : string.Empty; + var standardName = Cell(nameColumn); + var targetClass = Cell(classColumn); + if (standardName.Length == 0 || targetClass.Length == 0) continue; + var record = new StandardRecord { + StandardName = standardName, + TargetClass = targetClass, + LineNumber = index + 1, + PeakID = ParseInt(Cell(peakColumn), -1), + MolecularWeight = ParseDouble(Cell(weightColumn), 0d), + }; + var concentration = Cell(concentrationColumn); + if (!double.TryParse(concentration, NumberStyles.Float, CultureInfo.InvariantCulture, out var value)) { + throw new FormatException( + $"Line {record.LineNumber}: '{concentration}' is not a concentration for {standardName}."); + } + record.Concentration = value; + record.DilutionRate = ParseDouble(Cell(dilutionColumn), 1d); + if (record.DilutionRate <= 0d) record.DilutionRate = 1d; + records.Add(record); + } + return records; + } + + /// + /// Ties each standard to the aligned peak that carries it, by the alignment ID the + /// table gives or, failing that, by the name the alignment annotated. + /// + private static StandardResolution ResolveStandards( + IReadOnlyList records, IReadOnlyList spots) { + var result = new StandardResolution(); + var byId = spots.ToDictionary(spot => spot.MasterAlignmentID, spot => spot); + var byName = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var spot in spots) { + foreach (var alias in NameAliases(spot.Name)) { + if (!byName.TryGetValue(alias, out var bucket)) { + byName[alias] = bucket = new List(); + } + bucket.Add(spot); + } + } + + // One line per standard, not per class it covers: a standard covering forty + // classes repeated its own resolution forty times, and the handful of lines that + // needed a decision were interleaved somewhere in the middle of the rest. + var resolvedOnce = new Dictionary(StringComparer.OrdinalIgnoreCase); + var classesOf = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var unresolvedOnce = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + foreach (var record in records) { + AlignmentSpotProperty? spot = null; + var how = string.Empty; + if (record.PeakID >= 0 && byId.TryGetValue(record.PeakID, out var byIdSpot)) { + spot = byIdSpot; + how = $"alignment ID {record.PeakID}"; + // An ID naming a different compound is a table written for another run. + // Quantifying a class against whatever landed on that ID is a worse + // outcome than not quantifying it, so it stops rather than warns. + if (!NameAliases(byIdSpot.Name).Contains(record.StandardName, StringComparer.OrdinalIgnoreCase)) { + result.Mismatched.Add( + $"{record.StandardName} (alignment ID {record.PeakID} is annotated '{byIdSpot.Name}')"); + } + } + else if (byName.TryGetValue(record.StandardName, out var candidates)) { + spot = candidates.OrderByDescending(item => item.HeightAverage).First(); + how = $"annotation, alignment ID {spot.MasterAlignmentID}"; + if (candidates.Count > 1 && !resolvedOnce.ContainsKey(record.StandardName)) { + result.Ambiguous.Add( + $"{record.StandardName}: {candidates.Count} aligned peaks carry this annotation; " + + $"the most abundant (ID {spot.MasterAlignmentID}) was used"); + } + } + + if (spot is null) { + result.Unresolved.Add($"{record.StandardName} (for {record.TargetClass})"); + if (!unresolvedOnce.TryGetValue(record.StandardName, out var missingFor)) { + unresolvedOnce[record.StandardName] = missingFor = new List(); + } + missingFor.Add(record.TargetClass); + if (!record.TargetClass.Equals(StandardCompound.AnyOthers, StringComparison.OrdinalIgnoreCase)) { + result.UnresolvedClasses[record.TargetClass] = record.StandardName; + } + continue; + } + if (record.TargetClass.Equals(StandardCompound.AnyOthers, StringComparison.OrdinalIgnoreCase)) { + result.FallbackName = record.StandardName; + } + resolvedOnce[record.StandardName] = $"{how}, concentration {record.Concentration}"; + if (!classesOf.TryGetValue(record.StandardName, out var covered)) { + classesOf[record.StandardName] = covered = new List(); + } + covered.Add(record.TargetClass); + result.Compounds.Add(new StandardCompound { + StandardName = record.StandardName, + TargetClass = record.TargetClass, + Concentration = record.Concentration, + DilutionRate = record.DilutionRate, + MolecularWeight = record.MolecularWeight, + PeakID = spot.MasterAlignmentID, + }); + } + foreach (var pair in resolvedOnce.OrderBy(item => item.Key, StringComparer.OrdinalIgnoreCase)) { + var covered = classesOf.TryGetValue(pair.Key, out var list) ? list : new List(); + result.Report.Add($" {pair.Key} via {pair.Value}"); + result.Report.Add($" covers {covered.Count} class(es): {string.Join(", ", covered)}"); + } + foreach (var line in result.Ambiguous) { + result.Report.Add($" AMBIGUOUS {line}"); + } + // Last, so the lines that need a decision are the ones still on screen. + foreach (var pair in unresolvedOnce.OrderBy(item => item.Key, StringComparer.OrdinalIgnoreCase)) { + result.Report.Add( + $" UNRESOLVED {pair.Key} -- named for {pair.Value.Count} class(es): " + + string.Join(", ", pair.Value)); + } + return result; + } + + /// + /// The names an aligned peak answers to. MS-DIAL reports a lipid at two resolutions + /// separated by a bar, such as "PC 33:1(d7)|PC 15:0_18:1(d7)", and a standard table + /// may reasonably name either one. + /// + private static IEnumerable NameAliases(string? name) { + if (string.IsNullOrWhiteSpace(name)) yield break; + yield return name!.Trim(); + foreach (var part in name!.Split('|')) { + var trimmed = part.Trim(); + if (trimmed.Length > 0) yield return trimmed; + } + } + + private static string Normalize(string value) => + new string((value ?? string.Empty).Where(char.IsLetterOrDigit).ToArray()).ToLowerInvariant(); + + private static int ParseInt(string value, int fallback) => + int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed) ? parsed : fallback; + + private static double ParseDouble(string value, double fallback) => + double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out var parsed) ? parsed : fallback; +} diff --git a/tests/MSDIAL5/MsdialCoreTestApp/Program.cs b/tests/MSDIAL5/MsdialCoreTestApp/Program.cs index f93191095..363c80381 100644 --- a/tests/MSDIAL5/MsdialCoreTestApp/Program.cs +++ b/tests/MSDIAL5/MsdialCoreTestApp/Program.cs @@ -1,4 +1,4 @@ -using CompMs.App.MsdialConsole.Process; +using CompMs.App.MsdialConsole.Process; using CompMs.App.MsdialConsole.Properties; using System.CommandLine; using System.CommandLine.Invocation; @@ -182,6 +182,7 @@ public static Task Main(string[] args) { MainProcess.SetDimsCommand(root); MainProcess.SetImmsCommand(root); MainProcess.SetMsnCommand(root); + MainProcess.SetNormalizeCommand(root); MainProcess.SetEicCommand(root); MainProcess.SetRtCorrectionCommand(root); MainProcess.SetImageGenerationCommand(root);