From 7867c46da5990d28c9c012c759a5b7a561d40183 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Wed, 19 Feb 2025 15:04:53 +0100 Subject: [PATCH 01/14] #78: Added implementation for memory-mapping files beyond 2GB --- pom.xml | 6 +- .../com/riscure/trs/LargePreMappedFile.java | 105 ++++++++++++++++++ src/main/java/com/riscure/trs/TraceSet.java | 64 ++++------- 3 files changed, 131 insertions(+), 44 deletions(-) create mode 100644 src/main/java/com/riscure/trs/LargePreMappedFile.java diff --git a/pom.xml b/pom.xml index ada9def..c3112cc 100644 --- a/pom.xml +++ b/pom.xml @@ -36,9 +36,9 @@ - 8 - 8 - 8 + 11 + 11 + 11 UTF-8 diff --git a/src/main/java/com/riscure/trs/LargePreMappedFile.java b/src/main/java/com/riscure/trs/LargePreMappedFile.java new file mode 100644 index 0000000..4e9fac3 --- /dev/null +++ b/src/main/java/com/riscure/trs/LargePreMappedFile.java @@ -0,0 +1,105 @@ +package com.riscure.trs; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; + +public class LargePreMappedFile implements AutoCloseable{ + private final FileChannel channel; + + private final List buffers = new ArrayList<>(); + private final long readOffset; + private final long traceSize; + private final long fileSize; + + private long totalBufferSize; + + public LargePreMappedFile(FileChannel channel, long metaDataSize, long traceSize) throws IOException { + this.channel = channel; + this.readOffset = metaDataSize; + this.traceSize = traceSize; + this.fileSize = channel.size() - readOffset; + + mapBuffers(); + } + + public ByteBuffer getBuffer(int index) { + if (traceSize == 0) { + return ByteBuffer.wrap(new byte[0]); + } + return findBufferAndMoveToTrace(index); + } + + private ByteBuffer findBufferAndMoveToTrace(int traceIndex) { + MappedBuffer mappedBuffer = buffers.stream() + .filter(buffer -> traceIndex >= buffer.getFirstTraceIndex() && + traceIndex < buffer.getFirstTraceIndex() + buffer.getNumberOfTraces()) + .findFirst() + .orElseThrow(); + ByteBuffer buffer = mappedBuffer.getBuffer(); + int traceIndexInBuffer = traceIndex - mappedBuffer.getFirstTraceIndex(); + long positionInBuffer = traceIndexInBuffer * traceSize; + buffer.position((int) positionInBuffer); + return buffer; + } + + private void mapBuffers() { + if (traceSize > 0) { + int tracesPerBuffer = (int) (Integer.MAX_VALUE / traceSize); + long maximumBufferSize = tracesPerBuffer * traceSize; + + int firstTraceIndex = 0; + for (long offset = 0; offset < fileSize; offset += maximumBufferSize) { + buffers.add(mapBuffer(firstTraceIndex, maximumBufferSize)); + firstTraceIndex += tracesPerBuffer; + } + } + } + + private MappedBuffer mapBuffer(int firstTraceIndex, long bufferSize) { + try { + long bufferStart = firstTraceIndex * traceSize; + this.totalBufferSize += bufferSize; + long limitedBufferSize = Math.min(fileSize - bufferStart, bufferSize); + MappedBuffer mappedBuffer = new MappedBuffer(this.channel.map(FileChannel.MapMode.READ_ONLY, readOffset + bufferStart, limitedBufferSize), + firstTraceIndex, + (int) (traceSize > 0 ? (limitedBufferSize / traceSize) : 0)); + mappedBuffer.buffer.order(ByteOrder.LITTLE_ENDIAN); + return mappedBuffer; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + buffers.clear(); + } + + private static class MappedBuffer { + private final ByteBuffer buffer; + private final int firstTraceIndex; + private final int numberOfTraces; + + MappedBuffer(ByteBuffer buffer, int firstTraceIndex, int numberOfTraces) { + this.buffer = buffer; + this.firstTraceIndex = firstTraceIndex; + this.numberOfTraces = numberOfTraces; + } + + public ByteBuffer getBuffer() { + return buffer; + } + + public int getFirstTraceIndex() { + return firstTraceIndex; + } + + public int getNumberOfTraces() { + return numberOfTraces; + } + } +} diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index ee46acb..8ec00aa 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -34,18 +34,17 @@ public class TraceSet implements AutoCloseable { private static final String TRACE_LENGTH_DIFFERS = "All traces in a set need to be the same length, but current trace length (%d) differs from the previous trace(s) (%d)"; private static final String TRACE_DATA_LENGTH_DIFFERS = "All traces in a set need to have the same data length, but current trace data length (%d) differs from the previous trace(s) (%d)"; private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; - private static final long MAX_BUFFER_SIZE = Integer.MAX_VALUE; private static final String PARAMETER_NOT_DEFINED = "Parameter %s is saved in the trace, but was not found in the header definition"; + // This is excessive for the header, but it's only the initial maximum + private static final long MAX_METADATA_SIZE = 100_000_000L; //Reading variables private int metaDataSize; private FileInputStream readStream; - private FileChannel channel; - private ByteBuffer buffer; + private ByteBuffer metaDataBuffer; + private LargePreMappedFile mappedFile; - private long bufferStart; //the byte index of the file where the buffer window starts - private long bufferSize; //the number of bytes that are in the buffer window private long fileSize; //the total number of bytes in the underlying file //Writing variables @@ -66,16 +65,19 @@ private TraceSet(String inputFileName) throws IOException, TRSFormatException { this.open = true; this.path = Paths.get(inputFileName); this.readStream = new FileInputStream(inputFileName); - this.channel = readStream.getChannel(); + FileChannel channel = readStream.getChannel(); //the file might be bigger than the buffer, in which case we partially buffer it in memory - this.fileSize = this.channel.size(); - this.bufferStart = 0L; - this.bufferSize = Math.min(fileSize, MAX_BUFFER_SIZE); + this.fileSize = channel.size(); + long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE); - mapBuffer(); - this.metaData = TRSMetaDataUtils.readTRSMetaData(buffer); - this.metaDataSize = buffer.position(); + this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); + this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); + this.metaDataSize = metaDataBuffer.position(); + this.metaDataBuffer.limit(metaDataSize); + + long traceSize = calculateTraceSize(); + this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); } private TraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { @@ -93,23 +95,6 @@ public Path getPath() { return path; } - private void mapBuffer() throws IOException { - this.buffer = this.channel.map(FileChannel.MapMode.READ_ONLY, this.bufferStart, this.bufferSize); - } - - private void moveBufferIfNecessary(int traceIndex) throws IOException { - long traceSize = calculateTraceSize(); - long start = metaDataSize + (long) traceIndex * traceSize; - long end = start + traceSize; - - boolean moveRequired = start < this.bufferStart || this.bufferStart + this.bufferSize < end; - if (moveRequired) { - this.bufferStart = start; - this.bufferSize = Math.min(this.fileSize - start, MAX_BUFFER_SIZE); - this.mapBuffer(); - } - } - private long calculateTraceSize() { int sampleSize = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)).getSize(); long sampleSpace = metaData.getInt(NUMBER_OF_SAMPLES) * (long) sampleSize; @@ -140,12 +125,9 @@ public Trace get(int index) throws IOException { throw new IllegalStateException(msg); } - moveBufferIfNecessary(index); - - long absolutePosition = metaDataSize + index * traceSize; - buffer.position((int) (absolutePosition - this.bufferStart)); + ByteBuffer buffer = mappedFile.getBuffer(index); - String traceTitle = this.readTraceTitle(); + String traceTitle = this.readTraceTitle(buffer); if (traceTitle.trim().isEmpty()) { traceTitle = String.format("%s %d", metaData.getString(GLOBAL_TITLE), index); } @@ -160,14 +142,14 @@ public Trace get(int index) throws IOException { traceParameterMap = TraceParameterMap.deserialize(data, traceParameterDefinitionMap); } else { //legacy mode - byte[] data = readData(); + byte[] data = readData(buffer); traceParameterMap = new TraceParameterMap(); if (data.length > 0) { traceParameterMap.put("LEGACY_DATA", data); } } - float[] samples = readSamples(); + float[] samples = readSamples(buffer); return new Trace(traceTitle, samples, traceParameterMap); } catch (TRSFormatException ex) { throw new IOException(ex); @@ -339,7 +321,8 @@ private void checkValid(Trace trace) { } private void closeReader() throws IOException { - buffer = null; + metaDataBuffer = null; + mappedFile.close(); readStream.close(); } @@ -362,21 +345,20 @@ public TRSMetaData getMetaData() { return metaData; } - protected String readTraceTitle() { + protected String readTraceTitle(ByteBuffer buffer) { byte[] titleArray = new byte[metaData.getInt(TITLE_SPACE)]; buffer.get(titleArray); return new String(titleArray); } - protected byte[] readData() { + protected byte[] readData(ByteBuffer buffer) { int inputSize = metaData.getInt(DATA_LENGTH); byte[] comDataArray = new byte[inputSize]; buffer.get(comDataArray); return comDataArray; } - protected float[] readSamples() throws TRSFormatException { - buffer.order(ByteOrder.LITTLE_ENDIAN); + protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); float[] samples; switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { From 70c43188cc3f9aef00a9fc88a6e83b77a1baf8a1 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Thu, 20 Feb 2025 13:23:26 +0100 Subject: [PATCH 02/14] #78: Added pre-allocating of trace buffers for efficiency --- .../com/riscure/trs/LargePreMappedFile.java | 3 - src/main/java/com/riscure/trs/TraceSet.java | 71 +++++++++---------- 2 files changed, 32 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/riscure/trs/LargePreMappedFile.java b/src/main/java/com/riscure/trs/LargePreMappedFile.java index 4e9fac3..a182e54 100644 --- a/src/main/java/com/riscure/trs/LargePreMappedFile.java +++ b/src/main/java/com/riscure/trs/LargePreMappedFile.java @@ -15,8 +15,6 @@ public class LargePreMappedFile implements AutoCloseable{ private final long traceSize; private final long fileSize; - private long totalBufferSize; - public LargePreMappedFile(FileChannel channel, long metaDataSize, long traceSize) throws IOException { this.channel = channel; this.readOffset = metaDataSize; @@ -62,7 +60,6 @@ private void mapBuffers() { private MappedBuffer mapBuffer(int firstTraceIndex, long bufferSize) { try { long bufferStart = firstTraceIndex * traceSize; - this.totalBufferSize += bufferSize; long limitedBufferSize = Math.min(fileSize - bufferStart, bufferSize); MappedBuffer mappedBuffer = new MappedBuffer(this.channel.map(FileChannel.MapMode.READ_ONLY, readOffset + bufferStart, limitedBufferSize), firstTraceIndex, diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index 8ec00aa..2de90ef 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -44,6 +44,10 @@ public class TraceSet implements AutoCloseable { private ByteBuffer metaDataBuffer; private LargePreMappedFile mappedFile; + private float[] preallocatedSampleArray; + private byte[] preallocatedByteArray; + private short[] preallocatedShortArray; + private int[] preallocatedIntArray; private long fileSize; //the total number of bytes in the underlying file @@ -78,6 +82,9 @@ private TraceSet(String inputFileName) throws IOException, TRSFormatException { long traceSize = calculateTraceSize(); this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); + + int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); + this.preallocatedSampleArray = new float[numberOfSamples]; } private TraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { @@ -150,7 +157,8 @@ public Trace get(int index) throws IOException { } float[] samples = readSamples(buffer); - return new Trace(traceTitle, samples, traceParameterMap); + // Since we are using an internal sample array in this class, Trace.create() should duplicate it internally + return Trace.create(traceTitle, samples, traceParameterMap); } catch (TRSFormatException ex) { throw new IOException(ex); } @@ -358,61 +366,46 @@ protected byte[] readData(ByteBuffer buffer) { return comDataArray; } + /* + * We can reuse the buffers when not dealing with float samples. They are instantiated once just in time if needed. + */ protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { - int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); - float[] samples; switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { case BYTE: - byte[] byteData = new byte[numberOfSamples]; - buffer.get(byteData); - samples = toFloatArray(byteData); + this.preallocatedByteArray = this.preallocatedByteArray == null ? new byte[preallocatedSampleArray.length] : this.preallocatedByteArray; + buffer.get(preallocatedByteArray); + // Manual copy of byte[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedByteArray[k]; + } break; case SHORT: + this.preallocatedShortArray = this.preallocatedShortArray == null ? new short[preallocatedSampleArray.length] : this.preallocatedShortArray; ShortBuffer shortView = buffer.asShortBuffer(); - short[] shortData = new short[numberOfSamples]; - shortView.get(shortData); - samples = toFloatArray(shortData); + shortView.get(preallocatedShortArray); + // Manual copy of short[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedShortArray[k]; + } break; case FLOAT: FloatBuffer floatView = buffer.asFloatBuffer(); - samples = new float[numberOfSamples]; - floatView.get(samples); + floatView.get(preallocatedSampleArray); break; case INT: + this.preallocatedIntArray = this.preallocatedIntArray == null ? new int[preallocatedSampleArray.length] : this.preallocatedIntArray; IntBuffer intView = buffer.asIntBuffer(); - int[] intData = new int[numberOfSamples]; - intView.get(intData); - samples = toFloatArray(intData); + intView.get(preallocatedIntArray); + // Manual copy of int[] into float[] + for (int k = 0; k < preallocatedIntArray.length; k++) { + preallocatedSampleArray[k] = (float) preallocatedIntArray[k]; + } break; default: throw new TRSFormatException(String.format(UNKNOWN_SAMPLE_CODING, metaData.getInt(SAMPLE_CODING))); } - return samples; - } - - private float[] toFloatArray(byte[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = numbers[k]; - } - return result; - } - - private float[] toFloatArray(int[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = (float) numbers[k]; - } - return result; - } - - private float[] toFloatArray(short[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = numbers[k]; - } - return result; + return preallocatedSampleArray; } /** From 9e60abb4a71f175cb1c21c8c8e872a8d4b3b837e Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 19 Sep 2025 10:35:44 +0200 Subject: [PATCH 03/14] #79: Split off Read and Write mode from TraceSet --- .../com/riscure/trs/ReadOnlyTraceSet.java | 199 +++++++++ src/main/java/com/riscure/trs/TraceSet.java | 378 +----------------- .../com/riscure/trs/WritableTraceSet.java | 219 ++++++++++ 3 files changed, 433 insertions(+), 363 deletions(-) create mode 100644 src/main/java/com/riscure/trs/ReadOnlyTraceSet.java create mode 100644 src/main/java/com/riscure/trs/WritableTraceSet.java diff --git a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java new file mode 100644 index 0000000..b22b0c0 --- /dev/null +++ b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java @@ -0,0 +1,199 @@ +package com.riscure.trs; + +import com.riscure.trs.enums.Encoding; +import com.riscure.trs.parameter.trace.TraceParameterMap; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; + +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Paths; + +import static com.riscure.trs.enums.TRSTag.*; +import static com.riscure.trs.enums.TRSTag.TRS_VERSION; + +public class ReadOnlyTraceSet extends TraceSet { + private static final String TRACE_SET_IN_READ_MODE = "TraceSet is in read mode. Please open the TraceSet in write mode."; + private static final String ERROR_READING_FILE = "Error reading TRS file: file size (%d) != meta data (%d) + trace size (%d) * nr of traces (%d)"; + private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d)."; + private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; + // This is excessive for the header, but it's only the initial maximum + private static final long MAX_METADATA_SIZE = 100_000_000L; + + private final int metaDataSize; + private final FileInputStream readStream; + private final LargePreMappedFile mappedFile; + private final float[] preallocatedSampleArray; + private final TRSMetaData metaData; + private final long fileSize; //the total number of bytes in the underlying file + + private ByteBuffer metaDataBuffer; + private byte[] preallocatedByteArray; + private short[] preallocatedShortArray; + private int[] preallocatedIntArray; + + ReadOnlyTraceSet(String inputFileName) throws IOException, TRSFormatException { + super(Paths.get(inputFileName)); + + this.readStream = new FileInputStream(inputFileName); + FileChannel channel = readStream.getChannel(); + + //the file might be bigger than the buffer, in which case we partially buffer it in memory + this.fileSize = channel.size(); + long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE); + + this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); + this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); + this.metaDataSize = metaDataBuffer.position(); + this.metaDataBuffer.limit(metaDataSize); + + long traceSize = calculateTraceSize(); + this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); + + int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); + this.preallocatedSampleArray = new float[numberOfSamples]; + } + + /** + * Get a trace from the set at the specified index + * @param index the index of the Trace to read from the file + * @return the Trace at the requested trace index + * @throws IOException if a read error occurs + * @throws IllegalArgumentException if this TraceSet is not ready be read from + */ + @Override + public Trace get(int index) throws IOException { + if (!isOpen()) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); + + long traceSize = calculateTraceSize(); + long nrOfTraces = this.metaData.getInt(NUMBER_OF_TRACES); + if (index >= nrOfTraces) { + String msg = String.format(TRACE_INDEX_OUT_OF_BOUNDS, index, nrOfTraces); + throw new IllegalArgumentException(msg); + } + + long calculatedFileSize = metaDataSize + traceSize * nrOfTraces; + if (fileSize != calculatedFileSize) { + String msg = String.format(ERROR_READING_FILE, fileSize, metaDataSize, traceSize, nrOfTraces); + throw new IllegalStateException(msg); + } + + ByteBuffer buffer = mappedFile.getBuffer(index); + + String traceTitle = this.readTraceTitle(buffer); + if (traceTitle.trim().isEmpty()) { + traceTitle = String.format("%s %d", metaData.getString(GLOBAL_TITLE), index); + } + + try { + TraceParameterMap traceParameterMap; + if (metaData.getInt(TRS_VERSION) > 1) { + TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); + int size = traceParameterDefinitionMap.totalSize(); + byte[] data = new byte[size]; + buffer.get(data); + traceParameterMap = TraceParameterMap.deserialize(data, traceParameterDefinitionMap); + } else { + //legacy mode + byte[] data = readData(buffer); + traceParameterMap = new TraceParameterMap(); + if (data.length > 0) { + traceParameterMap.put("LEGACY_DATA", data); + } + } + + float[] samples = readSamples(buffer); + // Since we are using an internal sample array in this class, Trace.create() should duplicate it internally + return Trace.create(traceTitle, samples, traceParameterMap); + } catch (TRSFormatException ex) { + throw new IOException(ex); + } + } + + @Override + public void add(Trace trace) throws IOException, TRSFormatException { + throw new IllegalArgumentException(TRACE_SET_IN_READ_MODE); + } + + private long calculateTraceSize() { + int sampleSize = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)).getSize(); + long sampleSpace = metaData.getInt(NUMBER_OF_SAMPLES) * (long) sampleSize; + return sampleSpace + metaData.getInt(DATA_LENGTH) + metaData.getInt(TITLE_SPACE); + } + + @Override + public void close() throws IOException, TRSFormatException { + super.close(); + closeReader(); + } + + @Override + public TRSMetaData getMetaData() { + return metaData; + } + + private void closeReader() throws IOException { + metaDataBuffer = null; + mappedFile.close(); + readStream.close(); + } + + protected String readTraceTitle(ByteBuffer buffer) { + byte[] titleArray = new byte[metaData.getInt(TITLE_SPACE)]; + buffer.get(titleArray); + return new String(titleArray); + } + + protected byte[] readData(ByteBuffer buffer) { + int inputSize = metaData.getInt(DATA_LENGTH); + byte[] comDataArray = new byte[inputSize]; + buffer.get(comDataArray); + return comDataArray; + } + + /* + * We can reuse the buffers when not dealing with float samples. They are instantiated once just in time if needed. + */ + protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { + switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { + case BYTE: + this.preallocatedByteArray = this.preallocatedByteArray == null ? new byte[preallocatedSampleArray.length] : this.preallocatedByteArray; + buffer.get(preallocatedByteArray); + // Manual copy of byte[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedByteArray[k]; + } + break; + case SHORT: + this.preallocatedShortArray = this.preallocatedShortArray == null ? new short[preallocatedSampleArray.length] : this.preallocatedShortArray; + ShortBuffer shortView = buffer.asShortBuffer(); + shortView.get(preallocatedShortArray); + // Manual copy of short[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedShortArray[k]; + } + break; + case FLOAT: + FloatBuffer floatView = buffer.asFloatBuffer(); + floatView.get(preallocatedSampleArray); + break; + case INT: + this.preallocatedIntArray = this.preallocatedIntArray == null ? new int[preallocatedSampleArray.length] : this.preallocatedIntArray; + IntBuffer intView = buffer.asIntBuffer(); + intView.get(preallocatedIntArray); + // Manual copy of int[] into float[] + for (int k = 0; k < preallocatedIntArray.length; k++) { + preallocatedSampleArray[k] = (float) preallocatedIntArray[k]; + } + break; + default: + throw new TRSFormatException(String.format(UNKNOWN_SAMPLE_CODING, metaData.getInt(SAMPLE_CODING))); + } + + return preallocatedSampleArray; + } +} diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index 2de90ef..751187c 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -1,98 +1,21 @@ package com.riscure.trs; -import com.riscure.trs.enums.Encoding; -import com.riscure.trs.enums.ParameterType; -import com.riscure.trs.parameter.TraceParameter; -import com.riscure.trs.parameter.primitive.StringParameter; -import com.riscure.trs.parameter.trace.TraceParameterMap; -import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; -import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; - -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.nio.*; -import java.nio.channels.FileChannel; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; import java.util.List; -import java.util.Map; - -import static com.riscure.trs.enums.TRSTag.*; - -public class TraceSet implements AutoCloseable { - private static final String ERROR_READING_FILE = "Error reading TRS file: file size (%d) != meta data (%d) + trace size (%d) * nr of traces (%d)"; - private static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; - private static final String TRACE_SET_IN_WRITE_MODE = "TraceSet is in write mode. Please open the TraceSet in read mode."; - private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d)."; - private static final String TRACE_SET_IN_READ_MODE = "TraceSet is in read mode. Please open the TraceSet in write mode."; - private static final String TRACE_LENGTH_DIFFERS = "All traces in a set need to be the same length, but current trace length (%d) differs from the previous trace(s) (%d)"; - private static final String TRACE_DATA_LENGTH_DIFFERS = "All traces in a set need to have the same data length, but current trace data length (%d) differs from the previous trace(s) (%d)"; - private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; - private static final String PARAMETER_NOT_DEFINED = "Parameter %s is saved in the trace, but was not found in the header definition"; - // This is excessive for the header, but it's only the initial maximum - private static final long MAX_METADATA_SIZE = 100_000_000L; - - //Reading variables - private int metaDataSize; - private FileInputStream readStream; - - private ByteBuffer metaDataBuffer; - private LargePreMappedFile mappedFile; - private float[] preallocatedSampleArray; - private byte[] preallocatedByteArray; - private short[] preallocatedShortArray; - private int[] preallocatedIntArray; - private long fileSize; //the total number of bytes in the underlying file +import static com.riscure.trs.enums.TRSTag.TRS_VERSION; - //Writing variables - private FileOutputStream writeStream; - - private boolean firstTrace = true; +public abstract class TraceSet implements AutoCloseable { + protected static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; //Shared variables - private final TRSMetaData metaData; - private final boolean writing; //whether the trace is opened in write mode private final Path path; - private final CharsetDecoder utf8Decoder = StandardCharsets.UTF_8.newDecoder(); - private boolean open; - private TraceSet(String inputFileName) throws IOException, TRSFormatException { - this.writing = false; - this.open = true; - this.path = Paths.get(inputFileName); - this.readStream = new FileInputStream(inputFileName); - FileChannel channel = readStream.getChannel(); - - //the file might be bigger than the buffer, in which case we partially buffer it in memory - this.fileSize = channel.size(); - long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE); - - this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); - this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); - this.metaDataSize = metaDataBuffer.position(); - this.metaDataBuffer.limit(metaDataSize); - - long traceSize = calculateTraceSize(); - this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); - - int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); - this.preallocatedSampleArray = new float[numberOfSamples]; - } - - private TraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { + protected TraceSet(Path path) { + this.path = path; this.open = true; - this.writing = true; - this.metaData = metaData; - this.path = Paths.get(outputFileName); - this.writeStream = new FileOutputStream(outputFileName); } /** @@ -102,10 +25,11 @@ public Path getPath() { return path; } - private long calculateTraceSize() { - int sampleSize = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)).getSize(); - long sampleSpace = metaData.getInt(NUMBER_OF_SAMPLES) * (long) sampleSize; - return sampleSpace + metaData.getInt(DATA_LENGTH) + metaData.getInt(TITLE_SPACE); + /** + * @return whether this trace set is currently open + */ + public boolean isOpen() { + return open; } /** @@ -115,54 +39,7 @@ private long calculateTraceSize() { * @throws IOException if a read error occurs * @throws IllegalArgumentException if this TraceSet is not ready be read from */ - public Trace get(int index) throws IOException { - if (!open) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); - if (writing) throw new IllegalArgumentException(TRACE_SET_IN_WRITE_MODE); - - long traceSize = calculateTraceSize(); - long nrOfTraces = this.metaData.getInt(NUMBER_OF_TRACES); - if (index >= nrOfTraces) { - String msg = String.format(TRACE_INDEX_OUT_OF_BOUNDS, index, nrOfTraces); - throw new IllegalArgumentException(msg); - } - - long calculatedFileSize = metaDataSize + traceSize * nrOfTraces; - if (fileSize != calculatedFileSize) { - String msg = String.format(ERROR_READING_FILE, fileSize, metaDataSize, traceSize, nrOfTraces); - throw new IllegalStateException(msg); - } - - ByteBuffer buffer = mappedFile.getBuffer(index); - - String traceTitle = this.readTraceTitle(buffer); - if (traceTitle.trim().isEmpty()) { - traceTitle = String.format("%s %d", metaData.getString(GLOBAL_TITLE), index); - } - - try { - TraceParameterMap traceParameterMap; - if (metaData.getInt(TRS_VERSION) > 1) { - TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); - int size = traceParameterDefinitionMap.totalSize(); - byte[] data = new byte[size]; - buffer.get(data); - traceParameterMap = TraceParameterMap.deserialize(data, traceParameterDefinitionMap); - } else { - //legacy mode - byte[] data = readData(buffer); - traceParameterMap = new TraceParameterMap(); - if (data.length > 0) { - traceParameterMap.put("LEGACY_DATA", data); - } - } - - float[] samples = readSamples(buffer); - // Since we are using an internal sample array in this class, Trace.create() should duplicate it internally - return Trace.create(traceTitle, samples, traceParameterMap); - } catch (TRSFormatException ex) { - throw new IOException(ex); - } - } + public abstract Trace get(int index) throws IOException; /** * Add a trace to a writable TraceSet @@ -170,243 +47,18 @@ public Trace get(int index) throws IOException { * @throws IOException if any write error occurs * @throws TRSFormatException if the formatting of the trace is invalid */ - public void add(Trace trace) throws IOException, TRSFormatException { - if (!open) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); - if (!writing) throw new IllegalArgumentException(TRACE_SET_IN_READ_MODE); - if (firstTrace) { - int dataLength = trace.getData() == null ? 0 : trace.getData().length; - int titleLength = trace.getTitle() == null ? 0 : trace.getTitle().getBytes(StandardCharsets.UTF_8).length; - metaData.put(NUMBER_OF_SAMPLES, trace.getNumberOfSamples(), false); - metaData.put(DATA_LENGTH, dataLength, false); - metaData.put(TITLE_SPACE, titleLength, false); - metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false); - metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters())); - TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); - firstTrace = false; - } - truncateStrings(trace, metaData); - checkValid(trace); - - trace.setTraceSet(this); - writeTrace(trace); - - int numberOfTraces = metaData.getInt(NUMBER_OF_TRACES); - metaData.put(NUMBER_OF_TRACES, numberOfTraces + 1); - } - - /** - * This method makes sure that the trace title and any added string parameters adhere to the preset maximum length - * @param trace the trace to update - * @param metaData the metadata specifying the maximum string lengths - */ - private void truncateStrings(Trace trace, TRSMetaData metaData) { - int titleSpace = metaData.getInt(TITLE_SPACE); - trace.setTitle(fitUtf8StringToByteLength(trace.getTitle(), titleSpace)); - TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); - for (Map.Entry> definition : traceParameterDefinitionMap.entrySet()) { - TraceParameterDefinition value = definition.getValue(); - String key = definition.getKey(); - if (value.getType() == ParameterType.STRING) { - short stringLength = value.getLength(); - String stringValue = ((StringParameter) trace.getParameters().get(key)).getValue(); - if (stringLength != stringValue.getBytes(StandardCharsets.UTF_8).length) { - trace.getParameters().put(key, fitUtf8StringToByteLength(stringValue, stringLength)); - } - } - } - } - - /** - * Fits a string to the number of characters that fit in X bytes avoiding multi byte characters being cut in - * half at the cut off point. Also handles surrogate pairs where 2 characters in the string is actually one literal - * character. If the string is too long, it is truncated. If it's too short, it's padded with NUL characters. - * @param s the string to fit - * @param maxBytes the number of bytes required - */ - private String fitUtf8StringToByteLength(String s, int maxBytes) { - if (s == null) { - return null; - } - byte[] sba = s.getBytes(StandardCharsets.UTF_8); - if (sba.length <= maxBytes) { - return new String(Arrays.copyOf(sba, maxBytes)); - } - // Ensure truncation by having byte buffer = maxBytes - ByteBuffer bb = ByteBuffer.wrap(sba, 0, maxBytes); - CharBuffer cb = CharBuffer.allocate(maxBytes); - // Ignore an incomplete character - utf8Decoder.reset(); - utf8Decoder.onMalformedInput(CodingErrorAction.IGNORE); - utf8Decoder.decode(bb, cb, true); - utf8Decoder.flush(cb); - return new String(cb.array(), 0, cb.position()); - } - - private void writeTrace(Trace trace) throws TRSFormatException, IOException { - String title = trace.getTitle() == null ? "" : trace.getTitle(); - writeStream.write(title.getBytes(StandardCharsets.UTF_8)); - byte[] data = trace.getData() == null ? new byte[0] : trace.getData(); - writeStream.write(data); - Encoding encoding = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)); - writeStream.write(toByteArray(trace.getSample(), encoding)); - } - - private byte[] toByteArray(float[] samples, Encoding encoding) throws TRSFormatException { - byte[] result; - switch (encoding) { - case ILLEGAL: - throw new TRSFormatException("Illegal sample encoding"); - case BYTE: - result = new byte[samples.length]; - for (int k = 0; k < samples.length; k++) { - if (samples[k] != (byte)samples[k]) throw new IllegalArgumentException("Byte sample encoding too small"); - result[k] = (byte) samples[k]; - } - break; - case SHORT: - result = new byte[samples.length * 2]; - for (int k = 0; k < samples.length; k++) { - if (samples[k] != (short)samples[k]) throw new IllegalArgumentException("Short sample encoding too small"); - short value = (short) samples[k]; - result[2*k] = (byte) value; - result[2*k + 1] = (byte) (value >> 8); - } - break; - case INT: - result = new byte[samples.length * 4]; - for (int k = 0; k < samples.length; k++) { - int value = (int) samples[k]; - result[4*k] = (byte) value; - result[4*k + 1] = (byte) (value >> 8); - result[4*k + 2] = (byte) (value >> 16); - result[4*k + 3] = (byte) (value >> 24); - } - break; - case FLOAT: - result = new byte[samples.length * 4]; - for (int k = 0; k < samples.length; k++) { - int value = Float.floatToIntBits(samples[k]); - result[4*k] = (byte) value; - result[4*k + 1] = (byte) (value >> 8); - result[4*k + 2] = (byte) (value >> 16); - result[4*k + 3] = (byte) (value >> 24); - } - break; - default: - throw new TRSFormatException(String.format("Sample encoding not supported: %s", encoding.name())); - } - return result; - } + public abstract void add(Trace trace) throws IOException, TRSFormatException; @Override public void close() throws IOException, TRSFormatException { open = false; - if (writing) closeWriter(); - else closeReader(); - } - - private void checkValid(Trace trace) { - int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); - if (metaData.getInt(NUMBER_OF_SAMPLES) != trace.getNumberOfSamples()) { - throw new IllegalArgumentException(String.format(TRACE_LENGTH_DIFFERS, - trace.getNumberOfSamples(), - numberOfSamples)); - } - - int dataLength = metaData.getInt(DATA_LENGTH); - int traceDataLength = trace.getData() == null ? 0 : trace.getData().length; - if (metaData.getInt(DATA_LENGTH) != traceDataLength) { - throw new IllegalArgumentException(String.format(TRACE_DATA_LENGTH_DIFFERS, - traceDataLength, - dataLength)); - } - - for (Map.Entry entry : trace.getParameters().entrySet()) { - if (!metaData.getTraceParameterDefinitions().containsKey(entry.getKey())) { - throw new IllegalArgumentException(String.format(PARAMETER_NOT_DEFINED, entry.getKey())); - } - } - } - - private void closeReader() throws IOException { - metaDataBuffer = null; - mappedFile.close(); - readStream.close(); - } - - private void closeWriter() throws IOException, TRSFormatException { - try { - //reset writer to start of file and overwrite header - writeStream.getChannel().position(0); - TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); - writeStream.flush(); - } finally { - writeStream.close(); - } } /** * Get the metadata associated with this trace set * @return the metadata associated with this trace set */ - public TRSMetaData getMetaData() { - return metaData; - } - - protected String readTraceTitle(ByteBuffer buffer) { - byte[] titleArray = new byte[metaData.getInt(TITLE_SPACE)]; - buffer.get(titleArray); - return new String(titleArray); - } - - protected byte[] readData(ByteBuffer buffer) { - int inputSize = metaData.getInt(DATA_LENGTH); - byte[] comDataArray = new byte[inputSize]; - buffer.get(comDataArray); - return comDataArray; - } - - /* - * We can reuse the buffers when not dealing with float samples. They are instantiated once just in time if needed. - */ - protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { - switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { - case BYTE: - this.preallocatedByteArray = this.preallocatedByteArray == null ? new byte[preallocatedSampleArray.length] : this.preallocatedByteArray; - buffer.get(preallocatedByteArray); - // Manual copy of byte[] into float[] - for (int k = 0; k < preallocatedSampleArray.length; k++) { - preallocatedSampleArray[k] = preallocatedByteArray[k]; - } - break; - case SHORT: - this.preallocatedShortArray = this.preallocatedShortArray == null ? new short[preallocatedSampleArray.length] : this.preallocatedShortArray; - ShortBuffer shortView = buffer.asShortBuffer(); - shortView.get(preallocatedShortArray); - // Manual copy of short[] into float[] - for (int k = 0; k < preallocatedSampleArray.length; k++) { - preallocatedSampleArray[k] = preallocatedShortArray[k]; - } - break; - case FLOAT: - FloatBuffer floatView = buffer.asFloatBuffer(); - floatView.get(preallocatedSampleArray); - break; - case INT: - this.preallocatedIntArray = this.preallocatedIntArray == null ? new int[preallocatedSampleArray.length] : this.preallocatedIntArray; - IntBuffer intView = buffer.asIntBuffer(); - intView.get(preallocatedIntArray); - // Manual copy of int[] into float[] - for (int k = 0; k < preallocatedIntArray.length; k++) { - preallocatedSampleArray[k] = (float) preallocatedIntArray[k]; - } - break; - default: - throw new TRSFormatException(String.format(UNKNOWN_SAMPLE_CODING, metaData.getInt(SAMPLE_CODING))); - } - - return preallocatedSampleArray; - } + public abstract TRSMetaData getMetaData(); /** * Factory method. This creates a new open TraceSet for reading. @@ -418,7 +70,7 @@ protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { * @throws TRSFormatException when any incorrect formatting of the TRS file is encountered */ public static TraceSet open(String file) throws IOException, TRSFormatException { - return new TraceSet(file); + return new ReadOnlyTraceSet(file); } /** @@ -484,6 +136,6 @@ public static TraceSet create(String file) throws IOException { */ public static TraceSet create(String file, TRSMetaData metaData) throws IOException { metaData.put(TRS_VERSION, 2, false); - return new TraceSet(file, metaData); + return new WritableTraceSet(file, metaData); } } diff --git a/src/main/java/com/riscure/trs/WritableTraceSet.java b/src/main/java/com/riscure/trs/WritableTraceSet.java new file mode 100644 index 0000000..aef3306 --- /dev/null +++ b/src/main/java/com/riscure/trs/WritableTraceSet.java @@ -0,0 +1,219 @@ +package com.riscure.trs; + +import com.riscure.trs.enums.Encoding; +import com.riscure.trs.enums.ParameterType; +import com.riscure.trs.parameter.TraceParameter; +import com.riscure.trs.parameter.primitive.StringParameter; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Map; + +import static com.riscure.trs.enums.TRSTag.*; + +public class WritableTraceSet extends TraceSet { + private static final String TRACE_SET_IN_WRITE_MODE = "TraceSet is in write mode. Please open the TraceSet in read mode."; + private static final String TRACE_LENGTH_DIFFERS = "All traces in a set need to be the same length, but current trace length (%d) differs from the previous trace(s) (%d)"; + private static final String TRACE_DATA_LENGTH_DIFFERS = "All traces in a set need to have the same data length, but current trace data length (%d) differs from the previous trace(s) (%d)"; + private static final String PARAMETER_NOT_DEFINED = "Parameter %s is saved in the trace, but was not found in the header definition"; + + private final CharsetDecoder utf8Decoder = StandardCharsets.UTF_8.newDecoder(); + private final TRSMetaData metaData; + private final FileOutputStream writeStream; + + private boolean firstTrace = true; + + WritableTraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { + super(Paths.get(outputFileName)); + this.metaData = metaData; + this.writeStream = new FileOutputStream(outputFileName); + } + + @Override + public Trace get(int index) throws IOException { + throw new IllegalArgumentException(TRACE_SET_IN_WRITE_MODE); + } + + @Override + public void add(Trace trace) throws IOException, TRSFormatException { + if (!isOpen()) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); + if (firstTrace) { + int dataLength = trace.getData() == null ? 0 : trace.getData().length; + int titleLength = trace.getTitle() == null ? 0 : trace.getTitle().getBytes(StandardCharsets.UTF_8).length; + metaData.put(NUMBER_OF_SAMPLES, trace.getNumberOfSamples(), false); + metaData.put(DATA_LENGTH, dataLength, false); + metaData.put(TITLE_SPACE, titleLength, false); + metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false); + metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters())); + TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); + firstTrace = false; + } + truncateStrings(trace, metaData); + checkValid(trace); + + trace.setTraceSet(this); + writeTrace(trace); + + int numberOfTraces = metaData.getInt(NUMBER_OF_TRACES); + metaData.put(NUMBER_OF_TRACES, numberOfTraces + 1); + } + + @Override + public TRSMetaData getMetaData() { + return metaData; + } + + @Override + public void close() throws IOException, TRSFormatException { + super.close(); + closeWriter(); + } + + private void closeWriter() throws IOException, TRSFormatException { + try { + //reset writer to start of file and overwrite header + writeStream.getChannel().position(0); + TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); + writeStream.flush(); + } finally { + writeStream.close(); + } + } + + /** + * This method makes sure that the trace title and any added string parameters adhere to the preset maximum length + * @param trace the trace to update + * @param metaData the metadata specifying the maximum string lengths + */ + private void truncateStrings(Trace trace, TRSMetaData metaData) { + int titleSpace = metaData.getInt(TITLE_SPACE); + trace.setTitle(fitUtf8StringToByteLength(trace.getTitle(), titleSpace)); + TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); + for (Map.Entry> definition : traceParameterDefinitionMap.entrySet()) { + TraceParameterDefinition value = definition.getValue(); + String key = definition.getKey(); + if (value.getType() == ParameterType.STRING) { + short stringLength = value.getLength(); + String stringValue = ((StringParameter) trace.getParameters().get(key)).getValue(); + if (stringLength != stringValue.getBytes(StandardCharsets.UTF_8).length) { + trace.getParameters().put(key, fitUtf8StringToByteLength(stringValue, stringLength)); + } + } + } + } + + /** + * Fits a string to the number of characters that fit in X bytes avoiding multi byte characters being cut in + * half at the cut off point. Also handles surrogate pairs where 2 characters in the string is actually one literal + * character. If the string is too long, it is truncated. If it's too short, it's padded with NUL characters. + * @param s the string to fit + * @param maxBytes the number of bytes required + */ + private String fitUtf8StringToByteLength(String s, int maxBytes) { + if (s == null) { + return null; + } + byte[] sba = s.getBytes(StandardCharsets.UTF_8); + if (sba.length <= maxBytes) { + return new String(Arrays.copyOf(sba, maxBytes)); + } + // Ensure truncation by having byte buffer = maxBytes + ByteBuffer bb = ByteBuffer.wrap(sba, 0, maxBytes); + CharBuffer cb = CharBuffer.allocate(maxBytes); + // Ignore an incomplete character + utf8Decoder.reset(); + utf8Decoder.onMalformedInput(CodingErrorAction.IGNORE); + utf8Decoder.decode(bb, cb, true); + utf8Decoder.flush(cb); + return new String(cb.array(), 0, cb.position()); + } + + private void writeTrace(Trace trace) throws TRSFormatException, IOException { + String title = trace.getTitle() == null ? "" : trace.getTitle(); + writeStream.write(title.getBytes(StandardCharsets.UTF_8)); + byte[] data = trace.getData() == null ? new byte[0] : trace.getData(); + writeStream.write(data); + Encoding encoding = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)); + writeStream.write(toByteArray(trace.getSample(), encoding)); + } + + private byte[] toByteArray(float[] samples, Encoding encoding) throws TRSFormatException { + byte[] result; + switch (encoding) { + case ILLEGAL: + throw new TRSFormatException("Illegal sample encoding"); + case BYTE: + result = new byte[samples.length]; + for (int k = 0; k < samples.length; k++) { + if (samples[k] != (byte)samples[k]) throw new IllegalArgumentException("Byte sample encoding too small"); + result[k] = (byte) samples[k]; + } + break; + case SHORT: + result = new byte[samples.length * 2]; + for (int k = 0; k < samples.length; k++) { + if (samples[k] != (short)samples[k]) throw new IllegalArgumentException("Short sample encoding too small"); + short value = (short) samples[k]; + result[2*k] = (byte) value; + result[2*k + 1] = (byte) (value >> 8); + } + break; + case INT: + result = new byte[samples.length * 4]; + for (int k = 0; k < samples.length; k++) { + int value = (int) samples[k]; + result[4*k] = (byte) value; + result[4*k + 1] = (byte) (value >> 8); + result[4*k + 2] = (byte) (value >> 16); + result[4*k + 3] = (byte) (value >> 24); + } + break; + case FLOAT: + result = new byte[samples.length * 4]; + for (int k = 0; k < samples.length; k++) { + int value = Float.floatToIntBits(samples[k]); + result[4*k] = (byte) value; + result[4*k + 1] = (byte) (value >> 8); + result[4*k + 2] = (byte) (value >> 16); + result[4*k + 3] = (byte) (value >> 24); + } + break; + default: + throw new TRSFormatException(String.format("Sample encoding not supported: %s", encoding.name())); + } + return result; + } + + private void checkValid(Trace trace) { + int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); + if (metaData.getInt(NUMBER_OF_SAMPLES) != trace.getNumberOfSamples()) { + throw new IllegalArgumentException(String.format(TRACE_LENGTH_DIFFERS, + trace.getNumberOfSamples(), + numberOfSamples)); + } + + int dataLength = metaData.getInt(DATA_LENGTH); + int traceDataLength = trace.getData() == null ? 0 : trace.getData().length; + if (metaData.getInt(DATA_LENGTH) != traceDataLength) { + throw new IllegalArgumentException(String.format(TRACE_DATA_LENGTH_DIFFERS, + traceDataLength, + dataLength)); + } + + for (Map.Entry entry : trace.getParameters().entrySet()) { + if (!metaData.getTraceParameterDefinitions().containsKey(entry.getKey())) { + throw new IllegalArgumentException(String.format(PARAMETER_NOT_DEFINED, entry.getKey())); + } + } + } +} From cb3936e508c949f4a21757c3fa4a244774b0cdeb Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 19 Sep 2025 11:09:02 +0200 Subject: [PATCH 04/14] #77: Added space (up to 1M) to allow editing the metadata without fully re-saving the trace set --- .../com/riscure/trs/ReadOnlyTraceSet.java | 4 +-- .../com/riscure/trs/TRSMetaDataUtils.java | 14 ++++++++-- src/main/java/com/riscure/trs/TraceSet.java | 2 ++ .../java/com/riscure/trs/enums/TRSTag.java | 3 ++- .../TraceParameterDefinitionMap.java | 15 ++++++++--- src/test/java/TestTraceSet.java | 27 ++++++++++++++----- 6 files changed, 51 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java index b22b0c0..5fb8908 100644 --- a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java +++ b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java @@ -22,7 +22,7 @@ public class ReadOnlyTraceSet extends TraceSet { private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d)."; private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; // This is excessive for the header, but it's only the initial maximum - private static final long MAX_METADATA_SIZE = 100_000_000L; + private static final long INITIAL_MEMORY_SIZE = 100_000_000L; private final int metaDataSize; private final FileInputStream readStream; @@ -44,7 +44,7 @@ public class ReadOnlyTraceSet extends TraceSet { //the file might be bigger than the buffer, in which case we partially buffer it in memory this.fileSize = channel.size(); - long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE); + long initialBufferSize = Math.min(fileSize, INITIAL_MEMORY_SIZE); this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); diff --git a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java index dcc0ebf..75e194c 100644 --- a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java +++ b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java @@ -10,6 +10,8 @@ import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import static com.riscure.trs.TraceSet.DEFAULT_METADATA_SIZE; + public class TRSMetaDataUtils { private static final String IGNORED_UNKNOWN_TAG = "ignored unknown metadata tag '%02X' while reading a TRS file\n"; private static final String TAG_LENGTH_INVALID = "The length field following tag '%s' has value '%X', which is not between 0 and 0xffff"; @@ -30,7 +32,7 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData) fos.getChannel().position(0); } for (TRSTag tag : TRSTag.values()) { - if (tag.equals(TRSTag.TRACE_BLOCK)) continue; //TRACE BLOCK should be the last write + if (tag.equals(TRSTag.TRACE_BLOCK) || tag.equals(TRSTag.PADDING)) continue; //PADDING and TRACE BLOCK should be the last writes if (!tag.isRequired() && metaData.hasDefaultValue(tag)) continue; //ignore if default and not required fos.write(tag.getValue()); if (tag.getType() == String.class) { @@ -61,6 +63,14 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData) throw new TRSFormatException(String.format(UNSUPPORTED_TAG_TYPE, tag.getName(), tag.getType())); } } + // Grow the metadata up to 1M, creating an empty buffer in the trace set + // This allows us to grow the header without having to rewrite the whole file + while (fos.getChannel().position() < DEFAULT_METADATA_SIZE) { + byte[] bytes = new byte[(int) (DEFAULT_METADATA_SIZE - fos.getChannel().position())]; + fos.write(TRSTag.PADDING.getValue()); + writeLength(fos, bytes.length); + fos.write(bytes); + } fos.write(TRSTag.TRACE_BLOCK.getValue()); fos.write(TRSTag.TRACE_BLOCK.getLength()); } @@ -124,7 +134,7 @@ public static String readName(LittleEndianInputStream dis) throws IOException { private static void readAndStoreData(ByteBuffer buffer, byte tag, int length, TRSMetaData trsMD) throws TRSFormatException { - boolean hasValidLength = (0 <= length & length <= 0xffff); + boolean hasValidLength = (0 <= length & length <= 0xffffff); TRSTag trsTag; try { trsTag = TRSTag.fromValue(tag); diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index 751187c..1b172b0 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -8,6 +8,8 @@ public abstract class TraceSet implements AutoCloseable { protected static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; + // We want to pre-allocate 1M for the header, so we can grow it if needed without re-writing the whole file + public static final long DEFAULT_METADATA_SIZE = 1_000_000L; //Shared variables private final Path path; diff --git a/src/main/java/com/riscure/trs/enums/TRSTag.java b/src/main/java/com/riscure/trs/enums/TRSTag.java index 32038dd..0547d70 100644 --- a/src/main/java/com/riscure/trs/enums/TRSTag.java +++ b/src/main/java/com/riscure/trs/enums/TRSTag.java @@ -54,7 +54,8 @@ public enum TRSTag { XY_SCAN_HEIGHT (0x74, "HE", false, Integer.class, 4, 0, "Number of steps in the \"y\" direction during XY scan"), XY_MEASUREMENTS_PER_SPOT (0x75, "ME", false, Integer.class, 4, 0, "Number of consecutive measurements done per spot during XY scan"), TRACE_SET_PARAMETERS (0x76, "GP", false, TraceSetParameterMap.class, 0, UnmodifiableTraceSetParameterMap.of(new TraceSetParameterMap()), "The set of custom global trace set parameters"), - TRACE_PARAMETER_DEFINITIONS (0x77, "LP", false, TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()), "The set of custom local trace parameters"); + TRACE_PARAMETER_DEFINITIONS (0x77, "LP", false, TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()), "The set of custom local trace parameters"), + PADDING (0xFF, "FF", false, String.class, 0, 0, "Empty value to allow growing the metadata"); private static final String UNKNOWN_TAG = "Unknown tag: 0x%X"; diff --git a/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java b/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java index dedd43d..ddd1af4 100644 --- a/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java +++ b/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java @@ -1,5 +1,6 @@ package com.riscure.trs.parameter.trace.definition; +import com.riscure.trs.TRSFormatException; import com.riscure.trs.TRSMetaDataUtils; import com.riscure.trs.io.LittleEndianInputStream; import com.riscure.trs.io.LittleEndianOutputStream; @@ -9,7 +10,10 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.Map; @@ -18,6 +22,7 @@ * This explicitly implements LinkedHashMap to ensure that the data is retrieved in the same order as it was added */ public class TraceParameterDefinitionMap extends LinkedHashMap> { + private static final String NAME_TOO_LONG = "Name of length %d exceeds maximum length of %d bytes%nName will be truncated to the maximum length%n"; public TraceParameterDefinitionMap() { super(); @@ -43,7 +48,7 @@ public int totalSize() { * @return this map converted to a byte array, serialized according to the TRS V2 standard definition * @throws RuntimeException if the map failed to serialize correctly */ - public byte[] serialize() { + public byte[] serialize() throws IOException, TRSFormatException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (LittleEndianOutputStream dos = new LittleEndianOutputStream(baos)) { //Write NE @@ -51,6 +56,12 @@ public byte[] serialize() { for (Map.Entry> entry : entrySet()) { byte[] nameBytes = entry.getKey().getBytes(StandardCharsets.UTF_8); //Write NL + if (nameBytes.length > Short.MAX_VALUE) { + System.err.printf(NAME_TOO_LONG, nameBytes.length, Short.MAX_VALUE); + nameBytes = new byte[Short.MAX_VALUE]; + CharBuffer name = CharBuffer.wrap(entry.getKey()); + StandardCharsets.UTF_8.newEncoder().encode(name, ByteBuffer.wrap(nameBytes), true); + } dos.writeShort(nameBytes.length); //Write N dos.write(nameBytes); @@ -59,8 +70,6 @@ public byte[] serialize() { } dos.flush(); return baos.toByteArray(); - } catch (IOException ex) { - throw new RuntimeException(ex); } } diff --git a/src/test/java/TestTraceSet.java b/src/test/java/TestTraceSet.java index ba6e132..f5b2325 100644 --- a/src/test/java/TestTraceSet.java +++ b/src/test/java/TestTraceSet.java @@ -17,6 +17,7 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.opentest4j.AssertionFailedError; import java.io.ByteArrayInputStream; @@ -26,10 +27,11 @@ import java.nio.file.Path; import java.util.*; +import static com.riscure.trs.TraceSet.DEFAULT_METADATA_SIZE; import static org.junit.jupiter.api.Assertions.*; -public class TestTraceSet { +class TestTraceSet { private static Path tempDir; private static final String BYTES_TRS = "bytes.trs"; private static final String SHORTS_TRS = "shorts.trs"; @@ -208,7 +210,9 @@ void testWriteTraceSetParameters() throws IOException, TRSFormatException { } /** - * This tests adding a parameter with a name of 100000 characters + * This tests adding a parameter with a name of 100000 characters. + * Expectation: The name will be truncated to the maximum allowed length when writing, + * when reading back and comparing with the original metadata, the values will differ * * @throws IOException * @throws TRSFormatException @@ -218,15 +222,15 @@ void testWriteTraceParametersInvalidName() throws IOException, TRSFormatExceptio TRSMetaData metaData = TRSMetaData.create(); String parameterName = String.format("%100000s", "XYZ"); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put(parameterName, 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - assertThrows(TRSFormatException.class, () -> { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + assertThrows(AssertionFailedError.class, () -> { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); parameterDefinitions.forEach((key, parameter) -> assertEquals(parameterName, key)); } @@ -684,4 +688,15 @@ void testFileReleasing() throws IOException, TRSFormatException, InterruptedExce File file = new File(filePath); assert(file.delete()); } + + @Test + void testDefaultHeaderSize() throws IOException, TRSFormatException { + Path filePath = tempDir.resolve("large_header.trs"); + TRSMetaData metaData = new TRSMetaData(); + metaData.put(TRSTag.TRS_VERSION, 2); + try (TraceSet ts = TraceSet.create(filePath.toString(), metaData)) { + ts.add(new Trace(new float[]{})); + } + assertTrue(filePath.toFile().length() > DEFAULT_METADATA_SIZE); + } } From 723ce6ab2c9c2862b782f0541e6283bef3a18a6d Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 30 Jan 2026 15:08:41 +0100 Subject: [PATCH 05/14] #77: Added file unmapping delay as the default --- src/main/java/com/riscure/trs/ReadOnlyTraceSet.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java index 5fb8908..c99716f 100644 --- a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java +++ b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java @@ -129,6 +129,19 @@ private long calculateTraceSize() { public void close() throws IOException, TRSFormatException { super.close(); closeReader(); + awaitFileUnmapping(); + } + + private static void awaitFileUnmapping() throws IOException { + // Unfortunately, the current solution requires a garbage collect to have been performed before the issue is resolved. + // Other fixes required either a Java 8 Cleaner.clean() call not accessible from Java 21, or a Java 20 Arena.close(), + // which is not been finalized in Java 21. + System.gc(); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new IOException(e); + } } @Override From 903ff62e77c5da4e111d3d997ab30b84835d9426 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 30 Jan 2026 15:10:27 +0100 Subject: [PATCH 06/14] #77: Added test to check updating metadata and bumped to V3 after adding metadata padding --- src/test/java/TestTraceSet.java | 47 ++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/src/test/java/TestTraceSet.java b/src/test/java/TestTraceSet.java index f5b2325..72f6f69 100644 --- a/src/test/java/TestTraceSet.java +++ b/src/test/java/TestTraceSet.java @@ -76,10 +76,7 @@ public static void createTempDir() throws IOException, TRSFormatException { } @AfterAll - public static void cleanup() throws InterruptedException { - //We need to allow a little time for java to release all handles - System.gc(); - Thread.sleep(100); + public static void cleanup() { for (File file : Objects.requireNonNull(tempDir.toFile().listFiles())) { try { Files.delete(file.toPath()); @@ -679,24 +676,50 @@ void testFileReleasing() throws IOException, TRSFormatException, InterruptedExce try (TraceSet traceSet = TraceSet.open(filePath)) { traceSet.getMetaData().getTraceSetParameters(); } - // Unfortunately, the current solution requires a garbage collect to have been performed before the issue is resolved. - // Other fixes required either a Java 8 Cleaner.clean() call not accessible from Java 21, or a Java 20 Arena.close(), - // which is not been finalized in Java 21. - System.gc(); - Thread.sleep(1000); // Assert that the opened file has been closed again, by deleting it. File file = new File(filePath); - assert(file.delete()); + assertTrue(file.delete()); } + /** + * This test checks whether version 3 correctly allocates 1MB of header space by default + */ @Test void testDefaultHeaderSize() throws IOException, TRSFormatException { Path filePath = tempDir.resolve("large_header.trs"); TRSMetaData metaData = new TRSMetaData(); - metaData.put(TRSTag.TRS_VERSION, 2); + metaData.put(TRSTag.TRS_VERSION, 3); try (TraceSet ts = TraceSet.create(filePath.toString(), metaData)) { - ts.add(new Trace(new float[]{})); + ts.add(new Trace(new float[]{0})); } assertTrue(filePath.toFile().length() > DEFAULT_METADATA_SIZE); } + + /** + * This test checks whether we can successfully add information to the header of a traceset file without + * increasing its size + */ + @Test + void testOverwritingMetadata() throws IOException, TRSFormatException { + String filename = tempDir.toAbsolutePath() + File.separator + BYTES_TRS; + long originalFileSize = new File(filename).length(); + + TraceSetParameterMap tspm; + TraceParameterDefinitionMap tpdm; + try (TraceSet readable = TraceSet.open(filename)) { + assertFalse(readable.getMetaData().getTraceSetParameters().containsKey("test")); + + tspm = readable.getMetaData().getTraceSetParameters().copy(); + tpdm = readable.getMetaData().getTraceParameterDefinitions().copy(); + + tspm.put("test", "This value should exist afterwards"); + } + + TraceSet.updateParameterMaps(filename, tspm, tpdm); + + try (TraceSet readable = TraceSet.open(filename)) { + assertTrue(readable.getMetaData().getTraceSetParameters().containsKey("test")); + } + assertEquals(originalFileSize, new File(filename).length()); + } } From 9f12eaa15a6eff9a2a58459b2ca1b5a08442e32e Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 30 Jan 2026 15:12:22 +0100 Subject: [PATCH 07/14] #77: Added implementation for updating the metadata --- .../com/riscure/trs/TRSMetaDataUtils.java | 91 +++++++++++++++++++ src/main/java/com/riscure/trs/TraceSet.java | 37 +++++++- 2 files changed, 126 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java index 75e194c..bc48b37 100644 --- a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java +++ b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java @@ -7,6 +7,7 @@ import java.io.FileOutputStream; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; @@ -18,6 +19,64 @@ public class TRSMetaDataUtils { private static final String UNSUPPORTED_TAG_TYPE = "Unsupported tag type for tag '%s': %s"; private static final String REWINDING_STREAM = "The output stream is not at the start of the file. Rewinding stream."; + /** + * Writes the provided TRS metadata to the stream. + * + * @param raf the file output opened in random access mode + * @param metaData the metadata to write + * @throws IOException if any write error occurs + * @throws TRSFormatException if the metadata contains unsupported tags + */ + public static void writeTRSMetaData(RandomAccessFile raf, TRSMetaData metaData) throws IOException, TRSFormatException { + // We always write at the start of the file + raf.seek(0); + for (TRSTag tag : TRSTag.values()) { + if (tag.equals(TRSTag.TRACE_BLOCK) || tag.equals(TRSTag.PADDING)) continue; //PADDING and TRACE BLOCK should be the last writes + if (!tag.isRequired() && metaData.hasDefaultValue(tag)) continue; //ignore if default and not required + raf.write(tag.getValue()); + if (tag.getType() == String.class) { + String s = metaData.getString(tag); + byte[] stringBytes = s.getBytes(StandardCharsets.UTF_8); + writeLength(raf, stringBytes.length); + raf.write(stringBytes); + } else if (tag.getType() == Float.class) { + float f = metaData.getFloat(tag); + writeLength(raf, tag.getLength()); + writeInt(raf, Float.floatToIntBits(f), tag.getLength()); + } else if (tag.getType() == Boolean.class) { + int value = metaData.getBoolean(tag) ? 1 : 0; + writeLength(raf, tag.getLength()); + writeInt(raf, value, tag.getLength()); + } else if (tag.getType() == Integer.class) { + writeLength(raf, tag.getLength()); + writeInt(raf, metaData.getInt(tag), tag.getLength()); + } else if (tag.getType() == TraceSetParameterMap.class) { + byte[] serialized = metaData.getTraceSetParameters().serialize(); + writeLength(raf, serialized.length); + raf.write(serialized); + } else if (tag.getType() == TraceParameterDefinitionMap.class) { + byte[] serialized = metaData.getTraceParameterDefinitions().serialize(); + writeLength(raf, serialized.length); + raf.write(serialized); + } else { + throw new TRSFormatException(String.format(UNSUPPORTED_TAG_TYPE, tag.getName(), tag.getType())); + } + } + // Grow the metadata up to 1M, creating an empty buffer in the trace set + // This allows us to grow the header without having to rewrite the whole file + if (raf.getChannel().position() < DEFAULT_METADATA_SIZE) { + raf.write(TRSTag.PADDING.getValue()); + int expectedLength = (int) (DEFAULT_METADATA_SIZE - raf.getChannel().position()); + // The length of the padding will be the maximum size minus the current position minus the number of bytes used for the length tag minus the length of the trace block tag minus the length of the trace block length tag + int paddingLength = expectedLength - computeLengthBytes(expectedLength) - 2; + writeLength(raf, paddingLength); + byte[] bytes = new byte[paddingLength]; + raf.write(bytes); + } + raf.write(TRSTag.TRACE_BLOCK.getValue()); + raf.write(TRSTag.TRACE_BLOCK.getLength()); + } + /** * Writes the provided TRS metadata to the stream. * @@ -75,12 +134,32 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData) fos.write(TRSTag.TRACE_BLOCK.getLength()); } + private static int computeLengthBytes(int length) { + int lengthBytes = 0; + if (length > 0x7F) { + int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); + lengthBytes++; + for (int i = 0; i < lenlen; i++) { + lengthBytes++; + } + } else { + lengthBytes++; + } + return lengthBytes; + } + private static void writeInt(FileOutputStream fos, int value, int length) throws IOException { for (int i = 0; i < length; i++) { fos.write((byte) (value >> (i * 8))); } } + private static void writeInt(RandomAccessFile raf, int value, int length) throws IOException { + for (int i = 0; i < length; i++) { + raf.write((byte) (value >> (i * 8))); + } + } + private static void writeLength(FileOutputStream fos, long length) throws IOException { if (length > 0x7F) { int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); @@ -93,6 +172,18 @@ private static void writeLength(FileOutputStream fos, long length) throws IOExce } } + private static void writeLength(RandomAccessFile raf, long length) throws IOException { + if (length > 0x7F) { + int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); + raf.write((byte) (0x80 + lenlen)); + for (int i = 0; i < lenlen; i++) { + raf.write((byte) (length >> (i * 8))); + } + } else { + raf.write((byte) length); + } + } + /** * Reads the meta data of a TRS file. The {@code ByteBuffer} is assumed to be positioned at the start of the file; A * {@code TRSFormatException} will probably be thrown otherwise, since it cannot be parsed. diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index 1b172b0..9379b39 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -1,10 +1,14 @@ package com.riscure.trs; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; +import com.riscure.trs.parameter.traceset.TraceSetParameterMap; + import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.file.Path; import java.util.List; -import static com.riscure.trs.enums.TRSTag.TRS_VERSION; +import static com.riscure.trs.enums.TRSTag.*; public abstract class TraceSet implements AutoCloseable { protected static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; @@ -137,7 +141,36 @@ public static TraceSet create(String file) throws IOException { * @throws IOException if the file creation failed */ public static TraceSet create(String file, TRSMetaData metaData) throws IOException { - metaData.put(TRS_VERSION, 2, false); + metaData.put(TRS_VERSION, 3, false); return new WritableTraceSet(file, metaData); } + + /** + * Overwrite the metadata associated with this trace set + * If this traceset is in read mode, this is only possible under certain conditions: + * 1) The opened trace set is a V3 set + * 2) There is empty remaining space (i.e. padding) in the pre-allocated metadata + * + * TODO: We should probably limit the changes to specific tags. e.g. the number of traces should not be modified, + * TODO: but the TSPM is fine. The definition map may be updated, but the size must remain the same + */ + public static void updateParameterMaps(String file, TraceSetParameterMap tspm, TraceParameterDefinitionMap tpdm) throws IOException, TRSFormatException { + TRSMetaData metaData; + try (TraceSet ts = open(file)) { + metaData = ts.getMetaData(); + } + + if (metaData.getInt(TRS_VERSION) < 3) throw new IOException(String.format("This trace set is version %d. Only version 3 and upwards support updating metadata.", metaData.getInt(TRS_VERSION))); + // TODO check this + //if (metaDataSize > DEFAULT_METADATA_SIZE) throw new IOException("The meta data has already grown beyond the padding size. This trace set does not support updating the meta data."); + if (metaData.getTraceParameterDefinitions().totalSize() != tpdm.totalSize()) throw new IOException("The provided parameter definitions are of a different size than the current ones. While it's possible to change the definitions, the size must match."); + + metaData.put(TRACE_SET_PARAMETERS, tspm); + metaData.put(TRACE_PARAMETER_DEFINITIONS, tpdm); + + // Open the file in append mode so we can overwrite the header only + try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) { + TRSMetaDataUtils.writeTRSMetaData(raf, metaData); + } + } } From e1cff1e725f69e03300a6eca5e15662e7604ee0c Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 30 Jan 2026 15:13:08 +0100 Subject: [PATCH 08/14] #77: Changed while to if (while doesn't do anything here) --- src/main/java/com/riscure/trs/TRSMetaDataUtils.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java index bc48b37..7dd2565 100644 --- a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java +++ b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java @@ -124,10 +124,13 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData) } // Grow the metadata up to 1M, creating an empty buffer in the trace set // This allows us to grow the header without having to rewrite the whole file - while (fos.getChannel().position() < DEFAULT_METADATA_SIZE) { - byte[] bytes = new byte[(int) (DEFAULT_METADATA_SIZE - fos.getChannel().position())]; + if (fos.getChannel().position() < DEFAULT_METADATA_SIZE) { fos.write(TRSTag.PADDING.getValue()); - writeLength(fos, bytes.length); + int expectedLength = (int) (DEFAULT_METADATA_SIZE - fos.getChannel().position()); + // The length of the padding will be the maximum size minus the current position minus the number of bytes used for the length tag minus the length of the trace block tag minus the length of the trace block length tag + int paddingLength = expectedLength - computeLengthBytes(expectedLength) - 2; + writeLength(fos, paddingLength); + byte[] bytes = new byte[paddingLength]; fos.write(bytes); } fos.write(TRSTag.TRACE_BLOCK.getValue()); From d534237d6856774199b7e4fedefbc4ff1e5d2cbe Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 30 Jan 2026 15:13:17 +0100 Subject: [PATCH 09/14] #77: Cleanup tests --- src/test/java/TestTraceSet.java | 86 ++++++++++++--------------------- 1 file changed, 32 insertions(+), 54 deletions(-) diff --git a/src/test/java/TestTraceSet.java b/src/test/java/TestTraceSet.java index 72f6f69..29e3984 100644 --- a/src/test/java/TestTraceSet.java +++ b/src/test/java/TestTraceSet.java @@ -11,7 +11,6 @@ import com.riscure.trs.parameter.trace.TraceParameterMap; import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; -import com.riscure.trs.parameter.traceset.TraceSetParameter; import com.riscure.trs.parameter.traceset.TraceSetParameterMap; import com.riscure.trs.types.*; import org.junit.jupiter.api.AfterAll; @@ -50,25 +49,25 @@ class TestTraceSet { public static void createTempDir() throws IOException, TRSFormatException { tempDir = Files.createTempDirectory("TestTraceSet"); - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(BYTE_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + SHORTS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + SHORTS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(SHORT_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + INTS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + INTS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(INT_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + FLOATS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + FLOATS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(FLOAT_SAMPLES)); } @@ -81,21 +80,19 @@ public static void cleanup() { try { Files.delete(file.toPath()); } catch (IOException e) { - System.err.printf("Failed to delete temporary file '%s'%n", file.toPath().toAbsolutePath().toString()); - e.printStackTrace(); + System.err.printf("Failed to delete temporary file '%s'%n", file.toPath().toAbsolutePath()); } } try { Files.delete(tempDir); } catch (IOException e) { - System.err.printf("Failed to delete temporary folder '%s'%n", tempDir.toFile().toPath().toAbsolutePath().toString()); - e.printStackTrace(); + System.err.printf("Failed to delete temporary folder '%s'%n", tempDir.toFile().toPath().toAbsolutePath()); } } @Test void testOpenBytes() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.BYTE, encoding); @@ -110,7 +107,7 @@ void testOpenBytes() throws IOException, TRSFormatException { @Test void testOpenShorts() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + SHORTS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + SHORTS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.SHORT, encoding); @@ -125,7 +122,7 @@ void testOpenShorts() throws IOException, TRSFormatException { @Test void testOpenInts() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + INTS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + INTS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.INT, encoding); @@ -140,7 +137,7 @@ void testOpenInts() throws IOException, TRSFormatException { @Test void testOpenFloats() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + FLOATS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + FLOATS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.FLOAT, encoding); @@ -156,13 +153,11 @@ void testOpenFloats() throws IOException, TRSFormatException { @Test void testUTF8Title() throws IOException, TRSFormatException { String title = "씨브 크레그스만"; - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet ts = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet ts = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name)) { ts.add(Trace.create(title, new float[0], new TraceParameterMap())); - } catch (TRSFormatException e) { - throw e; } - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertEquals(title, readable.get(0).getTitle()); } } @@ -170,9 +165,6 @@ void testUTF8Title() throws IOException, TRSFormatException { /** * This tests adding several different types of information to the trace set header. The three parameters are chosen * to match the three major cases: Strings, primitives, and arbitrary (serializable) objects. - * - * @throws IOException - * @throws TRSFormatException */ @Test void testWriteTraceSetParameters() throws IOException, TRSFormatException { @@ -197,10 +189,10 @@ void testWriteTraceSetParameters() throws IOException, TRSFormatException { //parameters.put("XYZ offset", XYZ_TEST_VALUE); metaData.put(TRSTag.TRACE_SET_PARAMETERS, parameters); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData).close(); + String name = UUID.randomUUID() + TRS; + TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData).close(); //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceSetParameterMap readTraceSetParameterMap = readable.getMetaData().getTraceSetParameters(); parameters.forEach((s, traceSetParameter) -> assertEquals(traceSetParameter, readTraceSetParameterMap.get(s))); } @@ -210,9 +202,6 @@ void testWriteTraceSetParameters() throws IOException, TRSFormatException { * This tests adding a parameter with a name of 100000 characters. * Expectation: The name will be truncated to the maximum allowed length when writing, * when reading back and comparing with the original metadata, the values will differ - * - * @throws IOException - * @throws TRSFormatException */ @Test void testWriteTraceParametersInvalidName() throws IOException, TRSFormatException { @@ -239,9 +228,6 @@ void testWriteTraceParametersInvalidName() throws IOException, TRSFormatExceptio * - if no length is specified, the first string is leading * - if a string is longer than the length specified, it should be truncated * - when truncated, a string should still be valid UTF-8 (truncated at character level, not byte level) - * - * @throws IOException - * @throws TRSFormatException */ @Test void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormatException { @@ -253,8 +239,8 @@ void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormat strings.add("ab"); strings.add("abcdefgh汉字"); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { for (int k = 0; k < 25; k++) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTEARRAY", new byte[]{(byte) k, (byte) k, (byte) k}); @@ -270,17 +256,14 @@ void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormat /** * This tests whether all getters are working as expected - * - * @throws IOException - * @throws TRSFormatException */ @Test void testReadTraceParametersTyped() throws IOException, TRSFormatException { TRSMetaData metaData = TRSMetaData.create(); List testParameters = new ArrayList<>(); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { for (int k = 0; k < 25; k++) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTE", (byte) k); @@ -308,7 +291,7 @@ void testReadTraceParametersTyped() throws IOException, TRSFormatException { } private void readBackGeneric(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -323,7 +306,7 @@ private void readBackGeneric(List testParameters, String name } private void readBackTyped(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -392,7 +375,7 @@ private void readBackTyped(List testParameters, String name) } private void readBackTypedKeys(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -429,8 +412,8 @@ private void readBackTypedKeys(List testParameters, String na throw new RuntimeException("Unexpected type: " + parameter.getType()); } if (parameter.getLength() > 1 && typedKey.getCls().isArray()) { - assertArrayEquals(Arrays.asList(correctValue.getOrElseThrow(typedKey)).toArray(), - Arrays.asList(trace.getParameters().getOrElseThrow(typedKey)).toArray()); + assertArrayEquals(Collections.singletonList(correctValue.getOrElseThrow(typedKey)).toArray(), + Collections.singletonList(trace.getParameters().getOrElseThrow(typedKey)).toArray()); } else { assertEquals(correctValue.get(typedKey), trace.getParameters().get(typedKey)); } @@ -441,42 +424,37 @@ private void readBackTypedKeys(List testParameters, String na /** * This tests getting a value of the wrong type correctly throws an exception - * - * @throws IOException - * @throws TRSFormatException */ @Test void testExceptionWrongType() throws IOException, TRSFormatException { TRSMetaData metaData = TRSMetaData.create(); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTE", (byte) 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertThrows(ClassCastException.class, () -> readable.get(0).getParameters().getDouble("BYTE")); } } /** * This - * @throws IOException - * @throws TRSFormatException */ @Test void testContainsNonArray() throws IOException, TRSFormatException { ByteTypeKey byteKey = new ByteTypeKey("BYTE"); - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put(byteKey, (byte) 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertTrue(readable.get(0).getParameters().get(byteKey).isPresent()); } } @@ -504,7 +482,7 @@ void testInvalidParameterLength() { */ @Test void testModificationAfterReadback() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { assertThrows(UnsupportedOperationException.class, () -> readable.getMetaData().getTraceSetParameters().put("SHOULD_FAIL", 0)); assertThrows(UnsupportedOperationException.class, () -> readable.getMetaData().getTraceParameterDefinitions().put("SHOULD_FAIL", new TraceParameterDefinition(ParameterType.BYTE, (short)1, (short)1))); for (int k = 0; k < NUMBER_OF_TRACES; k++) { From 8d2f63b465293e129b0097ae9f6d319eca50a5f1 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 30 Jan 2026 16:03:27 +0100 Subject: [PATCH 10/14] #77: Changed parameter type so we can just write 0s instead of an empty string --- src/main/java/com/riscure/trs/enums/TRSTag.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/riscure/trs/enums/TRSTag.java b/src/main/java/com/riscure/trs/enums/TRSTag.java index 0547d70..e99ea33 100644 --- a/src/main/java/com/riscure/trs/enums/TRSTag.java +++ b/src/main/java/com/riscure/trs/enums/TRSTag.java @@ -55,7 +55,7 @@ public enum TRSTag { XY_MEASUREMENTS_PER_SPOT (0x75, "ME", false, Integer.class, 4, 0, "Number of consecutive measurements done per spot during XY scan"), TRACE_SET_PARAMETERS (0x76, "GP", false, TraceSetParameterMap.class, 0, UnmodifiableTraceSetParameterMap.of(new TraceSetParameterMap()), "The set of custom global trace set parameters"), TRACE_PARAMETER_DEFINITIONS (0x77, "LP", false, TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()), "The set of custom local trace parameters"), - PADDING (0xFF, "FF", false, String.class, 0, 0, "Empty value to allow growing the metadata"); + PADDING (0xFF, "FF", false, Integer.class, 0, 0, "Empty value to allow growing the metadata"); private static final String UNKNOWN_TAG = "Unknown tag: 0x%X"; From 84eb4fb9a5b086398bf47177389f6bc936dfbd7e Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 30 Jan 2026 16:04:12 +0100 Subject: [PATCH 11/14] #77: Reset the trace counter in case we created a copy of another MetaData object --- src/main/java/com/riscure/trs/WritableTraceSet.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/riscure/trs/WritableTraceSet.java b/src/main/java/com/riscure/trs/WritableTraceSet.java index aef3306..0e3b4a1 100644 --- a/src/main/java/com/riscure/trs/WritableTraceSet.java +++ b/src/main/java/com/riscure/trs/WritableTraceSet.java @@ -55,6 +55,7 @@ public void add(Trace trace) throws IOException, TRSFormatException { metaData.put(TITLE_SPACE, titleLength, false); metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false); metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters())); + metaData.put(NUMBER_OF_TRACES, 0); TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); firstTrace = false; } From 1a109a5a49e090353acb366f5b25fe8c9cf31b31 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Fri, 30 Jan 2026 16:04:39 +0100 Subject: [PATCH 12/14] #77: Allow duplicating a metadata object, making it modifiable --- src/main/java/com/riscure/trs/TRSMetaData.java | 13 +++++++++++++ src/main/java/com/riscure/trs/WritableTraceSet.java | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/riscure/trs/TRSMetaData.java b/src/main/java/com/riscure/trs/TRSMetaData.java index e2fb127..fa9829b 100644 --- a/src/main/java/com/riscure/trs/TRSMetaData.java +++ b/src/main/java/com/riscure/trs/TRSMetaData.java @@ -29,6 +29,19 @@ private void init() { } } + /** + * @return a modifiable copy of this metadata object + */ + public TRSMetaData modifiable() { + TRSMetaData copy = new TRSMetaData(); + for (TRSTag tag : TRSTag.values()) { + copy.put(tag, get(tag)); + } + copy.put(TRSTag.TRACE_SET_PARAMETERS, getTraceSetParameters().copy()); + copy.put(TRSTag.TRACE_PARAMETER_DEFINITIONS, getTraceParameterDefinitions().copy()); + return copy; + } + /** * Add the data associated with the supplied tag to this metadata. * This will overwrite any existing value diff --git a/src/main/java/com/riscure/trs/WritableTraceSet.java b/src/main/java/com/riscure/trs/WritableTraceSet.java index 0e3b4a1..050436c 100644 --- a/src/main/java/com/riscure/trs/WritableTraceSet.java +++ b/src/main/java/com/riscure/trs/WritableTraceSet.java @@ -35,7 +35,7 @@ public class WritableTraceSet extends TraceSet { WritableTraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { super(Paths.get(outputFileName)); - this.metaData = metaData; + this.metaData = metaData.modifiable(); this.writeStream = new FileOutputStream(outputFileName); } From b95efc5df57a41425b9d84a4ef3aa11057e47edf Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Wed, 12 Aug 2026 15:54:05 +0200 Subject: [PATCH 13/14] #77: Added specification to project --- TRS_FILE_FORMAT_SPECIFICATION.md | 171 +++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 TRS_FILE_FORMAT_SPECIFICATION.md diff --git a/TRS_FILE_FORMAT_SPECIFICATION.md b/TRS_FILE_FORMAT_SPECIFICATION.md new file mode 100644 index 0000000..ea4c0aa --- /dev/null +++ b/TRS_FILE_FORMAT_SPECIFICATION.md @@ -0,0 +1,171 @@ +# TRS File Format Specification + +This document describes the `.trs` file format implemented by this library. + +## 1. General rules + +- **Byte order:** little-endian for all multi-byte numeric values. +- **Strings:** UTF-8 encoded. +- **Container style:** the file starts with a metadata TLV stream, followed by packed trace records. + +## 2. File layout + +``` +[metadata TLVs ...][TRACE_BLOCK][trace 0][trace 1]...[trace N-1] +``` + +Metadata is read until tag `TRACE_BLOCK` (`0x5F`) is reached. + +Each trace record has fixed size: + +`TITLE_SPACE + DATA_LENGTH + (NUMBER_OF_SAMPLES * sample_byte_size)` + +## 3. Metadata TLV encoding + +Each metadata item is encoded as: + +1. `T`: 1 byte tag id +2. `L`: variable-length length field +3. `V`: payload bytes + +### 3.1 Length encoding + +- If `L <= 0x7F`, it is written as one byte. +- If `L > 0x7F`, the first byte is `0x80 + n`, where `n` is the number of following length bytes. +- Those `n` bytes store `L` in little-endian order. + +The reader accepts metadata lengths up to `0xFFFFFF`. + +## 4. Core metadata tags + +| Tag | Id | Payload | +|---|---:|---| +| NUMBER_OF_TRACES | `0x41` | int32 | +| NUMBER_OF_SAMPLES | `0x42` | int32 | +| SAMPLE_CODING | `0x43` | uint8 enum | +| DATA_LENGTH | `0x44` | uint16 | +| TITLE_SPACE | `0x45` | uint8 | +| GLOBAL_TITLE | `0x46` | string | +| DESCRIPTION | `0x47` | string | +| OFFSET_X | `0x48` | int32 | +| LABEL_X | `0x49` | string | +| LABEL_Y | `0x4A` | string | +| SCALE_X | `0x4B` | float32 | +| SCALE_Y | `0x4C` | float32 | +| TRACE_OFFSET | `0x4D` | int32 | +| LOGARITHMIC_SCALE | `0x4E` | boolean | +| TRS_VERSION | `0x4F` | uint8 | +| TRACE_BLOCK | `0x5F` | empty | +| TRACE_SET_PARAMETERS | `0x76` | serialized map | +| TRACE_PARAMETER_DEFINITIONS | `0x77` | serialized map | +| PADDING | `0xFF` | bytes | + +Unknown tags are skipped using their declared length. + +## 5. Sample coding + +`SAMPLE_CODING` uses these values: + +| Code | Meaning | Bytes/sample | +|---:|---|---:| +| `0x01` | BYTE | 1 | +| `0x02` | SHORT | 2 | +| `0x04` | INT | 4 | +| `0x14` | FLOAT | 4 | + +Samples are stored as raw little-endian values in the chosen encoding. + +## 6. Trace record layout + +For each trace: + +1. **Title**: exactly `TITLE_SPACE` bytes +2. **Data**: exactly `DATA_LENGTH` bytes +3. **Samples**: `NUMBER_OF_SAMPLES * sample_byte_size` bytes + +If a title is all whitespace when read, the reader may synthesize: + +` ` + +## 7. Version behavior + +### 7.1 Version 1 + +- The `DATA_LENGTH` bytes are treated as opaque legacy trace data. +- On read, non-empty legacy data is exposed as a trace parameter named `LEGACY_DATA`. + +### 7.2 Version 2+ + +Version 2 adds: + +- `TRACE_SET_PARAMETERS` +- `TRACE_PARAMETER_DEFINITIONS` + +Per-trace parameter data is stored only as the packed value bytes in the trace record; the names, types, lengths, and offsets live in the header definition map. + +### 7.3 Version 3 in this implementation + +- Writers default `TRS_VERSION` to `3`. +- Writers reserve about `1_000_000` bytes for metadata by inserting `PADDING` before `TRACE_BLOCK`. +- This allows later header rewrites without moving the trace data. + +## 8. Trace set parameters + +`TRACE_SET_PARAMETERS` stores global custom key/value pairs. + +Serialized form: + +1. `NE`: uint16 entry count +2. Repeated `NE` times: + - `NL`: uint16 name length + - `N`: UTF-8 name bytes + - `TYPE`: uint8 parameter type + - `LEN`: uint16 element count + - `VALUE`: serialized value bytes + +Supported parameter types: + +| Type | Code | Element size | +|---|---:|---:| +| BYTE | `0x01` | 1 | +| SHORT | `0x02` | 2 | +| INT | `0x04` | 4 | +| LONG | `0x08` | 8 | +| FLOAT | `0x14` | 4 | +| DOUBLE | `0x18` | 8 | +| STRING | `0x20` | UTF-8 bytes | +| BOOL | `0x31` | 1 | + +Length 1 values are treated as scalars by the API, but they still serialize through the same map structure. + +## 9. Trace parameter definitions + +`TRACE_PARAMETER_DEFINITIONS` defines the per-trace parameter block layout. + +Serialized form: + +1. `NE`: uint16 entry count +2. Repeated `NE` times: + - `NL`: uint16 name length + - `N`: UTF-8 name bytes + - `TYPE`: uint8 parameter type + - `LEN`: uint16 element count + - `OFFSET`: uint16 byte offset within the trace data block + +The order of entries is preserved and used when packing/unpacking trace parameter bytes. + +## 10. Per-trace parameter block + +For version 2+ files, the trace data block is the concatenation of the defined parameters in header order. + +`DATA_LENGTH` must equal: + +`sum(parameter_length * type_byte_size)` + +## 11. Practical constraints + +- All traces in a file must have the same number of samples. +- All traces must have the same data length. +- String fields are truncated or padded to their reserved byte length when writing. +- Unknown metadata tags are tolerated on read. + From f88475f70061a9ba43a3f0e026e909b544c44e20 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Wed, 12 Aug 2026 15:58:32 +0200 Subject: [PATCH 14/14] #77: Reduce default header size to 256k This will have less impact on very small trace sets, while still being plenty --- src/main/java/com/riscure/trs/TraceSet.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index 9379b39..9fcc6c7 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -12,8 +12,8 @@ public abstract class TraceSet implements AutoCloseable { protected static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; - // We want to pre-allocate 1M for the header, so we can grow it if needed without re-writing the whole file - public static final long DEFAULT_METADATA_SIZE = 1_000_000L; + // We want to pre-allocate 256k for the header, so we can grow it if needed without re-writing the whole file + public static final long DEFAULT_METADATA_SIZE = 256_000L; //Shared variables private final Path path;