diff --git a/TRS_FILE_FORMAT_SPECIFICATION.md b/TRS_FILE_FORMAT_SPECIFICATION.md new file mode 100644 index 0000000..ea4c0aa --- /dev/null +++ b/TRS_FILE_FORMAT_SPECIFICATION.md @@ -0,0 +1,171 @@ +# TRS File Format Specification + +This document describes the `.trs` file format implemented by this library. + +## 1. General rules + +- **Byte order:** little-endian for all multi-byte numeric values. +- **Strings:** UTF-8 encoded. +- **Container style:** the file starts with a metadata TLV stream, followed by packed trace records. + +## 2. File layout + +``` +[metadata TLVs ...][TRACE_BLOCK][trace 0][trace 1]...[trace N-1] +``` + +Metadata is read until tag `TRACE_BLOCK` (`0x5F`) is reached. + +Each trace record has fixed size: + +`TITLE_SPACE + DATA_LENGTH + (NUMBER_OF_SAMPLES * sample_byte_size)` + +## 3. Metadata TLV encoding + +Each metadata item is encoded as: + +1. `T`: 1 byte tag id +2. `L`: variable-length length field +3. `V`: payload bytes + +### 3.1 Length encoding + +- If `L <= 0x7F`, it is written as one byte. +- If `L > 0x7F`, the first byte is `0x80 + n`, where `n` is the number of following length bytes. +- Those `n` bytes store `L` in little-endian order. + +The reader accepts metadata lengths up to `0xFFFFFF`. + +## 4. Core metadata tags + +| Tag | Id | Payload | +|---|---:|---| +| NUMBER_OF_TRACES | `0x41` | int32 | +| NUMBER_OF_SAMPLES | `0x42` | int32 | +| SAMPLE_CODING | `0x43` | uint8 enum | +| DATA_LENGTH | `0x44` | uint16 | +| TITLE_SPACE | `0x45` | uint8 | +| GLOBAL_TITLE | `0x46` | string | +| DESCRIPTION | `0x47` | string | +| OFFSET_X | `0x48` | int32 | +| LABEL_X | `0x49` | string | +| LABEL_Y | `0x4A` | string | +| SCALE_X | `0x4B` | float32 | +| SCALE_Y | `0x4C` | float32 | +| TRACE_OFFSET | `0x4D` | int32 | +| LOGARITHMIC_SCALE | `0x4E` | boolean | +| TRS_VERSION | `0x4F` | uint8 | +| TRACE_BLOCK | `0x5F` | empty | +| TRACE_SET_PARAMETERS | `0x76` | serialized map | +| TRACE_PARAMETER_DEFINITIONS | `0x77` | serialized map | +| PADDING | `0xFF` | bytes | + +Unknown tags are skipped using their declared length. + +## 5. Sample coding + +`SAMPLE_CODING` uses these values: + +| Code | Meaning | Bytes/sample | +|---:|---|---:| +| `0x01` | BYTE | 1 | +| `0x02` | SHORT | 2 | +| `0x04` | INT | 4 | +| `0x14` | FLOAT | 4 | + +Samples are stored as raw little-endian values in the chosen encoding. + +## 6. Trace record layout + +For each trace: + +1. **Title**: exactly `TITLE_SPACE` bytes +2. **Data**: exactly `DATA_LENGTH` bytes +3. **Samples**: `NUMBER_OF_SAMPLES * sample_byte_size` bytes + +If a title is all whitespace when read, the reader may synthesize: + +` ` + +## 7. Version behavior + +### 7.1 Version 1 + +- The `DATA_LENGTH` bytes are treated as opaque legacy trace data. +- On read, non-empty legacy data is exposed as a trace parameter named `LEGACY_DATA`. + +### 7.2 Version 2+ + +Version 2 adds: + +- `TRACE_SET_PARAMETERS` +- `TRACE_PARAMETER_DEFINITIONS` + +Per-trace parameter data is stored only as the packed value bytes in the trace record; the names, types, lengths, and offsets live in the header definition map. + +### 7.3 Version 3 in this implementation + +- Writers default `TRS_VERSION` to `3`. +- Writers reserve about `1_000_000` bytes for metadata by inserting `PADDING` before `TRACE_BLOCK`. +- This allows later header rewrites without moving the trace data. + +## 8. Trace set parameters + +`TRACE_SET_PARAMETERS` stores global custom key/value pairs. + +Serialized form: + +1. `NE`: uint16 entry count +2. Repeated `NE` times: + - `NL`: uint16 name length + - `N`: UTF-8 name bytes + - `TYPE`: uint8 parameter type + - `LEN`: uint16 element count + - `VALUE`: serialized value bytes + +Supported parameter types: + +| Type | Code | Element size | +|---|---:|---:| +| BYTE | `0x01` | 1 | +| SHORT | `0x02` | 2 | +| INT | `0x04` | 4 | +| LONG | `0x08` | 8 | +| FLOAT | `0x14` | 4 | +| DOUBLE | `0x18` | 8 | +| STRING | `0x20` | UTF-8 bytes | +| BOOL | `0x31` | 1 | + +Length 1 values are treated as scalars by the API, but they still serialize through the same map structure. + +## 9. Trace parameter definitions + +`TRACE_PARAMETER_DEFINITIONS` defines the per-trace parameter block layout. + +Serialized form: + +1. `NE`: uint16 entry count +2. Repeated `NE` times: + - `NL`: uint16 name length + - `N`: UTF-8 name bytes + - `TYPE`: uint8 parameter type + - `LEN`: uint16 element count + - `OFFSET`: uint16 byte offset within the trace data block + +The order of entries is preserved and used when packing/unpacking trace parameter bytes. + +## 10. Per-trace parameter block + +For version 2+ files, the trace data block is the concatenation of the defined parameters in header order. + +`DATA_LENGTH` must equal: + +`sum(parameter_length * type_byte_size)` + +## 11. Practical constraints + +- All traces in a file must have the same number of samples. +- All traces must have the same data length. +- String fields are truncated or padded to their reserved byte length when writing. +- Unknown metadata tags are tolerated on read. + diff --git a/pom.xml b/pom.xml index ada9def..c3112cc 100644 --- a/pom.xml +++ b/pom.xml @@ -36,9 +36,9 @@ - 8 - 8 - 8 + 11 + 11 + 11 UTF-8 diff --git a/src/main/java/com/riscure/trs/LargePreMappedFile.java b/src/main/java/com/riscure/trs/LargePreMappedFile.java new file mode 100644 index 0000000..a182e54 --- /dev/null +++ b/src/main/java/com/riscure/trs/LargePreMappedFile.java @@ -0,0 +1,102 @@ +package com.riscure.trs; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; + +public class LargePreMappedFile implements AutoCloseable{ + private final FileChannel channel; + + private final List buffers = new ArrayList<>(); + private final long readOffset; + private final long traceSize; + private final long fileSize; + + public LargePreMappedFile(FileChannel channel, long metaDataSize, long traceSize) throws IOException { + this.channel = channel; + this.readOffset = metaDataSize; + this.traceSize = traceSize; + this.fileSize = channel.size() - readOffset; + + mapBuffers(); + } + + public ByteBuffer getBuffer(int index) { + if (traceSize == 0) { + return ByteBuffer.wrap(new byte[0]); + } + return findBufferAndMoveToTrace(index); + } + + private ByteBuffer findBufferAndMoveToTrace(int traceIndex) { + MappedBuffer mappedBuffer = buffers.stream() + .filter(buffer -> traceIndex >= buffer.getFirstTraceIndex() && + traceIndex < buffer.getFirstTraceIndex() + buffer.getNumberOfTraces()) + .findFirst() + .orElseThrow(); + ByteBuffer buffer = mappedBuffer.getBuffer(); + int traceIndexInBuffer = traceIndex - mappedBuffer.getFirstTraceIndex(); + long positionInBuffer = traceIndexInBuffer * traceSize; + buffer.position((int) positionInBuffer); + return buffer; + } + + private void mapBuffers() { + if (traceSize > 0) { + int tracesPerBuffer = (int) (Integer.MAX_VALUE / traceSize); + long maximumBufferSize = tracesPerBuffer * traceSize; + + int firstTraceIndex = 0; + for (long offset = 0; offset < fileSize; offset += maximumBufferSize) { + buffers.add(mapBuffer(firstTraceIndex, maximumBufferSize)); + firstTraceIndex += tracesPerBuffer; + } + } + } + + private MappedBuffer mapBuffer(int firstTraceIndex, long bufferSize) { + try { + long bufferStart = firstTraceIndex * traceSize; + long limitedBufferSize = Math.min(fileSize - bufferStart, bufferSize); + MappedBuffer mappedBuffer = new MappedBuffer(this.channel.map(FileChannel.MapMode.READ_ONLY, readOffset + bufferStart, limitedBufferSize), + firstTraceIndex, + (int) (traceSize > 0 ? (limitedBufferSize / traceSize) : 0)); + mappedBuffer.buffer.order(ByteOrder.LITTLE_ENDIAN); + return mappedBuffer; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + buffers.clear(); + } + + private static class MappedBuffer { + private final ByteBuffer buffer; + private final int firstTraceIndex; + private final int numberOfTraces; + + MappedBuffer(ByteBuffer buffer, int firstTraceIndex, int numberOfTraces) { + this.buffer = buffer; + this.firstTraceIndex = firstTraceIndex; + this.numberOfTraces = numberOfTraces; + } + + public ByteBuffer getBuffer() { + return buffer; + } + + public int getFirstTraceIndex() { + return firstTraceIndex; + } + + public int getNumberOfTraces() { + return numberOfTraces; + } + } +} diff --git a/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java new file mode 100644 index 0000000..c99716f --- /dev/null +++ b/src/main/java/com/riscure/trs/ReadOnlyTraceSet.java @@ -0,0 +1,212 @@ +package com.riscure.trs; + +import com.riscure.trs.enums.Encoding; +import com.riscure.trs.parameter.trace.TraceParameterMap; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; + +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.FloatBuffer; +import java.nio.IntBuffer; +import java.nio.ShortBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Paths; + +import static com.riscure.trs.enums.TRSTag.*; +import static com.riscure.trs.enums.TRSTag.TRS_VERSION; + +public class ReadOnlyTraceSet extends TraceSet { + private static final String TRACE_SET_IN_READ_MODE = "TraceSet is in read mode. Please open the TraceSet in write mode."; + private static final String ERROR_READING_FILE = "Error reading TRS file: file size (%d) != meta data (%d) + trace size (%d) * nr of traces (%d)"; + private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d)."; + private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; + // This is excessive for the header, but it's only the initial maximum + private static final long INITIAL_MEMORY_SIZE = 100_000_000L; + + private final int metaDataSize; + private final FileInputStream readStream; + private final LargePreMappedFile mappedFile; + private final float[] preallocatedSampleArray; + private final TRSMetaData metaData; + private final long fileSize; //the total number of bytes in the underlying file + + private ByteBuffer metaDataBuffer; + private byte[] preallocatedByteArray; + private short[] preallocatedShortArray; + private int[] preallocatedIntArray; + + ReadOnlyTraceSet(String inputFileName) throws IOException, TRSFormatException { + super(Paths.get(inputFileName)); + + this.readStream = new FileInputStream(inputFileName); + FileChannel channel = readStream.getChannel(); + + //the file might be bigger than the buffer, in which case we partially buffer it in memory + this.fileSize = channel.size(); + long initialBufferSize = Math.min(fileSize, INITIAL_MEMORY_SIZE); + + this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); + this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); + this.metaDataSize = metaDataBuffer.position(); + this.metaDataBuffer.limit(metaDataSize); + + long traceSize = calculateTraceSize(); + this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); + + int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); + this.preallocatedSampleArray = new float[numberOfSamples]; + } + + /** + * Get a trace from the set at the specified index + * @param index the index of the Trace to read from the file + * @return the Trace at the requested trace index + * @throws IOException if a read error occurs + * @throws IllegalArgumentException if this TraceSet is not ready be read from + */ + @Override + public Trace get(int index) throws IOException { + if (!isOpen()) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); + + long traceSize = calculateTraceSize(); + long nrOfTraces = this.metaData.getInt(NUMBER_OF_TRACES); + if (index >= nrOfTraces) { + String msg = String.format(TRACE_INDEX_OUT_OF_BOUNDS, index, nrOfTraces); + throw new IllegalArgumentException(msg); + } + + long calculatedFileSize = metaDataSize + traceSize * nrOfTraces; + if (fileSize != calculatedFileSize) { + String msg = String.format(ERROR_READING_FILE, fileSize, metaDataSize, traceSize, nrOfTraces); + throw new IllegalStateException(msg); + } + + ByteBuffer buffer = mappedFile.getBuffer(index); + + String traceTitle = this.readTraceTitle(buffer); + if (traceTitle.trim().isEmpty()) { + traceTitle = String.format("%s %d", metaData.getString(GLOBAL_TITLE), index); + } + + try { + TraceParameterMap traceParameterMap; + if (metaData.getInt(TRS_VERSION) > 1) { + TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); + int size = traceParameterDefinitionMap.totalSize(); + byte[] data = new byte[size]; + buffer.get(data); + traceParameterMap = TraceParameterMap.deserialize(data, traceParameterDefinitionMap); + } else { + //legacy mode + byte[] data = readData(buffer); + traceParameterMap = new TraceParameterMap(); + if (data.length > 0) { + traceParameterMap.put("LEGACY_DATA", data); + } + } + + float[] samples = readSamples(buffer); + // Since we are using an internal sample array in this class, Trace.create() should duplicate it internally + return Trace.create(traceTitle, samples, traceParameterMap); + } catch (TRSFormatException ex) { + throw new IOException(ex); + } + } + + @Override + public void add(Trace trace) throws IOException, TRSFormatException { + throw new IllegalArgumentException(TRACE_SET_IN_READ_MODE); + } + + private long calculateTraceSize() { + int sampleSize = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)).getSize(); + long sampleSpace = metaData.getInt(NUMBER_OF_SAMPLES) * (long) sampleSize; + return sampleSpace + metaData.getInt(DATA_LENGTH) + metaData.getInt(TITLE_SPACE); + } + + @Override + public void close() throws IOException, TRSFormatException { + super.close(); + closeReader(); + awaitFileUnmapping(); + } + + private static void awaitFileUnmapping() throws IOException { + // Unfortunately, the current solution requires a garbage collect to have been performed before the issue is resolved. + // Other fixes required either a Java 8 Cleaner.clean() call not accessible from Java 21, or a Java 20 Arena.close(), + // which is not been finalized in Java 21. + System.gc(); + try { + Thread.sleep(100); + } catch (InterruptedException e) { + throw new IOException(e); + } + } + + @Override + public TRSMetaData getMetaData() { + return metaData; + } + + private void closeReader() throws IOException { + metaDataBuffer = null; + mappedFile.close(); + readStream.close(); + } + + protected String readTraceTitle(ByteBuffer buffer) { + byte[] titleArray = new byte[metaData.getInt(TITLE_SPACE)]; + buffer.get(titleArray); + return new String(titleArray); + } + + protected byte[] readData(ByteBuffer buffer) { + int inputSize = metaData.getInt(DATA_LENGTH); + byte[] comDataArray = new byte[inputSize]; + buffer.get(comDataArray); + return comDataArray; + } + + /* + * We can reuse the buffers when not dealing with float samples. They are instantiated once just in time if needed. + */ + protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { + switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { + case BYTE: + this.preallocatedByteArray = this.preallocatedByteArray == null ? new byte[preallocatedSampleArray.length] : this.preallocatedByteArray; + buffer.get(preallocatedByteArray); + // Manual copy of byte[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedByteArray[k]; + } + break; + case SHORT: + this.preallocatedShortArray = this.preallocatedShortArray == null ? new short[preallocatedSampleArray.length] : this.preallocatedShortArray; + ShortBuffer shortView = buffer.asShortBuffer(); + shortView.get(preallocatedShortArray); + // Manual copy of short[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedShortArray[k]; + } + break; + case FLOAT: + FloatBuffer floatView = buffer.asFloatBuffer(); + floatView.get(preallocatedSampleArray); + break; + case INT: + this.preallocatedIntArray = this.preallocatedIntArray == null ? new int[preallocatedSampleArray.length] : this.preallocatedIntArray; + IntBuffer intView = buffer.asIntBuffer(); + intView.get(preallocatedIntArray); + // Manual copy of int[] into float[] + for (int k = 0; k < preallocatedIntArray.length; k++) { + preallocatedSampleArray[k] = (float) preallocatedIntArray[k]; + } + break; + default: + throw new TRSFormatException(String.format(UNKNOWN_SAMPLE_CODING, metaData.getInt(SAMPLE_CODING))); + } + + return preallocatedSampleArray; + } +} diff --git a/src/main/java/com/riscure/trs/TRSMetaData.java b/src/main/java/com/riscure/trs/TRSMetaData.java index e2fb127..fa9829b 100644 --- a/src/main/java/com/riscure/trs/TRSMetaData.java +++ b/src/main/java/com/riscure/trs/TRSMetaData.java @@ -29,6 +29,19 @@ private void init() { } } + /** + * @return a modifiable copy of this metadata object + */ + public TRSMetaData modifiable() { + TRSMetaData copy = new TRSMetaData(); + for (TRSTag tag : TRSTag.values()) { + copy.put(tag, get(tag)); + } + copy.put(TRSTag.TRACE_SET_PARAMETERS, getTraceSetParameters().copy()); + copy.put(TRSTag.TRACE_PARAMETER_DEFINITIONS, getTraceParameterDefinitions().copy()); + return copy; + } + /** * Add the data associated with the supplied tag to this metadata. * This will overwrite any existing value diff --git a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java index dcc0ebf..7dd2565 100644 --- a/src/main/java/com/riscure/trs/TRSMetaDataUtils.java +++ b/src/main/java/com/riscure/trs/TRSMetaDataUtils.java @@ -7,15 +7,76 @@ import java.io.FileOutputStream; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import static com.riscure.trs.TraceSet.DEFAULT_METADATA_SIZE; + public class TRSMetaDataUtils { private static final String IGNORED_UNKNOWN_TAG = "ignored unknown metadata tag '%02X' while reading a TRS file\n"; private static final String TAG_LENGTH_INVALID = "The length field following tag '%s' has value '%X', which is not between 0 and 0xffff"; private static final String UNSUPPORTED_TAG_TYPE = "Unsupported tag type for tag '%s': %s"; private static final String REWINDING_STREAM = "The output stream is not at the start of the file. Rewinding stream."; + /** + * Writes the provided TRS metadata to the stream. + * + * @param raf the file output opened in random access mode + * @param metaData the metadata to write + * @throws IOException if any write error occurs + * @throws TRSFormatException if the metadata contains unsupported tags + */ + public static void writeTRSMetaData(RandomAccessFile raf, TRSMetaData metaData) throws IOException, TRSFormatException { + // We always write at the start of the file + raf.seek(0); + for (TRSTag tag : TRSTag.values()) { + if (tag.equals(TRSTag.TRACE_BLOCK) || tag.equals(TRSTag.PADDING)) continue; //PADDING and TRACE BLOCK should be the last writes + if (!tag.isRequired() && metaData.hasDefaultValue(tag)) continue; //ignore if default and not required + raf.write(tag.getValue()); + if (tag.getType() == String.class) { + String s = metaData.getString(tag); + byte[] stringBytes = s.getBytes(StandardCharsets.UTF_8); + writeLength(raf, stringBytes.length); + raf.write(stringBytes); + } else if (tag.getType() == Float.class) { + float f = metaData.getFloat(tag); + writeLength(raf, tag.getLength()); + writeInt(raf, Float.floatToIntBits(f), tag.getLength()); + } else if (tag.getType() == Boolean.class) { + int value = metaData.getBoolean(tag) ? 1 : 0; + writeLength(raf, tag.getLength()); + writeInt(raf, value, tag.getLength()); + } else if (tag.getType() == Integer.class) { + writeLength(raf, tag.getLength()); + writeInt(raf, metaData.getInt(tag), tag.getLength()); + } else if (tag.getType() == TraceSetParameterMap.class) { + byte[] serialized = metaData.getTraceSetParameters().serialize(); + writeLength(raf, serialized.length); + raf.write(serialized); + } else if (tag.getType() == TraceParameterDefinitionMap.class) { + byte[] serialized = metaData.getTraceParameterDefinitions().serialize(); + writeLength(raf, serialized.length); + raf.write(serialized); + } else { + throw new TRSFormatException(String.format(UNSUPPORTED_TAG_TYPE, tag.getName(), tag.getType())); + } + } + // Grow the metadata up to 1M, creating an empty buffer in the trace set + // This allows us to grow the header without having to rewrite the whole file + if (raf.getChannel().position() < DEFAULT_METADATA_SIZE) { + raf.write(TRSTag.PADDING.getValue()); + int expectedLength = (int) (DEFAULT_METADATA_SIZE - raf.getChannel().position()); + // The length of the padding will be the maximum size minus the current position minus the number of bytes used for the length tag minus the length of the trace block tag minus the length of the trace block length tag + int paddingLength = expectedLength - computeLengthBytes(expectedLength) - 2; + writeLength(raf, paddingLength); + byte[] bytes = new byte[paddingLength]; + raf.write(bytes); + } + raf.write(TRSTag.TRACE_BLOCK.getValue()); + raf.write(TRSTag.TRACE_BLOCK.getLength()); + } + /** * Writes the provided TRS metadata to the stream. * @@ -30,7 +91,7 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData) fos.getChannel().position(0); } for (TRSTag tag : TRSTag.values()) { - if (tag.equals(TRSTag.TRACE_BLOCK)) continue; //TRACE BLOCK should be the last write + if (tag.equals(TRSTag.TRACE_BLOCK) || tag.equals(TRSTag.PADDING)) continue; //PADDING and TRACE BLOCK should be the last writes if (!tag.isRequired() && metaData.hasDefaultValue(tag)) continue; //ignore if default and not required fos.write(tag.getValue()); if (tag.getType() == String.class) { @@ -61,16 +122,47 @@ public static void writeTRSMetaData(FileOutputStream fos, TRSMetaData metaData) throw new TRSFormatException(String.format(UNSUPPORTED_TAG_TYPE, tag.getName(), tag.getType())); } } + // Grow the metadata up to 1M, creating an empty buffer in the trace set + // This allows us to grow the header without having to rewrite the whole file + if (fos.getChannel().position() < DEFAULT_METADATA_SIZE) { + fos.write(TRSTag.PADDING.getValue()); + int expectedLength = (int) (DEFAULT_METADATA_SIZE - fos.getChannel().position()); + // The length of the padding will be the maximum size minus the current position minus the number of bytes used for the length tag minus the length of the trace block tag minus the length of the trace block length tag + int paddingLength = expectedLength - computeLengthBytes(expectedLength) - 2; + writeLength(fos, paddingLength); + byte[] bytes = new byte[paddingLength]; + fos.write(bytes); + } fos.write(TRSTag.TRACE_BLOCK.getValue()); fos.write(TRSTag.TRACE_BLOCK.getLength()); } + private static int computeLengthBytes(int length) { + int lengthBytes = 0; + if (length > 0x7F) { + int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); + lengthBytes++; + for (int i = 0; i < lenlen; i++) { + lengthBytes++; + } + } else { + lengthBytes++; + } + return lengthBytes; + } + private static void writeInt(FileOutputStream fos, int value, int length) throws IOException { for (int i = 0; i < length; i++) { fos.write((byte) (value >> (i * 8))); } } + private static void writeInt(RandomAccessFile raf, int value, int length) throws IOException { + for (int i = 0; i < length; i++) { + raf.write((byte) (value >> (i * 8))); + } + } + private static void writeLength(FileOutputStream fos, long length) throws IOException { if (length > 0x7F) { int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); @@ -83,6 +175,18 @@ private static void writeLength(FileOutputStream fos, long length) throws IOExce } } + private static void writeLength(RandomAccessFile raf, long length) throws IOException { + if (length > 0x7F) { + int lenlen = 1 + (int) (Math.log(length) / Math.log(256)); + raf.write((byte) (0x80 + lenlen)); + for (int i = 0; i < lenlen; i++) { + raf.write((byte) (length >> (i * 8))); + } + } else { + raf.write((byte) length); + } + } + /** * Reads the meta data of a TRS file. The {@code ByteBuffer} is assumed to be positioned at the start of the file; A * {@code TRSFormatException} will probably be thrown otherwise, since it cannot be parsed. @@ -124,7 +228,7 @@ public static String readName(LittleEndianInputStream dis) throws IOException { private static void readAndStoreData(ByteBuffer buffer, byte tag, int length, TRSMetaData trsMD) throws TRSFormatException { - boolean hasValidLength = (0 <= length & length <= 0xffff); + boolean hasValidLength = (0 <= length & length <= 0xffffff); TRSTag trsTag; try { trsTag = TRSTag.fromValue(tag); diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index ee46acb..9fcc6c7 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -1,89 +1,27 @@ package com.riscure.trs; -import com.riscure.trs.enums.Encoding; -import com.riscure.trs.enums.ParameterType; -import com.riscure.trs.parameter.TraceParameter; -import com.riscure.trs.parameter.primitive.StringParameter; -import com.riscure.trs.parameter.trace.TraceParameterMap; -import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; +import com.riscure.trs.parameter.traceset.TraceSetParameterMap; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; -import java.nio.*; -import java.nio.channels.FileChannel; -import java.nio.charset.CharsetDecoder; -import java.nio.charset.CodingErrorAction; -import java.nio.charset.StandardCharsets; +import java.io.RandomAccessFile; import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; import java.util.List; -import java.util.Map; import static com.riscure.trs.enums.TRSTag.*; -public class TraceSet implements AutoCloseable { - private static final String ERROR_READING_FILE = "Error reading TRS file: file size (%d) != meta data (%d) + trace size (%d) * nr of traces (%d)"; - private static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; - private static final String TRACE_SET_IN_WRITE_MODE = "TraceSet is in write mode. Please open the TraceSet in read mode."; - private static final String TRACE_INDEX_OUT_OF_BOUNDS = "Requested trace index (%d) is larger than the total number of available traces (%d)."; - private static final String TRACE_SET_IN_READ_MODE = "TraceSet is in read mode. Please open the TraceSet in write mode."; - private static final String TRACE_LENGTH_DIFFERS = "All traces in a set need to be the same length, but current trace length (%d) differs from the previous trace(s) (%d)"; - private static final String TRACE_DATA_LENGTH_DIFFERS = "All traces in a set need to have the same data length, but current trace data length (%d) differs from the previous trace(s) (%d)"; - private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; - private static final long MAX_BUFFER_SIZE = Integer.MAX_VALUE; - private static final String PARAMETER_NOT_DEFINED = "Parameter %s is saved in the trace, but was not found in the header definition"; - - //Reading variables - private int metaDataSize; - private FileInputStream readStream; - private FileChannel channel; - - private ByteBuffer buffer; - - private long bufferStart; //the byte index of the file where the buffer window starts - private long bufferSize; //the number of bytes that are in the buffer window - private long fileSize; //the total number of bytes in the underlying file - - //Writing variables - private FileOutputStream writeStream; - - private boolean firstTrace = true; +public abstract class TraceSet implements AutoCloseable { + protected static final String TRACE_SET_NOT_OPEN = "TraceSet has not been opened or has been closed."; + // We want to pre-allocate 256k for the header, so we can grow it if needed without re-writing the whole file + public static final long DEFAULT_METADATA_SIZE = 256_000L; //Shared variables - private final TRSMetaData metaData; - private final boolean writing; //whether the trace is opened in write mode private final Path path; - private final CharsetDecoder utf8Decoder = StandardCharsets.UTF_8.newDecoder(); - private boolean open; - private TraceSet(String inputFileName) throws IOException, TRSFormatException { - this.writing = false; + protected TraceSet(Path path) { + this.path = path; this.open = true; - this.path = Paths.get(inputFileName); - this.readStream = new FileInputStream(inputFileName); - this.channel = readStream.getChannel(); - - //the file might be bigger than the buffer, in which case we partially buffer it in memory - this.fileSize = this.channel.size(); - this.bufferStart = 0L; - this.bufferSize = Math.min(fileSize, MAX_BUFFER_SIZE); - - mapBuffer(); - this.metaData = TRSMetaDataUtils.readTRSMetaData(buffer); - this.metaDataSize = buffer.position(); - } - - private TraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { - this.open = true; - this.writing = true; - this.metaData = metaData; - this.path = Paths.get(outputFileName); - this.writeStream = new FileOutputStream(outputFileName); } /** @@ -93,27 +31,11 @@ public Path getPath() { return path; } - private void mapBuffer() throws IOException { - this.buffer = this.channel.map(FileChannel.MapMode.READ_ONLY, this.bufferStart, this.bufferSize); - } - - private void moveBufferIfNecessary(int traceIndex) throws IOException { - long traceSize = calculateTraceSize(); - long start = metaDataSize + (long) traceIndex * traceSize; - long end = start + traceSize; - - boolean moveRequired = start < this.bufferStart || this.bufferStart + this.bufferSize < end; - if (moveRequired) { - this.bufferStart = start; - this.bufferSize = Math.min(this.fileSize - start, MAX_BUFFER_SIZE); - this.mapBuffer(); - } - } - - private long calculateTraceSize() { - int sampleSize = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)).getSize(); - long sampleSpace = metaData.getInt(NUMBER_OF_SAMPLES) * (long) sampleSize; - return sampleSpace + metaData.getInt(DATA_LENGTH) + metaData.getInt(TITLE_SPACE); + /** + * @return whether this trace set is currently open + */ + public boolean isOpen() { + return open; } /** @@ -123,56 +45,7 @@ private long calculateTraceSize() { * @throws IOException if a read error occurs * @throws IllegalArgumentException if this TraceSet is not ready be read from */ - public Trace get(int index) throws IOException { - if (!open) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); - if (writing) throw new IllegalArgumentException(TRACE_SET_IN_WRITE_MODE); - - long traceSize = calculateTraceSize(); - long nrOfTraces = this.metaData.getInt(NUMBER_OF_TRACES); - if (index >= nrOfTraces) { - String msg = String.format(TRACE_INDEX_OUT_OF_BOUNDS, index, nrOfTraces); - throw new IllegalArgumentException(msg); - } - - long calculatedFileSize = metaDataSize + traceSize * nrOfTraces; - if (fileSize != calculatedFileSize) { - String msg = String.format(ERROR_READING_FILE, fileSize, metaDataSize, traceSize, nrOfTraces); - throw new IllegalStateException(msg); - } - - moveBufferIfNecessary(index); - - long absolutePosition = metaDataSize + index * traceSize; - buffer.position((int) (absolutePosition - this.bufferStart)); - - String traceTitle = this.readTraceTitle(); - if (traceTitle.trim().isEmpty()) { - traceTitle = String.format("%s %d", metaData.getString(GLOBAL_TITLE), index); - } - - try { - TraceParameterMap traceParameterMap; - if (metaData.getInt(TRS_VERSION) > 1) { - TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); - int size = traceParameterDefinitionMap.totalSize(); - byte[] data = new byte[size]; - buffer.get(data); - traceParameterMap = TraceParameterMap.deserialize(data, traceParameterDefinitionMap); - } else { - //legacy mode - byte[] data = readData(); - traceParameterMap = new TraceParameterMap(); - if (data.length > 0) { - traceParameterMap.put("LEGACY_DATA", data); - } - } - - float[] samples = readSamples(); - return new Trace(traceTitle, samples, traceParameterMap); - } catch (TRSFormatException ex) { - throw new IOException(ex); - } - } + public abstract Trace get(int index) throws IOException; /** * Add a trace to a writable TraceSet @@ -180,258 +53,18 @@ public Trace get(int index) throws IOException { * @throws IOException if any write error occurs * @throws TRSFormatException if the formatting of the trace is invalid */ - public void add(Trace trace) throws IOException, TRSFormatException { - if (!open) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); - if (!writing) throw new IllegalArgumentException(TRACE_SET_IN_READ_MODE); - if (firstTrace) { - int dataLength = trace.getData() == null ? 0 : trace.getData().length; - int titleLength = trace.getTitle() == null ? 0 : trace.getTitle().getBytes(StandardCharsets.UTF_8).length; - metaData.put(NUMBER_OF_SAMPLES, trace.getNumberOfSamples(), false); - metaData.put(DATA_LENGTH, dataLength, false); - metaData.put(TITLE_SPACE, titleLength, false); - metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false); - metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters())); - TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); - firstTrace = false; - } - truncateStrings(trace, metaData); - checkValid(trace); - - trace.setTraceSet(this); - writeTrace(trace); - - int numberOfTraces = metaData.getInt(NUMBER_OF_TRACES); - metaData.put(NUMBER_OF_TRACES, numberOfTraces + 1); - } - - /** - * This method makes sure that the trace title and any added string parameters adhere to the preset maximum length - * @param trace the trace to update - * @param metaData the metadata specifying the maximum string lengths - */ - private void truncateStrings(Trace trace, TRSMetaData metaData) { - int titleSpace = metaData.getInt(TITLE_SPACE); - trace.setTitle(fitUtf8StringToByteLength(trace.getTitle(), titleSpace)); - TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); - for (Map.Entry> definition : traceParameterDefinitionMap.entrySet()) { - TraceParameterDefinition value = definition.getValue(); - String key = definition.getKey(); - if (value.getType() == ParameterType.STRING) { - short stringLength = value.getLength(); - String stringValue = ((StringParameter) trace.getParameters().get(key)).getValue(); - if (stringLength != stringValue.getBytes(StandardCharsets.UTF_8).length) { - trace.getParameters().put(key, fitUtf8StringToByteLength(stringValue, stringLength)); - } - } - } - } - - /** - * Fits a string to the number of characters that fit in X bytes avoiding multi byte characters being cut in - * half at the cut off point. Also handles surrogate pairs where 2 characters in the string is actually one literal - * character. If the string is too long, it is truncated. If it's too short, it's padded with NUL characters. - * @param s the string to fit - * @param maxBytes the number of bytes required - */ - private String fitUtf8StringToByteLength(String s, int maxBytes) { - if (s == null) { - return null; - } - byte[] sba = s.getBytes(StandardCharsets.UTF_8); - if (sba.length <= maxBytes) { - return new String(Arrays.copyOf(sba, maxBytes)); - } - // Ensure truncation by having byte buffer = maxBytes - ByteBuffer bb = ByteBuffer.wrap(sba, 0, maxBytes); - CharBuffer cb = CharBuffer.allocate(maxBytes); - // Ignore an incomplete character - utf8Decoder.reset(); - utf8Decoder.onMalformedInput(CodingErrorAction.IGNORE); - utf8Decoder.decode(bb, cb, true); - utf8Decoder.flush(cb); - return new String(cb.array(), 0, cb.position()); - } - - private void writeTrace(Trace trace) throws TRSFormatException, IOException { - String title = trace.getTitle() == null ? "" : trace.getTitle(); - writeStream.write(title.getBytes(StandardCharsets.UTF_8)); - byte[] data = trace.getData() == null ? new byte[0] : trace.getData(); - writeStream.write(data); - Encoding encoding = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)); - writeStream.write(toByteArray(trace.getSample(), encoding)); - } - - private byte[] toByteArray(float[] samples, Encoding encoding) throws TRSFormatException { - byte[] result; - switch (encoding) { - case ILLEGAL: - throw new TRSFormatException("Illegal sample encoding"); - case BYTE: - result = new byte[samples.length]; - for (int k = 0; k < samples.length; k++) { - if (samples[k] != (byte)samples[k]) throw new IllegalArgumentException("Byte sample encoding too small"); - result[k] = (byte) samples[k]; - } - break; - case SHORT: - result = new byte[samples.length * 2]; - for (int k = 0; k < samples.length; k++) { - if (samples[k] != (short)samples[k]) throw new IllegalArgumentException("Short sample encoding too small"); - short value = (short) samples[k]; - result[2*k] = (byte) value; - result[2*k + 1] = (byte) (value >> 8); - } - break; - case INT: - result = new byte[samples.length * 4]; - for (int k = 0; k < samples.length; k++) { - int value = (int) samples[k]; - result[4*k] = (byte) value; - result[4*k + 1] = (byte) (value >> 8); - result[4*k + 2] = (byte) (value >> 16); - result[4*k + 3] = (byte) (value >> 24); - } - break; - case FLOAT: - result = new byte[samples.length * 4]; - for (int k = 0; k < samples.length; k++) { - int value = Float.floatToIntBits(samples[k]); - result[4*k] = (byte) value; - result[4*k + 1] = (byte) (value >> 8); - result[4*k + 2] = (byte) (value >> 16); - result[4*k + 3] = (byte) (value >> 24); - } - break; - default: - throw new TRSFormatException(String.format("Sample encoding not supported: %s", encoding.name())); - } - return result; - } + public abstract void add(Trace trace) throws IOException, TRSFormatException; @Override public void close() throws IOException, TRSFormatException { open = false; - if (writing) closeWriter(); - else closeReader(); - } - - private void checkValid(Trace trace) { - int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); - if (metaData.getInt(NUMBER_OF_SAMPLES) != trace.getNumberOfSamples()) { - throw new IllegalArgumentException(String.format(TRACE_LENGTH_DIFFERS, - trace.getNumberOfSamples(), - numberOfSamples)); - } - - int dataLength = metaData.getInt(DATA_LENGTH); - int traceDataLength = trace.getData() == null ? 0 : trace.getData().length; - if (metaData.getInt(DATA_LENGTH) != traceDataLength) { - throw new IllegalArgumentException(String.format(TRACE_DATA_LENGTH_DIFFERS, - traceDataLength, - dataLength)); - } - - for (Map.Entry entry : trace.getParameters().entrySet()) { - if (!metaData.getTraceParameterDefinitions().containsKey(entry.getKey())) { - throw new IllegalArgumentException(String.format(PARAMETER_NOT_DEFINED, entry.getKey())); - } - } - } - - private void closeReader() throws IOException { - buffer = null; - readStream.close(); - } - - private void closeWriter() throws IOException, TRSFormatException { - try { - //reset writer to start of file and overwrite header - writeStream.getChannel().position(0); - TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); - writeStream.flush(); - } finally { - writeStream.close(); - } } /** * Get the metadata associated with this trace set * @return the metadata associated with this trace set */ - public TRSMetaData getMetaData() { - return metaData; - } - - protected String readTraceTitle() { - byte[] titleArray = new byte[metaData.getInt(TITLE_SPACE)]; - buffer.get(titleArray); - return new String(titleArray); - } - - protected byte[] readData() { - int inputSize = metaData.getInt(DATA_LENGTH); - byte[] comDataArray = new byte[inputSize]; - buffer.get(comDataArray); - return comDataArray; - } - - protected float[] readSamples() throws TRSFormatException { - buffer.order(ByteOrder.LITTLE_ENDIAN); - int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); - float[] samples; - switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { - case BYTE: - byte[] byteData = new byte[numberOfSamples]; - buffer.get(byteData); - samples = toFloatArray(byteData); - break; - case SHORT: - ShortBuffer shortView = buffer.asShortBuffer(); - short[] shortData = new short[numberOfSamples]; - shortView.get(shortData); - samples = toFloatArray(shortData); - break; - case FLOAT: - FloatBuffer floatView = buffer.asFloatBuffer(); - samples = new float[numberOfSamples]; - floatView.get(samples); - break; - case INT: - IntBuffer intView = buffer.asIntBuffer(); - int[] intData = new int[numberOfSamples]; - intView.get(intData); - samples = toFloatArray(intData); - break; - default: - throw new TRSFormatException(String.format(UNKNOWN_SAMPLE_CODING, metaData.getInt(SAMPLE_CODING))); - } - - return samples; - } - - private float[] toFloatArray(byte[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = numbers[k]; - } - return result; - } - - private float[] toFloatArray(int[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = (float) numbers[k]; - } - return result; - } - - private float[] toFloatArray(short[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = numbers[k]; - } - return result; - } + public abstract TRSMetaData getMetaData(); /** * Factory method. This creates a new open TraceSet for reading. @@ -443,7 +76,7 @@ private float[] toFloatArray(short[] numbers) { * @throws TRSFormatException when any incorrect formatting of the TRS file is encountered */ public static TraceSet open(String file) throws IOException, TRSFormatException { - return new TraceSet(file); + return new ReadOnlyTraceSet(file); } /** @@ -508,7 +141,36 @@ public static TraceSet create(String file) throws IOException { * @throws IOException if the file creation failed */ public static TraceSet create(String file, TRSMetaData metaData) throws IOException { - metaData.put(TRS_VERSION, 2, false); - return new TraceSet(file, metaData); + metaData.put(TRS_VERSION, 3, false); + return new WritableTraceSet(file, metaData); + } + + /** + * Overwrite the metadata associated with this trace set + * If this traceset is in read mode, this is only possible under certain conditions: + * 1) The opened trace set is a V3 set + * 2) There is empty remaining space (i.e. padding) in the pre-allocated metadata + * + * TODO: We should probably limit the changes to specific tags. e.g. the number of traces should not be modified, + * TODO: but the TSPM is fine. The definition map may be updated, but the size must remain the same + */ + public static void updateParameterMaps(String file, TraceSetParameterMap tspm, TraceParameterDefinitionMap tpdm) throws IOException, TRSFormatException { + TRSMetaData metaData; + try (TraceSet ts = open(file)) { + metaData = ts.getMetaData(); + } + + if (metaData.getInt(TRS_VERSION) < 3) throw new IOException(String.format("This trace set is version %d. Only version 3 and upwards support updating metadata.", metaData.getInt(TRS_VERSION))); + // TODO check this + //if (metaDataSize > DEFAULT_METADATA_SIZE) throw new IOException("The meta data has already grown beyond the padding size. This trace set does not support updating the meta data."); + if (metaData.getTraceParameterDefinitions().totalSize() != tpdm.totalSize()) throw new IOException("The provided parameter definitions are of a different size than the current ones. While it's possible to change the definitions, the size must match."); + + metaData.put(TRACE_SET_PARAMETERS, tspm); + metaData.put(TRACE_PARAMETER_DEFINITIONS, tpdm); + + // Open the file in append mode so we can overwrite the header only + try (RandomAccessFile raf = new RandomAccessFile(file, "rw")) { + TRSMetaDataUtils.writeTRSMetaData(raf, metaData); + } } } diff --git a/src/main/java/com/riscure/trs/WritableTraceSet.java b/src/main/java/com/riscure/trs/WritableTraceSet.java new file mode 100644 index 0000000..050436c --- /dev/null +++ b/src/main/java/com/riscure/trs/WritableTraceSet.java @@ -0,0 +1,220 @@ +package com.riscure.trs; + +import com.riscure.trs.enums.Encoding; +import com.riscure.trs.enums.ParameterType; +import com.riscure.trs.parameter.TraceParameter; +import com.riscure.trs.parameter.primitive.StringParameter; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; +import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Map; + +import static com.riscure.trs.enums.TRSTag.*; + +public class WritableTraceSet extends TraceSet { + private static final String TRACE_SET_IN_WRITE_MODE = "TraceSet is in write mode. Please open the TraceSet in read mode."; + private static final String TRACE_LENGTH_DIFFERS = "All traces in a set need to be the same length, but current trace length (%d) differs from the previous trace(s) (%d)"; + private static final String TRACE_DATA_LENGTH_DIFFERS = "All traces in a set need to have the same data length, but current trace data length (%d) differs from the previous trace(s) (%d)"; + private static final String PARAMETER_NOT_DEFINED = "Parameter %s is saved in the trace, but was not found in the header definition"; + + private final CharsetDecoder utf8Decoder = StandardCharsets.UTF_8.newDecoder(); + private final TRSMetaData metaData; + private final FileOutputStream writeStream; + + private boolean firstTrace = true; + + WritableTraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { + super(Paths.get(outputFileName)); + this.metaData = metaData.modifiable(); + this.writeStream = new FileOutputStream(outputFileName); + } + + @Override + public Trace get(int index) throws IOException { + throw new IllegalArgumentException(TRACE_SET_IN_WRITE_MODE); + } + + @Override + public void add(Trace trace) throws IOException, TRSFormatException { + if (!isOpen()) throw new IllegalArgumentException(TRACE_SET_NOT_OPEN); + if (firstTrace) { + int dataLength = trace.getData() == null ? 0 : trace.getData().length; + int titleLength = trace.getTitle() == null ? 0 : trace.getTitle().getBytes(StandardCharsets.UTF_8).length; + metaData.put(NUMBER_OF_SAMPLES, trace.getNumberOfSamples(), false); + metaData.put(DATA_LENGTH, dataLength, false); + metaData.put(TITLE_SPACE, titleLength, false); + metaData.put(SAMPLE_CODING, trace.getPreferredCoding(), false); + metaData.put(TRACE_PARAMETER_DEFINITIONS, TraceParameterDefinitionMap.createFrom(trace.getParameters())); + metaData.put(NUMBER_OF_TRACES, 0); + TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); + firstTrace = false; + } + truncateStrings(trace, metaData); + checkValid(trace); + + trace.setTraceSet(this); + writeTrace(trace); + + int numberOfTraces = metaData.getInt(NUMBER_OF_TRACES); + metaData.put(NUMBER_OF_TRACES, numberOfTraces + 1); + } + + @Override + public TRSMetaData getMetaData() { + return metaData; + } + + @Override + public void close() throws IOException, TRSFormatException { + super.close(); + closeWriter(); + } + + private void closeWriter() throws IOException, TRSFormatException { + try { + //reset writer to start of file and overwrite header + writeStream.getChannel().position(0); + TRSMetaDataUtils.writeTRSMetaData(writeStream, metaData); + writeStream.flush(); + } finally { + writeStream.close(); + } + } + + /** + * This method makes sure that the trace title and any added string parameters adhere to the preset maximum length + * @param trace the trace to update + * @param metaData the metadata specifying the maximum string lengths + */ + private void truncateStrings(Trace trace, TRSMetaData metaData) { + int titleSpace = metaData.getInt(TITLE_SPACE); + trace.setTitle(fitUtf8StringToByteLength(trace.getTitle(), titleSpace)); + TraceParameterDefinitionMap traceParameterDefinitionMap = metaData.getTraceParameterDefinitions(); + for (Map.Entry> definition : traceParameterDefinitionMap.entrySet()) { + TraceParameterDefinition value = definition.getValue(); + String key = definition.getKey(); + if (value.getType() == ParameterType.STRING) { + short stringLength = value.getLength(); + String stringValue = ((StringParameter) trace.getParameters().get(key)).getValue(); + if (stringLength != stringValue.getBytes(StandardCharsets.UTF_8).length) { + trace.getParameters().put(key, fitUtf8StringToByteLength(stringValue, stringLength)); + } + } + } + } + + /** + * Fits a string to the number of characters that fit in X bytes avoiding multi byte characters being cut in + * half at the cut off point. Also handles surrogate pairs where 2 characters in the string is actually one literal + * character. If the string is too long, it is truncated. If it's too short, it's padded with NUL characters. + * @param s the string to fit + * @param maxBytes the number of bytes required + */ + private String fitUtf8StringToByteLength(String s, int maxBytes) { + if (s == null) { + return null; + } + byte[] sba = s.getBytes(StandardCharsets.UTF_8); + if (sba.length <= maxBytes) { + return new String(Arrays.copyOf(sba, maxBytes)); + } + // Ensure truncation by having byte buffer = maxBytes + ByteBuffer bb = ByteBuffer.wrap(sba, 0, maxBytes); + CharBuffer cb = CharBuffer.allocate(maxBytes); + // Ignore an incomplete character + utf8Decoder.reset(); + utf8Decoder.onMalformedInput(CodingErrorAction.IGNORE); + utf8Decoder.decode(bb, cb, true); + utf8Decoder.flush(cb); + return new String(cb.array(), 0, cb.position()); + } + + private void writeTrace(Trace trace) throws TRSFormatException, IOException { + String title = trace.getTitle() == null ? "" : trace.getTitle(); + writeStream.write(title.getBytes(StandardCharsets.UTF_8)); + byte[] data = trace.getData() == null ? new byte[0] : trace.getData(); + writeStream.write(data); + Encoding encoding = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)); + writeStream.write(toByteArray(trace.getSample(), encoding)); + } + + private byte[] toByteArray(float[] samples, Encoding encoding) throws TRSFormatException { + byte[] result; + switch (encoding) { + case ILLEGAL: + throw new TRSFormatException("Illegal sample encoding"); + case BYTE: + result = new byte[samples.length]; + for (int k = 0; k < samples.length; k++) { + if (samples[k] != (byte)samples[k]) throw new IllegalArgumentException("Byte sample encoding too small"); + result[k] = (byte) samples[k]; + } + break; + case SHORT: + result = new byte[samples.length * 2]; + for (int k = 0; k < samples.length; k++) { + if (samples[k] != (short)samples[k]) throw new IllegalArgumentException("Short sample encoding too small"); + short value = (short) samples[k]; + result[2*k] = (byte) value; + result[2*k + 1] = (byte) (value >> 8); + } + break; + case INT: + result = new byte[samples.length * 4]; + for (int k = 0; k < samples.length; k++) { + int value = (int) samples[k]; + result[4*k] = (byte) value; + result[4*k + 1] = (byte) (value >> 8); + result[4*k + 2] = (byte) (value >> 16); + result[4*k + 3] = (byte) (value >> 24); + } + break; + case FLOAT: + result = new byte[samples.length * 4]; + for (int k = 0; k < samples.length; k++) { + int value = Float.floatToIntBits(samples[k]); + result[4*k] = (byte) value; + result[4*k + 1] = (byte) (value >> 8); + result[4*k + 2] = (byte) (value >> 16); + result[4*k + 3] = (byte) (value >> 24); + } + break; + default: + throw new TRSFormatException(String.format("Sample encoding not supported: %s", encoding.name())); + } + return result; + } + + private void checkValid(Trace trace) { + int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); + if (metaData.getInt(NUMBER_OF_SAMPLES) != trace.getNumberOfSamples()) { + throw new IllegalArgumentException(String.format(TRACE_LENGTH_DIFFERS, + trace.getNumberOfSamples(), + numberOfSamples)); + } + + int dataLength = metaData.getInt(DATA_LENGTH); + int traceDataLength = trace.getData() == null ? 0 : trace.getData().length; + if (metaData.getInt(DATA_LENGTH) != traceDataLength) { + throw new IllegalArgumentException(String.format(TRACE_DATA_LENGTH_DIFFERS, + traceDataLength, + dataLength)); + } + + for (Map.Entry entry : trace.getParameters().entrySet()) { + if (!metaData.getTraceParameterDefinitions().containsKey(entry.getKey())) { + throw new IllegalArgumentException(String.format(PARAMETER_NOT_DEFINED, entry.getKey())); + } + } + } +} diff --git a/src/main/java/com/riscure/trs/enums/TRSTag.java b/src/main/java/com/riscure/trs/enums/TRSTag.java index 32038dd..e99ea33 100644 --- a/src/main/java/com/riscure/trs/enums/TRSTag.java +++ b/src/main/java/com/riscure/trs/enums/TRSTag.java @@ -54,7 +54,8 @@ public enum TRSTag { XY_SCAN_HEIGHT (0x74, "HE", false, Integer.class, 4, 0, "Number of steps in the \"y\" direction during XY scan"), XY_MEASUREMENTS_PER_SPOT (0x75, "ME", false, Integer.class, 4, 0, "Number of consecutive measurements done per spot during XY scan"), TRACE_SET_PARAMETERS (0x76, "GP", false, TraceSetParameterMap.class, 0, UnmodifiableTraceSetParameterMap.of(new TraceSetParameterMap()), "The set of custom global trace set parameters"), - TRACE_PARAMETER_DEFINITIONS (0x77, "LP", false, TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()), "The set of custom local trace parameters"); + TRACE_PARAMETER_DEFINITIONS (0x77, "LP", false, TraceParameterDefinitionMap.class, 0, UnmodifiableTraceParameterDefinitionMap.of(new TraceParameterDefinitionMap()), "The set of custom local trace parameters"), + PADDING (0xFF, "FF", false, Integer.class, 0, 0, "Empty value to allow growing the metadata"); private static final String UNKNOWN_TAG = "Unknown tag: 0x%X"; diff --git a/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java b/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java index dedd43d..ddd1af4 100644 --- a/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java +++ b/src/main/java/com/riscure/trs/parameter/trace/definition/TraceParameterDefinitionMap.java @@ -1,5 +1,6 @@ package com.riscure.trs.parameter.trace.definition; +import com.riscure.trs.TRSFormatException; import com.riscure.trs.TRSMetaDataUtils; import com.riscure.trs.io.LittleEndianInputStream; import com.riscure.trs.io.LittleEndianOutputStream; @@ -9,7 +10,10 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.Map; @@ -18,6 +22,7 @@ * This explicitly implements LinkedHashMap to ensure that the data is retrieved in the same order as it was added */ public class TraceParameterDefinitionMap extends LinkedHashMap> { + private static final String NAME_TOO_LONG = "Name of length %d exceeds maximum length of %d bytes%nName will be truncated to the maximum length%n"; public TraceParameterDefinitionMap() { super(); @@ -43,7 +48,7 @@ public int totalSize() { * @return this map converted to a byte array, serialized according to the TRS V2 standard definition * @throws RuntimeException if the map failed to serialize correctly */ - public byte[] serialize() { + public byte[] serialize() throws IOException, TRSFormatException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (LittleEndianOutputStream dos = new LittleEndianOutputStream(baos)) { //Write NE @@ -51,6 +56,12 @@ public byte[] serialize() { for (Map.Entry> entry : entrySet()) { byte[] nameBytes = entry.getKey().getBytes(StandardCharsets.UTF_8); //Write NL + if (nameBytes.length > Short.MAX_VALUE) { + System.err.printf(NAME_TOO_LONG, nameBytes.length, Short.MAX_VALUE); + nameBytes = new byte[Short.MAX_VALUE]; + CharBuffer name = CharBuffer.wrap(entry.getKey()); + StandardCharsets.UTF_8.newEncoder().encode(name, ByteBuffer.wrap(nameBytes), true); + } dos.writeShort(nameBytes.length); //Write N dos.write(nameBytes); @@ -59,8 +70,6 @@ public byte[] serialize() { } dos.flush(); return baos.toByteArray(); - } catch (IOException ex) { - throw new RuntimeException(ex); } } diff --git a/src/test/java/TestTraceSet.java b/src/test/java/TestTraceSet.java index ba6e132..29e3984 100644 --- a/src/test/java/TestTraceSet.java +++ b/src/test/java/TestTraceSet.java @@ -11,12 +11,12 @@ import com.riscure.trs.parameter.trace.TraceParameterMap; import com.riscure.trs.parameter.trace.definition.TraceParameterDefinition; import com.riscure.trs.parameter.trace.definition.TraceParameterDefinitionMap; -import com.riscure.trs.parameter.traceset.TraceSetParameter; import com.riscure.trs.parameter.traceset.TraceSetParameterMap; import com.riscure.trs.types.*; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.opentest4j.AssertionFailedError; import java.io.ByteArrayInputStream; @@ -26,10 +26,11 @@ import java.nio.file.Path; import java.util.*; +import static com.riscure.trs.TraceSet.DEFAULT_METADATA_SIZE; import static org.junit.jupiter.api.Assertions.*; -public class TestTraceSet { +class TestTraceSet { private static Path tempDir; private static final String BYTES_TRS = "bytes.trs"; private static final String SHORTS_TRS = "shorts.trs"; @@ -48,25 +49,25 @@ public class TestTraceSet { public static void createTempDir() throws IOException, TRSFormatException { tempDir = Files.createTempDirectory("TestTraceSet"); - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(BYTE_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + SHORTS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + SHORTS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(SHORT_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + INTS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + INTS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(INT_SAMPLES)); } } - try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + FLOATS_TRS)) { + try (TraceSet writable = TraceSet.create(tempDir.toAbsolutePath() + File.separator + FLOATS_TRS)) { for (int k = 0; k < NUMBER_OF_TRACES; k++) { writable.add(Trace.create(FLOAT_SAMPLES)); } @@ -74,29 +75,24 @@ public static void createTempDir() throws IOException, TRSFormatException { } @AfterAll - public static void cleanup() throws InterruptedException { - //We need to allow a little time for java to release all handles - System.gc(); - Thread.sleep(100); + public static void cleanup() { for (File file : Objects.requireNonNull(tempDir.toFile().listFiles())) { try { Files.delete(file.toPath()); } catch (IOException e) { - System.err.printf("Failed to delete temporary file '%s'%n", file.toPath().toAbsolutePath().toString()); - e.printStackTrace(); + System.err.printf("Failed to delete temporary file '%s'%n", file.toPath().toAbsolutePath()); } } try { Files.delete(tempDir); } catch (IOException e) { - System.err.printf("Failed to delete temporary folder '%s'%n", tempDir.toFile().toPath().toAbsolutePath().toString()); - e.printStackTrace(); + System.err.printf("Failed to delete temporary folder '%s'%n", tempDir.toFile().toPath().toAbsolutePath()); } } @Test void testOpenBytes() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.BYTE, encoding); @@ -111,7 +107,7 @@ void testOpenBytes() throws IOException, TRSFormatException { @Test void testOpenShorts() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + SHORTS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + SHORTS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.SHORT, encoding); @@ -126,7 +122,7 @@ void testOpenShorts() throws IOException, TRSFormatException { @Test void testOpenInts() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + INTS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + INTS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.INT, encoding); @@ -141,7 +137,7 @@ void testOpenInts() throws IOException, TRSFormatException { @Test void testOpenFloats() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + FLOATS_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + FLOATS_TRS)) { int numberOfTracesRead = readable.getMetaData().getInt(TRSTag.NUMBER_OF_TRACES); Encoding encoding = Encoding.fromValue(readable.getMetaData().getInt(TRSTag.SAMPLE_CODING)); assertEquals(Encoding.FLOAT, encoding); @@ -157,13 +153,11 @@ void testOpenFloats() throws IOException, TRSFormatException { @Test void testUTF8Title() throws IOException, TRSFormatException { String title = "씨브 크레그스만"; - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet ts = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet ts = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name)) { ts.add(Trace.create(title, new float[0], new TraceParameterMap())); - } catch (TRSFormatException e) { - throw e; } - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertEquals(title, readable.get(0).getTitle()); } } @@ -171,9 +165,6 @@ void testUTF8Title() throws IOException, TRSFormatException { /** * This tests adding several different types of information to the trace set header. The three parameters are chosen * to match the three major cases: Strings, primitives, and arbitrary (serializable) objects. - * - * @throws IOException - * @throws TRSFormatException */ @Test void testWriteTraceSetParameters() throws IOException, TRSFormatException { @@ -198,35 +189,34 @@ void testWriteTraceSetParameters() throws IOException, TRSFormatException { //parameters.put("XYZ offset", XYZ_TEST_VALUE); metaData.put(TRSTag.TRACE_SET_PARAMETERS, parameters); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData).close(); + String name = UUID.randomUUID() + TRS; + TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData).close(); //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceSetParameterMap readTraceSetParameterMap = readable.getMetaData().getTraceSetParameters(); parameters.forEach((s, traceSetParameter) -> assertEquals(traceSetParameter, readTraceSetParameterMap.get(s))); } } /** - * This tests adding a parameter with a name of 100000 characters - * - * @throws IOException - * @throws TRSFormatException + * This tests adding a parameter with a name of 100000 characters. + * Expectation: The name will be truncated to the maximum allowed length when writing, + * when reading back and comparing with the original metadata, the values will differ */ @Test void testWriteTraceParametersInvalidName() throws IOException, TRSFormatException { TRSMetaData metaData = TRSMetaData.create(); String parameterName = String.format("%100000s", "XYZ"); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put(parameterName, 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - assertThrows(TRSFormatException.class, () -> { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + assertThrows(AssertionFailedError.class, () -> { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); parameterDefinitions.forEach((key, parameter) -> assertEquals(parameterName, key)); } @@ -238,9 +228,6 @@ void testWriteTraceParametersInvalidName() throws IOException, TRSFormatExceptio * - if no length is specified, the first string is leading * - if a string is longer than the length specified, it should be truncated * - when truncated, a string should still be valid UTF-8 (truncated at character level, not byte level) - * - * @throws IOException - * @throws TRSFormatException */ @Test void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormatException { @@ -252,8 +239,8 @@ void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormat strings.add("ab"); strings.add("abcdefgh汉字"); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { for (int k = 0; k < 25; k++) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTEARRAY", new byte[]{(byte) k, (byte) k, (byte) k}); @@ -269,17 +256,14 @@ void testWriteTraceParametersVaryingStringLength() throws IOException, TRSFormat /** * This tests whether all getters are working as expected - * - * @throws IOException - * @throws TRSFormatException */ @Test void testReadTraceParametersTyped() throws IOException, TRSFormatException { TRSMetaData metaData = TRSMetaData.create(); List testParameters = new ArrayList<>(); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { for (int k = 0; k < 25; k++) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTE", (byte) k); @@ -307,7 +291,7 @@ void testReadTraceParametersTyped() throws IOException, TRSFormatException { } private void readBackGeneric(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -322,7 +306,7 @@ private void readBackGeneric(List testParameters, String name } private void readBackTyped(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -391,7 +375,7 @@ private void readBackTyped(List testParameters, String name) } private void readBackTypedKeys(List testParameters, String name) throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterDefinitionMap parameterDefinitions = readable.getMetaData().getTraceParameterDefinitions(); for (int k = 0; k < 25; k++) { assertEquals(parameterDefinitions.size(), testParameters.get(k).size()); @@ -428,8 +412,8 @@ private void readBackTypedKeys(List testParameters, String na throw new RuntimeException("Unexpected type: " + parameter.getType()); } if (parameter.getLength() > 1 && typedKey.getCls().isArray()) { - assertArrayEquals(Arrays.asList(correctValue.getOrElseThrow(typedKey)).toArray(), - Arrays.asList(trace.getParameters().getOrElseThrow(typedKey)).toArray()); + assertArrayEquals(Collections.singletonList(correctValue.getOrElseThrow(typedKey)).toArray(), + Collections.singletonList(trace.getParameters().getOrElseThrow(typedKey)).toArray()); } else { assertEquals(correctValue.get(typedKey), trace.getParameters().get(typedKey)); } @@ -440,42 +424,37 @@ private void readBackTypedKeys(List testParameters, String na /** * This tests getting a value of the wrong type correctly throws an exception - * - * @throws IOException - * @throws TRSFormatException */ @Test void testExceptionWrongType() throws IOException, TRSFormatException { TRSMetaData metaData = TRSMetaData.create(); //CREATE TRACE - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name, metaData)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name, metaData)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put("BYTE", (byte) 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertThrows(ClassCastException.class, () -> readable.get(0).getParameters().getDouble("BYTE")); } } /** * This - * @throws IOException - * @throws TRSFormatException */ @Test void testContainsNonArray() throws IOException, TRSFormatException { ByteTypeKey byteKey = new ByteTypeKey("BYTE"); - String name = UUID.randomUUID().toString() + TRS; - try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath().toString() + File.separator + name)) { + String name = UUID.randomUUID() + TRS; + try (TraceSet traceWithParameters = TraceSet.create(tempDir.toAbsolutePath() + File.separator + name)) { TraceParameterMap parameters = new TraceParameterMap(); parameters.put(byteKey, (byte) 1); traceWithParameters.add(Trace.create("", FLOAT_SAMPLES, parameters)); } //READ BACK AND CHECK RESULT - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + name)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + name)) { assertTrue(readable.get(0).getParameters().get(byteKey).isPresent()); } } @@ -503,7 +482,7 @@ void testInvalidParameterLength() { */ @Test void testModificationAfterReadback() throws IOException, TRSFormatException { - try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath().toString() + File.separator + BYTES_TRS)) { + try (TraceSet readable = TraceSet.open(tempDir.toAbsolutePath() + File.separator + BYTES_TRS)) { assertThrows(UnsupportedOperationException.class, () -> readable.getMetaData().getTraceSetParameters().put("SHOULD_FAIL", 0)); assertThrows(UnsupportedOperationException.class, () -> readable.getMetaData().getTraceParameterDefinitions().put("SHOULD_FAIL", new TraceParameterDefinition(ParameterType.BYTE, (short)1, (short)1))); for (int k = 0; k < NUMBER_OF_TRACES; k++) { @@ -675,13 +654,50 @@ void testFileReleasing() throws IOException, TRSFormatException, InterruptedExce try (TraceSet traceSet = TraceSet.open(filePath)) { traceSet.getMetaData().getTraceSetParameters(); } - // Unfortunately, the current solution requires a garbage collect to have been performed before the issue is resolved. - // Other fixes required either a Java 8 Cleaner.clean() call not accessible from Java 21, or a Java 20 Arena.close(), - // which is not been finalized in Java 21. - System.gc(); - Thread.sleep(1000); // Assert that the opened file has been closed again, by deleting it. File file = new File(filePath); - assert(file.delete()); + assertTrue(file.delete()); + } + + /** + * This test checks whether version 3 correctly allocates 1MB of header space by default + */ + @Test + void testDefaultHeaderSize() throws IOException, TRSFormatException { + Path filePath = tempDir.resolve("large_header.trs"); + TRSMetaData metaData = new TRSMetaData(); + metaData.put(TRSTag.TRS_VERSION, 3); + try (TraceSet ts = TraceSet.create(filePath.toString(), metaData)) { + ts.add(new Trace(new float[]{0})); + } + assertTrue(filePath.toFile().length() > DEFAULT_METADATA_SIZE); + } + + /** + * This test checks whether we can successfully add information to the header of a traceset file without + * increasing its size + */ + @Test + void testOverwritingMetadata() throws IOException, TRSFormatException { + String filename = tempDir.toAbsolutePath() + File.separator + BYTES_TRS; + long originalFileSize = new File(filename).length(); + + TraceSetParameterMap tspm; + TraceParameterDefinitionMap tpdm; + try (TraceSet readable = TraceSet.open(filename)) { + assertFalse(readable.getMetaData().getTraceSetParameters().containsKey("test")); + + tspm = readable.getMetaData().getTraceSetParameters().copy(); + tpdm = readable.getMetaData().getTraceParameterDefinitions().copy(); + + tspm.put("test", "This value should exist afterwards"); + } + + TraceSet.updateParameterMaps(filename, tspm, tpdm); + + try (TraceSet readable = TraceSet.open(filename)) { + assertTrue(readable.getMetaData().getTraceSetParameters().containsKey("test")); + } + assertEquals(originalFileSize, new File(filename).length()); } }