From 7867c46da5990d28c9c012c759a5b7a561d40183 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Wed, 19 Feb 2025 15:04:53 +0100 Subject: [PATCH 1/3] #78: Added implementation for memory-mapping files beyond 2GB --- pom.xml | 6 +- .../com/riscure/trs/LargePreMappedFile.java | 105 ++++++++++++++++++ src/main/java/com/riscure/trs/TraceSet.java | 64 ++++------- 3 files changed, 131 insertions(+), 44 deletions(-) create mode 100644 src/main/java/com/riscure/trs/LargePreMappedFile.java diff --git a/pom.xml b/pom.xml index ada9def..c3112cc 100644 --- a/pom.xml +++ b/pom.xml @@ -36,9 +36,9 @@ - 8 - 8 - 8 + 11 + 11 + 11 UTF-8 diff --git a/src/main/java/com/riscure/trs/LargePreMappedFile.java b/src/main/java/com/riscure/trs/LargePreMappedFile.java new file mode 100644 index 0000000..4e9fac3 --- /dev/null +++ b/src/main/java/com/riscure/trs/LargePreMappedFile.java @@ -0,0 +1,105 @@ +package com.riscure.trs; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.List; + +public class LargePreMappedFile implements AutoCloseable{ + private final FileChannel channel; + + private final List buffers = new ArrayList<>(); + private final long readOffset; + private final long traceSize; + private final long fileSize; + + private long totalBufferSize; + + public LargePreMappedFile(FileChannel channel, long metaDataSize, long traceSize) throws IOException { + this.channel = channel; + this.readOffset = metaDataSize; + this.traceSize = traceSize; + this.fileSize = channel.size() - readOffset; + + mapBuffers(); + } + + public ByteBuffer getBuffer(int index) { + if (traceSize == 0) { + return ByteBuffer.wrap(new byte[0]); + } + return findBufferAndMoveToTrace(index); + } + + private ByteBuffer findBufferAndMoveToTrace(int traceIndex) { + MappedBuffer mappedBuffer = buffers.stream() + .filter(buffer -> traceIndex >= buffer.getFirstTraceIndex() && + traceIndex < buffer.getFirstTraceIndex() + buffer.getNumberOfTraces()) + .findFirst() + .orElseThrow(); + ByteBuffer buffer = mappedBuffer.getBuffer(); + int traceIndexInBuffer = traceIndex - mappedBuffer.getFirstTraceIndex(); + long positionInBuffer = traceIndexInBuffer * traceSize; + buffer.position((int) positionInBuffer); + return buffer; + } + + private void mapBuffers() { + if (traceSize > 0) { + int tracesPerBuffer = (int) (Integer.MAX_VALUE / traceSize); + long maximumBufferSize = tracesPerBuffer * traceSize; + + int firstTraceIndex = 0; + for (long offset = 0; offset < fileSize; offset += maximumBufferSize) { + buffers.add(mapBuffer(firstTraceIndex, maximumBufferSize)); + firstTraceIndex += tracesPerBuffer; + } + } + } + + private MappedBuffer mapBuffer(int firstTraceIndex, long bufferSize) { + try { + long bufferStart = firstTraceIndex * traceSize; + this.totalBufferSize += bufferSize; + long limitedBufferSize = Math.min(fileSize - bufferStart, bufferSize); + MappedBuffer mappedBuffer = new MappedBuffer(this.channel.map(FileChannel.MapMode.READ_ONLY, readOffset + bufferStart, limitedBufferSize), + firstTraceIndex, + (int) (traceSize > 0 ? (limitedBufferSize / traceSize) : 0)); + mappedBuffer.buffer.order(ByteOrder.LITTLE_ENDIAN); + return mappedBuffer; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() { + buffers.clear(); + } + + private static class MappedBuffer { + private final ByteBuffer buffer; + private final int firstTraceIndex; + private final int numberOfTraces; + + MappedBuffer(ByteBuffer buffer, int firstTraceIndex, int numberOfTraces) { + this.buffer = buffer; + this.firstTraceIndex = firstTraceIndex; + this.numberOfTraces = numberOfTraces; + } + + public ByteBuffer getBuffer() { + return buffer; + } + + public int getFirstTraceIndex() { + return firstTraceIndex; + } + + public int getNumberOfTraces() { + return numberOfTraces; + } + } +} diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index ee46acb..8ec00aa 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -34,18 +34,17 @@ public class TraceSet implements AutoCloseable { private static final String TRACE_LENGTH_DIFFERS = "All traces in a set need to be the same length, but current trace length (%d) differs from the previous trace(s) (%d)"; private static final String TRACE_DATA_LENGTH_DIFFERS = "All traces in a set need to have the same data length, but current trace data length (%d) differs from the previous trace(s) (%d)"; private static final String UNKNOWN_SAMPLE_CODING = "Error reading TRS file: unknown sample coding '%d'"; - private static final long MAX_BUFFER_SIZE = Integer.MAX_VALUE; private static final String PARAMETER_NOT_DEFINED = "Parameter %s is saved in the trace, but was not found in the header definition"; + // This is excessive for the header, but it's only the initial maximum + private static final long MAX_METADATA_SIZE = 100_000_000L; //Reading variables private int metaDataSize; private FileInputStream readStream; - private FileChannel channel; - private ByteBuffer buffer; + private ByteBuffer metaDataBuffer; + private LargePreMappedFile mappedFile; - private long bufferStart; //the byte index of the file where the buffer window starts - private long bufferSize; //the number of bytes that are in the buffer window private long fileSize; //the total number of bytes in the underlying file //Writing variables @@ -66,16 +65,19 @@ private TraceSet(String inputFileName) throws IOException, TRSFormatException { this.open = true; this.path = Paths.get(inputFileName); this.readStream = new FileInputStream(inputFileName); - this.channel = readStream.getChannel(); + FileChannel channel = readStream.getChannel(); //the file might be bigger than the buffer, in which case we partially buffer it in memory - this.fileSize = this.channel.size(); - this.bufferStart = 0L; - this.bufferSize = Math.min(fileSize, MAX_BUFFER_SIZE); + this.fileSize = channel.size(); + long initialBufferSize = Math.min(fileSize, MAX_METADATA_SIZE); - mapBuffer(); - this.metaData = TRSMetaDataUtils.readTRSMetaData(buffer); - this.metaDataSize = buffer.position(); + this.metaDataBuffer = channel.map(FileChannel.MapMode.READ_ONLY, 0, initialBufferSize); + this.metaData = TRSMetaDataUtils.readTRSMetaData(metaDataBuffer); + this.metaDataSize = metaDataBuffer.position(); + this.metaDataBuffer.limit(metaDataSize); + + long traceSize = calculateTraceSize(); + this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); } private TraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { @@ -93,23 +95,6 @@ public Path getPath() { return path; } - private void mapBuffer() throws IOException { - this.buffer = this.channel.map(FileChannel.MapMode.READ_ONLY, this.bufferStart, this.bufferSize); - } - - private void moveBufferIfNecessary(int traceIndex) throws IOException { - long traceSize = calculateTraceSize(); - long start = metaDataSize + (long) traceIndex * traceSize; - long end = start + traceSize; - - boolean moveRequired = start < this.bufferStart || this.bufferStart + this.bufferSize < end; - if (moveRequired) { - this.bufferStart = start; - this.bufferSize = Math.min(this.fileSize - start, MAX_BUFFER_SIZE); - this.mapBuffer(); - } - } - private long calculateTraceSize() { int sampleSize = Encoding.fromValue(metaData.getInt(SAMPLE_CODING)).getSize(); long sampleSpace = metaData.getInt(NUMBER_OF_SAMPLES) * (long) sampleSize; @@ -140,12 +125,9 @@ public Trace get(int index) throws IOException { throw new IllegalStateException(msg); } - moveBufferIfNecessary(index); - - long absolutePosition = metaDataSize + index * traceSize; - buffer.position((int) (absolutePosition - this.bufferStart)); + ByteBuffer buffer = mappedFile.getBuffer(index); - String traceTitle = this.readTraceTitle(); + String traceTitle = this.readTraceTitle(buffer); if (traceTitle.trim().isEmpty()) { traceTitle = String.format("%s %d", metaData.getString(GLOBAL_TITLE), index); } @@ -160,14 +142,14 @@ public Trace get(int index) throws IOException { traceParameterMap = TraceParameterMap.deserialize(data, traceParameterDefinitionMap); } else { //legacy mode - byte[] data = readData(); + byte[] data = readData(buffer); traceParameterMap = new TraceParameterMap(); if (data.length > 0) { traceParameterMap.put("LEGACY_DATA", data); } } - float[] samples = readSamples(); + float[] samples = readSamples(buffer); return new Trace(traceTitle, samples, traceParameterMap); } catch (TRSFormatException ex) { throw new IOException(ex); @@ -339,7 +321,8 @@ private void checkValid(Trace trace) { } private void closeReader() throws IOException { - buffer = null; + metaDataBuffer = null; + mappedFile.close(); readStream.close(); } @@ -362,21 +345,20 @@ public TRSMetaData getMetaData() { return metaData; } - protected String readTraceTitle() { + protected String readTraceTitle(ByteBuffer buffer) { byte[] titleArray = new byte[metaData.getInt(TITLE_SPACE)]; buffer.get(titleArray); return new String(titleArray); } - protected byte[] readData() { + protected byte[] readData(ByteBuffer buffer) { int inputSize = metaData.getInt(DATA_LENGTH); byte[] comDataArray = new byte[inputSize]; buffer.get(comDataArray); return comDataArray; } - protected float[] readSamples() throws TRSFormatException { - buffer.order(ByteOrder.LITTLE_ENDIAN); + protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); float[] samples; switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { From 70c43188cc3f9aef00a9fc88a6e83b77a1baf8a1 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Thu, 20 Feb 2025 13:23:26 +0100 Subject: [PATCH 2/3] #78: Added pre-allocating of trace buffers for efficiency --- .../com/riscure/trs/LargePreMappedFile.java | 3 - src/main/java/com/riscure/trs/TraceSet.java | 71 +++++++++---------- 2 files changed, 32 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/riscure/trs/LargePreMappedFile.java b/src/main/java/com/riscure/trs/LargePreMappedFile.java index 4e9fac3..a182e54 100644 --- a/src/main/java/com/riscure/trs/LargePreMappedFile.java +++ b/src/main/java/com/riscure/trs/LargePreMappedFile.java @@ -15,8 +15,6 @@ public class LargePreMappedFile implements AutoCloseable{ private final long traceSize; private final long fileSize; - private long totalBufferSize; - public LargePreMappedFile(FileChannel channel, long metaDataSize, long traceSize) throws IOException { this.channel = channel; this.readOffset = metaDataSize; @@ -62,7 +60,6 @@ private void mapBuffers() { private MappedBuffer mapBuffer(int firstTraceIndex, long bufferSize) { try { long bufferStart = firstTraceIndex * traceSize; - this.totalBufferSize += bufferSize; long limitedBufferSize = Math.min(fileSize - bufferStart, bufferSize); MappedBuffer mappedBuffer = new MappedBuffer(this.channel.map(FileChannel.MapMode.READ_ONLY, readOffset + bufferStart, limitedBufferSize), firstTraceIndex, diff --git a/src/main/java/com/riscure/trs/TraceSet.java b/src/main/java/com/riscure/trs/TraceSet.java index 8ec00aa..2de90ef 100644 --- a/src/main/java/com/riscure/trs/TraceSet.java +++ b/src/main/java/com/riscure/trs/TraceSet.java @@ -44,6 +44,10 @@ public class TraceSet implements AutoCloseable { private ByteBuffer metaDataBuffer; private LargePreMappedFile mappedFile; + private float[] preallocatedSampleArray; + private byte[] preallocatedByteArray; + private short[] preallocatedShortArray; + private int[] preallocatedIntArray; private long fileSize; //the total number of bytes in the underlying file @@ -78,6 +82,9 @@ private TraceSet(String inputFileName) throws IOException, TRSFormatException { long traceSize = calculateTraceSize(); this.mappedFile = new LargePreMappedFile(channel, metaDataSize, traceSize); + + int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); + this.preallocatedSampleArray = new float[numberOfSamples]; } private TraceSet(String outputFileName, TRSMetaData metaData) throws FileNotFoundException { @@ -150,7 +157,8 @@ public Trace get(int index) throws IOException { } float[] samples = readSamples(buffer); - return new Trace(traceTitle, samples, traceParameterMap); + // Since we are using an internal sample array in this class, Trace.create() should duplicate it internally + return Trace.create(traceTitle, samples, traceParameterMap); } catch (TRSFormatException ex) { throw new IOException(ex); } @@ -358,61 +366,46 @@ protected byte[] readData(ByteBuffer buffer) { return comDataArray; } + /* + * We can reuse the buffers when not dealing with float samples. They are instantiated once just in time if needed. + */ protected float[] readSamples(ByteBuffer buffer) throws TRSFormatException { - int numberOfSamples = metaData.getInt(NUMBER_OF_SAMPLES); - float[] samples; switch (Encoding.fromValue(metaData.getInt(SAMPLE_CODING))) { case BYTE: - byte[] byteData = new byte[numberOfSamples]; - buffer.get(byteData); - samples = toFloatArray(byteData); + this.preallocatedByteArray = this.preallocatedByteArray == null ? new byte[preallocatedSampleArray.length] : this.preallocatedByteArray; + buffer.get(preallocatedByteArray); + // Manual copy of byte[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedByteArray[k]; + } break; case SHORT: + this.preallocatedShortArray = this.preallocatedShortArray == null ? new short[preallocatedSampleArray.length] : this.preallocatedShortArray; ShortBuffer shortView = buffer.asShortBuffer(); - short[] shortData = new short[numberOfSamples]; - shortView.get(shortData); - samples = toFloatArray(shortData); + shortView.get(preallocatedShortArray); + // Manual copy of short[] into float[] + for (int k = 0; k < preallocatedSampleArray.length; k++) { + preallocatedSampleArray[k] = preallocatedShortArray[k]; + } break; case FLOAT: FloatBuffer floatView = buffer.asFloatBuffer(); - samples = new float[numberOfSamples]; - floatView.get(samples); + floatView.get(preallocatedSampleArray); break; case INT: + this.preallocatedIntArray = this.preallocatedIntArray == null ? new int[preallocatedSampleArray.length] : this.preallocatedIntArray; IntBuffer intView = buffer.asIntBuffer(); - int[] intData = new int[numberOfSamples]; - intView.get(intData); - samples = toFloatArray(intData); + intView.get(preallocatedIntArray); + // Manual copy of int[] into float[] + for (int k = 0; k < preallocatedIntArray.length; k++) { + preallocatedSampleArray[k] = (float) preallocatedIntArray[k]; + } break; default: throw new TRSFormatException(String.format(UNKNOWN_SAMPLE_CODING, metaData.getInt(SAMPLE_CODING))); } - return samples; - } - - private float[] toFloatArray(byte[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = numbers[k]; - } - return result; - } - - private float[] toFloatArray(int[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = (float) numbers[k]; - } - return result; - } - - private float[] toFloatArray(short[] numbers) { - float[] result = new float[numbers.length]; - for (int k = 0; k < numbers.length; k++) { - result[k] = numbers[k]; - } - return result; + return preallocatedSampleArray; } /** From e193bfa2acf2d557b95723d3f4167a3e41c5beb8 Mon Sep 17 00:00:00 2001 From: Siebe Krijgsman Date: Thu, 13 Aug 2026 16:09:50 +0200 Subject: [PATCH 3/3] #78: Added missing space --- src/main/java/com/riscure/trs/LargePreMappedFile.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/riscure/trs/LargePreMappedFile.java b/src/main/java/com/riscure/trs/LargePreMappedFile.java index a182e54..405f9e3 100644 --- a/src/main/java/com/riscure/trs/LargePreMappedFile.java +++ b/src/main/java/com/riscure/trs/LargePreMappedFile.java @@ -7,7 +7,7 @@ import java.util.ArrayList; import java.util.List; -public class LargePreMappedFile implements AutoCloseable{ +public class LargePreMappedFile implements AutoCloseable { private final FileChannel channel; private final List buffers = new ArrayList<>();