From 0aca26ce3c41f61e2234f7460cf128070774c0b9 Mon Sep 17 00:00:00 2001 From: dronov-dmitry Date: Mon, 17 Aug 2026 12:06:16 +0200 Subject: [PATCH] Detect text file encoding from byte order mark --- .../textpad/activities/EditorActivity.java | 42 +++++-- .../maxistar/textpad/utils/FileEncoding.java | 115 ++++++++++++++++++ .../textpad/utils/FileEncodingTest.java | 94 ++++++++++++++ 3 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java create mode 100644 app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java diff --git a/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java b/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java index dc2795b..f4795d2 100644 --- a/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java +++ b/app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java @@ -70,6 +70,7 @@ import com.maxistar.textpad.recovery.RecoveryWriter; import com.maxistar.textpad.utils.EditTextUndoRedo; import com.maxistar.textpad.utils.DocumentSaveValidator; +import com.maxistar.textpad.utils.FileEncoding; import com.maxistar.textpad.utils.FileNameHelper; import com.maxistar.textpad.utils.System; import com.maxistar.textpad.utils.TextConverter; @@ -130,6 +131,8 @@ public class EditorActivity extends AppCompatActivity { private ScrollView scrollView; private LinearLayout linearLayout; + private FileEncoding documentEncoding; + String urlFilename = TPStrings.EMPTY; Uri lastTriedSystemUri = null; @@ -261,6 +264,13 @@ private boolean simpleScrolling() { return settingsService.isUseSimpleScrolling(); } + private String resolveFileEncodingName() { + if (documentEncoding != null) { + return documentEncoding.getCharsetName(); + } + return settingsService.getFileEncoding(); + } + @RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM) private void applyEdgeToEdgeInsets() { View editorRoot = findViewById(R.id.editor_root); @@ -594,7 +604,7 @@ private RecoveryWriter.Snapshot createRecoverySnapshot() { identity, currentDisplayName(), identity == null, - settingsService.getFileEncoding(), + resolveFileEncodingName(), false, originalSize, originalLastModified, @@ -1159,6 +1169,7 @@ public void clearFile() { originalSize = null; originalLastModified = null; originalContentSha256 = null; + documentEncoding = null; selectionStart = 0; selectionEnd = 0; setEditorText(TPStrings.EMPTY, false); @@ -1340,7 +1351,7 @@ protected void saveFile(Uri uri) throws IOException { s = applyEndings(s); - outputStream.write(s.getBytes(settingsService.getFileEncoding())); + outputStream.write(FileEncoding.encode(s, documentEncoding, settingsService.getFileEncoding())); } finally { outputStream.close(); } @@ -1356,7 +1367,7 @@ private void guardedSaveNamedFile(boolean autosave) { SaveRequest request = new SaveRequest( editorGeneration, recoveryKey, - persistedText.getBytes(settingsService.getFileEncoding()) + FileEncoding.encode(persistedText, documentEncoding, settingsService.getFileEncoding()) ); boolean creatingDocument = nextSaveCreatesDocument || originalContentSha256 == null; nextSaveCreatesDocument = false; @@ -1551,8 +1562,11 @@ private void validateOpenDocumentOnForeground() { return; } - byte[] intendedBytes = applyEndings(mText.getText().toString()) - .getBytes(settingsService.getFileEncoding()); + byte[] intendedBytes = FileEncoding.encode( + applyEndings(mText.getText().toString()), + documentEncoding, + settingsService.getFileEncoding() + ); SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes); DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify( currentBytes, @@ -1572,7 +1586,8 @@ private void validateOpenDocumentOnForeground() { } private void applyExternalDocument(byte[] externalBytes) throws Exception { - String externalText = new String(externalBytes, settingsService.getFileEncoding()); + documentEncoding = FileEncoding.detect(externalBytes); + String externalText = FileEncoding.decode(externalBytes, documentEncoding, settingsService.getFileEncoding()); externalText = toUnixEndings(externalText); setEditorText(externalText, false); initEditor(); @@ -1588,8 +1603,11 @@ private void validateRestoredDraft() { return; } try { - byte[] intendedBytes = applyEndings(mText.getText().toString()) - .getBytes(settingsService.getFileEncoding()); + byte[] intendedBytes = FileEncoding.encode( + applyEndings(mText.getText().toString()), + documentEncoding, + settingsService.getFileEncoding() + ); SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes); byte[] currentBytes = readNamedDocumentBytes(); DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify( @@ -1664,9 +1682,8 @@ private void openNamedFileLegacyDirect(String filename) { dis.close(); fis.close(); - String ttt = new String(b, 0, length, - settingsService.getFileEncoding()); - + documentEncoding = FileEncoding.detect(b); + String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding()); ttt = toUnixEndings(ttt); setEditorText(ttt, false); @@ -1717,7 +1734,8 @@ private void openNamedFileDirect(final Uri uri) { } byte[] b = bytes.toByteArray(); - String ttt = new String(b, settingsService.getFileEncoding()); + documentEncoding = FileEncoding.detect(b); + String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding()); ttt = toUnixEndings(ttt); inputStream.close(); diff --git a/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java b/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java new file mode 100644 index 0000000..2e68a4c --- /dev/null +++ b/app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java @@ -0,0 +1,115 @@ +package com.maxistar.textpad.utils; + +import java.nio.charset.Charset; + +/** + * Detects a text file encoding from its byte order mark (BOM) and decodes or + * encodes the content keeping the original encoding. + */ +public class FileEncoding { + + public static final String UTF_8 = "UTF-8"; + public static final String UTF_16LE = "UTF-16LE"; + public static final String UTF_16BE = "UTF-16BE"; + public static final String UTF_32LE = "UTF-32LE"; + public static final String UTF_32BE = "UTF-32BE"; + + private static final byte[] BOM_UTF_32LE = {(byte) 0xFF, (byte) 0xFE, 0, 0}; + private static final byte[] BOM_UTF_32BE = {0, 0, (byte) 0xFE, (byte) 0xFF}; + private static final byte[] BOM_UTF_8 = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}; + private static final byte[] BOM_UTF_16BE = {(byte) 0xFE, (byte) 0xFF}; + private static final byte[] BOM_UTF_16LE = {(byte) 0xFF, (byte) 0xFE}; + + private final String charsetName; + private final byte[] bom; + + private FileEncoding(String charsetName, byte[] bom) { + this.charsetName = charsetName; + this.bom = bom; + } + + public String getCharsetName() { + return charsetName; + } + + public byte[] getBom() { + return bom; + } + + public boolean hasBom() { + return bom != null; + } + + public static FileEncoding detect(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return null; + } + if (startsWith(bytes, BOM_UTF_32LE)) { + return new FileEncoding(UTF_32LE, BOM_UTF_32LE); + } + if (startsWith(bytes, BOM_UTF_32BE)) { + return new FileEncoding(UTF_32BE, BOM_UTF_32BE); + } + if (startsWith(bytes, BOM_UTF_8)) { + return new FileEncoding(UTF_8, BOM_UTF_8); + } + if (startsWith(bytes, BOM_UTF_16BE)) { + return new FileEncoding(UTF_16BE, BOM_UTF_16BE); + } + if (startsWith(bytes, BOM_UTF_16LE)) { + return new FileEncoding(UTF_16LE, BOM_UTF_16LE); + } + return null; + } + + public static String decode(byte[] bytes, FileEncoding encoding, String fallbackCharsetName) { + if (bytes == null) { + return ""; + } + int offset = 0; + if (encoding != null && encoding.hasBom() && bytes.length >= encoding.getBom().length) { + offset = encoding.getBom().length; + } + String charsetName = encoding != null ? encoding.getCharsetName() : fallbackCharsetName; + try { + return new String(bytes, offset, bytes.length - offset, charsetForName(charsetName)); + } catch (Exception e) { + return new String(bytes); + } + } + + public static byte[] encode(String text, FileEncoding encoding, String fallbackCharsetName) { + String charsetName = encoding != null ? encoding.getCharsetName() : fallbackCharsetName; + byte[] body = text.getBytes(charsetForName(charsetName)); + if (encoding != null && encoding.hasBom()) { + byte[] result = new byte[encoding.getBom().length + body.length]; + java.lang.System.arraycopy(encoding.getBom(), 0, result, 0, encoding.getBom().length); + java.lang.System.arraycopy(body, 0, result, encoding.getBom().length, body.length); + return result; + } + return body; + } + + private static Charset charsetForName(String charsetName) { + if (charsetName == null) { + return Charset.defaultCharset(); + } + try { + return Charset.forName(charsetName); + } catch (Exception e) { + return Charset.defaultCharset(); + } + } + + private static boolean startsWith(byte[] bytes, byte[] prefix) { + if (bytes.length < prefix.length) { + return false; + } + for (int i = 0; i < prefix.length; i++) { + if (bytes[i] != prefix[i]) { + return false; + } + } + return true; + } +} \ No newline at end of file diff --git a/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java b/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java new file mode 100644 index 0000000..10e2850 --- /dev/null +++ b/app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java @@ -0,0 +1,94 @@ +package com.maxistar.textpad.utils; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class FileEncodingTest { + + private static byte[] bom(byte[]... parts) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + for (byte[] part : parts) { + out.write(part, 0, part.length); + } + return out.toByteArray(); + } + + @Test + public void detectUtf16LeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0x41, 0x00}); + assertEquals(FileEncoding.UTF_16LE, encoding.getCharsetName()); + assertEquals(2, encoding.getBom().length); + } + + @Test + public void detectUtf16BeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFE, (byte) 0xFF, 0x00, 0x41}); + assertEquals(FileEncoding.UTF_16BE, encoding.getCharsetName()); + } + + @Test + public void detectUtf8Bom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF, 0x41}); + assertEquals(FileEncoding.UTF_8, encoding.getCharsetName()); + } + + @Test + public void detectUtf32LeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0, 0, 0x41, 0, 0, 0}); + assertEquals(FileEncoding.UTF_32LE, encoding.getCharsetName()); + } + + @Test + public void detectUtf32BeBom() { + FileEncoding encoding = FileEncoding.detect(new byte[]{0, 0, (byte) 0xFE, (byte) 0xFF, 0, 0, 0, 0x41}); + assertEquals(FileEncoding.UTF_32BE, encoding.getCharsetName()); + } + + @Test + public void detectNoBom() { + assertNull(FileEncoding.detect("hello".getBytes(StandardCharsets.UTF_8))); + assertNull(FileEncoding.detect(new byte[0])); + assertNull(FileEncoding.detect(null)); + } + + @Test + public void decodeStripsBom() { + byte[] bytes = bom(new byte[]{(byte) 0xFF, (byte) 0xFE}, + "\u041f\u0440\u0438\u0432\u0435\u0442".getBytes(StandardCharsets.UTF_16LE)); + FileEncoding encoding = FileEncoding.detect(bytes); + String text = FileEncoding.decode(bytes, encoding, FileEncoding.UTF_8); + assertEquals("\u041f\u0440\u0438\u0432\u0435\u0442", text); + } + + @Test + public void decodeWithoutBomUsesFallback() { + byte[] bytes = "hello".getBytes(StandardCharsets.UTF_8); + String text = FileEncoding.decode(bytes, null, FileEncoding.UTF_8); + assertEquals("hello", text); + } + + @Test + public void encodeRestoresBom() { + byte[] bytes = bom(new byte[]{(byte) 0xFF, (byte) 0xFE}, + "\u041f\u0440\u0438\u0432\u0435\u0442".getBytes(StandardCharsets.UTF_16LE)); + FileEncoding encoding = FileEncoding.detect(bytes); + byte[] encoded = FileEncoding.encode("\u041f\u0440\u0438\u0432\u0435\u0442", encoding, FileEncoding.UTF_8); + assertArrayEquals(bytes, encoded); + } + + @Test + public void encodeRoundTrip() { + byte[] bytes = bom(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF}, + "test".getBytes(StandardCharsets.UTF_8)); + FileEncoding encoding = FileEncoding.detect(bytes); + String text = FileEncoding.decode(bytes, encoding, FileEncoding.UTF_8); + byte[] encoded = FileEncoding.encode(text, encoding, FileEncoding.UTF_8); + assertArrayEquals(bytes, encoded); + } +} \ No newline at end of file