Skip to content

Detect file encoding from byte order mark #235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dronov-dmitry wants to merge 1 commit into
base: master
Choose a base branch
from
+239 −12
Open

Detect file encoding from byte order mark #235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 30 additions & 12 deletions app/src/main/java/com/maxistar/textpad/activities/EditorActivity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@
import com.maxistar.textpad.recovery.RecoveryWriter;
import com.maxistar.textpad.utils.EditTextUndoRedo;
import com.maxistar.textpad.utils.DocumentSaveValidator;
import com.maxistar.textpad.utils.FileEncoding;
import com.maxistar.textpad.utils.FileNameHelper;
import com.maxistar.textpad.utils.System;
import com.maxistar.textpad.utils.TextConverter;
Expand Down Expand Up @@ -130,6 +131,8 @@ public class EditorActivity extends AppCompatActivity {
private ScrollView scrollView;
private LinearLayout linearLayout;

private FileEncoding documentEncoding;

String urlFilename = TPStrings.EMPTY;

Uri lastTriedSystemUri = null;
Expand Down Expand Up @@ -261,6 +264,13 @@ private boolean simpleScrolling() {
return settingsService.isUseSimpleScrolling();
}

private String resolveFileEncodingName() {
if (documentEncoding != null) {
return documentEncoding.getCharsetName();
}
return settingsService.getFileEncoding();
}

@RequiresApi(Build.VERSION_CODES.VANILLA_ICE_CREAM)
private void applyEdgeToEdgeInsets() {
View editorRoot = findViewById(R.id.editor_root);
Expand Down Expand Up @@ -594,7 +604,7 @@ private RecoveryWriter.Snapshot createRecoverySnapshot() {
identity,
currentDisplayName(),
identity == null,
settingsService.getFileEncoding(),
resolveFileEncodingName(),
false,
originalSize,
originalLastModified,
Expand Down Expand Up @@ -1159,6 +1169,7 @@ public void clearFile() {
originalSize = null;
originalLastModified = null;
originalContentSha256 = null;
documentEncoding = null;
selectionStart = 0;
selectionEnd = 0;
setEditorText(TPStrings.EMPTY, false);
Expand Down Expand Up @@ -1340,7 +1351,7 @@ protected void saveFile(Uri uri) throws IOException {

s = applyEndings(s);

outputStream.write(s.getBytes(settingsService.getFileEncoding()));
outputStream.write(FileEncoding.encode(s, documentEncoding, settingsService.getFileEncoding()));
} finally {
outputStream.close();
}
Expand All @@ -1356,7 +1367,7 @@ private void guardedSaveNamedFile(boolean autosave) {
SaveRequest request = new SaveRequest(
editorGeneration,
recoveryKey,
persistedText.getBytes(settingsService.getFileEncoding())
FileEncoding.encode(persistedText, documentEncoding, settingsService.getFileEncoding())
);
boolean creatingDocument = nextSaveCreatesDocument || originalContentSha256 == null;
nextSaveCreatesDocument = false;
Expand Down Expand Up @@ -1551,8 +1562,11 @@ private void validateOpenDocumentOnForeground() {
return;
}

byte[] intendedBytes = applyEndings(mText.getText().toString())
.getBytes(settingsService.getFileEncoding());
byte[] intendedBytes = FileEncoding.encode(
applyEndings(mText.getText().toString()),
documentEncoding,
settingsService.getFileEncoding()
);
SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes);
DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify(
currentBytes,
Expand All @@ -1572,7 +1586,8 @@ private void validateOpenDocumentOnForeground() {
}

private void applyExternalDocument(byte[] externalBytes) throws Exception {
String externalText = new String(externalBytes, settingsService.getFileEncoding());
documentEncoding = FileEncoding.detect(externalBytes);
String externalText = FileEncoding.decode(externalBytes, documentEncoding, settingsService.getFileEncoding());
externalText = toUnixEndings(externalText);
setEditorText(externalText, false);
initEditor();
Expand All @@ -1588,8 +1603,11 @@ private void validateRestoredDraft() {
return;
}
try {
byte[] intendedBytes = applyEndings(mText.getText().toString())
.getBytes(settingsService.getFileEncoding());
byte[] intendedBytes = FileEncoding.encode(
applyEndings(mText.getText().toString()),
documentEncoding,
settingsService.getFileEncoding()
);
SaveRequest request = new SaveRequest(editorGeneration, recoveryKey, intendedBytes);
byte[] currentBytes = readNamedDocumentBytes();
DocumentSaveValidator.Outcome outcome = DocumentSaveValidator.classify(
Expand Down Expand Up @@ -1664,9 +1682,8 @@ private void openNamedFileLegacyDirect(String filename) {
dis.close();
fis.close();

String ttt = new String(b, 0, length,
settingsService.getFileEncoding());

documentEncoding = FileEncoding.detect(b);
String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding());
ttt = toUnixEndings(ttt);

setEditorText(ttt, false);
Expand Down Expand Up @@ -1717,7 +1734,8 @@ private void openNamedFileDirect(final Uri uri) {
}
byte[] b = bytes.toByteArray();

String ttt = new String(b, settingsService.getFileEncoding());
documentEncoding = FileEncoding.detect(b);
String ttt = FileEncoding.decode(b, documentEncoding, settingsService.getFileEncoding());
ttt = toUnixEndings(ttt);

inputStream.close();
Expand Down
115 changes: 115 additions & 0 deletions app/src/main/java/com/maxistar/textpad/utils/FileEncoding.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package com.maxistar.textpad.utils;

import java.nio.charset.Charset;

/**
* Detects a text file encoding from its byte order mark (BOM) and decodes or
* encodes the content keeping the original encoding.
*/
public class FileEncoding {

public static final String UTF_8 = "UTF-8";
public static final String UTF_16LE = "UTF-16LE";
public static final String UTF_16BE = "UTF-16BE";
public static final String UTF_32LE = "UTF-32LE";
public static final String UTF_32BE = "UTF-32BE";

private static final byte[] BOM_UTF_32LE = {(byte) 0xFF, (byte) 0xFE, 0, 0};
private static final byte[] BOM_UTF_32BE = {0, 0, (byte) 0xFE, (byte) 0xFF};
private static final byte[] BOM_UTF_8 = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF};
private static final byte[] BOM_UTF_16BE = {(byte) 0xFE, (byte) 0xFF};
private static final byte[] BOM_UTF_16LE = {(byte) 0xFF, (byte) 0xFE};

private final String charsetName;
private final byte[] bom;

private FileEncoding(String charsetName, byte[] bom) {
this.charsetName = charsetName;
this.bom = bom;
}

public String getCharsetName() {
return charsetName;
}

public byte[] getBom() {
return bom;
}

public boolean hasBom() {
return bom != null;
}

public static FileEncoding detect(byte[] bytes) {
if (bytes == null || bytes.length == 0) {
return null;
}
if (startsWith(bytes, BOM_UTF_32LE)) {
return new FileEncoding(UTF_32LE, BOM_UTF_32LE);
}
if (startsWith(bytes, BOM_UTF_32BE)) {
return new FileEncoding(UTF_32BE, BOM_UTF_32BE);
}
if (startsWith(bytes, BOM_UTF_8)) {
return new FileEncoding(UTF_8, BOM_UTF_8);
}
if (startsWith(bytes, BOM_UTF_16BE)) {
return new FileEncoding(UTF_16BE, BOM_UTF_16BE);
}
if (startsWith(bytes, BOM_UTF_16LE)) {
return new FileEncoding(UTF_16LE, BOM_UTF_16LE);
}
return null;
}

public static String decode(byte[] bytes, FileEncoding encoding, String fallbackCharsetName) {
if (bytes == null) {
return "";
}
int offset = 0;
if (encoding != null && encoding.hasBom() && bytes.length >= encoding.getBom().length) {
offset = encoding.getBom().length;
}
String charsetName = encoding != null ? encoding.getCharsetName() : fallbackCharsetName;
try {
return new String(bytes, offset, bytes.length - offset, charsetForName(charsetName));
} catch (Exception e) {
return new String(bytes);
}
}

public static byte[] encode(String text, FileEncoding encoding, String fallbackCharsetName) {
String charsetName = encoding != null ? encoding.getCharsetName() : fallbackCharsetName;
byte[] body = text.getBytes(charsetForName(charsetName));
if (encoding != null && encoding.hasBom()) {
byte[] result = new byte[encoding.getBom().length + body.length];
java.lang.System.arraycopy(encoding.getBom(), 0, result, 0, encoding.getBom().length);
java.lang.System.arraycopy(body, 0, result, encoding.getBom().length, body.length);
return result;
}
return body;
}

private static Charset charsetForName(String charsetName) {
if (charsetName == null) {
return Charset.defaultCharset();
}
try {
return Charset.forName(charsetName);
} catch (Exception e) {
return Charset.defaultCharset();
}
}

private static boolean startsWith(byte[] bytes, byte[] prefix) {
if (bytes.length < prefix.length) {
return false;
}
for (int i = 0; i < prefix.length; i++) {
if (bytes[i] != prefix[i]) {
return false;
}
}
return true;
}
}
94 changes: 94 additions & 0 deletions app/src/test/java/com/maxistar/textpad/utils/FileEncodingTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package com.maxistar.textpad.utils;

import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;

import org.junit.jupiter.api.Test;

import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

public class FileEncodingTest {

private static byte[] bom(byte[]... parts) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
for (byte[] part : parts) {
out.write(part, 0, part.length);
}
return out.toByteArray();
}

@Test
public void detectUtf16LeBom() {
FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0x41, 0x00});
assertEquals(FileEncoding.UTF_16LE, encoding.getCharsetName());
assertEquals(2, encoding.getBom().length);
}

@Test
public void detectUtf16BeBom() {
FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFE, (byte) 0xFF, 0x00, 0x41});
assertEquals(FileEncoding.UTF_16BE, encoding.getCharsetName());
}

@Test
public void detectUtf8Bom() {
FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF, 0x41});
assertEquals(FileEncoding.UTF_8, encoding.getCharsetName());
}

@Test
public void detectUtf32LeBom() {
FileEncoding encoding = FileEncoding.detect(new byte[]{(byte) 0xFF, (byte) 0xFE, 0, 0, 0x41, 0, 0, 0});
assertEquals(FileEncoding.UTF_32LE, encoding.getCharsetName());
}

@Test
public void detectUtf32BeBom() {
FileEncoding encoding = FileEncoding.detect(new byte[]{0, 0, (byte) 0xFE, (byte) 0xFF, 0, 0, 0, 0x41});
assertEquals(FileEncoding.UTF_32BE, encoding.getCharsetName());
}

@Test
public void detectNoBom() {
assertNull(FileEncoding.detect("hello".getBytes(StandardCharsets.UTF_8)));
assertNull(FileEncoding.detect(new byte[0]));
assertNull(FileEncoding.detect(null));
}

@Test
public void decodeStripsBom() {
byte[] bytes = bom(new byte[]{(byte) 0xFF, (byte) 0xFE},
"\u041f\u0440\u0438\u0432\u0435\u0442".getBytes(StandardCharsets.UTF_16LE));
FileEncoding encoding = FileEncoding.detect(bytes);
String text = FileEncoding.decode(bytes, encoding, FileEncoding.UTF_8);
assertEquals("\u041f\u0440\u0438\u0432\u0435\u0442", text);
}

@Test
public void decodeWithoutBomUsesFallback() {
byte[] bytes = "hello".getBytes(StandardCharsets.UTF_8);
String text = FileEncoding.decode(bytes, null, FileEncoding.UTF_8);
assertEquals("hello", text);
}

@Test
public void encodeRestoresBom() {
byte[] bytes = bom(new byte[]{(byte) 0xFF, (byte) 0xFE},
"\u041f\u0440\u0438\u0432\u0435\u0442".getBytes(StandardCharsets.UTF_16LE));
FileEncoding encoding = FileEncoding.detect(bytes);
byte[] encoded = FileEncoding.encode("\u041f\u0440\u0438\u0432\u0435\u0442", encoding, FileEncoding.UTF_8);
assertArrayEquals(bytes, encoded);
}

@Test
public void encodeRoundTrip() {
byte[] bytes = bom(new byte[]{(byte) 0xEF, (byte) 0xBB, (byte) 0xBF},
"test".getBytes(StandardCharsets.UTF_8));
FileEncoding encoding = FileEncoding.detect(bytes);
String text = FileEncoding.decode(bytes, encoding, FileEncoding.UTF_8);
byte[] encoded = FileEncoding.encode(text, encoding, FileEncoding.UTF_8);
assertArrayEquals(bytes, encoded);
}
}