diff --git a/src/tagstudio/core/utils/encoding.py b/src/tagstudio/core/utils/encoding.py index 73265f011..94c948cbd 100644 --- a/src/tagstudio/core/utils/encoding.py +++ b/src/tagstudio/core/utils/encoding.py @@ -16,11 +16,22 @@ def detect_char_encoding(filepath: Path) -> str | None: Returns: str | None: The detected character encoding, if any. """ - detector = UniversalDetector() with open(filepath, "rb") as text_file: - for line in text_file.readlines(): - detector.feed(line) - if detector.done: - break - detector.close() + data = text_file.read() + + # chardet gives UTF-8 a low confidence when a file has only a few multi-byte + # characters (such as a couple of emoji), and then picks a single-byte + # encoding like Windows-1252 instead. Bytes that decode as UTF-8 are UTF-8. + try: + data.decode("utf-8") + return "utf-8" + except UnicodeDecodeError: + pass + + detector = UniversalDetector() + for line in data.splitlines(keepends=True): + detector.feed(line) + if detector.done: + break + detector.close() return detector.result["encoding"] diff --git a/tests/core/test_encoding.py b/tests/core/test_encoding.py new file mode 100644 index 000000000..dbc665694 --- /dev/null +++ b/tests/core/test_encoding.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: (c) TagStudio Contributors +# SPDX-License-Identifier: MIT + + +from pathlib import Path + +import pytest + +from tagstudio.core.utils.encoding import detect_char_encoding + + +@pytest.mark.parametrize( + "text", + [ + "Hello world 😀\n", + "Notes 🎉🎉 for the party\nsecond line\n", + "Grüße aus Köln 👋\n", + ], +) +def test_utf8_with_emoji_is_detected_as_utf8(tmp_path: Path, text: str): + path = tmp_path / "emoji.txt" + path.write_bytes(text.encode("utf-8")) + + encoding = detect_char_encoding(path) + + assert encoding == "utf-8" + assert path.read_text(encoding=encoding) == text + + +def test_non_utf8_file_falls_back_to_detection(tmp_path: Path): + text = "Ceci est un fichier en français, enregistré en Latin-1. " * 20 + path = tmp_path / "latin1.txt" + path.write_bytes(text.encode("latin-1")) + + encoding = detect_char_encoding(path) + + assert encoding is not None + assert encoding.lower() != "utf-8" + assert path.read_bytes().decode(encoding) == text