fix: decode UTF-8 text previews as UTF-8

src/tagstudio/core/utils/encoding.py

@@ -16,11 +16,22 @@ def detect_char_encoding(filepath: Path) -> str | None:
     Returns:
     str | None: The detected character encoding, if any.
     """
-    detector = UniversalDetector()
     with open(filepath, "rb") as text_file:
-        for line in text_file.readlines():
-            detector.feed(line)
-            if detector.done:
-                break
-        detector.close()
+        data = text_file.read()
+
+    # chardet gives UTF-8 a low confidence when a file has only a few multi-byte
+    # characters (such as a couple of emoji), and then picks a single-byte
+    # encoding like Windows-1252 instead. Bytes that decode as UTF-8 are UTF-8.
+    try:
+        data.decode("utf-8")
+        return "utf-8"
+    except UnicodeDecodeError:
+        pass
+
+    detector = UniversalDetector()
+    for line in data.splitlines(keepends=True):
+        detector.feed(line)
+        if detector.done:
+            break
+    detector.close()
     return detector.result["encoding"]

tests/core/test_encoding.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: (c) TagStudio Contributors
+# SPDX-License-Identifier: MIT
+
+
+from pathlib import Path
+
+import pytest
+
+from tagstudio.core.utils.encoding import detect_char_encoding
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "Hello world 😀\n",
+        "Notes 🎉🎉 for the party\nsecond line\n",
+        "Grüße aus Köln 👋\n",
+    ],
+)
+def test_utf8_with_emoji_is_detected_as_utf8(tmp_path: Path, text: str):
+    path = tmp_path / "emoji.txt"
+    path.write_bytes(text.encode("utf-8"))
+
+    encoding = detect_char_encoding(path)
+
+    assert encoding == "utf-8"
+    assert path.read_text(encoding=encoding) == text
+
+
+def test_non_utf8_file_falls_back_to_detection(tmp_path: Path):
+    text = "Ceci est un fichier en français, enregistré en Latin-1. " * 20
+    path = tmp_path / "latin1.txt"
+    path.write_bytes(text.encode("latin-1"))
+
+    encoding = detect_char_encoding(path)
+
+    assert encoding is not None
+    assert encoding.lower() != "utf-8"
+    assert path.read_bytes().decode(encoding) == text