From 826946e7ea48275a79dbbad3cc8323a495970a7b Mon Sep 17 00:00:00 2001
From: Abhinav Tarigoppula <abhinaaavvv07187@gmail.com>
Date: Thu, 12 Nov 2026 21:53:37 +0530
Subject: [PATCH] fix: decode UTF-8 text previews as UTF-8

chardet gives UTF-8 a low confidence when a file has only a few multi-byte
characters, so a short file with emoji was detected as Windows-1254 and
its preview showed mojibake. Use UTF-8 whenever the bytes decode as
UTF-8, and fall back to chardet otherwise.
---
 src/tagstudio/core/utils/encoding.py | 23 +++++++++++-----
 tests/core/test_encoding.py          | 39 ++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 6 deletions(-)
 create mode 100644 tests/core/test_encoding.py

diff --git a/src/tagstudio/core/utils/encoding.py b/src/tagstudio/core/utils/encoding.py
index 73265f011c..94c948cbd5 100644
--- a/src/tagstudio/core/utils/encoding.py
+++ b/src/tagstudio/core/utils/encoding.py
@@ -16,11 +16,22 @@ def detect_char_encoding(filepath: Path) -> str | None:
     Returns:
     str | None: The detected character encoding, if any.
     """
-    detector = UniversalDetector()
     with open(filepath, "rb") as text_file:
-        for line in text_file.readlines():
-            detector.feed(line)
-            if detector.done:
-                break
-        detector.close()
+        data = text_file.read()
+
+    # chardet gives UTF-8 a low confidence when a file has only a few multi-byte
+    # characters (such as a couple of emoji), and then picks a single-byte
+    # encoding like Windows-1252 instead. Bytes that decode as UTF-8 are UTF-8.
+    try:
+        data.decode("utf-8")
+        return "utf-8"
+    except UnicodeDecodeError:
+        pass
+
+    detector = UniversalDetector()
+    for line in data.splitlines(keepends=True):
+        detector.feed(line)
+        if detector.done:
+            break
+    detector.close()
     return detector.result["encoding"]
diff --git a/tests/core/test_encoding.py b/tests/core/test_encoding.py
new file mode 100644
index 0000000000..dbc6656940
--- /dev/null
+++ b/tests/core/test_encoding.py
@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: (c) TagStudio Contributors
+# SPDX-License-Identifier: MIT
+
+
+from pathlib import Path
+
+import pytest
+
+from tagstudio.core.utils.encoding import detect_char_encoding
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "Hello world 😀\n",
+        "Notes 🎉🎉 for the party\nsecond line\n",
+        "Grüße aus Köln 👋\n",
+    ],
+)
+def test_utf8_with_emoji_is_detected_as_utf8(tmp_path: Path, text: str):
+    path = tmp_path / "emoji.txt"
+    path.write_bytes(text.encode("utf-8"))
+
+    encoding = detect_char_encoding(path)
+
+    assert encoding == "utf-8"
+    assert path.read_text(encoding=encoding) == text
+
+
+def test_non_utf8_file_falls_back_to_detection(tmp_path: Path):
+    text = "Ceci est un fichier en français, enregistré en Latin-1. " * 20
+    path = tmp_path / "latin1.txt"
+    path.write_bytes(text.encode("latin-1"))
+
+    encoding = detect_char_encoding(path)
+
+    assert encoding is not None
+    assert encoding.lower() != "utf-8"
+    assert path.read_bytes().decode(encoding) == text