From 1fd0f92d0dff60e7230f8c44766074c9b51070f9 Mon Sep 17 00:00:00 2001 From: Ryan Walter Murray Date: Tue, 29 Sep 2026 10:28:12 +0200 Subject: [PATCH 1/4] Encode dfs file names for the platform, not as cp1252 The native library opens char* paths. Windows reads those in the ANSI code page, and Linux passes them to the filesystem as bytes (UTF-8), so hardcoding cp1252 broke paths outside cp1252 on Windows (e.g. Chinese) and any non-ASCII path on Linux. Names that can't be represented in the Windows ANSI code page now raise a ValueError that says so. Closes #23 Co-Authored-By: Claude Opus 5.5 --- mikecore/DfsBuilder.py | 2 +- mikecore/DfsFile.py | 19 +++++++++- tests/test_filename_encoding.py | 64 +++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 tests/test_filename_encoding.py diff --git a/mikecore/DfsBuilder.py b/mikecore/DfsBuilder.py index afee788..1150ee4 100644 --- a/mikecore/DfsBuilder.py +++ b/mikecore/DfsBuilder.py @@ -322,7 +322,7 @@ def CreateFile(self, filename): filePointer = ctypes.c_void_p(0) headerPointer = ctypes.c_void_p(0) - fnp = ctypes.c_char_p(filename.encode("cp1252")) + fnp = ctypes.c_char_p(DfsDLLUtil.EncodeFileName(filename)) try: headerPointer = self.__CreateHeader() DfsDLL.Wrapper.dfsFileCreate( diff --git a/mikecore/DfsFile.py b/mikecore/DfsFile.py index fc80ea9..db83f15 100644 --- a/mikecore/DfsFile.py +++ b/mikecore/DfsFile.py @@ -1,3 +1,4 @@ +import locale import os.path from enum import IntEnum import datetime @@ -815,7 +816,7 @@ def Open(self, filename, mode=DfsFileMode.Read, parameters=None): self.headPointer = ctypes.c_void_p() # Marshal filename string to C char* fnp = ctypes.c_char_p() - fnp.value = filename.encode("cp1252") + fnp.value = DfsDLLUtil.EncodeFileName(filename) if mode == DfsFileMode.Read: # Open file for reading @@ -1566,6 +1567,22 @@ def __GetStaticData(self, item): class DfsDLLUtil: """Utilities class, creating various Dfs classes based on pointers to DFS native data""" + @staticmethod + def EncodeFileName(filename) -> bytes: + # The native library opens char* paths, which Windows interprets in the + # ANSI code page ("mbcs"), not UTF-8. + if os.name != "nt": + return os.fsencode(filename) + filename = os.fspath(filename) + try: + return filename.encode("mbcs") + except UnicodeEncodeError: + raise ValueError( + "File name can not be represented in the Windows ANSI code page " + "({}): {}. Rename the file or folder, or enable 'Beta: Use Unicode " + "UTF-8 for worldwide language support' in Windows region settings." + .format(locale.getpreferredencoding(False), filename)) from None + # In case of incorrect eumUnitInt then return eumUmeter, the default axis unit @staticmethod def ToEumAxisUnit(eumUnitInt: int): diff --git a/tests/test_filename_encoding.py b/tests/test_filename_encoding.py new file mode 100644 index 0000000..a2e99e4 --- /dev/null +++ b/tests/test_filename_encoding.py @@ -0,0 +1,64 @@ +import os +import shutil + +import pytest + +from mikecore.DfsFileFactory import DfsFileFactory +from mikecore.DfsuFile import DfsuFile +from tests.examples_dfsu import ExamplesDfsu + + +def _representable(name): + if os.name != "nt": + return True + try: + name.encode("mbcs") + return True + except UnicodeEncodeError: + return False + + +# The Danish name is representable in cp1252 (the code page on GitHub's +# Windows runners); the Chinese one only on a Chinese code page, or on Linux. +NAMES = ["Øresund æøå", "项目 合界数模"] + + +@pytest.mark.parametrize("name", NAMES) +def test_open_non_ascii_path(tmp_path, name): + if not _representable(name): + pytest.skip("not representable in this machine's ANSI code page") + folder = tmp_path / name + folder.mkdir() + filename = folder / (name + ".dfsu") + shutil.copy("testdata/OresundHD.dfsu", filename) + + dfs = DfsuFile.Open(str(filename)) + assert dfs.NumberOfElements == 3636 + dfs.Close() + + +@pytest.mark.parametrize("name", NAMES) +def test_create_non_ascii_path(tmp_path, name): + if not _representable(name): + pytest.skip("not representable in this machine's ANSI code page") + folder = tmp_path / name + folder.mkdir() + filename = folder / (name + ".dfsu") + + ExamplesDfsu.CreateDfsuFile("testdata/OresundHD.dfsu", str(filename), True) + + assert filename.exists() + dfs = DfsFileFactory.DfsuFileOpen(str(filename)) + assert dfs.NumberOfElements == 3636 + dfs.Close() + + +@pytest.mark.skipif(os.name != "nt", reason="Windows ANSI code page only") +def test_unrepresentable_path_gives_clear_error(tmp_path): + name = next((n for n in NAMES if not _representable(n)), None) + if name is None: + pytest.skip("every test name is representable in this code page") + filename = tmp_path / (name + ".dfsu") + shutil.copy("testdata/OresundHD.dfsu", filename) + with pytest.raises(ValueError, match="ANSI code page"): + DfsuFile.Open(str(filename)) From b09e84fa187713b870db3015e4d5bd71424fab86 Mon Sep 17 00:00:00 2001 From: Ryan Walter Murray Date: Tue, 29 Sep 2026 10:33:54 +0200 Subject: [PATCH 2/4] Report the ANSI code page correctly; fsencode Linux lib paths getpreferredencoding returns utf-8 in Python UTF-8 mode, so the error message named the wrong code page; ask Windows for it instead. The eum and MzCart setup paths on Linux were encoded as ascii, failing for installs under a non-ASCII directory. Co-Authored-By: Claude Opus 5.5 --- mikecore/DfsFile.py | 9 +++++---- mikecore/Projections.py | 2 +- mikecore/eum.py | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/mikecore/DfsFile.py b/mikecore/DfsFile.py index db83f15..9fb750b 100644 --- a/mikecore/DfsFile.py +++ b/mikecore/DfsFile.py @@ -1,4 +1,3 @@ -import locale import os.path from enum import IntEnum import datetime @@ -1577,11 +1576,13 @@ def EncodeFileName(filename) -> bytes: try: return filename.encode("mbcs") except UnicodeEncodeError: + codePage = ctypes.windll.kernel32.GetACP() raise ValueError( "File name can not be represented in the Windows ANSI code page " - "({}): {}. Rename the file or folder, or enable 'Beta: Use Unicode " - "UTF-8 for worldwide language support' in Windows region settings." - .format(locale.getpreferredencoding(False), filename)) from None + f"(cp{codePage}): {filename}. Rename the file or folder, or enable " + "'Beta: Use Unicode UTF-8 for worldwide language support' in Windows " + "region settings." + ) from None # In case of incorrect eumUnitInt then return eumUmeter, the default axis unit @staticmethod diff --git a/mikecore/Projections.py b/mikecore/Projections.py index 37df105..7263192 100644 --- a/mikecore/Projections.py +++ b/mikecore/Projections.py @@ -76,7 +76,7 @@ def Init(libfilepath: str | None = None): else: lib = ctypes.CDLL(os.path.join(MzCartDLL.libfilepath, "libMzCart.so")) libfilepathe = MzCartDLL.libfilepath + "/" - libfilepatheP = ctypes.c_char_p(libfilepathe.encode("ascii")) + libfilepatheP = ctypes.c_char_p(os.fsencode(libfilepathe)) lib.CARTSETUPLINUX(libfilepatheP, libfilepatheP) lib.C_MZC_GETPROJECTION.restype = ctypes.c_void_p diff --git a/mikecore/eum.py b/mikecore/eum.py index a287061..4e4cb8e 100644 --- a/mikecore/eum.py +++ b/mikecore/eum.py @@ -1451,7 +1451,7 @@ def Init(libfilepath=None, libfilename=None): lib.eumSetupLoadLinux.argtypes = [ctypes.c_char_p] # TODO: Should this not be simpler? eumFilePath = path + "/EUM.xml" - eumFilePathP = ctypes.c_char_p(eumFilePath.encode("ascii")) + eumFilePathP = ctypes.c_char_p(os.fsencode(eumFilePath)) lib.eumSetupLoadLinux(eumFilePathP) lib.eumUnitGetParameters.argtypes = [ From 323690f65f66d82b33812b985ea46dccaead32fb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 12:01:05 +0200 Subject: [PATCH 3/4] Address review of the file name encoding Read the code page with locale.getencoding() instead of calling kernel32, type EncodeFileName's parameter, keep the UnicodeEncodeError as the cause and name the offending character, encode before any file system access in Open, close test files in finally, and base the test skip on the machine's code page rather than on the encoder. Co-Authored-By: Claude Opus 5.5 --- mikecore/DfsFile.py | 26 ++++++++++++++++---------- tests/test_filename_encoding.py | 17 ++++++++++++----- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/mikecore/DfsFile.py b/mikecore/DfsFile.py index 9fb750b..8d8dbb8 100644 --- a/mikecore/DfsFile.py +++ b/mikecore/DfsFile.py @@ -2,6 +2,7 @@ from enum import IntEnum import datetime import ctypes +import locale import numpy as np from mikecore.eum import * from mikecore.DfsDLL import DfsDLL @@ -791,6 +792,10 @@ def Open(self, filename, mode=DfsFileMode.Read, parameters=None): filename: name and path of file to open """ + # Encode first, so a name the native library can not open fails before + # any file system access. + encodedFileName = DfsDLLUtil.EncodeFileName(filename) + # Close file, if already open if self.filePointer.value is not None: self.Close() @@ -815,7 +820,7 @@ def Open(self, filename, mode=DfsFileMode.Read, parameters=None): self.headPointer = ctypes.c_void_p() # Marshal filename string to C char* fnp = ctypes.c_char_p() - fnp.value = DfsDLLUtil.EncodeFileName(filename) + fnp.value = encodedFileName if mode == DfsFileMode.Read: # Open file for reading @@ -1564,25 +1569,26 @@ def __GetStaticData(self, item): class DfsDLLUtil: - """Utilities class, creating various Dfs classes based on pointers to DFS native data""" + """Utilities class, marshalling values to and from the DFS native library, + and creating various Dfs classes based on pointers to DFS native data""" @staticmethod - def EncodeFileName(filename) -> bytes: + def EncodeFileName(filename: str | os.PathLike[str]) -> bytes: # The native library opens char* paths, which Windows interprets in the # ANSI code page ("mbcs"), not UTF-8. if os.name != "nt": return os.fsencode(filename) - filename = os.fspath(filename) + name: str = os.fspath(filename) try: - return filename.encode("mbcs") - except UnicodeEncodeError: - codePage = ctypes.windll.kernel32.GetACP() + return name.encode("mbcs") + except UnicodeEncodeError as e: raise ValueError( - "File name can not be represented in the Windows ANSI code page " - f"(cp{codePage}): {filename}. Rename the file or folder, or enable " + f"File name can not be represented in the Windows ANSI code page " + f"({locale.getencoding()}), character {e.object[e.start : e.end]!r} " + f"is not supported: {name}. Rename the file or folder, or enable " "'Beta: Use Unicode UTF-8 for worldwide language support' in Windows " "region settings." - ) from None + ) from e # In case of incorrect eumUnitInt then return eumUmeter, the default axis unit @staticmethod diff --git a/tests/test_filename_encoding.py b/tests/test_filename_encoding.py index a2e99e4..6074203 100644 --- a/tests/test_filename_encoding.py +++ b/tests/test_filename_encoding.py @@ -1,3 +1,4 @@ +import locale import os import shutil @@ -9,10 +10,12 @@ def _representable(name): + # Whether this machine can name the file at all: Linux paths are UTF-8, and + # Windows char* paths are in the ANSI code page, which getencoding reports. if os.name != "nt": return True try: - name.encode("mbcs") + name.encode(locale.getencoding()) return True except UnicodeEncodeError: return False @@ -33,8 +36,10 @@ def test_open_non_ascii_path(tmp_path, name): shutil.copy("testdata/OresundHD.dfsu", filename) dfs = DfsuFile.Open(str(filename)) - assert dfs.NumberOfElements == 3636 - dfs.Close() + try: + assert dfs.NumberOfElements == 3636 + finally: + dfs.Close() @pytest.mark.parametrize("name", NAMES) @@ -49,8 +54,10 @@ def test_create_non_ascii_path(tmp_path, name): assert filename.exists() dfs = DfsFileFactory.DfsuFileOpen(str(filename)) - assert dfs.NumberOfElements == 3636 - dfs.Close() + try: + assert dfs.NumberOfElements == 3636 + finally: + dfs.Close() @pytest.mark.skipif(os.name != "nt", reason="Windows ANSI code page only") From 31e7977da7b50be2daec04a56aa716e5860fb9eb Mon Sep 17 00:00:00 2001 From: Ryan Murray Date: Tue, 13 Oct 2026 12:45:15 +0200 Subject: [PATCH 4/4] Drop the Windows settings advice from the file name error It names a UI label that can change, and the setting was never verified. Co-Authored-By: Claude Opus 5.5 --- mikecore/DfsFile.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mikecore/DfsFile.py b/mikecore/DfsFile.py index 8d8dbb8..1e987e3 100644 --- a/mikecore/DfsFile.py +++ b/mikecore/DfsFile.py @@ -1585,9 +1585,7 @@ def EncodeFileName(filename: str | os.PathLike[str]) -> bytes: raise ValueError( f"File name can not be represented in the Windows ANSI code page " f"({locale.getencoding()}), character {e.object[e.start : e.end]!r} " - f"is not supported: {name}. Rename the file or folder, or enable " - "'Beta: Use Unicode UTF-8 for worldwide language support' in Windows " - "region settings." + f"is not supported: {name}" ) from e # In case of incorrect eumUnitInt then return eumUmeter, the default axis unit