diff --git a/mikecore/DfsBuilder.py b/mikecore/DfsBuilder.py index afee788..1150ee4 100644 --- a/mikecore/DfsBuilder.py +++ b/mikecore/DfsBuilder.py @@ -322,7 +322,7 @@ def CreateFile(self, filename): filePointer = ctypes.c_void_p(0) headerPointer = ctypes.c_void_p(0) - fnp = ctypes.c_char_p(filename.encode("cp1252")) + fnp = ctypes.c_char_p(DfsDLLUtil.EncodeFileName(filename)) try: headerPointer = self.__CreateHeader() DfsDLL.Wrapper.dfsFileCreate( diff --git a/mikecore/DfsFile.py b/mikecore/DfsFile.py index fc80ea9..1e987e3 100644 --- a/mikecore/DfsFile.py +++ b/mikecore/DfsFile.py @@ -2,6 +2,7 @@ from enum import IntEnum import datetime import ctypes +import locale import numpy as np from mikecore.eum import * from mikecore.DfsDLL import DfsDLL @@ -791,6 +792,10 @@ def Open(self, filename, mode=DfsFileMode.Read, parameters=None): filename: name and path of file to open """ + # Encode first, so a name the native library can not open fails before + # any file system access. + encodedFileName = DfsDLLUtil.EncodeFileName(filename) + # Close file, if already open if self.filePointer.value is not None: self.Close() @@ -815,7 +820,7 @@ def Open(self, filename, mode=DfsFileMode.Read, parameters=None): self.headPointer = ctypes.c_void_p() # Marshal filename string to C char* fnp = ctypes.c_char_p() - fnp.value = filename.encode("cp1252") + fnp.value = encodedFileName if mode == DfsFileMode.Read: # Open file for reading @@ -1564,7 +1569,24 @@ def __GetStaticData(self, item): class DfsDLLUtil: - """Utilities class, creating various Dfs classes based on pointers to DFS native data""" + """Utilities class, marshalling values to and from the DFS native library, + and creating various Dfs classes based on pointers to DFS native data""" + + @staticmethod + def EncodeFileName(filename: str | os.PathLike[str]) -> bytes: + # The native library opens char* paths, which Windows interprets in the + # ANSI code page ("mbcs"), not UTF-8. + if os.name != "nt": + return os.fsencode(filename) + name: str = os.fspath(filename) + try: + return name.encode("mbcs") + except UnicodeEncodeError as e: + raise ValueError( + f"File name can not be represented in the Windows ANSI code page " + f"({locale.getencoding()}), character {e.object[e.start : e.end]!r} " + f"is not supported: {name}" + ) from e # In case of incorrect eumUnitInt then return eumUmeter, the default axis unit @staticmethod diff --git a/mikecore/Projections.py b/mikecore/Projections.py index 37df105..7263192 100644 --- a/mikecore/Projections.py +++ b/mikecore/Projections.py @@ -76,7 +76,7 @@ def Init(libfilepath: str | None = None): else: lib = ctypes.CDLL(os.path.join(MzCartDLL.libfilepath, "libMzCart.so")) libfilepathe = MzCartDLL.libfilepath + "/" - libfilepatheP = ctypes.c_char_p(libfilepathe.encode("ascii")) + libfilepatheP = ctypes.c_char_p(os.fsencode(libfilepathe)) lib.CARTSETUPLINUX(libfilepatheP, libfilepatheP) lib.C_MZC_GETPROJECTION.restype = ctypes.c_void_p diff --git a/mikecore/eum.py b/mikecore/eum.py index a287061..4e4cb8e 100644 --- a/mikecore/eum.py +++ b/mikecore/eum.py @@ -1451,7 +1451,7 @@ def Init(libfilepath=None, libfilename=None): lib.eumSetupLoadLinux.argtypes = [ctypes.c_char_p] # TODO: Should this not be simpler? eumFilePath = path + "/EUM.xml" - eumFilePathP = ctypes.c_char_p(eumFilePath.encode("ascii")) + eumFilePathP = ctypes.c_char_p(os.fsencode(eumFilePath)) lib.eumSetupLoadLinux(eumFilePathP) lib.eumUnitGetParameters.argtypes = [ diff --git a/tests/test_filename_encoding.py b/tests/test_filename_encoding.py new file mode 100644 index 0000000..6074203 --- /dev/null +++ b/tests/test_filename_encoding.py @@ -0,0 +1,71 @@ +import locale +import os +import shutil + +import pytest + +from mikecore.DfsFileFactory import DfsFileFactory +from mikecore.DfsuFile import DfsuFile +from tests.examples_dfsu import ExamplesDfsu + + +def _representable(name): + # Whether this machine can name the file at all: Linux paths are UTF-8, and + # Windows char* paths are in the ANSI code page, which getencoding reports. + if os.name != "nt": + return True + try: + name.encode(locale.getencoding()) + return True + except UnicodeEncodeError: + return False + + +# The Danish name is representable in cp1252 (the code page on GitHub's +# Windows runners); the Chinese one only on a Chinese code page, or on Linux. +NAMES = ["Øresund æøå", "项目 合界数模"] + + +@pytest.mark.parametrize("name", NAMES) +def test_open_non_ascii_path(tmp_path, name): + if not _representable(name): + pytest.skip("not representable in this machine's ANSI code page") + folder = tmp_path / name + folder.mkdir() + filename = folder / (name + ".dfsu") + shutil.copy("testdata/OresundHD.dfsu", filename) + + dfs = DfsuFile.Open(str(filename)) + try: + assert dfs.NumberOfElements == 3636 + finally: + dfs.Close() + + +@pytest.mark.parametrize("name", NAMES) +def test_create_non_ascii_path(tmp_path, name): + if not _representable(name): + pytest.skip("not representable in this machine's ANSI code page") + folder = tmp_path / name + folder.mkdir() + filename = folder / (name + ".dfsu") + + ExamplesDfsu.CreateDfsuFile("testdata/OresundHD.dfsu", str(filename), True) + + assert filename.exists() + dfs = DfsFileFactory.DfsuFileOpen(str(filename)) + try: + assert dfs.NumberOfElements == 3636 + finally: + dfs.Close() + + +@pytest.mark.skipif(os.name != "nt", reason="Windows ANSI code page only") +def test_unrepresentable_path_gives_clear_error(tmp_path): + name = next((n for n in NAMES if not _representable(n)), None) + if name is None: + pytest.skip("every test name is representable in this code page") + filename = tmp_path / (name + ".dfsu") + shutil.copy("testdata/OresundHD.dfsu", filename) + with pytest.raises(ValueError, match="ANSI code page"): + DfsuFile.Open(str(filename))