Encode dfs file names for the platform instead of cp1252

mikecore/DfsBuilder.py

@@ -322,7 +322,7 @@ def CreateFile(self, filename):
 
         filePointer = ctypes.c_void_p(0)
         headerPointer = ctypes.c_void_p(0)
-        fnp = ctypes.c_char_p(filename.encode("cp1252"))
+        fnp = ctypes.c_char_p(DfsDLLUtil.EncodeFileName(filename))
         try:
             headerPointer = self.__CreateHeader()
             DfsDLL.Wrapper.dfsFileCreate(

mikecore/DfsFile.py

@@ -2,6 +2,7 @@
 from enum import IntEnum
 import datetime
 import ctypes
+import locale
 import numpy as np
 from mikecore.eum import *
 from mikecore.DfsDLL import DfsDLL
@@ -791,6 +792,10 @@ def Open(self, filename, mode=DfsFileMode.Read, parameters=None):
         filename: name and path of file to open
         """
 
+        # Encode first, so a name the native library can not open fails before
+        # any file system access.
+        encodedFileName = DfsDLLUtil.EncodeFileName(filename)
+
         # Close file, if already open
         if self.filePointer.value is not None:
             self.Close()
@@ -815,7 +820,7 @@ def Open(self, filename, mode=DfsFileMode.Read, parameters=None):
         self.headPointer = ctypes.c_void_p()
         # Marshal filename string to C char*
         fnp = ctypes.c_char_p()
-        fnp.value = filename.encode("cp1252")
+        fnp.value = encodedFileName
 
         if mode == DfsFileMode.Read:
             # Open file for reading
@@ -1564,7 +1569,24 @@ def __GetStaticData(self, item):
 
 
 class DfsDLLUtil:
-    """Utilities class, creating various Dfs classes based on pointers to DFS native data"""
+    """Utilities class, marshalling values to and from the DFS native library,
+    and creating various Dfs classes based on pointers to DFS native data"""
+
+    @staticmethod
+    def EncodeFileName(filename: str | os.PathLike[str]) -> bytes:
+        # The native library opens char* paths, which Windows interprets in the
+        # ANSI code page ("mbcs"), not UTF-8.
+        if os.name != "nt":
+            return os.fsencode(filename)
+        name: str = os.fspath(filename)
+        try:
+            return name.encode("mbcs")
+        except UnicodeEncodeError as e:
+            raise ValueError(
+                f"File name can not be represented in the Windows ANSI code page "
+                f"({locale.getencoding()}), character {e.object[e.start : e.end]!r} "
+                f"is not supported: {name}"
+            ) from e
 
     # In case of incorrect eumUnitInt then return eumUmeter, the default axis unit
     @staticmethod

mikecore/Projections.py

@@ -76,7 +76,7 @@ def Init(libfilepath: str | None = None):
         else:
             lib = ctypes.CDLL(os.path.join(MzCartDLL.libfilepath, "libMzCart.so"))
             libfilepathe = MzCartDLL.libfilepath + "/"
-            libfilepatheP = ctypes.c_char_p(libfilepathe.encode("ascii"))
+            libfilepatheP = ctypes.c_char_p(os.fsencode(libfilepathe))
             lib.CARTSETUPLINUX(libfilepatheP, libfilepatheP)
 
         lib.C_MZC_GETPROJECTION.restype = ctypes.c_void_p

mikecore/eum.py

@@ -1451,7 +1451,7 @@ def Init(libfilepath=None, libfilename=None):
             lib.eumSetupLoadLinux.argtypes = [ctypes.c_char_p]
             # TODO: Should this not be simpler?
             eumFilePath = path + "/EUM.xml"
-            eumFilePathP = ctypes.c_char_p(eumFilePath.encode("ascii"))
+            eumFilePathP = ctypes.c_char_p(os.fsencode(eumFilePath))
             lib.eumSetupLoadLinux(eumFilePathP)
 
         lib.eumUnitGetParameters.argtypes = [

tests/test_filename_encoding.py

@@ -0,0 +1,71 @@
+import locale
+import os
+import shutil
+
+import pytest
+
+from mikecore.DfsFileFactory import DfsFileFactory
+from mikecore.DfsuFile import DfsuFile
+from tests.examples_dfsu import ExamplesDfsu
+
+
+def _representable(name):
+    # Whether this machine can name the file at all: Linux paths are UTF-8, and
+    # Windows char* paths are in the ANSI code page, which getencoding reports.
+    if os.name != "nt":
+        return True
+    try:
+        name.encode(locale.getencoding())
+        return True
+    except UnicodeEncodeError:
+        return False
+
+
+# The Danish name is representable in cp1252 (the code page on GitHub's
+# Windows runners); the Chinese one only on a Chinese code page, or on Linux.
+NAMES = ["Øresund æøå", "项目 合界数模"]
+
+
+@pytest.mark.parametrize("name", NAMES)
+def test_open_non_ascii_path(tmp_path, name):
+    if not _representable(name):
+        pytest.skip("not representable in this machine's ANSI code page")
+    folder = tmp_path / name
+    folder.mkdir()
+    filename = folder / (name + ".dfsu")
+    shutil.copy("testdata/OresundHD.dfsu", filename)
+
+    dfs = DfsuFile.Open(str(filename))
+    try:
+        assert dfs.NumberOfElements == 3636
+    finally:
+        dfs.Close()
+
+
+@pytest.mark.parametrize("name", NAMES)
+def test_create_non_ascii_path(tmp_path, name):
+    if not _representable(name):
+        pytest.skip("not representable in this machine's ANSI code page")
+    folder = tmp_path / name
+    folder.mkdir()
+    filename = folder / (name + ".dfsu")
+
+    ExamplesDfsu.CreateDfsuFile("testdata/OresundHD.dfsu", str(filename), True)
+
+    assert filename.exists()
+    dfs = DfsFileFactory.DfsuFileOpen(str(filename))
+    try:
+        assert dfs.NumberOfElements == 3636
+    finally:
+        dfs.Close()
+
+
+@pytest.mark.skipif(os.name != "nt", reason="Windows ANSI code page only")
+def test_unrepresentable_path_gives_clear_error(tmp_path):
+    name = next((n for n in NAMES if not _representable(n)), None)
+    if name is None:
+        pytest.skip("every test name is representable in this code page")
+    filename = tmp_path / (name + ".dfsu")
+    shutil.copy("testdata/OresundHD.dfsu", filename)
+    with pytest.raises(ValueError, match="ANSI code page"):
+        DfsuFile.Open(str(filename))