FFMpeg: stream encrypted assets through MemBuf AVIO

.github/workflows/encrypt-sdk-release.yml

@@ -0,0 +1,65 @@
+name: Publish encryption SDK
+
+on:
+  push:
+    tags: ['encrypt-sdk-*']
+  workflow_dispatch:
+    inputs:
+      run_id:
+        description: Successful Encryption core run containing both SDK artifacts
+        required: true
+        default: '41587062927'
+      tag:
+        description: Existing SDK tag to publish
+        required: true
+        default: encrypt-sdk-openssl-3.6.5-v143
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: write
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      SDK_RUN: ${{ inputs.run_id || '41587062927' }}
+      SDK_TAG: ${{ inputs.tag || github.ref_name }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Verify the source CI run
+        run: |
+          gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SDK_RUN" > source-run.json
+          jq -e --arg repository "$GITHUB_REPOSITORY" \
+            '.conclusion == "success" and .head_repository.full_name == $repository and .path == ".github/workflows/encryption.yml"' source-run.json
+      - uses: actions/download-artifact@v4
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ env.SDK_RUN }}
+          pattern: Encrypt-sdk-*
+          merge-multiple: true
+          path: packages
+      - name: Verify archives and SDK layouts
+        run: |
+          for arch in x86 x64; do
+            archive="$PWD/packages/openssl-$arch.zip"
+            hash=$(tr -d '\r\n\357\273\277' < "$archive.sha256")
+            cmake "-DOFE_ARCH=$arch" "-DOFE_OPENSSL_ROOT=$RUNNER_TEMP/sdk-$arch" \
+              "-DOFE_WORK_DIR=$RUNNER_TEMP/stage" "-DOFE_PREBUILT_ARCHIVE=$archive" \
+              "-DOFE_PREBUILT_SHA256=$hash" -P Extensions/_3rdLib/Encrypt-build/EnsureSDK.cmake
+          done
+      - name: Publish versioned SDK assets
+        run: |
+          source_sha=$(jq -r .head_sha source-run.json)
+          cat > release-notes.md <<EOF
+          OpenSSL SDKs built and tested with Visual Studio 2022 (v143).
+
+          - Win32 and x64 static libraries, with /MD Release and /MDd Debug.
+          - Includes headers, licenses, SDK manifests and SHA256 files.
+          - Source commit: $source_sha
+          - Verified CI: https://github.com/$GITHUB_REPOSITORY/actions/runs/$SDK_RUN
+
+          These archives are consumed automatically by EncryptDependencies.
+          EOF
+          gh release create "$SDK_TAG" packages/openssl-*.zip packages/openssl-*.zip.sha256 \
+            --repo "$GITHUB_REPOSITORY" --verify-tag --latest=false \
+            --title "Encryption SDK: OpenSSL 3.6.5 (VS 2022)" --notes-file release-notes.md

.github/workflows/encryption.yml

@@ -0,0 +1,85 @@
+name: Encryption core
+on:
+  pull_request:
+    paths:
+      - 'Extensions/_DeLib/Encryption*'
+      - 'Extensions/_DeLib/UnicodeConversion.h'
+      - 'Extensions/_DeLib/Encryption/**'
+      - 'Extensions/_3rdLib/FFMpeg.h'
+      - 'Extensions/_3rdLib/MemBuf.h'
+      - 'Extensions/FFMpeg/**'
+      - 'Extensions/_3rdLib/Encrypt-build/**'
+      - '.github/workflows/encryption.yml'
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  windows:
+    runs-on: windows-2022
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: Win32
+            triplet: x86-windows-static-md
+          - platform: x64
+            triplet: x64-windows-static-md
+    steps:
+      - uses: actions/checkout@v4
+      - uses: microsoft/setup-msbuild@v2
+      - name: Prepare OpenSSL through the VS dependency project
+        shell: pwsh
+        run: |
+          msbuild Extensions/_3rdLib/Encrypt-build/EncryptDependencies.vcxproj /m /t:Build /p:Configuration=Release "/p:Platform=${{ matrix.platform }}"
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+      - name: Build with Visual Studio MSBuild (Release and Debug)
+        shell: pwsh
+        run: |
+          foreach ($configuration in @('Release', 'Debug')) {
+            & msbuild Extensions/_DeLib/Encryption/Encryption.sln /m /t:Build "/p:Configuration=$configuration" "/p:Platform=${{ matrix.platform }}"
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+            & "Extensions/_DeLib/Encryption/build/${{ matrix.platform }}/$configuration/EncryptionTests.exe"
+            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          }
+      - name: Analyze the core with MSVC
+        run: msbuild Extensions/_DeLib/Encryption/Encryption.vcxproj /m /t:Rebuild /p:Configuration=Debug "/p:Platform=${{ matrix.platform }}" /p:RunCodeAnalysis=true /p:EnablePREfast=true /p:CodeAnalysisTreatWarningsAsErrors=true
+      - name: Build a source SDK for future prebuilt releases
+        shell: pwsh
+        run: |
+          msbuild Extensions/_3rdLib/Encrypt-build/EncryptDependencies.vcxproj /m /t:Build /p:Configuration=Release "/p:Platform=${{ matrix.platform }}" /p:OfeEncryptBuildFromSource=true
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+      - name: Package and verify a prebuilt SDK
+        shell: pwsh
+        run: |
+          $arch = if ('${{ matrix.platform }}' -eq 'Win32') { 'x86' } else { 'x64' }
+          $archive = "$env:RUNNER_TEMP/openssl-$arch.zip"
+          Push-Location "Extensions/_3rdLib/Encrypt-sdk/$arch"
+          cmake -E tar cf $archive --format=zip include lib debug share ofe-encryption-sdk.json
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+          Pop-Location
+          $hash = (Get-FileHash $archive -Algorithm SHA256).Hash.ToLower()
+          Set-Content "$archive.sha256" $hash
+          cmake "-DOFE_ARCH=$arch" "-DOFE_OPENSSL_ROOT=$env:RUNNER_TEMP/prebuilt-$arch" "-DOFE_WORK_DIR=$env:RUNNER_TEMP/staging" "-DOFE_PREBUILT_ARCHIVE=$archive" "-DOFE_PREBUILT_SHA256=$hash" -P Extensions/_3rdLib/Encrypt-build/EnsureSDK.cmake
+          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+      - uses: actions/upload-artifact@v4
+        with:
+          name: Encrypt-sdk-${{ matrix.platform }}
+          path: ${{ runner.temp }}/openssl-*.zip*
+          retention-days: 7
+  linux:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: python3 Extensions/_3rdLib/Encrypt-build/tests/test_sdk.py
+      - run: sudo apt-get update && sudo apt-get install -y libssl-dev libavformat-dev libavcodec-dev libavutil-dev pkg-config
+      - run: cmake -S Extensions/_DeLib/Encryption -B build -DENCRYPTION_BUILD_TESTS=ON -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS='-fsanitize=address,undefined -fno-omit-frame-pointer' -DCMAKE_C_FLAGS='-fsanitize=address,undefined' -DCMAKE_EXE_LINKER_FLAGS='-fsanitize=address,undefined'
+      - run: cmake --build build --parallel
+      - run: ctest --test-dir build --output-on-failure
+  macos:
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: brew install openssl@3
+      - run: cmake -S Extensions/_DeLib/Encryption -B build -DENCRYPTION_BUILD_TESTS=ON -DOPENSSL_ROOT_DIR="$(brew --prefix openssl@3)" -DCMAKE_BUILD_TYPE=Release
+      - run: cmake --build build --parallel
+      - run: ctest --test-dir build --output-on-failure

.github/workflows/general-definition.yml

@@ -31,6 +31,6 @@ jobs:
       - name: Configure with Visual Studio
         run: cmake -S Extensions/Android_Gradle/tests/helpers -B "${{ runner.temp }}/delib-tests" -G "Visual Studio 17 2022" -A x64 -DCMAKE_CXX_FLAGS=/utf-8
       - name: Build standalone headers and compatibility checks
-        run: cmake --build "${{ runner.temp }}/delib-tests" --config Release --target general_definition_headers general_definition_test --parallel 2
+        run: cmake --build "${{ runner.temp }}/delib-tests" --config Release --target general_definition_headers general_definition_test unicode_conversion_test --parallel 2
       - name: Run compatibility checks
-        run: ctest --test-dir "${{ runner.temp }}/delib-tests" -C Release -R "^general_definition$" --output-on-failure
+        run: ctest --test-dir "${{ runner.temp }}/delib-tests" -C Release -R "^(general_definition|unicode_conversion)$" --output-on-failure

.gitignore

@@ -19,3 +19,7 @@ Extensions/Android_Extensions.lnk
 # Generated FFmpeg SDK and dependency downloads
 /Extensions/_3rdLib/FFmpeg-sdk/
 /Extensions/_3rdLib/FFmpeg-build/out/
+
+# Generated OpenSSL SDK, vcpkg checkout and dependency caches
+/Extensions/_3rdLib/Encrypt-sdk/
+/Extensions/_3rdLib/Encrypt-build/out/

Extensions/Android_Gradle/tests/helpers/CMakeLists.txt

@@ -6,7 +6,7 @@ enable_testing()
 # Each public header must compile as the first and only include in its own TU.
 set(general_definition_headers
     GeneralDefinition.h GeneralMacros.h PathUtilities.h MemoryConstants.h HashUtilities.h
-    StringTraits.h StringConversion.h StringUtilities.h StringSplit.h
+    StringTraits.h StringConversion.h UnicodeConversion.h StringUtilities.h StringSplit.h
     ArithmeticTraits.h ArithmeticUtilities.h ContainerUtilities.h)
 set(header_probes)
 foreach(header IN LISTS general_definition_headers)
@@ -17,6 +17,14 @@ endforeach()
 add_library(general_definition_headers OBJECT ${header_probes})
 target_link_libraries(general_definition_headers PRIVATE DeLib::Helpers)
 
+# The Unicode utility must also work without the umbrella's C++20/iconv requirements.
+add_executable(unicode_conversion_test UnicodeConversionTest.cpp)
+target_compile_features(unicode_conversion_test PRIVATE cxx_std_17)
+set_target_properties(unicode_conversion_test PROPERTIES
+    CXX_STANDARD 17 CXX_STANDARD_REQUIRED YES CXX_EXTENSIONS NO)
+target_include_directories(unicode_conversion_test PRIVATE "${FTW_CORE_INCLUDE}")
+add_test(NAME unicode_conversion COMMAND unicode_conversion_test)
+
 add_executable(general_definition_test GeneralDefinitionTest.cpp HelperOtherTU.cpp)
 target_link_libraries(general_definition_test PRIVATE DeLib::Helpers)
 add_test(NAME general_definition COMMAND general_definition_test)

Extensions/Android_Gradle/tests/helpers/UnicodeConversionTest.cpp

@@ -0,0 +1,49 @@
+#include "UnicodeConversion.h"
+
+#include <iostream>
+
+static int checks = 0;
+
+static void check(bool result) {
+    ++checks;
+    if (!result) throw std::runtime_error("Unicode conversion regression");
+}
+
+template<class Function>
+static void invalid(Function function) {
+    try { function(); }
+    catch (const std::invalid_argument&) { ++checks; return; }
+    throw std::runtime_error("Expected invalid Unicode input");
+}
+
+int main() {
+    const std::string utf8 = "A\xe4\xb8\xad\xf0\x9f\x98\x80";
+    const std::wstring wide = L"A\u4e2d\U0001f600";
+    const std::u16string units = u"A\u4e2d\U0001f600";
+    check(to_utf16_string(std::string_view(utf8)) == units);
+    check(to_utf16_string(std::wstring_view(wide)) == units);
+    check(to_utf16_bytes(wide) == std::vector<std::uint8_t>({0x41, 0, 0x2d, 0x4e, 0x3d, 0xd8, 0, 0xde}));
+    check(to_utf16_string(std::string_view()).empty());
+    check(to_utf16_string(std::wstring_view()).empty());
+    check(to_utf16_bytes(L"").empty());
+    check(to_utf16_string(std::string_view("A\0B", 3)) == std::u16string(u"A\0B", 3));
+    check(to_utf16_string(std::wstring_view(L"A\0B", 3)) == std::u16string(u"A\0B", 3));
+    check(to_utf16_bytes(std::wstring_view(L"A\0B", 3)) == std::vector<std::uint8_t>({0x41, 0, 0, 0, 0x42, 0}));
+    check(to_utf16_string(std::string_view("\xf4\x8f\xbf\xbf")) == u"\U0010ffff");
+
+    // Wide input historically preserves raw UTF-16 units, including isolated surrogates.
+    const std::wstring surrogate(1, static_cast<wchar_t>(0xd800));
+    check(to_utf16_string(std::wstring_view(surrogate)) == std::u16string(1, 0xd800));
+    check(to_utf16_bytes(surrogate) == std::vector<std::uint8_t>({0, 0xd8}));
+
+    for (const auto* text : {"\x80", "\xc0\x80", "\xe4\xb8", "\xe4x\xad",
+            "\xed\xa0\x80", "\xf0\x9f", "\xf4\x90\x80\x80", "\xf5\x80\x80\x80"}) {
+        invalid([&] { to_utf16_string(std::string_view(text)); });
+    }
+    if (sizeof(wchar_t) > 2) {
+        const std::wstring outside(1, static_cast<wchar_t>(0x110000));
+        invalid([&] { to_utf16_string(std::wstring_view(outside)); });
+        invalid([&] { to_utf16_bytes(outside); });
+    }
+    std::cout << "PASS: " << checks << " standalone C++17 Unicode checks\n";
+}

Extensions/Android_Gradle/tests/helpers/run_helper_tests.py

@@ -24,6 +24,7 @@ def run(command, env=None):
 run([args.cmake, '-S', tests, '-B', build / 'cmake', '-DCMAKE_BUILD_TYPE=Release', '-DDELIB_ACP_ENCODING=UTF-8'])
 run([args.cmake, '--build', build / 'cmake', '--parallel', '2'])
 run([build / 'cmake/general_definition_test'])
+run([build / 'cmake/unicode_conversion_test'])
 run([build / 'cmake/acp_override_test'])
 run([build / 'cmake/path_owner_thread_test'])
 run([build / 'cmake/map_data_test'])

Extensions/Deque2D/Deque2D.sln

@@ -1,9 +1,14 @@
-
+
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.31515.178
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Deque2D", "Deque2D.vcxproj", "{3BCB4C73-71F6-43B6-803D-41411F92652A}"
+	ProjectSection(ProjectDependencies) = postProject
+		{F408ED35-2D66-4E36-A659-580895179F86} = {F408ED35-2D66-4E36-A659-580895179F86}
+	EndProjectSection
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "EncryptDependencies", "..\_3rdLib\Encrypt-build\EncryptDependencies.vcxproj", "{F408ED35-2D66-4E36-A659-580895179F86}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -12,6 +17,12 @@ Global
 		Run_Only Unicode|Win32 = Run_Only Unicode|Win32
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F408ED35-2D66-4E36-A659-580895179F86}.Debug Unicode|Win32.ActiveCfg = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Debug Unicode|Win32.Build.0 = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Release Unicode|Win32.ActiveCfg = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Release Unicode|Win32.Build.0 = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Run_Only Unicode|Win32.ActiveCfg = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Run_Only Unicode|Win32.Build.0 = Release|Win32
 		{3BCB4C73-71F6-43B6-803D-41411F92652A}.Debug Unicode|Win32.ActiveCfg = Debug Unicode|Win32
 		{3BCB4C73-71F6-43B6-803D-41411F92652A}.Debug Unicode|Win32.Build.0 = Debug Unicode|Win32
 		{3BCB4C73-71F6-43B6-803D-41411F92652A}.Release Unicode|Win32.ActiveCfg = Release Unicode|Win32

Extensions/Deque2D/Deque2D.vcxproj

@@ -304,6 +304,7 @@ copy $(TargetPath) "C:\Steam\steamapps\common\Clickteam Fusion 2.5\Data\Runtime\
     <Image Include="EXTICON.BMP" />
     <Image Include="EXTIMG.BMP" />
   </ItemGroup>
+  <Import Project="..\_DeLib\Encryption.props" />
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>

Extensions/Encrypter/Encrypter.sln

@@ -1,9 +1,14 @@
-
+
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.31515.178
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Encrypter", "Encrypter.vcxproj", "{3BCB4C73-71F6-43B6-803D-41411F92652A}"
+	ProjectSection(ProjectDependencies) = postProject
+		{F408ED35-2D66-4E36-A659-580895179F86} = {F408ED35-2D66-4E36-A659-580895179F86}
+	EndProjectSection
+EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "EncryptDependencies", "..\_3rdLib\Encrypt-build\EncryptDependencies.vcxproj", "{F408ED35-2D66-4E36-A659-580895179F86}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -12,6 +17,12 @@ Global
 		Run_Only Unicode|Win32 = Run_Only Unicode|Win32
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{F408ED35-2D66-4E36-A659-580895179F86}.Debug Unicode|Win32.ActiveCfg = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Debug Unicode|Win32.Build.0 = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Release Unicode|Win32.ActiveCfg = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Release Unicode|Win32.Build.0 = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Run_Only Unicode|Win32.ActiveCfg = Release|Win32
+		{F408ED35-2D66-4E36-A659-580895179F86}.Run_Only Unicode|Win32.Build.0 = Release|Win32
 		{3BCB4C73-71F6-43B6-803D-41411F92652A}.Debug Unicode|Win32.ActiveCfg = Debug Unicode|Win32
 		{3BCB4C73-71F6-43B6-803D-41411F92652A}.Debug Unicode|Win32.Build.0 = Debug Unicode|Win32
 		{3BCB4C73-71F6-43B6-803D-41411F92652A}.Release Unicode|Win32.ActiveCfg = Release Unicode|Win32

Extensions/Encrypter/Encrypter.vcxproj

@@ -300,6 +300,7 @@ copy $(TargetPath) "C:\Steam\steamapps\common\Clickteam Fusion 2.5\Data\Runtime\
     <Image Include="RES\EXTICON.BMP" />
     <Image Include="RES\EXTIMG.BMP" />
   </ItemGroup>
+  <Import Project="..\_DeLib\Encryption.props" />
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
   <ImportGroup Label="ExtensionTargets">
   </ImportGroup>

Extensions/Encrypter/Func.h

@@ -1,27 +1,19 @@
 #pragma once
 
-#define USE_PROCESS_DIRECTLY
-
-inline bool EncryptCore(LPCTSTR pSrcFilePath, LPCTSTR pDesFilePath, LPCTSTR pKey, bool bEncrypt) {
+inline bool EncryptCore(LPCTSTR pSrcFilePath, LPCTSTR pDesFilePath, LPCTSTR pKey, encryption::Mode mode) {
     Encryption Encrypt;
-
     Encrypt.GenerateKey(pKey);
 
-#ifdef USE_PROCESS_DIRECTLY
-    auto ret = bEncrypt
-        ? Encrypt.EncryptFileDirectly(pSrcFilePath)
-        : Encrypt.DecryptFileDirectly(pSrcFilePath);
-#else
-    auto ret = bEncrypt
-        ? Encrypt.EncryptFile(pSrcFilePath)
-        : Encrypt.DecryptFile(pSrcFilePath);
-#endif
-
-    if (!ret) {
-        return false;
-    }
-
-    Encrypt.SaveFile(pDesFilePath);
+    if (![&]{
+		switch(mode){
+		case encryption::Mode::encrypt:
+			return Encrypt.EncryptFileDirectly(pSrcFilePath);
+		case encryption::Mode::decrypt:
+			return Encrypt.DecryptFileDirectly(pSrcFilePath);
+		default:
+			return false;
+		}
+	}()) { return false; }
 
-    return true;
-}
+    return Encrypt.SaveFile(pDesFilePath);
+}

Extensions/Encrypter/Main.cpp

@@ -57,7 +57,7 @@ short WINAPI DLLExport Encrypt(LPRDATA rdPtr, long param1, long param2) {
 	LPCTSTR pDesFilePath = (LPCTSTR)CNC_GetStringParameter(rdPtr);
 	LPCTSTR pKey = (LPCTSTR)CNC_GetStringParameter(rdPtr);
 
-	EncryptCore(pSrcFilePath, pDesFilePath, pKey, ENCRY);
+	EncryptCore(pSrcFilePath, pDesFilePath, pKey, encryption::Mode::encrypt);
 
 	return 0;
 }
@@ -67,7 +67,7 @@ short WINAPI DLLExport Decrypt(LPRDATA rdPtr, long param1, long param2) {
 	LPCTSTR pDesFilePath = (LPCTSTR)CNC_GetStringParameter(rdPtr);
 	LPCTSTR pKey = (LPCTSTR)CNC_GetStringParameter(rdPtr);
 
-	EncryptCore(pSrcFilePath, pDesFilePath, pKey, DECRY);
+	EncryptCore(pSrcFilePath, pDesFilePath, pKey, encryption::Mode::decrypt);
 
 	return 0;
 }