Harden RSA COSE key handling

.github/workflows/wolfssl-versions.yml

@@ -130,10 +130,20 @@ jobs:
         make -j$(nproc)
         make install
 
+    - name: Select wolfCOSE compatibility flags
+      run: |
+        VERSION_HEX=$(awk '/^#define LIBWOLFSSL_VERSION_HEX/{print $3}' \
+                          ~/wolfssl-install/include/wolfssl/version.h)
+        if (( VERSION_HEX < 0x05009002 )); then
+          echo "WOLFCOSE_COMPAT_FLAGS=-DWOLFCOSE_RSA_PUBLIC_ONLY -DWOLFCOSE_NO_MLDSA -DWOLFCOSE_NO_LMS" >> "$GITHUB_ENV"
+        else
+          echo "WOLFCOSE_COMPAT_FLAGS=" >> "$GITHUB_ENV"
+        fi
+
     - name: Build wolfCOSE
       run: |
         export WOLFSSL_DIR=$HOME/wolfssl-install
-        make CFLAGS="-std=c99 -DHAVE_ANONYMOUS_INLINE_AGGREGATES=1 -Os -Wall -Wextra -Wpedantic -Wshadow -Wconversion -I./include -isystem $WOLFSSL_DIR/include" \
+        make CFLAGS="-std=c99 -DHAVE_ANONYMOUS_INLINE_AGGREGATES=1 -Os -Wall -Wextra -Wpedantic -Wshadow -Wconversion $WOLFCOSE_COMPAT_FLAGS -I./include -isystem $WOLFSSL_DIR/include" \
              LDFLAGS="-L$WOLFSSL_DIR/lib -lwolfssl"
 
     - name: Verify legacy AES Key Wrap rejection
@@ -168,14 +178,14 @@ jobs:
       run: |
         export WOLFSSL_DIR=$HOME/wolfssl-install
         export LD_LIBRARY_PATH=$WOLFSSL_DIR/lib
-        make test CFLAGS="-std=c99 -DHAVE_ANONYMOUS_INLINE_AGGREGATES=1 -Os -Wall -Wextra -Wpedantic -Wshadow -Wconversion -I./include -isystem $WOLFSSL_DIR/include" \
+        make test CFLAGS="-std=c99 -DHAVE_ANONYMOUS_INLINE_AGGREGATES=1 -Os -Wall -Wextra -Wpedantic -Wshadow -Wconversion $WOLFCOSE_COMPAT_FLAGS -I./include -isystem $WOLFSSL_DIR/include" \
                   LDFLAGS="-L$WOLFSSL_DIR/lib -lwolfssl"
 
     - name: Run tool round-trip test
       run: |
         export WOLFSSL_DIR=$HOME/wolfssl-install
         export LD_LIBRARY_PATH=$WOLFSSL_DIR/lib
-        make tool-test CFLAGS="-std=c99 -DHAVE_ANONYMOUS_INLINE_AGGREGATES=1 -Os -Wall -Wextra -Wpedantic -Wshadow -Wconversion -I./include -isystem $WOLFSSL_DIR/include" \
+        make tool-test CFLAGS="-std=c99 -DHAVE_ANONYMOUS_INLINE_AGGREGATES=1 -Os -Wall -Wextra -Wpedantic -Wshadow -Wconversion $WOLFCOSE_COMPAT_FLAGS -I./include -isystem $WOLFSSL_DIR/include" \
                        LDFLAGS="-L$WOLFSSL_DIR/lib -lwolfssl"
 
     # ML-DSA / PQC-specific coverage: CLI keygen->sign->verify for all three

docs/Getting-Started.md

@@ -8,10 +8,18 @@ This guide covers prerequisites, building wolfCOSE, and basic usage examples.
 
 wolfCOSE requires wolfSSL 5.8.0 or later with the appropriate algorithms
 enabled. AES Key Wrap requires wolfSSL 5.9.0 or later because that release
-uses a constant-time integrity comparison during unwrap. ML-DSA requires a
-wolfSSL release newer than 5.9.1. HSS/LMS (RFC 8778) requires wolfSSL 5.9.2 or
-later, the first release whose public-key importer derives the parameter set
-from the key bytes.
+uses a constant-time integrity comparison during unwrap. Private RSA
+`COSE_Key` decoding requires wolfSSL 5.9.0 or later, and private RSA
+serialization requires wolfSSL 5.9.2 or later. ML-DSA requires a wolfSSL
+release newer than 5.9.1. HSS/LMS (RFC 8778) requires wolfSSL 5.9.2 or later,
+the first release whose public-key importer derives the parameter set from the
+key bytes.
+
+These dependency floors are enforced at compile time whenever wolfCOSE selects
+the corresponding feature. With an older wolfSSL, disable unused ML-DSA or LMS
+support with `WOLFCOSE_NO_MLDSA` or `WOLFCOSE_NO_LMS`. Define
+`WOLFCOSE_RSA_PUBLIC_ONLY` to retain RSA-PSS and public `COSE_Key` support
+without private RSA serialization.
 
 Here is a full-featured build using a release that meets those feature floors:
 
@@ -61,7 +69,9 @@ You can enable only the algorithms you need:
 | ECDH-ES key agreement | `--enable-ecc --enable-hkdf` |
 | AES Key Wrap | `--enable-aeskeywrap` (wolfSSL 5.9.0+) |
 | RSA-PSS signing | `--enable-rsapss --enable-keygen` |
-| ML-DSA (post-quantum) | `--enable-mldsa` |
+| Private RSA `COSE_Key` decoding | `--enable-rsapss` (wolfSSL 5.9.0+) |
+| Private RSA `COSE_Key` serialization | `--enable-rsapss --enable-keygen` (wolfSSL 5.9.2+) |
+| ML-DSA (post-quantum) | `--enable-mldsa` (wolfSSL newer than 5.9.1) |
 | HSS/LMS (stateful hash-based) | `--enable-lms` (wolfSSL 5.9.2+) |
 | AES-MAC | `--enable-aescbc` |
 

docs/Macros.md

@@ -63,7 +63,7 @@ Per-algorithm opt-outs for the default (non-lean) build. Each also has a `WOLFCO
 | `WOLFCOSE_NO_EDDSA` | Ed25519 | `HAVE_ED25519` |
 | `WOLFCOSE_NO_ED448` | Ed448 | `HAVE_ED448` |
 | `WOLFCOSE_NO_RSAPSS` | RSA-PSS (PS256/384/512) | `WC_RSA_PSS` |
-| `WOLFCOSE_NO_MLDSA` | ML-DSA (FIPS 204) | `WOLFSSL_HAVE_MLDSA` |
+| `WOLFCOSE_NO_MLDSA` | ML-DSA (FIPS 204) | `WOLFSSL_HAVE_MLDSA` (wolfSSL newer than 5.9.1) |
 | `WOLFCOSE_NO_LMS` | HSS/LMS (RFC 8778) | `WOLFSSL_HAVE_LMS` (wolfSSL 5.9.2+) |
 | `WOLFCOSE_NO_AESGCM` | AES-GCM | `HAVE_AESGCM` |
 | `WOLFCOSE_NO_AESCCM` | AES-CCM | `HAVE_AESCCM` |
@@ -269,6 +269,10 @@ Two limits worth knowing before designing around this:
 | `WOLFCOSE_KEY_DECODE` | Enable COSE_Key decoding | Enabled |
 | `WOLFCOSE_NO_KEY_DECODE` | Disable COSE_Key decoding | - |
 
+Private RSA `COSE_Key` serialization requires wolfSSL 5.9.2 or later. Define
+`WOLFCOSE_RSA_PUBLIC_ONLY` to retain RSA-PSS and public-key encoding on an older
+wolfSSL release.
+
 ---
 
 ## Size Configuration

include/wolfcose/settings.h

@@ -227,28 +227,40 @@ extern "C" {
 
 /* ML-DSA (44/65/87) — extension */
 #if defined(WOLFCOSE_ENABLE_MLDSA)
-    #ifndef WOLFSSL_HAVE_MLDSA
+    #if !defined(WOLFSSL_HAVE_MLDSA)
         #error "WOLFCOSE_ENABLE_MLDSA requires wolfSSL WOLFSSL_HAVE_MLDSA"
+    #elif LIBWOLFSSL_VERSION_HEX < 0x05009002
+        #error "wolfCOSE ML-DSA requires wolfSSL newer than 5.9.1"
+    #else
+        #define WOLFCOSE_HAVE_MLDSA
+    #endif
+#elif !defined(WOLFCOSE_LEAN) && !defined(WOLFCOSE_NO_MLDSA) && \
+      defined(WOLFSSL_HAVE_MLDSA)
+    #if LIBWOLFSSL_VERSION_HEX < 0x05009002
+        #error "wolfCOSE ML-DSA requires wolfSSL newer than 5.9.1"
+    #else
+        #define WOLFCOSE_HAVE_MLDSA
     #endif
-    #define WOLFCOSE_HAVE_MLDSA
-#elif !defined(WOLFCOSE_LEAN) && !defined(WOLFCOSE_NO_MLDSA) && defined(WOLFSSL_HAVE_MLDSA)
-    #define WOLFCOSE_HAVE_MLDSA
 #endif
 
 /* HSS/LMS (RFC 8778) — extension */
 /* wc_LmsKey_ImportPubRaw() derives the parameter set from the key bytes only
  * from wolfSSL 5.9.2; earlier releases dereference unset parameters. */
 #if defined(WOLFCOSE_ENABLE_LMS)
-    #ifndef WOLFSSL_HAVE_LMS
+    #if !defined(WOLFSSL_HAVE_LMS)
         #error "WOLFCOSE_ENABLE_LMS requires wolfSSL WOLFSSL_HAVE_LMS"
-    #endif
-    #if LIBWOLFSSL_VERSION_HEX < 0x05009002
+    #elif LIBWOLFSSL_VERSION_HEX < 0x05009002
         #error "WOLFCOSE_ENABLE_LMS requires wolfSSL 5.9.2 or later"
+    #else
+        #define WOLFCOSE_HAVE_LMS
     #endif
-    #define WOLFCOSE_HAVE_LMS
 #elif !defined(WOLFCOSE_LEAN) && !defined(WOLFCOSE_NO_LMS) && \
-      defined(WOLFSSL_HAVE_LMS) && (LIBWOLFSSL_VERSION_HEX >= 0x05009002)
-    #define WOLFCOSE_HAVE_LMS
+      defined(WOLFSSL_HAVE_LMS)
+    #if LIBWOLFSSL_VERSION_HEX < 0x05009002
+        #error "wolfCOSE HSS/LMS requires wolfSSL 5.9.2 or later"
+    #else
+        #define WOLFCOSE_HAVE_LMS
+    #endif
 #endif
 
 /* RSA-PSS (PS256/384/512) — extension */
@@ -278,11 +290,31 @@ extern "C" {
     defined(WOLFCOSE_HAVE_PS512)
     #define WOLFCOSE_HAVE_RSAPSS
 #endif
-/* Private RSA round-trip needs wc_export_int + RsaKey.u; else public-only. */
+/* Private-capable RSA builds enable private serialization with key encoding.
+ * Do not silently downgrade that configuration on an older backend. */
 #if defined(WOLFCOSE_HAVE_RSAPSS) && !defined(WOLFCOSE_RSA_PUBLIC_ONLY) && \
     !defined(WOLFSSL_RSA_PUBLIC_ONLY) && \
+    !defined(WOLFCOSE_NO_KEY_ENCODE) && \
+    (defined(HAVE_ECC) || defined(WOLFSSL_EXPORT_INT)) && \
+    (defined(WOLFSSL_KEY_GEN) || defined(OPENSSL_EXTRA) || \
+     !defined(RSA_LOW_MEM)) && \
+    (LIBWOLFSSL_VERSION_HEX < 0x05009002)
+    #error "Private RSA serialization requires wolfSSL 5.9.2 or later"
+#endif
+/* Private RSA decoding requires wolfSSL's hardened raw decoder from 5.9.0.
+ * Older backends explicitly reject private components. */
+#if defined(WOLFCOSE_HAVE_RSAPSS) && !defined(WOLFCOSE_RSA_PUBLIC_ONLY) && \
+    !defined(WOLFSSL_RSA_PUBLIC_ONLY) && \
+    (LIBWOLFSSL_VERSION_HEX >= 0x05009000)
+    #define WOLFCOSE_HAVE_RSA_PRIVATE_KEY_DECODE
+#endif
+/* Private RSA serialization additionally needs wc_export_int + RsaKey.u and
+ * wolfSSL 5.9.2's bounds-checked fixed-width export. */
+#if defined(WOLFCOSE_HAVE_RSA_PRIVATE_KEY_DECODE) && \
     (defined(HAVE_ECC) || defined(WOLFSSL_EXPORT_INT)) && \
-    (defined(WOLFSSL_KEY_GEN) || defined(OPENSSL_EXTRA) || !defined(RSA_LOW_MEM))
+    (defined(WOLFSSL_KEY_GEN) || defined(OPENSSL_EXTRA) || \
+     !defined(RSA_LOW_MEM)) && \
+    (LIBWOLFSSL_VERSION_HEX >= 0x05009002)
     #define WOLFCOSE_HAVE_RSA_PRIVATE_KEY
 #endif
 #if defined(WOLFCOSE_HAVE_ECDSA) || defined(WOLFCOSE_HAVE_EDDSA) || \

include/wolfcose/wolfcose.h

@@ -999,12 +999,6 @@ WOLFCOSE_API int wc_CoseKey_EncodeSize(const WOLFCOSE_KEY* key,
  * bound, so it can be used to size a buffer or to reject an oversized key
  * before committing storage.
  *
- * One configuration limit: in a build with neither HAVE_ECC nor
- * WOLFSSL_EXPORT_INT, reading the RSA public exponent needs a scratch copy of
- * the modulus, so an RSA key whose modulus exceeds WOLFCOSE_MAX_SCRATCH_SZ
- * returns WOLFCOSE_E_CRYPTO here even though wc_CoseKey_Encode_ex() encodes
- * it. Any build with ECC or WOLFSSL_EXPORT_INT enabled is unaffected.
- *
  * \param key     Key to size.
  * \param outLen  Output: exact encoded size in bytes.
  * \param flags   Bitmask of WOLFCOSE_KEY_* output options.

src/wolfcose_alg.c

@@ -546,21 +546,24 @@ int wolfCose_RsaPssCheckKey(const WOLFCOSE_KEY* key,
         }
         else if (modulusSz == (int)WOLFCOSE_RSA_PSS_MIN_SZ) {
             int modulusMaterialized = 0;
-            uint8_t modulus[WOLFCOSE_RSA_PSS_MIN_SZ] = {0};
+            int modulusExportRet;
+            uint8_t modulus[WOLFCOSE_RSA_PSS_MIN_SZ];
             word32 modulusLen32 = (word32)sizeof(modulus);
 #if !defined(HAVE_ECC) && !defined(WOLFSSL_EXPORT_INT) && \
     !defined(WOLFSSL_RSA_VERIFY_ONLY)
             word32 exponentLen = (word32)sizeof(modulus);
 #endif
+
+            (void)XMEMSET(modulus, 0, sizeof(modulus));
 #if defined(HAVE_ECC) || defined(WOLFSSL_EXPORT_INT)
-            int modulusExportRet = wc_export_int(&rsaKey->n, modulus,
+            modulusExportRet = wc_export_int(&rsaKey->n, modulus,
                 &modulusLen32, (word32)sizeof(modulus),
                 WC_TYPE_UNSIGNED_BIN);
 #elif defined(WOLFSSL_RSA_VERIFY_ONLY)
-            int modulusExportRet = -1;
+            modulusExportRet = -1;
 #else
             /* The modulus output overwrites the unused exponent output. */
-            int modulusExportRet = wc_RsaFlattenPublicKey(rsaKey,
+            modulusExportRet = wc_RsaFlattenPublicKey(rsaKey,
                 modulus, &exponentLen, modulus, &modulusLen32);
 #endif
             if (modulusExportRet == 0) {

src/wolfcose_internal.h

@@ -384,9 +384,8 @@ WOLFCOSE_LOCAL int wolfCose_ExtSign(const WOLFCOSE_KEY* key, int32_t alg,
 /* ----- Internal helpers shared across the split source files ----- */
 
 #ifdef WOLFCOSE_HAVE_RSAPSS
-/* Widest RSA public exponent wolfCOSE emits and the RFC 8230 RSA-PSS
- * minimum modulus width. These are shared by COSE_Key encoding and message
- * operations, including builds that disable COSE_Key encoding. */
+/* RSA component limits shared by COSE_Key encoding and message operations,
+ * including builds that disable COSE_Key encoding. */
 #define WOLFCOSE_RSA_E_MAX_SZ    8u
 #define WOLFCOSE_RSA_PSS_MIN_SZ  256u
 #endif