openssl: Add validation for XOF digests requiring explicit output length

Author: adapik

NEWS

@@ -41,6 +41,11 @@ PHP                                                                        NEWS
   . Fixed OSS-Fuzz #545352966 (default value AST of an SHM-persisted partial).
     (ndossche)
 
+- OpenSSL:
+  . openssl_digest() and openssl_x509_fingerprint() now emit a warning for
+    digest algorithms with no default output length (SHAKE128, SHAKE256 on
+    OpenSSL 3.4 and later) instead of failing silently. (Alexander Danilov)
+
 - PDO:
   . Fixed PDOStatement::getColumnMeta() reading out of bounds for an invalid
     column index. (Ilia Alshanetsky)

ext/openssl/openssl.c

@@ -4984,6 +4984,9 @@ PHP_FUNCTION(openssl_digest)
 		}
 	} else {
 		php_openssl_store_errors();
+		if (EVP_MD_flags(mdtype) & EVP_MD_FLAG_XOF) {
+			php_error_docref(NULL, E_WARNING, "Unsupported digest algorithm: output length must be specified");
+		}
 		zend_string_release_ex(sigbuf, 0);
 		RETVAL_FALSE;
 	}

ext/openssl/openssl_backend_common.c

@@ -581,6 +581,15 @@ X509 *php_openssl_x509_from_zval(
 	return cert;
 }
 
+static void php_openssl_fingerprint_warn(php_stream *stream, zend_enum_StreamErrorCode code, const char *msg)
+{
+	if (stream != NULL) {
+		php_stream_error(stream, NULL, E_WARNING, true, code, "%s", msg);
+	} else {
+		php_error_docref(NULL, E_WARNING, "%s", msg);
+	}
+}
+
 zend_string* php_openssl_x509_fingerprint(X509 *peer, const char *method, bool raw, php_stream *stream)
 {
 	unsigned char md[EVP_MAX_MD_SIZE];
@@ -589,19 +598,20 @@ zend_string* php_openssl_x509_fingerprint(X509 *peer, const char *method, bool r
 	zend_string *ret;
 
 	if (!(mdtype = php_openssl_get_evp_md_by_name(method))) {
-		if (stream != NULL) {
-			php_stream_warn(stream, Generic, "Unknown digest algorithm");
-		} else {
-			php_error_docref(NULL, E_WARNING, "Unknown digest algorithm");
-		}
+		php_openssl_fingerprint_warn(stream, PHP_STREAM_EC(Generic), "Unknown digest algorithm");
 		return NULL;
 	} else if (!X509_digest(peer, mdtype, md, &n)) {
+		bool is_xof = EVP_MD_flags(mdtype) & EVP_MD_FLAG_XOF;
 		php_openssl_release_evp_md(mdtype);
 		php_openssl_store_errors();
-		if (stream != NULL) {
-			php_stream_warn(stream, EncodingFailed, "Could not generate signature");
+		if (is_xof) {
+			php_openssl_fingerprint_warn(
+				stream,
+				PHP_STREAM_EC(Generic),
+			"Unsupported digest algorithm: output length must be specified"
+			);
 		} else {
-			php_error_docref(NULL, E_WARNING, "Could not generate signature");
+			php_openssl_fingerprint_warn(stream, PHP_STREAM_EC(EncodingFailed), "Could not generate signature");
 		}
 		return NULL;
 	}

ext/openssl/tests/openssl_digest_xof_no_length.phpt

@@ -0,0 +1,45 @@
+--TEST--
+openssl_digest() and openssl_x509_fingerprint() with XOF digests that have no default length
+--EXTENSIONS--
+openssl
+--SKIPIF--
+<?php
+if (OPENSSL_VERSION_NUMBER < 0x30400000) die("skip OpenSSL 3.4 or later required: SHAKE has a default length before");
+?>
+--FILE--
+<?php
+
+$cert = "file://" . __DIR__ . "/cert.crt";
+
+foreach (['shake128', 'shake256'] as $algo) {
+    var_dump(openssl_digest('abc', $algo));
+    var_dump(openssl_digest('abc', $algo, true));
+    var_dump(openssl_x509_fingerprint($cert, $algo));
+}
+
+echo "Fixed length digests are not affected\n";
+var_dump(openssl_digest('abc', 'sha3-256'));
+var_dump(strlen(openssl_x509_fingerprint($cert, 'sha3-256')));
+
+?>
+--EXPECTF--
+Warning: openssl_digest(): Unsupported digest algorithm: output length must be specified in %s on line %d
+bool(false)
+
+Warning: openssl_digest(): Unsupported digest algorithm: output length must be specified in %s on line %d
+bool(false)
+
+Warning: openssl_x509_fingerprint(): Unsupported digest algorithm: output length must be specified in %s on line %d
+bool(false)
+
+Warning: openssl_digest(): Unsupported digest algorithm: output length must be specified in %s on line %d
+bool(false)
+
+Warning: openssl_digest(): Unsupported digest algorithm: output length must be specified in %s on line %d
+bool(false)
+
+Warning: openssl_x509_fingerprint(): Unsupported digest algorithm: output length must be specified in %s on line %d
+bool(false)
+Fixed length digests are not affected
+string(64) "3a985da74fe225b2045c172d6bd390bd855f086e3e9d525b46bfe24511431532"
+int(64)