crypto: split OpenSSL 3, BoringSSL, and legacy backends

This separates the native crypto backend paths for OpenSSL >= 3,
BoringSSL, and legacy OpenSSL. The OpenSSL >= 3 path now builds with
`OPENSSL_API_COMPAT=30000` and `OPENSSL_NO_DEPRECATED`, moving normal
crypto/TLS code away from APIs that OpenSSL 3.0.0 marks deprecated.
BoringSSL remains on its own branch, and OpenSSL < 3 remains the legacy
fallback.

The exception is ENGINE support. ENGINE APIs are isolated into a
dedicated compatibility target so they can remain available while the
JS-facing engine APIs are runtime-deprecated in 27.x. That gives us a
clear removal point for 28.x, without letting ENGINE usage leak back
into the strict OpenSSL 3 path.

The split also makes the eventual OpenSSL 1.1.1 removal easier to reason
about. Once support for OpenSSL < 3 is dropped, the legacy branch can be
removed in a focused follow-up, possibly targeting 27.x, instead of
untangling mixed version guards throughout the crypto implementation.

No public crypto or TLS API behavior is intentionally changed.

Assisted-by: Codex:gpt-5
Signed-off-by: Filip Skokan <panva.ip@gmail.com>
PR-URL: #64211
Refs: #56733
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Richard Lau <richard.lau@ibm.com>
Reviewed-By: James M Snell <jasnell@gmail.com>

Author: panva

common.gypi

@@ -24,6 +24,7 @@
     'node_module_version%': '',
     'node_with_ltcg%': '',
     'node_shared_openssl%': 'false',
+    'openssl_is_boringssl%': 'false',
 
     'node_tag%': '',
     'uv_library%': 'static_library',

configure.py

@@ -1440,6 +1440,48 @@ def get_gas_version(cc):
   warn(f'Could not recognize `gas`: {gas_ret}')
   return '0.0'
 
+def get_openssl_macros(o):
+  """Extract OpenSSL preprocessor macros from the configured headers."""
+
+  # Use the C compiler to extract preprocessor macros from OpenSSL headers.
+  # crypto.h is included because BoringSSL declares OPENSSL_IS_BORINGSSL there.
+  args = ['-E', '-dM',
+          '-include', 'openssl/opensslv.h',
+          '-include', 'openssl/crypto.h',
+          '-']
+  if not options.shared_openssl:
+    args = ['-I', 'deps/openssl/openssl/include'] + args
+  elif options.shared_openssl_includes:
+    args = ['-I', options.shared_openssl_includes] + args
+  else:
+    for dir in o['include_dirs']:
+      args = ['-I', dir] + args
+
+  proc = subprocess.Popen(
+    shlex.split(CC) + args,
+    stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE
+  )
+  with proc:
+    proc.stdin.write(b'\n')
+    out = to_utf8(proc.communicate()[0])
+
+  if proc.returncode != 0:
+    warn('Failed to extract OpenSSL macros from headers')
+    return {}
+
+  macros = {}
+  for line in out.split('\n'):
+    if line.startswith('#define OPENSSL_'):
+      parts = line.split()
+      if len(parts) >= 2:
+        macro_name = parts[1]
+        macro_value = parts[2] if len(parts) >= 3 else '1'
+        macros[macro_name] = macro_value
+
+  return macros
+
 def get_openssl_version(o):
   """Parse OpenSSL version from opensslv.h header file.
 
@@ -1449,39 +1491,7 @@ def get_openssl_version(o):
   """
 
   try:
-    # Use the C compiler to extract preprocessor macros from opensslv.h
-    args = ['-E', '-dM', '-include', 'openssl/opensslv.h', '-']
-    if not options.shared_openssl:
-      args = ['-I', 'deps/openssl/openssl/include'] + args
-    elif options.shared_openssl_includes:
-      args = ['-I', options.shared_openssl_includes] + args
-    else:
-      for dir in o['include_dirs']:
-        args = ['-I', dir] + args
-
-    proc = subprocess.Popen(
-      shlex.split(CC) + args,
-      stdin=subprocess.PIPE,
-      stdout=subprocess.PIPE,
-      stderr=subprocess.PIPE
-    )
-    with proc:
-      proc.stdin.write(b'\n')
-      out = to_utf8(proc.communicate()[0])
-
-    if proc.returncode != 0:
-      warn('Failed to extract OpenSSL version from opensslv.h header')
-      return 0
-
-    # Parse the macro definitions
-    macros = {}
-    for line in out.split('\n'):
-      if line.startswith('#define OPENSSL_VERSION_'):
-        parts = line.split()
-        if len(parts) >= 3:
-          macro_name = parts[1]
-          macro_value = parts[2]
-          macros[macro_name] = macro_value
+    macros = get_openssl_macros(o)
 
     # Extract version components
     major = int(macros.get('OPENSSL_VERSION_MAJOR', '0'))
@@ -1511,6 +1521,13 @@ def get_openssl_version(o):
     warn(f'Failed to determine OpenSSL version from header: {e}')
     return 0
 
+def get_openssl_is_boringssl(o):
+  try:
+    return b('OPENSSL_IS_BORINGSSL' in get_openssl_macros(o))
+  except (OSError, ValueError, subprocess.SubprocessError) as e:
+    warn(f'Failed to determine whether OpenSSL headers are BoringSSL: {e}')
+    return 'false'
+
 def get_cargo_version(cargo):
   try:
     proc = subprocess.Popen(shlex.split(cargo) + ['--version'],
@@ -2315,6 +2332,7 @@ def without_ssl_error(option):
   configure_library('openssl', o)
 
   o['variables']['openssl_version'] = get_openssl_version(o)
+  o['variables']['openssl_is_boringssl'] = get_openssl_is_boringssl(o)
 
 def configure_lief(o):
   if options.without_lief:

deps/ncrypto/engine.cc

@@ -1,12 +1,18 @@
 #include "ncrypto.h"
 
+#if !defined(OPENSSL_NO_ENGINE) &&                                             \
+    ((defined(NCRYPTO_ENGINE_COMPAT) && NCRYPTO_ENGINE_COMPAT) ||              \
+     NCRYPTO_USE_LEGACY_OPENSSL)
+#include <openssl/engine.h>
+#endif
+
 namespace ncrypto {
 
 // ============================================================================
 // Engine
 
 #ifndef OPENSSL_NO_ENGINE
-EnginePointer::EnginePointer(ENGINE* engine_, bool finish_on_exit_)
+EnginePointer::EnginePointer(void* engine_, bool finish_on_exit_)
     : engine(engine_), finish_on_exit(finish_on_exit_) {}
 
 EnginePointer::EnginePointer(EnginePointer&& other) noexcept
@@ -24,21 +30,22 @@ EnginePointer& EnginePointer::operator=(EnginePointer&& other) noexcept {
   return *new (this) EnginePointer(std::move(other));
 }
 
-void EnginePointer::reset(ENGINE* engine_, bool finish_on_exit_) {
+void EnginePointer::reset(void* engine_, bool finish_on_exit_) {
   if (engine != nullptr) {
+    ENGINE* current = static_cast<ENGINE*>(engine);
     if (finish_on_exit) {
       // This also does the equivalent of ENGINE_free.
-      ENGINE_finish(engine);
+      ENGINE_finish(current);
     } else {
-      ENGINE_free(engine);
+      ENGINE_free(current);
     }
   }
   engine = engine_;
   finish_on_exit = finish_on_exit_;
 }
 
-ENGINE* EnginePointer::release() {
-  ENGINE* ret = engine;
+void* EnginePointer::release() {
+  void* ret = engine;
   engine = nullptr;
   finish_on_exit = false;
   return ret;
@@ -52,8 +59,9 @@ EnginePointer EnginePointer::getEngineByName(const char* name,
     // Engine not found, try loading dynamically.
     engine = EnginePointer(ENGINE_by_id("dynamic"));
     if (engine) {
-      if (!ENGINE_ctrl_cmd_string(engine.get(), "SO_PATH", name, 0) ||
-          !ENGINE_ctrl_cmd_string(engine.get(), "LOAD", nullptr, 0)) {
+      ENGINE* current = static_cast<ENGINE*>(engine.engine);
+      if (!ENGINE_ctrl_cmd_string(current, "SO_PATH", name, 0) ||
+          !ENGINE_ctrl_cmd_string(current, "LOAD", nullptr, 0)) {
         engine.reset();
       }
     }
@@ -64,19 +72,24 @@ EnginePointer EnginePointer::getEngineByName(const char* name,
 bool EnginePointer::setAsDefault(uint32_t flags, CryptoErrorList* errors) {
   if (engine == nullptr) return false;
   ClearErrorOnReturn clear_error_on_return(errors);
-  return ENGINE_set_default(engine, flags) != 0;
+  return ENGINE_set_default(static_cast<ENGINE*>(engine), flags) != 0;
 }
 
 bool EnginePointer::init(bool finish_on_exit) {
   if (engine == nullptr) return false;
   if (finish_on_exit) setFinishOnExit();
-  return ENGINE_init(engine) == 1;
+  return ENGINE_init(static_cast<ENGINE*>(engine)) == 1;
 }
 
 EVPKeyPointer EnginePointer::loadPrivateKey(const char* key_name) {
   if (engine == nullptr) return EVPKeyPointer();
-  return EVPKeyPointer(
-      ENGINE_load_private_key(engine, key_name, nullptr, nullptr));
+  return EVPKeyPointer(ENGINE_load_private_key(
+      static_cast<ENGINE*>(engine), key_name, nullptr, nullptr));
+}
+
+bool EnginePointer::setClientCertEngine(SSL_CTX* ctx) {
+  if (engine == nullptr || ctx == nullptr) return false;
+  return SSL_CTX_set_client_cert_engine(ctx, static_cast<ENGINE*>(engine)) == 1;
 }
 
 void EnginePointer::initEnginesOnce() {