diff --git a/thirdparty/patches/brpc-Support-Certificate-Hot-Reload.patch b/thirdparty/patches/brpc-Support-Certificate-Hot-Reload.patch new file mode 100644 index 00000000000000..130cddcc01fc05 --- /dev/null +++ b/thirdparty/patches/brpc-Support-Certificate-Hot-Reload.patch @@ -0,0 +1,603 @@ +From 016b0d1e4956b995c8169fcb2a47d1fd2d3d5753 Mon Sep 17 00:00:00 2001 +From: koarz +Date: Fri, 10 Oct 2025 17:01:46 +0800 +Subject: [PATCH] feat: Support Certificate Hot Reload + +--- + src/brpc/details/ssl_helper.cpp | 107 +++++++++++- + src/brpc/details/ssl_helper.h | 8 + + src/brpc/server.cpp | 282 +++++++++++++++++++++++++++++++- + src/brpc/server.h | 18 ++ + src/brpc/ssl_options.cpp | 2 + + src/brpc/ssl_options.h | 14 ++ + 6 files changed, 428 insertions(+), 3 deletions(-) + +diff --git a/src/brpc/details/ssl_helper.cpp b/src/brpc/details/ssl_helper.cpp +index 76a73547..f305f813 100644 +--- a/src/brpc/details/ssl_helper.cpp ++++ b/src/brpc/details/ssl_helper.cpp +@@ -88,6 +88,96 @@ static int ParseSSLProtocols(const std::string& str_protocol) { + return protocol_flag; + } + ++bool IsPemFormattedKey(const std::string& input) { ++ static const char kPemPrefix[] = "-----BEGIN"; ++ for (const char* s = input.c_str(); *s != '\0'; ++s) { ++ if (*s == '\n' || *s == '\r') { ++ continue; ++ } ++ return strncmp(s, kPemPrefix, sizeof(kPemPrefix) - 1) == 0; ++ } ++ return false; ++} ++ ++int PasswordCallback(char *buf, int size, int rwflag, void *userdata) { ++ const char *pass = static_cast(userdata); ++ int len = strlen(pass); ++ if (len > size) len = size; ++ memcpy(buf, pass, len); ++ return len; ++} ++ ++bool ParsePrivateKeyWithPassword(const std::string& source, ++ const std::string& password, ++ std::string* parsed_key) { ++ if (parsed_key == NULL) { ++ LOG(ERROR) << "Output parameter for parsed key is null"; ++ return false; ++ } ++ parsed_key->clear(); ++ if (source.empty()) { ++ LOG(ERROR) << "Private key is empty"; ++ return false; ++ } ++ ++ using BIOPtr = std::unique_ptr; ++ BIOPtr key_bio(NULL, BIO_free); ++ ++ if (IsPemFormattedKey(source)) { ++ key_bio.reset(BIO_new_mem_buf(const_cast(source.c_str()), -1)); ++ } else { ++ key_bio.reset(BIO_new(BIO_s_file())); ++ if (!key_bio) { ++ LOG(ERROR) << "Fail to create BIO for private key"; ++ return false; ++ } ++ if (BIO_read_filename(key_bio.get(), source.c_str()) <= 0) { ++ LOG(ERROR) << "Fail to read private key from " << source ++ << ": " << SSLError(ERR_get_error()); ++ ERR_clear_error(); ++ return false; ++ } ++ } ++ ++ if (!key_bio) { ++ LOG(ERROR) << "Fail to initialize BIO for private key"; ++ return false; ++ } ++ ++ void* passwd_ptr = static_cast(const_cast(password.c_str())); ++ using EVPKeyPtr = std::unique_ptr; ++ EVP_PKEY* raw_key = PEM_read_bio_PrivateKey(key_bio.get(), NULL, PasswordCallback, passwd_ptr); ++ EVPKeyPtr key(raw_key, EVP_PKEY_free); ++ if (!key) { ++ LOG(ERROR) << "Fail to parse private key: " << SSLError(ERR_get_error()); ++ ERR_clear_error(); ++ return false; ++ } ++ ++ using BIOBufferPtr = std::unique_ptr; ++ BIOBufferPtr buffer(BIO_new(BIO_s_mem()), BIO_free); ++ if (!buffer) { ++ LOG(ERROR) << "Fail to allocate memory BIO for private key"; ++ return false; ++ } ++ ++ if (PEM_write_bio_PrivateKey(buffer.get(), key.get(), NULL, NULL, 0, NULL, NULL) != 1) { ++ LOG(ERROR) << "Fail to export private key: " << SSLError(ERR_get_error()); ++ ERR_clear_error(); ++ return false; ++ } ++ ++ char* data = NULL; ++ long len = BIO_get_mem_data(buffer.get(), &data); ++ if (len <= 0 || data == NULL) { ++ LOG(ERROR) << "Fail to retrieve private key buffer"; ++ return false; ++ } ++ ++ parsed_key->assign(data, static_cast(len)); ++ return true; ++} ++ + std::ostream& operator<<(std::ostream& os, const SSLError& ssl) { + char buf[128]; // Should be enough + ERR_error_string_n(ssl.error, buf, sizeof(buf)); +@@ -449,10 +539,25 @@ SSL_CTX* CreateClientSSLContext(const ChannelSSLOptions& options) { + return NULL; + } + ++ if (!options.client_cert.private_key_passwd.empty()) { ++ ++ } ++ std::string parsed_private_key; ++ const std::string* private_key_in_use = &options.client_cert.private_key; ++ if (!options.client_cert.private_key_passwd.empty()) { ++ if (!ParsePrivateKeyWithPassword(options.client_cert.private_key, ++ options.client_cert.private_key_passwd, ++ &parsed_private_key)) { ++ LOG(ERROR) << "Failed to parse private key with password"; ++ return NULL; ++ } ++ private_key_in_use = &parsed_private_key; ++ } ++ + if (!options.client_cert.certificate.empty() + && LoadCertificate(ssl_ctx.get(), + options.client_cert.certificate, +- options.client_cert.private_key, NULL) != 0) { ++ *private_key_in_use, NULL) != 0) { + return NULL; + } + +diff --git a/src/brpc/details/ssl_helper.h b/src/brpc/details/ssl_helper.h +index 8f09aae2..78bd664d 100644 +--- a/src/brpc/details/ssl_helper.h ++++ b/src/brpc/details/ssl_helper.h +@@ -102,6 +102,14 @@ SSLState DetectSSLState(int fd, int* error_code); + void Print(std::ostream& os, SSL* ssl, const char* sep); + void Print(std::ostream& os, X509* cert, const char* sep); + ++bool IsPemFormattedKey(const std::string& input); ++ ++int PasswordCallback(char *buf, int size, int rwflag, void *userdata); ++ ++bool ParsePrivateKeyWithPassword(const std::string& source, ++ const std::string& password, ++ std::string* parsed_key); ++ + } // namespace brpc + + #endif // BRPC_SSL_HELPER_H +diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp +index a5100745..4839fd26 100644 +--- a/src/brpc/server.cpp ++++ b/src/brpc/server.cpp +@@ -18,6 +18,8 @@ + + #include // wordexp + #include ++#include ++#include + #include // inet_aton + #include // O_CREAT + #include // mkdir +@@ -38,6 +40,13 @@ + #include "brpc/socket_map.h" // SocketMapList + #include "brpc/acceptor.h" // Acceptor + #include "brpc/details/ssl_helper.h" // CreateServerSSLContext ++#ifndef USE_MESALINK ++#include ++#include ++#else ++#include ++#include ++#endif + #include "brpc/protocol.h" // ListProtocols + #include "brpc/nshead_service.h" // NsheadService + #ifdef ENABLE_THRIFT_FRAMED_PROTOCOL +@@ -401,7 +410,10 @@ Server::Server(ProfilerLinker) + , _derivative_thread(INVALID_BTHREAD) + , _keytable_pool(NULL) + , _eps_bvar(&_nerror_bvar) +- , _concurrency(0) { ++ , _concurrency(0) ++ , _ssl_reload_cert_thread(INVALID_BTHREAD) ++ , _ssl_reload_cert_thread_started(false) ++ , _ssl_reload_cert_thread_should_stop(false) { + BAIDU_CASSERT(offsetof(Server, _concurrency) % 64 == 0, + Server_concurrency_must_be_aligned_by_cacheline); + } +@@ -735,6 +747,12 @@ int Server::StartInternal(const butil::EndPoint& endpoint, + const PortRange& port_range, + const ServerOptions *opt) { + std::unique_ptr revert_server(this); ++ ++ { ++ BAIDU_SCOPED_LOCK(_ssl_reload_cert_mutex); ++ _ssl_reload_cert_thread_should_stop = false; ++ } ++ + if (_failed_to_set_max_concurrency_of_method) { + _failed_to_set_max_concurrency_of_method = false; + LOG(ERROR) << "previous call to MaxConcurrencyOf() was failed, " +@@ -1089,6 +1107,16 @@ int Server::StartInternal(const butil::EndPoint& endpoint, + return -1; + } + ++ if (_options.has_ssl_options() && ++ _options.ssl_options().enable_certificate_reload) { ++ if (_options.ssl_options().certificate_reload_interval_s <= 0) { ++ LOG(WARNING) << "SSL certificate reload is enabled but certificate_reload_interval_s<=0, skip watcher"; ++ } else if (!StartSSLCertificateReloadThread()) { ++ LOG(ERROR) << "Fail to start SSL certificate reload thread"; ++ return -1; ++ } ++ } ++ + // Print tips to server launcher. + if (butil::is_endpoint_extended(_listen_addr)) { + const char* builtin_msg = _options.has_builtin_services ? " with builtin service" : ""; +@@ -1222,6 +1250,17 @@ int Server::Join() { + _derivative_thread = INVALID_BTHREAD; + } + ++ if (_ssl_reload_cert_thread_started) { ++ { ++ BAIDU_SCOPED_LOCK(_ssl_reload_cert_mutex); ++ _ssl_reload_cert_thread_should_stop = true; ++ } ++ bthread_stop(_ssl_reload_cert_thread); ++ bthread_join(_ssl_reload_cert_thread, NULL); ++ _ssl_reload_cert_thread_started = false; ++ _ssl_reload_cert_thread = INVALID_BTHREAD; ++ } ++ + g_running_server_count.fetch_sub(1, butil::memory_order_relaxed); + _status = READY; + return 0; +@@ -1897,6 +1936,99 @@ Server::FindServicePropertyByName(const butil::StringPiece& name) const { + return _service_map.seek(name); + } + ++static void* DelayedFreeSSLContext(void* arg) { ++ SSL_CTX* ctx = static_cast(arg); ++ if (ctx == NULL) { ++ return NULL; ++ } ++ ++ const int64_t cleanup_delay_us = 300000000; // 300s grace period ++ bthread_usleep(cleanup_delay_us); ++ SSL_CTX_free(ctx); ++ LOG(INFO) << "Released previous SSL_CTX after certificate reload"; ++ return NULL; ++} ++ ++void Server::ReloadCertificate() { ++ // check open SSL ++ if (_default_ssl_ctx == NULL) { ++ LOG(WARNING) << "Server not start with ssl"; ++ return; ++ } ++ ++ // get default cert ++ const CertInfo& default_cert = _options.ssl_options().default_cert; ++ if (default_cert.certificate.empty()) { ++ LOG(ERROR) << "Default certificate is empty"; ++ return; ++ } ++ ++ // create a temp SSL context for verify new cert ++ std::string parsed_private_key; ++ const std::string* private_key_in_use = &default_cert.private_key; ++ if (!default_cert.private_key_passwd.empty()) { ++ if (!ParsePrivateKeyWithPassword(default_cert.private_key, ++ default_cert.private_key_passwd, ++ &parsed_private_key)) { ++ LOG(ERROR) << "Failed to parse private key with password"; ++ return; ++ } ++ private_key_in_use = &parsed_private_key; ++ } ++ ++ std::vector temp_filters = default_cert.sni_filters; ++ SSL_CTX* new_raw_ctx = CreateServerSSLContext( ++ default_cert.certificate, *private_key_in_use, ++ _options.ssl_options(), &_raw_alpns, &temp_filters); ++ ++ if (new_raw_ctx == NULL) { ++ LOG(ERROR) << "Failed to create new SSL context"; ++ return; ++ } ++ ++ // get origin SSL_CTX for release defer ++ SSL_CTX* origin_ctx = _default_ssl_ctx->raw_ctx; ++ ++ { ++ // replace default_ssl_ctx's raw_ctx ++ BAIDU_SCOPED_LOCK(_default_ssl_ctx_mutex); ++ _default_ssl_ctx->raw_ctx = new_raw_ctx; ++ } ++ ++ // set SNI call back ++#ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME ++ SSL_CTX_set_tlsext_servername_callback(new_raw_ctx, SSLSwitchCTXByHostname); ++ SSL_CTX_set_tlsext_servername_arg(new_raw_ctx, this); ++#endif ++ ++ // update SSL context mapping ++ std::string cert_key(default_cert.certificate); ++ cert_key.append(default_cert.private_key); ++ SSLContext* ssl_ctx = _ssl_ctx_map.seek(cert_key); ++ if (ssl_ctx != NULL) { ++ std::atomic_store_explicit( ++ reinterpret_cast*>(&ssl_ctx->ctx->raw_ctx), ++ new_raw_ctx, ++ std::memory_order_release ++ ); ++ } ++ ++ // defer release the old SSL_CTX to avoid connection suffer trouble ++ if (origin_ctx != NULL) { ++ bthread_t cleanup_thread = INVALID_BTHREAD; ++ bthread_attr_t cleanup_attr = BTHREAD_ATTR_NORMAL; ++ if (bthread_start_background(&cleanup_thread, &cleanup_attr, ++ DelayedFreeSSLContext, ++ reinterpret_cast(origin_ctx)) != 0) { ++ LOG(ERROR) << "Failed to start cleanup bthread for previous SSL_CTX, " ++ "releasing immediately"; ++ SSL_CTX_free(origin_ctx); ++ } ++ } ++ ++ LOG(INFO) << "Successfully reloaded default SSL certificate"; ++} ++ + int Server::AddCertificate(const CertInfo& cert) { + if (!_options.has_ssl_options()) { + LOG(ERROR) << "ServerOptions.ssl_options is not configured yet"; +@@ -1909,11 +2041,23 @@ int Server::AddCertificate(const CertInfo& cert) { + return 0; + } + ++ std::string parsed_private_key; ++ const std::string* private_key_in_use = &cert.private_key; ++ if (!cert.private_key_passwd.empty()) { ++ if (!ParsePrivateKeyWithPassword(cert.private_key, ++ cert.private_key_passwd, ++ &parsed_private_key)) { ++ LOG(ERROR) << "Failed to parse private key with password"; ++ return -1; ++ } ++ private_key_in_use = &parsed_private_key; ++ } ++ + SSLContext ssl_ctx; + ssl_ctx.filters = cert.sni_filters; + ssl_ctx.ctx = std::make_shared(); + SSL_CTX* raw_ctx = CreateServerSSLContext( +- cert.certificate, cert.private_key, ++ cert.certificate, *private_key_in_use, + _options.ssl_options(), &_raw_alpns, &ssl_ctx.filters); + if (raw_ctx == NULL) { + return -1; +@@ -2227,4 +2371,138 @@ int Server::SSLSwitchCTXByHostname(struct ssl_st* ssl, + } + #endif // SSL_CTRL_SET_TLSEXT_HOSTNAME + ++bool Server::StartSSLCertificateReloadThread() { ++ BAIDU_SCOPED_LOCK(_ssl_reload_cert_mutex); ++ if (_ssl_reload_cert_thread_started) { ++ return true; ++ } ++ ++ if (_options.ssl_options().certificate_reload_interval_s <= 0) { ++ return false; ++ } ++ ++ _ssl_reload_cert_thread_should_stop = false; ++ ++ bthread_attr_t attr = BTHREAD_ATTR_NORMAL; ++ if (bthread_start_background(&_ssl_reload_cert_thread, &attr, ++ Server::SSLCertificateReloadWorkerWrapper, this) != 0) { ++ _ssl_reload_cert_thread = INVALID_BTHREAD; ++ return false; ++ } ++ ++ _ssl_reload_cert_thread_started = true; ++ LOG(INFO) << "Started SSL certificate reload worker thread"; ++ return true; ++} ++ ++void* Server::SSLCertificateReloadWorkerWrapper(void* arg) { ++ Server* server = static_cast(arg); ++ return server->SSLCertificateReloadWorker(arg); ++} ++ ++void* Server::SSLCertificateReloadWorker(void* arg) { ++ (void)arg; ++#if defined(__linux__) ++ pthread_setname_np(pthread_self(), "brpc_cert_monitor"); ++#elif defined(__APPLE__) ++ pthread_setname_np("brpc_cert_monitor"); ++#endif ++ if (!_options.has_ssl_options()) { ++ return nullptr; ++ } ++ ++ const ServerSSLOptions& ssl_options = _options.ssl_options(); ++ const std::string& cert_path = ssl_options.default_cert.certificate; ++ const std::string& key_path = ssl_options.default_cert.private_key; ++ const std::string& ca_path = ssl_options.verify.ca_file_path; ++ ++ int interval_s = ssl_options.certificate_reload_interval_s; ++ if (interval_s <= 0) { ++ LOG(WARNING) << "SSL certificate reload thread cancle because certificate_reload_interval_s<=0"; ++ return nullptr; ++ } ++ const int64_t sleep_us = static_cast(interval_s) * 1000000L; ++ ++ time_t last_cert_mtime = 0; ++ time_t last_key_mtime = 0; ++ time_t last_ca_mtime = 0; ++ bool cert_time_valid = false; ++ bool key_time_valid = false; ++ bool ca_time_valid = false; ++ ++ while (true) { ++ { ++ BAIDU_SCOPED_LOCK(_ssl_reload_cert_mutex); ++ if (_ssl_reload_cert_thread_should_stop) { ++ break; ++ } ++ } ++ ++ bool cert_changed = false; ++ bool key_changed = false; ++ bool ca_changed = false; ++ ++ if (!cert_path.empty()) { ++ struct stat cert_stat; ++ if (stat(cert_path.c_str(), &cert_stat) == 0) { ++ if (!cert_time_valid) { ++ last_cert_mtime = cert_stat.st_mtime; ++ cert_time_valid = true; ++ } else if (cert_stat.st_mtime != last_cert_mtime) { ++ last_cert_mtime = cert_stat.st_mtime; ++ cert_changed = true; ++ } ++ } else { ++ PLOG_EVERY_N(WARNING, 60) ++ << "Fail to stat certificate file `" << cert_path << "'"; ++ } ++ } ++ ++ if (!key_path.empty()) { ++ struct stat key_stat; ++ if (stat(key_path.c_str(), &key_stat) == 0) { ++ if (!key_time_valid) { ++ last_key_mtime = key_stat.st_mtime; ++ key_time_valid = true; ++ } else if (key_stat.st_mtime != last_key_mtime) { ++ last_key_mtime = key_stat.st_mtime; ++ key_changed = true; ++ } ++ } else { ++ PLOG_EVERY_N(WARNING, 60) ++ << "Fail to stat private key file `" << key_path << "'"; ++ } ++ } ++ ++ ++ if (!ca_path.empty()) { ++ struct stat ca_stat; ++ if (stat(ca_path.c_str(), &ca_stat) == 0) { ++ if (!ca_time_valid) { ++ last_ca_mtime = ca_stat.st_mtime; ++ ca_time_valid = true; ++ } else if (ca_stat.st_mtime != last_ca_mtime) { ++ last_ca_mtime = ca_stat.st_mtime; ++ ca_changed = true; ++ } ++ } else { ++ PLOG_EVERY_N(WARNING, 60) ++ << "Fail to stat ca file `" << ca_path << "'"; ++ } ++ } ++ ++ if (cert_changed || key_changed || ca_changed) { ++ LOG(INFO) << "Detected certificate changed, reloading SSL context..."; ++ ReloadCertificate(); ++ } ++ ++ if (bthread_usleep(sleep_us) != 0 && errno == ESTOP) { ++ break; ++ } ++ } ++ ++ LOG(INFO) << "SSL certificate reload worker thread exiting"; ++ return NULL; ++} ++ + } // namespace brpc +diff --git a/src/brpc/server.h b/src/brpc/server.h +index 0974ce12..337978a7 100644 +--- a/src/brpc/server.h ++++ b/src/brpc/server.h +@@ -449,6 +449,12 @@ public: + // NOTE: clearing services when server is running is forbidden. + void ClearServices(); + ++ // Reload the default SSL certificate without restarting the server. ++ // This method creates a new SSL context with the updated certificate ++ // and safely replaces the old one with delayed cleanup. ++ // Can be called while the server is running, but it's not thread-safe by itself. ++ void ReloadCertificate(); ++ + // Dynamically add a new certificate into server. It can be called + // while the server is running, but it's not thread-safe by itself. + // Returns 0 on success, -1 otherwise. +@@ -636,6 +642,10 @@ friend class Controller; + static bool ResetCertMappings(CertMaps& bg, const SSLContextMap& ctx_map); + static bool ClearCertMapping(CertMaps& bg); + ++ bool StartSSLCertificateReloadThread(); ++ void* SSLCertificateReloadWorker(void* arg); ++ static void* SSLCertificateReloadWorkerWrapper(void* arg); ++ + AdaptiveMaxConcurrency& MaxConcurrencyOf(MethodProperty*); + int MaxConcurrencyOf(const MethodProperty*) const; + +@@ -696,6 +706,14 @@ friend class Controller; + mutable bvar::Adder _nerror_bvar; + mutable bvar::PerSecond > _eps_bvar; + BAIDU_CACHELINE_ALIGNMENT mutable int32_t _concurrency; ++ ++ // SSL context cleanup members ++ butil::Mutex _ssl_cleanup_mutex; ++ butil::Mutex _default_ssl_ctx_mutex; ++ butil::Mutex _ssl_reload_cert_mutex; ++ bthread_t _ssl_reload_cert_thread; ++ bool _ssl_reload_cert_thread_started; ++ bool _ssl_reload_cert_thread_should_stop; + }; + + // Get the data attached to current searching thread. The data is created by +diff --git a/src/brpc/ssl_options.cpp b/src/brpc/ssl_options.cpp +index e3b8f5b1..dfcb6f85 100644 +--- a/src/brpc/ssl_options.cpp ++++ b/src/brpc/ssl_options.cpp +@@ -34,6 +34,8 @@ ServerSSLOptions::ServerSSLOptions() + , session_lifetime_s(300) + , session_cache_size(20480) + , ecdhe_curve_name("prime256v1") ++ , enable_certificate_reload(false) ++ , certificate_reload_interval_s(3600) + {} + + } // namespace brpc +diff --git a/src/brpc/ssl_options.h b/src/brpc/ssl_options.h +index 4e5d19c0..f5f0696e 100644 +--- a/src/brpc/ssl_options.h ++++ b/src/brpc/ssl_options.h +@@ -36,6 +36,8 @@ struct CertInfo { + // Supported both file path and raw string based on prefix: + std::string private_key; + ++ std::string private_key_passwd; ++ + // Additional hostnames besides those inside the certificate. Wildcards + // are supported but it can only appear once at the beginning (i.e. *.xxx.com). + std::vector sni_filters; +@@ -154,6 +156,18 @@ struct ServerSSLOptions { + // Default: empty + std::string alpns; + ++ // Whether to enable automatic certificate reload. ++ // If true, brpc will watch the certificate and private key files, ++ // and reload them dynamically when they are modified on disk. ++ // This allows certificate rotation without restarting the server. ++ // Default: false (certificate reload is disabled). ++ bool enable_certificate_reload; ++ ++ // Interval in seconds for checking certificate changes when reload is enabled. ++ // Values <= 0 cancle monitor thread ++ // Default: 3600 (1h) ++ int certificate_reload_interval_s; ++ + // TODO: Support OSCP stapling + }; + +-- +2.43.5 + diff --git a/thirdparty/patches/brpc-zzzz-fix-certificate-hot-reload-safety.patch b/thirdparty/patches/brpc-zzzz-fix-certificate-hot-reload-safety.patch new file mode 100644 index 00000000000000..b61d1924a29ea7 --- /dev/null +++ b/thirdparty/patches/brpc-zzzz-fix-certificate-hot-reload-safety.patch @@ -0,0 +1,549 @@ +diff --git a/src/brpc/channel.cpp b/src/brpc/channel.cpp +index a94c09b5b8c7132aa061310390029cd19197cad9..9be42224792beb13b56002f91668ba30e2ef98ef 100644 +--- a/src/brpc/channel.cpp ++++ b/src/brpc/channel.cpp +@@ -301,7 +301,7 @@ static int CreateSocketSSLContext(const ChannelOptions& options, + return -1; + } + *ssl_ctx = std::make_shared(); +- (*ssl_ctx)->raw_ctx = raw_ctx; ++ (*ssl_ctx)->ResetRawContext(raw_ctx); + (*ssl_ctx)->sni_name = options.ssl_options().sni_name; + } else { + (*ssl_ctx) = NULL; +diff --git a/src/brpc/details/mesalink_ssl_helper.cpp b/src/brpc/details/mesalink_ssl_helper.cpp +index afc3861359b13ec173c6d39257f3b0bc1ec27334..8192d329f841ceed76bc70306d1fc73ae438577a 100644 +--- a/src/brpc/details/mesalink_ssl_helper.cpp ++++ b/src/brpc/details/mesalink_ssl_helper.cpp +@@ -266,6 +266,11 @@ SSL_CTX* CreateClientSSLContext(const ChannelSSLOptions& options) { + return NULL; + } + ++ if (!options.client_cert.private_key_passwd.empty()) { ++ LOG(ERROR) << "Encrypted private keys are not supported with MesaLink"; ++ return NULL; ++ } ++ + if (!options.client_cert.certificate.empty() + && LoadCertificate(ssl_ctx.get(), + options.client_cert.certificate, +diff --git a/src/brpc/details/ssl_helper.cpp b/src/brpc/details/ssl_helper.cpp +index 0a715aa34a6ea385990db6738c63a141221c3555..7c57e6366ee94e78c8e2601885387f8741e0f59e 100644 +--- a/src/brpc/details/ssl_helper.cpp ++++ b/src/brpc/details/ssl_helper.cpp +@@ -573,9 +573,6 @@ SSL_CTX* CreateClientSSLContext(const ChannelSSLOptions& options) { + return NULL; + } + +- if (!options.client_cert.private_key_passwd.empty()) { +- +- } + std::string parsed_private_key; + const std::string* private_key_in_use = &options.client_cert.private_key; + if (!options.client_cert.private_key_passwd.empty()) { +diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp +index a32cfe253b107435ce8d68dcb013015a406007ac..c07c4752c73c1ff3fe1bfe7282394692a6cd5d78 100644 +--- a/src/brpc/server.cpp ++++ b/src/brpc/server.cpp +@@ -1980,44 +1980,48 @@ Server::FindServicePropertyByName(const butil::StringPiece& name) const { + return _service_map.seek(name); + } + +-static void* DelayedFreeSSLContext(void* arg) { +- SSL_CTX* ctx = static_cast(arg); +- if (ctx == NULL) { +- return NULL; ++static bool PreparePrivateKey(const CertInfo& cert, ++ std::string* parsed_private_key, ++ const std::string** private_key_in_use) { ++ *private_key_in_use = &cert.private_key; ++ if (cert.private_key_passwd.empty()) { ++ return true; + } +- +- const int64_t cleanup_delay_us = 300000000; // 300s grace period +- bthread_usleep(cleanup_delay_us); +- SSL_CTX_free(ctx); +- LOG(INFO) << "Released previous SSL_CTX after certificate reload"; +- return NULL; ++#ifdef USE_MESALINK ++ LOG(ERROR) << "Encrypted private keys are not supported with MesaLink"; ++ return false; ++#else ++ if (!ParsePrivateKeyWithPassword(cert.private_key, ++ cert.private_key_passwd, ++ parsed_private_key)) { ++ LOG(ERROR) << "Failed to parse private key with password"; ++ return false; ++ } ++ *private_key_in_use = parsed_private_key; ++ return true; ++#endif + } + +-void Server::ReloadCertificate() { ++bool Server::ReloadCertificate() { + // check open SSL + if (_default_ssl_ctx == NULL) { + LOG(WARNING) << "Server not start with ssl"; +- return; ++ return false; + } + + // get default cert + const CertInfo& default_cert = _options.ssl_options().default_cert; + if (default_cert.certificate.empty()) { + LOG(ERROR) << "Default certificate is empty"; +- return; ++ return false; + } + + // create a temp SSL context for verify new cert + std::string parsed_private_key; +- const std::string* private_key_in_use = &default_cert.private_key; +- if (!default_cert.private_key_passwd.empty()) { +- if (!ParsePrivateKeyWithPassword(default_cert.private_key, +- default_cert.private_key_passwd, +- &parsed_private_key)) { +- LOG(ERROR) << "Failed to parse private key with password"; +- return; +- } +- private_key_in_use = &parsed_private_key; ++ const std::string* private_key_in_use = NULL; ++ if (!PreparePrivateKey(default_cert, &parsed_private_key, ++ &private_key_in_use)) { ++ return false; + } + + std::vector temp_filters = default_cert.sni_filters; +@@ -2027,50 +2031,31 @@ void Server::ReloadCertificate() { + + if (new_raw_ctx == NULL) { + LOG(ERROR) << "Failed to create new SSL context"; +- return; ++ return false; + } + +- // get origin SSL_CTX for release defer +- SSL_CTX* origin_ctx = _default_ssl_ctx->raw_ctx; +- +- { +- // replace default_ssl_ctx's raw_ctx +- BAIDU_SCOPED_LOCK(_default_ssl_ctx_mutex); +- _default_ssl_ctx->raw_ctx = new_raw_ctx; +- } +- +- // set SNI call back ++ // Finish configuring the new context before making it visible to any ++ // concurrently accepted connection. + #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME + SSL_CTX_set_tlsext_servername_callback(new_raw_ctx, SSLSwitchCTXByHostname); + SSL_CTX_set_tlsext_servername_arg(new_raw_ctx, this); + #endif +- +- // update SSL context mapping ++ + std::string cert_key(default_cert.certificate); + cert_key.append(default_cert.private_key); + SSLContext* ssl_ctx = _ssl_ctx_map.seek(cert_key); +- if (ssl_ctx != NULL) { +- std::atomic_store_explicit( +- reinterpret_cast*>(&ssl_ctx->ctx->raw_ctx), +- new_raw_ctx, +- std::memory_order_release +- ); +- } +- +- // defer release the old SSL_CTX to avoid connection suffer trouble +- if (origin_ctx != NULL) { +- bthread_t cleanup_thread = INVALID_BTHREAD; +- bthread_attr_t cleanup_attr = BTHREAD_ATTR_NORMAL; +- if (bthread_start_background(&cleanup_thread, &cleanup_attr, +- DelayedFreeSSLContext, +- reinterpret_cast(origin_ctx)) != 0) { +- LOG(ERROR) << "Failed to start cleanup bthread for previous SSL_CTX, " +- "releasing immediately"; +- SSL_CTX_free(origin_ctx); +- } +- } +- ++ if (ssl_ctx == NULL || ssl_ctx->ctx != _default_ssl_ctx) { ++ LOG(ERROR) << "Default SSL context is missing from the certificate map"; ++ SSL_CTX_free(new_raw_ctx); ++ return false; ++ } ++ ++ // ResetRawContext serializes against SSL_new/SSL_set_SSL_CTX. Those calls ++ // take their own OpenSSL reference before the old owner reference is freed. ++ _default_ssl_ctx->ResetRawContext(new_raw_ctx); ++ + LOG(INFO) << "Successfully reloaded default SSL certificate"; ++ return true; + } + + int Server::AddCertificate(const CertInfo& cert) { +@@ -2086,15 +2071,9 @@ int Server::AddCertificate(const CertInfo& cert) { + } + + std::string parsed_private_key; +- const std::string* private_key_in_use = &cert.private_key; +- if (!cert.private_key_passwd.empty()) { +- if (!ParsePrivateKeyWithPassword(cert.private_key, +- cert.private_key_passwd, +- &parsed_private_key)) { +- LOG(ERROR) << "Failed to parse private key with password"; +- return -1; +- } +- private_key_in_use = &parsed_private_key; ++ const std::string* private_key_in_use = NULL; ++ if (!PreparePrivateKey(cert, &parsed_private_key, &private_key_in_use)) { ++ return -1; + } + + SSLContext ssl_ctx; +@@ -2106,12 +2085,11 @@ int Server::AddCertificate(const CertInfo& cert) { + if (raw_ctx == NULL) { + return -1; + } +- ssl_ctx.ctx->raw_ctx = raw_ctx; +- + #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME +- SSL_CTX_set_tlsext_servername_callback(ssl_ctx.ctx->raw_ctx, SSLSwitchCTXByHostname); +- SSL_CTX_set_tlsext_servername_arg(ssl_ctx.ctx->raw_ctx, this); ++ SSL_CTX_set_tlsext_servername_callback(raw_ctx, SSLSwitchCTXByHostname); ++ SSL_CTX_set_tlsext_servername_arg(raw_ctx, this); + #endif ++ ssl_ctx.ctx->ResetRawContext(raw_ctx); + + if (!_reload_cert_maps.Modify(AddCertMapping, ssl_ctx)) { + LOG(ERROR) << "Fail to add mappings into _reload_cert_maps"; +@@ -2225,17 +2203,24 @@ int Server::ResetCertificates(const std::vector& certs) { + SSLContext ssl_ctx; + ssl_ctx.filters = certs[i].sni_filters; + ssl_ctx.ctx = std::make_shared(); +- ssl_ctx.ctx->raw_ctx = CreateServerSSLContext( +- certs[i].certificate, certs[i].private_key, ++ std::string parsed_private_key; ++ const std::string* private_key_in_use = NULL; ++ if (!PreparePrivateKey(certs[i], &parsed_private_key, ++ &private_key_in_use)) { ++ return -1; ++ } ++ SSL_CTX* raw_ctx = CreateServerSSLContext( ++ certs[i].certificate, *private_key_in_use, + _options.ssl_options(), &_raw_alpns, &ssl_ctx.filters); +- if (ssl_ctx.ctx->raw_ctx == NULL) { ++ if (raw_ctx == NULL) { + return -1; + } + + #ifdef SSL_CTRL_SET_TLSEXT_HOSTNAME +- SSL_CTX_set_tlsext_servername_callback(ssl_ctx.ctx->raw_ctx, SSLSwitchCTXByHostname); +- SSL_CTX_set_tlsext_servername_arg(ssl_ctx.ctx->raw_ctx, this); ++ SSL_CTX_set_tlsext_servername_callback(raw_ctx, SSLSwitchCTXByHostname); ++ SSL_CTX_set_tlsext_servername_arg(raw_ctx, this); + #endif ++ ssl_ctx.ctx->ResetRawContext(raw_ctx); + tmp_map[cert_key] = ssl_ctx; + } + +@@ -2410,8 +2395,8 @@ int Server::SSLSwitchCTXByHostname(struct ssl_st* ssl, + } + + // Switch SSL_CTX to the one with correct hostname +- SSL_set_SSL_CTX(ssl, (*pctx)->raw_ctx); +- return SSL_TLSEXT_ERR_OK; ++ return (*pctx)->SetSessionContext(ssl) ++ ? SSL_TLSEXT_ERR_OK : SSL_TLSEXT_ERR_ALERT_FATAL; + } + #endif // SSL_CTRL_SET_TLSEXT_HOSTNAME + +@@ -2485,6 +2470,9 @@ void* Server::SSLCertificateReloadWorker(void* arg) { + bool cert_changed = false; + bool key_changed = false; + bool ca_changed = false; ++ time_t observed_cert_mtime = last_cert_mtime; ++ time_t observed_key_mtime = last_key_mtime; ++ time_t observed_ca_mtime = last_ca_mtime; + + if (!cert_path.empty()) { + struct stat cert_stat; +@@ -2493,7 +2481,7 @@ void* Server::SSLCertificateReloadWorker(void* arg) { + last_cert_mtime = cert_stat.st_mtime; + cert_time_valid = true; + } else if (cert_stat.st_mtime != last_cert_mtime) { +- last_cert_mtime = cert_stat.st_mtime; ++ observed_cert_mtime = cert_stat.st_mtime; + cert_changed = true; + } + } else { +@@ -2509,7 +2497,7 @@ void* Server::SSLCertificateReloadWorker(void* arg) { + last_key_mtime = key_stat.st_mtime; + key_time_valid = true; + } else if (key_stat.st_mtime != last_key_mtime) { +- last_key_mtime = key_stat.st_mtime; ++ observed_key_mtime = key_stat.st_mtime; + key_changed = true; + } + } else { +@@ -2526,7 +2514,7 @@ void* Server::SSLCertificateReloadWorker(void* arg) { + last_ca_mtime = ca_stat.st_mtime; + ca_time_valid = true; + } else if (ca_stat.st_mtime != last_ca_mtime) { +- last_ca_mtime = ca_stat.st_mtime; ++ observed_ca_mtime = ca_stat.st_mtime; + ca_changed = true; + } + } else { +@@ -2537,7 +2525,19 @@ void* Server::SSLCertificateReloadWorker(void* arg) { + + if (cert_changed || key_changed || ca_changed) { + LOG(INFO) << "Detected certificate changed, reloading SSL context..."; +- ReloadCertificate(); ++ if (ReloadCertificate()) { ++ if (cert_changed) { ++ last_cert_mtime = observed_cert_mtime; ++ } ++ if (key_changed) { ++ last_key_mtime = observed_key_mtime; ++ } ++ if (ca_changed) { ++ last_ca_mtime = observed_ca_mtime; ++ } ++ } else { ++ LOG(WARNING) << "Failed to reload SSL certificate; will retry"; ++ } + } + + if (bthread_usleep(sleep_us) != 0 && errno == ESTOP) { +diff --git a/src/brpc/server.h b/src/brpc/server.h +index fdbc5e5a9ece3f46f0e82bf6e3185571af914863..7dd89e1e012e4a17c6ad26d03c4247a859177dc0 100644 +--- a/src/brpc/server.h ++++ b/src/brpc/server.h +@@ -462,10 +462,12 @@ public: + void ClearServices(); + + // Reload the default SSL certificate without restarting the server. +- // This method creates a new SSL context with the updated certificate +- // and safely replaces the old one with delayed cleanup. +- // Can be called while the server is running, but it's not thread-safe by itself. +- void ReloadCertificate(); ++ // This method creates a fully configured SSL context and atomically replaces ++ // the default context. Existing sessions keep their OpenSSL references. ++ // Can be called while the server is running. Concurrent calls to this method ++ // or changes to the certificate options must be serialized by the caller. ++ // Returns true on success, false otherwise. ++ bool ReloadCertificate(); + + // Dynamically add a new certificate into server. It can be called + // while the server is running, but it's not thread-safe by itself. +@@ -725,9 +727,6 @@ friend class Controller; + mutable bvar::PerSecond > _eps_bvar; + BAIDU_CACHELINE_ALIGNMENT mutable int32_t _concurrency; + +- // SSL context cleanup members +- butil::Mutex _ssl_cleanup_mutex; +- butil::Mutex _default_ssl_ctx_mutex; + butil::Mutex _ssl_reload_cert_mutex; + bthread_t _ssl_reload_cert_thread; + bool _ssl_reload_cert_thread_started; +diff --git a/src/brpc/socket.cpp b/src/brpc/socket.cpp +index e9e934635dc568664db357104e56e46f9ead065f..e421ad0383b753fac52f518fcda1ec73b5793988 100644 +--- a/src/brpc/socket.cpp ++++ b/src/brpc/socket.cpp +@@ -1869,7 +1869,7 @@ int Socket::SSLHandshake(int fd, bool server_mode) { + // Free the last session, which may be deprecated when socket failed + SSL_free(_ssl_session); + } +- _ssl_session = CreateSSLSession(_ssl_ctx->raw_ctx, id(), fd, server_mode); ++ _ssl_session = _ssl_ctx->CreateSession(id(), fd, server_mode); + if (_ssl_session == NULL) { + LOG(ERROR) << "Fail to CreateSSLSession"; + return -1; +@@ -2278,7 +2278,7 @@ void Socket::DebugSocket(std::ostream& os, SocketId id) { + << "\nssl_state=" << SSLStateToString(ssl_state); + const SocketSSLContext* ssl_ctx = ptr->_ssl_ctx.get(); + if (ssl_ctx) { +- os << "\ninitial_ssl_ctx=" << ssl_ctx->raw_ctx; ++ os << "\ninitial_ssl_ctx=" << ssl_ctx->raw_ctx_address(); + if (!ssl_ctx->sni_name.empty()) { + os << "\nsni_name=" << ssl_ctx->sni_name; + } +@@ -2798,15 +2798,41 @@ std::string Socket::description() const { + } + + SocketSSLContext::SocketSSLContext() +- : raw_ctx(NULL) ++ : _raw_ctx(NULL) + {} + + SocketSSLContext::~SocketSSLContext() { +- if (raw_ctx) { +- SSL_CTX_free(raw_ctx); ++ ResetRawContext(NULL); ++} ++ ++SSL* SocketSSLContext::CreateSession(SocketId socket_id, int fd, ++ bool server_mode) const { ++ BAIDU_SCOPED_LOCK(_raw_ctx_mutex); ++ return CreateSSLSession(_raw_ctx, socket_id, fd, server_mode); ++} ++ ++bool SocketSSLContext::SetSessionContext(SSL* ssl) const { ++ BAIDU_SCOPED_LOCK(_raw_ctx_mutex); ++ return SSL_set_SSL_CTX(ssl, _raw_ctx) != NULL; ++} ++ ++void SocketSSLContext::ResetRawContext(SSL_CTX* raw_ctx) { ++ SSL_CTX* old_raw_ctx = NULL; ++ { ++ BAIDU_SCOPED_LOCK(_raw_ctx_mutex); ++ old_raw_ctx = _raw_ctx; ++ _raw_ctx = raw_ctx; ++ } ++ if (old_raw_ctx != NULL) { ++ SSL_CTX_free(old_raw_ctx); + } + } + ++const void* SocketSSLContext::raw_ctx_address() const { ++ BAIDU_SCOPED_LOCK(_raw_ctx_mutex); ++ return _raw_ctx; ++} ++ + } // namespace brpc + + +diff --git a/src/brpc/socket.h b/src/brpc/socket.h +index c535669c20434d9e8826f62a5f9d663519f53f01..5dbc33cf377d3ef734f469caa41e71920f5ebf37 100644 +--- a/src/brpc/socket.h ++++ b/src/brpc/socket.h +@@ -24,6 +24,7 @@ + #include // std::set + #include "butil/atomicops.h" // butil::atomic + #include "bthread/types.h" // bthread_id_t ++#include "bthread/mutex.h" // butil::Mutex + #include "butil/iobuf.h" // butil::IOBuf, IOPortal + #include "butil/macros.h" // DISALLOW_COPY_AND_ASSIGN + #include "butil/endpoint.h" // butil::EndPoint +@@ -169,8 +170,19 @@ struct SocketSSLContext { + SocketSSLContext(); + ~SocketSSLContext(); + +- SSL_CTX* raw_ctx; // owned ++ // All operations which acquire or replace the owned SSL_CTX are ++ // serialized so a reload cannot free the context between a reader ++ // loading the pointer and OpenSSL taking its own reference. ++ SSL* CreateSession(SocketId socket_id, int fd, bool server_mode) const; ++ bool SetSessionContext(SSL* ssl) const; ++ void ResetRawContext(SSL_CTX* raw_ctx); ++ const void* raw_ctx_address() const; ++ + std::string sni_name; // useful for clients ++ ++private: ++ mutable butil::Mutex _raw_ctx_mutex; ++ SSL_CTX* _raw_ctx; // owned + }; + + // TODO: Comment fields +diff --git a/test/brpc_ssl_unittest.cpp b/test/brpc_ssl_unittest.cpp +index e101f534ec23e7ff231ad4e5e9ddc8e84fe68f0c..b0022cc17f5804bc9c94a177ecca179af11b8106 100644 +--- a/test/brpc_ssl_unittest.cpp ++++ b/test/brpc_ssl_unittest.cpp +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + #include "brpc/global.h" + #include "brpc/socket.h" + #include "brpc/server.h" +@@ -338,6 +339,87 @@ TEST_F(SSLTest, ssl_reload) { + server.Join(); + } + ++struct CertificateReloadTraffic { ++ int port; ++ butil::atomic stop; ++ ++ CertificateReloadTraffic() : port(0), stop(false) {} ++}; ++ ++void* SendRPCWhileReloadingCertificate(void* arg) { ++ CertificateReloadTraffic* traffic = ++ static_cast(arg); ++ while (!traffic->stop.load(butil::memory_order_acquire)) { ++ brpc::Channel channel; ++ brpc::ChannelOptions options; ++ options.connection_type = brpc::CONNECTION_TYPE_SHORT; ++ options.mutable_ssl_options()->sni_name = "reload.test"; ++ if (channel.Init("127.0.0.1", traffic->port, &options) != 0) { ++ ADD_FAILURE() << "Fail to initialize TLS channel during reload"; ++ break; ++ } ++ ++ brpc::Controller cntl; ++ test::EchoRequest req; ++ test::EchoResponse res; ++ req.set_message(EXP_REQUEST); ++ test::EchoService_Stub stub(&channel); ++ stub.Echo(&cntl, &req, &res, NULL); ++ if (cntl.Failed() || res.message() != EXP_RESPONSE) { ++ ADD_FAILURE() << cntl.ErrorText(); ++ break; ++ } ++ } ++ return NULL; ++} ++ ++TEST_F(SSLTest, hot_reload_default_certificate_during_handshakes) { ++ const int port = 8613; ++ const std::string cert1 = GetRawPemString("cert1.crt"); ++ const std::string key1 = GetRawPemString("cert1.key"); ++ const std::string cert2 = GetRawPemString("cert2.crt"); ++ const std::string key2 = GetRawPemString("cert2.key"); ++ butil::TempFile cert_file("crt"); ++ butil::TempFile key_file("key"); ++ ASSERT_EQ(0, cert_file.save_bin(cert1.data(), cert1.size())); ++ ASSERT_EQ(0, key_file.save_bin(key1.data(), key1.size())); ++ ++ brpc::Server server; ++ brpc::ServerOptions options; ++ options.mutable_ssl_options()->default_cert.certificate = cert_file.fname(); ++ options.mutable_ssl_options()->default_cert.private_key = key_file.fname(); ++ EchoServiceImpl echo_svc; ++ ASSERT_EQ(0, server.AddService( ++ &echo_svc, brpc::SERVER_DOESNT_OWN_SERVICE)); ++ ASSERT_EQ(0, server.Start(port, &options)); ++ ++ CertificateReloadTraffic traffic; ++ traffic.port = port; ++ pthread_t tid; ++ ASSERT_EQ(0, pthread_create( ++ &tid, NULL, SendRPCWhileReloadingCertificate, &traffic)); ++ ++ bool reload_ok = true; ++ for (int i = 0; i < 20; ++i) { ++ const std::string& cert = i % 2 == 0 ? cert2 : cert1; ++ const std::string& key = i % 2 == 0 ? key2 : key1; ++ if (cert_file.save_bin(cert.data(), cert.size()) != 0 ++ || key_file.save_bin(key.data(), key.size()) != 0 ++ || !server.ReloadCertificate()) { ++ reload_ok = false; ++ break; ++ } ++ } ++ ++ traffic.stop.store(true, butil::memory_order_release); ++ ASSERT_EQ(0, pthread_join(tid, NULL)); ++ ASSERT_TRUE(reload_ok); ++ ASSERT_GT(echo_svc.count.load(butil::memory_order_relaxed), 0); ++ CheckCert("reload.test", "cert1"); ++ ASSERT_EQ(0, server.Stop(0)); ++ ASSERT_EQ(0, server.Join()); ++} ++ + #endif // SSL_CTRL_SET_TLSEXT_HOSTNAME + + const int BUFSIZE[] = {64, 128, 256, 1024, 4096};