From b60d14a1ed8f6e51d5fa4d348a09cbefefef396c Mon Sep 17 00:00:00 2001 From: chala2001 Date: Sat, 29 Aug 2026 14:38:25 +0530 Subject: [PATCH] Add an opt-in TCP keepalive option to the sync client Long lived requests such as watches are dropped silently when an idle proxy or load balancer closes the connection, because the client never sends anything on it. Setting keep_alive on the Configuration now asks the kernel for the same keepalive timings client-go dials with: probe after 30s idle, then every 15s, giving up after 9 probes. socket_options still wins if it is set, so existing callers are unaffected. --- kubernetes/client/configuration.py | 9 ++ kubernetes/client/rest.py | 3 + kubernetes/utils/__init__.py | 1 + kubernetes/utils/keepalive.py | 74 +++++++++++++++ kubernetes/utils/keepalive_test.py | 140 +++++++++++++++++++++++++++++ scripts/keepalive_patch.diff | 39 ++++++++ scripts/update-client.sh | 3 + 7 files changed, 269 insertions(+) create mode 100644 kubernetes/utils/keepalive.py create mode 100644 kubernetes/utils/keepalive_test.py create mode 100644 scripts/keepalive_patch.diff diff --git a/kubernetes/client/configuration.py b/kubernetes/client/configuration.py index 9d67b322b3..205c7e2edc 100644 --- a/kubernetes/client/configuration.py +++ b/kubernetes/client/configuration.py @@ -387,6 +387,15 @@ def __init__( self.socket_options = socket_options """Options to pass down to the underlying urllib3 socket """ + self.keep_alive = False + """Enable TCP keepalive on the underlying urllib3 sockets. + + Long lived requests such as watches are otherwise dropped + silently by an idle proxy or load balancer. When enabled, the + client asks the kernel for the same keepalive timings client-go + uses. Ignored if ``socket_options`` is set, which takes + precedence. + """ self.datetime_format = datetime_format """datetime format diff --git a/kubernetes/client/rest.py b/kubernetes/client/rest.py index 736c33f3cb..38ce2f838f 100644 --- a/kubernetes/client/rest.py +++ b/kubernetes/client/rest.py @@ -27,6 +27,7 @@ on_retry_after_error, retry_after_backoff, ) +from kubernetes.utils.keepalive import tcp_keepalive_socket_options from kubernetes.client.exceptions import ApiException, ApiValueError SUPPORTED_SOCKS_PROXIES = {"socks5", "socks5h", "socks4", "socks4a"} @@ -160,6 +161,8 @@ def __init__(self, configuration) -> None: if configuration.socket_options is not None: pool_args['socket_options'] = configuration.socket_options + elif getattr(configuration, 'keep_alive', False): + pool_args['socket_options'] = tcp_keepalive_socket_options() if configuration.connection_pool_maxsize is not None: pool_args['maxsize'] = configuration.connection_pool_maxsize diff --git a/kubernetes/utils/__init__.py b/kubernetes/utils/__init__.py index 681123a57c..4b8c07f312 100644 --- a/kubernetes/utils/__init__.py +++ b/kubernetes/utils/__init__.py @@ -25,3 +25,4 @@ on_retry_after_error, retry_after_backoff, retry_after_max_retries, retry_on_conflict, retry_after_seconds) +from .keepalive import tcp_keepalive_socket_options diff --git a/kubernetes/utils/keepalive.py b/kubernetes/utils/keepalive.py new file mode 100644 index 0000000000..0e54a43ea6 --- /dev/null +++ b/kubernetes/utils/keepalive.py @@ -0,0 +1,74 @@ +# Copyright 2026 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +from typing import List, Tuple + +import urllib3 + + +# client-go dials the API server with a 30 second keepalive: +# https://github.com/kubernetes/client-go/blob/master/transport/cache.go +# Go folds that single duration into the idle time and leaves the probe +# interval and count at its own defaults, 15 seconds and 9 probes: +# https://github.com/golang/go/blob/master/src/net/tcpsock.go +# https://github.com/golang/go/blob/master/src/net/dial.go +DEFAULT_IDLE = 30 +DEFAULT_INTERVAL = 15 +DEFAULT_COUNT = 9 + +SocketOptions = List[Tuple[int, int, int]] + + +def tcp_keepalive_socket_options( + idle: int = DEFAULT_IDLE, + interval: int = DEFAULT_INTERVAL, + count: int = DEFAULT_COUNT, +) -> SocketOptions: + """Build urllib3 socket options that enable TCP keepalive. + + The defaults match what client-go asks the kernel for, so an idle + watch is probed after ``idle`` seconds and dropped after ``count`` + unanswered probes ``interval`` seconds apart. + + The returned list starts from ``urllib3``'s own default socket + options, which disable Nagle's algorithm. urllib3 replaces its + defaults with whatever list it is given rather than merging, so + building on them keeps that behaviour. + + Options the platform does not define are left out: macOS spells the + idle time ``TCP_KEEPALIVE``, and Windows only grew the idle and + interval options in Windows 10 1709. + """ + + for name, value in (('idle', idle), ('interval', interval), + ('count', count)): + if value < 1: + raise ValueError('%s must be at least 1' % name) + + options = list(urllib3.connection.HTTPConnection.default_socket_options) + options.append((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)) + + if hasattr(socket, 'TCP_KEEPIDLE'): + options.append((socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, idle)) + elif hasattr(socket, 'TCP_KEEPALIVE'): + options.append((socket.IPPROTO_TCP, socket.TCP_KEEPALIVE, idle)) + + if hasattr(socket, 'TCP_KEEPINTVL'): + options.append((socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, interval)) + + if hasattr(socket, 'TCP_KEEPCNT'): + options.append((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, count)) + + return options diff --git a/kubernetes/utils/keepalive_test.py b/kubernetes/utils/keepalive_test.py new file mode 100644 index 0000000000..005808fb83 --- /dev/null +++ b/kubernetes/utils/keepalive_test.py @@ -0,0 +1,140 @@ +# Copyright 2026 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import socket +import types +import unittest +from unittest import mock + +import urllib3 + +from kubernetes.client import Configuration +from kubernetes.client.rest import RESTClientObject +from kubernetes.utils import keepalive +from kubernetes.utils.keepalive import tcp_keepalive_socket_options + + +def fake_socket_module(**names): + """A stand-in for the socket module exposing only the given names.""" + + defaults = { + 'SOL_SOCKET': socket.SOL_SOCKET, + 'SO_KEEPALIVE': socket.SO_KEEPALIVE, + 'IPPROTO_TCP': socket.IPPROTO_TCP, + } + defaults.update(names) + return types.SimpleNamespace(**defaults) + + +class TestTcpKeepaliveSocketOptions(unittest.TestCase): + + def test_defaults_match_client_go(self): + options = tcp_keepalive_socket_options() + + self.assertIn((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), options) + self.assertIn( + (socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 30), options) + self.assertIn( + (socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 15), options) + self.assertIn((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 9), options) + + def test_keeps_the_urllib3_defaults(self): + options = tcp_keepalive_socket_options() + + defaults = urllib3.connection.HTTPConnection.default_socket_options + for default in defaults: + self.assertIn(default, options) + + def test_custom_timings(self): + options = tcp_keepalive_socket_options(idle=5, interval=2, count=3) + + self.assertIn((socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 5), options) + self.assertIn((socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 2), options) + self.assertIn((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, 3), options) + + def test_timings_must_be_positive(self): + for kwargs in ({'idle': 0}, {'interval': 0}, {'count': 0}): + with self.assertRaises(ValueError): + tcp_keepalive_socket_options(**kwargs) + + def test_options_are_setsockopt_triples(self): + for option in tcp_keepalive_socket_options(): + self.assertEqual(3, len(option)) + for item in option: + self.assertIsInstance(item, int) + + def test_options_apply_to_a_socket(self): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + for level, name, value in tcp_keepalive_socket_options(): + sock.setsockopt(level, name, value) + + self.assertEqual( + 1, sock.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE)) + self.assertEqual( + 30, sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE)) + + def test_falls_back_to_tcp_keepalive_on_macos(self): + macos = fake_socket_module( + TCP_KEEPALIVE=0x10, + TCP_KEEPINTVL=socket.TCP_KEEPINTVL, + TCP_KEEPCNT=socket.TCP_KEEPCNT, + ) + with mock.patch.object(keepalive, 'socket', macos): + options = tcp_keepalive_socket_options() + + self.assertIn((socket.IPPROTO_TCP, 0x10, 30), options) + + def test_skips_options_the_platform_lacks(self): + with mock.patch.object(keepalive, 'socket', fake_socket_module()): + options = tcp_keepalive_socket_options() + + self.assertIn((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1), options) + defaults = urllib3.connection.HTTPConnection.default_socket_options + self.assertEqual(len(defaults) + 1, len(options)) + + +class TestConfigurationKeepAlive(unittest.TestCase): + + def pool_socket_options(self, configuration): + rest_client = RESTClientObject(configuration) + return rest_client.pool_manager.connection_pool_kw.get( + 'socket_options') + + def test_off_by_default(self): + configuration = Configuration() + + self.assertFalse(configuration.keep_alive) + self.assertIsNone(self.pool_socket_options(configuration)) + + def test_enabled(self): + configuration = Configuration() + configuration.keep_alive = True + + self.assertEqual( + tcp_keepalive_socket_options(), + self.pool_socket_options(configuration)) + + def test_socket_options_win(self): + configuration = Configuration() + configuration.keep_alive = True + configuration.socket_options = [ + (socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)] + + self.assertEqual( + configuration.socket_options, + self.pool_socket_options(configuration)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/keepalive_patch.diff b/scripts/keepalive_patch.diff new file mode 100644 index 0000000000..1583fde05c --- /dev/null +++ b/scripts/keepalive_patch.diff @@ -0,0 +1,39 @@ +diff --git a/kubernetes/client/configuration.py b/kubernetes/client/configuration.py +--- a/kubernetes/client/configuration.py ++++ b/kubernetes/client/configuration.py +@@ -387,6 +387,15 @@ + self.socket_options = socket_options + """Options to pass down to the underlying urllib3 socket + """ ++ self.keep_alive = False ++ """Enable TCP keepalive on the underlying urllib3 sockets. ++ ++ Long lived requests such as watches are otherwise dropped ++ silently by an idle proxy or load balancer. When enabled, the ++ client asks the kernel for the same keepalive timings client-go ++ uses. Ignored if ``socket_options`` is set, which takes ++ precedence. ++ """ + + self.datetime_format = datetime_format + """datetime format +diff --git a/kubernetes/client/rest.py b/kubernetes/client/rest.py +--- a/kubernetes/client/rest.py ++++ b/kubernetes/client/rest.py +@@ -27,6 +27,7 @@ + on_retry_after_error, + retry_after_backoff, + ) ++from kubernetes.utils.keepalive import tcp_keepalive_socket_options + from kubernetes.client.exceptions import ApiException, ApiValueError + + SUPPORTED_SOCKS_PROXIES = {"socks5", "socks5h", "socks4", "socks4a"} +@@ -160,6 +161,8 @@ + + if configuration.socket_options is not None: + pool_args['socket_options'] = configuration.socket_options ++ elif getattr(configuration, 'keep_alive', False): ++ pool_args['socket_options'] = tcp_keepalive_socket_options() + + if configuration.connection_pool_maxsize is not None: + pool_args['maxsize'] = configuration.connection_pool_maxsize diff --git a/scripts/update-client.sh b/scripts/update-client.sh index e128e8454c..102db9a1b7 100755 --- a/scripts/update-client.sh +++ b/scripts/update-client.sh @@ -65,6 +65,9 @@ git apply "${SCRIPT_ROOT}/rest_client_patch.diff" echo ">>> restoring Kubernetes client-go retry integration..." git apply "${SCRIPT_ROOT}/client_go_retry_patch.diff" +echo ">>> restoring Kubernetes TCP keepalive option..." +git apply "${SCRIPT_ROOT}/keepalive_patch.diff" + echo ">>> updating version information..." sed -i'' "s/^CLIENT_VERSION = .*/CLIENT_VERSION = \\\"${CLIENT_VERSION}\\\"/" "${SCRIPT_ROOT}/../setup.py" sed -i'' "s/^__version__ = .*/__version__ = \\\"${CLIENT_VERSION}\\\"/" "${CLIENT_ROOT}/__init__.py"