From 5210019046b76f136dad17cd43e301b656ff3d5e Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 26 Aug 2026 20:09:57 -0700 Subject: [PATCH 1/7] Added migration helper project --- CHANGELOG.md | 2 + migration-helpers/pom.xml | 79 +++++++ .../migration/config/ConfigPropertyCache.java | 170 ++++++++++++++ .../config/ConfigurationMigrationHelper.java | 221 ++++++++++++++++++ .../JdbcConfigurationMigrationExample.java | 92 ++++++++ .../config/v1-known-properties.properties | 121 ++++++++++ .../config/v1-to-v2-mappings.properties | 9 + .../config/v2-known-properties.properties | 87 +++++++ .../ConfigurationMigrationHelperTest.java | 119 ++++++++++ ...JdbcConfigurationMigrationExampleTest.java | 28 +++ 10 files changed, 928 insertions(+) create mode 100644 migration-helpers/pom.xml create mode 100644 migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java create mode 100644 migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java create mode 100644 migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java create mode 100644 migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties create mode 100644 migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties create mode 100644 migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties create mode 100644 migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java create mode 100644 migration-helpers/src/test/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExampleTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 861c20280..d35dd6600 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### New Features +- **[migration-helpers]** Added `migration-helpers` module containing `ConfigurationMigrationHelper` and `ConfigPropertyCache` to convert configuration properties and connection URLs from v1 (0.7.1) format to v2 (0.9.8+) format (automatically prefixing ClickHouse server settings with `clickhouse_setting_`, custom headers with `http_header_`, and mapping renamed property keys). + - **[client-v2]** Added an OpenTelemetry implementation of the observability SPI. `Client.Builder.setSpanRecorder(new OpenTelemetrySpanRecorder(openTelemetry))` reports every client operation and every transport request as an OpenTelemetry `CLIENT` span: an operation span is diff --git a/migration-helpers/pom.xml b/migration-helpers/pom.xml new file mode 100644 index 000000000..df998480d --- /dev/null +++ b/migration-helpers/pom.xml @@ -0,0 +1,79 @@ + + + 4.0.0 + + + com.clickhouse + clickhouse-java + ${revision} + + + migration-helpers + jar + + ClickHouse Migration Helpers + Helper utilities for migrating from ClickHouse v1 client/driver to v2 + https://github.com/ClickHouse/clickhouse-java/tree/main/migration-helpers + + + + ${project.parent.groupId} + client-v2 + ${revision} + + + + ${project.parent.groupId} + jdbc-v2 + ${revision} + + + + ${project.parent.groupId} + clickhouse-client + ${revision} + + + + ${project.parent.groupId} + clickhouse-jdbc + ${revision} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.slf4j + slf4j-simple + ${slf4j.version} + + + + + org.testng + testng + ${testng.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java new file mode 100644 index 000000000..49cc8b5f6 --- /dev/null +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java @@ -0,0 +1,170 @@ +package com.clickhouse.migration.config; + +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.http.config.ClickHouseHttpOption; +import com.clickhouse.jdbc.DriverProperties; +import com.clickhouse.jdbc.JdbcConfig; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.sql.DriverPropertyInfo; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Cache for v1 and v2 configuration properties, loaded from resource files and pre-loaded into memory. + */ +public class ConfigPropertyCache { + + private static final Logger log = LoggerFactory.getLogger(ConfigPropertyCache.class); + + private static final String V1_KNOWN_RESOURCE = "/com/clickhouse/migration/config/v1-known-properties.properties"; + private static final String V2_KNOWN_RESOURCE = "/com/clickhouse/migration/config/v2-known-properties.properties"; + private static final String MAPPINGS_RESOURCE = "/com/clickhouse/migration/config/v1-to-v2-mappings.properties"; + + private static final ConfigPropertyCache INSTANCE = new ConfigPropertyCache(); + + private final Set v1KnownProperties; + private final Set v2KnownProperties; + private final Map v1ToV2Mappings; + + public static ConfigPropertyCache getInstance() { + return INSTANCE; + } + + private ConfigPropertyCache() { + Set v1Props = new HashSet<>(); + Set v2Props = new HashSet<>(); + Map mappings = new HashMap<>(); + + // 1. Load properties from resource files + loadPropertiesResource(V1_KNOWN_RESOURCE, v1Props, null); + loadPropertiesResource(V2_KNOWN_RESOURCE, v2Props, null); + loadPropertiesResource(MAPPINGS_RESOURCE, null, mappings); + + // 2. Pre-load / enrich with runtime enum keys from v1 and v2 + enrichWithRuntimeEnums(v1Props, v2Props); + + this.v1KnownProperties = Collections.unmodifiableSet(v1Props); + this.v2KnownProperties = Collections.unmodifiableSet(v2Props); + this.v1ToV2Mappings = Collections.unmodifiableMap(mappings); + + log.debug("Pre-loaded {} v1 properties, {} v2 properties, {} mappings into cache.", + v1KnownProperties.size(), v2KnownProperties.size(), v1ToV2Mappings.size()); + } + + private void loadPropertiesResource(String resourcePath, Set targetSet, Map targetMap) { + try (InputStream in = getClass().getResourceAsStream(resourcePath)) { + if (in != null) { + Properties props = new Properties(); + try (InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) { + props.load(reader); + } + for (String key : props.stringPropertyNames()) { + if (targetSet != null) { + targetSet.add(key.trim()); + } + if (targetMap != null) { + targetMap.put(key.trim(), props.getProperty(key).trim()); + } + } + } else { + log.warn("Migration resource file not found on classpath: {}", resourcePath); + } + } catch (Exception e) { + log.error("Failed to load migration resource file: {}", resourcePath, e); + } + } + + private void enrichWithRuntimeEnums(Set v1Props, Set v2Props) { + // v2 ClientConfigProperties + try { + for (ClientConfigProperties prop : ClientConfigProperties.values()) { + if (prop.getKey() != null) { + v2Props.add(prop.getKey()); + } + } + } catch (Throwable t) { + log.debug("Could not inspect ClientConfigProperties: {}", t.getMessage()); + } + + // v2 DriverProperties + try { + for (DriverProperties prop : DriverProperties.values()) { + if (prop.getKey() != null) { + v2Props.add(prop.getKey()); + } + } + } catch (Throwable t) { + log.debug("Could not inspect DriverProperties: {}", t.getMessage()); + } + + // v1 ClickHouseClientOption + try { + for (ClickHouseClientOption prop : ClickHouseClientOption.values()) { + if (prop.getKey() != null) { + v1Props.add(prop.getKey()); + } + } + } catch (Throwable t) { + log.debug("Could not inspect ClickHouseClientOption: {}", t.getMessage()); + } + + // v1 ClickHouseHttpOption + try { + for (ClickHouseHttpOption prop : ClickHouseHttpOption.values()) { + if (prop.getKey() != null) { + v1Props.add(prop.getKey()); + } + } + } catch (Throwable t) { + log.debug("Could not inspect ClickHouseHttpOption: {}", t.getMessage()); + } + + // v1 JdbcConfig + try { + for (DriverPropertyInfo info : JdbcConfig.getDriverProperties()) { + if (info.name != null) { + v1Props.add(info.name); + } + } + } catch (Throwable t) { + log.debug("Could not inspect JdbcConfig properties: {}", t.getMessage()); + } + } + + public boolean isV1KnownProperty(String key) { + return key != null && v1KnownProperties.contains(key); + } + + public boolean isV2KnownProperty(String key) { + return key != null && v2KnownProperties.contains(key); + } + + public String getV2MappedKey(String v1Key) { + if (v1Key == null) { + return null; + } + return v1ToV2Mappings.getOrDefault(v1Key, v1Key); + } + + public Set getV1KnownProperties() { + return v1KnownProperties; + } + + public Set getV2KnownProperties() { + return v2KnownProperties; + } + + public Map getV1ToV2Mappings() { + return v1ToV2Mappings; + } +} diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java new file mode 100644 index 000000000..56688ba50 --- /dev/null +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java @@ -0,0 +1,221 @@ +package com.clickhouse.migration.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Migration helper for converting ClickHouse configuration properties and connection URLs + * from v1 (0.7.1) format to v2 (0.9.8+) format. + */ +public class ConfigurationMigrationHelper { + + private static final Logger log = LoggerFactory.getLogger(ConfigurationMigrationHelper.class); + + public static final String SERVER_SETTING_PREFIX = "clickhouse_setting_"; + public static final String HTTP_HEADER_PREFIX = "http_header_"; + + /** + * Converts a {@link Properties} object from v1 format to v2 format. + * + * @param v1Properties source properties in v1 format + * @return converted properties in v2 format + */ + public static Properties convertProperties(Properties v1Properties) { + if (v1Properties == null) { + return new Properties(); + } + Properties v2Properties = new Properties(); + Map convertedMap = convertMap(propertiesToMap(v1Properties)); + for (Map.Entry entry : convertedMap.entrySet()) { + if (entry.getKey() != null && entry.getValue() != null) { + v2Properties.setProperty(entry.getKey(), entry.getValue()); + } + } + return v2Properties; + } + + /** + * Converts a configuration map from v1 format to v2 format. + * Unprefixed ClickHouse server settings are automatically prefixed with {@code clickhouse_setting_}. + * Custom HTTP headers are prefixed with {@code http_header_}. + * Renamed v1 client/driver properties are mapped to their corresponding v2 names. + * + * @param v1Config map of configuration key-values in v1 format + * @return converted map in v2 format + */ + public static Map convertMap(Map v1Config) { + if (v1Config == null) { + return new LinkedHashMap<>(); + } + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + Map v2Config = new LinkedHashMap<>(); + + for (Map.Entry entry : v1Config.entrySet()) { + String origKey = entry.getKey(); + String value = entry.getValue(); + + if (origKey == null) { + continue; + } + + String key = origKey.trim(); + + // 1. If key already starts with clickhouse_setting_ or http_header_, preserve it as is. + if (key.toLowerCase().startsWith(SERVER_SETTING_PREFIX) || key.toLowerCase().startsWith(HTTP_HEADER_PREFIX)) { + v2Config.put(key, value); + continue; + } + + // 2. Handle legacy composite setting properties: custom_settings, custom_http_params, custom_params + if ("custom_settings".equalsIgnoreCase(key) || "custom_http_params".equalsIgnoreCase(key) || "custom_params".equalsIgnoreCase(key)) { + parseAndAddKeyValuePairs(value, SERVER_SETTING_PREFIX, v2Config); + continue; + } + + // Handle legacy composite header properties: custom_http_headers, custom_headers + if ("custom_http_headers".equalsIgnoreCase(key) || "custom_headers".equalsIgnoreCase(key)) { + parseAndAddKeyValuePairs(value, HTTP_HEADER_PREFIX, v2Config); + continue; + } + + // 3. Check for mapped renamed key (e.g. connect_timeout -> connection_timeout) + String mappedKey = cache.getV2MappedKey(key); + + // 4. If key or mappedKey is a known v2 property, keep as client/driver property + if (cache.isV2KnownProperty(mappedKey)) { + v2Config.put(mappedKey, value); + } else if (cache.isV2KnownProperty(key)) { + v2Config.put(key, value); + } else { + // 5. Unrecognized key: in v1 this was implicitly treated as a ClickHouse server setting. + // In v2, it must be explicitly prefixed with clickhouse_setting_ + String serverSettingKey = SERVER_SETTING_PREFIX + key; + v2Config.put(serverSettingKey, value); + } + } + + return v2Config; + } + + /** + * Converts an Object-valued configuration map from v1 to v2 format. + * + * @param v1Config map with Object values + * @return converted map with Object values + */ + public static Map convertObjectMap(Map v1Config) { + if (v1Config == null) { + return new LinkedHashMap<>(); + } + Map strMap = new LinkedHashMap<>(); + for (Map.Entry entry : v1Config.entrySet()) { + if (entry.getKey() != null) { + strMap.put(entry.getKey(), entry.getValue() != null ? entry.getValue().toString() : null); + } + } + Map convertedStrMap = convertMap(strMap); + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : convertedStrMap.entrySet()) { + result.put(entry.getKey(), entry.getValue()); + } + return result; + } + + /** + * Converts a connection URL containing query parameters from v1 format to v2 format. + * Example: + * {@code jdbc:clickhouse://localhost:8123/db?max_threads=8&connect_timeout=5000} + * -> {@code jdbc:clickhouse://localhost:8123/db?clickhouse_setting_max_threads=8&connection_timeout=5000} + * + * @param url connection string/URL + * @return converted connection string/URL in v2 format + */ + public static String convertUrl(String url) { + if (url == null || url.trim().isEmpty()) { + return url; + } + + int queryIndex = url.indexOf('?'); + if (queryIndex < 0 || queryIndex == url.length() - 1) { + return url; + } + + String baseUrl = url.substring(0, queryIndex); + String queryString = url.substring(queryIndex + 1); + + Map queryParams = parseQueryString(queryString); + Map convertedParams = convertMap(queryParams); + + StringBuilder sb = new StringBuilder(baseUrl).append('?'); + boolean first = true; + for (Map.Entry entry : convertedParams.entrySet()) { + if (!first) { + sb.append('&'); + } + sb.append(entry.getKey()); + if (entry.getValue() != null) { + sb.append('=').append(entry.getValue()); + } + first = false; + } + + return sb.toString(); + } + + private static Map propertiesToMap(Properties props) { + Map map = new LinkedHashMap<>(); + for (String name : props.stringPropertyNames()) { + map.put(name, props.getProperty(name)); + } + return map; + } + + private static void parseAndAddKeyValuePairs(String valueStr, String prefix, Map targetMap) { + if (valueStr == null || valueStr.trim().isEmpty()) { + return; + } + String[] pairs = valueStr.split(","); + for (String pair : pairs) { + String trimmed = pair.trim(); + if (trimmed.isEmpty()) { + continue; + } + int eqIndex = trimmed.indexOf('='); + if (eqIndex > 0) { + String k = trimmed.substring(0, eqIndex).trim(); + String v = trimmed.substring(eqIndex + 1).trim(); + if (!k.isEmpty()) { + if (!k.toLowerCase().startsWith(prefix)) { + k = prefix + k; + } + targetMap.put(k, v); + } + } + } + } + + private static Map parseQueryString(String queryString) { + Map map = new LinkedHashMap<>(); + if (queryString == null || queryString.trim().isEmpty()) { + return map; + } + String[] pairs = queryString.split("&"); + for (String pair : pairs) { + if (pair.isEmpty()) { + continue; + } + int eqIdx = pair.indexOf('='); + if (eqIdx >= 0) { + String k = pair.substring(0, eqIdx); + String v = pair.substring(eqIdx + 1); + map.put(k, v); + } else { + map.put(pair, ""); + } + } + return map; + } +} diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java b/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java new file mode 100644 index 000000000..6d8d99579 --- /dev/null +++ b/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java @@ -0,0 +1,92 @@ +package com.clickhouse.migration.examples; + +import com.clickhouse.migration.config.ConfigurationMigrationHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +/** + * Example demonstrating how to migrate v1 JDBC configuration properties and connection URLs + * to v2 format using {@link ConfigurationMigrationHelper} before establishing a JDBC connection. + */ +public class JdbcConfigurationMigrationExample { + + private static final Logger log = LoggerFactory.getLogger(JdbcConfigurationMigrationExample.class); + + /** + * Converts legacy v1 connection properties to v2 format and creates a JDBC connection. + * + * @param url connection URL (e.g., "jdbc:clickhouse://localhost:8123/default") + * @param v1Properties legacy properties containing v1 option names and un-prefixed server settings + * @return active JDBC connection + * @throws SQLException if a database access error occurs + */ + public Connection createConnectionWithConvertedProperties(String url, Properties v1Properties) throws SQLException { + // Convert v1 properties (un-prefixed server settings, renamed keys, custom_settings) to v2 format + Properties v2Properties = ConfigurationMigrationHelper.convertProperties(v1Properties); + + // Connect using standard JDBC DriverManager with converted v2 properties + return DriverManager.getConnection(url, v2Properties); + } + + /** + * Converts a legacy v1 connection URL (containing query parameters) to v2 format + * and creates a JDBC connection. + * + * @param v1Url v1 connection URL containing query parameters (e.g. "jdbc:clickhouse://localhost:8123/default?max_threads=8&connect_timeout=5000") + * @return active JDBC connection + * @throws SQLException if a database access error occurs + */ + public Connection createConnectionWithConvertedUrl(String v1Url) throws SQLException { + // Convert v1 URL query parameters to v2 format (e.g. max_threads -> clickhouse_setting_max_threads) + String v2Url = ConfigurationMigrationHelper.convertUrl(v1Url); + + // Connect using standard JDBC DriverManager with the converted v2 URL + return DriverManager.getConnection(v2Url); + } + + /** + * Demonstrates complete workflow: migrating v1 configuration and executing a query with the v2 JDBC driver. + * + * @throws SQLException if a database access error occurs + */ + public void executeQueryWithMigratedConfig() throws SQLException { + // 1. Build legacy v1 properties + Properties v1Props = new Properties(); + v1Props.setProperty("user", "default"); + v1Props.setProperty("password", ""); + v1Props.setProperty("connect_timeout", "10000"); // Renamed in v2 to connection_timeout + v1Props.setProperty("max_threads", "4"); // Server setting in v1; needs clickhouse_setting_ prefix in v2 + v1Props.setProperty("custom_settings", "join_use_nulls=1"); // Legacy custom_settings property + + // 2. Convert to v2 properties + Properties v2Props = ConfigurationMigrationHelper.convertProperties(v1Props); + log.info("v2Props: {}", v2Props); + + // 3. Connect and execute query with try-with-resources + String url = "jdbc:clickhouse://localhost:8123/default"; + try (Connection conn = DriverManager.getConnection(url, v2Props); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT 1")) { + + while (rs.next()) { + int value = rs.getInt(1); + // process result + } + } + } + + public static void main(String[] args) { + try { + new JdbcConfigurationMigrationExample().executeQueryWithMigratedConfig(); + } catch (Exception e) { + log.error("failed to query with migration config", e); + } + } +} diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties new file mode 100644 index 000000000..d1b4e156d --- /dev/null +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties @@ -0,0 +1,121 @@ +# Known client and driver options in v1 (0.7.1) +async=async +auto_discovery=auto_discovery +custom_settings=custom_settings +custom_socket_factory=custom_socket_factory +custom_socket_factory_options=custom_socket_factory_options +load_balancing_policy=load_balancing_policy +load_balancing_tags=load_balancing_tags +health_check_interval=health_check_interval +health_check_method=health_check_method +node_discovery_interval=node_discovery_interval +node_discovery_limit=node_discovery_limit +node_check_interval=node_check_interval +node_group_size=node_group_size +check_all_nodes=check_all_nodes +buffer_size=buffer_size +buffer_queue_variation=buffer_queue_variation +read_buffer_size=read_buffer_size +write_buffer_size=write_buffer_size +request_chunk_size=request_chunk_size +request_buffering=request_buffering +response_buffering=response_buffering +client_name=client_name +compress=compress +decompress=decompress +compress_algorithm=compress_algorithm +decompress_algorithm=decompress_algorithm +compress_level=compress_level +decompress_level=decompress_level +connect_timeout=connect_timeout +database=database +failover=failover +format=format +log_leading_comment=log_leading_comment +max_buffer_size=max_buffer_size +max_mapper_cache=max_mapper_cache +max_execution_time=max_execution_time +max_queued_buffers=max_queued_buffers +max_queued_requests=max_queued_requests +max_result_rows=max_result_rows +result_overflow_mode=result_overflow_mode +max_threads_per_client=max_threads_per_client +max_core_thread_ttl=max_core_thread_ttl +product_name=product_name +rename_response_column=rename_response_column +retry=retry +repeat_on_session_lock=repeat_on_session_lock +reuse_value_wrapper=reuse_value_wrapper +server_revision=server_revision +server_time_zone=server_time_zone +server_version=server_version +session_id=session_id +session_check=session_check +session_timeout=session_timeout +socket_timeout=socket_timeout +socket_reuseaddr=socket_reuseaddr +socket_keepalive=socket_keepalive +socket_linger=socket_linger +socket_ip_tos=socket_ip_tos +socket_tcp_nodelay=socket_tcp_nodelay +socket_rcvbuf=socket_rcvbuf +socket_sndbuf=socket_sndbuf +ssl=ssl +sslmode=sslmode +sslrootcert=sslrootcert +sslcert=sslcert +sslkey=sslkey +key_store_type=key_store_type +trust_store=trust_store +key_store_password=key_store_password +transaction_timeout=transaction_timeout +widen_unsigned_types=widen_unsigned_types +use_binary_string=use_binary_string +use_blocking_queue=use_blocking_queue +use_compilation=use_compilation +use_objects_in_arrays=use_objects_in_arrays +proxy_type=proxy_type +proxy_host=proxy_host +proxy_port=proxy_port +proxy_username=proxy_username +proxy_password=proxy_password +use_server_time_zone=use_server_time_zone +use_server_time_zone_for_dates=use_server_time_zone_for_dates +use_time_zone=use_time_zone +query_id=query_id +connection_ttl=connection_ttl +debug_measure_request_time=debug_measure_request_time +ssl_socket_sni=ssl_socket_sni +http_connection_provider=http_connection_provider +custom_http_headers=custom_http_headers +custom_headers=custom_headers +custom_http_params=custom_http_params +custom_params=custom_params +http_server_default_response=http_server_default_response +http_keep_alive=http_keep_alive +max_open_connections=max_open_connections +receive_query_progress=receive_query_progress +send_http_client_id=send_http_client_id +wait_end_of_query=wait_end_of_query +remember_last_set_roles=remember_last_set_roles +ahc_validate_after_inactivity=ahc_validate_after_inactivity +ahc_retry_on_failure=ahc_retry_on_failure +connection_reuse_strategy=connection_reuse_strategy +alive_timeout=alive_timeout +http_use_basic_auth=http_use_basic_auth +autoCommit=autoCommit +createDatabaseIfNotExist=createDatabaseIfNotExist +continueBatchOnError=continueBatchOnError +databaseTerm=databaseTerm +dialect=dialect +externalDatabase=externalDatabase +fetchSize=fetchSize +localFile=localFile +jdbcCompliant=jdbcCompliant +namedParameter=namedParameter +nullAsDefault=nullAsDefault +transactionSupport=transactionSupport +typeMappings=typeMappings +wrapperObject=wrapperObject +user=user +password=password diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties new file mode 100644 index 000000000..80d076a45 --- /dev/null +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties @@ -0,0 +1,9 @@ +# Property mappings from v1 keys to v2 keys +connect_timeout=connection_timeout +buffer_size=client_network_buffer_size +sslmode=ssl_mode +sslkey=ssl_key +proxy_username=proxy_user +alive_timeout=http_keep_alive_timeout +typeMappings=jdbc_type_mappings +databaseTerm=jdbc_schema_term diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties new file mode 100644 index 000000000..87ae5f808 --- /dev/null +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties @@ -0,0 +1,87 @@ +# Known client and driver options in v2 (0.9.8) +session_db_roles=session_db_roles +http_use_basic_auth=http_use_basic_auth +user=user +password=password +max_open_connections=max_open_connections +http_keep_alive_timeout=http_keep_alive_timeout +use_server_time_zone=use_server_time_zone +use_time_zone=use_time_zone +server_version=server_version +server_time_zone=server_time_zone +async=async +connection_ttl=connection_ttl +connection_timeout=connection_timeout +connection_reuse_strategy=connection_reuse_strategy +socket_timeout=socket_timeout +socket_rcvbuf=socket_rcvbuf +socket_sndbuf=socket_sndbuf +socket_reuseaddr=socket_reuseaddr +socket_keepalive=socket_keepalive +socket_tcp_nodelay=socket_tcp_nodelay +socket_linger=socket_linger +database=database +compress=compress +decompress=decompress +client.use_http_compression=client.use_http_compression +compression.lz4.uncompressed_buffer_size=compression.lz4.uncompressed_buffer_size +disable_native_compression=disable_native_compression +proxy_type=proxy_type +proxy_host=proxy_host +proxy_port=proxy_port +proxy_user=proxy_user +proxy_password=proxy_password +max_execution_time=max_execution_time +trust_store=trust_store +key_store_type=key_store_type +ssl_key_store=ssl_key_store +key_store_password=key_store_password +ssl_key=ssl_key +sslrootcert=sslrootcert +sslcert=sslcert +ssl_mode=ssl_mode +ssl_context=ssl_context +retry=retry +format=format +max_threads_per_client=max_threads_per_client +query_id=query_id +client_network_buffer_size=client_network_buffer_size +access_token=access_token +ssl_authentication=ssl_authentication +connection_pool_enabled=connection_pool_enabled +connection_request_timeout=connection_request_timeout +client_retry_on_failures=client_retry_on_failures +client_name=client_name +product_name=product_name +bearer_token=bearer_token +app_compressed_data=app_compressed_data +metrics_name=metrics_name +client.http.cookies_enabled=client.http.cookies_enabled +client_allow_binary_reader_to_reuse_buffers=client_allow_binary_reader_to_reuse_buffers +type_hint_mapping=type_hint_mapping +binary_string_support=binary_string_support +ssl_socket_sni=ssl_socket_sni +client.http.use_form_request_for_query=client.http.use_form_request_for_query +json_disable_number_quoting=json_disable_number_quoting +custom_settings_prefix=custom_settings_prefix +ssl_cipher_suites=ssl_cipher_suites + +# JDBC Driver V2 specific properties +jdbc_ignore_unsupported_values=jdbc_ignore_unsupported_values +jdbc_schema_term=jdbc_schema_term +ssl=ssl +default_query_settings=default_query_settings +beta.row_binary_for_simple_insert=beta.row_binary_for_simple_insert +jdbc_resultset_auto_close=jdbc_resultset_auto_close +jdbc_use_max_result_rows=jdbc_use_max_result_rows +jdbc_sql_parser=jdbc_sql_parser +jdbc_query_id_generator=jdbc_query_id_generator +remember_last_set_roles=remember_last_set_roles +custom_http_params=custom_http_params +custom_settings=custom_settings +use_server_time_zone_for_dates=use_server_time_zone_for_dates +http_connection_provider=http_connection_provider +jdbc_cluster_name=jdbc_cluster_name +jdbc_type_mappings=jdbc_type_mappings +typeMappings=typeMappings +jdbc_json_parser_factory=jdbc_json_parser_factory diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java new file mode 100644 index 000000000..422ea7a0c --- /dev/null +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java @@ -0,0 +1,119 @@ +package com.clickhouse.migration.config; + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +public class ConfigurationMigrationHelperTest { + + @DataProvider(name = "propertyConversionData") + public Object[][] providePropertyConversionData() { + return new Object[][]{ + // Standard known v2 properties should remain as-is + {"user", "default", "user", "default"}, + {"password", "secret", "password", "secret"}, + {"database", "analytics", "database", "analytics"}, + {"ssl", "true", "ssl", "true"}, + {"async", "false", "async", "false"}, + + // Unprefixed server settings in v1 must be prefixed with clickhouse_setting_ in v2 + {"max_threads", "8", "clickhouse_setting_max_threads", "8"}, + {"date_time_input_format", "best_effort", "clickhouse_setting_date_time_input_format", "best_effort"}, + {"join_use_nulls", "1", "clickhouse_setting_join_use_nulls", "1"}, + + // Renamed properties in v1 should be converted to v2 names + {"connect_timeout", "10000", "connection_timeout", "10000"}, + {"buffer_size", "65536", "client_network_buffer_size", "65536"}, + {"sslmode", "strict", "ssl_mode", "strict"}, + {"sslkey", "/path/to/key", "ssl_key", "/path/to/key"}, + {"proxy_username", "puser", "proxy_user", "puser"}, + + // Existing v2 prefixed properties should be preserved + {"clickhouse_setting_max_execution_time", "60", "clickhouse_setting_max_execution_time", "60"}, + {"http_header_X-Custom-Header", "custom-val", "http_header_X-Custom-Header", "custom-val"} + }; + } + + @Test(dataProvider = "propertyConversionData") + public void testConvertSingleProperty(String inputKey, String inputValue, String expectedKey, String expectedValue) { + Map input = new HashMap<>(); + input.put(inputKey, inputValue); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.size(), 1); + Assert.assertTrue(result.containsKey(expectedKey), "Expected key missing: " + expectedKey); + Assert.assertEquals(result.get(expectedKey), expectedValue); + } + + @Test + public void testConvertCustomSettingsAndHeaders() { + Map input = new LinkedHashMap<>(); + input.put("user", "my_user"); + input.put("custom_settings", "max_threads=4, join_use_nulls=1"); + input.put("custom_http_headers", "X-Trace-Id=123, X-App-Name=demo"); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.get("user"), "my_user"); + Assert.assertEquals(result.get("clickhouse_setting_max_threads"), "4"); + Assert.assertEquals(result.get("clickhouse_setting_join_use_nulls"), "1"); + Assert.assertEquals(result.get("http_header_X-Trace-Id"), "123"); + Assert.assertEquals(result.get("http_header_X-App-Name"), "demo"); + } + + @Test + public void testConvertPropertiesObject() { + Properties v1Props = new Properties(); + v1Props.setProperty("user", "default"); + v1Props.setProperty("connect_timeout", "5000"); + v1Props.setProperty("max_threads", "16"); + + Properties v2Props = ConfigurationMigrationHelper.convertProperties(v1Props); + + Assert.assertEquals(v2Props.getProperty("user"), "default"); + Assert.assertEquals(v2Props.getProperty("connection_timeout"), "5000"); + Assert.assertEquals(v2Props.getProperty("clickhouse_setting_max_threads"), "16"); + } + + @DataProvider(name = "urlConversionData") + public Object[][] provideUrlConversionData() { + return new Object[][]{ + { + "jdbc:clickhouse://localhost:8123/default?user=default&connect_timeout=5000&max_threads=8", + "jdbc:clickhouse://localhost:8123/default?user=default&connection_timeout=5000&clickhouse_setting_max_threads=8" + }, + { + "http://localhost:8123/?ssl=true&date_time_input_format=best_effort", + "http://localhost:8123/?ssl=true&clickhouse_setting_date_time_input_format=best_effort" + }, + { + "jdbc:clickhouse://localhost:8123/db", + "jdbc:clickhouse://localhost:8123/db" + } + }; + } + + @Test(dataProvider = "urlConversionData") + public void testConvertUrl(String inputUrl, String expectedUrl) { + String resultUrl = ConfigurationMigrationHelper.convertUrl(inputUrl); + Assert.assertEquals(resultUrl, expectedUrl); + } + + @Test + public void testCacheInitializationAndPreload() { + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + + Assert.assertTrue(cache.isV1KnownProperty("connect_timeout")); + Assert.assertTrue(cache.isV2KnownProperty("connection_timeout")); + Assert.assertTrue(cache.isV2KnownProperty("user")); + + Assert.assertEquals(cache.getV2MappedKey("connect_timeout"), "connection_timeout"); + Assert.assertEquals(cache.getV2MappedKey("buffer_size"), "client_network_buffer_size"); + } +} diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExampleTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExampleTest.java new file mode 100644 index 000000000..77a1e053e --- /dev/null +++ b/migration-helpers/src/test/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExampleTest.java @@ -0,0 +1,28 @@ +package com.clickhouse.migration.examples; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.Properties; + +public class JdbcConfigurationMigrationExampleTest { + + @Test + public void testExampleMethods() throws Exception { + JdbcConfigurationMigrationExample example = new JdbcConfigurationMigrationExample(); + + Properties v1Props = new Properties(); + v1Props.setProperty("user", "default"); + v1Props.setProperty("connect_timeout", "5000"); + v1Props.setProperty("max_threads", "8"); + + // Verify conversion logic works as demonstrated in example + String v1Url = "jdbc:clickhouse://localhost:8123/default?connect_timeout=5000&max_threads=8"; + String convertedUrl = com.clickhouse.migration.config.ConfigurationMigrationHelper.convertUrl(v1Url); + Assert.assertEquals(convertedUrl, "jdbc:clickhouse://localhost:8123/default?connection_timeout=5000&clickhouse_setting_max_threads=8"); + + Properties convertedProps = com.clickhouse.migration.config.ConfigurationMigrationHelper.convertProperties(v1Props); + Assert.assertEquals(convertedProps.getProperty("connection_timeout"), "5000"); + Assert.assertEquals(convertedProps.getProperty("clickhouse_setting_max_threads"), "8"); + } +} From 5314c1c66a2c7e61df53a5140b19eac8c2d5a059 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 28 Aug 2026 16:34:15 -0700 Subject: [PATCH 2/7] Added properties migration notes. Updated migration helper --- migration-helpers/migrating_properties.md | 133 ++++++++++++++ migration-helpers/pom.xml | 12 ++ .../migration/config/ConfigPropertyCache.java | 170 +++++++++++++----- .../config/ConfigurationMigrationHelper.java | 29 ++- .../JdbcConfigurationMigrationExample.java | 7 +- .../v1-deprecated-properties.properties | 58 ++++++ .../config/v1-known-properties.properties | 10 ++ .../config/v1-to-v2-mappings.properties | 6 + .../config/v2-known-properties.properties | 2 - ...lickHouseJdbcMigrationIntegrationTest.java | 94 ++++++++++ .../config/ClickHouseOptionMigrationTest.java | 130 ++++++++++++++ .../ConfigurationMigrationHelperTest.java | 51 ++++++ 12 files changed, 649 insertions(+), 53 deletions(-) create mode 100644 migration-helpers/migrating_properties.md create mode 100644 migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties create mode 100644 migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseJdbcMigrationIntegrationTest.java create mode 100644 migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseOptionMigrationTest.java diff --git a/migration-helpers/migrating_properties.md b/migration-helpers/migrating_properties.md new file mode 100644 index 000000000..2fb0f2054 --- /dev/null +++ b/migration-helpers/migrating_properties.md @@ -0,0 +1,133 @@ +# Migration Notes for ugprade from 0.6.x to 0.9.x + +## Host specification + +- No multihost allowed in jdbc URL like `jdbc:ch://host1,host2:8123/`. +- `ssl_mode` was redefined in `0.10.0`. In `0.9.x` it is handled by JDBC and can be only `STRICT` +- HTTP protocol is not guessed by port anymore. Default is plain HTTP. Otherwise should be like `jdbc:ch:https://cloud.com:8443/ + +## Authentication + +- Plain user-password is unchanged. +- New SSL modes for self-signed certificates added in `0.10.0`. +- `http_use_basic_auth` (default: `true`) to send authentication credentials. was introduced in V2 and backported to V1 + + +## Protocol Configuration + +- `protocol` - deprecated. Only http supported. Can be ignored in JDBC case because URL defines protocol. + +### Connection + +- `sslcerttype` - is deprecated. But will be re-introduced soon with a new name. Can be ignored if X.509 requested +- `sslkeyalg` - is deprecated. But will be re-introduced soon with a new name. Can be ignored if RSA requested. +- `sslprotocol` - is deprecated. Currently only latest protocol is available. Can be ignored. +- `custom_socket_factory` - is deprecated. +- `custom_socket_factory_options` - is deprecated. +- `connect_timeout` - is replaced with `connection_request_timeout` and `connection_timeout`. V1 used same timeout for getting + connection from pool and timeouting establishing new connection. +- `ssl` - is deprecated. Ignored. +- `sslmode` - replaced by `ssl_mode` with more values. + +### TCP Socket Configuration + +- `socket_ip_tos` - is deprecated + +### HTTP Configuration + +- `http_connection_provider` - is deprecated and has to be ignored. +- `custom_http_headers` - is deprecated. Custom headers should be set by one and with `http_header_` prefix. +- `custom_http_params` - is deprecated. If custom http parameter is clickhouse setting it should be set with `clickhouse_setting_` prefix. + There is another case when query parameters have user define meaning. The should be with prefix set in DB configuration like `custom_` (see more https://clickhouse.com/docs/reference/settings/server-settings/settings/custom#custom_settings_prefixes). + +- `http_server_default_response` - is deprecated. Can be ignored for JDBC case. +- `receive_query_progress` - is deprecated. Not supported and can be ignored for JDBC case. +- `send_http_client_id` - is deprecated. Can be ignored for JDBC case. +- `wait_end_of_query` - is really a server setting - should be prefixed with `clickhouse_setting_` +- `remember_last_set_roles` - valid for JDBC only. List of roles should be set via `session_db_roles` if working with client directly. +- `ahc_validate_after_inactivity` - is deprecated. Can be ignored. Validation made automatically. +- `ahc_retry_on_failure` - is deprecated. Two new properties `retry` (for number of retries) and `client_retry_on_failures` (to configure when to retry. Possible values: `NoHttpResponse`, `ConnectTimeout`, `ConnectionRequestTimeout`, `ServerRetryable`) + +- `alive_timeout` and `http_keep_alive` - are deprecated and joined into `http_keep_alive_timeout`. + + +## Client Operation Side + +- `use_compilation` - is deprecated. +- `debug_measure_request_time` - is deprecated. + + +### Multithreading +- `async` - this defined if each operation is run in separate thread. V2 switched to `false` by default. +- `max_scheduler_threads` - is deprecated. scheduler is set via configuration and defined by user. +- `max_threads` - is deprecated. +- `max_requests` - is deprecated. +- `thread_keepalive_timeout` - is deprecated. +- `max_core_thread_ttl` - is deprecated. + + +### Server Endpoints + +- `auto_discovery` - is deprecated. +- `load_balancing_policy` - is deprecated. Load balancing is not part of Client main functionality. +- `load_balancing_tags` - is deprecated. +- `health_check_interval` - is deprecated. +- `health_check_method` - is deprecated. +- `node_discovery_interval` - is deprecated. +- `node_discovery_limit` - is deprecated. +- `node_check_interval` - is deprecated. +- `node_group_size` - is deprecated. +- `check_all_nodes` - is deprecated. +- `version` - is replaced by `server_version`. +- `server_revision` - is replaced by `server_version`. +- `failover` - is deprecated. + +### Server Interaction + +- `custom_settings` - is deprecated. Was used to define client wide list of server settings. Now each settings should be set separatly and + with `clickhouse_setting_` prefix. +- `time_zone` - is replaced by `server_time_zone` +- `auto_session` - is deprecated. Sessions are created using client API. JDBC has no direct control over it. +- `log_leading_comment` - is deprecated and was applicable for JDBC. When true JDBC was parsing leading comment and sent to + server via `log_comment`. +- `max_execution_time` - Should be replaced with server setting (`clickhouse_setting_max_execution_time`). However V2 + client has similar setting with another meaning for async operations. +- `max_result_rows` - Should be replaced with server setting (`clickhouse_setting_max_result_rows`). +- `result_overflow_mode` - Should be replaced with server setting (`clickhouse_setting_result_overflow_mode`). +- `product_name` - replaced by `client_name`. +- `rename_response_column` - is deprecated. +- `transaction_timeout` - is deprecated. + +### Sessions +- `repeat_on_session_lock` - is deprecated. But need to be implemented as part of retry logic. +- `session_id` - Should be replaced with server setting (`clickhouse_setting_session_id`). +- `session_check` - Should be replaced with server setting (`clickhouse_setting_session_check`). +- `session_timeout` - Should be replaced with server setting (`clickhouse_setting_session_timeout`). + + +### Data Transfer + +- `buffering` - is deprecated. +- `buffer_size` - is deprecated. +- `buffer_queue_variation` - is deprecated. +- `use_blocking_queue` - is deprecated. +- `read_buffer_size` - is deprecated. +- `write_buffer_size` - is deprecated. +- `request_chunk_size` - is deprecated. +- `request_buffering` - is deprecated. +- `response_buffering`- is deprecated. +- `compress_algorithm` - is deprecated. +- `decompress_algorithm` - is deprecated. +- `compress_level` - is deprecated. +- `decompress_level` - is deprecated. +- `max_buffer_size` - is deprecated. +- `max_mapper_cache` - is deprecated. +- `max_queued_buffers` - is deprecated. +- `max_queued_requests` - is deprecated. +- `rounding_mode` - is deprecated. +- `srv_resolve` - is deprecated. +- `reuse_value_wrapper` - is deprecated. +- `widen_unsigned_types` - is deprecated. +- `use_binary_string` - replaced with `binary_string_support`. applicable only for `0.10.0` +- `use_objects_in_arrays` - is deprecated. +- `use_server_time_zone_for_dates` - is deprecated. diff --git a/migration-helpers/pom.xml b/migration-helpers/pom.xml index df998480d..f86333bcf 100644 --- a/migration-helpers/pom.xml +++ b/migration-helpers/pom.xml @@ -56,6 +56,18 @@ + + ${project.parent.groupId} + clickhouse-client + ${revision} + test-jar + test + + + org.testcontainers + testcontainers + test + org.testng testng diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java index 49cc8b5f6..ca9366ad7 100644 --- a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java @@ -1,15 +1,11 @@ package com.clickhouse.migration.config; -import com.clickhouse.client.api.ClientConfigProperties; -import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.http.config.ClickHouseHttpOption; -import com.clickhouse.jdbc.DriverProperties; -import com.clickhouse.jdbc.JdbcConfig; - import java.io.InputStream; import java.io.InputStreamReader; +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; -import java.sql.DriverPropertyInfo; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -28,14 +24,21 @@ public class ConfigPropertyCache { private static final String V1_KNOWN_RESOURCE = "/com/clickhouse/migration/config/v1-known-properties.properties"; private static final String V2_KNOWN_RESOURCE = "/com/clickhouse/migration/config/v2-known-properties.properties"; + private static final String V1_DEPRECATED_RESOURCE = "/com/clickhouse/migration/config/v1-deprecated-properties.properties"; private static final String MAPPINGS_RESOURCE = "/com/clickhouse/migration/config/v1-to-v2-mappings.properties"; private static final ConfigPropertyCache INSTANCE = new ConfigPropertyCache(); private final Set v1KnownProperties; private final Set v2KnownProperties; + private final Set v1DeprecatedProperties; private final Map v1ToV2Mappings; + /** + * Gets the singleton instance of {@link ConfigPropertyCache}. + * + * @return cache singleton instance + */ public static ConfigPropertyCache getInstance() { return INSTANCE; } @@ -43,22 +46,25 @@ public static ConfigPropertyCache getInstance() { private ConfigPropertyCache() { Set v1Props = new HashSet<>(); Set v2Props = new HashSet<>(); + Set deprecatedProps = new HashSet<>(); Map mappings = new HashMap<>(); // 1. Load properties from resource files loadPropertiesResource(V1_KNOWN_RESOURCE, v1Props, null); loadPropertiesResource(V2_KNOWN_RESOURCE, v2Props, null); + loadPropertiesResource(V1_DEPRECATED_RESOURCE, deprecatedProps, null); loadPropertiesResource(MAPPINGS_RESOURCE, null, mappings); - // 2. Pre-load / enrich with runtime enum keys from v1 and v2 + // 2. Pre-load / enrich with runtime enum keys from v1 and v2 if present on classpath enrichWithRuntimeEnums(v1Props, v2Props); this.v1KnownProperties = Collections.unmodifiableSet(v1Props); this.v2KnownProperties = Collections.unmodifiableSet(v2Props); + this.v1DeprecatedProperties = Collections.unmodifiableSet(deprecatedProps); this.v1ToV2Mappings = Collections.unmodifiableMap(mappings); - log.debug("Pre-loaded {} v1 properties, {} v2 properties, {} mappings into cache.", - v1KnownProperties.size(), v2KnownProperties.size(), v1ToV2Mappings.size()); + log.debug("Pre-loaded {} v1 properties, {} v2 properties, {} deprecated properties, {} mappings into cache.", + v1KnownProperties.size(), v2KnownProperties.size(), v1DeprecatedProperties.size(), v1ToV2Mappings.size()); } private void loadPropertiesResource(String resourcePath, Set targetSet, Map targetMap) { @@ -85,55 +91,75 @@ private void loadPropertiesResource(String resourcePath, Set targetSet, } private void enrichWithRuntimeEnums(Set v1Props, Set v2Props) { - // v2 ClientConfigProperties - try { - for (ClientConfigProperties prop : ClientConfigProperties.values()) { - if (prop.getKey() != null) { - v2Props.add(prop.getKey()); - } - } - } catch (Throwable t) { - log.debug("Could not inspect ClientConfigProperties: {}", t.getMessage()); - } + // v2 ClientConfigProperties and ClientConfigurationProperties + loadEnumKeysFromClasspath("com.clickhouse.client.api.ClientConfigProperties", v2Props); + loadEnumKeysFromClasspath("com.clickhouse.client.api.ClientConfigurationProperties", v2Props); // v2 DriverProperties - try { - for (DriverProperties prop : DriverProperties.values()) { - if (prop.getKey() != null) { - v2Props.add(prop.getKey()); - } - } - } catch (Throwable t) { - log.debug("Could not inspect DriverProperties: {}", t.getMessage()); - } + loadEnumKeysFromClasspath("com.clickhouse.jdbc.DriverProperties", v2Props); // v1 ClickHouseClientOption - try { - for (ClickHouseClientOption prop : ClickHouseClientOption.values()) { - if (prop.getKey() != null) { - v1Props.add(prop.getKey()); - } - } - } catch (Throwable t) { - log.debug("Could not inspect ClickHouseClientOption: {}", t.getMessage()); - } + loadEnumKeysFromClasspath("com.clickhouse.client.config.ClickHouseClientOption", v1Props); // v1 ClickHouseHttpOption + loadEnumKeysFromClasspath("com.clickhouse.client.http.config.ClickHouseHttpOption", v1Props); + + // v1 JdbcConfig + loadJdbcConfigFromClasspath(v1Props); + } + + private void loadEnumKeysFromClasspath(String className, Set targetSet) { try { - for (ClickHouseHttpOption prop : ClickHouseHttpOption.values()) { - if (prop.getKey() != null) { - v1Props.add(prop.getKey()); + Class clazz = Class.forName(className, false, getClass().getClassLoader()); + if (clazz.isEnum()) { + Object[] constants = clazz.getEnumConstants(); + if (constants != null) { + Method getKeyMethod = null; + try { + getKeyMethod = clazz.getMethod("getKey"); + } catch (NoSuchMethodException ignored) { + // ignore if getKey() is missing + } + + for (Object obj : constants) { + if (obj != null) { + if (getKeyMethod != null) { + try { + Object keyObj = getKeyMethod.invoke(obj); + if (keyObj != null) { + targetSet.add(keyObj.toString()); + } + } catch (Exception e) { + targetSet.add(obj.toString()); + } + } else { + targetSet.add(obj.toString()); + } + } + } } } } catch (Throwable t) { - log.debug("Could not inspect ClickHouseHttpOption: {}", t.getMessage()); + log.debug("Class {} is not present on classpath or could not be loaded: {}", className, t.getMessage()); } + } - // v1 JdbcConfig + private void loadJdbcConfigFromClasspath(Set v1Props) { try { - for (DriverPropertyInfo info : JdbcConfig.getDriverProperties()) { - if (info.name != null) { - v1Props.add(info.name); + Class clazz = Class.forName("com.clickhouse.jdbc.JdbcConfig", false, getClass().getClassLoader()); + Method getDriverPropertiesMethod = clazz.getMethod("getDriverProperties"); + Object driverProps = getDriverPropertiesMethod.invoke(null); + if (driverProps != null && driverProps.getClass().isArray()) { + int length = Array.getLength(driverProps); + for (int i = 0; i < length; i++) { + Object info = Array.get(driverProps, i); + if (info != null) { + Field nameField = info.getClass().getField("name"); + Object nameObj = nameField.get(info); + if (nameObj != null) { + v1Props.add(nameObj.toString()); + } + } } } } catch (Throwable t) { @@ -141,14 +167,42 @@ private void enrichWithRuntimeEnums(Set v1Props, Set v2Props) { } } + /** + * Checks if the key is a known v1 configuration property. + * + * @param key property name + * @return true if key is known in v1 + */ public boolean isV1KnownProperty(String key) { return key != null && v1KnownProperties.contains(key); } + /** + * Checks if the key is a known v2 configuration property. + * + * @param key property name + * @return true if key is known in v2 + */ public boolean isV2KnownProperty(String key) { return key != null && v2KnownProperties.contains(key); } + /** + * Checks if the property is deprecated in v2 without direct conversion. + * + * @param key property name + * @return true if property is deprecated + */ + public boolean isDeprecatedProperty(String key) { + return key != null && (v1DeprecatedProperties.contains(key) || v1DeprecatedProperties.contains(key.toLowerCase())); + } + + /** + * Gets the mapped v2 key name for a given v1 property key. + * + * @param v1Key property name in v1 format + * @return mapped property name in v2 format, or original key if no explicit mapping exists + */ public String getV2MappedKey(String v1Key) { if (v1Key == null) { return null; @@ -156,14 +210,38 @@ public String getV2MappedKey(String v1Key) { return v1ToV2Mappings.getOrDefault(v1Key, v1Key); } + /** + * Gets the unmodifiable set of known v1 property keys. + * + * @return set of v1 property keys + */ public Set getV1KnownProperties() { return v1KnownProperties; } + /** + * Gets the unmodifiable set of known v2 property keys. + * + * @return set of v2 property keys + */ public Set getV2KnownProperties() { return v2KnownProperties; } + /** + * Gets the unmodifiable set of deprecated v1 property keys without conversion. + * + * @return set of deprecated property keys + */ + public Set getV1DeprecatedProperties() { + return v1DeprecatedProperties; + } + + /** + * Gets the unmodifiable map of v1-to-v2 property mappings. + * + * @return map of v1-to-v2 property mappings + */ public Map getV1ToV2Mappings() { return v1ToV2Mappings; } diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java index 56688ba50..7544fd144 100644 --- a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java @@ -14,7 +14,14 @@ public class ConfigurationMigrationHelper { private static final Logger log = LoggerFactory.getLogger(ConfigurationMigrationHelper.class); + /** + * Prefix used for ClickHouse server settings in v2. + */ public static final String SERVER_SETTING_PREFIX = "clickhouse_setting_"; + + /** + * Prefix used for custom HTTP headers in v2. + */ public static final String HTTP_HEADER_PREFIX = "http_header_"; /** @@ -83,14 +90,24 @@ public static Map convertMap(Map v1Config) { // 3. Check for mapped renamed key (e.g. connect_timeout -> connection_timeout) String mappedKey = cache.getV2MappedKey(key); + boolean isMapped = mappedKey != null && !mappedKey.equalsIgnoreCase(key); - // 4. If key or mappedKey is a known v2 property, keep as client/driver property - if (cache.isV2KnownProperty(mappedKey)) { + if (isMapped) { v2Config.put(mappedKey, value); - } else if (cache.isV2KnownProperty(key)) { + continue; + } + + // 4. Check if key is deprecated in v2 without conversion + if (cache.isDeprecatedProperty(key)) { + log.debug("Property '{}' is deprecated in v2 without conversion and will be ignored.", key); + continue; + } + + // 5. If key is a known v2 property, keep as client/driver property + if (cache.isV2KnownProperty(key)) { v2Config.put(key, value); } else { - // 5. Unrecognized key: in v1 this was implicitly treated as a ClickHouse server setting. + // 6. Unrecognized key: in v1 this was implicitly treated as a ClickHouse server setting. // In v2, it must be explicitly prefixed with clickhouse_setting_ String serverSettingKey = SERVER_SETTING_PREFIX + key; v2Config.put(serverSettingKey, value); @@ -149,6 +166,10 @@ public static String convertUrl(String url) { Map queryParams = parseQueryString(queryString); Map convertedParams = convertMap(queryParams); + if (convertedParams.isEmpty()) { + return baseUrl; + } + StringBuilder sb = new StringBuilder(baseUrl).append('?'); boolean first = true; for (Map.Entry entry : convertedParams.entrySet()) { diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java b/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java index 6d8d99579..54b598b53 100644 --- a/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java +++ b/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java @@ -39,7 +39,7 @@ public Connection createConnectionWithConvertedProperties(String url, Properties * Converts a legacy v1 connection URL (containing query parameters) to v2 format * and creates a JDBC connection. * - * @param v1Url v1 connection URL containing query parameters (e.g. "jdbc:clickhouse://localhost:8123/default?max_threads=8&connect_timeout=5000") + * @param v1Url v1 connection URL containing query parameters (e.g. {@code "jdbc:clickhouse://localhost:8123/default?max_threads=8&connect_timeout=5000"}) * @return active JDBC connection * @throws SQLException if a database access error occurs */ @@ -82,6 +82,11 @@ public void executeQueryWithMigratedConfig() throws SQLException { } } + /** + * Main entry point demonstrating configuration migration execution. + * + * @param args command line arguments + */ public static void main(String[] args) { try { new JdbcConfigurationMigrationExample().executeQueryWithMigratedConfig(); diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties new file mode 100644 index 000000000..fa6b73323 --- /dev/null +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties @@ -0,0 +1,58 @@ +# Deprecated v1 properties without conversion in v2 +protocol=protocol +sslcerttype=sslcerttype +sslkeyalg=sslkeyalg +sslprotocol=sslprotocol +custom_socket_factory=custom_socket_factory +custom_socket_factory_options=custom_socket_factory_options +socket_ip_tos=socket_ip_tos +http_connection_provider=http_connection_provider +http_server_default_response=http_server_default_response +receive_query_progress=receive_query_progress +send_http_client_id=send_http_client_id +ahc_validate_after_inactivity=ahc_validate_after_inactivity +ahc_retry_on_failure=ahc_retry_on_failure +use_compilation=use_compilation +debug_measure_request_time=debug_measure_request_time +max_scheduler_threads=max_scheduler_threads +max_requests=max_requests +thread_keepalive_timeout=thread_keepalive_timeout +max_core_thread_ttl=max_core_thread_ttl +auto_discovery=auto_discovery +load_balancing_policy=load_balancing_policy +load_balancing_tags=load_balancing_tags +health_check_interval=health_check_interval +health_check_method=health_check_method +node_discovery_interval=node_discovery_interval +node_discovery_limit=node_discovery_limit +node_check_interval=node_check_interval +node_group_size=node_group_size +check_all_nodes=check_all_nodes +failover=failover +auto_session=auto_session +log_leading_comment=log_leading_comment +rename_response_column=rename_response_column +transaction_timeout=transaction_timeout +repeat_on_session_lock=repeat_on_session_lock +buffering=buffering +buffer_queue_variation=buffer_queue_variation +use_blocking_queue=use_blocking_queue +read_buffer_size=read_buffer_size +write_buffer_size=write_buffer_size +request_chunk_size=request_chunk_size +request_buffering=request_buffering +response_buffering=response_buffering +compress_algorithm=compress_algorithm +decompress_algorithm=decompress_algorithm +compress_level=compress_level +decompress_level=decompress_level +max_buffer_size=max_buffer_size +max_mapper_cache=max_mapper_cache +max_queued_buffers=max_queued_buffers +max_queued_requests=max_queued_requests +rounding_mode=rounding_mode +srv_resolve=srv_resolve +reuse_value_wrapper=reuse_value_wrapper +widen_unsigned_types=widen_unsigned_types +use_objects_in_arrays=use_objects_in_arrays +use_server_time_zone_for_dates=use_server_time_zone_for_dates diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties index d1b4e156d..dc13fac38 100644 --- a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties @@ -119,3 +119,13 @@ typeMappings=typeMappings wrapperObject=wrapperObject user=user password=password +protocol=protocol +sslcerttype=sslcerttype +sslkeyalg=sslkeyalg +sslprotocol=sslprotocol +max_scheduler_threads=max_scheduler_threads +max_requests=max_requests +thread_keepalive_timeout=thread_keepalive_timeout +buffering=buffering +time_zone=time_zone +version=version diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties index 80d076a45..2fb9d807b 100644 --- a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties @@ -5,5 +5,11 @@ sslmode=ssl_mode sslkey=ssl_key proxy_username=proxy_user alive_timeout=http_keep_alive_timeout +http_keep_alive=http_keep_alive_timeout +version=server_version +server_revision=server_version +time_zone=server_time_zone +product_name=client_name +use_binary_string=binary_string_support typeMappings=jdbc_type_mappings databaseTerm=jdbc_schema_term diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties index 87ae5f808..1457254e8 100644 --- a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties @@ -79,8 +79,6 @@ jdbc_query_id_generator=jdbc_query_id_generator remember_last_set_roles=remember_last_set_roles custom_http_params=custom_http_params custom_settings=custom_settings -use_server_time_zone_for_dates=use_server_time_zone_for_dates -http_connection_provider=http_connection_provider jdbc_cluster_name=jdbc_cluster_name jdbc_type_mappings=jdbc_type_mappings typeMappings=typeMappings diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseJdbcMigrationIntegrationTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseJdbcMigrationIntegrationTest.java new file mode 100644 index 000000000..f08b155e4 --- /dev/null +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseJdbcMigrationIntegrationTest.java @@ -0,0 +1,94 @@ +package com.clickhouse.migration.config; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseServerForTest; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +public class ClickHouseJdbcMigrationIntegrationTest extends BaseIntegrationTest { + + @BeforeClass + public static void setUpContainer() { + ClickHouseServerForTest.beforeSuite(); + try { + Class.forName("com.clickhouse.jdbc.Driver"); + } catch (ClassNotFoundException e) { + throw new RuntimeException("ClickHouse JDBC Driver not found on classpath", e); + } + } + + @AfterClass + public static void tearDownContainer() { + ClickHouseServerForTest.afterSuite(); + } + + @Test + public void testMinimalJdbcConnectivityWithConvertedProperties() throws SQLException { + String hostAndPort = ClickHouseServerForTest.getClickHouseAddress(ClickHouseProtocol.HTTP, false); + String database = ClickHouseServerForTest.getDatabase(); + String url = "jdbc:clickhouse:http://" + hostAndPort + "/" + database; + + // Legacy v1 properties containing renamed properties, un-prefixed server settings, and deprecated properties + Properties v1Props = new Properties(); + v1Props.setProperty("user", ClickHouseServerForTest.getUsername()); + v1Props.setProperty("password", ClickHouseServerForTest.getPassword()); + v1Props.setProperty("connect_timeout", "5000"); // Renamed to connection_timeout + v1Props.setProperty("buffer_size", "65536"); // Renamed to client_network_buffer_size + v1Props.setProperty("max_threads", "4"); // Server setting -> clickhouse_setting_max_threads + v1Props.setProperty("protocol", "http"); // Deprecated -> ignored + v1Props.setProperty("use_compilation", "true"); // Deprecated -> ignored + v1Props.setProperty("custom_settings", "join_use_nulls=1"); + + Properties v2Props = ConfigurationMigrationHelper.convertProperties(v1Props); + + // Verify minimal JDBC connectivity with converted properties + try (Connection conn = DriverManager.getConnection(url, v2Props); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT 1")) { + + Assert.assertTrue(rs.next(), "ResultSet should contain at least one row."); + Assert.assertEquals(rs.getInt(1), 1, "First column should be 1."); + } + } + + @Test + public void testMinimalJdbcConnectivityWithConvertedUrl() throws SQLException { + String hostAndPort = ClickHouseServerForTest.getClickHouseAddress(ClickHouseProtocol.HTTP, false); + String database = ClickHouseServerForTest.getDatabase(); + String password = ClickHouseServerForTest.getPassword(); + String username = ClickHouseServerForTest.getUsername(); + + // Legacy v1 connection URL containing query parameters + String v1Url = "jdbc:clickhouse:http://" + hostAndPort + "/" + database + + "?user=" + username + + "&password=" + password + + "&connect_timeout=5000&max_threads=4&protocol=http&use_compilation=true"; + + String v2Url = ConfigurationMigrationHelper.convertUrl(v1Url); + + Assert.assertFalse(v2Url.contains("protocol="), "Converted URL should not contain deprecated protocol param."); + Assert.assertFalse(v2Url.contains("use_compilation="), "Converted URL should not contain deprecated use_compilation param."); + Assert.assertTrue(v2Url.contains("connection_timeout=5000"), "Converted URL should contain connection_timeout=5000."); + Assert.assertTrue(v2Url.contains("clickhouse_setting_max_threads=4"), "Converted URL should contain clickhouse_setting_max_threads=4."); + + // Verify minimal JDBC connectivity with converted URL + try (Connection conn = DriverManager.getConnection(v2Url); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT 1, 'test_container'")) { + + Assert.assertTrue(rs.next(), "ResultSet should contain at least one row."); + Assert.assertEquals(rs.getInt(1), 1, "First column should be 1."); + Assert.assertEquals(rs.getString(2), "test_container", "Second column should match 'test_container'."); + } + } +} diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseOptionMigrationTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseOptionMigrationTest.java new file mode 100644 index 000000000..58bbd88ed --- /dev/null +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseOptionMigrationTest.java @@ -0,0 +1,130 @@ +package com.clickhouse.migration.config; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.client.http.config.ClickHouseHttpOption; +import com.clickhouse.config.ClickHouseOption; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +public class ClickHouseOptionMigrationTest { + + @Test + public void testAllClickHouseOptionDerivativesAreConvertedOrCleanedUp() { + Map v1OptionsMap = new LinkedHashMap<>(); + + // 1. Add all enum options from ClickHouseClientOption + for (ClickHouseClientOption opt : ClickHouseClientOption.values()) { + if (opt.getKey() != null) { + v1OptionsMap.put(opt.getKey(), getSampleValueForOption(opt.getKey())); + } + } + + // 2. Add all enum options from ClickHouseHttpOption + for (ClickHouseHttpOption opt : ClickHouseHttpOption.values()) { + if (opt.getKey() != null) { + v1OptionsMap.put(opt.getKey(), getSampleValueForOption(opt.getKey())); + } + } + + // 3. Add all enum options from ClickHouseDefaults + for (ClickHouseDefaults opt : ClickHouseDefaults.values()) { + if (opt.getKey() != null) { + v1OptionsMap.put(opt.getKey(), getSampleValueForOption(opt.getKey())); + } + } + + // 4. Add all known v1 options from cache + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + for (String v1Key : cache.getV1KnownProperties()) { + v1OptionsMap.put(v1Key, getSampleValueForOption(v1Key)); + } + + Assert.assertFalse(v1OptionsMap.isEmpty(), "v1OptionsMap should contain ClickHouseOption derivatives."); + + // Convert options map + Map convertedMap = ConfigurationMigrationHelper.convertMap(v1OptionsMap); + Set deprecatedProps = cache.getV1DeprecatedProperties(); + + // Verify conversion rules for every single v1 option + for (Map.Entry entry : v1OptionsMap.entrySet()) { + String origKey = entry.getKey(); + + if (deprecatedProps.contains(origKey) || deprecatedProps.contains(origKey.toLowerCase())) { + // Deprecated options without conversion must NOT be in the converted map + Assert.assertFalse(convertedMap.containsKey(origKey), + "Deprecated option '" + origKey + "' should have been cleaned up/ignored."); + Assert.assertFalse(convertedMap.containsKey(ConfigurationMigrationHelper.SERVER_SETTING_PREFIX + origKey), + "Deprecated option '" + origKey + "' should not be converted to a server setting."); + } else if ("custom_settings".equalsIgnoreCase(origKey) || "custom_http_params".equalsIgnoreCase(origKey) || "custom_params".equalsIgnoreCase(origKey) || "custom_http_headers".equalsIgnoreCase(origKey) || "custom_headers".equalsIgnoreCase(origKey)) { + // Legacy composite properties are unpacked into individual clickhouse_setting_ / http_header_ entries + Assert.assertFalse(convertedMap.containsKey(origKey), + "Composite option '" + origKey + "' should be unpacked rather than remaining as a raw property."); + } else { + String mappedKey = cache.getV2MappedKey(origKey); + if (mappedKey != null && !mappedKey.equalsIgnoreCase(origKey)) { + // Mapped keys should be converted to their v2 name + Assert.assertTrue(convertedMap.containsKey(mappedKey), + "Mapped option '" + origKey + "' -> '" + mappedKey + "' should be present in converted map."); + } else if (cache.isV2KnownProperty(origKey)) { + // Known v2 property should remain present + Assert.assertTrue(convertedMap.containsKey(origKey), + "Known v2 option '" + origKey + "' should be present in converted map."); + } else { + // Un-prefixed server settings should receive clickhouse_setting_ prefix + String expectedSettingKey = ConfigurationMigrationHelper.SERVER_SETTING_PREFIX + origKey; + Assert.assertTrue(convertedMap.containsKey(expectedSettingKey), + "Unrecognized option '" + origKey + "' should be prefixed with " + expectedSettingKey); + } + } + } + } + + @Test + public void testClientInstantiationWithConvertedOptionsMap() { + Map v1Props = new LinkedHashMap<>(); + v1Props.put("user", "default"); + v1Props.put("password", "secret"); + v1Props.put("database", "default"); + v1Props.put("connect_timeout", "5000"); + v1Props.put("buffer_size", "65536"); + v1Props.put("max_threads", "8"); + v1Props.put("protocol", "http"); // deprecated - ignored + v1Props.put("use_compilation", "true"); // deprecated - ignored + v1Props.put("custom_settings", "join_use_nulls=1"); + v1Props.put("custom_http_headers", "X-App-Name=test"); + + Map convertedMap = ConfigurationMigrationHelper.convertMap(v1Props); + + // Client in client-v2 should be successfully instantiated with converted options + try (Client client = new Client.Builder() + .addEndpoint("http://localhost:8123") + .setOptions(convertedMap) + .build()) { + + Assert.assertNotNull(client, "Client instance should be successfully created."); + } + } + + private String getSampleValueForOption(String key) { + if ("connect_timeout".equalsIgnoreCase(key) || "socket_timeout".equalsIgnoreCase(key) || "alive_timeout".equalsIgnoreCase(key)) { + return "5000"; + } + if ("buffer_size".equalsIgnoreCase(key) || "read_buffer_size".equalsIgnoreCase(key)) { + return "65536"; + } + if ("ssl".equalsIgnoreCase(key) || "async".equalsIgnoreCase(key) || "compress".equalsIgnoreCase(key)) { + return "true"; + } + if ("port".equalsIgnoreCase(key)) { + return "8123"; + } + return "sample_value"; + } +} diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java index 422ea7a0c..722a5fdef 100644 --- a/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java @@ -32,6 +32,13 @@ public Object[][] providePropertyConversionData() { {"sslmode", "strict", "ssl_mode", "strict"}, {"sslkey", "/path/to/key", "ssl_key", "/path/to/key"}, {"proxy_username", "puser", "proxy_user", "puser"}, + {"alive_timeout", "60000", "http_keep_alive_timeout", "60000"}, + {"http_keep_alive", "60000", "http_keep_alive_timeout", "60000"}, + {"version", "23.8", "server_version", "23.8"}, + {"server_revision", "54460", "server_version", "54460"}, + {"time_zone", "UTC", "server_time_zone", "UTC"}, + {"product_name", "my-app", "client_name", "my-app"}, + {"use_binary_string", "true", "binary_string_support", "true"}, // Existing v2 prefixed properties should be preserved {"clickhouse_setting_max_execution_time", "60", "clickhouse_setting_max_execution_time", "60"}, @@ -105,6 +112,38 @@ public void testConvertUrl(String inputUrl, String expectedUrl) { Assert.assertEquals(resultUrl, expectedUrl); } + @Test + public void testDeprecatedPropertiesWithoutConversionAreIgnored() { + Map input = new LinkedHashMap<>(); + input.put("user", "default"); + input.put("protocol", "http"); + input.put("use_compilation", "true"); + input.put("socket_ip_tos", "0"); + input.put("http_connection_provider", "custom"); + input.put("auto_discovery", "true"); + input.put("buffering", "true"); + input.put("max_requests", "10"); + input.put("failover", "2"); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.get("user"), "default"); + Assert.assertFalse(result.containsKey("protocol")); + Assert.assertFalse(result.containsKey("clickhouse_setting_protocol")); + Assert.assertFalse(result.containsKey("use_compilation")); + Assert.assertFalse(result.containsKey("buffering")); + } + + @Test + public void testConvertUrlWithDeprecatedProperties() { + String inputUrl = "jdbc:clickhouse://localhost:8123/default?protocol=http&use_compilation=true&connect_timeout=5000"; + String expectedUrl = "jdbc:clickhouse://localhost:8123/default?connection_timeout=5000"; + + String resultUrl = ConfigurationMigrationHelper.convertUrl(inputUrl); + Assert.assertEquals(resultUrl, expectedUrl); + } + @Test public void testCacheInitializationAndPreload() { ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); @@ -112,8 +151,20 @@ public void testCacheInitializationAndPreload() { Assert.assertTrue(cache.isV1KnownProperty("connect_timeout")); Assert.assertTrue(cache.isV2KnownProperty("connection_timeout")); Assert.assertTrue(cache.isV2KnownProperty("user")); + Assert.assertTrue(cache.isDeprecatedProperty("protocol")); + Assert.assertTrue(cache.isDeprecatedProperty("use_compilation")); Assert.assertEquals(cache.getV2MappedKey("connect_timeout"), "connection_timeout"); Assert.assertEquals(cache.getV2MappedKey("buffer_size"), "client_network_buffer_size"); } + + @Test + public void testClasspathReflectionHandlesMissingClassesGracefully() { + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + + // Verify cache instance is non-null and functioning even when checking optional classpath classes + Assert.assertNotNull(cache, "Cache should initialize without throwing exceptions when checking classpath enums."); + Assert.assertNotNull(cache.getV2KnownProperties()); + Assert.assertNotNull(cache.getV1KnownProperties()); + } } From 2c9378ddfdf2938bf83e5014952f23f870d635e6 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 9 Sep 2026 13:14:44 -0700 Subject: [PATCH 3/7] Fixed issues in helpers code --- .github/workflows/build.yml | 7 +- CHANGELOG.md | 1 + migration-helpers/migrating_properties.md | 3 +- .../migration/config/ConfigPropertyCache.java | 12 ++- .../config/ConfigurationMigrationHelper.java | 34 +++---- .../v1-deprecated-properties.properties | 1 + .../config/v1-to-v2-mappings.properties | 1 - .../ConfigurationMigrationHelperTest.java | 90 ++++++++++++++++++- 8 files changed, 129 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ccec4fb70..2183e871a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -66,7 +66,12 @@ jobs: for d in $(ls -d `pwd`/examples/*/); do \ if [ -e $d/pom.xml ]; then cd $d && mvn --batch-mode --no-transfer-progress clean compile; fi; done - - name: Save clickhouse-jdbc-all for tests + - name: Compile miscellaneous + run: | + cd migration-helpers + mvn --batch-mode --no-transfer-progress clean compile + cd .. + - name: Save clickhouse-jdbc-all for tests uses: actions/upload-artifact@v4 with: name: clickhouse-jdbc-archive diff --git a/CHANGELOG.md b/CHANGELOG.md index cd8fd922a..b0c52cf89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 0.11.0-rc1 [Release Migration Guide](docs/releases/0_11_0.md) +[Migration Helpers](migration-helpers) - small code helpers to convert old configuration to a new one. ### Breaking Changes diff --git a/migration-helpers/migrating_properties.md b/migration-helpers/migrating_properties.md index 2fb0f2054..ef5fb3441 100644 --- a/migration-helpers/migrating_properties.md +++ b/migration-helpers/migrating_properties.md @@ -48,7 +48,8 @@ - `ahc_validate_after_inactivity` - is deprecated. Can be ignored. Validation made automatically. - `ahc_retry_on_failure` - is deprecated. Two new properties `retry` (for number of retries) and `client_retry_on_failures` (to configure when to retry. Possible values: `NoHttpResponse`, `ConnectTimeout`, `ConnectionRequestTimeout`, `ServerRetryable`) -- `alive_timeout` and `http_keep_alive` - are deprecated and joined into `http_keep_alive_timeout`. +- `alive_timeout` - maps to `http_keep_alive_timeout`. +- `http_keep_alive` - is deprecated (setting to false or 0 sets `http_keep_alive_timeout=0` to disable keep-alive). ## Client Operation Side diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java index ca9366ad7..b854fc379 100644 --- a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java @@ -149,7 +149,17 @@ private void loadJdbcConfigFromClasspath(Set v1Props) { Class clazz = Class.forName("com.clickhouse.jdbc.JdbcConfig", false, getClass().getClassLoader()); Method getDriverPropertiesMethod = clazz.getMethod("getDriverProperties"); Object driverProps = getDriverPropertiesMethod.invoke(null); - if (driverProps != null && driverProps.getClass().isArray()) { + if (driverProps instanceof Iterable) { + for (Object info : (Iterable) driverProps) { + if (info != null) { + Field nameField = info.getClass().getField("name"); + Object nameObj = nameField.get(info); + if (nameObj != null) { + v1Props.add(nameObj.toString()); + } + } + } + } else if (driverProps != null && driverProps.getClass().isArray()) { int length = Array.getLength(driverProps); for (int i = 0; i < length; i++) { Object info = Array.get(driverProps, i); diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java index 7544fd144..2215fc11c 100644 --- a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java @@ -1,5 +1,6 @@ package com.clickhouse.migration.config; +import com.clickhouse.client.api.ClientConfigProperties; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; @@ -88,6 +89,16 @@ public static Map convertMap(Map v1Config) { continue; } + // Handle legacy boolean keep-alive property: + // false or 0 disables keep-alive in v2 (http_keep_alive_timeout = 0) + // true or 1 keeps default keep-alive in v2 (does not set http_keep_alive_timeout to boolean string) + if ("http_keep_alive".equalsIgnoreCase(key)) { + if ("false".equalsIgnoreCase(value) || "0".equals(value)) { + v2Config.put("http_keep_alive_timeout", "0"); + } + continue; + } + // 3. Check for mapped renamed key (e.g. connect_timeout -> connection_timeout) String mappedKey = cache.getV2MappedKey(key); boolean isMapped = mappedKey != null && !mappedKey.equalsIgnoreCase(key); @@ -198,22 +209,15 @@ private static void parseAndAddKeyValuePairs(String valueStr, String prefix, Map if (valueStr == null || valueStr.trim().isEmpty()) { return; } - String[] pairs = valueStr.split(","); - for (String pair : pairs) { - String trimmed = pair.trim(); - if (trimmed.isEmpty()) { - continue; - } - int eqIndex = trimmed.indexOf('='); - if (eqIndex > 0) { - String k = trimmed.substring(0, eqIndex).trim(); - String v = trimmed.substring(eqIndex + 1).trim(); - if (!k.isEmpty()) { - if (!k.toLowerCase().startsWith(prefix)) { - k = prefix + k; - } - targetMap.put(k, v); + Map pairs = ClientConfigProperties.toKeyValuePairs(valueStr); + for (Map.Entry entry : pairs.entrySet()) { + String k = entry.getKey(); + String v = entry.getValue(); + if (k != null && !k.isEmpty()) { + if (!k.toLowerCase().startsWith(prefix)) { + k = prefix + k; } + targetMap.put(k, v); } } } diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties index fa6b73323..520ce4915 100644 --- a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties @@ -7,6 +7,7 @@ custom_socket_factory=custom_socket_factory custom_socket_factory_options=custom_socket_factory_options socket_ip_tos=socket_ip_tos http_connection_provider=http_connection_provider +http_keep_alive=http_keep_alive http_server_default_response=http_server_default_response receive_query_progress=receive_query_progress send_http_client_id=send_http_client_id diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties index 2fb9d807b..ab00dd4a9 100644 --- a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties @@ -5,7 +5,6 @@ sslmode=ssl_mode sslkey=ssl_key proxy_username=proxy_user alive_timeout=http_keep_alive_timeout -http_keep_alive=http_keep_alive_timeout version=server_version server_revision=server_version time_zone=server_time_zone diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java index 722a5fdef..280411c4f 100644 --- a/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java @@ -1,5 +1,6 @@ package com.clickhouse.migration.config; +import com.clickhouse.client.api.ClientConfigProperties; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -33,7 +34,8 @@ public Object[][] providePropertyConversionData() { {"sslkey", "/path/to/key", "ssl_key", "/path/to/key"}, {"proxy_username", "puser", "proxy_user", "puser"}, {"alive_timeout", "60000", "http_keep_alive_timeout", "60000"}, - {"http_keep_alive", "60000", "http_keep_alive_timeout", "60000"}, + {"http_keep_alive", "false", "http_keep_alive_timeout", "0"}, + {"http_keep_alive", "0", "http_keep_alive_timeout", "0"}, {"version", "23.8", "server_version", "23.8"}, {"server_revision", "54460", "server_version", "54460"}, {"time_zone", "UTC", "server_time_zone", "UTC"}, @@ -74,6 +76,59 @@ public void testConvertCustomSettingsAndHeaders() { Assert.assertEquals(result.get("http_header_X-App-Name"), "demo"); } + @DataProvider(name = "escapedCompositeSettingsData") + public Object[][] provideEscapedCompositeSettingsData() { + return new Object[][]{ + { + "custom_settings", + "format_csv_delimiter=\\,, max_threads=4", + "clickhouse_setting_format_csv_delimiter", + ",", + "clickhouse_setting_max_threads", + "4" + }, + { + "custom_settings", + "setting_with_eq=val\\=123, max_threads=4", + "clickhouse_setting_setting_with_eq", + "val=123", + "clickhouse_setting_max_threads", + "4" + }, + { + "custom_http_headers", + "X-Header-1=val1\\,val2, X-Header-2=a\\=b", + "http_header_X-Header-1", + "val1,val2", + "http_header_X-Header-2", + "a=b" + }, + { + "custom_settings", + "complex_setting=a\\,b\\=c\\,d", + "clickhouse_setting_complex_setting", + "a,b=c,d", + null, + null + } + }; + } + + @Test(dataProvider = "escapedCompositeSettingsData") + public void testConvertEscapedCustomSettingsAndHeaders(String propertyKey, String rawValue, + String expectedKey1, String expectedValue1, + String expectedKey2, String expectedValue2) { + Map input = new LinkedHashMap<>(); + input.put(propertyKey, rawValue); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.get(expectedKey1), expectedValue1); + if (expectedKey2 != null) { + Assert.assertEquals(result.get(expectedKey2), expectedValue2); + } + } + @Test public void testConvertPropertiesObject() { Properties v1Props = new Properties(); @@ -149,6 +204,10 @@ public void testCacheInitializationAndPreload() { ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); Assert.assertTrue(cache.isV1KnownProperty("connect_timeout")); + Assert.assertTrue(cache.isV1KnownProperty("autoCommit")); + Assert.assertTrue(cache.isV1KnownProperty("createDatabaseIfNotExist")); + Assert.assertTrue(cache.isV1KnownProperty("continueBatchOnError")); + Assert.assertTrue(cache.isV1KnownProperty("jdbcCompliant")); Assert.assertTrue(cache.isV2KnownProperty("connection_timeout")); Assert.assertTrue(cache.isV2KnownProperty("user")); Assert.assertTrue(cache.isDeprecatedProperty("protocol")); @@ -158,6 +217,35 @@ public void testCacheInitializationAndPreload() { Assert.assertEquals(cache.getV2MappedKey("buffer_size"), "client_network_buffer_size"); } + @Test + public void testHttpKeepAliveTrueDoesNotSetTimeoutOrCauseParseException() { + Map input = new LinkedHashMap<>(); + input.put("http_keep_alive", "true"); + input.put("alive_timeout", "60000"); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.get("http_keep_alive_timeout"), "60000"); + Assert.assertFalse(result.containsKey("http_keep_alive")); + + // Verify parsing converted map in v2 ClientConfigProperties does not fail with NumberFormatException + Map parsed = ClientConfigProperties.parseConfigMap(result); + Assert.assertEquals(parsed.get("http_keep_alive_timeout"), 60000L); + } + + @Test + public void testHttpKeepAliveFalseSetsTimeoutToZero() { + Map input = new LinkedHashMap<>(); + input.put("http_keep_alive", "false"); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.get("http_keep_alive_timeout"), "0"); + + Map parsed = ClientConfigProperties.parseConfigMap(result); + Assert.assertEquals(parsed.get("http_keep_alive_timeout"), 0L); + } + @Test public void testClasspathReflectionHandlesMissingClassesGracefully() { ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); From 4a8640c1ae4cb6d92591f1079317dc34a4545808 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 9 Sep 2026 13:22:20 -0700 Subject: [PATCH 4/7] fixed the build issue with migration helpers --- migration-helpers/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/migration-helpers/pom.xml b/migration-helpers/pom.xml index f86333bcf..062444abb 100644 --- a/migration-helpers/pom.xml +++ b/migration-helpers/pom.xml @@ -81,10 +81,6 @@ org.apache.maven.plugins maven-compiler-plugin - - 1.8 - 1.8 - From 98336c4580d221d5d36d07270fa787d722ff9c1e Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 9 Sep 2026 14:07:51 -0700 Subject: [PATCH 5/7] Fixed issues in migrations helpers and CI --- .github/workflows/build.yml | 7 +- migration-helpers/pom.xml | 10 ++ .../migration/config/ConfigPropertyCache.java | 106 ++++++++++++++---- .../config/ConfigurationMigrationHelper.java | 28 ++++- .../v1-deprecated-properties.properties | 23 ++++ .../config/v1-to-v2-mappings.properties | 1 + .../ConfigurationMigrationHelperTest.java | 66 +++++++++++ 7 files changed, 206 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2183e871a..530515d8a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,16 +61,11 @@ jobs: cp -rf $HOME/.m2/repository/com/clickhouse/clickhouse-jdbc/* ./clickhouse-jdbc-artifacts/ - name: Compile examples run: | - export LIB_VER=$(grep '' pom.xml | sed -e 's|[[:space:]]*<[/]*revision>[[:space:]]*||g') + export LIB_VER=$(cat VERSION) find `pwd`/examples -type f -name pom.xml -exec sed -i -e "s|\(\).*\(<\)|\1$LIB_VER\2|g" {} \; for d in $(ls -d `pwd`/examples/*/); do \ if [ -e $d/pom.xml ]; then cd $d && mvn --batch-mode --no-transfer-progress clean compile; fi; done - - name: Compile miscellaneous - run: | - cd migration-helpers - mvn --batch-mode --no-transfer-progress clean compile - cd .. - name: Save clickhouse-jdbc-all for tests uses: actions/upload-artifact@v4 with: diff --git a/migration-helpers/pom.xml b/migration-helpers/pom.xml index 062444abb..9e8c8f125 100644 --- a/migration-helpers/pom.xml +++ b/migration-helpers/pom.xml @@ -78,6 +78,16 @@ + + org.apache.maven.plugins + maven-toolchains-plugin + + + default + none + + + org.apache.maven.plugins maven-compiler-plugin diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java index b854fc379..6c5f5ade1 100644 --- a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java @@ -33,6 +33,7 @@ public class ConfigPropertyCache { private final Set v2KnownProperties; private final Set v1DeprecatedProperties; private final Map v1ToV2Mappings; + private final Map v2CanonicalKeys; /** * Gets the singleton instance of {@link ConfigPropertyCache}. @@ -48,26 +49,28 @@ private ConfigPropertyCache() { Set v2Props = new HashSet<>(); Set deprecatedProps = new HashSet<>(); Map mappings = new HashMap<>(); + Map v2Canonical = new HashMap<>(); // 1. Load properties from resource files - loadPropertiesResource(V1_KNOWN_RESOURCE, v1Props, null); - loadPropertiesResource(V2_KNOWN_RESOURCE, v2Props, null); - loadPropertiesResource(V1_DEPRECATED_RESOURCE, deprecatedProps, null); - loadPropertiesResource(MAPPINGS_RESOURCE, null, mappings); + loadPropertiesResource(V1_KNOWN_RESOURCE, v1Props, null, null); + loadPropertiesResource(V2_KNOWN_RESOURCE, v2Props, null, v2Canonical); + loadPropertiesResource(V1_DEPRECATED_RESOURCE, deprecatedProps, null, null); + loadPropertiesResource(MAPPINGS_RESOURCE, null, mappings, null); // 2. Pre-load / enrich with runtime enum keys from v1 and v2 if present on classpath - enrichWithRuntimeEnums(v1Props, v2Props); + enrichWithRuntimeEnums(v1Props, v2Props, v2Canonical); this.v1KnownProperties = Collections.unmodifiableSet(v1Props); this.v2KnownProperties = Collections.unmodifiableSet(v2Props); this.v1DeprecatedProperties = Collections.unmodifiableSet(deprecatedProps); this.v1ToV2Mappings = Collections.unmodifiableMap(mappings); + this.v2CanonicalKeys = Collections.unmodifiableMap(v2Canonical); log.debug("Pre-loaded {} v1 properties, {} v2 properties, {} deprecated properties, {} mappings into cache.", v1KnownProperties.size(), v2KnownProperties.size(), v1DeprecatedProperties.size(), v1ToV2Mappings.size()); } - private void loadPropertiesResource(String resourcePath, Set targetSet, Map targetMap) { + private void loadPropertiesResource(String resourcePath, Set targetSet, Map targetMap, Map canonicalMap) { try (InputStream in = getClass().getResourceAsStream(resourcePath)) { if (in != null) { Properties props = new Properties(); @@ -75,11 +78,18 @@ private void loadPropertiesResource(String resourcePath, Set targetSet, props.load(reader); } for (String key : props.stringPropertyNames()) { + String trimmedKey = key.trim(); + String val = props.getProperty(key).trim(); if (targetSet != null) { - targetSet.add(key.trim()); + addKeyToSet(targetSet, trimmedKey); } if (targetMap != null) { - targetMap.put(key.trim(), props.getProperty(key).trim()); + targetMap.put(trimmedKey, val); + targetMap.put(trimmedKey.toLowerCase(), val); + } + if (canonicalMap != null) { + canonicalMap.putIfAbsent(trimmedKey, trimmedKey); + canonicalMap.putIfAbsent(trimmedKey.toLowerCase(), trimmedKey); } } } else { @@ -90,25 +100,25 @@ private void loadPropertiesResource(String resourcePath, Set targetSet, } } - private void enrichWithRuntimeEnums(Set v1Props, Set v2Props) { + private void enrichWithRuntimeEnums(Set v1Props, Set v2Props, Map v2Canonical) { // v2 ClientConfigProperties and ClientConfigurationProperties - loadEnumKeysFromClasspath("com.clickhouse.client.api.ClientConfigProperties", v2Props); - loadEnumKeysFromClasspath("com.clickhouse.client.api.ClientConfigurationProperties", v2Props); + loadEnumKeysFromClasspath("com.clickhouse.client.api.ClientConfigProperties", v2Props, v2Canonical); + loadEnumKeysFromClasspath("com.clickhouse.client.api.ClientConfigurationProperties", v2Props, v2Canonical); // v2 DriverProperties - loadEnumKeysFromClasspath("com.clickhouse.jdbc.DriverProperties", v2Props); + loadEnumKeysFromClasspath("com.clickhouse.jdbc.DriverProperties", v2Props, v2Canonical); // v1 ClickHouseClientOption - loadEnumKeysFromClasspath("com.clickhouse.client.config.ClickHouseClientOption", v1Props); + loadEnumKeysFromClasspath("com.clickhouse.client.config.ClickHouseClientOption", v1Props, null); // v1 ClickHouseHttpOption - loadEnumKeysFromClasspath("com.clickhouse.client.http.config.ClickHouseHttpOption", v1Props); + loadEnumKeysFromClasspath("com.clickhouse.client.http.config.ClickHouseHttpOption", v1Props, null); // v1 JdbcConfig loadJdbcConfigFromClasspath(v1Props); } - private void loadEnumKeysFromClasspath(String className, Set targetSet) { + private void loadEnumKeysFromClasspath(String className, Set targetSet, Map canonicalMap) { try { Class clazz = Class.forName(className, false, getClass().getClassLoader()); if (clazz.isEnum()) { @@ -123,17 +133,22 @@ private void loadEnumKeysFromClasspath(String className, Set targetSet) for (Object obj : constants) { if (obj != null) { + String keyStr = null; if (getKeyMethod != null) { try { Object keyObj = getKeyMethod.invoke(obj); if (keyObj != null) { - targetSet.add(keyObj.toString()); + keyStr = keyObj.toString(); } } catch (Exception e) { - targetSet.add(obj.toString()); + keyStr = obj.toString(); } } else { - targetSet.add(obj.toString()); + keyStr = obj.toString(); + } + + if (keyStr != null) { + addKeyToSetAndCanonicalMap(targetSet, canonicalMap, keyStr); } } } @@ -155,7 +170,7 @@ private void loadJdbcConfigFromClasspath(Set v1Props) { Field nameField = info.getClass().getField("name"); Object nameObj = nameField.get(info); if (nameObj != null) { - v1Props.add(nameObj.toString()); + addKeyToSet(v1Props, nameObj.toString()); } } } @@ -167,7 +182,7 @@ private void loadJdbcConfigFromClasspath(Set v1Props) { Field nameField = info.getClass().getField("name"); Object nameObj = nameField.get(info); if (nameObj != null) { - v1Props.add(nameObj.toString()); + addKeyToSet(v1Props, nameObj.toString()); } } } @@ -177,6 +192,30 @@ private void loadJdbcConfigFromClasspath(Set v1Props) { } } + private void addKeyToSet(Set targetSet, String key) { + if (key != null) { + String trimmed = key.trim(); + if (!trimmed.isEmpty()) { + targetSet.add(trimmed); + targetSet.add(trimmed.toLowerCase()); + } + } + } + + private void addKeyToSetAndCanonicalMap(Set targetSet, Map canonicalMap, String key) { + if (key != null) { + String trimmed = key.trim(); + if (!trimmed.isEmpty()) { + targetSet.add(trimmed); + targetSet.add(trimmed.toLowerCase()); + if (canonicalMap != null) { + canonicalMap.putIfAbsent(trimmed, trimmed); + canonicalMap.putIfAbsent(trimmed.toLowerCase(), trimmed); + } + } + } + } + /** * Checks if the key is a known v1 configuration property. * @@ -184,7 +223,7 @@ private void loadJdbcConfigFromClasspath(Set v1Props) { * @return true if key is known in v1 */ public boolean isV1KnownProperty(String key) { - return key != null && v1KnownProperties.contains(key); + return key != null && (v1KnownProperties.contains(key) || v1KnownProperties.contains(key.toLowerCase())); } /** @@ -194,7 +233,7 @@ public boolean isV1KnownProperty(String key) { * @return true if key is known in v2 */ public boolean isV2KnownProperty(String key) { - return key != null && v2KnownProperties.contains(key); + return key != null && (v2KnownProperties.contains(key) || v2KnownProperties.contains(key.toLowerCase())); } /** @@ -217,7 +256,28 @@ public String getV2MappedKey(String v1Key) { if (v1Key == null) { return null; } - return v1ToV2Mappings.getOrDefault(v1Key, v1Key); + String mapped = v1ToV2Mappings.get(v1Key); + if (mapped == null) { + mapped = v1ToV2Mappings.get(v1Key.toLowerCase()); + } + return mapped != null ? mapped : v1Key; + } + + /** + * Gets the canonical v2 property key for a known v2 key. + * + * @param key property name in v2 format + * @return canonical v2 key name, or null if key is not known in v2 + */ + public String getV2CanonicalKey(String key) { + if (key == null) { + return null; + } + String canonical = v2CanonicalKeys.get(key); + if (canonical == null) { + canonical = v2CanonicalKeys.get(key.toLowerCase()); + } + return canonical; } /** diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java index 2215fc11c..0c475ebe3 100644 --- a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java @@ -73,7 +73,7 @@ public static Map convertMap(Map v1Config) { // 1. If key already starts with clickhouse_setting_ or http_header_, preserve it as is. if (key.toLowerCase().startsWith(SERVER_SETTING_PREFIX) || key.toLowerCase().startsWith(HTTP_HEADER_PREFIX)) { - v2Config.put(key, value); + v2Config.put(key, normalizePropertyValue(key, value)); continue; } @@ -104,7 +104,7 @@ public static Map convertMap(Map v1Config) { boolean isMapped = mappedKey != null && !mappedKey.equalsIgnoreCase(key); if (isMapped) { - v2Config.put(mappedKey, value); + v2Config.put(mappedKey, normalizePropertyValue(mappedKey, value)); continue; } @@ -114,14 +114,18 @@ public static Map convertMap(Map v1Config) { continue; } - // 5. If key is a known v2 property, keep as client/driver property + // 5. If key is a known v2 property, keep as client/driver property using its canonical name if (cache.isV2KnownProperty(key)) { - v2Config.put(key, value); + String v2Key = cache.getV2CanonicalKey(key); + if (v2Key == null) { + v2Key = key.toLowerCase(); + } + v2Config.put(v2Key, normalizePropertyValue(v2Key, value)); } else { // 6. Unrecognized key: in v1 this was implicitly treated as a ClickHouse server setting. // In v2, it must be explicitly prefixed with clickhouse_setting_ String serverSettingKey = SERVER_SETTING_PREFIX + key; - v2Config.put(serverSettingKey, value); + v2Config.put(serverSettingKey, normalizePropertyValue(serverSettingKey, value)); } } @@ -217,9 +221,21 @@ private static void parseAndAddKeyValuePairs(String valueStr, String prefix, Map if (!k.toLowerCase().startsWith(prefix)) { k = prefix + k; } - targetMap.put(k, v); + targetMap.put(k, normalizePropertyValue(k, v)); + } + } + } + + private static String normalizePropertyValue(String key, String value) { + if (value == null) { + return null; + } + if ("ssl_mode".equalsIgnoreCase(key) || "sslmode".equalsIgnoreCase(key)) { + if ("none".equalsIgnoreCase(value)) { + return "TRUST"; } } + return value; } private static Map parseQueryString(String queryString) { diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties index 520ce4915..1cc2686fd 100644 --- a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties @@ -57,3 +57,26 @@ reuse_value_wrapper=reuse_value_wrapper widen_unsigned_types=widen_unsigned_types use_objects_in_arrays=use_objects_in_arrays use_server_time_zone_for_dates=use_server_time_zone_for_dates +autoCommit=autoCommit +auto_commit=auto_commit +createDatabaseIfNotExist=createDatabaseIfNotExist +create_database_if_not_exist=create_database_if_not_exist +continueBatchOnError=continueBatchOnError +continue_batch_on_error=continue_batch_on_error +dialect=dialect +externalDatabase=externalDatabase +external_database=external_database +fetchSize=fetchSize +fetch_size=fetch_size +localFile=localFile +local_file=local_file +jdbcCompliant=jdbcCompliant +jdbc_compliant=jdbc_compliant +namedParameter=namedParameter +named_parameter=named_parameter +nullAsDefault=nullAsDefault +null_as_default=null_as_default +transactionSupport=transactionSupport +transaction_support=transaction_support +wrapperObject=wrapperObject +wrapper_object=wrapper_object diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties index ab00dd4a9..3b3bd2719 100644 --- a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties @@ -12,3 +12,4 @@ product_name=client_name use_binary_string=binary_string_support typeMappings=jdbc_type_mappings databaseTerm=jdbc_schema_term +max_execution_time=clickhouse_setting_max_execution_time diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java index 280411c4f..b564811f5 100644 --- a/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java @@ -31,6 +31,9 @@ public Object[][] providePropertyConversionData() { {"connect_timeout", "10000", "connection_timeout", "10000"}, {"buffer_size", "65536", "client_network_buffer_size", "65536"}, {"sslmode", "strict", "ssl_mode", "strict"}, + {"sslmode", "none", "ssl_mode", "TRUST"}, + {"ssl_mode", "none", "ssl_mode", "TRUST"}, + {"max_execution_time", "60", "clickhouse_setting_max_execution_time", "60"}, {"sslkey", "/path/to/key", "ssl_key", "/path/to/key"}, {"proxy_username", "puser", "proxy_user", "puser"}, {"alive_timeout", "60000", "http_keep_alive_timeout", "60000"}, @@ -179,6 +182,12 @@ public void testDeprecatedPropertiesWithoutConversionAreIgnored() { input.put("buffering", "true"); input.put("max_requests", "10"); input.put("failover", "2"); + input.put("autoCommit", "true"); + input.put("fetchSize", "1000"); + input.put("nullAsDefault", "1"); + input.put("jdbcCompliant", "true"); + input.put("createDatabaseIfNotExist", "true"); + input.put("continueBatchOnError", "true"); Map result = ConfigurationMigrationHelper.convertMap(input); @@ -188,6 +197,18 @@ public void testDeprecatedPropertiesWithoutConversionAreIgnored() { Assert.assertFalse(result.containsKey("clickhouse_setting_protocol")); Assert.assertFalse(result.containsKey("use_compilation")); Assert.assertFalse(result.containsKey("buffering")); + Assert.assertFalse(result.containsKey("autoCommit")); + Assert.assertFalse(result.containsKey("clickhouse_setting_autoCommit")); + Assert.assertFalse(result.containsKey("fetchSize")); + Assert.assertFalse(result.containsKey("clickhouse_setting_fetchSize")); + Assert.assertFalse(result.containsKey("nullAsDefault")); + Assert.assertFalse(result.containsKey("clickhouse_setting_nullAsDefault")); + Assert.assertFalse(result.containsKey("jdbcCompliant")); + Assert.assertFalse(result.containsKey("clickhouse_setting_jdbcCompliant")); + Assert.assertFalse(result.containsKey("createDatabaseIfNotExist")); + Assert.assertFalse(result.containsKey("clickhouse_setting_createDatabaseIfNotExist")); + Assert.assertFalse(result.containsKey("continueBatchOnError")); + Assert.assertFalse(result.containsKey("clickhouse_setting_continueBatchOnError")); } @Test @@ -255,4 +276,49 @@ public void testClasspathReflectionHandlesMissingClassesGracefully() { Assert.assertNotNull(cache.getV2KnownProperties()); Assert.assertNotNull(cache.getV1KnownProperties()); } + + @Test + public void testCaseInsensitivePropertyLookupsAndConversion() { + Map input = new LinkedHashMap<>(); + input.put("USER", "default"); + input.put("PASSWORD", "secret"); + input.put("CONNECT_TIMEOUT", "10000"); + input.put("SSLMODE", "NONE"); + input.put("PROTOCOL", "http"); + input.put("HTTP_KEEP_ALIVE", "false"); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.get("user"), "default"); + Assert.assertEquals(result.get("password"), "secret"); + Assert.assertEquals(result.get("connection_timeout"), "10000"); + Assert.assertEquals(result.get("ssl_mode"), "TRUST"); + Assert.assertFalse(result.containsKey("protocol")); + Assert.assertFalse(result.containsKey("PROTOCOL")); + Assert.assertEquals(result.get("http_keep_alive_timeout"), "0"); + + // Verify parsing converted map with client-v2 ClientConfigProperties succeeds without exceptions + Map parsed = ClientConfigProperties.parseConfigMap(result); + Assert.assertEquals(parsed.get("user"), "default"); + Assert.assertEquals(parsed.get("connection_timeout"), 10000L); + Assert.assertEquals(parsed.get("ssl_mode"), com.clickhouse.client.api.enums.SSLMode.TRUST); + } + + @Test + public void testRuntimeEnumKeysInCacheHaveLowercasedAliases() { + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + + Assert.assertTrue(cache.isV1KnownProperty("CONNECT_TIMEOUT")); + Assert.assertTrue(cache.isV1KnownProperty("connect_timeout")); + + Assert.assertTrue(cache.isV2KnownProperty("USER")); + Assert.assertTrue(cache.isV2KnownProperty("user")); + + Assert.assertTrue(cache.isDeprecatedProperty("PROTOCOL")); + Assert.assertTrue(cache.isDeprecatedProperty("protocol")); + + Assert.assertEquals(cache.getV2MappedKey("CONNECT_TIMEOUT"), "connection_timeout"); + Assert.assertEquals(cache.getV2CanonicalKey("USER"), "user"); + Assert.assertEquals(cache.getV2CanonicalKey("User"), "user"); + } } From 0caba445692b31469128a6faf6c68c1de2177f5c Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 9 Sep 2026 14:14:21 -0700 Subject: [PATCH 6/7] fixed version in main pom --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 171380a10..99fbfc46d 100644 --- a/pom.xml +++ b/pom.xml @@ -73,7 +73,7 @@ - 0.10.0-rc1-SNAPSHOT + 0.11.0-rc1-SNAPSHOT 2026 UTF-8 UTF-8 From 2d8cdaaaabd7653db8d7c9a38a739e1dc8350763 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Wed, 9 Sep 2026 14:18:59 -0700 Subject: [PATCH 7/7] fix building examples --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 530515d8a..6d6393d6c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,7 +61,7 @@ jobs: cp -rf $HOME/.m2/repository/com/clickhouse/clickhouse-jdbc/* ./clickhouse-jdbc-artifacts/ - name: Compile examples run: | - export LIB_VER=$(cat VERSION) + export LIB_VER=$(grep '' pom.xml | sed -e 's|[[:space:]]*<[/]*revision>[[:space:]]*||g') find `pwd`/examples -type f -name pom.xml -exec sed -i -e "s|\(\).*\(<\)|\1$LIB_VER\2|g" {} \; for d in $(ls -d `pwd`/examples/*/); do \ if [ -e $d/pom.xml ]; then cd $d && mvn --batch-mode --no-transfer-progress clean compile; fi;