diff --git a/VERSION b/VERSION index d4a0b71..7fe52d3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.2.1-dev +v2.2.1 diff --git a/docker/clickhouse/config.d/retention.xml b/docker/clickhouse/config.d/retention.xml new file mode 100644 index 0000000..4107339 --- /dev/null +++ b/docker/clickhouse/config.d/retention.xml @@ -0,0 +1,16 @@ + + + + 900 + + + + + 100M + 3 + + diff --git a/docker/docker-compose/base/docker-compose.clickhouse.yml b/docker/docker-compose/base/docker-compose.clickhouse.yml index 4e60c21..813ed13 100644 --- a/docker/docker-compose/base/docker-compose.clickhouse.yml +++ b/docker/docker-compose/base/docker-compose.clickhouse.yml @@ -4,6 +4,7 @@ services: container_name: clickhouse-server volumes: - ../../create_tables:/docker-entrypoint-initdb.d + - ../../clickhouse/config.d/retention.xml:/etc/clickhouse-server/config.d/retention.xml:ro - ch_data:/var/lib/clickhouse/ - ch_logs:/var/log/clickhouse-server/ environment: diff --git a/docker/docker-compose/base/docker-compose.kafka.yml b/docker/docker-compose/base/docker-compose.kafka.yml index 01a9688..7434380 100644 --- a/docker/docker-compose/base/docker-compose.kafka.yml +++ b/docker/docker-compose/base/docker-compose.kafka.yml @@ -9,10 +9,19 @@ x-kafka-broker-env: &kafka-broker-env KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 3 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 2 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 3 - KAFKA_LOG_RETENTION_HOURS: 4 - KAFKA_LOG_RETENTION_BYTES: 10737418240 - KAFKA_LOG_SEGMENT_BYTES: 1073741824 - KAFKA_LOG_CLEANUP_POLICY: delete + # Retention is evaluated only for closed log segments. Keep the segment + # smaller and roll it frequently so that the four-hour retention period is + # effective even for low-volume partitions. + # + # log.retention.bytes applies to *each partition replica*, not to the broker. + # With 12 partitions and replication factor 3, a 10 GiB value can consume + # tens of GiB across the three broker volumes. + KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" + KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-16777216}" + KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-4194304}" + KAFKA_LOG_SEGMENT_MS: "${KAFKA_LOG_SEGMENT_MS:-900000}" + KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: "${KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS:-300000}" + KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" services: kafka1: diff --git a/docker/docker_swarm/docker-compose.swarm.yml b/docker/docker_swarm/docker-compose.swarm.yml index 784c76d..da95e78 100644 --- a/docker/docker_swarm/docker-compose.swarm.yml +++ b/docker/docker_swarm/docker-compose.swarm.yml @@ -11,9 +11,14 @@ x-kafka-broker-env: &kafka-broker-env KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "${KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR:-3}" KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "${KAFKA_TRANSACTION_STATE_LOG_MIN_ISR:-2}" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "${KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR:-3}" + # Retention applies only to closed segments and retention.bytes is per + # partition replica. Frequent, small segments make the retention window + # effective and keep the replicated pipeline topics bounded on disk. KAFKA_LOG_RETENTION_HOURS: "${KAFKA_LOG_RETENTION_HOURS:-4}" - KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-10737418240}" - KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-1073741824}" + KAFKA_LOG_RETENTION_BYTES: "${KAFKA_LOG_RETENTION_BYTES:-16777216}" + KAFKA_LOG_SEGMENT_BYTES: "${KAFKA_LOG_SEGMENT_BYTES:-4194304}" + KAFKA_LOG_SEGMENT_MS: "${KAFKA_LOG_SEGMENT_MS:-900000}" + KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: "${KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS:-300000}" KAFKA_LOG_CLEANUP_POLICY: "${KAFKA_LOG_CLEANUP_POLICY:-delete}" x-hamstring-kafka-wait-env: &hamstring-kafka-wait-env @@ -113,6 +118,7 @@ services: mode: ingress volumes: - "${HAMSTRING_ROOT:-../..}/docker/create_tables:/docker-entrypoint-initdb.d" + - "${HAMSTRING_ROOT:-../..}/docker/clickhouse/config.d/retention.xml:/etc/clickhouse-server/config.d/retention.xml:ro" - ch_data:/var/lib/clickhouse/ - ch_logs:/var/log/clickhouse-server/ healthcheck: diff --git a/docs/monitoring.rst b/docs/monitoring.rst index 356e7aa..9b2c27c 100644 --- a/docs/monitoring.rst +++ b/docs/monitoring.rst @@ -33,6 +33,32 @@ All modules send their monitoring-relevant information to Kafka, from which it i default configuration defined in ``docker-compose.yml``, `Grafana` automatically loads the dashboard views and fills them with the data in `ClickHouse`. The dashboard views can then be observed on ``localhost:3000`` (by default). +Storage retention +----------------- + +HAMSTRING treats Kafka as transient transport. By default each Kafka partition replica has a four-hour / 16 MiB +retention target, whichever limit is reached first. Kafka can delete only *closed* log segments, so the Compose files +also roll a segment every 15 minutes (or after 4 MiB); this makes time-based retention work for low-volume topics. +All Kafka values can be overridden at startup, for example: + +.. code-block:: console + + KAFKA_LOG_RETENTION_HOURS=12 KAFKA_LOG_RETENTION_BYTES=67108864 \\ + HOST_IP=127.0.0.1 docker compose -f docker/docker-compose.yml --profile prod up + +``KAFKA_LOG_RETENTION_BYTES`` is a limit **per partition replica**, rather than a per-broker or per-volume limit. +Increasing the partition count, replication factor, or this value increases the possible disk usage accordingly. + +ClickHouse monitoring tables have a one-day TTL (alerts have a 60-day TTL). TTL deletion happens in background +merges; the supplied ClickHouse configuration checks for TTL work every 15 minutes. Its server logs are also rotated +at 100 MiB with three retained files. + +The new retention settings apply after restarting Kafka and ClickHouse. They reclaim data as the brokers close new +segments and ClickHouse performs TTL merges; they do not remove a stopped stack's volume files immediately. If this is +a disposable local environment and its historical data is not needed, stop the stack and remove its volumes with +``docker compose -f docker/docker-compose.yml down -v`` before starting it again. Do not remove production volumes as +a retention operation. + `Datatest` mode ---------------