diff --git a/docker/README.md b/docker/README.md index 0bb74cf81f..8ed7ea8faf 100644 --- a/docker/README.md +++ b/docker/README.md @@ -243,6 +243,17 @@ The Hubble `latest` image is expected to work with HugeGraph Server 1.7 and Server `latest`; compatibility with versions older than 1.7 is not promised. Pin immutable image references when reproducibility is required. +### Server startup timeout + +Every topology gives each Server 120 seconds to answer on its REST port before the container gives up. Raise it on a slow or contended host with `HG_SERVER_STARTUP_TIMEOUT_S=300 docker compose -f docker-compose-hstore.yml up -d`. Leaving it unset keeps 120; an empty value is rejected rather than treated as a silent default, so a missing value in your own script is not mistaken for a deliberate one. + +
+Keeping it inside the health check budget + +The Server health check keeps a separate budget of roughly 360 seconds that this variable does not move. `up -d --wait` gives up there, and so does a plain `up -d`, because Hubble waits on the Server with `depends_on: condition: service_healthy` in every topology. Keep the startup timeout inside that budget, or raise the Server health check in the Compose file alongside it. [The Server docker README](../hugegraph-server/hugegraph-dist/docker/README.md#6-process-supervision--health-checks) has the accepted range and the `docker run` equivalents. + +
+ ### Data persistence Each topology creates its own normal Compose network and named volumes. No diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index 6f599c6870..9c066242e4 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -68,6 +68,9 @@ x-server-environment: &server-environment HG_SERVER_MIN_FREE_MEMORY: "0" HG_SERVER_INIT_STORE_ENABLED: "false" HG_SERVER_REQUIRE_AUTH_TOKEN_SECRET: "true" + # Unset-only default: ":-" would turn an empty host value into 120 + # silently, which the entrypoint rejects on purpose. + HG_SERVER_STARTUP_TIMEOUT_S: ${HG_SERVER_STARTUP_TIMEOUT_S-120} HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:-} diff --git a/docker/docker-compose-hstore.yml b/docker/docker-compose-hstore.yml index d201430692..f6f20bd896 100644 --- a/docker/docker-compose-hstore.yml +++ b/docker/docker-compose-hstore.yml @@ -92,6 +92,9 @@ services: HG_SERVER_REST_URL: http://server:8080 HG_SERVER_MIN_FREE_MEMORY: "0" HG_SERVER_INIT_STORE_ENABLED: "false" + # Unset-only default: ":-" would turn an empty host value into 120 + # silently, which the entrypoint rejects on purpose. + HG_SERVER_STARTUP_TIMEOUT_S: ${HG_SERVER_STARTUP_TIMEOUT_S-120} HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:-} ports: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 828ffe42b1..d85cf5b1c6 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -34,6 +34,10 @@ services: PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:-} HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:-} HG_SERVER_MIN_FREE_MEMORY: "0" + # Unset-only default: a host value overrides it, an absent one + # renders the entrypoint default. ":-" would turn an empty host + # value into 120 silently, which the entrypoint rejects on purpose. + HG_SERVER_STARTUP_TIMEOUT_S: ${HG_SERVER_STARTUP_TIMEOUT_S-120} ports: - "8080:8080" volumes: diff --git a/docker/test-compose.sh b/docker/test-compose.sh index ecd5ab5b2d..7673036481 100644 --- a/docker/test-compose.sh +++ b/docker/test-compose.sh @@ -29,8 +29,15 @@ ACTIVE_PROJECT="" ACTIVE_FILES=() RENDER_DIR="" +# How the render sees HG_SERVER_STARTUP_TIMEOUT_S. Stripped by default, so a +# baseline render shows the Compose default whatever the developer exported; +# render_with_timeout swaps in a value. Kept as one array rather than a second +# copy of the environment below, so a variable added there reaches both renders. +STARTUP_TIMEOUT_ENV=(-u HG_SERVER_STARTUP_TIMEOUT_S) + compose_auth() { - env HUGEGRAPH_VERSION="${VERSION}" \ + env "${STARTUP_TIMEOUT_ENV[@]}" \ + HUGEGRAPH_VERSION="${VERSION}" \ HUBBLE_IMAGE="${RENDER_HUBBLE_IMAGE}" \ HUGEGRAPH_ADMIN_PASSWORD="${PASSWORD}" \ HUGEGRAPH_AUTH_TOKEN_SECRET="${SECRET}" \ @@ -43,6 +50,33 @@ render() { compose_auth "$@" config --format json > "${output}" } +# Same render with an explicit host value, to prove the variable reaches the +# Server environment rather than only defaulting there. An empty value is a +# set value and must survive to the container, which is what separates the +# Compose files' "-" from ":-". +render_with_timeout() { + local output="$1" timeout="$2" + shift 2 + ( + STARTUP_TIMEOUT_ENV=(HG_SERVER_STARTUP_TIMEOUT_S="${timeout}") + compose_auth "$@" config --format json + ) > "${output}" +} + +assert_startup_timeout() { + local rendered="$1" expected="$2" service + shift 2 + for service in "$@"; do + jq -e --arg s "${service}" --arg v "${expected}" \ + '.services[$s].environment.HG_SERVER_STARTUP_TIMEOUT_S == $v' \ + "${rendered}" >/dev/null || { + echo "expected ${service} HG_SERVER_STARTUP_TIMEOUT_S=${expected}" \ + "in ${rendered}" >&2 + return 1 + } + done +} + assert_file_property() { local file="$1" local property="$2" @@ -263,11 +297,46 @@ run_render() { assert_ha "${RENDER_DIR}/ha.json" assert_dev_override "${RENDER_DIR}/dev.json" \ "${RENDER_DIR}/override.json" + + # An absent host value renders the entrypoint default in every topology and + # a host value overrides it, so the container budget can be lowered to meet + # the healthcheck budget without editing the Compose files. Raising it past + # that budget still needs a file edit: the healthcheck literals give every + # Server about 360 seconds, and the documented "up -d --wait" gives up + # there, so this knob aligns the two downward only. + assert_startup_timeout "${RENDER_DIR}/standalone.json" 120 server + assert_startup_timeout "${RENDER_DIR}/hstore.json" 120 server + assert_startup_timeout "${RENDER_DIR}/ha.json" 120 server0 server1 server2 + render_with_timeout "${RENDER_DIR}/standalone-timeout.json" 450 \ + -f "${DOCKER_DIR}/docker-compose.yml" + render_with_timeout "${RENDER_DIR}/hstore-timeout.json" 450 \ + -f "${DOCKER_DIR}/docker-compose-hstore.yml" + render_with_timeout "${RENDER_DIR}/ha-timeout.json" 450 \ + -f "${DOCKER_DIR}/docker-compose-3pd-3store-3server.yml" + assert_startup_timeout "${RENDER_DIR}/standalone-timeout.json" 450 server + assert_startup_timeout "${RENDER_DIR}/hstore-timeout.json" 450 server + assert_startup_timeout "${RENDER_DIR}/ha-timeout.json" 450 \ + server0 server1 server2 + + # The unset and override cases above hold under ":-" too. Only an empty + # host value separates the spellings, and it has to reach the container so + # the entrypoint can reject it, as README section 6 promises. + render_with_timeout "${RENDER_DIR}/standalone-empty.json" "" \ + -f "${DOCKER_DIR}/docker-compose.yml" + render_with_timeout "${RENDER_DIR}/hstore-empty.json" "" \ + -f "${DOCKER_DIR}/docker-compose-hstore.yml" + render_with_timeout "${RENDER_DIR}/ha-empty.json" "" \ + -f "${DOCKER_DIR}/docker-compose-3pd-3store-3server.yml" + assert_startup_timeout "${RENDER_DIR}/standalone-empty.json" "" server + assert_startup_timeout "${RENDER_DIR}/hstore-empty.json" "" server + assert_startup_timeout "${RENDER_DIR}/ha-empty.json" "" \ + server0 server1 server2 echo "Compose render contracts passed" } compose_active() { - env HUGEGRAPH_VERSION="${HUGEGRAPH_VERSION:-latest}" \ + env -u HG_SERVER_STARTUP_TIMEOUT_S \ + HUGEGRAPH_VERSION="${HUGEGRAPH_VERSION:-latest}" \ HUBBLE_IMAGE="${HUBBLE_IMAGE:-hugegraph/hubble:latest}" \ HUGEGRAPH_ADMIN_PASSWORD="${PASSWORD}" \ HUGEGRAPH_AUTH_TOKEN_SECRET="${SECRET}" \ diff --git a/hugegraph-server/hugegraph-dist/docker/README.md b/hugegraph-server/hugegraph-dist/docker/README.md index 9214aa830e..aa76092b82 100644 --- a/hugegraph-server/hugegraph-dist/docker/README.md +++ b/hugegraph-server/hugegraph-dist/docker/README.md @@ -144,3 +144,18 @@ native `HEALTHCHECK` instructions. `docker ps` shows real health status: | `hugegraph/hugegraph-store` | `GET /v1/health` on port 8520 | The entrypoints supervise the Java process directly — when Java exits, the container exits. If started with a restart policy (the provided compose files use `restart: unless-stopped`), Docker will bring it back automatically. The old cron-based monitor (`-m true`) is for VM/bare-metal deployments only and is not used in Docker images. + +`HG_SERVER_STARTUP_TIMEOUT_S` sets how long the Server entrypoint waits for that REST port before ending the container: whole seconds from 1 to 86400, 120 by default. Anything outside the range, an empty value included, stops the container at startup rather than falling back silently. The Compose topologies under `docker/` pass it through with an unset-only default, so `HG_SERVER_STARTUP_TIMEOUT_S=300 docker compose up -d` raises it without editing a Compose file. + +
+Startup budget and health check budget are two clocks + +Raise the startup budget on slow or contended hosts, and wherever an orchestrator already owns it through a probe of its own: a startup probe cannot extend a container that has already ended the JVM it was waiting for. + +```bash +docker run -itd --name=graph -p 8080:8080 -e HG_SERVER_STARTUP_TIMEOUT_S=450 hugegraph/hugegraph:1.7.0 +``` + +Raising it does not move the health check above. The images set `--interval=15s --start-period=90s --retries=3`, so a container given a longer startup budget is reported `unhealthy` around 135 seconds while the entrypoint is still legitimately waiting; raise it with `--health-start-period` on `docker run`. The Compose files replace those values with their own (`start_period: 60s`, `interval: 10s`, `retries: 30`, so roughly 360 seconds), and anything gated on `depends_on: condition: service_healthy`, Hubble included, waits on that budget rather than on this variable. Move the two together. + +
diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh index 6e22885ebe..42137dc3b5 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint-test.sh @@ -35,6 +35,10 @@ backend=rocksdb EOF cat > "${TEST_HOME}/bin/start-hugegraph.sh" <<'EOF' #!/usr/bin/env bash +# One argument per line, so an assertion can read the exact -t value rather +# than substring-matching a flattened "$*", where -t 1200 contains -t 120. +printf '%s\n' "$@" > ./docker/start-hugegraph-argv +printf 'called\n' >> ./docker/start-hugegraph-calls exit 0 EOF cat > "${TEST_HOME}/bin/init-store.sh" <<'EOF' @@ -232,4 +236,92 @@ rm -f "${TEST_HOME}/docker/init_complete" ) grep -Fqx -- '-n' "${TEST_HOME}/docker/init-store-password" +# The value start-hugegraph.sh actually received for -t, read from the +# recorded argument vector so that -t 1200 can never satisfy an assertion +# that wants 120. +last_start_timeout() { + local previous="" argument + while IFS= read -r argument; do + if [[ "${previous}" == "-t" ]]; then + printf '%s\n' "${argument}" + return 0 + fi + previous="${argument}" + done < "${TEST_HOME}/docker/start-hugegraph-argv" + return 1 +} + +# Spelled with an explicit exit rather than a bare [[ ]]: bash 3.2, still the +# /bin/bash of macOS, does not apply set -e to a failing [[ ]], so a bare +# assertion reports PASS there while CI catches the regression. +assert_start_timeout() { + local expected="$1" actual + if ! actual=$(last_start_timeout); then + echo "start-hugegraph.sh received no -t argument" >&2 + exit 1 + fi + if [[ "${actual}" != "${expected}" ]]; then + echo "expected start-hugegraph.sh -t ${expected}, got -t ${actual}" >&2 + exit 1 + fi +} + +# An absent variable keeps the historical default. env -u rather than a bare +# subshell: a child shell inherits an exported HG_SERVER_STARTUP_TIMEOUT_S, so +# without it this case would silently exercise whatever the developer exported. +( + cd "${TEST_HOME}" + env -u HG_SERVER_STARTUP_TIMEOUT_S bash ./docker-entrypoint.sh +) +assert_start_timeout 120 + +( + cd "${TEST_HOME}" + HG_SERVER_STARTUP_TIMEOUT_S=450 bash ./docker-entrypoint.sh +) +assert_start_timeout 450 + +( + cd "${TEST_HOME}" + HG_SERVER_STARTUP_TIMEOUT_S=86400 bash ./docker-entrypoint.sh +) +assert_start_timeout 86400 + +# An empty value is a set value, not an absent one: Compose writes it whenever +# an interpolated host variable is missing. 2m is the shape of a typo, and the +# two large values bracket the point where the deadline arithmetic in +# wait_for_startup would wrap negative and end the wait before its first probe. +for invalid_timeout in "" " " 0 +5 2m 86401 9223372036854775807; do + start_calls_before_invalid=$(wc -l < "${TEST_HOME}/docker/start-hugegraph-calls") + init_calls_before_invalid=$(wc -l < "${TEST_HOME}/docker/init-store-calls") + if ( + cd "${TEST_HOME}" + HG_SERVER_STARTUP_TIMEOUT_S="${invalid_timeout}" \ + bash ./docker-entrypoint.sh + ); then + echo "startup timeout '${invalid_timeout}' unexpectedly succeeded" >&2 + exit 1 + fi + # The server must not have started, and the guard must have run ahead of + # init-store, as the comment above it in the entrypoint claims. + if [[ "$(wc -l < "${TEST_HOME}/docker/start-hugegraph-calls")" -ne \ + "${start_calls_before_invalid}" ]]; then + echo "startup timeout '${invalid_timeout}' started the server" >&2 + exit 1 + fi + if [[ "$(wc -l < "${TEST_HOME}/docker/init-store-calls")" -ne \ + "${init_calls_before_invalid}" ]]; then + echo "startup timeout '${invalid_timeout}' was rejected only after" \ + "init-store ran" >&2 + exit 1 + fi +done + +# Still the default once the rejected values are out of the way. +( + cd "${TEST_HOME}" + env -u HG_SERVER_STARTUP_TIMEOUT_S bash ./docker-entrypoint.sh +) +assert_start_timeout 120 + echo "PASS: Docker entrypoint configures HStore discovery and authentication" diff --git a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh index fe9974c430..b5ba2de34f 100755 --- a/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh +++ b/hugegraph-server/hugegraph-dist/docker/docker-entrypoint.sh @@ -98,6 +98,29 @@ if [[ -n "${HG_SERVER_AUTH_TOKEN_SECRET:-}" ]]; then fi fi +# How long the entrypoint lets the server take to answer on its REST port +# before it gives up and ends the container. An orchestrator that already +# owns this budget through a startup probe needs to raise it, otherwise the +# container terminates a JVM that is still starting and the probe never gets +# to decide. Validated here so a bad value fails before init-store runs, +# rather than reaching the arithmetic in wait_for_startup: that deadline is +# $((now_s + timeout_s)), which wraps negative near the 64-bit ceiling and +# makes the wait exit before its first probe, the very failure this variable +# exists to avoid. The five-digit bound keeps this comparison in range too, +# and a day is already far past any real start. Plain '-' rather than ':-', +# so an explicitly empty value is rejected instead of quietly becoming the +# default: Compose interpolation such as ${SOME_VAR:-} yields empty, not +# unset, whenever the host variable is missing. +SERVER_STARTUP_TIMEOUT_MAX_S=86400 +SERVER_STARTUP_TIMEOUT_S="${HG_SERVER_STARTUP_TIMEOUT_S-120}" +if [[ ! "${SERVER_STARTUP_TIMEOUT_S}" =~ ^[1-9][0-9]{0,4}$ ]] || + (( SERVER_STARTUP_TIMEOUT_S > SERVER_STARTUP_TIMEOUT_MAX_S )); then + log "ERROR: HG_SERVER_STARTUP_TIMEOUT_S must be a whole number of" \ + "seconds from 1 to ${SERVER_STARTUP_TIMEOUT_MAX_S}," \ + "got '${SERVER_STARTUP_TIMEOUT_S}'" + exit 1 +fi + if [[ -n "${PASSWORD:-}" && "${HG_SERVER_REQUIRE_AUTH_TOKEN_SECRET:-false}" == "true" && -z "${HG_SERVER_AUTH_TOKEN_SECRET:-}" ]]; then @@ -217,7 +240,7 @@ else ./bin/init-store.sh fi -./bin/start-hugegraph.sh -j "${JAVA_OPTS:-}" -t 120 +./bin/start-hugegraph.sh -j "${JAVA_OPTS:-}" -t "${SERVER_STARTUP_TIMEOUT_S}" # Post-startup cluster stabilization check (hstore only — rocksdb has no partitions) ACTUAL_BACKEND=$(grep -E '^[[:space:]]*backend[[:space:]]*=' "${GRAPH_CONF}" | head -n 1 | sed 's/.*=//' | tr -d '[:space:]' || true)