From 28f59b837de9871eaaa2b96493871cb6f3007716 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Tue, 29 Sep 2026 11:33:11 +0300 Subject: [PATCH 1/5] [#1086] Join replication in the background on every start of the Docker image --- .github/workflows/build.yml | 163 ++++++--- opendj-packages/opendj-docker/Dockerfile | 9 +- .../opendj-docker/Dockerfile-alpine | 9 +- opendj-packages/opendj-docker/README.md | 69 +++- .../opendj-docker/bootstrap/join.sh | 343 ++++++++++++++++++ .../opendj-docker/bootstrap/replicate.sh | 49 +-- opendj-packages/opendj-docker/run.sh | 51 ++- 7 files changed, 589 insertions(+), 104 deletions(-) create mode 100755 opendj-packages/opendj-docker/bootstrap/join.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 327a02a931..544caae771 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -581,72 +581,133 @@ jobs: - name: Docker test replication shell: bash run: | + set -E IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }} - REPLICAS="test_replica test_replica_sdsr" - cleanup() { docker rm -f test_master $REPLICAS >/dev/null 2>&1 || true; docker network rm test_replication >/dev/null 2>&1 || true; } + NODES="dj-0 dj-1 dj-2 dj-x dj-sdsr dj-shm-probe dj-shm" + VOLUMES="vol-dj-0 vol-dj-1 vol-dj-2 vol-dj-x" + cleanup() { docker rm -f $NODES >/dev/null 2>&1 || true; docker network rm test_replication >/dev/null 2>&1 || true; docker volume rm -f $VOLUMES >/dev/null 2>&1 || true; } cleanup - trap 'code=$?; for c in test_master $REPLICAS; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR + trap 'code=$?; for c in $NODES; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR # every tool reads the root password from a file (#1084, #1092); dsreplication run with -n prints # no command line, so a password put back on one would pass every check below - rc=0; docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword|rootUserPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh || rc=$? - if [ $rc -ne 1 ]; then echo "::error::setup.sh or replicate.sh passes the root password on a command line, or grep could not read them"; false; fi + rc=0; docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword|rootUserPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh /opt/opendj/bootstrap/join.sh || rc=$? + if [ $rc -ne 1 ]; then echo "::error::a bootstrap script passes the root password on a command line, or grep could not read them"; false; fi # the password file goes to /dev/shm, off the writable layer of the container, and the mktemp of the image puts it there + docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.' /opt/opendj/bootstrap/join.sh || { echo "::error::join.sh no longer puts the password file on /dev/shm"; false; } docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.' /opt/opendj/bootstrap/replicate.sh || { echo "::error::replicate.sh no longer puts the password file on /dev/shm"; false; } - docker run --rm --entrypoint sh "$IMAGE" -c 'f=$(mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.XXXXXX") && rm -f "$f" && case $f in /dev/shm/opendj-replicate.4444.*) ;; *) exit 1;; esac' || { echo "::error::mktemp in the image does not create the password file on /dev/shm"; false; } + docker run --rm --entrypoint sh "$IMAGE" -c 'f=$(mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.XXXXXX") && rm -f "$f" && case $f in /dev/shm/opendj-join.4444.*) ;; *) exit 1;; esac' || { echo "::error::mktemp in the image does not create the password file on /dev/shm"; false; } # a password with a space in it reaches every tool as one value ROOT_PASSWORD='replication secret' docker network create test_replication - docker run --rm -it -d --memory="512m" --network test_replication --ipc=shareable --name=test_master --hostname=dj-master -e ADD_BASE_ENTRY="--addBaseEntry" -e ROOT_PASSWORD="$ROOT_PASSWORD" "$IMAGE" - timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_master | grep -q \"healthy\"; do sleep 10; done' - # a replica reports itself healthy only once replicate.sh has succeeded; the sdsr replica joins after - # the simple one, as two dsreplication enable at once would both rewrite the admin data of the master - # a Kubernetes pod keeps its /dev/shm across container restarts: the replica shares the /dev/shm of the master, where - # a password file waits as a killed replicate.sh would have left it, and its run.sh has to remove it (checked below); - # the file of another container of the pod, which listens on another admin port, has to be kept - docker exec test_master sh -c 'printf "%s\n" "$ROOT_PASSWORD" >/dev/shm/opendj-replicate.4444.killed' - docker exec test_master sh -c ': >/dev/shm/opendj-replicate.5444.other' - docker run --rm -it -d --memory="512m" --network test_replication --ipc=container:test_master --name=test_replica --hostname=dj-replica -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=simple "$IMAGE" - # on a first start the master stops the server its bootstrap started and starts it again, and a replica - # started together with it can reach it in between: replicate.sh tries a dsreplication that could not - # connect again. The master is taken off the network while the replica sleeps before its first try, and - # comes back only once the replica has said it will try again - timeout 5m bash -c 'until docker logs test_replica 2>&1 | grep -q "Will sleep for a bit"; do sleep 0.2; done' - docker network disconnect test_replication test_master - timeout 2m bash -c 'until docker logs test_replica 2>&1 | grep -q "exited with 8, trying again"; do sleep 1; done' - docker network connect --alias dj-master test_replication test_master - timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica | grep -q \"healthy\"; do sleep 10; done' - docker run --rm -it -d --memory="512m" --network test_replication --name=test_replica_sdsr --hostname=dj-replica-sdsr -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" - # the same for the sdsr replica, whose dsreplication enable is another command of replicate.sh - timeout 5m bash -c 'until docker logs test_replica_sdsr 2>&1 | grep -q "Will sleep for a bit"; do sleep 0.2; done' - docker network disconnect test_replication test_master - timeout 2m bash -c 'until docker logs test_replica_sdsr 2>&1 | grep -q "exited with 8, trying again"; do sleep 1; done' - docker network connect --alias dj-master test_replication test_master - timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica_sdsr | grep -q \"healthy\"; do sleep 10; done' - # the replicas were initialized from the master, and a change made on the master reaches them - for c in $REPLICAS; do - docker exec $c /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 - done - printf 'dn: ou=replicated,dc=example,dc=com\nobjectClass: organizationalUnit\nou: replicated\n' | docker exec -i test_master /opt/opendj/bin/ldapmodify --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd - for c in $REPLICAS; do - timeout 1m bash -c 'until docker exec $1 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" $c + # small retry values keep the seed decision and the failed-join case quick; the volume holds + # the instance so that a container can be replaced with or without its data surviving + start_node() { # + docker volume create "vol-$1" >/dev/null + docker run -d --memory="512m" --network test_replication --name "$1" --hostname "$1" \ + -v "vol-$1:/opt/opendj/data" \ + -e ROOT_PASSWORD="$ROOT_PASSWORD" -e ADD_BASE_ENTRY="--addBaseEntry" \ + -e OPENDJ_REPLICATION_TYPE=simple -e REPLICATION_PEERS="$2" \ + -e REPLICATION_RETRY_COUNT=10 -e REPLICATION_RETRY_INTERVAL=3 -e REPLICATION_ATTEMPT_TIMEOUT=90 \ + "$IMAGE" + } + wait_healthy() { timeout 8m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" "$0" | grep -q \"healthy\"; do sleep 5; done' "$1"; } + # the first peer of REPLICATION_PEERS, and only it, seeds a topology its retries could not find + start_node dj-0 dj-0,dj-1 + wait_healthy dj-0 + docker logs dj-0 2>&1 | grep -q "seeding it with this server's data" || { echo "::error::dj-0 did not seed the topology"; false; } + # a joining server tries again while its peer is unreachable, and decides membership from what + # is there, not from the exit code of dsreplication enable + docker network disconnect test_replication dj-0 + start_node dj-1 dj-0,dj-1 + timeout 5m bash -c 'until docker logs dj-1 2>&1 | grep -q "trying again in"; do sleep 2; done' + docker network connect --alias dj-0 test_replication dj-0 + wait_healthy dj-1 + docker logs dj-1 2>&1 | grep -q "joined the replication topology through dj-0" || { echo "::error::dj-1 did not join through dj-0"; false; } + # the bootstrapped volume of dj-1 was initialized from the topology although its BASE_DN + # held the imported base entry - entries in BASE_DN say nothing about who holds the data + docker logs dj-1 2>&1 | grep -q "initializing from dj-0" || { echo "::error::dj-1 did not initialize from the topology"; false; } + # a change made on the seed reaches the replica + printf 'dn: ou=replicated,dc=example,dc=com\nobjectClass: organizationalUnit\nou: replicated\n' | docker exec -i dj-0 /opt/opendj/bin/ldapmodify --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd + timeout 1m bash -c 'until docker exec dj-1 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" + # a member is ready again right after a restart, without waiting for its peers, and + # replication still flows + docker restart dj-0 dj-1 + wait_healthy dj-0 + wait_healthy dj-1 + printf 'dn: ou=replicated2,dc=example,dc=com\nobjectClass: organizationalUnit\nou: replicated2\n' | docker exec -i dj-1 /opt/opendj/bin/ldapmodify --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd + timeout 1m bash -c 'until docker exec dj-0 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated2,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" + # a Kubernetes pod keeps its /dev/shm across container restarts, shared by all its containers: + # a starting container removes the join password files a killed join left there - only those of + # its own ADMIN_PORT, the files of the other containers of the pod are not its to remove + docker run -d --memory="64m" --ipc=shareable --name dj-shm --entrypoint sleep "$IMAGE" 600 + docker exec dj-shm sh -c ': >/dev/shm/opendj-join.4444.killed && : >/dev/shm/opendj-join.5444.other' + docker run -d --memory="512m" --network test_replication --ipc=container:dj-shm --name dj-shm-probe --hostname dj-shm-probe -e ROOT_PASSWORD="$ROOT_PASSWORD" "$IMAGE" + timeout 1m bash -c 'while docker exec dj-shm test -e /dev/shm/opendj-join.4444.killed; do sleep 2; done' + docker exec dj-shm test -e /dev/shm/opendj-join.5444.other || { echo "::error::run.sh removed the join password file of another container"; false; } + docker rm -f dj-shm-probe dj-shm + # a join that cannot succeed keeps the container from reporting itself healthy - across a + # restart too, where the health marker used to follow the upgrade and a failed join turned + # into a healthy, unreplicated server. dj-x may not seed: it is not the first peer of its list + start_node dj-x dj-absent,dj-x + timeout 5m bash -c 'until docker logs dj-x 2>&1 | grep -q "could not join the replication topology"; do sleep 5; done' + [ "$(docker inspect --format='{{.State.Health.Status}}' dj-x)" != healthy ] || { echo "::error::dj-x reports itself healthy although its join failed"; false; } + docker restart dj-x + timeout 5m bash -c 'until docker logs dj-x 2>&1 | grep -q "never joined its replication topology"; do sleep 2; done' + timeout 5m bash -c 'until [ "$(docker logs dj-x 2>&1 | grep -c "could not join the replication topology")" -ge 2 ]; do sleep 5; done' + [ "$(docker inspect --format='{{.State.Health.Status}}' dj-x)" != healthy ] || { echo "::error::a restart turned the never-joined dj-x healthy"; false; } + docker rm -f dj-x + docker volume rm vol-dj-x + # the seed lost its volume: behind the same name, a fresh dj-0 finds the topology at dj-1 and + # takes its data from it instead of seeding an empty one next to it + docker rm -f dj-0 + docker volume rm vol-dj-0 + start_node dj-0 dj-0,dj-1 + wait_healthy dj-0 + docker logs dj-0 2>&1 | grep -q "initializing from dj-1" || { echo "::error::the reborn dj-0 did not initialize from dj-1"; false; } + if docker logs dj-0 2>&1 | grep -q "seeding it with this server's data"; then echo "::error::the reborn dj-0 seeded a topology although dj-1 held it"; false; fi + docker exec dj-0 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 + # scale up to three - the peer list is configuration, so the running servers are replaced with + # the longer list before the third one starts, as a rolling update would + docker rm -f dj-0 dj-1 + start_node dj-0 dj-0,dj-1,dj-2 + start_node dj-1 dj-0,dj-1,dj-2 + wait_healthy dj-0 + wait_healthy dj-1 + start_node dj-2 dj-0,dj-1,dj-2 + wait_healthy dj-2 + docker exec dj-2 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 + # scale down to two: the survivors, restarted with the shorter list, remove dj-2 from + # cn=admin data and from their replication server lists - dsreplication disable cannot, + # dj-2 being already gone + docker rm -f dj-2 + docker volume rm vol-dj-2 + docker rm -f dj-0 dj-1 + start_node dj-0 dj-0,dj-1 + start_node dj-1 dj-0,dj-1 + wait_healthy dj-0 + wait_healthy dj-1 + # whichever survivor's join ran first deletes the cn=admin data entry, the delete replicates + # to the other; each survivor prunes its own replication server lists + timeout 3m bash -c 'until docker logs dj-0 2>&1 | grep -q "removing departed server dj-2" || docker logs dj-1 2>&1 | grep -q "removing departed server dj-2"; do sleep 5; done' + for c in dj-0 dj-1; do + timeout 2m bash -c 'while docker exec '"$c"' /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 4444 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$0" --baseDN "cn=Servers,cn=admin data" --searchScope one "(objectClass=*)" hostname | grep -q dj-2; do sleep 5; done' "$ROOT_PASSWORD" + timeout 2m bash -c 'while docker exec '"$c"' /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 4444 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$0" --baseDN "cn=config" --searchScope sub "(|(objectClass=ds-cfg-replication-server)(objectClass=ds-cfg-replication-domain))" ds-cfg-replication-server | grep -q dj-2; do sleep 5; done' "$ROOT_PASSWORD" done - # replicate.sh tries dsreplication enable again only when it exits 8, and a failed enable ends it: run once more - # on the replica, the enable of a base DN already replicated exits 5 and nothing is tried again or initialized - rc=0; out=$(docker exec -e BASE_DN=dc=example,dc=com -e ROOT_USER_DN="cn=Directory Manager" test_replica timeout 90 /opt/opendj/bootstrap/replicate.sh 2>&1) || rc=$? - if [ $rc -ne 5 ] || grep -qE "trying again|initializing replication" <<<"$out"; then - echo "$out"; echo "::error::a second replicate.sh exited with $rc, not with the 5 of its dsreplication enable, or went on after it"; false - fi - # the root password shows in no container log, and the files setup.sh and replicate.sh passed it in are gone (#1084, #1092) - for c in test_master $REPLICAS; do + # replication between the survivors is intact + printf 'dn: ou=replicated3,dc=example,dc=com\nobjectClass: organizationalUnit\nou: replicated3\n' | docker exec -i dj-0 /opt/opendj/bin/ldapmodify --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd + timeout 1m bash -c 'until docker exec dj-1 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated3,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" + # the deprecated one-shot sdsr path of replicate.sh still bootstraps a replica + docker run --rm -it -d --memory="512m" --network test_replication --name dj-sdsr --hostname dj-sdsr -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-0 -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" + wait_healthy dj-sdsr + timeout 1m bash -c 'until docker exec dj-sdsr /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated3,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" + # the root password shows in no container log, and the files the tools read it from are gone + for c in dj-0 dj-1 dj-sdsr; do if docker logs $c 2>&1 | grep -F "$ROOT_PASSWORD"; then echo "::error::The root password is in the log of $c"; false; fi done - for c in test_master $REPLICAS; do - # a JVM keeps its command line in /tmp/hsperfdata_* while it runs; the HEALTHCHECK no longer binds as root (#1092), - # so no process left running has the root password on it + for c in dj-0 dj-1 dj-sdsr; do left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm || true) if [ -n "$left" ]; then echo "::error::The root password is left in $left of $c"; false; fi done - docker exec test_replica test -e /dev/shm/opendj-replicate.5444.other || { echo "::error::run.sh of test_replica removed the password file of another container"; false; } cleanup - name: Docker test secret volume # a keystore mounted at SECRET_VOLUME is what LDAPS serves from the first start on, a diff --git a/opendj-packages/opendj-docker/Dockerfile b/opendj-packages/opendj-docker/Dockerfile index 3ffaac8231..05188d5c65 100644 --- a/opendj-packages/opendj-docker/Dockerfile +++ b/opendj-packages/opendj-docker/Dockerfile @@ -29,6 +29,13 @@ ENV ROOT_USER_DN="cn=Directory Manager" ENV OPENDJ_SSL_OPTIONS="--generateSelfSignedCertificate" #ENV MASTER_SERVER #ENV OPENDJ_REPLICATION_TYPE +# every server of the replication topology, comma separated; the first entry may seed a new +# topology. MASTER_SERVER keeps working as a one-element list. See bootstrap/join.sh +#ENV REPLICATION_PEERS +#ENV REPLICATION_PORT=8989 +#ENV REPLICATION_RETRY_COUNT=30 +#ENV REPLICATION_RETRY_INTERVAL=10 +#ENV REPLICATION_ATTEMPT_TIMEOUT=120 ENV OPENDJ_USER="opendj" #ENV OPENDJ_JAVA_ARGS="" ENV BACKEND_TYPE="je" @@ -68,7 +75,7 @@ COPY --chown=$OPENDJ_USER:0 bootstrap/ /opt/opendj/bootstrap/ COPY --chown=$OPENDJ_USER:0 run.sh /opt/opendj/run.sh COPY --chown=$OPENDJ_USER:0 healthcheck.sh /opt/opendj/healthcheck.sh -RUN chmod +x /opt/opendj/run.sh /opt/opendj/healthcheck.sh /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh +RUN chmod +x /opt/opendj/run.sh /opt/opendj/healthcheck.sh /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh /opt/opendj/bootstrap/join.sh EXPOSE $PORT/tcp $LDAPS_PORT/tcp $ADMIN_PORT/tcp diff --git a/opendj-packages/opendj-docker/Dockerfile-alpine b/opendj-packages/opendj-docker/Dockerfile-alpine index 55658cb1ea..cdfa282d9f 100644 --- a/opendj-packages/opendj-docker/Dockerfile-alpine +++ b/opendj-packages/opendj-docker/Dockerfile-alpine @@ -29,6 +29,13 @@ ENV ROOT_USER_DN="cn=Directory Manager" ENV OPENDJ_SSL_OPTIONS="--generateSelfSignedCertificate" #ENV MASTER_SERVER #ENV OPENDJ_REPLICATION_TYPE +# every server of the replication topology, comma separated; the first entry may seed a new +# topology. MASTER_SERVER keeps working as a one-element list. See bootstrap/join.sh +#ENV REPLICATION_PEERS +#ENV REPLICATION_PORT=8989 +#ENV REPLICATION_RETRY_COUNT=30 +#ENV REPLICATION_RETRY_INTERVAL=10 +#ENV REPLICATION_ATTEMPT_TIMEOUT=120 ENV OPENDJ_USER="opendj" #ENV OPENDJ_JAVA_ARGS="" ENV BACKEND_TYPE="je" @@ -72,7 +79,7 @@ COPY --chown=$OPENDJ_USER:0 bootstrap/ /opt/opendj/bootstrap/ COPY --chown=$OPENDJ_USER:0 run.sh /opt/opendj/run.sh COPY --chown=$OPENDJ_USER:0 healthcheck.sh /opt/opendj/healthcheck.sh -RUN chmod +x /opt/opendj/run.sh /opt/opendj/healthcheck.sh /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh +RUN chmod +x /opt/opendj/run.sh /opt/opendj/healthcheck.sh /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh /opt/opendj/bootstrap/join.sh EXPOSE $PORT/tcp $LDAPS_PORT/tcp $ADMIN_PORT/tcp diff --git a/opendj-packages/opendj-docker/README.md b/opendj-packages/opendj-docker/README.md index dcc0f2502c..8c49cc60fb 100644 --- a/opendj-packages/opendj-docker/README.md +++ b/opendj-packages/opendj-docker/README.md @@ -16,9 +16,10 @@ docker run -d -p 1389:1389 -p 1636:1636 -p 4444:4444 --name opendj openidentityp The image reports itself `healthy` once the server answers on `LDAPS_PORT` *and* the whole bootstrap has succeeded - the instance, the `userRoot` backend over `BASE_DN`, whatever -`ADD_BASE_ENTRY` and `SAMPLE_DATA` asked to be imported into it, and the replication asked -for by `MASTER_SERVER`. Waiting for that status is therefore enough before the first search -of what the bootstrap was told to create: +`ADD_BASE_ENTRY` and `SAMPLE_DATA` asked to be imported into it, and, where replication is +asked for, the join of the replication topology (see [Replication](#replication)). Waiting +for that status is therefore enough before the first search of what the bootstrap was told +to create: ```bash docker run -d --name opendj -e ADD_BASE_ENTRY=--addBaseEntry openidentityplatform/opendj @@ -52,14 +53,7 @@ set them before the upgrade. The server answering is not enough on a first start: the bootstrap starts the server, and once it is done that server is stopped and started again in the foreground, so a client -that only waits for the port can have its first requests fail in between. A replica set up -with `MASTER_SERVER` tries a master it cannot connect to again, every 10 s for up to 5 -minutes, so a master that is down when the replica reaches it does not fail its replication -setup; one that stops while `dsreplication enable` is writing to it still does. - -With `OPENDJ_REPLICATION_TYPE=srs`, start the directory server replicas one at a time, each -once the previous one is healthy: every replica pushes its data to the replicas connected at -that moment, and one that is restarting at the end of its own first start misses it. +that only waits for the port can have its first requests fail in between. A bootstrap that imports `SAMPLE_DATA` can take minutes on a small container, which is what the start period allows for. A bootstrap that fails - or an upgrade that fails when starting @@ -72,6 +66,50 @@ behind to it - those of a health check that ran past its timeout, say. Run the c with `docker run --init` (`init: true` in Compose) to put a PID 1 in front of the server that reaps them and passes SIGTERM on to it. +## Replication + +With `OPENDJ_REPLICATION_TYPE=simple`, the container joins its replication topology in the +background, next to the running server, on every start - not as a step of the first +bootstrap, so a join that could not complete is tried again on the next start, and +membership that changed while no container ran is repaired. `REPLICATION_PEERS` lists every +server of the topology, comma separated; all of them share `BASE_DN`, `ROOT_USER_DN`, +`ROOT_PASSWORD`, `ADMIN_PORT` and `REPLICATION_PORT`. A Kubernetes StatefulSet derives the +list from its ordinals (`-0.,…,-N-1.`), so it needs +no registry; plain `docker run` passes the container names. `MASTER_SERVER` keeps working +as a one-element list. A server recognises itself in the list by its `hostname -f` or by +the first DNS label of it - a pod named `-N` is listed as `-N.`. + +The join decides from what is there, not from an exit code: this server is a member once +its configuration holds the replication domain for `BASE_DN` and it is registered in +`cn=admin data`. Until then it runs `dsreplication enable` through every listed peer, +`REPLICATION_RETRY_COUNT` times every `REPLICATION_RETRY_INTERVAL` seconds, each attempt +bounded by `REPLICATION_ATTEMPT_TIMEOUT` seconds, and the container reports itself healthy +only once the join succeeded - across restarts too. A server that already is a member is +ready as soon as it serves, without waiting for its peers, so a whole cluster restart does +not deadlock under `OrderedReady`. + +A volume the container bootstrapped is initialized from the topology once the join +succeeds, whether or not `BASE_DN` already holds entries: every fresh volume holds what +`ADD_BASE_ENTRY` or `SAMPLE_DATA` imported, which says nothing about which server holds +the data of the topology. Only the first entry of `REPLICATION_PEERS` may decide - once +its retries are exhausted without finding anyone - that there is no topology yet, and seed +it with its own data. The residual risk of that rule: with every server down at once *and* +the volume of the first peer lost, the first peer seeds an empty topology and the others +initialize from it; keep backups accordingly. + +With `REPLICATION_PEERS` set explicitly, a joined server also removes every server that is +registered in the topology but no longer listed: its entries in `cn=admin data` and its +values in the local replication server lists. A scale-down therefore needs no `preStop` +hook - `dsreplication disable` on termination would take the server out on every rolling +restart, and cannot clean up a server that is already gone. With only `MASTER_SERVER` set +nothing is removed, so servers joined by hand stay. + +The one-shot types `srs`, `sdsr` and `rg` keep their previous behaviour - they run once, +during the first bootstrap only - and are deprecated in favour of `simple`. With +`OPENDJ_REPLICATION_TYPE=srs`, start the directory server replicas one at a time, each +once the previous one is healthy: every replica pushes its data to the replicas connected +at that moment, and one that is restarting at the end of its own first start misses it. + ## Certificates With the default `OPENDJ_SSL_OPTIONS` the instance serves LDAPS and StartTLS with a @@ -150,10 +188,15 @@ with `subPath` when the Secret changes. | ROOT_PASSWORD | password | Initial root user password; the bootstrap fails if it contains a line break (CR or LF) | | SECRET_VOLUME | /var/secrets/opendj | Mounted keystore volume, if present its `key*` and `trust*` files are copied into the instance on every start, see [Certificates](#certificates) | | SECRET_VOLUME_REFRESH | 60 | While the server runs, `SECRET_VOLUME` is checked again every that many seconds and changed files are copied again; `0` copies them on start only | -| MASTER_SERVER | - | Replication master server | +| MASTER_SERVER | - | Replication master server; with `simple` it works as a one-element `REPLICATION_PEERS` | +| REPLICATION_PEERS | value of MASTER_SERVER | every server of the replication topology, comma separated; the first entry may seed a new topology, and servers no longer listed are removed from it, see [Replication](#replication) | +| REPLICATION_PORT | 8989 | replication port, the same on every server of the topology | +| REPLICATION_RETRY_COUNT | 30 | rounds of join attempts through every peer before the join gives up: the first peer of `REPLICATION_PEERS` then seeds the topology, any other server stays `unhealthy` | +| REPLICATION_RETRY_INTERVAL | 10 | seconds between rounds of join attempts | +| REPLICATION_ATTEMPT_TIMEOUT | 120 | seconds a single `dsreplication` run may take before it is killed and tried again; `dsreplication enable` can hang on a peer that stops mid-operation | | VERSION | - | OpenDJ version | | OPENDJ_USER | opendj | user which runs OpenDJ | -| OPENDJ_REPLICATION_TYPE | - | OpenDJ Replication type, valid values are:
  • simple - standart replication
  • srs - standalone replication servers
  • sdsr - Standalone Directory Server Replicas
  • rg - Replication Groups
Other values will be ignored | +| OPENDJ_REPLICATION_TYPE | - | OpenDJ Replication type, valid values are:
  • simple - standard replication, joined in the background on every start, see [Replication](#replication)
  • srs - standalone replication servers (one-shot, deprecated)
  • sdsr - Standalone Directory Server Replicas (one-shot, deprecated)
  • rg - Replication Groups (one-shot, deprecated)
Other values will be ignored | | OPENDJ_SSL_OPTIONS | --generateSelfSignedCertificate | you can replace ssl options at here, like : "--usePkcs12keyStore /opt/domain.pfx --keyStorePassword domain" | | OPENDJ_JAVA_ARGS | -server | extra instance java args | | BACKEND_TYPE | je | OpenDJ backend type, see [dsconfig create-backend](https://doc.openidentityplatform.org/opendj/reference/dsconfig-subcommands-ref#dsconfig-create-backend) documentation | diff --git a/opendj-packages/opendj-docker/bootstrap/join.sh b/opendj-packages/opendj-docker/bootstrap/join.sh new file mode 100755 index 0000000000..97636d5c67 --- /dev/null +++ b/opendj-packages/opendj-docker/bootstrap/join.sh @@ -0,0 +1,343 @@ +#!/usr/bin/env bash +# The contents of this file are subject to the terms of the Common Development and +# Distribution License (the License). You may not use this file except in compliance with the +# License. +# +# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the +# specific language governing permission and limitations under the License. +# +# When distributing Covered Software, include this CDDL Header Notice in each file and include +# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL +# Header, with the fields enclosed by brackets [] replaced by your own identifying +# information: "Portions copyright [year] [name of copyright owner]". +# +# Copyright 2026 3A Systems, LLC. + +# Joins this server to the replication topology (#1086). run.sh starts it in the background +# next to the server on every start, not as a step of the first bootstrap: a join that could +# not complete is tried again on the next start, and membership that changed while no +# container ran is repaired. It covers OPENDJ_REPLICATION_TYPE=simple; the one-shot srs, sdsr +# and rg paths stay in replicate.sh. +# +# The contract with the operator: REPLICATION_PEERS lists every server of the topology, and +# all of them share BASE_DN, ROOT_USER_DN, ROOT_PASSWORD, ADMIN_PORT and REPLICATION_PORT. A +# StatefulSet chart derives the list from its ordinals (-0. ... -N-1.), +# plain docker run passes the container names. MASTER_SERVER keeps working as a one-element +# list. This server recognises itself in the list by comparing a peer with hostname -f, or +# the peer's first DNS label with its own: a StatefulSet pod is named -N but listed as +# -N.. The anchored comparison is deliberate - the grep of /etc/hosts +# it replaces took opendj-1 for opendj-10, and a master that lost its volume for a replica +# of itself. +# +# Membership is decided from what is there, never from an exit code: exit 5 of dsreplication +# enable (REPLICATION_CANNOT_BE_ENABLED_ON_BASEDN) means "no suffix was left to enable", +# which covers a base DN that is already replicated and a base DN that one of the two +# servers does not hold - the race with a peer whose bootstrap has not created the backend +# yet. The step is a member once the replication domain for BASE_DN exists in its cn=config +# and the server is registered in cn=admin data; anything else is tried again, whatever the +# exit code, because a repeated enable is safe exactly when it is decided this way. +# +# Initialization follows what the volume went through, not whether BASE_DN has entries: +# every fresh volume has entries (setup.sh imports the base entry or SAMPLE_DATA), and two +# freshly bootstrapped volumes even share a generation ID, so replication silently carries +# nothing that reached the seed by import-ldif. run.sh marks a volume it bootstrapped with +# $INITIALIZE_PENDING; the join runs dsreplication initialize from a peer while the marker +# is there and removes it once the data of the topology has arrived, so a marker that +# outlived a killed container still gets its initialize on the next start. The seed - only +# the first entry of REPLICATION_PEERS, and only once the retries are exhausted without +# finding a topology - removes the marker instead: its data is what the others initialize +# from. The residual risk is documented in the README: every server down at once and the +# first peer's volume lost means the first peer seeds empty. +# +# The health marker is written only once the join succeeded or the seed rule fired, so a +# container that never joined stays unhealthy - across restarts too, which the bootstrap-only +# replicate.sh could not say (a restart wrote the marker right after upgrade and a failed +# join turned into a healthy, unreplicated server). + +cd /opt/opendj || exit 1 +export PATH=/opt/opendj/bin:$PATH + +BASE_DN=${BASE_DN:-"dc=example,dc=com"} +ROOT_USER_DN=${ROOT_USER_DN:-"cn=Directory Manager"} +ROOT_PASSWORD=${ROOT_PASSWORD:-password} +ADMIN_PORT=${ADMIN_PORT:-4444} +REPLICATION_PORT=${REPLICATION_PORT:-8989} +MYHOSTNAME=${MYHOSTNAME:-$(hostname -f)} +BOOTSTRAP_COMPLETE=${BOOTSTRAP_COMPLETE:-/opt/opendj/.bootstrap-complete} +INITIALIZE_PENDING=${INITIALIZE_PENDING:-/opt/opendj/data/.replication-initialize-pending} + +REPLICATION_RETRY_COUNT=${REPLICATION_RETRY_COUNT:-30} +REPLICATION_RETRY_INTERVAL=${REPLICATION_RETRY_INTERVAL:-10} +# dsreplication can hang rather than fail (a peer that stops mid-operation), so no attempt +# runs without a bound +REPLICATION_ATTEMPT_TIMEOUT=${REPLICATION_ATTEMPT_TIMEOUT:-120} + +# Removing servers that left the topology needs the operator's word on who belongs to it: +# with only MASTER_SERVER set, a third server joined by hand would be "not in the list" and +# thrown out. So the cleanup runs only when REPLICATION_PEERS itself is set. +PEERS_ARE_EXPLICIT=${REPLICATION_PEERS:+yes} +REPLICATION_PEERS=${REPLICATION_PEERS:-$MASTER_SERVER} + +if [ -z "$REPLICATION_PEERS" ]; then + echo "join: neither REPLICATION_PEERS nor MASTER_SERVER is set, nothing to join" + exit 1 +fi + +IFS=',' read -r -a RAW_PEERS <<<"$REPLICATION_PEERS" +PEERS=() +for peer in "${RAW_PEERS[@]}"; do + peer=$(echo "$peer" | tr -d '[:space:]') + [ -n "$peer" ] && PEERS+=("$peer") +done +if [ "${#PEERS[@]}" -eq 0 ]; then + echo "join: REPLICATION_PEERS holds no peer" + exit 1 +fi + +# The tools read the root password from a file (#1084); the file lives on the tmpfs of +# /dev/shm where there is one, and run.sh removes what a killed join left behind, by the +# ADMIN_PORT in the name, on every start +PASSWORD_FILE=$(mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.XXXXXX" 2>/dev/null || mktemp) || exit 1 +trap 'rm -f "$PASSWORD_FILE"' EXIT +printf '%s\n' "$ROOT_PASSWORD" >"$PASSWORD_FILE" || exit 1 + +lower() { + printf '%s' "$1" | tr '[:upper:]' '[:lower:]' +} + +# DNS names compare case-insensitively, and a peer of a StatefulSet is the pod's hostname +# plus the headless service: self when the peer equals hostname -f, or its first label +# equals the first label of hostname -f +is_self() { + local peer host + peer=$(lower "$1") + host=$(lower "$MYHOSTNAME") + [ "$peer" = "$host" ] || [ "${peer%%.*}" = "${host%%.*}" ] +} + +# every LDAP operation goes through the administration connector, which always serves TLS; +# the root user may change its password after the bootstrap, so unlike the health check +# these tools run only while the join still has the bootstrap's ROOT_PASSWORD to work with +search() { + local host=$1 + shift + ldapsearch --noPropertiesFile --hostname "$host" --port "$ADMIN_PORT" --useSsl --trustAll \ + --bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" "$@" 2>/dev/null +} + +server_up() { + search localhost --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null +} + +# a member holds the replication domain for BASE_DN in its configuration and is registered +# in cn=admin data; both are made by one dsreplication enable, so requiring both keeps a +# half-done enable in the retry loop instead of declaring it a success +is_member() { + search localhost --baseDN "cn=config" --searchScope sub \ + "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN))" 1.1 | grep -q "^dn:" || return 1 + search localhost --baseDN "cn=Servers,cn=admin data" --searchScope one \ + "(hostname=$MYHOSTNAME)" 1.1 | grep -q "^dn:" +} + +enable_through() { + timeout "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication enable \ + --host1 "$1" --port1 "$ADMIN_PORT" --bindDN1 "$ROOT_USER_DN" \ + --bindPasswordFile1 "$PASSWORD_FILE" --replicationPort1 "$REPLICATION_PORT" \ + --host2 "$MYHOSTNAME" --port2 "$ADMIN_PORT" --bindDN2 "$ROOT_USER_DN" \ + --bindPasswordFile2 "$PASSWORD_FILE" --replicationPort2 "$REPLICATION_PORT" \ + --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \ + --baseDN "$BASE_DN" -X -n +} + +# the generation ID of the replication domain, from the domain's monitor entry (the +# connected-to attribute is what tells it from the entries of a replication server) +generation_id() { + search "$1" --baseDN "cn=monitor" --searchScope sub \ + "(&(domain-name=$BASE_DN)(connected-to=*))" generation-id \ + | awk '/^generation-id: /{print $2; exit}' +} + +initialize_from() { + timeout "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication initialize --baseDN "$BASE_DN" \ + --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \ + --hostSource "$1" --portSource "$ADMIN_PORT" \ + --hostDestination "$MYHOSTNAME" --portDestination "$ADMIN_PORT" -X -n +} + +# initialize while the marker of a bootstrapped, never initialized volume is there; without +# it only cross-check the generation IDs, so a volume that already carried data is never +# overwritten by a peer +ensure_initialized() { + local source=$1 peer local_id peer_id rc i + if [ ! -f "$INITIALIZE_PENDING" ]; then + if [ -n "$source" ]; then + local_id=$(generation_id localhost) + peer_id=$(generation_id "$source") + if [ -n "$local_id" ] && [ -n "$peer_id" ] && [ "$local_id" != "$peer_id" ]; then + echo "join: generation ID $local_id does not match $peer_id of $source; replication will not flow until this server or that one is initialized by hand" + fi + fi + return 0 + fi + if [ -z "$source" ]; then + for peer in "${PEERS[@]}"; do + is_self "$peer" && continue + if [ -n "$(generation_id "$peer")" ]; then + source=$peer + break + fi + done + source=${source:-} + fi + if [ -z "$source" ]; then + echo "join: no peer to initialize from" + return 1 + fi + for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do + echo "join: initializing from $source" + if initialize_from "$source"; then + rm -f "$INITIALIZE_PENDING" + local_id=$(generation_id localhost) + peer_id=$(generation_id "$source") + if [ -n "$local_id" ] && [ -n "$peer_id" ] && [ "$local_id" != "$peer_id" ]; then + echo "join: generation ID $local_id does not match $peer_id of $source after initialize" + fi + return 0 + fi + rc=$? + [ "$i" -eq "$REPLICATION_RETRY_COUNT" ] && return $rc + echo "join: initialize from $source exited with $rc, trying again in $REPLICATION_RETRY_INTERVAL s" + sleep "$REPLICATION_RETRY_INTERVAL" + done +} + +ldapmodify_local() { + ldapmodify --noPropertiesFile --hostname localhost --port "$ADMIN_PORT" --useSsl --trustAll \ + --bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" 2>/dev/null +} + +in_peers() { + local host peer + host=$(lower "${1%%.*}") + for peer in "${PEERS[@]}"; do + [ "$(lower "${peer%%.*}")" = "$host" ] && return 0 + done + return 1 +} + +# Removes every server that is registered in the topology but no longer listed in +# REPLICATION_PEERS: its entry and group membership in cn=admin data (which is replicated, +# so one survivor's delete reaches the others) and its values in this server's +# replication-server lists (which are configuration of this server alone, so every survivor +# prunes its own on its next start). dsreplication disable cannot do this for a dead server: +# it changes only the servers it can reach, and OpenDJ 4 has no cleanup subcommand. Every +# step tolerates losing the race to another survivor. +cleanup_departed() { + local dn host value domain + [ "$PEERS_ARE_EXPLICIT" = yes ] || return 0 + search localhost --baseDN "cn=Servers,cn=admin data" --searchScope one "(objectClass=*)" hostname \ + | awk '/^dn: /{dn=substr($0,5)} /^hostname: /{print dn "\t" $2}' \ + | while IFS=$'\t' read -r dn host; do + [ -z "$host" ] && continue + if ! in_peers "$host" && ! is_self "$host"; then + echo "join: removing departed server $host from cn=admin data" + printf 'dn: %s\nchangetype: delete\n' "$dn" | ldapmodify_local \ + || echo "join: could not remove $dn (already removed by another survivor?)" + printf 'dn: cn=all-servers,cn=Server Groups,cn=admin data\nchangetype: modify\ndelete: uniqueMember\nuniqueMember: %s\n' "${dn%%,cn=Servers,cn=admin data}" \ + | ldapmodify_local || true + fi + done + search localhost --baseDN "cn=config" --searchScope sub \ + "(objectClass=ds-cfg-replication-server)" ds-cfg-replication-server \ + | awk '/^ds-cfg-replication-server: /{print $2}' \ + | while read -r value; do + host=${value%:*} + if ! in_peers "$host" && ! is_self "$host"; then + echo "join: removing departed replication server $value from the replication server configuration" + dsconfig set-replication-server-prop --provider-name "Multimaster Synchronization" \ + --remove "replication-server:$value" \ + --hostname localhost --port "$ADMIN_PORT" --bindDN "$ROOT_USER_DN" \ + --bindPasswordFile "$PASSWORD_FILE" --trustAll --no-prompt || true + fi + done + domain=$(search localhost --baseDN "cn=config" --searchScope sub \ + "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN))" cn \ + | awk '/^cn: /{print substr($0,5); exit}') + [ -z "$domain" ] && return 0 + search localhost --baseDN "cn=config" --searchScope sub \ + "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN))" ds-cfg-replication-server \ + | awk '/^ds-cfg-replication-server: /{print $2}' \ + | while read -r value; do + host=${value%:*} + if ! in_peers "$host" && ! is_self "$host"; then + echo "join: removing departed replication server $value from the replication domain configuration" + dsconfig set-replication-domain-prop --provider-name "Multimaster Synchronization" \ + --domain-name "$domain" --remove "replication-server:$value" \ + --hostname localhost --port "$ADMIN_PORT" --bindDN "$ROOT_USER_DN" \ + --bindPasswordFile "$PASSWORD_FILE" --trustAll --no-prompt || true + fi + done +} + +joined() { + ensure_initialized "$1" || return 1 + cleanup_departed + touch "$BOOTSTRAP_COMPLETE" + echo "join: this server is a member of the replication topology, the health check may probe it" +} + +echo "join: waiting for the server on the administration connector" +for i in $(seq 1 150); do + server_up && break + if [ "$i" -eq 150 ]; then + echo "join: the server did not come up, giving up" + exit 1 + fi + sleep 2 +done + +if is_member; then + echo "join: already a member of the replication topology" + joined "" + exit +fi + +# a list without a single other server - MASTER_SERVER pointing at this server, the way the +# old replicate.sh master recognised itself - leaves nothing to retry against, so the seed +# rule below decides at once instead of sitting out the retries +OTHERS=no +for peer in "${PEERS[@]}"; do + is_self "$peer" || OTHERS=yes +done + +[ "$OTHERS" = yes ] && for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do + for peer in "${PEERS[@]}"; do + is_self "$peer" && continue + echo "join: enabling replication with $peer" + enable_through "$peer" + rc=$? + if is_member; then + echo "join: joined the replication topology through $peer" + joined "$peer" + exit + fi + echo "join: dsreplication enable with $peer exited with $rc and membership is not there" + done + if [ "$i" -lt "$REPLICATION_RETRY_COUNT" ]; then + echo "join: no peer joined this server yet, trying again in $REPLICATION_RETRY_INTERVAL s ($i of $REPLICATION_RETRY_COUNT)" + sleep "$REPLICATION_RETRY_INTERVAL" + fi +done + +# Only the first peer of the list may decide that there is no topology to join and that its +# own data seeds one; anyone else staying unhealthy is what surfaces a lost topology instead +# of forking it. The seed keeps its bootstrap data, so the initialize marker goes away. +if is_self "${PEERS[0]}"; then + echo "join: no topology found after $REPLICATION_RETRY_COUNT attempts, seeding it with this server's data" + rm -f "$INITIALIZE_PENDING" + touch "$BOOTSTRAP_COMPLETE" + exit 0 +fi + +echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this container will not report itself healthy" +exit 1 diff --git a/opendj-packages/opendj-docker/bootstrap/replicate.sh b/opendj-packages/opendj-docker/bootstrap/replicate.sh index 470e721b41..bc63f3e2f0 100755 --- a/opendj-packages/opendj-docker/bootstrap/replicate.sh +++ b/opendj-packages/opendj-docker/bootstrap/replicate.sh @@ -15,12 +15,18 @@ # Replicate to the master server hostname defined in $1 # If that server is ourself this is a no-op +# +# This is the one-shot bootstrap path of the srs, sdsr and rg replication types, kept as it +# is for compatibility; OPENDJ_REPLICATION_TYPE=simple is served by bootstrap/join.sh, which +# runs in the background next to the server on every start (#1086) # This is a bit kludgy. # The hostname has to be a fully resolvable DNS name in the cluster # If the service is called MYHOSTNAME=${MYHOSTNAME:-$(hostname -f)} +ADMIN_PORT=${ADMIN_PORT:-4444} +REPLICATION_PORT=${REPLICATION_PORT:-8989} export PATH=/opt/opendj/bin:$PATH echo "Setting up replication from $MYHOSTNAME to $MASTER_SERVER" @@ -77,43 +83,22 @@ echo "Will sleep for a bit to ensure master is up" sleep 5 -if [ "$OPENDJ_REPLICATION_TYPE" == "simple" ]; then - echo "Enabling Standard Replication..." - retry 8 /opt/opendj/bin/dsreplication \ - enable \ - --host1 $MASTER_SERVER \ - --port1 4444 \ - --bindDN1 "$ROOT_USER_DN" \ - --bindPasswordFile1 "$PASSWORD_FILE" --replicationPort1 8989 \ - --host2 $MYHOSTNAME --port2 4444 --bindDN2 "$ROOT_USER_DN" \ - --bindPasswordFile2 "$PASSWORD_FILE" --replicationPort2 8989 \ - --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \ - --baseDN "$BASE_DN" -X -n || exit - - echo "initializing replication" - - # replicating data in MASTER_SERVER to MYHOSTNAME: - retry any /opt/opendj/bin/dsreplication initialize --baseDN "$BASE_DN" \ - --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \ - --hostSource $MASTER_SERVER --portSource 4444 \ - --hostDestination $MYHOSTNAME --portDestination 4444 -X -n - -elif [ "$OPENDJ_REPLICATION_TYPE" == "srs" ]; then +if [ "$OPENDJ_REPLICATION_TYPE" == "srs" ]; then echo "Enabling Standalone Replication Servers..." retry 8 dsreplication enable \ --adminUID admin \ --adminPasswordFile "$PASSWORD_FILE" \ --baseDN "$BASE_DN" \ --host1 $MYHOSTNAME \ - --port1 4444 \ + --port1 "$ADMIN_PORT" \ --bindDN1 "$ROOT_USER_DN" \ --bindPasswordFile1 "$PASSWORD_FILE" \ --noReplicationServer1 \ --host2 $MASTER_SERVER \ - --port2 4444 \ + --port2 "$ADMIN_PORT" \ --bindDN2 "$ROOT_USER_DN" \ --bindPasswordFile2 "$PASSWORD_FILE" \ - --replicationPort2 8989 \ + --replicationPort2 "$REPLICATION_PORT" \ --onlyReplicationServer2 \ --trustAll \ --no-prompt || exit @@ -126,7 +111,7 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "srs" ]; then --adminPasswordFile "$PASSWORD_FILE" \ --baseDN "$BASE_DN" \ --hostname $MYHOSTNAME \ - --port 4444 \ + --port "$ADMIN_PORT" \ --trustAll \ --no-prompt @@ -138,11 +123,11 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "sdsr" ]; then --adminPasswordFile "$PASSWORD_FILE" \ --baseDN "$BASE_DN" \ --host1 $MASTER_SERVER \ - --port1 4444 \ + --port1 "$ADMIN_PORT" \ --bindDN1 "$ROOT_USER_DN" \ --bindPasswordFile1 "$PASSWORD_FILE" \ --host2 $MYHOSTNAME \ - --port2 4444 \ + --port2 "$ADMIN_PORT" \ --bindDN2 "$ROOT_USER_DN" \ --bindPasswordFile2 "$PASSWORD_FILE" \ --noReplicationServer2 \ @@ -157,9 +142,9 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "sdsr" ]; then --adminPasswordFile "$PASSWORD_FILE" \ --baseDN "$BASE_DN" \ --hostSource $MASTER_SERVER \ - --portSource 4444 \ + --portSource "$ADMIN_PORT" \ --hostDestination $MYHOSTNAME \ - --portDestination 4444 \ + --portDestination "$ADMIN_PORT" \ --trustAll \ --no-prompt @@ -168,7 +153,7 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "rg" ]; then dsconfig \ set-replication-domain-prop \ - --port 4444 \ + --port "$ADMIN_PORT" \ --hostname $MYHOSTNAME \ --bindDN "$ROOT_USER_DN" \ --bindPasswordFile "$PASSWORD_FILE" \ @@ -180,7 +165,7 @@ elif [ "$OPENDJ_REPLICATION_TYPE" == "rg" ]; then retry any dsconfig \ set-replication-server-prop \ - --port 4444 \ + --port "$ADMIN_PORT" \ --hostname $MASTER_SERVER \ --bindDN "$ROOT_USER_DN" \ --bindPasswordFile "$PASSWORD_FILE" \ diff --git a/opendj-packages/opendj-docker/run.sh b/opendj-packages/opendj-docker/run.sh index d7f19434e1..b294554b8e 100755 --- a/opendj-packages/opendj-docker/run.sh +++ b/opendj-packages/opendj-docker/run.sh @@ -42,9 +42,20 @@ rm -f "$BOOTSTRAP_COMPLETE" # network namespace. Containers in distinct network namespaces that share /dev/shm through # --ipc=host, and listen on the same ADMIN_PORT, are not told apart. /tmp belongs to this # container alone, so a password file left there is removed whatever its name. -rm -f /dev/shm/opendj-replicate."$ADMIN_PORT".* +rm -f /dev/shm/opendj-replicate."$ADMIN_PORT".* /dev/shm/opendj-join."$ADMIN_PORT".* rm -f /dev/shm/opendj-setup-password."$ADMIN_PORT".* /tmp/opendj-setup-password.* +# What the volume went through is recorded next to the instance: a volume this script +# bootstrapped carries the marker until the join has initialized it from the topology +# (see bootstrap/join.sh), however many restarts that takes +export INITIALIZE_PENDING=${INITIALIZE_PENDING:-/opt/opendj/data/.replication-initialize-pending} + +# The background join (bootstrap/join.sh) serves OPENDJ_REPLICATION_TYPE=simple on every +# start; srs, sdsr and rg keep the one-shot replicate.sh of the first bootstrap +join_requested() { + [ "$OPENDJ_REPLICATION_TYPE" = "simple" ] && [ -n "${REPLICATION_PEERS:-$MASTER_SERVER}" ] +} + # Keystores and truststores mounted as a volume (a Kubernetes Secret, say) are copied into # the instance on every start, not only on the one that bootstraps it: the instance lives on # a persistent volume, and a renewed certificate in the Secret has to reach it. @@ -122,7 +133,21 @@ if [ -d ./data/config ]; then # nothing is bootstrapped here, the instance is already there - but a half-migrated one # is not ready to serve either, so the marker follows the upgrade if sh ./upgrade -n; then - touch "$BOOTSTRAP_COMPLETE" + # A server whose replication domain is already configured is ready as soon as it + # serves: gating it on its peers would deadlock a whole-cluster restart under + # OrderedReady, where -0 would wait for peers the StatefulSet starts only once -0 is + # ready. Only a volume that was asked to join and never did waits for the join, so a + # join that failed before a restart no longer turns into a healthy, unreplicated + # server the way it did when the marker followed the upgrade alone. + if ! join_requested || grep -q "ds-cfg-replication-domain" ./data/config/config.ldif; then + touch "$BOOTSTRAP_COMPLETE" + else + echo "This instance never joined its replication topology, the join decides whether it is healthy" + fi + # the join also repairs membership that changed while no container ran, on every start + if join_requested; then + /opt/opendj/bootstrap/join.sh & + fi else echo "Upgrade failed, this container will not report itself healthy" fi @@ -146,7 +171,14 @@ if ! sh "${BOOTSTRAP}"; then fi # Check if OPENDJ_REPLICATION_TYPE var is set. If it is - replicate to that server -if [ -n "${MASTER_SERVER}" ] && [ -n "${OPENDJ_REPLICATION_TYPE}" ]; then +if join_requested; then + # the join runs in the background next to the server, below; a bootstrapped volume has + # entries whether or not it was asked for any, so it is marked to be initialized from + # the topology (see bootstrap/join.sh) + if [ "$BOOTSTRAPPED" = true ]; then + touch "$INITIALIZE_PENDING" + fi +elif [ -n "${MASTER_SERVER}" ] && [ -n "${OPENDJ_REPLICATION_TYPE}" ]; then if ! /opt/opendj/bootstrap/replicate.sh; then BOOTSTRAPPED=false echo "Replication setup failed, this container will not report itself healthy" @@ -170,10 +202,17 @@ if ! ./bin/stop-ds; then fi # Everything the instance was asked to be set up with - its backend, its base entry, its -# replication - is in place from here on, so the health check may start probing the server +# replication - is in place from here on, so the health check may start probing the server. +# With the background join, replication is the one thing still outstanding: the join writes +# the health marker once this server is a member of its topology, and not before. if [ "$BOOTSTRAPPED" = true ]; then - touch "$BOOTSTRAP_COMPLETE" - echo "The instance is bootstrapped, the health check may probe it" + if join_requested; then + /opt/opendj/bootstrap/join.sh & + echo "The instance is bootstrapped, joining the replication topology decides whether it is healthy" + else + touch "$BOOTSTRAP_COMPLETE" + echo "The instance is bootstrapped, the health check may probe it" + fi fi start_server From 0c93c12ca27ff10a56748d7922d599b12eee02e3 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Tue, 29 Sep 2026 15:27:05 +0300 Subject: [PATCH 2/5] [#1086] Publish whether a volume holds the topology's data, and clean every replication list Review round 1 of #1115: - join.sh: a failed dsreplication initialize is no longer taken for a success (the exit status was that of an if without else), and it is bounded by its own REPLICATION_INITIALIZE_TIMEOUT, none by default; REPLICATION_ATTEMPT_TIMEOUT bounds the enable only, and the bound sits on the dsreplication call, as timeout cannot run a shell function. - join.sh publishes pending/ready in the local entry cn=Docker Join,cn=config; a pending server enables and initializes only through a ready peer, the first peer seeds at once when every other one is pending and after the retries only while no peer that could hold the data answers. Fresh servers started together no longer take each other's bootstrap data for the topology's. - The cleanup prunes departed servers from every replication list (replication server, BASE_DN, cn=schema, cn=admin data), and the lists get every listed, registered peer they lack; dsconfig inside the read loops reads /dev/null. - Self-recognition: a name equal to hostname -f or cut from it at a dot, and the container's own addresses and their /etc/hosts names, compared whole; is_member checks every registered hostname; the generation ID comes from the domain's own monitor entry. - run.sh stays unhealthy on a restart while $INITIALIZE_PENDING is on the volume. - Dockerfile-alpine installs coreutils, whose timeout signals the whole process group. - CI: both image jobs run .github/scripts/docker-test-replication.sh, which adds the initialize pin, held unhealthy checks, the full-list scale-down check, three fresh servers started at once, a master named by its own address, and the replicate.sh pins on the /dev/shm glob and the retry on exit 8. - README documents all of the above. --- .github/scripts/docker-test-replication.sh | 360 +++++++++++ .github/workflows/build.yml | 203 +------ opendj-packages/opendj-docker/Dockerfile | 1 + .../opendj-docker/Dockerfile-alpine | 3 +- opendj-packages/opendj-docker/README.md | 71 ++- .../opendj-docker/bootstrap/join.sh | 568 +++++++++++++----- opendj-packages/opendj-docker/run.sh | 18 +- 7 files changed, 835 insertions(+), 389 deletions(-) create mode 100755 .github/scripts/docker-test-replication.sh diff --git a/.github/scripts/docker-test-replication.sh b/.github/scripts/docker-test-replication.sh new file mode 100755 index 0000000000..4c9a9dabac --- /dev/null +++ b/.github/scripts/docker-test-replication.sh @@ -0,0 +1,360 @@ +#!/usr/bin/env bash +# +# The contents of this file are subject to the terms of the Common Development and +# Distribution License (the License). You may not use this file except in compliance with the +# License. +# +# You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the +# specific language governing permission and limitations under the License. +# +# When distributing Covered Software, include this CDDL Header Notice in each file and include +# the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL +# Header, with the fields enclosed by brackets [] replaced by your own identifying +# information: "Portions copyright [year] [name of copyright owner]". +# +# Copyright 2026 3A Systems, LLC. + +# Tests the replication of the Docker image (#1086): the background join of +# OPENDJ_REPLICATION_TYPE=simple (bootstrap/join.sh) and the deprecated one-shot sdsr path of +# bootstrap/replicate.sh. Both image jobs of build.yml run it, each with its own image. +# +# Usage: docker-test-replication.sh + +set -eE -o pipefail + +IMAGE=${1:?usage: $0 } +NODES="dj-0 dj-1 dj-2 dj-x dj-p0 dj-p1 dj-p2 dj-ipm dj-ipr dj-sdsr dj-shm-probe dj-shm" +VOLUMES="vol-dj-0 vol-dj-1 vol-dj-2 vol-dj-x vol-dj-p0 vol-dj-p1 vol-dj-p2" +NETWORK=test_replication + +cleanup() { + docker rm -f $NODES >/dev/null 2>&1 || true + docker network rm $NETWORK >/dev/null 2>&1 || true + docker volume rm -f $VOLUMES >/dev/null 2>&1 || true +} +cleanup +trap 'code=$?; for c in $NODES; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR + +fail() { + echo "::error::$1" + false +} + +# grep reads the whole log rather than stopping at the first match (-q), so docker logs never +# writes into a closed pipe, which pipefail would report as a failure of the check +logs_have() { # + docker logs "$1" 2>&1 | grep -F -- "$2" >/dev/null +} + +logs_count() { # + docker logs "$1" 2>&1 | grep -cF -- "$2" || true +} + +wait_until() { # [...] + local deadline=$((SECONDS + $1)) what=$2 + shift 2 + until "$@"; do + [ "$SECONDS" -lt "$deadline" ] || fail "timed out waiting until $what" + sleep 2 + done +} + +health() { + docker inspect --format='{{.State.Health.Status}}' "$1" +} + +is_healthy() { + [ "$(health "$1")" = healthy ] +} + +wait_healthy() { + wait_until 480 "$1 is healthy" is_healthy "$1" +} + +# A container reports itself healthy only after a probe that finds the marker, and the +# probes run every 30 s: a single look right after a failure reads "starting" whatever the +# join did, so the container is watched over more than two probe cycles +stays_unhealthy() { # + local i + for i in $(seq 1 15); do + is_healthy "$1" && fail "$1 reports itself healthy although $2" + sleep 5 + done +} + +ldaps_has() { # + docker exec "$1" /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 1636 \ + --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll \ + --baseDN "$2" --searchScope base "(objectClass=*)" 1.1 >/dev/null 2>&1 +} + +wait_has() { # + wait_until 60 "$2 is on $1" ldaps_has "$1" "$2" +} + +admin_search() { # [...] + local container=$1 base=$2 scope=$3 filter=$4 + shift 4 + docker exec "$container" /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 4444 \ + --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \ + --baseDN "$base" --searchScope "$scope" "$filter" "$@" 2>/dev/null +} + +add_ou() { # + printf 'dn: ou=%s,dc=example,dc=com\nobjectClass: organizationalUnit\nou: %s\n' "$2" "$2" \ + | docker exec -i "$1" /opt/opendj/bin/ldapmodify --noPropertiesFile --hostname localhost --port 1636 \ + --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd >/dev/null +} + +# the replication server lists of a container: the one of its replication server and one per +# replication domain (BASE_DN, cn=schema, cn=admin data) +replication_lists() { + admin_search "$1" "cn=config" sub \ + "(|(objectClass=ds-cfg-replication-server)(objectClass=ds-cfg-replication-domain))" ds-cfg-replication-server +} + +lists_lack() { # + ! replication_lists "$1" | grep -F -- "$2" >/dev/null +} + +admin_data_lacks() { # + ! admin_search "$1" "cn=Servers,cn=admin data" one "(objectClass=*)" hostname | grep -F -- "$2" >/dev/null +} + +# the replication server of the container lists every one of the other names +server_lists() { # ... + local container=$1 list name + shift + list=$(admin_search "$container" "cn=config" sub "(objectClass=ds-cfg-replication-server)" ds-cfg-replication-server) + for name in "$@"; do + grep -F -- "ds-cfg-replication-server: $name:" <<<"$list" >/dev/null || return 1 + done +} + +# every tool reads the root password from a file (#1084, #1092); dsreplication run with -n +# prints no command line, so a password put back on one would pass every check below +rc=0 +docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword|rootUserPassword))([[:space:]=]|$)' \ + /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh /opt/opendj/bootstrap/join.sh || rc=$? +[ "$rc" -eq 1 ] || fail "a bootstrap script passes the root password on a command line, or grep could not read them" +# the password files go to /dev/shm, off the writable layer of the container, and the mktemp +# of the image puts them there +docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.' /opt/opendj/bootstrap/join.sh \ + || fail "join.sh no longer puts the password file on /dev/shm" +docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.' /opt/opendj/bootstrap/replicate.sh \ + || fail "replicate.sh no longer puts the password file on /dev/shm" +docker run --rm --entrypoint sh "$IMAGE" -c 'f=$(mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.XXXXXX") && rm -f "$f" && case $f in /dev/shm/opendj-join.4444.*) ;; *) exit 1;; esac' \ + || fail "mktemp in the image does not create the password file on /dev/shm" +# a bounded dsreplication has to take its JVM down with it: the timeout of BusyBox signals +# only the shell script that starts java, that of coreutils the whole process group +docker run --rm --entrypoint sh "$IMAGE" -c 'timeout --version 2>&1 | grep -q "GNU coreutils"' \ + || fail "the image has no timeout of coreutils" + +# a password with a space in it reaches every tool as one value +ROOT_PASSWORD='replication secret' +# a subnet of its own, so that a container can be given a known address +MASTER_ADDRESS=172.30.99.50 +docker network create --subnet 172.30.99.0/24 $NETWORK >/dev/null + +# small retry values keep the seed decision and the failed-join case quick; the volume holds +# the instance so that a container can be replaced with or without its data surviving +start_node() { # [...] + local name=$1 peers=$2 + shift 2 + docker volume create "vol-$name" >/dev/null + docker run -d --memory="512m" --network $NETWORK --name "$name" --hostname "$name" \ + -v "vol-$name:/opt/opendj/data" \ + -e ROOT_PASSWORD="$ROOT_PASSWORD" -e ADD_BASE_ENTRY="--addBaseEntry" \ + -e OPENDJ_REPLICATION_TYPE=simple -e REPLICATION_PEERS="$peers" \ + -e REPLICATION_RETRY_COUNT=10 -e REPLICATION_RETRY_INTERVAL=3 -e REPLICATION_ATTEMPT_TIMEOUT=90 \ + "$@" "$IMAGE" >/dev/null +} + +# the first peer of REPLICATION_PEERS, and only it, seeds a topology its retries could not +# find; it imports entries no other server ever gets but from an initialize +start_node dj-0 dj-0,dj-1 -e SAMPLE_DATA=10 +wait_healthy dj-0 +logs_have dj-0 "seeding it with this server's data" || fail "dj-0 did not seed the topology" + +# a joining server tries again while its peer is unreachable +docker network disconnect $NETWORK dj-0 +start_node dj-1 dj-0,dj-1 +wait_until 300 "dj-1 tries again" logs_have dj-1 "trying again in" +docker network connect --alias dj-0 $NETWORK dj-0 +wait_healthy dj-1 +logs_have dj-1 "joined the replication topology through dj-0" || fail "dj-1 did not join through dj-0" +# the bootstrapped volume of dj-1 was initialized from the topology although its BASE_DN held +# the imported base entry - entries in BASE_DN say nothing about who holds the data. The +# sample entries reached dj-0 by import-ldif, never through the changelog, so only an +# initialize carries them +logs_have dj-1 "initializing from dj-0" || fail "dj-1 did not initialize from the topology" +ldaps_has dj-1 "uid=user.0,ou=People,dc=example,dc=com" || fail "dj-1 lacks the entries only an initialize from dj-0 brings" + +# a change made on the seed reaches the replica +add_ou dj-0 replicated +wait_has dj-1 "ou=replicated,dc=example,dc=com" + +# a member is ready again right after a restart, without waiting for its peers, and +# replication still flows +docker restart dj-0 dj-1 >/dev/null +wait_healthy dj-0 +wait_healthy dj-1 +add_ou dj-1 replicated2 +wait_has dj-0 "ou=replicated2,dc=example,dc=com" + +# a Kubernetes pod keeps its /dev/shm across container restarts, shared by all its +# containers: a starting container removes the password files a killed join or replicate.sh +# left there - only those of its own ADMIN_PORT, the files of the other containers of the pod +# are not its to remove +docker run -d --memory="64m" --ipc=shareable --name dj-shm --entrypoint sleep "$IMAGE" 600 >/dev/null +docker exec dj-shm sh -c ': >/dev/shm/opendj-join.4444.killed && : >/dev/shm/opendj-replicate.4444.killed && : >/dev/shm/opendj-join.5444.other' +docker run -d --memory="512m" --network $NETWORK --ipc=container:dj-shm --name dj-shm-probe --hostname dj-shm-probe \ + -e ROOT_PASSWORD="$ROOT_PASSWORD" "$IMAGE" >/dev/null +# the planted files only: the probe's own bootstrap keeps a password file of its ADMIN_PORT +# there while it runs +shm_cleared() { ! docker exec dj-shm sh -c 'test -e /dev/shm/opendj-join.4444.killed || test -e /dev/shm/opendj-replicate.4444.killed'; } +wait_until 60 "run.sh removes the password files of its ADMIN_PORT" shm_cleared +docker exec dj-shm test -e /dev/shm/opendj-join.5444.other || fail "run.sh removed the password file of another container" +docker rm -f dj-shm-probe dj-shm >/dev/null + +# a join that cannot succeed keeps the container from reporting itself healthy - across a +# restart too, where the health marker used to follow the upgrade and a failed join turned +# into a healthy, unreplicated server. dj-x may not seed: it is not the first peer of its list +start_node dj-x dj-absent,dj-x +wait_until 300 "the join of dj-x gives up" logs_have dj-x "could not join the replication topology" +stays_unhealthy dj-x "its join failed" +docker restart dj-x >/dev/null +wait_until 300 "the restarted dj-x waits for its join" logs_have dj-x "never joined its replication topology" +second_failure() { [ "$(logs_count dj-x "could not join the replication topology")" -ge 2 ]; } +wait_until 300 "the second join of dj-x gives up" second_failure +stays_unhealthy dj-x "it never joined" +docker rm -f dj-x >/dev/null +docker volume rm vol-dj-x >/dev/null + +# the seed lost its volume: behind the same name, a fresh dj-0 finds the topology at dj-1 and +# takes its data from it instead of seeding an empty one next to it +docker rm -f dj-0 >/dev/null +docker volume rm vol-dj-0 >/dev/null +start_node dj-0 dj-0,dj-1 +wait_healthy dj-0 +logs_have dj-0 "initializing from dj-1" || fail "the reborn dj-0 did not initialize from dj-1" +if logs_have dj-0 "seeding it with this server's data"; then fail "the reborn dj-0 seeded a topology although dj-1 held it"; fi +ldaps_has dj-0 "ou=replicated,dc=example,dc=com" || fail "the reborn dj-0 lacks the data of the topology" +ldaps_has dj-0 "uid=user.0,ou=People,dc=example,dc=com" || fail "the reborn dj-0 lacks the entries of the seed" + +# scale up to three - the peer list is configuration, so the running servers are replaced +# with the longer list before the third one starts, as a rolling update would +docker rm -f dj-0 dj-1 >/dev/null +start_node dj-0 dj-0,dj-1,dj-2 +start_node dj-1 dj-0,dj-1,dj-2 +wait_healthy dj-0 +wait_healthy dj-1 +start_node dj-2 dj-0,dj-1,dj-2 +wait_healthy dj-2 +ldaps_has dj-2 "ou=replicated,dc=example,dc=com" || fail "dj-2 lacks the data of the topology" + +# scale down to two: the survivors, restarted with the shorter list, remove dj-2 from +# cn=admin data and from every replication server list they hold - that of BASE_DN, and +# those of cn=schema and cn=admin data that dsreplication enable configures next to it. +# dsreplication disable cannot, dj-2 being already gone +docker rm -f dj-2 >/dev/null +docker volume rm vol-dj-2 >/dev/null +docker rm -f dj-0 dj-1 >/dev/null +start_node dj-0 dj-0,dj-1 +start_node dj-1 dj-0,dj-1 +wait_healthy dj-0 +wait_healthy dj-1 +# whichever survivor's join ran first deletes the cn=admin data entry, the delete replicates +# to the other; each survivor prunes its own replication server lists +removed_dj2() { logs_have dj-0 "removing departed server dj-2" || logs_have dj-1 "removing departed server dj-2"; } +wait_until 180 "a survivor removes dj-2" removed_dj2 +for c in dj-0 dj-1; do + wait_until 120 "dj-2 leaves cn=admin data of $c" admin_data_lacks $c dj-2 + wait_until 120 "dj-2 leaves the replication server lists of $c" lists_lack $c dj-2 +done +# replication between the survivors is intact +add_ou dj-0 replicated3 +wait_has dj-1 "ou=replicated3,dc=example,dc=com" + +# the deprecated one-shot sdsr path of replicate.sh still bootstraps a replica. On a first +# start a server stops the server its bootstrap started and starts it again, and a replica +# can reach it in between: replicate.sh tries a dsreplication enable that could not connect +# again (exit 8). dj-0 is taken off the network while the replica sleeps before its first try, +# and comes back only once the replica has said it will try again +docker run -d --memory="512m" --network $NETWORK --name dj-sdsr --hostname dj-sdsr \ + -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-0 -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" >/dev/null +wait_until 300 "dj-sdsr sleeps before its first try" logs_have dj-sdsr "Will sleep for a bit" +docker network disconnect $NETWORK dj-0 +wait_until 120 "dj-sdsr tries again" logs_have dj-sdsr "exited with 8, trying again" +docker network connect --alias dj-0 $NETWORK dj-0 +wait_healthy dj-sdsr +wait_has dj-sdsr "ou=replicated3,dc=example,dc=com" +# replicate.sh tries dsreplication enable again only when it exits 8, and a failed enable +# ends it: run once more on the replica for a base DN neither server holds, the enable exits +# 5 (REPLICATION_CANNOT_BE_ENABLED_ON_BASEDN) and nothing is tried again or initialized +rc=0 +out=$(docker exec -e BASE_DN=dc=absent,dc=com -e ROOT_USER_DN="cn=Directory Manager" dj-sdsr \ + timeout 90 /opt/opendj/bootstrap/replicate.sh 2>&1) || rc=$? +if [ "$rc" -ne 5 ] || grep -E "trying again|initializing replication" <<<"$out" >/dev/null; then + echo "$out" + fail "replicate.sh for a base DN nobody holds exited with $rc, not with the 5 of its dsreplication enable, or went on after it" +fi + +# the root password shows in no container log, and the files the tools read it from are gone +for c in dj-0 dj-1 dj-sdsr; do + if docker logs $c 2>&1 | grep -F -- "$ROOT_PASSWORD"; then fail "the root password is in the log of $c"; fi + left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm || true) + [ -z "$left" ] || fail "the root password is left in $left of $c" +done +docker rm -f dj-0 dj-1 dj-sdsr >/dev/null +docker volume rm vol-dj-0 vol-dj-1 >/dev/null + +# MASTER_SERVER may name the master by its address, as the grep of /etc/hosts in the base +# replicate.sh allowed: a master given its own address recognises itself and seeds at once, +# and a replica joins and initializes through that address +for n in dj-ipm dj-ipr; do + if [ $n = dj-ipm ]; then extra="--ip $MASTER_ADDRESS -e SAMPLE_DATA=10"; else extra=; fi + docker run -d --memory="512m" --network $NETWORK --name $n --hostname $n $extra \ + -e ROOT_PASSWORD="$ROOT_PASSWORD" -e ADD_BASE_ENTRY="--addBaseEntry" \ + -e OPENDJ_REPLICATION_TYPE=simple -e MASTER_SERVER=$MASTER_ADDRESS \ + -e REPLICATION_RETRY_COUNT=10 -e REPLICATION_RETRY_INTERVAL=3 "$IMAGE" >/dev/null + wait_healthy $n +done +logs_have dj-ipm "seeding it with this server's data" || fail "dj-ipm did not recognise itself by its address" +logs_have dj-ipr "initializing from $MASTER_ADDRESS" || fail "dj-ipr did not initialize from the master's address" +ldaps_has dj-ipr "uid=user.0,ou=People,dc=example,dc=com" || fail "dj-ipr lacks the entries of the master" +docker rm -f dj-ipm dj-ipr >/dev/null + +# three fresh servers started together, as Compose or a StatefulSet with +# podManagementPolicy: Parallel start them: none of them takes the bootstrap data of another +# fresh one for the topology's. The first peer seeds, the others initialize from it - its +# sample entries reach them only that way - and the replication servers know each other +# although the joins ran at the same time +for n in dj-p0 dj-p1 dj-p2; do + if [ $n = dj-p0 ]; then extra="-e SAMPLE_DATA=10"; else extra=; fi + start_node $n dj-p0,dj-p1,dj-p2 -e REPLICATION_RETRY_COUNT=20 $extra +done +for n in dj-p0 dj-p1 dj-p2; do + wait_healthy $n +done +logs_have dj-p0 "seeding it with this server's data" || fail "dj-p0 did not seed the topology" +for n in dj-p1 dj-p2; do + if logs_have $n "seeding it with this server's data"; then fail "$n seeded a topology although it is not the first peer"; fi + logs_have $n "initializing from dj-p0" || fail "$n did not initialize from dj-p0" + if logs_have $n "initializing from dj-p1" || logs_have $n "initializing from dj-p2"; then + fail "$n initialized from a server that holds only bootstrap data" + fi + ldaps_has $n "uid=user.0,ou=People,dc=example,dc=com" || fail "$n lacks the entries of the seed" +done +wait_until 120 "the replication server of dj-p0 knows dj-p1 and dj-p2" server_lists dj-p0 dj-p1 dj-p2 +wait_until 120 "the replication server of dj-p1 knows dj-p0 and dj-p2" server_lists dj-p1 dj-p0 dj-p2 +wait_until 120 "the replication server of dj-p2 knows dj-p0 and dj-p1" server_lists dj-p2 dj-p0 dj-p1 +add_ou dj-p1 parallel +wait_has dj-p0 "ou=parallel,dc=example,dc=com" +wait_has dj-p2 "ou=parallel,dc=example,dc=com" +for c in dj-p0 dj-p1 dj-p2; do + if docker logs $c 2>&1 | grep -F -- "$ROOT_PASSWORD"; then fail "the root password is in the log of $c"; fi +done + +cleanup +echo "Docker replication test passed" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 544caae771..0328313eae 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -579,136 +579,10 @@ jobs: docker exec test_uid 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1' docker kill test_uid - name: Docker test replication + # the background join of OPENDJ_REPLICATION_TYPE=simple and the one-shot sdsr path, the same + # scenarios for both images shell: bash - run: | - set -E - IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }} - NODES="dj-0 dj-1 dj-2 dj-x dj-sdsr dj-shm-probe dj-shm" - VOLUMES="vol-dj-0 vol-dj-1 vol-dj-2 vol-dj-x" - cleanup() { docker rm -f $NODES >/dev/null 2>&1 || true; docker network rm test_replication >/dev/null 2>&1 || true; docker volume rm -f $VOLUMES >/dev/null 2>&1 || true; } - cleanup - trap 'code=$?; for c in $NODES; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR - # every tool reads the root password from a file (#1084, #1092); dsreplication run with -n prints - # no command line, so a password put back on one would pass every check below - rc=0; docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword|rootUserPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh /opt/opendj/bootstrap/join.sh || rc=$? - if [ $rc -ne 1 ]; then echo "::error::a bootstrap script passes the root password on a command line, or grep could not read them"; false; fi - # the password file goes to /dev/shm, off the writable layer of the container, and the mktemp of the image puts it there - docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.' /opt/opendj/bootstrap/join.sh || { echo "::error::join.sh no longer puts the password file on /dev/shm"; false; } - docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.' /opt/opendj/bootstrap/replicate.sh || { echo "::error::replicate.sh no longer puts the password file on /dev/shm"; false; } - docker run --rm --entrypoint sh "$IMAGE" -c 'f=$(mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.XXXXXX") && rm -f "$f" && case $f in /dev/shm/opendj-join.4444.*) ;; *) exit 1;; esac' || { echo "::error::mktemp in the image does not create the password file on /dev/shm"; false; } - # a password with a space in it reaches every tool as one value - ROOT_PASSWORD='replication secret' - docker network create test_replication - # small retry values keep the seed decision and the failed-join case quick; the volume holds - # the instance so that a container can be replaced with or without its data surviving - start_node() { # - docker volume create "vol-$1" >/dev/null - docker run -d --memory="512m" --network test_replication --name "$1" --hostname "$1" \ - -v "vol-$1:/opt/opendj/data" \ - -e ROOT_PASSWORD="$ROOT_PASSWORD" -e ADD_BASE_ENTRY="--addBaseEntry" \ - -e OPENDJ_REPLICATION_TYPE=simple -e REPLICATION_PEERS="$2" \ - -e REPLICATION_RETRY_COUNT=10 -e REPLICATION_RETRY_INTERVAL=3 -e REPLICATION_ATTEMPT_TIMEOUT=90 \ - "$IMAGE" - } - wait_healthy() { timeout 8m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" "$0" | grep -q \"healthy\"; do sleep 5; done' "$1"; } - # the first peer of REPLICATION_PEERS, and only it, seeds a topology its retries could not find - start_node dj-0 dj-0,dj-1 - wait_healthy dj-0 - docker logs dj-0 2>&1 | grep -q "seeding it with this server's data" || { echo "::error::dj-0 did not seed the topology"; false; } - # a joining server tries again while its peer is unreachable, and decides membership from what - # is there, not from the exit code of dsreplication enable - docker network disconnect test_replication dj-0 - start_node dj-1 dj-0,dj-1 - timeout 5m bash -c 'until docker logs dj-1 2>&1 | grep -q "trying again in"; do sleep 2; done' - docker network connect --alias dj-0 test_replication dj-0 - wait_healthy dj-1 - docker logs dj-1 2>&1 | grep -q "joined the replication topology through dj-0" || { echo "::error::dj-1 did not join through dj-0"; false; } - # the bootstrapped volume of dj-1 was initialized from the topology although its BASE_DN - # held the imported base entry - entries in BASE_DN say nothing about who holds the data - docker logs dj-1 2>&1 | grep -q "initializing from dj-0" || { echo "::error::dj-1 did not initialize from the topology"; false; } - # a change made on the seed reaches the replica - printf 'dn: ou=replicated,dc=example,dc=com\nobjectClass: organizationalUnit\nou: replicated\n' | docker exec -i dj-0 /opt/opendj/bin/ldapmodify --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd - timeout 1m bash -c 'until docker exec dj-1 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" - # a member is ready again right after a restart, without waiting for its peers, and - # replication still flows - docker restart dj-0 dj-1 - wait_healthy dj-0 - wait_healthy dj-1 - printf 'dn: ou=replicated2,dc=example,dc=com\nobjectClass: organizationalUnit\nou: replicated2\n' | docker exec -i dj-1 /opt/opendj/bin/ldapmodify --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd - timeout 1m bash -c 'until docker exec dj-0 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated2,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" - # a Kubernetes pod keeps its /dev/shm across container restarts, shared by all its containers: - # a starting container removes the join password files a killed join left there - only those of - # its own ADMIN_PORT, the files of the other containers of the pod are not its to remove - docker run -d --memory="64m" --ipc=shareable --name dj-shm --entrypoint sleep "$IMAGE" 600 - docker exec dj-shm sh -c ': >/dev/shm/opendj-join.4444.killed && : >/dev/shm/opendj-join.5444.other' - docker run -d --memory="512m" --network test_replication --ipc=container:dj-shm --name dj-shm-probe --hostname dj-shm-probe -e ROOT_PASSWORD="$ROOT_PASSWORD" "$IMAGE" - timeout 1m bash -c 'while docker exec dj-shm test -e /dev/shm/opendj-join.4444.killed; do sleep 2; done' - docker exec dj-shm test -e /dev/shm/opendj-join.5444.other || { echo "::error::run.sh removed the join password file of another container"; false; } - docker rm -f dj-shm-probe dj-shm - # a join that cannot succeed keeps the container from reporting itself healthy - across a - # restart too, where the health marker used to follow the upgrade and a failed join turned - # into a healthy, unreplicated server. dj-x may not seed: it is not the first peer of its list - start_node dj-x dj-absent,dj-x - timeout 5m bash -c 'until docker logs dj-x 2>&1 | grep -q "could not join the replication topology"; do sleep 5; done' - [ "$(docker inspect --format='{{.State.Health.Status}}' dj-x)" != healthy ] || { echo "::error::dj-x reports itself healthy although its join failed"; false; } - docker restart dj-x - timeout 5m bash -c 'until docker logs dj-x 2>&1 | grep -q "never joined its replication topology"; do sleep 2; done' - timeout 5m bash -c 'until [ "$(docker logs dj-x 2>&1 | grep -c "could not join the replication topology")" -ge 2 ]; do sleep 5; done' - [ "$(docker inspect --format='{{.State.Health.Status}}' dj-x)" != healthy ] || { echo "::error::a restart turned the never-joined dj-x healthy"; false; } - docker rm -f dj-x - docker volume rm vol-dj-x - # the seed lost its volume: behind the same name, a fresh dj-0 finds the topology at dj-1 and - # takes its data from it instead of seeding an empty one next to it - docker rm -f dj-0 - docker volume rm vol-dj-0 - start_node dj-0 dj-0,dj-1 - wait_healthy dj-0 - docker logs dj-0 2>&1 | grep -q "initializing from dj-1" || { echo "::error::the reborn dj-0 did not initialize from dj-1"; false; } - if docker logs dj-0 2>&1 | grep -q "seeding it with this server's data"; then echo "::error::the reborn dj-0 seeded a topology although dj-1 held it"; false; fi - docker exec dj-0 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 - # scale up to three - the peer list is configuration, so the running servers are replaced with - # the longer list before the third one starts, as a rolling update would - docker rm -f dj-0 dj-1 - start_node dj-0 dj-0,dj-1,dj-2 - start_node dj-1 dj-0,dj-1,dj-2 - wait_healthy dj-0 - wait_healthy dj-1 - start_node dj-2 dj-0,dj-1,dj-2 - wait_healthy dj-2 - docker exec dj-2 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 - # scale down to two: the survivors, restarted with the shorter list, remove dj-2 from - # cn=admin data and from their replication server lists - dsreplication disable cannot, - # dj-2 being already gone - docker rm -f dj-2 - docker volume rm vol-dj-2 - docker rm -f dj-0 dj-1 - start_node dj-0 dj-0,dj-1 - start_node dj-1 dj-0,dj-1 - wait_healthy dj-0 - wait_healthy dj-1 - # whichever survivor's join ran first deletes the cn=admin data entry, the delete replicates - # to the other; each survivor prunes its own replication server lists - timeout 3m bash -c 'until docker logs dj-0 2>&1 | grep -q "removing departed server dj-2" || docker logs dj-1 2>&1 | grep -q "removing departed server dj-2"; do sleep 5; done' - for c in dj-0 dj-1; do - timeout 2m bash -c 'while docker exec '"$c"' /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 4444 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$0" --baseDN "cn=Servers,cn=admin data" --searchScope one "(objectClass=*)" hostname | grep -q dj-2; do sleep 5; done' "$ROOT_PASSWORD" - timeout 2m bash -c 'while docker exec '"$c"' /opt/opendj/bin/ldapsearch --noPropertiesFile --hostname localhost --port 4444 --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$0" --baseDN "cn=config" --searchScope sub "(|(objectClass=ds-cfg-replication-server)(objectClass=ds-cfg-replication-domain))" ds-cfg-replication-server | grep -q dj-2; do sleep 5; done' "$ROOT_PASSWORD" - done - # replication between the survivors is intact - printf 'dn: ou=replicated3,dc=example,dc=com\nobjectClass: organizationalUnit\nou: replicated3\n' | docker exec -i dj-0 /opt/opendj/bin/ldapmodify --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd - timeout 1m bash -c 'until docker exec dj-1 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated3,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" - # the deprecated one-shot sdsr path of replicate.sh still bootstraps a replica - docker run --rm -it -d --memory="512m" --network test_replication --name dj-sdsr --hostname dj-sdsr -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-0 -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" - wait_healthy dj-sdsr - timeout 1m bash -c 'until docker exec dj-sdsr /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated3,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" - # the root password shows in no container log, and the files the tools read it from are gone - for c in dj-0 dj-1 dj-sdsr; do - if docker logs $c 2>&1 | grep -F "$ROOT_PASSWORD"; then echo "::error::The root password is in the log of $c"; false; fi - done - for c in dj-0 dj-1 dj-sdsr; do - left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm || true) - if [ -n "$left" ]; then echo "::error::The root password is left in $left of $c"; false; fi - done - cleanup + run: .github/scripts/docker-test-replication.sh "localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}" - name: Docker test secret volume # a keystore mounted at SECRET_VOLUME is what LDAPS serves from the first start on, a # renewed one - a new password included - is copied while the server runs and served @@ -1086,75 +960,10 @@ jobs: docker exec test_uid 'sh' '-c' '/opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword password --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1' docker kill test_uid - name: Docker test replication + # the background join of OPENDJ_REPLICATION_TYPE=simple and the one-shot sdsr path, the same + # scenarios for both images shell: bash - run: | - IMAGE=localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine - REPLICAS="test_replica test_replica_sdsr" - cleanup() { docker rm -f test_master $REPLICAS >/dev/null 2>&1 || true; docker network rm test_replication >/dev/null 2>&1 || true; } - cleanup - trap 'code=$?; for c in test_master $REPLICAS; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR - # every tool reads the root password from a file (#1084, #1092); dsreplication run with -n prints - # no command line, so a password put back on one would pass every check below - rc=0; docker run --rm --entrypoint grep "$IMAGE" -nE -- '(^|[[:space:]])(-w|--(bindPassword[12]?|adminPassword|rootUserPassword))([[:space:]=]|$)' /opt/opendj/bootstrap/setup.sh /opt/opendj/bootstrap/replicate.sh || rc=$? - if [ $rc -ne 1 ]; then echo "::error::setup.sh or replicate.sh passes the root password on a command line, or grep could not read them"; false; fi - # the password file goes to /dev/shm, off the writable layer of the container, and the mktemp of the image puts it there - docker run --rm --entrypoint grep "$IMAGE" -qF -- 'mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.' /opt/opendj/bootstrap/replicate.sh || { echo "::error::replicate.sh no longer puts the password file on /dev/shm"; false; } - docker run --rm --entrypoint sh "$IMAGE" -c 'f=$(mktemp -p /dev/shm "opendj-replicate.$ADMIN_PORT.XXXXXX") && rm -f "$f" && case $f in /dev/shm/opendj-replicate.4444.*) ;; *) exit 1;; esac' || { echo "::error::mktemp in the image does not create the password file on /dev/shm"; false; } - # a password with a space in it reaches every tool as one value - ROOT_PASSWORD='replication secret' - docker network create test_replication - docker run --rm -it -d --memory="1g" --network test_replication --ipc=shareable --name=test_master --hostname=dj-master -e ADD_BASE_ENTRY="--addBaseEntry" -e ROOT_PASSWORD="$ROOT_PASSWORD" "$IMAGE" - timeout 3m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_master | grep -q \"healthy\"; do sleep 10; done' - # a replica reports itself healthy only once replicate.sh has succeeded; the sdsr replica joins after - # the simple one, as two dsreplication enable at once would both rewrite the admin data of the master - # a Kubernetes pod keeps its /dev/shm across container restarts: the replica shares the /dev/shm of the master, where - # a password file waits as a killed replicate.sh would have left it, and its run.sh has to remove it (checked below); - # the file of another container of the pod, which listens on another admin port, has to be kept - docker exec test_master sh -c 'printf "%s\n" "$ROOT_PASSWORD" >/dev/shm/opendj-replicate.4444.killed' - docker exec test_master sh -c ': >/dev/shm/opendj-replicate.5444.other' - docker run --rm -it -d --memory="1g" --network test_replication --ipc=container:test_master --name=test_replica --hostname=dj-replica -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=simple "$IMAGE" - # on a first start the master stops the server its bootstrap started and starts it again, and a replica - # started together with it can reach it in between: replicate.sh tries a dsreplication that could not - # connect again. The master is taken off the network while the replica sleeps before its first try, and - # comes back only once the replica has said it will try again - timeout 5m bash -c 'until docker logs test_replica 2>&1 | grep -q "Will sleep for a bit"; do sleep 0.2; done' - docker network disconnect test_replication test_master - timeout 2m bash -c 'until docker logs test_replica 2>&1 | grep -q "exited with 8, trying again"; do sleep 1; done' - docker network connect --alias dj-master test_replication test_master - timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica | grep -q \"healthy\"; do sleep 10; done' - docker run --rm -it -d --memory="1g" --network test_replication --name=test_replica_sdsr --hostname=dj-replica-sdsr -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-master -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" - # the same for the sdsr replica, whose dsreplication enable is another command of replicate.sh - timeout 5m bash -c 'until docker logs test_replica_sdsr 2>&1 | grep -q "Will sleep for a bit"; do sleep 0.2; done' - docker network disconnect test_replication test_master - timeout 2m bash -c 'until docker logs test_replica_sdsr 2>&1 | grep -q "exited with 8, trying again"; do sleep 1; done' - docker network connect --alias dj-master test_replication test_master - timeout 5m bash -c 'until docker inspect --format="{{json .State.Health.Status}}" test_replica_sdsr | grep -q \"healthy\"; do sleep 10; done' - # the replicas were initialized from the master, and a change made on the master reaches them - for c in $REPLICAS; do - docker exec $c /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --baseDN "dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1 - done - printf 'dn: ou=replicated,dc=example,dc=com\nobjectClass: organizationalUnit\nou: replicated\n' | docker exec -i test_master /opt/opendj/bin/ldapmodify --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" --useSsl --trustAll --defaultAdd - for c in $REPLICAS; do - timeout 1m bash -c 'until docker exec $1 /opt/opendj/bin/ldapsearch --hostname localhost --port 1636 --bindDN "cn=Directory Manager" --bindPassword "$0" --useSsl --trustAll --baseDN "ou=replicated,dc=example,dc=com" --searchScope base "(objectClass=*)" 1.1; do sleep 5; done' "$ROOT_PASSWORD" $c - done - # replicate.sh tries dsreplication enable again only when it exits 8, and a failed enable ends it: run once more - # on the replica, the enable of a base DN already replicated exits 5 and nothing is tried again or initialized - rc=0; out=$(docker exec -e BASE_DN=dc=example,dc=com -e ROOT_USER_DN="cn=Directory Manager" test_replica timeout 90 /opt/opendj/bootstrap/replicate.sh 2>&1) || rc=$? - if [ $rc -ne 5 ] || grep -qE "trying again|initializing replication" <<<"$out"; then - echo "$out"; echo "::error::a second replicate.sh exited with $rc, not with the 5 of its dsreplication enable, or went on after it"; false - fi - # the root password shows in no container log, and the files setup.sh and replicate.sh passed it in are gone (#1084, #1092) - for c in test_master $REPLICAS; do - if docker logs $c 2>&1 | grep -F "$ROOT_PASSWORD"; then echo "::error::The root password is in the log of $c"; false; fi - done - for c in test_master $REPLICAS; do - # a JVM keeps its command line in /tmp/hsperfdata_* while it runs; the HEALTHCHECK no longer binds as root (#1092), - # so no process left running has the root password on it - left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm || true) - if [ -n "$left" ]; then echo "::error::The root password is left in $left of $c"; false; fi - done - docker exec test_replica test -e /dev/shm/opendj-replicate.5444.other || { echo "::error::run.sh of test_replica removed the password file of another container"; false; } - cleanup + run: .github/scripts/docker-test-replication.sh "localhost:5000/${GITHUB_REPOSITORY,,}:${{ env.release_version }}-alpine" - name: Docker test secret volume # a keystore mounted at SECRET_VOLUME is what LDAPS serves from the first start on, a # renewed one - a new password included - is copied while the server runs and served diff --git a/opendj-packages/opendj-docker/Dockerfile b/opendj-packages/opendj-docker/Dockerfile index 05188d5c65..0ebaf0a5b9 100644 --- a/opendj-packages/opendj-docker/Dockerfile +++ b/opendj-packages/opendj-docker/Dockerfile @@ -36,6 +36,7 @@ ENV OPENDJ_SSL_OPTIONS="--generateSelfSignedCertificate" #ENV REPLICATION_RETRY_COUNT=30 #ENV REPLICATION_RETRY_INTERVAL=10 #ENV REPLICATION_ATTEMPT_TIMEOUT=120 +#ENV REPLICATION_INITIALIZE_TIMEOUT=0 ENV OPENDJ_USER="opendj" #ENV OPENDJ_JAVA_ARGS="" ENV BACKEND_TYPE="je" diff --git a/opendj-packages/opendj-docker/Dockerfile-alpine b/opendj-packages/opendj-docker/Dockerfile-alpine index cdfa282d9f..fba79d46fb 100644 --- a/opendj-packages/opendj-docker/Dockerfile-alpine +++ b/opendj-packages/opendj-docker/Dockerfile-alpine @@ -36,6 +36,7 @@ ENV OPENDJ_SSL_OPTIONS="--generateSelfSignedCertificate" #ENV REPLICATION_RETRY_COUNT=30 #ENV REPLICATION_RETRY_INTERVAL=10 #ENV REPLICATION_ATTEMPT_TIMEOUT=120 +#ENV REPLICATION_INITIALIZE_TIMEOUT=0 ENV OPENDJ_USER="opendj" #ENV OPENDJ_JAVA_ARGS="" ENV BACKEND_TYPE="je" @@ -57,7 +58,7 @@ WORKDIR /opt RUN apk add --update --no-cache --virtual builddeps curl unzip \ && apk upgrade --update --no-cache \ && if [ "$TARGETARCH" = "386" ]; then JDK=openjdk11-jre; else JDK=openjdk25-jre; fi \ - && apk add bash "$JDK" \ + && apk add bash coreutils "$JDK" \ && if [ -z "$VERSION" ] ; then VERSION="$(curl -i -o - --silent https://api.github.com/repos/OpenIdentityPlatform/OpenDJ/releases/latest | grep -m1 "\"name\"" | cut -d\" -f4)"; fi \ && if [ ! -f "$OPENDJ_DIST_FILENAME" ]; then echo file exists && curl -L https://github.com/OpenIdentityPlatform/OpenDJ/releases/download/$VERSION/opendj-$VERSION.zip --output $OPENDJ_DIST_FILENAME; fi \ && unzip $OPENDJ_DIST_FILENAME \ diff --git a/opendj-packages/opendj-docker/README.md b/opendj-packages/opendj-docker/README.md index 8c49cc60fb..5d18c7d43a 100644 --- a/opendj-packages/opendj-docker/README.md +++ b/opendj-packages/opendj-docker/README.md @@ -72,37 +72,63 @@ With `OPENDJ_REPLICATION_TYPE=simple`, the container joins its replication topol background, next to the running server, on every start - not as a step of the first bootstrap, so a join that could not complete is tried again on the next start, and membership that changed while no container ran is repaired. `REPLICATION_PEERS` lists every -server of the topology, comma separated; all of them share `BASE_DN`, `ROOT_USER_DN`, -`ROOT_PASSWORD`, `ADMIN_PORT` and `REPLICATION_PORT`. A Kubernetes StatefulSet derives the -list from its ordinals (`-0.,…,-N-1.`), so it needs -no registry; plain `docker run` passes the container names. `MASTER_SERVER` keeps working -as a one-element list. A server recognises itself in the list by its `hostname -f` or by -the first DNS label of it - a pod named `-N` is listed as `-N.`. +server of the topology by DNS name, comma separated; all of them share `BASE_DN`, +`ROOT_USER_DN`, `ROOT_PASSWORD`, `ADMIN_PORT` and `REPLICATION_PORT`. A Kubernetes +StatefulSet derives the list from its ordinals +(`-0.,…,-N-1.`), so it needs no registry; plain +`docker run` passes the container names, each container started with `--hostname` equal to +its `--name` (or with `MYHOSTNAME` set to it). `MASTER_SERVER` keeps working as a +one-element list, and may name the master by an address or an `/etc/hosts` alias as before: +the master recognises itself by those as well. + +A server recognises itself in the list by a name that equals its `hostname -f`, or that is +its `hostname -f` cut at a dot (or the other way round) - a pod whose FQDN is +`-N...svc.cluster.local` is listed as +`-N.` - and by one of its own addresses or a name `/etc/hosts` gives one +of them (an `--add-host` or a `hostAliases` entry for itself). Names are compared whole: +`opendj-1` is not `opendj-10`, and `opendj-0.opendj.east` is not `opendj-0.opendj.west`. +List DNS names rather than addresses in `REPLICATION_PEERS`: the servers register in +`cn=admin data` by name, and a server listed by address alone would be taken for one that +left the topology and removed from it (see below). The join decides from what is there, not from an exit code: this server is a member once its configuration holds the replication domain for `BASE_DN` and it is registered in -`cn=admin data`. Until then it runs `dsreplication enable` through every listed peer, -`REPLICATION_RETRY_COUNT` times every `REPLICATION_RETRY_INTERVAL` seconds, each attempt +`cn=admin data`. Until then it runs `dsreplication enable` through the listed peers, +`REPLICATION_RETRY_COUNT` times every `REPLICATION_RETRY_INTERVAL` seconds, each enable bounded by `REPLICATION_ATTEMPT_TIMEOUT` seconds, and the container reports itself healthy -only once the join succeeded - across restarts too. A server that already is a member is -ready as soon as it serves, without waiting for its peers, so a whole cluster restart does -not deadlock under `OrderedReady`. +only once the join succeeded and the volume holds the data of the topology - across +restarts too. A server that already is a member and holds that data is ready as soon as it +serves, without waiting for its peers, so a whole cluster restart does not deadlock under +`OrderedReady`. A volume the container bootstrapped is initialized from the topology once the join succeeds, whether or not `BASE_DN` already holds entries: every fresh volume holds what `ADD_BASE_ENTRY` or `SAMPLE_DATA` imported, which says nothing about which server holds -the data of the topology. Only the first entry of `REPLICATION_PEERS` may decide - once -its retries are exhausted without finding anyone - that there is no topology yet, and seed -it with its own data. The residual risk of that rule: with every server down at once *and* -the volume of the first peer lost, the first peer seeds an empty topology and the others -initialize from it; keep backups accordingly. +the data of the topology. `dsreplication initialize` is a full import of `BASE_DN` and runs +without a bound unless `REPLICATION_INITIALIZE_TIMEOUT` sets one. Each server publishes to +its peers whether its volume still waits for that data (`pending`) or holds it (`ready`), in +the local entry `cn=Docker Join,cn=config`, and a pending server joins and initializes only +through a ready one. So servers may start together - Compose, or a StatefulSet with +`podManagementPolicy: Parallel` - without any of them taking the bootstrap data of another +fresh one for the topology's. Only the first entry of `REPLICATION_PEERS` may decide that +there is no topology yet and seed it with its own data: at once when every other peer +answers and waits for data as well, or once its retries are exhausted without any peer +that could hold the data answering. The residual risk of that rule: with every other server +down *and* the volume of the first peer lost, the first peer seeds an empty topology and the +others initialize from it; keep backups accordingly. A volume that joined but whose +initialize never completed keeps waiting for a ready peer on every start, so under +`OrderedReady` a `-0` in that state stays unready until a peer that holds the data is +started by hand - which is where its data has to come from anyway. With `REPLICATION_PEERS` set explicitly, a joined server also removes every server that is registered in the topology but no longer listed: its entries in `cn=admin data` and its -values in the local replication server lists. A scale-down therefore needs no `preStop` -hook - `dsreplication disable` on termination would take the server out on every rolling -restart, and cannot clean up a server that is already gone. With only `MASTER_SERVER` set -nothing is removed, so servers joined by hand stay. +values in every local replication server list - those of `BASE_DN`, and of `cn=schema` and +`cn=admin data`, which `dsreplication enable` replicates next to it. A scale-down therefore +needs no `preStop` hook - `dsreplication disable` on termination would take the server out +on every rolling restart, and cannot clean up a server that is already gone. It also adds +every listed peer registered in the topology to the local lists that lack it, so joins that +ran at the same time cannot leave two replication servers that know nothing of each other. +With only `MASTER_SERVER` set nothing is removed or added, so servers joined by hand stay. The one-shot types `srs`, `sdsr` and `rg` keep their previous behaviour - they run once, during the first bootstrap only - and are deprecated in favour of `simple`. With @@ -189,11 +215,12 @@ with `subPath` when the Secret changes. | SECRET_VOLUME | /var/secrets/opendj | Mounted keystore volume, if present its `key*` and `trust*` files are copied into the instance on every start, see [Certificates](#certificates) | | SECRET_VOLUME_REFRESH | 60 | While the server runs, `SECRET_VOLUME` is checked again every that many seconds and changed files are copied again; `0` copies them on start only | | MASTER_SERVER | - | Replication master server; with `simple` it works as a one-element `REPLICATION_PEERS` | -| REPLICATION_PEERS | value of MASTER_SERVER | every server of the replication topology, comma separated; the first entry may seed a new topology, and servers no longer listed are removed from it, see [Replication](#replication) | +| REPLICATION_PEERS | value of MASTER_SERVER | every server of the replication topology by DNS name, comma separated; the first entry may seed a new topology, and servers no longer listed are removed from it, see [Replication](#replication) | | REPLICATION_PORT | 8989 | replication port, the same on every server of the topology | | REPLICATION_RETRY_COUNT | 30 | rounds of join attempts through every peer before the join gives up: the first peer of `REPLICATION_PEERS` then seeds the topology, any other server stays `unhealthy` | | REPLICATION_RETRY_INTERVAL | 10 | seconds between rounds of join attempts | -| REPLICATION_ATTEMPT_TIMEOUT | 120 | seconds a single `dsreplication` run may take before it is killed and tried again; `dsreplication enable` can hang on a peer that stops mid-operation | +| REPLICATION_ATTEMPT_TIMEOUT | 120 | seconds a single `dsreplication enable` may take before it is killed and tried again; it can hang on a peer that stops mid-operation | +| REPLICATION_INITIALIZE_TIMEOUT | 0 | seconds a single `dsreplication initialize` - a full import of `BASE_DN` - may take before it is killed and tried again; `0` sets no bound | | VERSION | - | OpenDJ version | | OPENDJ_USER | opendj | user which runs OpenDJ | | OPENDJ_REPLICATION_TYPE | - | OpenDJ Replication type, valid values are:
  • simple - standard replication, joined in the background on every start, see [Replication](#replication)
  • srs - standalone replication servers (one-shot, deprecated)
  • sdsr - Standalone Directory Server Replicas (one-shot, deprecated)
  • rg - Replication Groups (one-shot, deprecated)
Other values will be ignored | diff --git a/opendj-packages/opendj-docker/bootstrap/join.sh b/opendj-packages/opendj-docker/bootstrap/join.sh index 97636d5c67..75396b1d03 100755 --- a/opendj-packages/opendj-docker/bootstrap/join.sh +++ b/opendj-packages/opendj-docker/bootstrap/join.sh @@ -19,15 +19,18 @@ # container ran is repaired. It covers OPENDJ_REPLICATION_TYPE=simple; the one-shot srs, sdsr # and rg paths stay in replicate.sh. # -# The contract with the operator: REPLICATION_PEERS lists every server of the topology, and -# all of them share BASE_DN, ROOT_USER_DN, ROOT_PASSWORD, ADMIN_PORT and REPLICATION_PORT. A -# StatefulSet chart derives the list from its ordinals (-0. ... -N-1.), -# plain docker run passes the container names. MASTER_SERVER keeps working as a one-element -# list. This server recognises itself in the list by comparing a peer with hostname -f, or -# the peer's first DNS label with its own: a StatefulSet pod is named -N but listed as -# -N.. The anchored comparison is deliberate - the grep of /etc/hosts -# it replaces took opendj-1 for opendj-10, and a master that lost its volume for a replica -# of itself. +# The contract with the operator: REPLICATION_PEERS lists every server of the topology by DNS +# name, and all of them share BASE_DN, ROOT_USER_DN, ROOT_PASSWORD, ADMIN_PORT and +# REPLICATION_PORT. A StatefulSet chart derives the list from its ordinals +# (-0. ... -N-1.), plain docker run passes the container names of +# containers whose --hostname is their --name. MASTER_SERVER keeps working as a one-element +# list. This server recognises itself in the list by a name that equals hostname -f, or is +# hostname -f cut at a dot (a StatefulSet pod whose FQDN is -N...svc. +# is listed as -N.), or the other way round; and, as the grep of /etc/hosts it +# replaces did, by one of its own addresses or a name /etc/hosts gives one of them. Names +# are compared whole - that grep took opendj-1 for opendj-10, a replica for its master when +# an --add-host named the master, and a master that lost its volume for a replica of itself - +# and cut only at a dot, so opendj-0.opendj.east is not opendj-0.opendj.west. # # Membership is decided from what is there, never from an exit code: exit 5 of dsreplication # enable (REPLICATION_CANNOT_BE_ENABLED_ON_BASEDN) means "no suffix was left to enable", @@ -37,17 +40,22 @@ # and the server is registered in cn=admin data; anything else is tried again, whatever the # exit code, because a repeated enable is safe exactly when it is decided this way. # -# Initialization follows what the volume went through, not whether BASE_DN has entries: -# every fresh volume has entries (setup.sh imports the base entry or SAMPLE_DATA), and two -# freshly bootstrapped volumes even share a generation ID, so replication silently carries -# nothing that reached the seed by import-ldif. run.sh marks a volume it bootstrapped with -# $INITIALIZE_PENDING; the join runs dsreplication initialize from a peer while the marker -# is there and removes it once the data of the topology has arrived, so a marker that -# outlived a killed container still gets its initialize on the next start. The seed - only -# the first entry of REPLICATION_PEERS, and only once the retries are exhausted without -# finding a topology - removes the marker instead: its data is what the others initialize -# from. The residual risk is documented in the README: every server down at once and the -# first peer's volume lost means the first peer seeds empty. +# Whose data the topology carries follows what each volume went through, not whether BASE_DN +# has entries: every fresh volume has entries (setup.sh imports the base entry or +# SAMPLE_DATA), and two freshly bootstrapped volumes even share a generation ID, so +# replication silently carries nothing that reached one of them by import-ldif. run.sh marks +# a volume it bootstrapped with $INITIALIZE_PENDING, and the join publishes that to its peers +# in $STATE_DN, a local entry of cn=config: "pending" while the marker is there, "ready" once +# the volume holds the data of the topology - it was never bootstrapped by run.sh, it was +# initialized from a ready peer, or it seeded the topology. A pending server joins and +# initializes only through a ready peer, so two fresh servers that start together never take +# each other's bootstrap data for the topology's; where the list is only MASTER_SERVER, a +# peer that publishes no state (an image before this one) counts as ready, as the old +# replicate.sh trusted its master. The seed is only the first entry of REPLICATION_PEERS: at +# once when every other peer answers and is pending (all of them start from scratch, +# together or not), or once the retries are exhausted without any peer it could trust. The +# residual risk is documented in the README: every other server down *and* the first peer's +# volume lost means the first peer seeds empty. # # The health marker is written only once the join succeeded or the seed rule fired, so a # container that never joined stays unhealthy - across restarts too, which the bootstrap-only @@ -65,16 +73,24 @@ REPLICATION_PORT=${REPLICATION_PORT:-8989} MYHOSTNAME=${MYHOSTNAME:-$(hostname -f)} BOOTSTRAP_COMPLETE=${BOOTSTRAP_COMPLETE:-/opt/opendj/.bootstrap-complete} INITIALIZE_PENDING=${INITIALIZE_PENDING:-/opt/opendj/data/.replication-initialize-pending} +# not replicated: cn=config is this server's alone, and it lives on the volume next to the +# marker, so the two cannot tell different stories +STATE_DN="cn=Docker Join,cn=config" REPLICATION_RETRY_COUNT=${REPLICATION_RETRY_COUNT:-30} REPLICATION_RETRY_INTERVAL=${REPLICATION_RETRY_INTERVAL:-10} -# dsreplication can hang rather than fail (a peer that stops mid-operation), so no attempt -# runs without a bound +# dsreplication enable can hang rather than fail (a peer that stops mid-operation), so no +# enable runs without a bound REPLICATION_ATTEMPT_TIMEOUT=${REPLICATION_ATTEMPT_TIMEOUT:-120} +# a total update is a full import of BASE_DN, which takes as long as the data needs; a killed +# dsreplication initialize leaves its task running on the server, and the next attempt asks +# for another full import, so by default it runs without a bound +REPLICATION_INITIALIZE_TIMEOUT=${REPLICATION_INITIALIZE_TIMEOUT:-0} # Removing servers that left the topology needs the operator's word on who belongs to it: # with only MASTER_SERVER set, a third server joined by hand would be "not in the list" and -# thrown out. So the cleanup runs only when REPLICATION_PEERS itself is set. +# thrown out. So the cleanup, and the repair of the replication server lists, run only when +# REPLICATION_PEERS itself is set. PEERS_ARE_EXPLICIT=${REPLICATION_PEERS:+yes} REPLICATION_PEERS=${REPLICATION_PEERS:-$MASTER_SERVER} @@ -105,14 +121,61 @@ lower() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]' } -# DNS names compare case-insensitively, and a peer of a StatefulSet is the pod's hostname -# plus the headless service: self when the peer equals hostname -f, or its first label -# equals the first label of hostname -f +# an IPv4 address is digits and dots, an IPv6 one has a colon +is_address() { + case $1 in + *:*) return 0 ;; + *[!0-9.]* | '') return 1 ;; + *) return 0 ;; + esac +} + +# Two names (lower case) are one server when they are equal, or when one of them is the +# other cut at a dot; addresses only when they are equal +names_match() { + [ "$1" = "$2" ] && return 0 + if is_address "$1" || is_address "$2"; then + return 1 + fi + case $1 in "$2".*) return 0 ;; esac + case $2 in "$1".*) return 0 ;; esac + return 1 +} + +SELF_NAME=$(lower "$MYHOSTNAME") +# the addresses of this container (loopback aside) and the names /etc/hosts gives them - an +# --add-host or a hostAliases entry for itself - as the base replicate.sh recognised them +SELF_ADDRESSES=() +SELF_ALIASES=() +own_names=" $(lower "$(hostname)") $(lower "$(hostname -f 2>/dev/null)") $SELF_NAME " +for address in $(hostname -i 2>/dev/null) \ + $(awk -v names="$own_names" '!/^[[:space:]]*#/ { for (i = 2; i <= NF; i++) if (index(names, " " tolower($i) " ")) { print $1; break } }' /etc/hosts 2>/dev/null); do + case $address in 127.* | ::1 | 0.0.0.0) continue ;; esac + SELF_ADDRESSES+=("$(lower "$address")") +done +if [ "${#SELF_ADDRESSES[@]}" -gt 0 ]; then + for name in $(awk -v addresses=" ${SELF_ADDRESSES[*]} " '!/^[[:space:]]*#/ && index(addresses, " " tolower($1) " ") { for (i = 2; i <= NF; i++) print tolower($i) }' /etc/hosts 2>/dev/null); do + SELF_ALIASES+=("$name") + done +fi + is_self() { - local peer host + local peer candidate peer=$(lower "$1") - host=$(lower "$MYHOSTNAME") - [ "$peer" = "$host" ] || [ "${peer%%.*}" = "${host%%.*}" ] + names_match "$peer" "$SELF_NAME" && return 0 + for candidate in "${SELF_ADDRESSES[@]}" "${SELF_ALIASES[@]}"; do + [ "$peer" = "$candidate" ] && return 0 + done + return 1 +} + +in_peers() { + local host peer + host=$(lower "$1") + for peer in "${PEERS[@]}"; do + names_match "$(lower "$peer")" "$host" && return 0 + done + return 1 } # every LDAP operation goes through the administration connector, which always serves TLS; @@ -125,22 +188,102 @@ search() { --bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" "$@" 2>/dev/null } +ldapmodify_local() { + ldapmodify --noPropertiesFile --hostname localhost --port "$ADMIN_PORT" --useSsl --trustAll \ + --bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" "$@" 2>/dev/null +} + +dsconfig_local() { + dsconfig "$@" --hostname localhost --port "$ADMIN_PORT" --bindDN "$ROOT_USER_DN" \ + --bindPasswordFile "$PASSWORD_FILE" --trustAll --no-prompt +} + +# runs a command within that many seconds, or without a bound for 0; timeout executes the +# command, so it has to be a program, not a function of this script +bounded() { + local limit=$1 + shift + if [ "$limit" -gt 0 ] 2>/dev/null; then + timeout "$limit" "$@" + else + "$@" + fi +} + server_up() { search localhost --baseDN "" --searchScope base "(objectClass=*)" 1.1 >/dev/null } +publish_state() { + printf 'dn: %s\nchangetype: modify\nreplace: description\ndescription: %s\n' "$STATE_DN" "$1" \ + | ldapmodify_local >/dev/null && return 0 + printf 'dn: %s\nobjectClass: top\nobjectClass: ds-cfg-branch\nobjectClass: extensibleObject\ncn: Docker Join\ndescription: %s\n' "$STATE_DN" "$1" \ + | ldapmodify_local --defaultAdd >/dev/null && return 0 + echo "join: could not publish the state $1 of this server to its peers" + return 1 +} + +# ready or pending as the peer publishes it, absent when it answers without publishing any, +# down when it does not answer +peer_state() { + local out state + out=$(search "$1" --baseDN "$STATE_DN" --searchScope base "(objectClass=*)" description) + case $? in + 0) + state=$(printf '%s\n' "$out" | awk 'tolower($1) == "description:" { print $2; exit }') + case $state in + ready | pending) echo "$state" ;; + *) echo absent ;; + esac + ;; + 32) echo absent ;; + *) echo down ;; + esac +} + +trusted() { + case $1 in + ready) return 0 ;; + absent) [ "$PEERS_ARE_EXPLICIT" != yes ] ;; + *) return 1 ;; + esac +} + +registered_hosts() { + search localhost --baseDN "cn=Servers,cn=admin data" --searchScope one "(objectClass=*)" hostname \ + | awk 'tolower($1) == "hostname:" { print $2 }' +} + +is_registered() { + local host peer + peer=$(lower "$1") + for host in $(registered_hosts); do + names_match "$peer" "$(lower "$host")" && return 0 + done + return 1 +} + # a member holds the replication domain for BASE_DN in its configuration and is registered -# in cn=admin data; both are made by one dsreplication enable, so requiring both keeps a -# half-done enable in the retry loop instead of declaring it a success +# in cn=admin data - under the name the enable that registered it connected with, which for +# a server that never ran an enable of its own is the name a peer listed it by; both are made +# by one dsreplication enable, so requiring both keeps a half-done enable in the retry loop +# instead of declaring it a success +replicates_base_dn() { # + search "$1" --baseDN "cn=config" --searchScope sub \ + "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN))" 1.1 | grep -q "^dn:" +} + is_member() { - search localhost --baseDN "cn=config" --searchScope sub \ - "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN))" 1.1 | grep -q "^dn:" || return 1 - search localhost --baseDN "cn=Servers,cn=admin data" --searchScope one \ - "(hostname=$MYHOSTNAME)" 1.1 | grep -q "^dn:" + local host + replicates_base_dn localhost || return 1 + for host in $(registered_hosts); do + is_self "$host" && return 0 + done + return 1 } enable_through() { - timeout "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication enable \ + bounded "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication enable \ --host1 "$1" --port1 "$ADMIN_PORT" --bindDN1 "$ROOT_USER_DN" \ --bindPasswordFile1 "$PASSWORD_FILE" --replicationPort1 "$REPLICATION_PORT" \ --host2 "$MYHOSTNAME" --port2 "$ADMIN_PORT" --bindDN2 "$ROOT_USER_DN" \ @@ -149,141 +292,180 @@ enable_through() { --baseDN "$BASE_DN" -X -n } -# the generation ID of the replication domain, from the domain's monitor entry (the -# connected-to attribute is what tells it from the entries of a replication server) -generation_id() { - search "$1" --baseDN "cn=monitor" --searchScope sub \ - "(&(domain-name=$BASE_DN)(connected-to=*))" generation-id \ - | awk '/^generation-id: /{print $2; exit}' -} - initialize_from() { - timeout "$REPLICATION_ATTEMPT_TIMEOUT" dsreplication initialize --baseDN "$BASE_DN" \ + bounded "$REPLICATION_INITIALIZE_TIMEOUT" dsreplication initialize --baseDN "$BASE_DN" \ --adminUID admin --adminPasswordFile "$PASSWORD_FILE" \ --hostSource "$1" --portSource "$ADMIN_PORT" \ --hostDestination "$MYHOSTNAME" --portDestination "$ADMIN_PORT" -X -n } -# initialize while the marker of a bootstrapped, never initialized volume is there; without -# it only cross-check the generation IDs, so a volume that already carried data is never -# overwritten by a peer -ensure_initialized() { - local source=$1 peer local_id peer_id rc i - if [ ! -f "$INITIALIZE_PENDING" ]; then - if [ -n "$source" ]; then - local_id=$(generation_id localhost) - peer_id=$(generation_id "$source") - if [ -n "$local_id" ] && [ -n "$peer_id" ] && [ "$local_id" != "$peer_id" ]; then - echo "join: generation ID $local_id does not match $peer_id of $source; replication will not flow until this server or that one is initialized by hand" - fi - fi - return 0 - fi - if [ -z "$source" ]; then - for peer in "${PEERS[@]}"; do - is_self "$peer" && continue - if [ -n "$(generation_id "$peer")" ]; then - source=$peer - break - fi - done - source=${source:-} - fi - if [ -z "$source" ]; then - echo "join: no peer to initialize from" - return 1 +# the generation ID of the replication domain, from the domain's own monitor entry: a +# replication server also puts domain-name, connected-to and generation-id on the entry of +# every directory server connected to it, but only the domain publishes replayed-updates +generation_id() { + search "$1" --baseDN "cn=monitor" --searchScope sub \ + "(&(domain-name=$BASE_DN)(replayed-updates=*))" generation-id \ + | awk 'tolower($1) == "generation-id:" { print $2; exit }' +} + +cross_check() { # + local local_id peer_id + local_id=$(generation_id localhost) + peer_id=$(generation_id "$1") + if [ -n "$local_id" ] && [ -n "$peer_id" ] && [ "$local_id" != "$peer_id" ]; then + echo "join: generation ID $local_id does not match $peer_id of $1$2" fi - for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do - echo "join: initializing from $source" - if initialize_from "$source"; then - rm -f "$INITIALIZE_PENDING" - local_id=$(generation_id localhost) - peer_id=$(generation_id "$source") - if [ -n "$local_id" ] && [ -n "$peer_id" ] && [ "$local_id" != "$peer_id" ]; then - echo "join: generation ID $local_id does not match $peer_id of $source after initialize" - fi - return 0 - fi - rc=$? - [ "$i" -eq "$REPLICATION_RETRY_COUNT" ] && return $rc - echo "join: initialize from $source exited with $rc, trying again in $REPLICATION_RETRY_INTERVAL s" - sleep "$REPLICATION_RETRY_INTERVAL" +} + +# the first other peer, in the order of the list, that holds the data of the topology and +# replicates BASE_DN - asked of the peer itself, as cn=admin data registers a server under +# whatever name the enable that registered it used, which need not be the listed one +trusted_source() { + local peer + for peer in "${PEERS[@]}"; do + is_self "$peer" && continue + trusted "$(peer_state "$peer")" && replicates_base_dn "$peer" && { echo "$peer"; return 0; } done + return 1 } -ldapmodify_local() { - ldapmodify --noPropertiesFile --hostname localhost --port "$ADMIN_PORT" --useSsl --trustAll \ - --bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" 2>/dev/null +# every other peer answers and waits for the data of the topology itself; true as well when +# the list names no other server +all_others_pending() { + local peer + for peer in "${PEERS[@]}"; do + is_self "$peer" && continue + [ "$(peer_state "$peer")" = pending ] || return 1 + done + return 0 } -in_peers() { - local host peer - host=$(lower "${1%%.*}") +any_other_trusted() { + local peer for peer in "${PEERS[@]}"; do - [ "$(lower "${peer%%.*}")" = "$host" ] && return 0 + is_self "$peer" && continue + trusted "$(peer_state "$peer")" && return 0 done return 1 } +# The values of the replication server lists this server holds: one "\t" line for +# every list, its replication server entry (kind server) and each of its replication domains +# (kind domain) - BASE_DN, and cn=schema and cn=admin data, which dsreplication enable +# configures next to it - followed by a "\t\t" line for every value +replication_server_lists() { + search localhost --baseDN "cn=config" --searchScope sub \ + "(|(objectClass=ds-cfg-replication-server)(objectClass=ds-cfg-replication-domain))" \ + objectClass cn ds-cfg-replication-server \ + | awk -v OFS='\t' ' + function flush( i) { + if (kind != "") { print kind, cn; for (i = 1; i <= n; i++) print kind, cn, values[i] } + kind = ""; cn = ""; n = 0 + } + /^dn: / { flush() } + tolower($0) == "objectclass: ds-cfg-replication-domain" { kind = "domain" } + tolower($0) == "objectclass: ds-cfg-replication-server" { kind = "server" } + tolower($1) == "cn:" { cn = substr($0, 5) } + tolower($1) == "ds-cfg-replication-server:" { values[++n] = $2 } + END { flush() }' +} + +# it runs inside loops that read their lines from stdin, which dsconfig is kept away from +change_replication_servers() { # --add|--remove + if [ "$1" = server ]; then + dsconfig_local set-replication-server-prop --provider-name "Multimaster Synchronization" \ + "$3" "replication-server:$4" /dev/null \ || echo "join: could not remove $dn (already removed by another survivor?)" printf 'dn: cn=all-servers,cn=Server Groups,cn=admin data\nchangetype: modify\ndelete: uniqueMember\nuniqueMember: %s\n' "${dn%%,cn=Servers,cn=admin data}" \ - | ldapmodify_local || true + | ldapmodify_local >/dev/null || true fi done - search localhost --baseDN "cn=config" --searchScope sub \ - "(objectClass=ds-cfg-replication-server)" ds-cfg-replication-server \ - | awk '/^ds-cfg-replication-server: /{print $2}' \ - | while read -r value; do - host=${value%:*} - if ! in_peers "$host" && ! is_self "$host"; then - echo "join: removing departed replication server $value from the replication server configuration" - dsconfig set-replication-server-prop --provider-name "Multimaster Synchronization" \ - --remove "replication-server:$value" \ - --hostname localhost --port "$ADMIN_PORT" --bindDN "$ROOT_USER_DN" \ - --bindPasswordFile "$PASSWORD_FILE" --trustAll --no-prompt || true - fi + replication_server_lists | while IFS=$'\t' read -r kind cn value; do + [ -z "$value" ] && continue + host=${value%:*} + if ! in_peers "$host" && ! is_self "$host"; then + echo "join: removing departed replication server $value from the $kind list of $cn" + change_replication_servers "$kind" "$cn" --remove "$value" || true + fi + done +} + +# Adds every listed peer that is registered in the topology to each replication server list +# of this server that lacks it, under the name it is registered by - the name dsreplication +# enable writes as well, so no server is listed twice. Joins that run at the same time each +# configure the topology as they read it, and two of them can leave a pair of replication +# servers that know nothing of each other, cutting the updates of one off from the other; a +# replication server connects to a server added to its list at once. A peer that is not +# registered yet is waited for, as long as the retries last, after this server already +# reports itself healthy. +repair_replication_servers() { + local i peer waiting hosts host lists kind cn value found + [ "$PEERS_ARE_EXPLICIT" = yes ] || return 0 + for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do + waiting= + for peer in "${PEERS[@]}"; do + is_self "$peer" || is_registered "$peer" || waiting="$waiting $peer" done - domain=$(search localhost --baseDN "cn=config" --searchScope sub \ - "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN))" cn \ - | awk '/^cn: /{print substr($0,5); exit}') - [ -z "$domain" ] && return 0 - search localhost --baseDN "cn=config" --searchScope sub \ - "(&(objectClass=ds-cfg-replication-domain)(ds-cfg-base-dn=$BASE_DN))" ds-cfg-replication-server \ - | awk '/^ds-cfg-replication-server: /{print $2}' \ - | while read -r value; do - host=${value%:*} - if ! in_peers "$host" && ! is_self "$host"; then - echo "join: removing departed replication server $value from the replication domain configuration" - dsconfig set-replication-domain-prop --provider-name "Multimaster Synchronization" \ - --domain-name "$domain" --remove "replication-server:$value" \ - --hostname localhost --port "$ADMIN_PORT" --bindDN "$ROOT_USER_DN" \ - --bindPasswordFile "$PASSWORD_FILE" --trustAll --no-prompt || true + [ -z "$waiting" ] && break + [ "$i" -eq "$REPLICATION_RETRY_COUNT" ] && break + echo "join: waiting for$waiting to register in the topology before the replication server lists are checked" + sleep "$REPLICATION_RETRY_INTERVAL" + done + hosts=$(registered_hosts) + lists=$(replication_server_lists) + while IFS=$'\t' read -r kind cn value; do + [ -n "$value" ] && continue + for host in $hosts; do + is_self "$host" && continue + in_peers "$host" || continue + found=no + while IFS=$'\t' read -r k c v; do + [ "$k" = "$kind" ] && [ "$c" = "$cn" ] && [ -n "$v" ] || continue + names_match "$(lower "${v%:*}")" "$(lower "$host")" && { found=yes; break; } + done <<<"$lists" + if [ "$found" = no ]; then + echo "join: adding replication server $host:$REPLICATION_PORT to the $kind list of $cn" + change_replication_servers "$kind" "$cn" --add "$host:$REPLICATION_PORT" || true fi done + done <<<"$lists" } joined() { - ensure_initialized "$1" || return 1 cleanup_departed touch "$BOOTSTRAP_COMPLETE" echo "join: this server is a member of the replication topology, the health check may probe it" + repair_replication_servers +} + +seed() { # + echo "join: $1, seeding it with this server's data" + rm -f "$INITIALIZE_PENDING" + publish_state ready + touch "$BOOTSTRAP_COMPLETE" } echo "join: waiting for the server on the administration connector" @@ -296,46 +478,110 @@ for i in $(seq 1 150); do sleep 2 done -if is_member; then - echo "join: already a member of the replication topology" - joined "" - exit -fi - -# a list without a single other server - MASTER_SERVER pointing at this server, the way the -# old replicate.sh master recognised itself - leaves nothing to retry against, so the seed -# rule below decides at once instead of sitting out the retries -OTHERS=no -for peer in "${PEERS[@]}"; do - is_self "$peer" || OTHERS=yes -done +FIRST=no +is_self "${PEERS[0]}" && FIRST=yes -[ "$OTHERS" = yes ] && for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do - for peer in "${PEERS[@]}"; do - is_self "$peer" && continue - echo "join: enabling replication with $peer" - enable_through "$peer" - rc=$? - if is_member; then - echo "join: joined the replication topology through $peer" - joined "$peer" - exit +if [ ! -f "$INITIALIZE_PENDING" ]; then + # This volume holds the data of the topology, or is the one that seeds it: it may join + # through any peer that answers + publish_state ready + if is_member; then + echo "join: already a member of the replication topology" + joined + exit + fi + for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do + for peer in "${PEERS[@]}"; do + is_self "$peer" && continue + echo "join: enabling replication with $peer" + enable_through "$peer" + rc=$? + if is_member; then + echo "join: joined the replication topology through $peer" + cross_check "$peer" "; replication will not flow until this server or that one is initialized by hand" + joined + exit + fi + echo "join: dsreplication enable with $peer exited with $rc and membership is not there" + done + # a list without a single other server leaves nothing to retry against, and peers that + # all wait for data join through this one rather than the other way round + if [ "$FIRST" = yes ] && all_others_pending; then + seed "no other peer holds the data of the topology" + exit 0 + fi + if [ "$i" -lt "$REPLICATION_RETRY_COUNT" ]; then + echo "join: no peer joined this server yet, trying again in $REPLICATION_RETRY_INTERVAL s ($i of $REPLICATION_RETRY_COUNT)" + sleep "$REPLICATION_RETRY_INTERVAL" fi - echo "join: dsreplication enable with $peer exited with $rc and membership is not there" done + if [ "$FIRST" = yes ]; then + seed "no topology found after $REPLICATION_RETRY_COUNT attempts" + exit 0 + fi + echo "join: could not join the replication topology after $REPLICATION_RETRY_COUNT attempts, this container will not report itself healthy" + exit 1 +fi + +# This volume was bootstrapped and never received the data of the topology: it joins and +# initializes only through a peer that holds that data, and only the first peer may decide +# that nobody does +publish_state pending +for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do + if ! is_member; then + for peer in "${PEERS[@]}"; do + is_self "$peer" && continue + state=$(peer_state "$peer") + if ! trusted "$state"; then + echo "join: $peer is $state, not a peer to join through" + continue + fi + echo "join: enabling replication with $peer" + enable_through "$peer" + rc=$? + if is_member; then + echo "join: joined the replication topology through $peer" + break + fi + echo "join: dsreplication enable with $peer exited with $rc and membership is not there" + done + fi + if is_member; then + source=$(trusted_source) + if [ -n "$source" ]; then + echo "join: initializing from $source" + initialize_from "$source" + rc=$? + if [ "$rc" -eq 0 ]; then + rm -f "$INITIALIZE_PENDING" + publish_state ready + cross_check "$source" " after initialize" + joined + exit + fi + echo "join: initialize from $source exited with $rc" + elif [ "$FIRST" = yes ] && all_others_pending; then + seed "every other peer waits for the data of the topology as well" + joined + exit 0 + else + echo "join: no peer holds the data of the topology yet" + fi + elif [ "$FIRST" = yes ] && all_others_pending; then + seed "every other peer waits for the data of the topology as well" + exit 0 + fi if [ "$i" -lt "$REPLICATION_RETRY_COUNT" ]; then - echo "join: no peer joined this server yet, trying again in $REPLICATION_RETRY_INTERVAL s ($i of $REPLICATION_RETRY_COUNT)" + echo "join: not joined and initialized yet, trying again in $REPLICATION_RETRY_INTERVAL s ($i of $REPLICATION_RETRY_COUNT)" sleep "$REPLICATION_RETRY_INTERVAL" fi done -# Only the first peer of the list may decide that there is no topology to join and that its -# own data seeds one; anyone else staying unhealthy is what surfaces a lost topology instead -# of forking it. The seed keeps its bootstrap data, so the initialize marker goes away. -if is_self "${PEERS[0]}"; then - echo "join: no topology found after $REPLICATION_RETRY_COUNT attempts, seeding it with this server's data" - rm -f "$INITIALIZE_PENDING" - touch "$BOOTSTRAP_COMPLETE" +# Only the first peer may decide that there is no topology to join and that its own data +# seeds one, and only while no peer that could hold the topology's data answers; anyone +# else staying unhealthy is what surfaces a lost topology instead of forking it +if [ "$FIRST" = yes ] && ! is_member && ! any_other_trusted; then + seed "no topology found after $REPLICATION_RETRY_COUNT attempts" exit 0 fi diff --git a/opendj-packages/opendj-docker/run.sh b/opendj-packages/opendj-docker/run.sh index b294554b8e..da9071e3fb 100755 --- a/opendj-packages/opendj-docker/run.sh +++ b/opendj-packages/opendj-docker/run.sh @@ -133,16 +133,18 @@ if [ -d ./data/config ]; then # nothing is bootstrapped here, the instance is already there - but a half-migrated one # is not ready to serve either, so the marker follows the upgrade if sh ./upgrade -n; then - # A server whose replication domain is already configured is ready as soon as it - # serves: gating it on its peers would deadlock a whole-cluster restart under - # OrderedReady, where -0 would wait for peers the StatefulSet starts only once -0 is - # ready. Only a volume that was asked to join and never did waits for the join, so a - # join that failed before a restart no longer turns into a healthy, unreplicated - # server the way it did when the marker followed the upgrade alone. - if ! join_requested || grep -q "ds-cfg-replication-domain" ./data/config/config.ldif; then + # A server whose replication domain is already configured, and whose volume holds the + # data of the topology, is ready as soon as it serves: gating it on its peers would + # deadlock a whole-cluster restart under OrderedReady, where -0 would wait for peers + # the StatefulSet starts only once -0 is ready. A volume that was asked to join and + # never did, or that joined but never received the data of the topology, waits for the + # join, so a join or an initialize that failed before a restart no longer turns into a + # healthy server with bootstrap data only, the way it did when the marker followed the + # upgrade alone. + if ! join_requested || { grep -q "ds-cfg-replication-domain" ./data/config/config.ldif && [ ! -f "$INITIALIZE_PENDING" ]; }; then touch "$BOOTSTRAP_COMPLETE" else - echo "This instance never joined its replication topology, the join decides whether it is healthy" + echo "This instance never joined its replication topology or never received its data, the join decides whether it is healthy" fi # the join also repairs membership that changed while no container ran, on every start if join_requested; then From 26c82087713edbdcb686078d968303dd43a1d68f Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Tue, 29 Sep 2026 18:02:41 +0300 Subject: [PATCH 3/5] [#1086] Keep the ERR trap of the replication test out of command substitutions set -E hands the ERR trap to every $(...): a command that failed there, as the dsreplication enable the replicate.sh pin expects to fail, dumped the container logs into the captured output and removed the containers under the running test. The trap now acts in the main shell only. join.sh waits for the peers to register with one search a round instead of one per peer: every ldapsearch starts a JVM. --- .github/scripts/docker-test-replication.sh | 5 ++++- .../opendj-docker/bootstrap/join.sh | 18 ++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/scripts/docker-test-replication.sh b/.github/scripts/docker-test-replication.sh index 4c9a9dabac..442fdbd585 100755 --- a/.github/scripts/docker-test-replication.sh +++ b/.github/scripts/docker-test-replication.sh @@ -33,7 +33,10 @@ cleanup() { docker volume rm -f $VOLUMES >/dev/null 2>&1 || true } cleanup -trap 'code=$?; for c in $NODES; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR +# set -E hands the trap to every $(...) as well, where a failing command would dump the logs +# into the value being captured and remove the containers under the running test: there the +# failure only ends the subshell, and the main shell decides +trap 'code=$?; [ "$BASH_SUBSHELL" -eq 0 ] || exit $code; for c in $NODES; do echo "::group::container logs ($c)"; docker logs $c 2>&1 || true; echo "::endgroup::"; done; cleanup; exit $code' ERR fail() { echo "::error::$1" diff --git a/opendj-packages/opendj-docker/bootstrap/join.sh b/opendj-packages/opendj-docker/bootstrap/join.sh index 75396b1d03..45895871e6 100755 --- a/opendj-packages/opendj-docker/bootstrap/join.sh +++ b/opendj-packages/opendj-docker/bootstrap/join.sh @@ -254,15 +254,6 @@ registered_hosts() { | awk 'tolower($1) == "hostname:" { print $2 }' } -is_registered() { - local host peer - peer=$(lower "$1") - for host in $(registered_hosts); do - names_match "$peer" "$(lower "$host")" && return 0 - done - return 1 -} - # a member holds the replication domain for BASE_DN in its configuration and is registered # in cn=admin data - under the name the enable that registered it connected with, which for # a server that never ran an enable of its own is the name a peer listed it by; both are made @@ -426,8 +417,15 @@ repair_replication_servers() { [ "$PEERS_ARE_EXPLICIT" = yes ] || return 0 for i in $(seq 1 "$REPLICATION_RETRY_COUNT"); do waiting= + # one search a round: every ldapsearch starts a JVM + hosts=$(registered_hosts) for peer in "${PEERS[@]}"; do - is_self "$peer" || is_registered "$peer" || waiting="$waiting $peer" + is_self "$peer" && continue + found=no + for host in $hosts; do + names_match "$(lower "$peer")" "$(lower "$host")" && { found=yes; break; } + done + [ "$found" = yes ] || waiting="$waiting $peer" done [ -z "$waiting" ] && break [ "$i" -eq "$REPLICATION_RETRY_COUNT" ] && break From 68f3e33b9f28795be50898ec0352761116d978c5 Mon Sep 17 00:00:00 2001 From: Valera V Harseko Date: Wed, 30 Sep 2026 18:40:02 +0300 Subject: [PATCH 4/5] [#1086] Mark a volume pending before its bootstrap, keep a seed ready across restarts, and take turns to enable Review round 2 of #1115: - both image jobs check out .github/scripts, so the replication test runs at all - run.sh marks the volume pending before the bootstrap, so a bootstrap that failed or was killed is initialized from the topology rather than published ready - on a restart the volume is ready unless it is pending; a seed that no peer joined has no replication domain, and the join cannot bind once the root password was changed - the first peer seeds after its retries on either road only while no other peer answers that may hold the data: ready, or publishing no state but replicating BASE_DN - the state reaches the peers before the pending marker goes, or the join fails - a server the survivors removed while it was away takes its replication configuration down and enables anew: dsreplication enable does not register a server whose replicated cn=admin data already matches the peer's - servers registered by an address are not removed as departed - the /tmp fallback of the password files has a name run.sh removes - an empty list of replication servers changes nothing - joins through one peer take turns, with a lock entry in the peer's cn=config: two enable runs at once leave the loser a member only in part, for good - README: one ADMIN_PORT for the one-shot types, address-registered servers, the lock - CI: a member ready while its peer is down, a seed ready after its root password changed, a failed bootstrap and a failed initialize, a scale-up on a retained volume, no lock left after the parallel start, and sdsr's exit 8 without a race: the replica starts on a network of its own rather than dj-0 leaving the topology's, which left dj-0's replication server a dead connection of its own directory server to route the replica's initialize into --- .github/scripts/docker-test-replication.sh | 101 ++++++-- .github/workflows/build.yml | 10 +- opendj-packages/opendj-docker/README.md | 35 ++- .../opendj-docker/bootstrap/join.sh | 225 +++++++++++++++--- .../opendj-docker/bootstrap/replicate.sh | 13 +- opendj-packages/opendj-docker/run.sh | 48 ++-- 6 files changed, 335 insertions(+), 97 deletions(-) diff --git a/.github/scripts/docker-test-replication.sh b/.github/scripts/docker-test-replication.sh index 442fdbd585..1ec9c48dd9 100755 --- a/.github/scripts/docker-test-replication.sh +++ b/.github/scripts/docker-test-replication.sh @@ -23,13 +23,18 @@ set -eE -o pipefail IMAGE=${1:?usage: $0 } -NODES="dj-0 dj-1 dj-2 dj-x dj-p0 dj-p1 dj-p2 dj-ipm dj-ipr dj-sdsr dj-shm-probe dj-shm" -VOLUMES="vol-dj-0 vol-dj-1 vol-dj-2 vol-dj-x vol-dj-p0 vol-dj-p1 vol-dj-p2" +NODES="dj-0 dj-1 dj-2 dj-x dj-solo dj-bf dj-fi dj-p0 dj-p1 dj-p2 dj-ipm dj-ipr dj-sdsr dj-shm-probe dj-shm" +VOLUMES="vol-dj-0 vol-dj-1 vol-dj-2 vol-dj-x vol-dj-solo vol-dj-bf vol-dj-fi vol-dj-p0 vol-dj-p1 vol-dj-p2" +# a bootstrap that imports the base entry and then fails, mounted into dj-bf +FAILING_BOOTSTRAP=$(mktemp) NETWORK=test_replication +# where dj-sdsr starts: it resolves its own name there, and no other server +NETWORK_ALONE=test_replication_alone cleanup() { + rm -f "$FAILING_BOOTSTRAP" docker rm -f $NODES >/dev/null 2>&1 || true - docker network rm $NETWORK >/dev/null 2>&1 || true + docker network rm $NETWORK $NETWORK_ALONE >/dev/null 2>&1 || true docker volume rm -f $VOLUMES >/dev/null 2>&1 || true } cleanup @@ -197,14 +202,31 @@ ldaps_has dj-1 "uid=user.0,ou=People,dc=example,dc=com" || fail "dj-1 lacks the add_ou dj-0 replicated wait_has dj-1 "ou=replicated,dc=example,dc=com" -# a member is ready again right after a restart, without waiting for its peers, and -# replication still flows -docker restart dj-0 dj-1 >/dev/null -wait_healthy dj-0 +# a member is ready again right after a restart, without waiting for its peers - gating it +# on them would deadlock a whole-cluster restart under OrderedReady - and replication still +# flows +docker stop dj-1 >/dev/null +docker restart dj-0 >/dev/null +wait_until 120 "dj-0 is healthy while dj-1 is down" is_healthy dj-0 +docker start dj-1 >/dev/null wait_healthy dj-1 add_ou dj-1 replicated2 wait_has dj-0 "ou=replicated2,dc=example,dc=com" +# a seed that no peer joined holds the data without a replication domain, and its join binds +# with the ROOT_PASSWORD of the bootstrap: once the root password is changed, a restart is +# still healthy, as it is without replication +start_node dj-solo dj-solo +wait_healthy dj-solo +logs_have dj-solo "seeding it with this server's data" || fail "dj-solo did not seed its topology" +docker exec dj-solo /opt/opendj/bin/ldappasswordmodify --noPropertiesFile --hostname localhost --port 1636 \ + --useSsl --trustAll --bindDN "cn=Directory Manager" --bindPassword "$ROOT_PASSWORD" \ + --authzID "dn:cn=Directory Manager" --currentPassword "$ROOT_PASSWORD" --newPassword "changed $ROOT_PASSWORD" >/dev/null +docker restart dj-solo >/dev/null +wait_until 180 "dj-solo is healthy again with its root password changed" is_healthy dj-solo +docker rm -f dj-solo >/dev/null +docker volume rm vol-dj-solo >/dev/null + # a Kubernetes pod keeps its /dev/shm across container restarts, shared by all its # containers: a starting container removes the password files a killed join or replicate.sh # left there - only those of its own ADMIN_PORT, the files of the other containers of the pod @@ -245,6 +267,31 @@ if logs_have dj-0 "seeding it with this server's data"; then fail "the reborn dj ldaps_has dj-0 "ou=replicated,dc=example,dc=com" || fail "the reborn dj-0 lacks the data of the topology" ldaps_has dj-0 "uid=user.0,ou=People,dc=example,dc=com" || fail "the reborn dj-0 lacks the entries of the seed" +# a bootstrap that failed after it imported the base entry leaves a volume that never counts +# as holding the data of the topology: once restarted, dj-bf initializes from dj-0 rather +# than joining with what its bootstrap left +printf 'sh /opt/opendj/bootstrap/setup.sh\nexit 1\n' >"$FAILING_BOOTSTRAP" +chmod 644 "$FAILING_BOOTSTRAP" +start_node dj-bf dj-0,dj-1,dj-bf -v "$FAILING_BOOTSTRAP:/opt/opendj/failing-bootstrap.sh:ro" \ + -e BOOTSTRAP=/opt/opendj/failing-bootstrap.sh +wait_until 300 "the bootstrap of dj-bf fails" logs_have dj-bf "failing-bootstrap.sh failed" +docker restart dj-bf >/dev/null +wait_healthy dj-bf +logs_have dj-bf "initializing from dj-0" || fail "dj-bf joined with the data of its failed bootstrap" +ldaps_has dj-bf "uid=user.0,ou=People,dc=example,dc=com" || fail "dj-bf lacks the entries only an initialize from dj-0 brings" +docker rm -f dj-bf >/dev/null +docker volume rm vol-dj-bf >/dev/null + +# an initialize that fails keeps the volume waiting for the data of the topology: a +# one-second bound cuts the dsreplication JVM off before it connects +start_node dj-fi dj-0,dj-fi -e REPLICATION_INITIALIZE_TIMEOUT=1 -e REPLICATION_RETRY_COUNT=2 +wait_until 300 "the initialize of dj-fi is cut off" logs_have dj-fi "initialize from dj-0 exited with 124" +stays_unhealthy dj-fi "its initialize failed" +docker exec dj-fi test -f /opt/opendj/data/.replication-initialize-pending \ + || fail "dj-fi dropped its pending marker after a failed initialize" +docker rm -f dj-fi >/dev/null +docker volume rm vol-dj-fi >/dev/null + # scale up to three - the peer list is configuration, so the running servers are replaced # with the longer list before the third one starts, as a rolling update would docker rm -f dj-0 dj-1 >/dev/null @@ -259,9 +306,9 @@ ldaps_has dj-2 "ou=replicated,dc=example,dc=com" || fail "dj-2 lacks the data of # scale down to two: the survivors, restarted with the shorter list, remove dj-2 from # cn=admin data and from every replication server list they hold - that of BASE_DN, and # those of cn=schema and cn=admin data that dsreplication enable configures next to it. -# dsreplication disable cannot, dj-2 being already gone +# dsreplication disable cannot, dj-2 being already gone. dj-2 keeps its volume, for the +# scale-up below docker rm -f dj-2 >/dev/null -docker volume rm vol-dj-2 >/dev/null docker rm -f dj-0 dj-1 >/dev/null start_node dj-0 dj-0,dj-1 start_node dj-1 dj-0,dj-1 @@ -279,17 +326,29 @@ done add_ou dj-0 replicated3 wait_has dj-1 "ou=replicated3,dc=example,dc=com" +# scaled up again on the volume it kept: dj-2 still replicates, but the survivors no longer +# register it, and an enable between two servers whose cn=admin data is replicated registers +# nobody - so dj-2 takes its replication configuration down, enables again and registers +start_node dj-2 dj-0,dj-1,dj-2 +wait_healthy dj-2 +registered_dj2() { ! admin_data_lacks dj-0 dj-2; } +wait_until 300 "dj-2 registers in the topology again" registered_dj2 +add_ou dj-2 rejoined +wait_has dj-0 "ou=rejoined,dc=example,dc=com" + # the deprecated one-shot sdsr path of replicate.sh still bootstraps a replica. On a first # start a server stops the server its bootstrap started and starts it again, and a replica # can reach it in between: replicate.sh tries a dsreplication enable that could not connect -# again (exit 8). dj-0 is taken off the network while the replica sleeps before its first try, -# and comes back only once the replica has said it will try again -docker run -d --memory="512m" --network $NETWORK --name dj-sdsr --hostname dj-sdsr \ +# again (exit 8). The replica starts on a network of its own, where dj-0 does not resolve, and +# moves to the network of the topology only once it has said it will try again: taking dj-0 +# off the network for as long instead would leave its replication server holding the dead +# connection of dj-0's own directory server, and route the initialize of the replica into it +docker network create $NETWORK_ALONE >/dev/null +docker run -d --memory="512m" --network $NETWORK_ALONE --name dj-sdsr --hostname dj-sdsr \ -e ROOT_PASSWORD="$ROOT_PASSWORD" -e MASTER_SERVER=dj-0 -e OPENDJ_REPLICATION_TYPE=sdsr "$IMAGE" >/dev/null -wait_until 300 "dj-sdsr sleeps before its first try" logs_have dj-sdsr "Will sleep for a bit" -docker network disconnect $NETWORK dj-0 -wait_until 120 "dj-sdsr tries again" logs_have dj-sdsr "exited with 8, trying again" -docker network connect --alias dj-0 $NETWORK dj-0 +wait_until 420 "dj-sdsr tries again" logs_have dj-sdsr "exited with 8, trying again" +docker network connect $NETWORK dj-sdsr +docker network disconnect $NETWORK_ALONE dj-sdsr wait_healthy dj-sdsr wait_has dj-sdsr "ou=replicated3,dc=example,dc=com" # replicate.sh tries dsreplication enable again only when it exits 8, and a failed enable @@ -304,13 +363,13 @@ if [ "$rc" -ne 5 ] || grep -E "trying again|initializing replication" <<<"$out" fi # the root password shows in no container log, and the files the tools read it from are gone -for c in dj-0 dj-1 dj-sdsr; do +for c in dj-0 dj-1 dj-2 dj-sdsr; do if docker logs $c 2>&1 | grep -F -- "$ROOT_PASSWORD"; then fail "the root password is in the log of $c"; fi left=$(docker exec $c grep -rlsF -- "$ROOT_PASSWORD" /tmp /dev/shm || true) [ -z "$left" ] || fail "the root password is left in $left of $c" done -docker rm -f dj-0 dj-1 dj-sdsr >/dev/null -docker volume rm vol-dj-0 vol-dj-1 >/dev/null +docker rm -f dj-0 dj-1 dj-2 dj-sdsr >/dev/null +docker volume rm vol-dj-0 vol-dj-1 vol-dj-2 >/dev/null # MASTER_SERVER may name the master by its address, as the grep of /etc/hosts in the base # replicate.sh allowed: a master given its own address recognises itself and seeds at once, @@ -355,6 +414,10 @@ wait_until 120 "the replication server of dj-p2 knows dj-p0 and dj-p1" server_li add_ou dj-p1 parallel wait_has dj-p0 "ou=parallel,dc=example,dc=com" wait_has dj-p2 "ou=parallel,dc=example,dc=com" +# dj-p1 and dj-p2 enabled through dj-p0 one at a time, and each removed its lock after +if admin_search dj-p0 "cn=Docker Join Lock,cn=config" base "(objectClass=*)" 1.1 | grep "^dn:" >/dev/null; then + fail "a join left its lock on dj-p0" +fi for c in dj-p0 dj-p1 dj-p2; do if docker logs $c 2>&1 | grep -F -- "$ROOT_PASSWORD"; then fail "the root password is in the log of $c"; fi done diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0328313eae..99ada09a00 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -476,9 +476,12 @@ jobs: ports: - 5000:5000 steps: + # .github/scripts holds the replication test that both image jobs run - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - sparse-checkout: .github/benchmark + sparse-checkout: | + .github/benchmark + .github/scripts - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -856,9 +859,12 @@ jobs: ports: - 5000:5000 steps: + # .github/scripts holds the replication test that both image jobs run - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - sparse-checkout: .github/benchmark + sparse-checkout: | + .github/benchmark + .github/scripts - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: diff --git a/opendj-packages/opendj-docker/README.md b/opendj-packages/opendj-docker/README.md index 5d18c7d43a..92fe7944eb 100644 --- a/opendj-packages/opendj-docker/README.md +++ b/opendj-packages/opendj-docker/README.md @@ -89,7 +89,11 @@ of them (an `--add-host` or a `hostAliases` entry for itself). Names are compare `opendj-1` is not `opendj-10`, and `opendj-0.opendj.east` is not `opendj-0.opendj.west`. List DNS names rather than addresses in `REPLICATION_PEERS`: the servers register in `cn=admin data` by name, and a server listed by address alone would be taken for one that -left the topology and removed from it (see below). +left the topology and removed from it (see below). A server registered by an address - a +master that `MASTER_SERVER=
` named, which its replicas registered under that address +- is never removed that way, as nothing tells it from a listed name: moving such a topology +to a `REPLICATION_PEERS` of names keeps the master, and once it really is gone, it is +removed by hand. The join decides from what is there, not from an exit code: this server is a member once its configuration holds the replication domain for `BASE_DN` and it is registered in @@ -97,12 +101,15 @@ its configuration holds the replication domain for `BASE_DN` and it is registere `REPLICATION_RETRY_COUNT` times every `REPLICATION_RETRY_INTERVAL` seconds, each enable bounded by `REPLICATION_ATTEMPT_TIMEOUT` seconds, and the container reports itself healthy only once the join succeeded and the volume holds the data of the topology - across -restarts too. A server that already is a member and holds that data is ready as soon as it -serves, without waiting for its peers, so a whole cluster restart does not deadlock under -`OrderedReady`. - -A volume the container bootstrapped is initialized from the topology once the join -succeeds, whether or not `BASE_DN` already holds entries: every fresh volume holds what +restarts too. A server whose volume holds that data is ready as soon as it serves, without +waiting for its peers or for its join, so a whole cluster restart does not deadlock under +`OrderedReady`, and a changed root password does not keep it unready either (the join, +which binds with `ROOT_PASSWORD`, then only logs that it cannot). + +A volume the container bootstraps is marked before the bootstrap starts, and initialized +from the topology once the join succeeds, whether or not `BASE_DN` already holds entries - so +a bootstrap that failed or was killed half-way is initialized as well, never taken for data +of the topology: every fresh volume holds what `ADD_BASE_ENTRY` or `SAMPLE_DATA` imported, which says nothing about which server holds the data of the topology. `dsreplication initialize` is a full import of `BASE_DN` and runs without a bound unless `REPLICATION_INITIALIZE_TIMEOUT` sets one. Each server publishes to @@ -110,10 +117,15 @@ its peers whether its volume still waits for that data (`pending`) or holds it ( the local entry `cn=Docker Join,cn=config`, and a pending server joins and initializes only through a ready one. So servers may start together - Compose, or a StatefulSet with `podManagementPolicy: Parallel` - without any of them taking the bootstrap data of another -fresh one for the topology's. Only the first entry of `REPLICATION_PEERS` may decide that +fresh one for the topology's. Servers that join through the same peer take turns: two +`dsreplication enable` runs through one server at once leave the loser a member only in +part, for good. The turn is the entry `cn=Docker Join Lock,cn=config` on that peer, which a +join adds before its enable and removes after it; one left behind by a killed join is taken +over once it is older than `REPLICATION_ATTEMPT_TIMEOUT` plus a minute. Only the first entry of `REPLICATION_PEERS` may decide that there is no topology yet and seed it with its own data: at once when every other peer answers and waits for data as well, or once its retries are exhausted without any peer -that could hold the data answering. The residual risk of that rule: with every other server +that could hold the data answering - a ready one, or a server of an earlier image that +publishes no state but replicates `BASE_DN`. The residual risk of that rule: with every other server down *and* the volume of the first peer lost, the first peer seeds an empty topology and the others initialize from it; keep backups accordingly. A volume that joined but whose initialize never completed keeps waiting for a ready peer on every start, so under @@ -131,7 +143,10 @@ ran at the same time cannot leave two replication servers that know nothing of e With only `MASTER_SERVER` set nothing is removed or added, so servers joined by hand stay. The one-shot types `srs`, `sdsr` and `rg` keep their previous behaviour - they run once, -during the first bootstrap only - and are deprecated in favour of `simple`. With +during the first bootstrap only - and are deprecated in favour of `simple`. Like `simple`, +they need one `ADMIN_PORT` and one `REPLICATION_PORT` on every server: the tools reach the +master on the ports of the container they run in (images before this one used 4444 and +8989 on both sides, whatever the container was set up with). With `OPENDJ_REPLICATION_TYPE=srs`, start the directory server replicas one at a time, each once the previous one is healthy: every replica pushes its data to the replicas connected at that moment, and one that is restarting at the end of its own first start misses it. diff --git a/opendj-packages/opendj-docker/bootstrap/join.sh b/opendj-packages/opendj-docker/bootstrap/join.sh index 45895871e6..9bff5a6e82 100755 --- a/opendj-packages/opendj-docker/bootstrap/join.sh +++ b/opendj-packages/opendj-docker/bootstrap/join.sh @@ -53,9 +53,10 @@ # peer that publishes no state (an image before this one) counts as ready, as the old # replicate.sh trusted its master. The seed is only the first entry of REPLICATION_PEERS: at # once when every other peer answers and is pending (all of them start from scratch, -# together or not), or once the retries are exhausted without any peer it could trust. The -# residual risk is documented in the README: every other server down *and* the first peer's -# volume lost means the first peer seeds empty. +# together or not), or once the retries are exhausted while no other peer answers that may +# hold the data - a ready one, or one of an image before this one that publishes no state but +# replicates BASE_DN. The residual risk is documented in the README: every other server down +# *and* the first peer's volume lost means the first peer seeds empty. # # The health marker is written only once the join succeeded or the seed rule fired, so a # container that never joined stays unhealthy - across restarts too, which the bootstrap-only @@ -112,8 +113,9 @@ fi # The tools read the root password from a file (#1084); the file lives on the tmpfs of # /dev/shm where there is one, and run.sh removes what a killed join left behind, by the -# ADMIN_PORT in the name, on every start -PASSWORD_FILE=$(mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.XXXXXX" 2>/dev/null || mktemp) || exit 1 +# name, on every start - in /dev/shm those of its ADMIN_PORT, in /tmp all of them +PASSWORD_FILE=$(mktemp -p /dev/shm "opendj-join.$ADMIN_PORT.XXXXXX" 2>/dev/null \ + || mktemp "/tmp/opendj-join.$ADMIN_PORT.XXXXXX") || exit 1 trap 'rm -f "$PASSWORD_FILE"' EXIT printf '%s\n' "$ROOT_PASSWORD" >"$PASSWORD_FILE" || exit 1 @@ -188,11 +190,17 @@ search() { --bindDN "$ROOT_USER_DN" --bindPasswordFile "$PASSWORD_FILE" "$@" 2>/dev/null } -ldapmodify_local() { - ldapmodify --noPropertiesFile --hostname localhost --port "$ADMIN_PORT" --useSsl --trustAll \ +ldapmodify_on() { # [