From e2fa56adea9aa3ec26d4842f9bc28f1abcd6fb08 Mon Sep 17 00:00:00 2001 From: Gautier DI FOLCO Date: Fri, 11 Sep 2026 19:46:54 +0200 Subject: [PATCH] feat(brig): remove ElasticSearch; query user search from Postgres ElasticSearch backed exactly one feature area: brig user indexing. Delete the projection entirely and query brig's Postgres wire_user table directly via a new UserSearchStore Polysemy effect. New: - Wire.UserSearchStore effect: searchUsers, searchUsersFederated, paginateTeamMembers, getTeamSize, setTeamSearchVisibilityInbound - Wire.UserSearchStore.Postgres interpreter porting the ES semantics: shouldIndex candidate filter, documented swagger rank order, visibility via team_search_visibility (default own-team-only), app exclusion, tokenized prefix matching with LIKE escaping, keyset pagination - Wire.UserSearch.Normalize (ICU Any-Latin;Latin-ASCII;Lower, formerly the ES 'normalized' field) now stored in wire_user.name_normalized - migration: name_normalized + text_pattern_ops indexes + team_search_visibility table - brig-index reduced to 'backfill-normalized-names' (run once before cutover; pre-backfill rows lack name_normalized) - in-memory mock for MiniBackend unit tests Deleted: - Wire.IndexedUserStore*, Wire.UserSearch.{Types,Migration,Metrics}, Wire.UserStore.IndexUser, GetIndexUser/GetIndexUsersPaginated store ops - Brig.Index.*, Brig.User.Search.{Index,SearchIndex}, IndexEnv/mkIndexEnv - ISearchIndexAPI routes (/i/index/*), galley updateSearchIndex push, index-sync call sites, counter metrics - bloodhound dependency, tools/db/find-undead, elasticsearch/kibana charts, opensearch releases, dockerephemeral ES services, ES test fixtures and ES-specific integration tests Deployment: run `brig-index backfill-normalized-names --pg-settings ...` after this migration and before search cutover. Known env-only build failures (untouched packages): integration Certs.hs (crypton-x509/pem solver skew), spar/federator/stern integration exes (hidden Test.Hspec.JUnit), extended/saml2 (missing shared libssl). --- Makefile | 41 +- cabal.project | 1 - charts/databases-ephemeral/requirements.yaml | 9 +- .../databases-ephemeral/templates/NOTES.txt | 3 +- charts/elasticsearch-curator/Chart.yaml | 4 - .../elasticsearch-curator/requirements.yaml | 4 - charts/elasticsearch-curator/values.yaml | 31 - charts/elasticsearch-ephemeral/Chart.yaml | 4 - .../templates/_helpers.tpl | 16 - .../templates/cert.yaml | 30 - .../templates/es-svc.yaml | 23 - .../elasticsearch-ephemeral/templates/es.yaml | 72 -- charts/elasticsearch-ephemeral/values.yaml | 23 - charts/elasticsearch-external/Chart.yaml | 4 - .../templates/endpoint.yaml | 38 - .../templates/helpers.tpl | 11 - charts/elasticsearch-external/values.yaml | 6 - charts/elasticsearch-index/.helmignore | 21 - charts/elasticsearch-index/Chart.yaml | 4 - .../templates/_helpers.tpl | 54 -- .../templates/cassandra-secret.yaml | 14 - .../templates/create-index.yaml | 118 --- .../templates/elasticsearch-ca-secret.yaml | 14 - .../elasticsearch-index/templates/helpers.tpl | 7 - .../templates/migrate-data.yaml | 108 --- .../elasticsearch-index/templates/secret.yaml | 23 - charts/elasticsearch-index/values.yaml | 75 -- charts/fluent-bit/values.yaml | 20 - .../templates/integration-integration.yaml | 13 - charts/integration/values.yaml | 3 - charts/kibana/Chart.yaml | 4 - charts/kibana/requirements.yaml | 4 - .../kibana/templates/basic-auth-secret.yaml | 13 - charts/kibana/values.yaml | 22 - charts/wire-server/requirements.yaml | 5 - charts/wire-server/templates/_helpers.tpl | 40 - .../wire-server/templates/brig/configmap.yaml | 24 - .../templates/brig/deployment.yaml | 18 - .../brig/elasticsearch-ca-secret.yaml | 30 - charts/wire-server/templates/brig/secret.yaml | 6 - .../brig/tests/brig-integration.yaml | 9 - .../templates/brig/tests/configmap.yaml | 2 - charts/wire-server/values.yaml | 28 - .../dockerephemeral/db-migrate/brig-index.sh | 9 - deploy/dockerephemeral/docker-compose.yaml | 71 -- .../docker/elasticsearch-ca.pem | 20 - .../docker/elasticsearch-cert.pem | 20 - .../docker/elasticsearch-key.pem | 28 - .../opensearch-security/action_groups.yml | 3 - .../opensearch-security/allowlist.yml | 6 - .../opensearch/opensearch-security/config.yml | 17 - .../opensearch-security/internal_users.yml | 12 - .../opensearch-security/nodes_dn.yml | 3 - .../opensearch/opensearch-security/roles.yml | 3 - .../opensearch-security/roles_mapping.yml | 9 - .../opensearch-security/tenants.yml | 3 - .../docker/opensearch/opensearch.yml | 45 - .../opensearch/opensearch_dashboards.yml | 8 - .../dockerephemeral/federation-v0/brig.yaml | 4 - .../federation-v0/nginz/conf/nginx.conf | 20 - .../dockerephemeral/federation-v1/brig.yaml | 4 - .../federation-v1/nginz/conf/nginx.conf | 20 - .../dockerephemeral/federation-v2/brig.yaml | 8 - .../elasticsearch-credentials.yaml | 2 - docs/src/developer/developer/building.md | 1 - .../src/developer/reference/config-options.md | 108 --- .../install/infrastructure-configuration.md | 5 +- docs/src/how-to/install/troubleshooting.md | 1 - flake.lock | 18 - flake.nix | 5 - hack/bin/create-helm-sboms.sh | 1 - hack/bin/gen-certs.sh | 9 - hack/bin/set-wire-server-image-version.sh | 2 +- hack/helm_vars/certs/elasticsearch-ca-key.pem | 28 - hack/helm_vars/certs/elasticsearch-ca.pem | 20 - hack/helm_vars/certs/values.yaml.gotmpl | 17 - hack/helm_vars/opensearch/values.yaml.gotmpl | 192 ---- .../wire-federation-v0/values.yaml.gotmpl | 10 - hack/helm_vars/wire-server/values.yaml.gotmpl | 45 - hack/helmfile.yaml.gotmpl | 57 -- ...integration-dynamic-backends-brig-index.sh | 10 - integration/test/API/BrigInternal.hs | 6 - integration/test/SetupHelpers.hs | 114 --- integration/test/Test/Apps.hs | 3 - integration/test/Test/Brig.hs | 1 - integration/test/Test/Demo.hs | 7 +- integration/test/Test/Migration/User.hs | 59 -- integration/test/Test/Search.hs | 102 --- integration/test/Test/Teams.hs | 2 - integration/test/Testlib/ModService.hs | 6 - integration/test/Testlib/ResourcePool.hs | 1 - integration/test/Testlib/RunServices.hs | 2 - integration/test/Testlib/Types.hs | 1 - .../src/Wire/API/Routes/Internal/Brig.hs | 3 - .../API/Routes/Internal/Brig/SearchIndex.hs | 41 - .../src/Wire/API/Routes/Public/Brig.hs | 1 - libs/wire-api/src/Wire/API/User/Search.hs | 90 ++ libs/wire-api/wire-api.cabal | 1 - libs/wire-subsystems/default.nix | 3 - ...000-search-store-without-elasticsearch.sql | 24 + .../src/Wire/AppSubsystem/Interpreter.hs | 6 +- .../wire-subsystems/src/Wire/BrigAPIAccess.hs | 1 - .../src/Wire/BrigAPIAccess/Rpc.hs | 11 - .../src/Wire/IndexedUserStore.hs | 63 -- .../IndexedUserStore/Bulk/ElasticSearch.hs | 188 ---- .../Wire/IndexedUserStore/ElasticSearch.hs | 689 -------------- .../Wire/IndexedUserStore/MigrationStore.hs | 30 - .../MigrationStore/ElasticSearch.hs | 90 -- .../src/Wire/UserSearch/Metrics.hs | 61 -- .../src/Wire/UserSearch/Migration.hs | 47 - .../src/Wire/UserSearch/Normalize.hs | 26 +- .../src/Wire/UserSearch/Types.hs | 275 ------ .../src/Wire/UserSearchStore.hs | 70 ++ .../src/Wire/UserSearchStore/Postgres.hs | 864 ++++++++++++++++++ libs/wire-subsystems/src/Wire/UserStore.hs | 5 - .../src/Wire/UserStore/Cassandra.hs | 50 - .../src/Wire/UserStore/IndexUser.hs | 213 ----- .../src/Wire/UserStore/Postgres.hs | 170 +--- .../wire-subsystems/src/Wire/UserSubsystem.hs | 5 - .../src/Wire/UserSubsystem/Interpreter.hs | 134 +-- .../test/resources/elasticsearch-ca.pem | 1 - .../resources/elasticsearch-credentials.yaml | 2 - .../test/unit/Wire/MiniBackend.hs | 20 +- .../test/unit/Wire/MockInterpreters.hs | 2 +- .../Wire/MockInterpreters/IndexedUserStore.hs | 191 ---- .../Wire/MockInterpreters/UserSearchStore.hs | 407 +++++++++ .../unit/Wire/MockInterpreters/UserStore.hs | 34 +- .../Wire/MockInterpreters/UserSubsystem.hs | 1 - .../test/unit/Wire/UserSearch/TypesSpec.hs | 73 -- .../Wire/UserSubsystem/InterpreterSpec.hs | 41 +- libs/wire-subsystems/wire-subsystems.cabal | 18 +- nix/haskell-pins.nix | 4 - nix/local-haskell-packages.nix | 1 - nix/manual-overrides.nix | 3 - services/brig/.env | 2 - services/brig/brig.cabal | 15 +- services/brig/default.nix | 3 - services/brig/index/src/Main.hs | 107 ++- services/brig/src/Brig/API/Auth.hs | 4 - services/brig/src/Brig/API/Federation.hs | 10 +- services/brig/src/Brig/API/Internal.hs | 27 +- services/brig/src/Brig/API/Public.hs | 7 +- services/brig/src/Brig/API/User.hs | 13 +- services/brig/src/Brig/App.hs | 50 +- .../brig/src/Brig/CanonicalInterpreter.hs | 35 +- services/brig/src/Brig/Index/Eval.hs | 271 ------ services/brig/src/Brig/Index/Options.hs | 548 ----------- services/brig/src/Brig/Options.hs | 37 - services/brig/src/Brig/Team/API.hs | 12 +- services/brig/src/Brig/User/Auth.hs | 5 - services/brig/src/Brig/User/Search/Index.hs | 538 ----------- .../brig/src/Brig/User/Search/SearchIndex.hs | 279 ------ .../brig/test/integration/API/Federation.hs | 6 - services/brig/test/integration/API/Search.hs | 651 +------------ .../brig/test/integration/API/Search/Util.hs | 26 +- services/brig/test/integration/API/Team.hs | 4 - .../test/integration/API/TeamUserSearch.hs | 29 +- .../brig/test/integration/API/User/Account.hs | 6 - .../brig/test/integration/API/User/Handles.hs | 2 - .../brig/test/integration/Federation/Util.hs | 1 - .../brig/test/integration/Index/Create.hs | 150 --- services/brig/test/integration/Run.hs | 13 +- services/brig/test/integration/Util.hs | 12 +- services/galley/src/Galley/API/Teams.hs | 2 - services/integration.yaml | 1 - tools/db/find-undead/.ormolu | 1 - tools/db/find-undead/README.md | 23 - tools/db/find-undead/default.nix | 47 - tools/db/find-undead/find-undead.cabal | 84 -- tools/db/find-undead/src/Main.hs | 62 -- tools/db/find-undead/src/Options.hs | 88 -- tools/db/find-undead/src/Work.hs | 125 --- tools/stern/src/Stern/Intra.hs | 3 +- 173 files changed, 1750 insertions(+), 7655 deletions(-) delete mode 100644 charts/elasticsearch-curator/Chart.yaml delete mode 100644 charts/elasticsearch-curator/requirements.yaml delete mode 100644 charts/elasticsearch-curator/values.yaml delete mode 100644 charts/elasticsearch-ephemeral/Chart.yaml delete mode 100644 charts/elasticsearch-ephemeral/templates/_helpers.tpl delete mode 100644 charts/elasticsearch-ephemeral/templates/cert.yaml delete mode 100644 charts/elasticsearch-ephemeral/templates/es-svc.yaml delete mode 100644 charts/elasticsearch-ephemeral/templates/es.yaml delete mode 100644 charts/elasticsearch-ephemeral/values.yaml delete mode 100644 charts/elasticsearch-external/Chart.yaml delete mode 100644 charts/elasticsearch-external/templates/endpoint.yaml delete mode 100644 charts/elasticsearch-external/templates/helpers.tpl delete mode 100644 charts/elasticsearch-external/values.yaml delete mode 100644 charts/elasticsearch-index/.helmignore delete mode 100644 charts/elasticsearch-index/Chart.yaml delete mode 100644 charts/elasticsearch-index/templates/_helpers.tpl delete mode 100644 charts/elasticsearch-index/templates/cassandra-secret.yaml delete mode 100644 charts/elasticsearch-index/templates/create-index.yaml delete mode 100644 charts/elasticsearch-index/templates/elasticsearch-ca-secret.yaml delete mode 100644 charts/elasticsearch-index/templates/helpers.tpl delete mode 100644 charts/elasticsearch-index/templates/migrate-data.yaml delete mode 100644 charts/elasticsearch-index/templates/secret.yaml delete mode 100644 charts/elasticsearch-index/values.yaml delete mode 100644 charts/kibana/Chart.yaml delete mode 100644 charts/kibana/requirements.yaml delete mode 100644 charts/kibana/templates/basic-auth-secret.yaml delete mode 100644 charts/kibana/values.yaml delete mode 100644 charts/wire-server/templates/brig/elasticsearch-ca-secret.yaml delete mode 100755 deploy/dockerephemeral/db-migrate/brig-index.sh delete mode 100644 deploy/dockerephemeral/docker/elasticsearch-ca.pem delete mode 100755 deploy/dockerephemeral/docker/elasticsearch-cert.pem delete mode 100755 deploy/dockerephemeral/docker/elasticsearch-key.pem delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch-security/action_groups.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch-security/allowlist.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch-security/config.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch-security/internal_users.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch-security/nodes_dn.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch-security/roles.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch-security/roles_mapping.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch-security/tenants.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch.yml delete mode 100644 deploy/dockerephemeral/docker/opensearch/opensearch_dashboards.yml delete mode 100644 deploy/dockerephemeral/federation-v2/elasticsearch-credentials.yaml delete mode 100644 hack/helm_vars/certs/elasticsearch-ca-key.pem delete mode 100644 hack/helm_vars/certs/elasticsearch-ca.pem delete mode 100644 hack/helm_vars/opensearch/values.yaml.gotmpl delete mode 100755 integration/scripts/integration-dynamic-backends-brig-index.sh delete mode 100644 libs/wire-api/src/Wire/API/Routes/Internal/Brig/SearchIndex.hs create mode 100644 libs/wire-subsystems/postgres-migrations/20260911000000-search-store-without-elasticsearch.sql delete mode 100644 libs/wire-subsystems/src/Wire/IndexedUserStore.hs delete mode 100644 libs/wire-subsystems/src/Wire/IndexedUserStore/Bulk/ElasticSearch.hs delete mode 100644 libs/wire-subsystems/src/Wire/IndexedUserStore/ElasticSearch.hs delete mode 100644 libs/wire-subsystems/src/Wire/IndexedUserStore/MigrationStore.hs delete mode 100644 libs/wire-subsystems/src/Wire/IndexedUserStore/MigrationStore/ElasticSearch.hs delete mode 100644 libs/wire-subsystems/src/Wire/UserSearch/Metrics.hs delete mode 100644 libs/wire-subsystems/src/Wire/UserSearch/Migration.hs rename services/brig/src/Brig/Index/Types.hs => libs/wire-subsystems/src/Wire/UserSearch/Normalize.hs (51%) delete mode 100644 libs/wire-subsystems/src/Wire/UserSearch/Types.hs create mode 100644 libs/wire-subsystems/src/Wire/UserSearchStore.hs create mode 100644 libs/wire-subsystems/src/Wire/UserSearchStore/Postgres.hs delete mode 100644 libs/wire-subsystems/src/Wire/UserStore/IndexUser.hs delete mode 120000 libs/wire-subsystems/test/resources/elasticsearch-ca.pem delete mode 100644 libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml delete mode 100644 libs/wire-subsystems/test/unit/Wire/MockInterpreters/IndexedUserStore.hs create mode 100644 libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserSearchStore.hs delete mode 100644 libs/wire-subsystems/test/unit/Wire/UserSearch/TypesSpec.hs delete mode 100644 services/brig/src/Brig/Index/Eval.hs delete mode 100644 services/brig/src/Brig/Index/Options.hs delete mode 100644 services/brig/src/Brig/User/Search/Index.hs delete mode 100644 services/brig/src/Brig/User/Search/SearchIndex.hs delete mode 100644 services/brig/test/integration/Index/Create.hs delete mode 120000 tools/db/find-undead/.ormolu delete mode 100644 tools/db/find-undead/README.md delete mode 100644 tools/db/find-undead/default.nix delete mode 100644 tools/db/find-undead/find-undead.cabal delete mode 100644 tools/db/find-undead/src/Main.hs delete mode 100644 tools/db/find-undead/src/Options.hs delete mode 100644 tools/db/find-undead/src/Work.hs diff --git a/Makefile b/Makefile index 0e2e771c983..a45783fd905 100644 --- a/Makefile +++ b/Makefile @@ -7,16 +7,15 @@ DOCKER_TAG ?= $(USER) # default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything) HELM_SEMVER ?= 0.0.42 # The list of helm charts needed on internal kubernetes testing environments -CHARTS_INTEGRATION := wire-server databases-ephemeral rabbitmq fake-aws ingress-nginx-controller nginx-ingress-services wire-ingress fluent-bit kibana k8ssandra-test-cluster wire-server-enterprise +CHARTS_INTEGRATION := wire-server databases-ephemeral rabbitmq fake-aws ingress-nginx-controller nginx-ingress-services wire-ingress fluent-bit k8ssandra-test-cluster wire-server-enterprise # The list of helm charts to publish on S3 # FUTUREWORK: after we "inline local subcharts", # (e.g. move charts/brig to charts/wire-server/brig) # this list could be generated from the folder names under ./charts/ like so: # CHARTS_RELEASE := $(shell find charts/ -maxdepth 1 -type d | xargs -n 1 basename | grep -v charts) CHARTS_RELEASE := wire-server rabbitmq rabbitmq-external databases-ephemeral \ -fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit kibana backoffice \ -calling-test demo-smtp elasticsearch-curator elasticsearch-external \ -elasticsearch-ephemeral minio-external cassandra-external \ +fake-aws fake-aws-s3 fake-aws-sqs aws-ingress fluent-bit backoffice \ +calling-test demo-smtp minio-external cassandra-external \ ingress-nginx-controller nginx-ingress-services \ k8ssandra-test-cluster ldap-scim-bridge wire-server-enterprise \ wire-ingress @@ -393,41 +392,10 @@ postgres-migrate: c ./dist/brig -c ./services/brig/brig.integration.yaml migrate-postgres --dbname dyn-2 ./dist/brig -c ./services/brig/brig.integration.yaml migrate-postgres --dbname dyn-3 -.PHONY: es-reset -es-reset: c - ./dist/brig-index reset \ - --elasticsearch-index-prefix directory \ - --elasticsearch-server https://localhost:9200 \ - --elasticsearch-ca-cert ./libs/wire-subsystems/test/resources/elasticsearch-ca.pem \ - --elasticsearch-credentials ./libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml > /dev/null - ./dist/brig-index reset \ - --elasticsearch-index-prefix directory2 \ - --elasticsearch-server https://localhost:9200 \ - --elasticsearch-ca-cert ./libs/wire-subsystems/test/resources/elasticsearch-ca.pem \ - --elasticsearch-credentials ./libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml > /dev/null - ./integration/scripts/integration-dynamic-backends-brig-index.sh \ - --elasticsearch-server https://localhost:9200 \ - --elasticsearch-ca-cert ./libs/wire-subsystems/test/resources/elasticsearch-ca.pem \ - --elasticsearch-credentials ./libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml > /dev/null - @echo -e "\n'brig-index reset' only deletes the index and regenerates the mapping, but doesn't generate or populate a new index, so you need to call 'make es-reindex explicitly now!\n" - -.PHONY: es-reindex -es-reindex: c - ./dist/brig-index reindex \ - --pg-pool-size 10 \ - --pg-pool-acquisition-timeout 10s \ - --pg-pool-aging-timeout 1d \ - --pg-pool-idleness-timeout 1h \ - --pg-settings '{"host":"127.0.0.1","port":"5432","user":"wire-server","dbname":"backendA"}' \ - --pg-password-file ./libs/wire-subsystems/test/resources/postgres-credentials.yaml \ - --elasticsearch-server https://localhost:9200 \ - --elasticsearch-ca-cert ./libs/wire-subsystems/test/resources/elasticsearch-ca.pem \ - --elasticsearch-credentials ./libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml > /dev/null - .PHONY: rabbitmq-reset rabbitmq-reset: rabbit-clean -# Migrate all keyspaces and reset the ES index +# Migrate all keyspaces # Does not migrate postgres as brig does that on startup. .PHONY: db-migrate db-migrate: c postgres-migrate @@ -440,7 +408,6 @@ db-migrate: c postgres-migrate ./dist/gundeck-schema --keyspace gundeck_test2 --replication-factor 1 > /dev/null ./dist/spar-schema --keyspace spar_test2 --replication-factor 1 > /dev/null ./integration/scripts/integration-dynamic-backends-db-schemas.sh --replication-factor 1 > /dev/null - make es-reset ################################# ## dependencies diff --git a/cabal.project b/cabal.project index 345b1d52076..353bf7ec8c9 100644 --- a/cabal.project +++ b/cabal.project @@ -45,7 +45,6 @@ packages: , services/spar/ , tools/db/assets/ , tools/db/auto-whitelist/ - , tools/db/find-undead/ , tools/db/inconsistencies/ , tools/db/migrate-sso-feature-flag/ , tools/db/migrate-features/ diff --git a/charts/databases-ephemeral/requirements.yaml b/charts/databases-ephemeral/requirements.yaml index fff1534c387..81d0989fd90 100644 --- a/charts/databases-ephemeral/requirements.yaml +++ b/charts/databases-ephemeral/requirements.yaml @@ -3,7 +3,7 @@ dependencies: ## dependent (demo, non-persistent, non-HA) databases # # Note: why are these charts not part of the wire-server chart? -# These charts, in particular cassandra/elasticsearch +# These charts, in particular cassandra # cannot be part of the wire-server chart, because of the required ordering of # 1. install databases, wait for them to be ready # 2. run database cassandra-migrations (done as a pre-install/pre-upgrade hook) @@ -13,13 +13,6 @@ dependencies: # since cassandra-migrations did not yet run; but the cassandra-migrations hook # requires all pods to be in a 'Ready' state before starting (condition for post-install); this is impossible. ##################################################### -- name: elasticsearch-ephemeral - version: "0.0.42" - repository: "file://../elasticsearch-ephemeral" - tags: - - elasticsearch-ephemeral - - databases-ephemeral - - demo - name: cassandra-ephemeral version: "0.0.42" repository: "file://../cassandra-ephemeral" diff --git a/charts/databases-ephemeral/templates/NOTES.txt b/charts/databases-ephemeral/templates/NOTES.txt index 7f07b9ed7ca..105d0fec1de 100644 --- a/charts/databases-ephemeral/templates/NOTES.txt +++ b/charts/databases-ephemeral/templates/NOTES.txt @@ -1,10 +1,9 @@ You now have an in-memory, non-persistent, non-highly-available set of databases: * cassandra-ephemeral -* elasticsearch-ephemeral !! WARNING WARNING !! This is fine for testing and demo purposes, but NOT for a production use case. !! WARNING WARNING !! -Note that before use of these databases for wire-server components, an index (in the case of elasticsearch) and a set of cassandra-migrations (in the case of cassandra) have to be applied. This comes bundled with the wire-server chart (see cassandra-migrations and elasticsearch-index charts for details) +Note that before use of these databases for wire-server components, a set of cassandra-migrations have to be applied. This comes bundled with the wire-server chart (see cassandra-migrations chart for details) diff --git a/charts/elasticsearch-curator/Chart.yaml b/charts/elasticsearch-curator/Chart.yaml deleted file mode 100644 index 69c6a06c6dc..00000000000 --- a/charts/elasticsearch-curator/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: Wrapper chart for stable/elasticsearch-curator -name: elasticsearch-curator -version: 0.0.42 diff --git a/charts/elasticsearch-curator/requirements.yaml b/charts/elasticsearch-curator/requirements.yaml deleted file mode 100644 index 68ae9975e2e..00000000000 --- a/charts/elasticsearch-curator/requirements.yaml +++ /dev/null @@ -1,4 +0,0 @@ -dependencies: -- name: elasticsearch-curator - version: 1.5.0 - repository: https://charts.helm.sh/stable diff --git a/charts/elasticsearch-curator/values.yaml b/charts/elasticsearch-curator/values.yaml deleted file mode 100644 index 45ca082109c..00000000000 --- a/charts/elasticsearch-curator/values.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# See defaults in https://github.com/helm/charts/tree/master/stable/elasticsearch-curator -elasticsearch-curator: - configMaps: - action_file_yml: |- - --- - actions: - 1: - action: delete_indices - description: "Clean up ES by deleting old indices" - options: - timeout_override: - continue_if_exception: False - disable_action: False - ignore_empty_list: True - filters: - - filtertype: age - source: name - direction: older - timestring: '%Y.%m.%d' - unit: days - unit_count: 3 - field: - stats_result: - epoch: - exclude: False - config_yml: |- - --- - client: - hosts: - - elasticsearch-ephemeral - port: 9200 diff --git a/charts/elasticsearch-ephemeral/Chart.yaml b/charts/elasticsearch-ephemeral/Chart.yaml deleted file mode 100644 index 0beb68ceac5..00000000000 --- a/charts/elasticsearch-ephemeral/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: Dummy ephemeral elasticsearch -name: elasticsearch-ephemeral -version: 0.0.42 diff --git a/charts/elasticsearch-ephemeral/templates/_helpers.tpl b/charts/elasticsearch-ephemeral/templates/_helpers.tpl deleted file mode 100644 index 6ecbd30d5a9..00000000000 --- a/charts/elasticsearch-ephemeral/templates/_helpers.tpl +++ /dev/null @@ -1,16 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 53 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 53 chars (63 - len("-discovery")) because some Kubernetes name fields are limited to 63 (by the DNS naming spec). -*/}} -{{- define "fullname" -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- printf "%s" $name | trunc 53 | trimSuffix "-" -}} -{{- end -}} diff --git a/charts/elasticsearch-ephemeral/templates/cert.yaml b/charts/elasticsearch-ephemeral/templates/cert.yaml deleted file mode 100644 index bae69529d25..00000000000 --- a/charts/elasticsearch-ephemeral/templates/cert.yaml +++ /dev/null @@ -1,30 +0,0 @@ - -{{- if .Values.tls.enabled -}} -apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: {{ template "fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -spec: - issuerRef: {{ required "Please specify .Values.tls.issuerRef when .Values.tls.enabled is true" .Values.tls.issuerRef | toJson }} - usages: - - server auth - duration: 2160h # 90d - renewBefore: 360h # 15d - isCA: false - secretName: {{ template "fullname" . }}-certificate - - privateKey: - algorithm: ECDSA - size: 384 - encoding: PKCS1 - rotationPolicy: Always - - dnsNames: - - {{ template "fullname" . }} - - {{ template "fullname" . }}.{{ .Release.Namespace }}.svc.cluster.local -{{- end -}} diff --git a/charts/elasticsearch-ephemeral/templates/es-svc.yaml b/charts/elasticsearch-ephemeral/templates/es-svc.yaml deleted file mode 100644 index 499652ee77d..00000000000 --- a/charts/elasticsearch-ephemeral/templates/es-svc.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ template "fullname" . }} - labels: - app: {{ template "fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - component: {{ template "fullname" . }} -spec: - type: ClusterIP - selector: - component: {{ template "fullname" . }} - ports: - - name: http - port: {{ .Values.service.httpPort }} - targetPort: 9200 - protocol: TCP - - name: transport - port: {{ .Values.service.transportPort }} - targetPort: 9300 - protocol: TCP diff --git a/charts/elasticsearch-ephemeral/templates/es.yaml b/charts/elasticsearch-ephemeral/templates/es.yaml deleted file mode 100644 index ae8f5cfca5d..00000000000 --- a/charts/elasticsearch-ephemeral/templates/es.yaml +++ /dev/null @@ -1,72 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "fullname" . }} - labels: - app: {{ template "fullname" . }} - chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - component: {{ template "fullname" . }} -spec: - replicas: 1 - selector: - matchLabels: - component: {{ template "fullname" . }} - template: - metadata: - labels: - component: {{ template "fullname" . }} - spec: - automountServiceAccountToken: false - containers: - - name: es - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - env: - - name: MAX_HEAP_SIZE - value: "2048" - - name: HEAP_NEWSIZE - value: "800M" - - name: "bootstrap.system_call_filter" - value: "false" - - name: "discovery.type" - value: "single-node" - - name: "action.auto_create_index" - value: ".watches,.triggered_watches,.watcher-history-*,pod-*,node-*" - - name: "xpack.security.enabled" - value: "true" - - name: "ELASTIC_PASSWORD" - value: {{ .Values.secrets.password }} - {{- if .Values.tls.enabled }} - - name: "xpack.security.http.ssl.enabled" - value: "true" - - name: "xpack.security.http.ssl.certificate" - value: "certs/tls.crt" - - name: "xpack.security.http.ssl.key" - value: "certs/tls.key" - {{- end }} - ports: - - containerPort: 9200 - name: http - protocol: TCP - - containerPort: 9300 - name: transport - protocol: TCP - volumeMounts: - - name: storage - mountPath: /data - {{- if .Values.tls.enabled }} - - name: certificate - mountPath: /usr/share/elasticsearch/config/certs - {{- end }} - resources: -{{ toYaml .Values.resources | indent 12 }} - volumes: - - emptyDir: - medium: "" - name: "storage" - {{- if .Values.tls.enabled }} - - name: certificate - secret: - secretName: {{ template "fullname" . }}-certificate - {{- end }} diff --git a/charts/elasticsearch-ephemeral/values.yaml b/charts/elasticsearch-ephemeral/values.yaml deleted file mode 100644 index 1543bd897fa..00000000000 --- a/charts/elasticsearch-ephemeral/values.yaml +++ /dev/null @@ -1,23 +0,0 @@ -image: - repository: elasticsearch - # Keep this aligned with the Elastic Search version in wire-server-deploy! - tag: 6.8.23 - -service: - httpPort: 9200 - transportPort: 9300 - -resources: - limits: - cpu: "2000m" - memory: "4Gi" - requests: - cpu: "250m" - memory: "500Mi" - -tls: - enabled: false - # issuerRef: .. - -secrets: - password: "changeme" diff --git a/charts/elasticsearch-external/Chart.yaml b/charts/elasticsearch-external/Chart.yaml deleted file mode 100644 index 5d817bef923..00000000000 --- a/charts/elasticsearch-external/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: Refer to elasticsearch IPs located outside kubernetes by specifying IPs manually -name: elasticsearch-external -version: 0.0.42 diff --git a/charts/elasticsearch-external/templates/endpoint.yaml b/charts/elasticsearch-external/templates/endpoint.yaml deleted file mode 100644 index 04a4b1a0417..00000000000 --- a/charts/elasticsearch-external/templates/endpoint.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# create a headless clusterIP service to create dns name "elasticsearch-external" -# and a custom endpoint, thus forwarding traffic when resolving DNS to custom IPs -kind: Service -apiVersion: v1 -metadata: - name: {{ .Chart.Name }} - labels: - app: {{ .Chart.Name }} - chart: {{ template "elasticsearch-external.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - type: ClusterIP - clusterIP: None # headless service - ports: - - name: http - port: {{ .Values.portHttp }} - targetPort: {{ .Values.portHttp }} ---- -kind: Endpoints -apiVersion: v1 -metadata: - name: {{ .Chart.Name }} - labels: - app: {{ .Chart.Name }} - chart: {{ template "elasticsearch-external.chart" . }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -subsets: - - addresses: - {{- range .Values.IPs }} - - ip: {{ . }} - {{- end }} - ports: - # port and name in the endpoint must match port and name in the service - # see also https://docs.openshift.com/enterprise/3.0/dev_guide/integrating_external_services.html - - name: http - port: {{ .Values.portHttp }} diff --git a/charts/elasticsearch-external/templates/helpers.tpl b/charts/elasticsearch-external/templates/helpers.tpl deleted file mode 100644 index 8c545ceb55b..00000000000 --- a/charts/elasticsearch-external/templates/helpers.tpl +++ /dev/null @@ -1,11 +0,0 @@ -{{- define "elasticsearch-external.fullname" -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "elasticsearch-external.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} diff --git a/charts/elasticsearch-external/values.yaml b/charts/elasticsearch-external/values.yaml deleted file mode 100644 index b6f296f6d3f..00000000000 --- a/charts/elasticsearch-external/values.yaml +++ /dev/null @@ -1,6 +0,0 @@ -portHttp: 9200 - -## Configure this helm chart with: -# IPs: -# - 1.2.3.4 -# - 5.6.7.8 diff --git a/charts/elasticsearch-index/.helmignore b/charts/elasticsearch-index/.helmignore deleted file mode 100644 index f0c13194444..00000000000 --- a/charts/elasticsearch-index/.helmignore +++ /dev/null @@ -1,21 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*~ -# Various IDEs -.project -.idea/ -*.tmproj diff --git a/charts/elasticsearch-index/Chart.yaml b/charts/elasticsearch-index/Chart.yaml deleted file mode 100644 index 92624ae7262..00000000000 --- a/charts/elasticsearch-index/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: Elasticsearch index for brig -name: elasticsearch-index -version: 0.0.42 diff --git a/charts/elasticsearch-index/templates/_helpers.tpl b/charts/elasticsearch-index/templates/_helpers.tpl deleted file mode 100644 index a3581b09d50..00000000000 --- a/charts/elasticsearch-index/templates/_helpers.tpl +++ /dev/null @@ -1,54 +0,0 @@ - -{{/* Allow KubeVersion to be overridden. */}} -{{- define "kubeVersion" -}} - {{- default .Capabilities.KubeVersion.Version .Values.kubeVersionOverride -}} -{{- end -}} - -{{- define "includeSecurityContext" -}} - {{- (semverCompare ">= 1.24-0" (include "kubeVersion" .)) -}} -{{- end -}} - -{{- define "useCassandraTLS" -}} -{{ or (hasKey .cassandra "tlsCa") (hasKey .cassandra "tlsCaSecretRef") }} -{{- end -}} - -{{/* Return a Dict of TLS CA secret name and key -This is used to switch between provided secret (e.g. by cert-manager) and -created one (in case the CA is provided as PEM string.) -*/}} - -{{- define "cassandraTlsSecretName" -}} -{{- if .cassandra.tlsCaSecretRef -}} -{{ .cassandra.tlsCaSecretRef.name }} -{{- else }} -{{- print "elasticsearch-index-migrate-cassandra-client-ca" -}} -{{- end -}} -{{- end -}} - -{{- define "cassandraTlsSecretKey" -}} -{{- if .cassandra.tlsCaSecretRef -}} -{{ .cassandra.tlsCaSecretRef.key }} -{{- else }} -{{- print "ca.pem" -}} -{{- end -}} -{{- end -}} - -{{- define "configureElasticsearchCa" -}} -{{ or (hasKey .elasticsearch "tlsCa") (hasKey .elasticsearch "tlsCaSecretRef") }} -{{- end -}} - -{{- define "elasticsearchTlsSecretName" -}} -{{- if .elasticsearch.tlsCaSecretRef -}} -{{ .elasticsearch.tlsCaSecretRef.name }} -{{- else }} -{{- printf "%s-ca" (include "fullname" .) -}} -{{- end -}} -{{- end -}} - -{{- define "elasticsearchTlsSecretKey" -}} -{{- if .elasticsearch.tlsCaSecretRef -}} -{{ .elasticsearch.tlsCaSecretRef.key }} -{{- else }} -{{- print "ca.pem" -}} -{{- end -}} -{{- end -}} diff --git a/charts/elasticsearch-index/templates/cassandra-secret.yaml b/charts/elasticsearch-index/templates/cassandra-secret.yaml deleted file mode 100644 index 93486dd962a..00000000000 --- a/charts/elasticsearch-index/templates/cassandra-secret.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if not (empty .Values.cassandra.tlsCa) }} -apiVersion: v1 -kind: Secret -metadata: - name: elasticsearch-index-migrate-cassandra-client-ca - labels: - app: elasticsearch-index-migrate-data - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -type: Opaque -data: - ca.pem: {{ .Values.cassandra.tlsCa | b64enc | quote }} -{{- end }} diff --git a/charts/elasticsearch-index/templates/create-index.yaml b/charts/elasticsearch-index/templates/create-index.yaml deleted file mode 100644 index 225ecf82c9b..00000000000 --- a/charts/elasticsearch-index/templates/create-index.yaml +++ /dev/null @@ -1,118 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: elasticsearch-index-create - labels: - app: elasticsearch-index-create - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-delete-policy": "before-hook-creation" -spec: - template: - metadata: - name: "{{.Release.Name}}" - labels: - app: elasticsearch-index-create - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - spec: - restartPolicy: OnFailure - {{- if or (eq (include "configureElasticsearchCa" .Values) "true") (hasKey .Values.secrets "elasticsearch") }} - volumes: - {{- if hasKey .Values.secrets "elasticsearch" }} - - name: elasticsearch-index-secrets - secret: - secretName: elasticsearch-index - {{- end }} - {{- if eq (include "configureElasticsearchCa" .Values) "true" }} - - name: elasticsearch-ca - secret: - secretName: {{ include "elasticsearchTlsSecretName" .Values }} - {{- end }} - {{- end }} - initContainers: - # Creates index in elasticsearch only when it doesn't exist. - # Does nothing if the index exists. - - name: brig-index-create - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- if or (eq (include "configureElasticsearchCa" .Values) "true") (hasKey .Values.secrets "elasticsearch") }} - volumeMounts: - {{- if hasKey .Values.secrets "elasticsearch" }} - - name: "elasticsearch-index-secrets" - mountPath: "/etc/wire/elasticsearch-index/secrets" - {{- end }} - - {{- if eq (include "configureElasticsearchCa" .Values) "true" }} - - name: elasticsearch-ca - mountPath: "/certs/elasticsearch" - {{- end }} - {{- end }} - {{- if eq (include "includeSecurityContext" .) "true" }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 12 }} - {{- end }} - args: - - create - - --elasticsearch-server - - "{{ .Values.elasticsearch.scheme }}://{{ required "missing elasticsearch-index.elasticsearch.host!" .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port }}" - {{- if hasKey .Values.secrets "elasticsearch" }} - - --elasticsearch-credentials - - "/etc/wire/elasticsearch-index/secrets/elasticsearch-credentials.yaml" - {{- end }} - - --elasticsearch-index - - "{{ or (.Values.elasticsearch.additionalWriteIndex) (.Values.elasticsearch.index) }}" - - --elasticsearch-shards=5 - - --elasticsearch-replicas=2 - - --elasticsearch-refresh-interval=5 - {{- if .Values.elasticsearch.delete_template }} - - --delete-template - - "{{ .Values.elasticsearch.delete_template }}" - {{- end }} - {{- if eq (include "configureElasticsearchCa" .Values) "true" }} - - --elasticsearch-ca-cert - - /certs/elasticsearch/{{- include "elasticsearchTlsSecretKey" .Values}} - {{- end }} - {{- if .Values.elasticsearch.insecureSkipTlsVerify }} - - --elasticsearch-insecure-skip-tls-verify - {{- end }} - containers: - - name: brig-index-update-mapping - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }} - {{- if or (eq (include "configureElasticsearchCa" .Values) "true") (hasKey .Values.secrets "elasticsearch") }} - volumeMounts: - {{- if hasKey .Values.secrets "elasticsearch" }} - - name: "elasticsearch-index-secrets" - mountPath: "/etc/wire/elasticsearch-index/secrets" - {{- end }} - - {{- if eq (include "configureElasticsearchCa" .Values) "true" }} - - name: elasticsearch-ca - mountPath: "/certs/elasticsearch" - {{- end }} - {{- end }} - {{- if eq (include "includeSecurityContext" .) "true" }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 12 }} - {{- end }} - args: - - update-mapping - - --elasticsearch-server - - "{{ .Values.elasticsearch.scheme }}://{{ required "missing elasticsearch-index.elasticsearch.host!" .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port }}" - {{- if hasKey .Values.secrets "elasticsearch" }} - - --elasticsearch-credentials - - "/etc/wire/elasticsearch-index/secrets/elasticsearch-credentials.yaml" - {{- end }} - - --elasticsearch-index - - "{{ or (.Values.elasticsearch.additionalWriteIndex) (.Values.elasticsearch.index) }}" - {{- if eq (include "configureElasticsearchCa" .Values) "true" }} - - --elasticsearch-ca-cert - - /certs/elasticsearch/{{- include "elasticsearchTlsSecretKey" .Values}} - {{- end }} - {{- if .Values.elasticsearch.insecureSkipTlsVerify }} - - --elasticsearch-insecure-skip-tls-verify - {{- end }} diff --git a/charts/elasticsearch-index/templates/elasticsearch-ca-secret.yaml b/charts/elasticsearch-index/templates/elasticsearch-ca-secret.yaml deleted file mode 100644 index eef7f10de60..00000000000 --- a/charts/elasticsearch-index/templates/elasticsearch-ca-secret.yaml +++ /dev/null @@ -1,14 +0,0 @@ -{{- if not (empty .Values.elasticsearch.tlsCa) }} -apiVersion: v1 -kind: Secret -metadata: - name: "{{ include "fullname" . }}-ca" - labels: - app: elasticsearch-index - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -type: Opaque -data: - ca.pem: {{ .Values.elasticsearch.tlsCa | b64enc | quote }} -{{- end }} diff --git a/charts/elasticsearch-index/templates/helpers.tpl b/charts/elasticsearch-index/templates/helpers.tpl deleted file mode 100644 index fb1ddbd472e..00000000000 --- a/charts/elasticsearch-index/templates/helpers.tpl +++ /dev/null @@ -1,7 +0,0 @@ -{{/* -override default fullname template to remove the .Release.Name from the definition -*/}} -{{- define "fullname" -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- printf "%s" $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} \ No newline at end of file diff --git a/charts/elasticsearch-index/templates/migrate-data.yaml b/charts/elasticsearch-index/templates/migrate-data.yaml deleted file mode 100644 index bb59d83ff6d..00000000000 --- a/charts/elasticsearch-index/templates/migrate-data.yaml +++ /dev/null @@ -1,108 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: brig-index-migrate-data - labels: - app: elasticsearch-index-migrate-data - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-delete-policy": "before-hook-creation" -spec: - template: - metadata: - name: "{{.Release.Name}}" - labels: - app: elasticsearch-index-migrate-data - heritage: {{.Release.Service | quote }} - release: {{.Release.Name | quote }} - chart: "{{.Chart.Name}}-{{.Chart.Version}}" - spec: - restartPolicy: OnFailure - containers: - # Reindexes all users when a new migration is detected. - - name: brig-index - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }} - args: - - migrate-data - - --elasticsearch-server - - "{{ .Values.elasticsearch.scheme }}://{{ required "missing elasticsearch-index.elasticsearch.host!" .Values.elasticsearch.host }}:{{ .Values.elasticsearch.port }}" - {{- if hasKey .Values.secrets "elasticsearch" }} - - --elasticsearch-credentials - - "/etc/wire/elasticsearch-index/secrets/elasticsearch-credentials.yaml" - {{- end }} - - --elasticsearch-index - - "{{ or (.Values.elasticsearch.additionalWriteIndex) (.Values.elasticsearch.index) }}" - - --cassandra-host - - "{{ required "missing elasticsearch-index.cassandra.host!" .Values.cassandra.host }}" - - --cassandra-port - - "{{ required "missing elasticsearch-index.cassandra.port!" .Values.cassandra.port }}" - - --cassandra-keyspace - - "{{ required "missing elasticsearch-index.cassandra.keyspace!" .Values.cassandra.keyspace }}" - - --galley-host - - "{{ required "missing elasticsearch-index.galley.host!" .Values.galley.host }}" - - --galley-port - - "{{ required "missing elasticsearch-index.galley.port!" .Values.galley.port }}" - {{- if eq (include "useCassandraTLS" .Values) "true" }} - - --cassandra-ca-cert - - /certs/cassandra/{{- include "cassandraTlsSecretKey" .Values }} - {{- end }} - {{- if eq (include "configureElasticsearchCa" .Values) "true" }} - - --elasticsearch-ca-cert - - /certs/elasticsearch/{{- include "elasticsearchTlsSecretKey" .Values}} - {{- end }} - {{- if .Values.elasticsearch.insecureSkipTlsVerify }} - - --elasticsearch-insecure-skip-tls-verify - {{- end }} - - --pg-pool-size - - {{ .Values.postgresqlPool.size | quote }} - - --pg-pool-acquisition-timeout - - {{ .Values.postgresqlPool.acquisitionTimeout | quote }} - - --pg-pool-idleness-timeout - - {{ .Values.postgresqlPool.idlenessTimeout | quote }} - {{- if hasKey $.Values.secrets "pgPassword" }} - - --pg-password-file - - /etc/wire/elasticsearch-index/secrets/pgPassword - {{- end }} - - --pg-settings - - {{ toJson .Values.postgresql | quote }} - - --user-storage-location - - {{ .Values.postgresMigration.user }} - volumeMounts: - {{- if hasKey .Values.secrets "elasticsearch" }} - - name: "elasticsearch-index-secrets" - mountPath: "/etc/wire/elasticsearch-index/secrets" - {{- end }} - {{- if eq (include "useCassandraTLS" .Values) "true" }} - - name: elasticsearch-index-migrate-cassandra-client-ca - mountPath: "/certs/cassandra" - {{- end }} - {{- if eq (include "configureElasticsearchCa" .Values) "true" }} - - name: elasticsearch-ca - mountPath: "/certs/elasticsearch" - {{- end }} - {{- if .Values.migrateData.additionalVolumeMounts }} - {{ toYaml .Values.migrateData.additionalVolumeMounts | nindent 10 }} - {{- end }} - volumes: - {{- if hasKey .Values.secrets "elasticsearch" }} - - name: elasticsearch-index-secrets - secret: - secretName: elasticsearch-index - {{- end }} - {{- if eq (include "useCassandraTLS" .Values) "true" }} - - name: elasticsearch-index-migrate-cassandra-client-ca - secret: - secretName: {{ include "cassandraTlsSecretName" .Values }} - {{- end }} - {{- if eq (include "configureElasticsearchCa" .Values) "true" }} - - name: elasticsearch-ca - secret: - secretName: {{ include "elasticsearchTlsSecretName" .Values }} - {{- end }} - {{- if .Values.migrateData.additionalVolumes }} - {{ toYaml .Values.migrateData.additionalVolumes | nindent 8 }} - {{- end }} diff --git a/charts/elasticsearch-index/templates/secret.yaml b/charts/elasticsearch-index/templates/secret.yaml deleted file mode 100644 index a88a8f6a8bc..00000000000 --- a/charts/elasticsearch-index/templates/secret.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if hasKey .Values.secrets "elasticsearch" }} -apiVersion: v1 -kind: Secret -metadata: - name: elasticsearch-index - labels: - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" - annotations: - "helm.sh/hook": pre-install,pre-upgrade - "helm.sh/hook-delete-policy": "before-hook-creation" -type: Opaque -data: - {{- with .Values.secrets }} - {{- if .elasticsearch }} - elasticsearch-credentials.yaml: {{ .elasticsearch | toYaml | b64enc }} - {{- end }} - {{- if .pgPassword }} - pgPassword: {{ .pgPassword | b64enc | quote }} - {{- end }} - {{- end }} -{{- end }} diff --git a/charts/elasticsearch-index/values.yaml b/charts/elasticsearch-index/values.yaml deleted file mode 100644 index b12673ed181..00000000000 --- a/charts/elasticsearch-index/values.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Default values for elasticsearch-index -elasticsearch: - scheme: http - #host: # elasticsearch-client|elasticsearch-ephemeral - port: 9200 - index: directory - delete_template: directory -# To enable TLS verification with a custom CA: -# tlsCa: -# -# Or refer to an existing secret (containing the CA): -# tlsCaSecretRef: -# name: -# key: - insecureSkipTlsVerify: false - -cassandra: - # host: - port: 9042 - keyspace: brig - # To enable TLS provide a CA: - # tlsCa: - # - # Or refer to an existing secret (containing the CA): - # tlsCaSecretRef: - # name: - # key: - -# Postgres connection settings -# -# Values are described in https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS -# To set the password via a brig secret see `secrets.pgPassword`. -# -# `additionalVolumeMounts` and `additionalVolumes` under `migrateData` can be used to mount -# additional files (e.g. certificates) into the brig-index-migrate-data container. This way -# does not work for password files (parameter `passfile`), because -# libpq-connect requires access rights (mask 0600) for them that we cannot -# provide for random uids. -# -# Below is an example configuration we're using for our CI tests. -postgresql: - host: postgresql # DNS name without protocol - port: "5432" - user: wire-server - dbname: wire-server -postgresqlPool: - size: 100 - acquisitionTimeout: 10s - idlenessTimeout: 10m - -postgresMigration: - user: cassandra - -galley: - host: galley - port: 8080 - -image: - repository: quay.io/wire/brig-index - tag: do-not-use - -podSecurityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - -migrateData: - additionalVolumes: [] - additionalVolumeMounts: [] - -secrets: {} diff --git a/charts/fluent-bit/values.yaml b/charts/fluent-bit/values.yaml index 26fe5d5dd02..a4e8dd181fb 100644 --- a/charts/fluent-bit/values.yaml +++ b/charts/fluent-bit/values.yaml @@ -2,26 +2,6 @@ fluent-bit: config: outputs: | - [OUTPUT] - Name es - Match kube.* - Host elasticsearch-ephemeral - Generate_ID On - Logstash_Format On - Logstash_Prefix pod - Retry_Limit False - Trace_Error On - Replace_Dots On - [OUTPUT] - Name es - Match host.* - Host elasticsearch-ephemeral - Generate_ID On - Logstash_Format On - Logstash_Prefix node - Retry_Limit False - Trace_Error On - Replace_Dots On # syslog output reference - https://docs.fluentbit.io/manual/pipeline/outputs/syslog # Uncomment this section to enable syslog output # [OUTPUT] diff --git a/charts/integration/templates/integration-integration.yaml b/charts/integration/templates/integration-integration.yaml index 4841ef372b7..0d3f966ef93 100644 --- a/charts/integration/templates/integration-integration.yaml +++ b/charts/integration/templates/integration-integration.yaml @@ -93,11 +93,6 @@ spec: secret: secretName: "nginz" - - name: elasticsearch-ca - secret: - secretName: {{ .Values.config.elasticsearch.tlsCaSecretRef.name }} - - - name: rabbitmq-ca secret: secretName: {{ .Values.config.rabbitmq.tlsCaSecretRef.name }} @@ -122,8 +117,6 @@ spec: {{- toYaml .Values.podSecurityContext | nindent 6 }} {{- end }} volumeMounts: - - name: elasticsearch-ca - mountPath: "/certs/elasticsearch" {{- if eq (include "useCassandraTLS" .Values.config) "true" }} - name: "integration-cassandra" mountPath: "/certs/cassandra" @@ -163,9 +156,6 @@ spec: --tls-ca-certificate-file /certs/cassandra/{{- include "cassandraTlsSecretKey" .Values.config }} {{ end }} - integration-dynamic-backends-brig-index.sh \ - --elasticsearch-server https://elastic:changeme@{{ .Values.config.elasticsearch.host }}:9200 \ - --elasticsearch-ca-cert /certs/elasticsearch/{{ .Values.config.elasticsearch.tlsCaSecretRef.key }} integration-dynamic-backends-ses.sh {{ .Values.config.sesEndpointUrl }} integration-dynamic-backends-s3.sh {{ .Values.config.s3EndpointUrl }} {{- range $name, $dynamicBackend := .Values.config.dynamicBackends }} @@ -277,9 +267,6 @@ spec: - name: nginz-secrets mountPath: /etc/wire/nginz/secrets - - name: elasticsearch-ca - mountPath: /etc/wire/brig/elasticsearch-ca - - name: rabbitmq-ca mountPath: /etc/wire/brig/rabbitmq-ca diff --git a/charts/integration/values.yaml b/charts/integration/values.yaml index 65c7963b0d8..1373f54a90d 100644 --- a/charts/integration/values.yaml +++ b/charts/integration/values.yaml @@ -112,9 +112,6 @@ config: port: 9042 replicationFactor: 1 - elasticsearch: - host: elasticsearch-ephemeral - sqsEndpointUrl: http://fake-aws-sqs:4568 sesEndpointUrl: http://fake-aws-ses:4569 s3EndpointUrl: http://fake-aws-s3:9000 diff --git a/charts/kibana/Chart.yaml b/charts/kibana/Chart.yaml deleted file mode 100644 index 5a6f2dc00dc..00000000000 --- a/charts/kibana/Chart.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -description: Wrapper chart for stable/kibana -name: kibana -version: 0.0.42 diff --git a/charts/kibana/requirements.yaml b/charts/kibana/requirements.yaml deleted file mode 100644 index 53ccd8b99bb..00000000000 --- a/charts/kibana/requirements.yaml +++ /dev/null @@ -1,4 +0,0 @@ -dependencies: -- name: kibana - version: 6.8.18 - repository: https://helm.elastic.co diff --git a/charts/kibana/templates/basic-auth-secret.yaml b/charts/kibana/templates/basic-auth-secret.yaml deleted file mode 100644 index fef8ca38041..00000000000 --- a/charts/kibana/templates/basic-auth-secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if (hasKey .Values "basicAuthSecret") }} -apiVersion: v1 -kind: Secret -metadata: - name: kibana-basic-auth - labels: - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -type: Opaque -data: - auth: {{ .Values.basicAuthSecret | b64enc | quote }} -{{- end }} diff --git a/charts/kibana/values.yaml b/charts/kibana/values.yaml deleted file mode 100644 index 41789450124..00000000000 --- a/charts/kibana/values.yaml +++ /dev/null @@ -1,22 +0,0 @@ -## When this is configured, a secret called kibana-basic-auth is created with key -## `auth` and value of this key. -# basicAuthSecret: - -# See defaults in https://github.com/elastic/helm-charts/tree/main/kibana -kibana: - elasticsearchHosts: "http://elasticsearch-ephemeral:9200" - - lifecycle: - postStart: - exec: - command: - - bash - - -c - - | - #!/bin/bash - KB_URL=http://localhost:5601 - # Wait for kibana to be ready - while [[ "$(curl -s -o /dev/null -w '%{http_code}\n' -L $KB_URL)" != "200" ]]; do sleep 1; done - # Import index patterns for pods logs and node logs, for kibana <7, - # we have to use the dashboard import API. - curl -XPOST "$KB_URL/api/kibana/dashboards/import" -H "Content-Type: application/json" -H 'kbn-xsrf: true' -d'{"objects":[{"type": "index-pattern", "id": "7e7061cc-7d7e-4287-b631-a7c5257f73e5", "attributes": {"title": "pod-*", "timeFieldName": "@timestamp"}},{"type": "index-pattern", "id": "b1a2866f-70ec-40fb-bfea-d78e9662b741", "attributes": {"title": "node-*", "timeFieldName": "@timestamp"}}]}' diff --git a/charts/wire-server/requirements.yaml b/charts/wire-server/requirements.yaml index fc10d8fa195..469ab615dc2 100644 --- a/charts/wire-server/requirements.yaml +++ b/charts/wire-server/requirements.yaml @@ -7,11 +7,6 @@ dependencies: repository: "file://../cassandra-migrations" tags: - cassandra-migrations -- name: elasticsearch-index - version: "0.0.42" - repository: "file://../elasticsearch-index" - tags: - - elasticsearch-index ######################## ## wire-servers/services ######################## diff --git a/charts/wire-server/templates/_helpers.tpl b/charts/wire-server/templates/_helpers.tpl index 94aca197dc7..63a82060bad 100644 --- a/charts/wire-server/templates/_helpers.tpl +++ b/charts/wire-server/templates/_helpers.tpl @@ -48,46 +48,6 @@ {{- end -}} {{- end -}} -{{- define "brig.configureElasticSearchCa" -}} -{{ or (hasKey .elasticsearch "tlsCa") (hasKey .elasticsearch "tlsCaSecretRef") }} -{{- end -}} - -{{- define "brig.elasticsearchTlsSecretName" -}} -{{- if .elasticsearch.tlsCaSecretRef -}} -{{ .elasticsearch.tlsCaSecretRef.name }} -{{- else }} -{{- print "brig-elasticsearch-ca" -}} -{{- end -}} -{{- end -}} - -{{- define "brig.elasticsearchTlsSecretKey" -}} -{{- if .elasticsearch.tlsCaSecretRef -}} -{{ .elasticsearch.tlsCaSecretRef.key }} -{{- else }} -{{- print "ca.pem" -}} -{{- end -}} -{{- end -}} - -{{- define "brig.configureAdditionalElasticSearchCa" -}} -{{ or (hasKey .elasticsearch "additionalTlsCa") (hasKey .elasticsearch "additionalTlsCaSecretRef") }} -{{- end -}} - -{{- define "brig.additionalElasticsearchTlsSecretName" -}} -{{- if .elasticsearch.additionalTlsCaSecretRef -}} -{{ .elasticsearch.additionalTlsCaSecretRef.name }} -{{- else }} -{{- print "brig-additional-elasticsearch-ca" -}} -{{- end -}} -{{- end -}} - -{{- define "brig.additionalElasticsearchTlsSecretKey" -}} -{{- if .elasticsearch.additionalTlsCaSecretRef -}} -{{ .elasticsearch.additionalTlsCaSecretRef.key }} -{{- else }} -{{- print "ca.pem" -}} -{{- end -}} -{{- end -}} - {{/* CANNON */}} {{- define "cannon.tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} diff --git a/charts/wire-server/templates/brig/configmap.yaml b/charts/wire-server/templates/brig/configmap.yaml index 7a43c8244ce..321f1f8634b 100644 --- a/charts/wire-server/templates/brig/configmap.yaml +++ b/charts/wire-server/templates/brig/configmap.yaml @@ -39,30 +39,6 @@ data: {{- end }} postgresMigration: {{- toYaml $.Values.galley.config.postgresMigration | nindent 6 }} - elasticsearch: - url: {{ .elasticsearch.scheme }}://{{ .elasticsearch.host }}:{{ .elasticsearch.port }} - index: {{ .elasticsearch.index }} - {{- if .elasticsearch.additionalWriteHost }} - additionalWriteIndexUrl: {{ .elasticsearch.additionalWriteScheme }}://{{ .elasticsearch.additionalWriteHost }}:{{ .elasticsearch.additionalWritePort }} - {{- end }} - {{- if .elasticsearch.additionalWriteIndex }} - additionalWriteIndex: {{ .elasticsearch.additionalWriteIndex }} - {{- end }} - {{- if $.Values.brig.secrets.elasticsearch }} - credentials: /etc/wire/brig/secrets/elasticsearch-credentials.yaml - {{- end }} - {{- if eq (include "brig.configureElasticSearchCa" .) "true" }} - caCert: /etc/wire/brig/elasticsearch-ca/{{ include "brig.elasticsearchTlsSecretKey" .}} - {{- end }} - {{- if eq (include "brig.configureAdditionalElasticSearchCa" .) "true" }} - additionalCaCert: /etc/wire/brig/additional-elasticsearch-ca/{{ include "brig.additionalElasticsearchTlsSecretKey" .}} - {{- end }} - {{- if $.Values.brig.secrets.elasticsearchAdditional }} - additionalCredentials: /etc/wire/brig/secrets/elasticsearch-additional-credentials.yaml - {{- end }} - insecureSkipVerifyTls: {{ .elasticsearch.insecureSkipVerifyTls }} - additionalInsecureSkipVerifyTls: {{ .elasticsearch.additionalInsecureSkipVerifyTls }} - cargohold: host: cargohold port: 8080 diff --git a/charts/wire-server/templates/brig/deployment.yaml b/charts/wire-server/templates/brig/deployment.yaml index cbf6b56ff75..37ae5429022 100644 --- a/charts/wire-server/templates/brig/deployment.yaml +++ b/charts/wire-server/templates/brig/deployment.yaml @@ -55,16 +55,6 @@ spec: secret: secretName: {{ (include "brig.tlsSecretRef" .Values.brig.config | fromYaml).name }} {{- end}} - {{- if eq (include "brig.configureElasticSearchCa" .Values.brig.config) "true" }} - - name: "elasticsearch-ca" - secret: - secretName: {{ include "brig.elasticsearchTlsSecretName" .Values.brig.config }} - {{- end }} - {{- if eq (include "brig.configureAdditionalElasticSearchCa" .Values.brig.config) "true" }} - - name: "additional-elasticsearch-ca" - secret: - secretName: {{ include "brig.additionalElasticsearchTlsSecretName" .Values.brig.config }} - {{- end }} {{- if and .Values.brig.config.rabbitmq .Values.brig.config.rabbitmq.tlsCaSecretRef }} - name: "rabbitmq-ca" secret: @@ -94,14 +84,6 @@ spec: - name: "brig-cassandra" mountPath: "/etc/wire/brig/cassandra" {{- end }} - {{- if eq (include "brig.configureElasticSearchCa" .Values.brig.config) "true" }} - - name: "elasticsearch-ca" - mountPath: "/etc/wire/brig/elasticsearch-ca/" - {{- end }} - {{- if eq (include "brig.configureAdditionalElasticSearchCa" .Values.brig.config) "true" }} - - name: "additional-elasticsearch-ca" - mountPath: "/etc/wire/brig/additional-elasticsearch-ca/" - {{- end }} {{- if and .Values.brig.config.rabbitmq .Values.brig.config.rabbitmq.tlsCaSecretRef }} - name: "rabbitmq-ca" mountPath: "/etc/wire/brig/rabbitmq-ca/" diff --git a/charts/wire-server/templates/brig/elasticsearch-ca-secret.yaml b/charts/wire-server/templates/brig/elasticsearch-ca-secret.yaml deleted file mode 100644 index cc080f7c7c9..00000000000 --- a/charts/wire-server/templates/brig/elasticsearch-ca-secret.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -{{- if not (empty .Values.brig.config.elasticsearch.tlsCa) }} -apiVersion: v1 -kind: Secret -metadata: - name: "brig-elasticsearch-ca" - labels: - app: brig - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -type: Opaque -data: - ca.pem: {{ .Values.brig.config.elasticsearch.tlsCa | b64enc | quote }} -{{- end }} ---- -{{- if not (empty .Values.brig.config.elasticsearch.additionalTlsCa) }} -apiVersion: v1 -kind: Secret -metadata: - name: "brig-additional-elasticsearch-ca" - labels: - app: brig - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: "{{ .Release.Name }}" - heritage: "{{ .Release.Service }}" -type: Opaque -data: - ca.pem: {{ .Values.brig.config.elasticsearch.additionalTlsCa | b64enc | quote }} -{{- end }} diff --git a/charts/wire-server/templates/brig/secret.yaml b/charts/wire-server/templates/brig/secret.yaml index 5175276fda1..e35cebc888f 100644 --- a/charts/wire-server/templates/brig/secret.yaml +++ b/charts/wire-server/templates/brig/secret.yaml @@ -34,12 +34,6 @@ data: {{- end }} rabbitmqUsername: {{ .rabbitmq.username | b64enc | quote }} rabbitmqPassword: {{ .rabbitmq.password | b64enc | quote }} - {{- if .elasticsearch }} - elasticsearch-credentials.yaml: {{ .elasticsearch | toYaml | b64enc }} - {{- end }} - {{- if .elasticsearchAdditional }} - elasticsearch-additional-credentials.yaml: {{ .elasticsearchAdditional | toYaml | b64enc }} - {{- end }} {{- if .pgPassword }} pgPassword: {{ .pgPassword | b64enc | quote }} {{- end }} diff --git a/charts/wire-server/templates/brig/tests/brig-integration.yaml b/charts/wire-server/templates/brig/tests/brig-integration.yaml index 079e71b20c5..2ac64faa968 100644 --- a/charts/wire-server/templates/brig/tests/brig-integration.yaml +++ b/charts/wire-server/templates/brig/tests/brig-integration.yaml @@ -44,11 +44,6 @@ spec: - name: "brig-integration-secrets" secret: secretName: "brig-integration" - {{- if eq (include "brig.configureElasticSearchCa" .Values.brig.config) "true" }} - - name: elasticsearch-ca - secret: - secretName: {{ include "brig.elasticsearchTlsSecretName" .Values.brig.config }} - {{- end }} {{- if eq (include "useCassandraTLS" .Values.brig.config.cassandra) "true" }} - name: "brig-cassandra" secret: @@ -116,10 +111,6 @@ spec: # non-default locations # (see corresp. TODO in galley.) mountPath: "/etc/wire/integration-secrets" - {{- if eq (include "brig.configureElasticSearchCa" .Values.brig.config) "true" }} - - name: elasticsearch-ca - mountPath: "/etc/wire/brig/elasticsearch-ca" - {{- end }} {{- if eq (include "useCassandraTLS" .Values.brig.config.cassandra) "true" }} - name: "brig-cassandra" mountPath: "/etc/wire/brig/cassandra" diff --git a/charts/wire-server/templates/brig/tests/configmap.yaml b/charts/wire-server/templates/brig/tests/configmap.yaml index b327f32dbde..bbeb4325032 100644 --- a/charts/wire-server/templates/brig/tests/configmap.yaml +++ b/charts/wire-server/templates/brig/tests/configmap.yaml @@ -89,5 +89,3 @@ data: federatorExternal: host: federator.{{ .Release.Namespace }}-fed2.svc.cluster.local port: 8081 - - additionalElasticSearch: https://{{ .Values.brig.test.elasticsearch.additionalHost }}:9200 diff --git a/charts/wire-server/values.yaml b/charts/wire-server/values.yaml index 9a98be5d528..6438467fc6a 100644 --- a/charts/wire-server/values.yaml +++ b/charts/wire-server/values.yaml @@ -1111,31 +1111,6 @@ brig: # name: # key: - elasticsearch: - scheme: http - host: elasticsearch-client - port: 9200 - index: directory - insecureSkipVerifyTls: false - # To configure custom TLS CA, please provide one of these: - # tlsCa: - # - # Or refer to an existing secret (containing the CA): - # tlsCaSecretRef: - # name: - # key: - additionalWriteScheme: http - # additionalWriteHost: - additionalWritePort: 9200 - # additionalWriteIndex: - additionalInsecureSkipVerifyTls: false - # To configure custom TLS CA, please provide one of these: - # additionalTlsCa: - # - # Or refer to an existing secret (containing the CA): - # additionalTlsCaSecretRef: - # name: - # key: aws: region: "eu-west-1" sesEndpoint: https://email.eu-west-1.amazonaws.com @@ -1383,6 +1358,3 @@ brig: -----END CERTIFICATE----- # pgPassword: - test: - elasticsearch: - additionalHost: elasticsearch-ephemeral diff --git a/deploy/dockerephemeral/db-migrate/brig-index.sh b/deploy/dockerephemeral/db-migrate/brig-index.sh deleted file mode 100755 index 81a459292be..00000000000 --- a/deploy/dockerephemeral/db-migrate/brig-index.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh - -until_ready() { - cmd=$1 - until $cmd; do echo 'service not ready yet'; sleep 5; done - return 0 -} - -until_ready "brig-index reset --elasticsearch-server http://elasticsearch:9200" diff --git a/deploy/dockerephemeral/docker-compose.yaml b/deploy/dockerephemeral/docker-compose.yaml index a8a9ab66d5c..426926896fc 100644 --- a/deploy/dockerephemeral/docker-compose.yaml +++ b/deploy/dockerephemeral/docker-compose.yaml @@ -72,77 +72,6 @@ services: networks: - demo_wire - elasticsearch: - container_name: demo_wire_elasticsearch - image: elasticsearch:6.8.23 - ulimits: - nofile: - soft: 65536 - hard: 65536 - ports: - - "127.0.0.1:9200:9200" - - "127.0.0.1:9300:9300" - environment: - - "xpack.ml.enabled=false" - - "xpack.security.enabled=true" - - "xpack.security.http.ssl.enabled=true" - - "xpack.ssl.certificate=certs/elasticsearch-cert.pem" - - "xpack.ssl.key=certs/elasticsearch-key.pem" - - "bootstrap.system_call_filter=false" - - "JVM_OPTIONS_ES=-Xmx512m -Xms512m" - - "discovery.type=single-node" - - "ELASTIC_PASSWORD=changeme" - volumes: - - ./docker/elasticsearch-cert.pem:/usr/share/elasticsearch/config/certs/elasticsearch-cert.pem - - ./docker/elasticsearch-key.pem:/usr/share/elasticsearch/config/certs/elasticsearch-key.pem - networks: - - demo_wire - - opensearch: - container_name: opensearch - image: opensearchproject/opensearch:1.3.20 - ulimits: - nofile: - soft: 65536 - hard: 65536 - ports: - - "127.0.0.1:9201:9200" - - "127.0.0.1:9301:9300" - environment: - - "bootstrap.system_call_filter=false" - - "JVM_OPTIONS_ES=-Xmx512m -Xms512m" - - "discovery.type=single-node" - - - "DISABLE_INSTALL_DEMO_CONFIG=true" - - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=Ch4ng3m3Secr3t!" - volumes: - - ./docker/elasticsearch-cert.pem:/usr/share/opensearch/config/certs/tls.crt - - ./docker/elasticsearch-key.pem:/usr/share/opensearch/config/certs/tls.key - - ./docker/elasticsearch-ca.pem:/usr/share/opensearch/config/certs/ca.crt - - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml - - ./docker/opensearch/opensearch-security/config.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/config.yml - - ./docker/opensearch/opensearch-security/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml - - ./docker/opensearch/opensearch-security/roles_mapping.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml - - ./docker/opensearch/opensearch-security/allowlist.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/allowlist.yml - - ./docker/opensearch/opensearch-security/roles.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles.yml - - ./docker/opensearch/opensearch-security/nodes_dn.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/nodes_dn.yml - - ./docker/opensearch/opensearch-security/action_groups.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/action_groups.yml - - ./docker/opensearch/opensearch-security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml - networks: - - demo_wire - - opensearch-dashboard: - image: opensearchproject/opensearch-dashboards:1 - container_name: opensearch-dashboards - ports: - - 5601:5601 - expose: - - "5601" - volumes: - - ./docker/opensearch/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml - networks: - - demo_wire - postgres: container_name: postgres image: 'postgres:17-alpine' diff --git a/deploy/dockerephemeral/docker/elasticsearch-ca.pem b/deploy/dockerephemeral/docker/elasticsearch-ca.pem deleted file mode 100644 index 6511f688a73..00000000000 --- a/deploy/dockerephemeral/docker/elasticsearch-ca.pem +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDLzCCAhegAwIBAgIUUOLn63PL3FEyGdhOK1ocDAn8dC8wDQYJKoZIhvcNAQEL -BQAwJzElMCMGA1UEAwwcZWxhc3RpY3NlYXJjaC5jYS5leGFtcGxlLmNvbTAeFw0y -NDA5MDMxMjAzMzhaFw0zNDA5MDExMjAzMzhaMCcxJTAjBgNVBAMMHGVsYXN0aWNz -ZWFyY2guY2EuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDhfp2hokSJ88qb6Gl8BBwg4jMbkt4l2ynOa/lO6DZFRheFlWZBaUZAcj1o -jBgbd3EoOXygZReL9TGBRrc5iACzytlOJkwMgNUUIq1KsPwWII41VJw3h3NO07tM -Tsf0kFvH1pEllJqorhQ1eZnU1SISyQcjk0oRQEyWd6arF7TLHna69OHF2ybYYXMD -MFWNr+O6t8RUfYs4kb7z1Nx3OnUKrUhIyaYeoyvBBOdXA5/G5GenDu/G4iVozsuL -gofYWu+77EnpProF0KRK+XlKakvF7bD26Qm1ol5qsXbdXOXKYq/c4KDdC938HNd/ -FNPz7EsnW0brBXqtz8TeQmxHix7DAgMBAAGjUzBRMB0GA1UdDgQWBBTWzp/VRTGq -s/IwlrROQGDwavTpyzAfBgNVHSMEGDAWgBTWzp/VRTGqs/IwlrROQGDwavTpyzAP -BgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBFue/SGpAVZ0TOVp8l -bdaaY/e9wUBSdCH5b39Nzd3rKmH3lIHqcafLlFCx+scKUIlFbohJr0aTK339wFfl -L0LzBJVUT9JzeDNPhk8pl/jBJk+eGP3fiykFMCgxxGvHtccHu8E/y8U0SeEtKqDn -Xy0ZbC3M54UedhDpHMovfHEsfN24Ev0DK13sBR2T8fmXCyCrfq887cCqJyP2ODgb -xAY/R4F8Ueadn0ywHYSY3MqmDsvDul0QlaOu2J5A0+k5oy4hAfFB8PzPYZrmPkeU -N5oxudTTihIZ+0JiL2JmWGBzMGzgtmD1rHC6lugUlWq+BoPu2+/+hn8RcVHBCFDk -WMSU ------END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/elasticsearch-cert.pem b/deploy/dockerephemeral/docker/elasticsearch-cert.pem deleted file mode 100755 index 99fe3c464aa..00000000000 --- a/deploy/dockerephemeral/docker/elasticsearch-cert.pem +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDMDCCAhigAwIBAgIBADANBgkqhkiG9w0BAQsFADAnMSUwIwYDVQQDDBxlbGFz -dGljc2VhcmNoLmNhLmV4YW1wbGUuY29tMB4XDTI0MDkwMzEyMDMzOFoXDTM0MDkw -MTEyMDMzOFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF -AAOCAQ8AMIIBCgKCAQEAwDpnRRMmtz2g/5RBTO7FcuOsy3ss5g3E1npSOyQ4MebM -vM3fIZd+xVPNj6ZDxIHotLS17av41YmHCEtSgu0BtOda9hvn62HBR3Gyl6TDEvvC -ptHqxc+5ttrdKc5XqpI2lJsNUOUQhwQjYyTvbGwF5YxtHGc0mtUEJ7d/1qoT7A/y -W0KxTfclo7LPPBMiIF6Qzjn5iEguVPQm7jvQs9UbHad/ffMKDEBkBNs5joYTzLil -NFrnxxQKpxf4qS/cA62zeBS2dVvgZHqNeHfOF+v1DHItB/uoB9zRKFPvArI+Y+RR -uPsXuPNZgN210V55iDyDdU31Eh78ndShCSg6X3rfgQIDAQABo3oweDAdBgNVHSUE -FjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwFwYDVR0RAQH/BA0wC4IJbG9jYWxob3N0 -MB0GA1UdDgQWBBTxqOE025egTOExeT139PqO6dxGKDAfBgNVHSMEGDAWgBTWzp/V -RTGqs/IwlrROQGDwavTpyzANBgkqhkiG9w0BAQsFAAOCAQEAWyJLKHLcz3oKVZnH -KP7AR0ty0m9H4yeHVPT7/IjfUsemDkFhk9xcSHlqVEqNu7CHL/VjZ6wke79yGm4L -zBIqiTGKgHmFRTn+19bNg/K/IodAXaTWayEzAwrJmEU2W6aarxhL6IiyHHnDba2J -u9h/cVV2OGODdg3+QuEr/3UV5XQX6X3hVGa3YUb/sTt1tuj4Rs9e1UCoSL2+4NtM -20De1G5zF0z05SP5z9H98sryf69PysJjmSWc601S4iR22o2nGDA/JrPBnVHfL0Cj -Aah5YYqY4m2llOwTGTrQdrzX2Oe2Qwcm1ofmn0P8Y4uYvqg9sUXKR1yf92PjytoE -/ZIJPA== ------END CERTIFICATE----- diff --git a/deploy/dockerephemeral/docker/elasticsearch-key.pem b/deploy/dockerephemeral/docker/elasticsearch-key.pem deleted file mode 100755 index b4346d3579c..00000000000 --- a/deploy/dockerephemeral/docker/elasticsearch-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDAOmdFEya3PaD/ -lEFM7sVy46zLeyzmDcTWelI7JDgx5sy8zd8hl37FU82PpkPEgei0tLXtq/jViYcI -S1KC7QG051r2G+frYcFHcbKXpMMS+8Km0erFz7m22t0pzleqkjaUmw1Q5RCHBCNj -JO9sbAXljG0cZzSa1QQnt3/WqhPsD/JbQrFN9yWjss88EyIgXpDOOfmISC5U9Cbu -O9Cz1Rsdp3998woMQGQE2zmOhhPMuKU0WufHFAqnF/ipL9wDrbN4FLZ1W+Bkeo14 -d84X6/UMci0H+6gH3NEoU+8Csj5j5FG4+xe481mA3bXRXnmIPIN1TfUSHvyd1KEJ -KDpfet+BAgMBAAECggEADItTJWgSSPdx2/PcDg3v3yc55b54R9wCqh9p4dejfigq -WLDTnJDTEkP9gGAQgJCcs7wuOiAUmTTEFde6fvZB/AD0B+b6y7rBnuytw6UaINFC -mtnMiROc8jCWqa2APY6Ulr6GkC6elT0BJS1qHWhwOxJepXGbtnXrsz7Pjh3jtm4X -8tZ6/msANBKGzaEKaIg41YxUbr4tWr1G73IlnWkJwVrfLTHm1sAjDTqoYFo3xdlK -Ow3vm5+Zscv6sZmRzZDUuEdaBuyqkHr5X3L9lncfxG6WdpCi5JUFfihxegnV5yi6 -5dL3Spu+ZdOrnHsQ/leaSEwzJOWAQlYvCrYoiOSKxQKBgQDuYeyff9JQkOxSlnzs -UO849VFQceltgjpiNF8D3O4bF6eE09qPljs0zWiGBvpwNLX+L/xJTEVv1625IB8I -W2SvBu+WhfwqlThwNBW0HWAn6V0ehEgu7dWaU2X5cNdyGdr8wqkLyZhFi8BepOyR -ytvO7Azeks/XnpJtZwhJzy1qFQKBgQDOb0MDwnEdYMmEZ3aDSvXO8xoifblrCTBl -ysCtj1jmplp0FV9VeWsuJawovzF8DaDtuCUOLBSimRU/56BflWRPivwSy2b70gJr -LXcvN1Wws2zBd6YJjqNCyu/d3eqLV8+YTL/ZpjyI5uEREfoR6mK3ze2S4KkOxPW1 -snvtcq7WvQKBgQDPfnMtvl/9ergJhy4DsMsZpAb8Y7rQdDuHgZh2z1Z+RI+vAYzL -0POGGYlyqB5Tjr4fG/uYfYgvOuffLQN2Db9Mzle7iLKfCjYPDHcbyToKY4mHZ5NB -Lgnwg8lOXxdZHQJNYs8sEHS3jFaMyzeUC6Rar4LgNaAuSbug+L7xKCGapQKBgHF1 -jWufjvQKojd3dherN3bK/m4+k45Uupj32vaJdt8uR0DODlu4JER0yC6NBvGbu/tr -3lHvwFets5QwBmEChuOBDBJ4YN2/Cz1E++CjlSFNPFUJIeTW1Lx9NWDH+4UieiLG -7Br/1v2Xh9QOAVefbyp+sDit6b0IW9PFiX90LMwxAoGBANorN+N/857GHDfZNadH -3z1TI3vFQCC99OZr98IUVJ2KR5/bxhnlknzY1BOqX0KVZxCelXUlyc1bPc9NZvDv -dpE2tuMf9Yi3QqUDMAz06NsBbJ07b/7Te+nPFFWnts6MLApM8BPDyvkqpWAiWMdd -5kIXKnYdDAyHqhfz46+yzxQa ------END PRIVATE KEY----- diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch-security/action_groups.yml b/deploy/dockerephemeral/docker/opensearch/opensearch-security/action_groups.yml deleted file mode 100644 index 7c40612b836..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch-security/action_groups.yml +++ /dev/null @@ -1,3 +0,0 @@ -_meta: - type: "actiongroups" - config_version: 2 diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch-security/allowlist.yml b/deploy/dockerephemeral/docker/opensearch/opensearch-security/allowlist.yml deleted file mode 100644 index dd09dc80656..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch-security/allowlist.yml +++ /dev/null @@ -1,6 +0,0 @@ -_meta: - type: "allowlist" - config_version: 2 - -config: - enabled: false diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch-security/config.yml b/deploy/dockerephemeral/docker/opensearch/opensearch-security/config.yml deleted file mode 100644 index fdbeb97420b..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch-security/config.yml +++ /dev/null @@ -1,17 +0,0 @@ -_meta: - type: "config" - config_version: 2 - -config: - dynamic: - authc: - basic_internal_auth_domain: - description: "Authenticate using HTTP basic against the internal users database" - http_enabled: true - transport_enabled: true - order: 1 - http_authenticator: - type: basic - challenge: true - authentication_backend: - type: internal diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch-security/internal_users.yml b/deploy/dockerephemeral/docker/opensearch/opensearch-security/internal_users.yml deleted file mode 100644 index dc7023779fa..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch-security/internal_users.yml +++ /dev/null @@ -1,12 +0,0 @@ -_meta: - type: "internalusers" - config_version: 2 - -# User: elastic -# Password: changeme -elastic: - hash: "$2y$12$GRc68jkEX1m4uQpTVbwURu79xHxZ7vsbyEctOAADQwPjlhYS4LJVa" - reserved: true - description: "Wire User" - backend_roles: - - index_manager diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch-security/nodes_dn.yml b/deploy/dockerephemeral/docker/opensearch/opensearch-security/nodes_dn.yml deleted file mode 100644 index 09afda4a1f3..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch-security/nodes_dn.yml +++ /dev/null @@ -1,3 +0,0 @@ -_meta: - type: "nodesdn" - config_version: 2 diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch-security/roles.yml b/deploy/dockerephemeral/docker/opensearch/opensearch-security/roles.yml deleted file mode 100644 index 9bbe7b23f39..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch-security/roles.yml +++ /dev/null @@ -1,3 +0,0 @@ -_meta: - type: "roles" - config_version: 2 diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch-security/roles_mapping.yml b/deploy/dockerephemeral/docker/opensearch/opensearch-security/roles_mapping.yml deleted file mode 100644 index e7627c3e67b..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch-security/roles_mapping.yml +++ /dev/null @@ -1,9 +0,0 @@ -_meta: - type: "rolesmapping" - config_version: 2 - -all_access: - reserved: false - backend_roles: - - index_manager - description: "Map index_manager to full_access" diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch-security/tenants.yml b/deploy/dockerephemeral/docker/opensearch/opensearch-security/tenants.yml deleted file mode 100644 index e9582d70b59..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch-security/tenants.yml +++ /dev/null @@ -1,3 +0,0 @@ -_meta: - type: "tenants" - config_version: 2 diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch.yml b/deploy/dockerephemeral/docker/opensearch/opensearch.yml deleted file mode 100644 index b02910412b9..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch.yml +++ /dev/null @@ -1,45 +0,0 @@ -cluster.name: opensearch-cluster - -# Bind to all interfaces because we don't know what IP address Docker will assign to us. -network.host: 0.0.0.0 - -# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again. -discovery.type: single-node - -path.data: /usr/share/opensearch/data - -# WARNING: This is not a production-ready config! (Good enough for testing, -# though.) -plugins: - security: - ssl: - transport: - pemcert_filepath: certs/tls.crt - pemkey_filepath: certs/tls.key - pemtrustedcas_filepath: certs/ca.crt - enforce_hostname_verification: false - http: - enabled: true - pemcert_filepath: certs/tls.crt - pemkey_filepath: certs/tls.key - pemtrustedcas_filepath: certs/ca.crt - allow_unsafe_democertificates: true - allow_default_init_securityindex: true - audit.type: internal_opensearch - restapi: - roles_enabled: ["all_access", "security_rest_api_access"] - system_indices: - enabled: true - indices: - [ - ".opendistro-alerting-config", - ".opendistro-alerting-alert*", - ".opendistro-anomaly-results*", - ".opendistro-anomaly-detector*", - ".opendistro-anomaly-checkpoints", - ".opendistro-anomaly-detection-state", - ".opendistro-reports-*", - ".opendistro-notifications-*", - ".opendistro-notebooks", - ".opendistro-asynchronous-search-response*", - ] diff --git a/deploy/dockerephemeral/docker/opensearch/opensearch_dashboards.yml b/deploy/dockerephemeral/docker/opensearch/opensearch_dashboards.yml deleted file mode 100644 index 240fb646f57..00000000000 --- a/deploy/dockerephemeral/docker/opensearch/opensearch_dashboards.yml +++ /dev/null @@ -1,8 +0,0 @@ -opensearch.hosts: [https://opensearch:9200] -opensearch.ssl.verificationMode: none -opensearch.username: elastic -opensearch.password: changeme - -# Use this setting if you are running opensearch-dashboards without https -opensearch_security.cookie.secure: false -server.host: '0.0.0.0' diff --git a/deploy/dockerephemeral/federation-v0/brig.yaml b/deploy/dockerephemeral/federation-v0/brig.yaml index a236c83b90e..747c4bdff16 100644 --- a/deploy/dockerephemeral/federation-v0/brig.yaml +++ b/deploy/dockerephemeral/federation-v0/brig.yaml @@ -9,10 +9,6 @@ cassandra: keyspace: brig_test_federation_v0 # filterNodesByDatacentre: datacenter1 -elasticsearch: - url: http://nginz-federation-v0:9201 - index: directory_test - rabbitmq: host: rabbitmq port: 5672 diff --git a/deploy/dockerephemeral/federation-v0/nginz/conf/nginx.conf b/deploy/dockerephemeral/federation-v0/nginz/conf/nginx.conf index cd4ec97a1a7..bef49f1d046 100644 --- a/deploy/dockerephemeral/federation-v0/nginz/conf/nginx.conf +++ b/deploy/dockerephemeral/federation-v0/nginz/conf/nginx.conf @@ -102,26 +102,6 @@ http { # Locations # - server { - # elastic search does not support running http and https listeners - # at the same time. so our instance only runs https, but - # federation-v0 only supports http. this proxy rule helps with - # that. - # - # see also: git grep -Hn 'elasticsearch:' ../../brig.yaml - listen 9201; - - zauth_keystore /etc/wire/zauth-pubkeys.txt; - zauth_acl /etc/wire/nginz/conf/zauth_acl.txt; - - location "" { - zauth off; - - proxy_pass https://demo_wire_elasticsearch:9200; - proxy_set_header Authorization "Basic ZWxhc3RpYzpjaGFuZ2VtZQ=="; - } - } - server { include integration.conf; diff --git a/deploy/dockerephemeral/federation-v1/brig.yaml b/deploy/dockerephemeral/federation-v1/brig.yaml index 62a6c8c2a8f..a2eb09ea513 100644 --- a/deploy/dockerephemeral/federation-v1/brig.yaml +++ b/deploy/dockerephemeral/federation-v1/brig.yaml @@ -9,10 +9,6 @@ cassandra: keyspace: brig_test_federation_v1 # filterNodesByDatacentre: datacenter1 -elasticsearch: - url: http://nginz-federation-v1:9201 - index: directory_test - rabbitmq: host: rabbitmq port: 5672 diff --git a/deploy/dockerephemeral/federation-v1/nginz/conf/nginx.conf b/deploy/dockerephemeral/federation-v1/nginz/conf/nginx.conf index 43f8c68b306..bef49f1d046 100644 --- a/deploy/dockerephemeral/federation-v1/nginz/conf/nginx.conf +++ b/deploy/dockerephemeral/federation-v1/nginz/conf/nginx.conf @@ -102,26 +102,6 @@ http { # Locations # - server { - # elastic search does not support running http and https listeners - # at the same time. so our instance only runs https, but - # federation-v1 only supports http. this proxy rule helps with - # that. - # - # see also: git grep -Hn 'elasticsearch:' ../../brig.yaml - listen 9201; - - zauth_keystore /etc/wire/zauth-pubkeys.txt; - zauth_acl /etc/wire/nginz/conf/zauth_acl.txt; - - location "" { - zauth off; - - proxy_pass https://demo_wire_elasticsearch:9200; - proxy_set_header Authorization "Basic ZWxhc3RpYzpjaGFuZ2VtZQ=="; - } - } - server { include integration.conf; diff --git a/deploy/dockerephemeral/federation-v2/brig.yaml b/deploy/dockerephemeral/federation-v2/brig.yaml index 1bfd0896152..21acf92f052 100644 --- a/deploy/dockerephemeral/federation-v2/brig.yaml +++ b/deploy/dockerephemeral/federation-v2/brig.yaml @@ -9,14 +9,6 @@ cassandra: keyspace: brig_test_federation_v2 # filterNodesByDatacentre: datacenter1 -elasticsearch: - url: https://demo_wire_elasticsearch:9200 - index: directory_test_federation_v2 - credentials: /etc/wire/brig/conf/elasticsearch-credentials.yaml - insecureSkipVerifyTls: true - additionalCredentials: /etc/wire/brig/conf/elasticsearch-credentials.yaml - additionalInsecureSkipVerifyTls: true - rabbitmq: host: rabbitmq port: 5672 diff --git a/deploy/dockerephemeral/federation-v2/elasticsearch-credentials.yaml b/deploy/dockerephemeral/federation-v2/elasticsearch-credentials.yaml deleted file mode 100644 index 47846ea1017..00000000000 --- a/deploy/dockerephemeral/federation-v2/elasticsearch-credentials.yaml +++ /dev/null @@ -1,2 +0,0 @@ -username: "elastic" -password: changeme diff --git a/docs/src/developer/developer/building.md b/docs/src/developer/developer/building.md index 4cbdba83822..1dece284175 100644 --- a/docs/src/developer/developer/building.md +++ b/docs/src/developer/developer/building.md @@ -155,7 +155,6 @@ These services require most of the deployment dependencies as seen in the archit - Required internal dependencies: - cassandra (with the correct schema) - - elasticsearch (with the correct schema) - Required external dependencies are the following configured AWS services (or “fake” replacements providing the same API): - SES - SQS diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 9154c97ea73..45be306967e 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -1875,114 +1875,6 @@ accessible to services (and not the private key.) The corresponding Cassandra options are described in Cassandra’s documentation: [client_encryption_options](https://cassandra.apache.org/doc/stable/cassandra/configuration/cass_yaml_file.html#client_encryption_options) -## Configure Elasticsearch basic authentication - -When the Wire backend is configured to work against a custom Elasticsearch -instance, it may be desired to enable basic authentication for the internal -communication between the Wire backend and the ES instance. To do so the -Elasticsearch credentials can be set in wire-server’s secrets for `brig` and -`elasticsearch-index` as follows: - -```yaml -brig: - secrets: - elasticsearch: - username: elastic - password: changeme - -elasticsearch-index: - secrets: - elasticsearch: - username: elastic - password: changeme -``` - -In some cases an additional Elasticsearch instance is needed (e.g. for index -migrations). To configure credentials for the additional ES instance add the -secret as follows: - -```yaml -brig: - secrets: - elasticsearchAdditional: - username: elastic - password: changeme -``` - -## Configure TLS for Elasticsearch - -If the elasticsearch instance requires TLS, it can be configured like this: - -```yaml -brig: - config: - elasticsearch: - scheme: https - -elasticsearch-index: - elasticsearch: - scheme: https -``` - -In case a custom CA certificate is required it can be provided like this: - -```yaml -brig: - config: - elasticsearch: - tlsCa: -elasticsearch-index: - elasticsearch: - tlsCa: -``` - -There is another way to provide this, in case there already exists a kubernetes -secret containing the CA certificate(s): - -```yaml -brig: - config: - elasticsearch: - tlsCaSecretRef: - name: - key: -elasticsearch-index: - elasticsearch: - tlsCaSecretRef: - name: - key: -``` - -For configuring `addtionalWriteIndex` in brig (this is required during a -migration from one index to another or one ES instance to another), the settings -need to be like this: - -```yaml -brig: - config: - elasticsearch: - additionalWriteScheme: https - # One or none of these: - # addtionalTlsCa: - # addtionalTlsCaSecretRef: -``` - -**WARNING:** Please do this only if you know what you’re doing. - -In case it is not possible to verify TLS certificate of the elasticsearch -server, it can be turned off without tuning off TLS like this: - -```yaml -brig: - config: - elasticsearch: - insecureSkipVerifyTls: true - addtionalInsecureSkipVerifyTls: true # only required when addtional index is being used. -elasticsearch-index: - elasticsearch: - insecureSkipVerifyTls: true -``` - ## Configure RabbitMQ RabbitMQ authentication must be configured on brig, galley and background-worker. For example: diff --git a/docs/src/how-to/install/infrastructure-configuration.md b/docs/src/how-to/install/infrastructure-configuration.md index 06ef7a218d1..04fab0494a1 100644 --- a/docs/src/how-to/install/infrastructure-configuration.md +++ b/docs/src/how-to/install/infrastructure-configuration.md @@ -25,7 +25,6 @@ gundeck: - "localhost" - "127.0.0.1" - "10.0.0.0/8" - - "elasticsearch-external" - "cassandra-external" - "fake-aws-sqs" - "fake-aws-dynamodb" @@ -414,8 +413,8 @@ cassandra cannot reliably be installed on kubernetes. Some people have tried, e.g. [this project](https://github.com/instaclustr/cassandra-operator) though at the time of writing (Nov 2018), this does not yet work as advertised. We -recommend therefore to install cassandra, (possibly also elasticsearch) -separately, i.e. outside of kubernetes (using 3 nodes each). +recommend therefore to install cassandra separately, i.e. outside of +kubernetes (using 3 nodes). For further higher-availability: diff --git a/docs/src/how-to/install/troubleshooting.md b/docs/src/how-to/install/troubleshooting.md index 3703500a9fd..e28d87d4e8e 100644 --- a/docs/src/how-to/install/troubleshooting.md +++ b/docs/src/how-to/install/troubleshooting.md @@ -473,7 +473,6 @@ Open a shell inside the SNS pod, and make sure you can resolve the following thr * `minio-external` * `cassandra-external` -* `elasticsearch-external` First get a list of all pods: diff --git a/flake.lock b/flake.lock index 67d13b227d8..38aef894c3a 100644 --- a/flake.lock +++ b/flake.lock @@ -33,23 +33,6 @@ "type": "github" } }, - "bloodhound": { - "flake": false, - "locked": { - "lastModified": 1739958389, - "narHash": "sha256-E3co9FGZP135T3RocX4vbUELbbgGbYddD8CcVNUzHu8=", - "owner": "wireapp", - "repo": "bloodhound", - "rev": "dac0f1384b335ce35dc026bf8154e574b1a15d62", - "type": "github" - }, - "original": { - "owner": "wireapp", - "ref": "wire-fork", - "repo": "bloodhound", - "type": "github" - } - }, "cql": { "flake": false, "locked": { @@ -385,7 +368,6 @@ "inputs": { "amazonka": "amazonka", "arbiter": "arbiter", - "bloodhound": "bloodhound", "cql": "cql", "cql-io": "cql-io", "cryptostore": "cryptostore", diff --git a/flake.nix b/flake.nix index 46255be3e45..143f4762917 100644 --- a/flake.nix +++ b/flake.nix @@ -16,11 +16,6 @@ inputs.nixpkgs.follows = "nixpkgs"; }; - bloodhound = { - url = "github:wireapp/bloodhound?ref=wire-fork"; - flake = false; - }; - http-client = { url = "github:wireapp/http-client?ref=master"; flake = false; diff --git a/hack/bin/create-helm-sboms.sh b/hack/bin/create-helm-sboms.sh index c9493b07826..024934ee350 100755 --- a/hack/bin/create-helm-sboms.sh +++ b/hack/bin/create-helm-sboms.sh @@ -52,7 +52,6 @@ proxy: {secrets: {proxy_config: placeholder}} cannon: {secrets: {rabbitmq: {username: placeholder, password: placeholder}}} gundeck: {secrets: {rabbitmq: {username: placeholder, password: placeholder}}} cassandra-migrations: {cassandra: {host: placeholder}} -elasticsearch-index: {elasticsearch: {host: placeholder}, cassandra: {host: placeholder}} spar: {config: {appUri: 'https://placeholder', ssoUri: 'https://placeholder', contacts: [placeholder]}} galley: {config: {settings: {conversationCodeURI: 'https://placeholder'}}, secrets: {rabbitmq: {username: placeholder, password: placeholder}}} EOF diff --git a/hack/bin/gen-certs.sh b/hack/bin/gen-certs.sh index d4840f8af6d..a8fdb647539 100755 --- a/hack/bin/gen-certs.sh +++ b/hack/bin/gen-certs.sh @@ -72,15 +72,6 @@ install_certs "$TEMP/federation" "$ROOT_DIR/deploy/dockerephemeral/federation-v0 install_certs "$TEMP/federation" "$ROOT_DIR/deploy/dockerephemeral/federation-v1" \ integration-ca "" integration-leaf integration-leaf-key -# elasticsearch -mkdir -p "$TEMP/es" -gen_ca "$TEMP/es" elasticsearch.ca.example.com -gen_cert "$TEMP/es" "DNS:localhost" localhost -install_certs "$TEMP/es" "$ROOT_DIR/deploy/dockerephemeral/docker" \ - elasticsearch-ca "" elasticsearch-cert elasticsearch-key -install_certs "$TEMP/es" "$ROOT_DIR/hack/helm_vars/certs" \ - elasticsearch-ca elasticsearch-ca-key - # rabbitmq RABBITMQ="$ROOT_DIR/deploy/dockerephemeral/rabbitmq-config/certificates" gen_ca "$RABBITMQ" rabbitmq.ca.example.com diff --git a/hack/bin/set-wire-server-image-version.sh b/hack/bin/set-wire-server-image-version.sh index 7c924909d41..e2d7f525819 100755 --- a/hack/bin/set-wire-server-image-version.sh +++ b/hack/bin/set-wire-server-image-version.sh @@ -6,7 +6,7 @@ target_version=${1?$USAGE} TOP_LEVEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )" CHARTS_DIR="$TOP_LEVEL/.local/charts" -charts=(proxy cassandra-migrations elasticsearch-index federator backoffice integration wire-server-enterprise) +charts=(proxy cassandra-migrations federator backoffice integration wire-server-enterprise) for chart in "${charts[@]}"; do values_file="$CHARTS_DIR/$chart/values.yaml" diff --git a/hack/helm_vars/certs/elasticsearch-ca-key.pem b/hack/helm_vars/certs/elasticsearch-ca-key.pem deleted file mode 100644 index f59d94e52b6..00000000000 --- a/hack/helm_vars/certs/elasticsearch-ca-key.pem +++ /dev/null @@ -1,28 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDhfp2hokSJ88qb -6Gl8BBwg4jMbkt4l2ynOa/lO6DZFRheFlWZBaUZAcj1ojBgbd3EoOXygZReL9TGB -Rrc5iACzytlOJkwMgNUUIq1KsPwWII41VJw3h3NO07tMTsf0kFvH1pEllJqorhQ1 -eZnU1SISyQcjk0oRQEyWd6arF7TLHna69OHF2ybYYXMDMFWNr+O6t8RUfYs4kb7z -1Nx3OnUKrUhIyaYeoyvBBOdXA5/G5GenDu/G4iVozsuLgofYWu+77EnpProF0KRK -+XlKakvF7bD26Qm1ol5qsXbdXOXKYq/c4KDdC938HNd/FNPz7EsnW0brBXqtz8Te -QmxHix7DAgMBAAECggEAOEemxiG+44OCbRk7wqUv9BEg2l/0rBQgQhH23nfcm7ub -wU6BgA/rZchdhUt59NkB2B1I+qtgjiD7Yx2oO2azbixRwkySrIg3JlhlUgAMWuVz -OOJOPxnCcMkttSTwiRzCm4T1IyEM3M7d4l7gQxuS7odYDcwEL3wR4XgplAhNqmgP -AxBayjX/POTsIoH2xPhjYsILPnRjDihQdxIWJoqZHUfIvM925tFi7WtzaFvyX8VU -s8t2zcByiqi1q9MLDv+uwhweP7tYGUj1RoQYoMuFjVRrm09nCRmAix2Evok1zXqV -6jPqgDe4+p03+816n8EWKgCTcAiZvFb7YhYkAO1gWQKBgQD7IKkUzvK6qKrN6Fg4 -bX4U6tAvKVGRhiNEfPRWv44OatusI1K14ndtLbZesQ+R2WgjwO1P2bhs6BDeV/oo -Y5T2ETBRcAV154NJxv5ktGw5IaI0IAlYBy/daaTY9wZBZ8+Qy5+FHra0ClHOMzKl -qQ3UeSMNZ+E5mH9ofMfhRFs8lwKBgQDl3qMqaBEDg/2O58IqwjWG9QGEGfoyqEi4 -C1DwQhvPAFa8PJ/mqwFZtYd1WpLV+wMm6WZWhVTpjQIYo7jYX54Vz2/y94AqtRC6 -sqlJgMCDSGjVeVgzIw2L1yaxdl+WzhbtGQUPmKxh9BM1WmJysst2/f7rk9MjyTFg -auhtAkH4tQKBgQCqA92UsdrRFjm093Uqlq5CWQqiszV+8TJVPsdpJ3x0NFIOg0eO -zgiOiOEr0HG7C1YexpGjesIKMT6iWSuKRojl4pM0v0NjJF7VBvzZjvCp6SRYZ8wL -pan5G3m4Td0VUMPMwp530GhfEZF6qVzDnOU5EN3zSH3JsX2obrofv1iJdwKBgDnO -iFfkvcqVidFDRRf9qPpcaNowsjPFECyAZAVXiqi+3BEQaeHXRUqrFPqVIXIAYuWJ -Mnw1oYnuNQW/Pn/jY9z2Qp/mT+vthtx8i4f5gfBB6GMu1dheS0zMeWWNcDJ7d1Z+ -wUAP0+H6QE5dgX54qiQtccsKbMGGGg22NOcc9zw1AoGAGiacZEXrdHsg1piMlHjw -LE96b7mynZfOO4LPm/0Xl5FJ8mElNnZGWlpMFVq+Hi2WnHQ6MRRIZmOS3YTdS4FB -cBvwiGGn18QEgFGOI7JonzPDp1LZWnxy5nmZhmTuI4GiXyYAqGw6QLGiqqQo4p08 -J7OLSRyQ7aiG97iUS1QgJFU= ------END PRIVATE KEY----- diff --git a/hack/helm_vars/certs/elasticsearch-ca.pem b/hack/helm_vars/certs/elasticsearch-ca.pem deleted file mode 100644 index 6511f688a73..00000000000 --- a/hack/helm_vars/certs/elasticsearch-ca.pem +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDLzCCAhegAwIBAgIUUOLn63PL3FEyGdhOK1ocDAn8dC8wDQYJKoZIhvcNAQEL -BQAwJzElMCMGA1UEAwwcZWxhc3RpY3NlYXJjaC5jYS5leGFtcGxlLmNvbTAeFw0y -NDA5MDMxMjAzMzhaFw0zNDA5MDExMjAzMzhaMCcxJTAjBgNVBAMMHGVsYXN0aWNz -ZWFyY2guY2EuZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK -AoIBAQDhfp2hokSJ88qb6Gl8BBwg4jMbkt4l2ynOa/lO6DZFRheFlWZBaUZAcj1o -jBgbd3EoOXygZReL9TGBRrc5iACzytlOJkwMgNUUIq1KsPwWII41VJw3h3NO07tM -Tsf0kFvH1pEllJqorhQ1eZnU1SISyQcjk0oRQEyWd6arF7TLHna69OHF2ybYYXMD -MFWNr+O6t8RUfYs4kb7z1Nx3OnUKrUhIyaYeoyvBBOdXA5/G5GenDu/G4iVozsuL -gofYWu+77EnpProF0KRK+XlKakvF7bD26Qm1ol5qsXbdXOXKYq/c4KDdC938HNd/ -FNPz7EsnW0brBXqtz8TeQmxHix7DAgMBAAGjUzBRMB0GA1UdDgQWBBTWzp/VRTGq -s/IwlrROQGDwavTpyzAfBgNVHSMEGDAWgBTWzp/VRTGqs/IwlrROQGDwavTpyzAP -BgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBFue/SGpAVZ0TOVp8l -bdaaY/e9wUBSdCH5b39Nzd3rKmH3lIHqcafLlFCx+scKUIlFbohJr0aTK339wFfl -L0LzBJVUT9JzeDNPhk8pl/jBJk+eGP3fiykFMCgxxGvHtccHu8E/y8U0SeEtKqDn -Xy0ZbC3M54UedhDpHMovfHEsfN24Ev0DK13sBR2T8fmXCyCrfq887cCqJyP2ODgb -xAY/R4F8Ueadn0ywHYSY3MqmDsvDul0QlaOu2J5A0+k5oy4hAfFB8PzPYZrmPkeU -N5oxudTTihIZ+0JiL2JmWGBzMGzgtmD1rHC6lugUlWq+BoPu2+/+hn8RcVHBCFDk -WMSU ------END CERTIFICATE----- diff --git a/hack/helm_vars/certs/values.yaml.gotmpl b/hack/helm_vars/certs/values.yaml.gotmpl index 7cd8a633653..7dd8b6c27b5 100644 --- a/hack/helm_vars/certs/values.yaml.gotmpl +++ b/hack/helm_vars/certs/values.yaml.gotmpl @@ -1,21 +1,4 @@ resources: - - apiVersion: v1 - kind: Secret - metadata: - name: elasticsearch-ca - namespace: '{{ .Release.Namespace }}' - data: - tls.crt: {{ readFile "./elasticsearch-ca.pem" | b64enc | quote }} - tls.key: {{ readFile "./elasticsearch-ca-key.pem" | b64enc | quote }} - - apiVersion: cert-manager.io/v1 - kind: Issuer - metadata: - name: elasticsearch - namespace: '{{ .Release.Namespace }}' - spec: - ca: - secretName: elasticsearch-ca - # RabbitMQ CA and certificate - apiVersion: cert-manager.io/v1 kind: Issuer diff --git a/hack/helm_vars/opensearch/values.yaml.gotmpl b/hack/helm_vars/opensearch/values.yaml.gotmpl deleted file mode 100644 index 72fa275b163..00000000000 --- a/hack/helm_vars/opensearch/values.yaml.gotmpl +++ /dev/null @@ -1,192 +0,0 @@ -singleNode: true - -# Helm labels and annotations are automatically added for these Kubernetes -# manifests. -extraObjects: - - apiVersion: cert-manager.io/v1 - kind: Certificate - metadata: - name: opensearch-cert - namespace: {{ .Release.Namespace }} - spec: - issuerRef: - name: elasticsearch - kind: Issuer - - usages: - - server auth - - client auth - duration: 2160h # 90d - renewBefore: 360h # 15d - isCA: false - secretName: opensearch-ephemeral-certificate - - privateKey: - algorithm: ECDSA - size: 384 - encoding: PKCS8 - rotationPolicy: Always - - dnsNames: - - opensearch-cluster-master - - opensearch-cluster-master.{{ .Release.Namespace }}.svc.cluster.local - - commonName: opensearch-cluster-master - -opensearchHome: /usr/share/opensearch - -config: - opensearch.yml: | - cluster.name: opensearch-cluster - - # Bind to all interfaces because we don't know what IP address Docker will assign to us. - network.host: 0.0.0.0 - - discovery.type: single-node - - action.auto_create_index: true - - # WARNING: This config is not meant to be used as prod setup! Revise all - # lines before you copy them. - plugins: - security: - nodes_dn: - - '/CN=opensearch-cluster-master.*/' - ssl: - transport: - pemcert_filepath: esnode.pem - pemkey_filepath: esnode-key.pem - pemtrustedcas_filepath: root-ca.pem - enforce_hostname_verification: false - http: - enabled: true - pemcert_filepath: esnode.pem - pemkey_filepath: esnode-key.pem - pemtrustedcas_filepath: root-ca.pem - allow_unsafe_democertificates: true - allow_default_init_securityindex: true - audit.type: internal_opensearch - enable_snapshot_restore_privilege: true - check_snapshot_restore_write_privileges: true - restapi: - roles_enabled: ["all_access", "security_rest_api_access"] - system_indices: - enabled: true - indices: - [ - ".opendistro-alerting-config", - ".opendistro-alerting-alert*", - ".opendistro-anomaly-results*", - ".opendistro-anomaly-detector*", - ".opendistro-anomaly-checkpoints", - ".opendistro-anomaly-detection-state", - ".opendistro-reports-*", - ".opendistro-notifications-*", - ".opendistro-notebooks", - ".opendistro-asynchronous-search-response*", - ] - -securityConfig: - enabled: true - # The path will be different for OpenSearch 2.x.x! - path: "/usr/share/opensearch/plugins/opensearch-security/securityconfig" - - # Configure one user with full access (this could be refined in future.) - # Credentials: elastic:changeme - config: - dataComplete: "true" - data: - config.yml: | - _meta: - type: "config" - config_version: 2 - - config: - dynamic: - authc: - basic_internal_auth_domain: - description: "Authenticate using HTTP basic against the internal users database" - http_enabled: true - transport_enabled: true - order: 1 - http_authenticator: - type: basic - challenge: true - authentication_backend: - type: internal - - internal_users.yml: | - _meta: - type: "internalusers" - config_version: 2 - - elastic: - hash: "$2y$12$GRc68jkEX1m4uQpTVbwURu79xHxZ7vsbyEctOAADQwPjlhYS4LJVa" - reserved: true - description: "Wire User" - backend_roles: - - index_manager - - roles_mapping.yml: | - _meta: - type: "rolesmapping" - config_version: 2 - - all_access: - reserved: false - backend_roles: - - index_manager - description: "Map index_manager to full_access" - - allowlist.yml: | - _meta: - type: "allowlist" - config_version: 2 - - config: - enabled: false - - roles.yml: | - _meta: - type: "roles" - config_version: 2 - - nodes_dn.yml: | - _meta: - type: "nodesdn" - config_version: 2 - - action_groups.yml: | - _meta: - type: "actiongroups" - config_version: 2 - - tenants.yml: | - _meta: - type: "tenants" - config_version: 2 - -extraEnvs: - - name: OPENSEARCH_INITIAL_ADMIN_PASSWORD - value: "Ch4ng3m3Secr3t!" - - name: DISABLE_INSTALL_DEMO_CONFIG - value: "true" - -persistence: - enabled: false - -secretMounts: - - name: node-pem - secretName: opensearch-ephemeral-certificate - path: /usr/share/opensearch/config/esnode.pem - subPath: tls.crt - - - name: node-key - secretName: opensearch-ephemeral-certificate - path: /usr/share/opensearch/config/esnode-key.pem - subPath: tls.key - - - name: root-cacert - secretName: opensearch-ephemeral-certificate - path: /usr/share/opensearch/config/root-ca.pem - subPath: ca.crt diff --git a/hack/helm_vars/wire-federation-v0/values.yaml.gotmpl b/hack/helm_vars/wire-federation-v0/values.yaml.gotmpl index 52abc30d9b6..c2ccc68c4d3 100644 --- a/hack/helm_vars/wire-federation-v0/values.yaml.gotmpl +++ b/hack/helm_vars/wire-federation-v0/values.yaml.gotmpl @@ -19,13 +19,6 @@ cassandra-migrations: cassandra: host: cassandra-ephemeral replicationFactor: 1 -elasticsearch-index: - elasticsearch: - host: {{ .Values.elasticsearch.host }} - index: directory_test - cassandra: - host: cassandra-ephemeral - brig: replicaCount: 1 resources: @@ -40,9 +33,6 @@ brig: cassandra: host: cassandra-ephemeral replicaCount: 1 - elasticsearch: - host: {{ .Values.elasticsearch.host }} - index: directory_test authSettings: userTokenTimeout: 120 sessionTokenTimeout: 20 diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 43373b1cf2e..fa20a56c0a5 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -20,27 +20,6 @@ cassandra-migrations: key: "ca.crt" {{- end }} -elasticsearch-index: - imagePullPolicy: {{ .Values.imagePullPolicy }} - elasticsearch: - scheme: https - host: {{ .Values.elasticsearch.host }} - index: directory_test - tlsCaSecretRef: - name: {{ .Values.elasticsearch.caSecretName }} - key: "ca.crt" - cassandra: - host: {{ .Values.cassandraHost }} - {{- if .Values.useK8ssandraSSL.enabled }} - tlsCaSecretRef: - name: "cassandra-jks-keystore" - key: "ca.crt" - {{- end }} - secrets: - elasticsearch: - username: "elastic" - password: "changeme" - brig: replicaCount: 1 imagePullPolicy: {{ .Values.imagePullPolicy }} @@ -67,16 +46,6 @@ brig: port: "5432" user: wire-server dbname: wire-server - elasticsearch: - scheme: https - host: {{ .Values.elasticsearch.host }} - index: directory_test - tlsCaSecretRef: - name: {{ .Values.elasticsearch.caSecretName }} - key: "ca.crt" - additionalTlsCaSecretRef: - name: {{ .Values.elasticsearch.caSecretName }} - key: "ca.crt" rabbitmq: port: 5671 enableTls: true @@ -171,9 +140,6 @@ brig: tActivationUrl: https://example.com/verify/?key=${key}&code=${code} tCreatorWelcomeUrl: https://example.com/login tMemberWelcomeUrl: https://example.com/download - test: - elasticsearch: - additionalHost: {{ .Values.elasticsearch.additionalHost }} secrets: # these secrets are only used during integration tests and should therefore be safe to include unencrypted in git. # Normally these would live in a separately-encrypted secrets.yaml file and incorporated using the helm secrets plugin (wrapper around mozilla sops) @@ -201,12 +167,6 @@ brig: rabbitmq: username: {{ .Values.rabbitmqUsername }} password: {{ .Values.rabbitmqPassword }} - elasticsearch: - username: "elastic" - password: "changeme" - elasticsearchAdditional: - username: "elastic" - password: "changeme" pgPassword: "posty-the-gres" tests: enableFederationTests: true @@ -721,11 +681,6 @@ integration: name: cassandra-jks-keystore key: ca.crt {{- end }} - elasticsearch: - host: {{ .Values.elasticsearch.host }} - tlsCaSecretRef: - name: {{ .Values.elasticsearch.caSecretName }} - key: "ca.crt" rabbitmq: tlsCaSecretRef: name: "rabbitmq-certificate" diff --git a/hack/helmfile.yaml.gotmpl b/hack/helmfile.yaml.gotmpl index f28ed995af5..71fffb82a20 100644 --- a/hack/helmfile.yaml.gotmpl +++ b/hack/helmfile.yaml.gotmpl @@ -20,10 +20,6 @@ environments: - cassandraHost: cassandra-ephemeral - useK8ssandraSSL: enabled: false - - elasticsearch: - host: opensearch-cluster-master - additionalHost: elasticsearch-ephemeral - caSecretName: opensearch-ephemeral-certificate default-ssl: values: - ./helm_vars/common.yaml.gotmpl @@ -32,10 +28,6 @@ environments: - cassandraHost: k8ssandra-cluster-datacenter-1-service - useK8ssandraSSL: enabled: true - - elasticsearch: - host: elasticsearch-ephemeral - additionalHost: opensearch-cluster-master - caSecretName: elasticsearch-ephemeral-certificate kind: values: - ./helm_vars/common.yaml.gotmpl @@ -44,10 +36,6 @@ environments: - cassandraHost: cassandra-ephemeral - useK8ssandraSSL: enabled: false - - elasticsearch: - host: elasticsearch-ephemeral - additionalHost: opensearch-cluster-master - caSecretName: elasticsearch-ephemeral-certificate kind-ssl: values: - ./helm_vars/common.yaml.gotmpl @@ -56,10 +44,6 @@ environments: - cassandraHost: k8ssandra-cluster-datacenter-1-service - useK8ssandraSSL: enabled: true - - elasticsearch: - host: elasticsearch-ephemeral - additionalHost: opensearch-cluster-master - caSecretName: elasticsearch-ephemeral-certificate --- repositories: - name: incubator @@ -77,9 +61,6 @@ repositories: - name: obeone url: 'https://charts.obeone.cloud' - - name: opensearch - url: 'https://opensearch-project.github.io/helm-charts/' - - name: bitnami url: registry-1.docker.io/bitnamicharts oci: true @@ -118,26 +99,6 @@ releases: namespace: '{{ .Values.namespace1 }}' chart: '../.local/charts/cassandra-ephemeral' - - name: 'elasticsearch-ephemeral' - namespace: '{{ .Values.namespace1 }}' - chart: '../.local/charts/elasticsearch-ephemeral' - values: - - tls: - enabled: true - issuerRef: - name: elasticsearch - kind: Issuer - - - name: 'elasticsearch-ephemeral' - namespace: '{{ .Values.namespace2 }}' - chart: '../.local/charts/elasticsearch-ephemeral' - values: - - tls: - enabled: true - issuerRef: - name: elasticsearch - kind: Issuer - - name: 'cassandra-ephemeral' namespace: '{{ .Values.namespace2 }}' chart: '../.local/charts/cassandra-ephemeral' @@ -180,24 +141,6 @@ releases: GRANT ALL PRIVILEGES ON DATABASE "wire-server" TO "wire-server"; - - name: 'opensearch-ephemeral' - namespace: '{{ .Values.namespace1 }}' - chart: 'opensearch/opensearch' - # The 1.x.x and 2.x.x chart versions belong to the OpenSearch versions 1.x.x - # and 2.x.x respectively. I.e. both strains are actively maintained. - version: "1.31.0" - values: - - './helm_vars/opensearch/values.yaml.gotmpl' - - - name: 'opensearch-ephemeral-2' - namespace: '{{ .Values.namespace2 }}' - chart: 'opensearch/opensearch' - # The 1.x.x and 2.x.x chart versions belong to the OpenSearch versions 1.x.x - # and 2.x.x respectively. I.e. both strains are actively maintained. - version: "1.31.0" - values: - - './helm_vars/opensearch/values.yaml.gotmpl' - - name: 'certs' namespace: '{{ .Values.namespace2 }}' chart: bedag/raw diff --git a/integration/scripts/integration-dynamic-backends-brig-index.sh b/integration/scripts/integration-dynamic-backends-brig-index.sh deleted file mode 100755 index ea7f0f4eeac..00000000000 --- a/integration/scripts/integration-dynamic-backends-brig-index.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC3040 -# -set -eo pipefail - -for i in $(seq "$INTEGRATION_DYNAMIC_BACKENDS_POOLSIZE"); do - cmd="brig-index reset --elasticsearch-index-prefix directory_dyn_$i $*" - echo "$cmd" - $cmd -done diff --git a/integration/test/API/BrigInternal.hs b/integration/test/API/BrigInternal.hs index 4407e1a043b..816ed1e2871 100644 --- a/integration/test/API/BrigInternal.hs +++ b/integration/test/API/BrigInternal.hs @@ -190,12 +190,6 @@ getInvitationCodeForTeam domain tid inv = do "i/teams/invitation-code?team=" <> tid <> "&invitation_id=" <> invId submit "GET" req -refreshIndex :: (HasCallStack, MakesValue domain) => domain -> App () -refreshIndex domain = do - req <- baseRequest domain Brig Unversioned "i/index/refresh" - res <- submit "POST" req - res.status `shouldMatchInt` 200 - getTeamSize :: (HasCallStack, MakesValue caller) => caller -> String -> App Response getTeamSize caller tid = do req <- baseRequest caller Brig Unversioned $ joinHttpPath ["i", "teams", tid, "size"] diff --git a/integration/test/SetupHelpers.hs b/integration/test/SetupHelpers.hs index b5aed2c98a2..36a0f40675a 100644 --- a/integration/test/SetupHelpers.hs +++ b/integration/test/SetupHelpers.hs @@ -53,9 +53,6 @@ import qualified SAML2.WebSSO as SAML import qualified SAML2.WebSSO.API.Example as SAML import qualified SAML2.WebSSO.Test.MockResponse as SAML import SAML2.WebSSO.Test.Util (SampleIdP (..), makeSampleIdPMetadata) -import System.Exit -import System.FilePath (()) -import System.Process import System.Random import Test.DNSMock import Testlib.JSON @@ -66,7 +63,6 @@ import qualified Text.XML as XML import qualified Text.XML.Cursor as XML import qualified Text.XML.DSig as SAML import UnliftIO (pooledForConcurrentlyN) -import UnliftIO.Concurrent (forkIO) randomUser :: (HasCallStack, MakesValue domain) => domain -> CreateUser -> App Value randomUser domain cu = createUser domain cu >>= getJSON 201 @@ -821,113 +817,3 @@ getMetrics domain service = do req <- rawBaseRequest domain service Unversioned "/i/metrics" submit "GET" req -createNewIndex :: (HasCallStack) => App String -createNewIndex = do - testName <- asks (fromMaybe "NoTest" . (.currentTestName)) - indexName <- ("temp-" <>) . take 10 . randomRs ('a', 'z') <$> newStdGen - let prefix = "[create-new-index:" <> indexName <> ":" <> testName <> "] " - brigConfig <- readServiceConfig Brig - esServer <- brigConfig %. "elasticsearch.url" & asString - esCredentials <- brigConfig %. "elasticsearch.credentials" & asString - esCaCert <- brigConfig %. "elasticsearch.caCert" & asString - cwd <- ( "brig") <$$> asks (.servicesCwdBase) - - (_, Just stdoutHdl, Just stderrHdl, ph) <- - liftIO $ - createProcess - ( proc - "brig-index" - [ "create", - "--elasticsearch-server", - esServer, - "--elasticsearch-index", - indexName, - "--elasticsearch-ca-cert", - esCaCert, - "--elasticsearch-credentials", - esCredentials - ] - ) - { cwd = cwd, - std_out = CreatePipe, - std_err = CreatePipe - } - - void $ forkIO $ liftIO $ logToConsole (colored green) prefix stdoutHdl - void $ forkIO $ liftIO $ logToConsole id prefix stderrHdl - exitCode <- liftIO $ waitForProcess ph - case exitCode of - ExitFailure _ -> assertFailure $ prefix <> "failed to create index" - ExitSuccess -> pure indexName - -reindexUsers :: (HasCallStack) => BackendResource -> ServiceOverrides -> Int -> App () -reindexUsers ber serviceOverrides pageSize = do - testName <- asks (fromMaybe "NoTest" . (.currentTestName)) - let indexName = ber.berElasticsearchIndex - let prefix = "[reindex-users:" <> indexName <> ":" <> testName <> "] " - getBrigConfig <- readAndUpdateConfig (defaultOverrides ber <> serviceOverrides) ber Brig - brigConfig <- liftIO $ getBrigConfig - esServer <- brigConfig %. "elasticsearch.url" & asString - esCredentials <- brigConfig %. "elasticsearch.credentials" & asString - esCaCert <- brigConfig %. "elasticsearch.caCert" & asString - cwd <- ( "brig") <$$> asks (.servicesCwdBase) - pgSettings <- cs . Aeson.encode <$> brigConfig %. "postgresql" - mPgPasswordFile <- brigConfig `lookupField` "postgresqlPassword" & asStringM - let pgPasswordOpts = case mPgPasswordFile of - Nothing -> [] - Just pwFile -> ["--pg-password-file", pwFile] - userStorageLocation <- brigConfig %. "postgresMigration.user" & asString - galleyHost <- brigConfig %. "galley.host" & asString - galleyPort <- brigConfig %. "galley.port" & asInt <&> show - casHost <- brigConfig %. "cassandra.endpoint.host" & asString - casPort <- brigConfig %. "cassandra.endpoint.port" & asInt <&> show - casKeySpace <- brigConfig %. "cassandra.keyspace" & asString - let indexOpts = - [ "reindex", - "--elasticsearch-server", - esServer, - "--elasticsearch-index", - indexName, - "--elasticsearch-ca-cert", - esCaCert, - "--elasticsearch-credentials", - esCredentials, - "--pg-pool-size", - "10", - "--pg-pool-acquisition-timeout", - "10s", - "--pg-pool-idleness-timeout", - "1h", - "--pg-settings", - pgSettings, - "--user-storage-location", - userStorageLocation, - "--galley-host", - galleyHost, - "--galley-port", - galleyPort, - "--cassandra-host", - casHost, - "--cassandra-port", - casPort, - "--cassandra-keyspace", - casKeySpace, - "--page-size", - show pageSize - ] - <> pgPasswordOpts - (_, Just stdoutHdl, Just stderrHdl, ph) <- - liftIO $ - createProcess - (proc "brig-index" indexOpts) - { cwd = cwd, - std_out = CreatePipe, - std_err = CreatePipe - } - - void $ forkIO $ liftIO $ logToConsole (colored green) prefix stdoutHdl - void $ forkIO $ liftIO $ logToConsole (colored green) prefix stderrHdl - exitCode <- liftIO $ waitForProcess ph - case exitCode of - ExitFailure _ -> assertFailure $ prefix <> "failed to reindex users" - ExitSuccess -> pure () diff --git a/integration/test/Test/Apps.hs b/integration/test/Test/Apps.hs index 4428f5e732f..f6de10968b0 100644 --- a/integration/test/Test/Apps.hs +++ b/integration/test/Test/Apps.hs @@ -464,7 +464,6 @@ testFindApp sameOrOtherDomain = do (appA1Id) <- bindResponse (createApp ownerA1 tidA1 newAppA1) $ \resp -> do resp.status `shouldMatchInt` 200 resp.json %. "user.id" & asString - BrigI.refreshIndex domainA (ownerA2, _, [regularMemberA2]) <- createTeam domainA 2 (ownerB1, _, [regularMemberB1]) <- createTeam domainB 2 @@ -589,13 +588,11 @@ testTeamSizeWithApps (TaggedBool testInternalApi) = do resp.json %. "teamSizeRegulars" `shouldMatchInt` (1 + wantRegulars) resp.json %. "teamSizeApps" `shouldMatchInt` wantApps - BrigI.refreshIndex domain eventually $ do checkSize numRegulars numApps deleteTeamMember tid owner (head apps) >>= assertSuccess deleteTeamMember tid owner (head extraMembers) >>= assertSuccess - BrigI.refreshIndex domain eventually $ do checkSize (numRegulars - 1) (numApps - 1) diff --git a/integration/test/Test/Brig.hs b/integration/test/Test/Brig.hs index 962274a4b2c..56958d4c7c7 100644 --- a/integration/test/Test/Brig.hs +++ b/integration/test/Test/Brig.hs @@ -276,7 +276,6 @@ testDeleteEmail = do searchShouldBe :: (HasCallStack) => String -> App () searchShouldBe expected = do - BrigI.refreshIndex OwnDomain bindResponse (BrigP.searchTeamWithSearchTerm owner email) $ \resp -> do resp.status `shouldMatchInt` 200 numDocs <- length <$> (resp.json %. "documents" >>= asList) diff --git a/integration/test/Test/Demo.hs b/integration/test/Test/Demo.hs index 8d93872208d..a7b61e80404 100644 --- a/integration/test/Test/Demo.hs +++ b/integration/test/Test/Demo.hs @@ -4,7 +4,6 @@ module Test.Demo where import qualified API.Brig as BrigP -import qualified API.BrigInternal as BrigI import qualified API.GalleyInternal as GalleyI import qualified API.Nginz as Nginz import GHC.Stack @@ -129,13 +128,12 @@ testStartMultipleDynamicBackends = do (resp.json %. "domain") `shouldMatch` domain startDynamicBackends [def, def, def] $ mapM_ assertCorrectDomain -testIndependentESIndices :: (HasCallStack) => App () -testIndependentESIndices = do +testIndependentBackends :: (HasCallStack) => App () +testIndependentBackends = do u1 <- randomUser OwnDomain def u2 <- randomUser OwnDomain def uid2 <- objId u2 connectTwoUsers u1 u2 - BrigI.refreshIndex OwnDomain bindResponse (BrigP.searchContacts u1 (u2 %. "name") OwnDomain) $ \resp -> do resp.status `shouldMatchInt` 200 docs <- resp.json %. "documents" >>= asList @@ -153,7 +151,6 @@ testIndependentESIndices = do uD2 <- randomUser dynDomain def uidD2 <- objId uD2 connectTwoUsers uD1 uD2 - BrigI.refreshIndex dynDomain -- searching for uD2 on the dyn backend should yield a result bindResponse (BrigP.searchContacts uD1 (uD2 %. "name") dynDomain) $ \resp -> do resp.status `shouldMatchInt` 200 diff --git a/integration/test/Test/Migration/User.hs b/integration/test/Test/Migration/User.hs index 573b3f46e7c..20a77d8725c 100644 --- a/integration/test/Test/Migration/User.hs +++ b/integration/test/Test/Migration/User.hs @@ -44,7 +44,6 @@ import SetupHelpers hiding (deleteUser) import Test.Bot (mkBotService) import Test.Migration.Util import Test.QuickCheck -import Test.Search import Testlib.MockIntegrationService (MockServerSettings (..), withMockServer) import Testlib.Prelude import Testlib.ResourcePool @@ -556,64 +555,6 @@ testUserMigrationToPostgres = withMockServer botServiceSettings mkBotService $ \ bid <- bot %. "qualified_id.id" & asString rmBotSelf mel bid cid >>= assertSuccess --- | This test creates users in PG and Cassandra separately to simulate a --- situation where there are users in both DBs. Then tries to index them into ES --- to make sure the pagination over these users works. -testReindexingUsersDuringMigration :: (HasCallStack) => App () -testReindexingUsersDuringMigration = do - resourcePool <- asks (.resourcePool) - - runCodensity (acquireResources 1 resourcePool) $ \[backend] -> do - let domain = backend.berDomain - -- Create users in cassandra using 'phase1Overrides' - (casSearcher, casExistingUsers, casDeletedUsers) <- - runCodensity (startDynamicBackend backend phase1Overrides) - $ \_ -> setupUsers domain - - -- Create users in postgres using 'phase5Overrides' - (pgSearcher, pgExistingUsers, pgDeletedUsers) <- - runCodensity (startDynamicBackend backend phase5Overrides) - $ \_ -> setupUsers domain - - -- Test that searching in the already existing index works with in - -- 'phase2Overrides', which should work with data in cassandra and postgres - runCodensity (startDynamicBackend backend phase2Overrides) $ \_ -> do - I.refreshIndex domain - checkSearchWorks domain casSearcher casExistingUsers casDeletedUsers - checkSearchWorks domain pgSearcher pgExistingUsers pgDeletedUsers - - newIndex <- createNewIndex - let backendWithNewIndex = backend {berElasticsearchIndex = newIndex} - runCodensity (startDynamicBackend backendWithNewIndex phase2Overrides) $ \_ -> do - reindexUsers backendWithNewIndex phase2Overrides 5 - I.refreshIndex domain - checkSearchWorks domain casSearcher casExistingUsers casDeletedUsers - checkSearchWorks domain pgSearcher pgExistingUsers pgDeletedUsers - where - n = 5 - parallelism = 16 - - setupUsers :: (HasCallStack) => String -> App (Value, [Value], [Value]) - setupUsers domain = do - searcher <- randomUser domain def - existingUsers <- pooledReplicateConcurrentlyN parallelism n $ randomUser domain def - deletedUsers <- pooledReplicateConcurrentlyN parallelism n $ do - u <- randomUser domain def - connectTwoUsers searcher u - pure u - withWebSocket searcher $ \ws -> do - pooledForConcurrentlyN_ parallelism deletedUsers deleteUser - void $ awaitNMatches n isDeleteUserNotif ws - pure (searcher, existingUsers, deletedUsers) - - checkSearchWorks :: (HasCallStack) => String -> Value -> [Value] -> [Value] -> App () - checkSearchWorks domain searcher existingUsers deletedUsers = do - pooledForConcurrentlyN_ parallelism existingUsers $ \u -> - assertCanFind searcher u (u %. "name") domain - - pooledForConcurrentlyN_ parallelism deletedUsers $ \u -> - assertCannotFind searcher u (u %. "name") domain - -- handleA: Alice and Anna have the same handle, but the handle claims table -- supports Alice's claim. After the migration Bob loses their handle. -- diff --git a/integration/test/Test/Search.hs b/integration/test/Test/Search.hs index b4b3edb4b79..22856faa263 100644 --- a/integration/test/Test/Search.hs +++ b/integration/test/Test/Search.hs @@ -26,15 +26,11 @@ import qualified API.Common as API import API.Galley import qualified API.Galley as Galley import qualified API.GalleyInternal as GalleyI -import Control.Monad.Codensity (Codensity (runCodensity)) -import Control.Monad.Reader import qualified Data.Set as Set import GHC.Stack import SetupHelpers import Testlib.Assertions import Testlib.Prelude -import Testlib.ResourcePool (acquireResources) -import UnliftIO (pooledForConcurrentlyN, pooledForConcurrentlyN_) -- * Local Search @@ -84,7 +80,6 @@ testEphemeralUsersSearch :: (HasCallStack) => App () testEphemeralUsersSearch = do userEphemeral <- ephemeralUser OwnDomain [user1, user2] <- replicateM 2 $ randomUser OwnDomain def - BrigI.refreshIndex OwnDomain -- user1 can find user2 BrigP.searchContacts user1 (user2 %. "name") OwnDomain >>= \resp -> do @@ -182,7 +177,6 @@ checkUserSearch d1 d2 = do foundNames :: [String] <- ((%. "name") >=> asString) `mapM` foundDocs foundNames `shouldMatchSet` names - BrigI.refreshIndex d2 forM_ [owner, remoteSearcher] $ \searcher -> do filterByType searcher "chappie" Nothing ["chappie"] @@ -203,7 +197,6 @@ federatedUserSearch d1 d2 test = do u2Handle <- API.randomHandle bindResponse (BrigP.putHandle u2 u2Handle) $ assertSuccess - BrigI.refreshIndex d2 bindResponse (BrigP.searchContacts u1 u2Handle d2) $ \resp -> do resp.status `shouldMatchInt` 200 @@ -255,7 +248,6 @@ testFederatedUserSearchNonTeamSearcher = do u2Handle <- API.randomHandle bindResponse (BrigP.putHandle u2 u2Handle) $ assertSuccess - BrigI.refreshIndex d2 bindResponse (BrigP.searchContacts u1 u2Handle d2) $ \resp -> do resp.status `shouldMatchInt` 200 @@ -284,7 +276,6 @@ testFederatedUserSearchForNonTeamUser = do u2Handle <- API.randomHandle bindResponse (BrigP.putHandle u2 u2Handle) $ assertSuccess - BrigI.refreshIndex d2 bindResponse (BrigP.searchContacts u1 u2Handle d2) $ \resp -> do resp.status `shouldMatchInt` 200 @@ -309,7 +300,6 @@ testSearchForTeamMembersWithRoles = do (owner, tid, m1 : m2 : m3 : m4 : _) <- createTeam OwnDomain 5 [ownerId, m1Id, m2Id, m3Id, m4Id] <- for [owner, m1, m2, m3, m4] objId - BrigI.refreshIndex OwnDomain bindResponse (BrigP.searchTeamAll owner) $ \resp -> do resp.status `shouldMatchInt` 200 docs <- resp.json %. "documents" >>= asList @@ -334,7 +324,6 @@ testSearchForTeamMembersWithRoles = do expectedUserToRoleMapping = expectedRoles >>= \(role, uids) -> [(uid, role) | uid <- uids] toUidRoleTuple doc = (,) <$> (doc %. "id" & asString) <*> (doc %. "role" & asString) - BrigI.refreshIndex OwnDomain bindResponse (BrigP.searchTeamAll owner) $ \resp -> do resp.status `shouldMatchInt` 200 docs <- resp.json %. "documents" >>= asList @@ -368,8 +357,6 @@ testSearchWithDifferentEndpoints = do updateTeamMember tid owner m3 Partner >>= assertSuccess updateTeamMember tid owner m4 Admin >>= assertSuccess - BrigI.refreshIndex dom - (allOfThemUnqualified, allOfThemQualified) <- bindResponse (BrigP.searchTeamAll owner) $ \resp -> do resp.status `shouldMatchInt` 200 docs <- resp.json %. "documents" >>= asList @@ -407,8 +394,6 @@ testTeamSearchEmailFilter = do newUnverified <- API.randomEmail BrigP.updateEmail mem newUnverified cookie token >>= assertSuccess - BrigI.refreshIndex OwnDomain - -- email=verified returns users with verified email and no unverified (owner only) BrigP.searchTeam owner [("email", "verified"), ("size", "100"), ("q", "")] `bindResponse` \resp -> do resp.status `shouldMatchInt` 200 @@ -443,8 +428,6 @@ testTeamSearchUserIncludesUserGroups = do ug2 <- BrigP.createUserGroup owner (object ["name" .= "group 2", "members" .= [mem2id, mem3id, mem4id]]) >>= getJSON 200 >>= objId ug3 <- BrigP.createUserGroup owner (object ["name" .= "group 3", "members" .= [mem2id, mem3id]]) >>= getJSON 200 >>= objId - BrigI.refreshIndex OwnDomain - bindResponse (BrigP.searchTeamAll owner) \resp -> do resp.status `shouldMatchInt` 200 docs <- resp.json %. "documents" >>= asList @@ -492,7 +475,6 @@ testUserSearchable = do -- By default created team members are found. u2id <- u2 %. "id" & asString - BrigI.refreshIndex OwnDomain withFoundDocs u1 (u2 %. "name") $ \docs -> do foundUids <- for docs objId assertBool "u1 must find u2 as they are searchable by default" $ u2id `elem` foundUids @@ -500,7 +482,6 @@ testUserSearchable = do -- User set to non-searchable is not found by other team members. u3id <- u3 %. "id" & asString BrigP.setUserSearchable owner u3id False `bindResponse` \resp -> resp.status `shouldMatchInt` 200 - BrigI.refreshIndex OwnDomain withFoundDocs u1 (u3 %. "name") $ \docs -> do foundUids <- for docs objId assertBool "u1 must not find u3 as they are set non-searchable" $ notElem u3id foundUids @@ -592,19 +573,16 @@ testStealthUsersWithFederation = do assertSuccess =<< GalleyI.setTeamFeatureStatus OtherDomain tid "searchVisibilityInbound" "enabled" BrigP.putSelf searchee (def {BrigP.name = Just searchTerm}) >>= assertSuccess - BrigI.refreshIndex OtherDomain assertCanFind searcher searchee searchTerm OtherDomain BrigP.setUserSearchable owner searcheeId False >>= assertSuccess - BrigI.refreshIndex OtherDomain assertCannotFind searcher searchee searchTerm OtherDomain testSuspendedUserSearch :: (HasCallStack) => App () testSuspendedUserSearch = do [searcher, searchee] <- replicateM 2 $ randomUser OwnDomain def - BrigI.refreshIndex OwnDomain searcheeQid <- objQidObject searchee -- The searcher can find the searchee by default @@ -615,7 +593,6 @@ testSuspendedUserSearch = do BrigI.getAccountStatus searchee `bindResponse` \resp -> do resp.status `shouldMatchInt` 200 resp.json %. "status" `shouldMatch` "suspended" - BrigI.refreshIndex OwnDomain assertCannotFind searcher searcheeQid (searchee %. "name") OwnDomain -- The searcher can find the searchee once the searchee is unsuspended @@ -623,87 +600,8 @@ testSuspendedUserSearch = do BrigI.getAccountStatus searchee `bindResponse` \resp -> do resp.status `shouldMatchInt` 200 resp.json %. "status" `shouldMatch` "active" - BrigI.refreshIndex OwnDomain assertCanFind searcher searcheeQid (searchee %. "name") OwnDomain -testReindexAllUsers :: (HasCallStack) => App () -testReindexAllUsers = do - resourcePool <- asks (.resourcePool) - runCodensity (acquireResources 1 resourcePool) $ \[testBackend] -> do - let domain = testBackend.berDomain - usersOfEachType = 5 - parallelism = 8 - - -- The name changers change their name when the backend is writing to a new - -- ES index. The deleters delete their own account during the same time. - (alice, nameChangers, deleters) <- runCodensity (startDynamicBackend testBackend def) $ \_ -> do - alice <- randomUser domain def - nameChangers <- replicateM usersOfEachType $ randomUser domain def - deleters <- replicateM usersOfEachType $ randomUser domain def - - BrigI.refreshIndex domain - pooledForConcurrentlyN_ parallelism (nameChangers <> deleters) $ \user -> do - assertCanFind alice user (user %. "name") domain - pure (alice, nameChangers, deleters) - - -- Temporarily use a new index, so new users, updates and deletes get - -- written there. - tempIndex <- createNewIndex - (newUsers, changedNames) <- runCodensity (startDynamicBackend (testBackend {berElasticsearchIndex = tempIndex}) def) $ \_ -> do - newUsers <- replicateM usersOfEachType $ randomUser domain def - - BrigI.refreshIndex domain - pooledForConcurrentlyN_ parallelism (nameChangers <> deleters) $ \user -> - assertCannotFind alice user (user %. "name") domain - - pooledForConcurrentlyN_ parallelism (newUsers) $ \user -> - assertCanFind alice user (user %. "name") domain - - changedNames <- pooledForConcurrentlyN parallelism nameChangers $ \user -> do - newName <- API.randomName - BrigP.putSelf user (def {BrigP.name = Just newName}) >>= assertSuccess - BrigI.refreshIndex domain - assertCanFind alice user newName domain - pure newName - - pooledForConcurrentlyN_ parallelism deleters $ \user -> do - deleteUser user - BrigI.refreshIndex domain - assertCannotFind alice user (user %. "name") domain - - pure (newUsers, changedNames) - - let context = - ("alice", alice) - : zipWith (\n user -> ("nameChanger" <> show n, user)) [1 :: Int ..] nameChangers - <> zipWith (\n user -> ("deleter" <> show n, user)) [1 :: Int ..] deleters - <> zipWith (\n user -> ("newUser" <> show n, user)) [1 :: Int ..] newUsers - addUsersToFailureContext context $ do - -- Now if we use the old index, things shouldn't work as expected until a - -- re-index is done. - runCodensity (startDynamicBackend testBackend def) $ \_ -> do - -- Can find people with stale info - pooledForConcurrentlyN_ parallelism (nameChangers <> deleters) $ \user -> do - assertCanFind alice user (user %. "name") domain - - -- New things don't work - pooledForConcurrentlyN_ parallelism (zip changedNames nameChangers) $ \(newName, user) -> - assertCannotFind alice user newName domain - pooledForConcurrentlyN_ parallelism newUsers $ \user -> - assertCannotFind alice user (user %. "name") domain - - -- Reindex users using a small page size so pagination gets excersiced - reindexUsers testBackend def 5 - BrigI.refreshIndex domain - - -- Now things should work as expected - pooledForConcurrentlyN_ parallelism (nameChangers <> deleters) $ \user -> do - assertCannotFind alice user (user %. "name") domain - pooledForConcurrentlyN_ parallelism (zip changedNames nameChangers) $ \(newName, user) -> - assertCanFind alice user newName domain - pooledForConcurrentlyN_ parallelism newUsers $ \user -> - assertCanFind alice user (user %. "name") domain - -- * Assertion Helpers assertCanFind :: diff --git a/integration/test/Test/Teams.hs b/integration/test/Test/Teams.hs index 1553a7da096..40a8db5defb 100644 --- a/integration/test/Test/Teams.hs +++ b/integration/test/Test/Teams.hs @@ -119,7 +119,6 @@ testInvitePersonalUserToTeam = do ids <- for documents ((%. "id") >=> asString) ids `shouldContain` [ownerId] - I.refreshIndex domain -- a team member can now search for the former personal user bindResponse (searchContacts tm (user %. "name") domain) $ \resp -> do resp.status `shouldMatchInt` 200 @@ -356,7 +355,6 @@ testUpgradePersonalToTeam = do shouldBeNull $ owner %. "created_by" mem <- createTeamMember alice' def - I.refreshIndex OwnDomain bindResponse (searchTeamAll alice') $ \resp -> do resp.status `shouldMatchInt` 200 diff --git a/integration/test/Testlib/ModService.hs b/integration/test/Testlib/ModService.hs index a02f921639a..cb85b63bc27 100644 --- a/integration/test/Testlib/ModService.hs +++ b/integration/test/Testlib/ModService.hs @@ -179,7 +179,6 @@ defaultOverrides :: BackendResource -> ServiceOverrides defaultOverrides resource = mconcat [ setKeyspace, - setEsIndex, setPgDb, setFederationSettings, setAwsConfigs, @@ -258,11 +257,6 @@ defaultOverrides resource = backgroundWorkerCfg = setField "postgresql.dbname" resource.berPostgresqlDBName } - setEsIndex :: ServiceOverrides - setEsIndex = - def - { brigCfg = setField "elasticsearch.index" resource.berElasticsearchIndex - } setMlsPrivateKeyPaths :: ServiceOverrides setMlsPrivateKeyPaths = diff --git a/integration/test/Testlib/ResourcePool.hs b/integration/test/Testlib/ResourcePool.hs index 236803a5e77..776a5a4592e 100644 --- a/integration/test/Testlib/ResourcePool.hs +++ b/integration/test/Testlib/ResourcePool.hs @@ -118,7 +118,6 @@ backendResources dynConfs = berGalleyKeyspace = "galley_test_dyn_" <> show i, berSparKeyspace = "spar_test_dyn_" <> show i, berGundeckKeyspace = "gundeck_test_dyn_" <> show i, - berElasticsearchIndex = "directory_dyn_" <> show i <> "_test", berPostgresqlDBName = "dyn-" <> show i, berFederatorInternal = portForDyn (ServiceInternal FederatorInternal) i, berFederatorExternal = dynConf.federatorExternalPort, diff --git a/integration/test/Testlib/RunServices.hs b/integration/test/Testlib/RunServices.hs index 4a9f6403d46..139cb8f8e2d 100644 --- a/integration/test/Testlib/RunServices.hs +++ b/integration/test/Testlib/RunServices.hs @@ -112,7 +112,6 @@ backendA = berGalleyKeyspace = "galley_test", berSparKeyspace = "spar_test", berGundeckKeyspace = "gundeck_test", - berElasticsearchIndex = "directory_test", berPostgresqlDBName = "backendA", berFederatorInternal = servicePort (ServiceInternal FederatorInternal) BackendA, berFederatorExternal = servicePort FederatorExternal BackendA, @@ -150,7 +149,6 @@ backendB = berGalleyKeyspace = "galley_test2", berSparKeyspace = "spar_test2", berGundeckKeyspace = "gundeck_test2", - berElasticsearchIndex = "directory2_test", berPostgresqlDBName = "backendB", berFederatorInternal = servicePort (ServiceInternal FederatorInternal) BackendB, berFederatorExternal = servicePort FederatorExternal BackendB, diff --git a/integration/test/Testlib/Types.hs b/integration/test/Testlib/Types.hs index 2919803d459..48a8feef8d9 100644 --- a/integration/test/Testlib/Types.hs +++ b/integration/test/Testlib/Types.hs @@ -79,7 +79,6 @@ data BackendResource = BackendResource berGalleyKeyspace :: String, berSparKeyspace :: String, berGundeckKeyspace :: String, - berElasticsearchIndex :: String, berPostgresqlDBName :: String, berFederatorInternal :: Word16, berFederatorExternal :: Word16, diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs index 7ef9eead11a..20923ac8023 100644 --- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs +++ b/libs/wire-api/src/Wire/API/Routes/Internal/Brig.hs @@ -31,7 +31,6 @@ module Wire.API.Routes.Internal.Brig AuthAPI, FederationRemotesAPI, EJPDRequest, - ISearchIndexAPI, ProviderAPI, GetAccountConferenceCallingConfig, PutAccountConferenceCallingConfig, @@ -86,7 +85,6 @@ import Wire.API.Routes.Internal.Brig.Connection import Wire.API.Routes.Internal.Brig.EJPD import Wire.API.Routes.Internal.Brig.EnterpriseLogin (EnterpriseLoginApi) import Wire.API.Routes.Internal.Brig.OAuth (OAuthAPI) -import Wire.API.Routes.Internal.Brig.SearchIndex (ISearchIndexAPI) import Wire.API.Routes.Internal.Galley.TeamFeatureNoConfigMulti qualified as Multi import Wire.API.Routes.MultiVerb import Wire.API.Routes.Named @@ -724,7 +722,6 @@ type API = :<|> ClientAPI :<|> AuthAPI :<|> OAuthAPI - :<|> ISearchIndexAPI :<|> FederationRemotesAPI :<|> ProviderAPI :<|> EnterpriseLoginApi diff --git a/libs/wire-api/src/Wire/API/Routes/Internal/Brig/SearchIndex.hs b/libs/wire-api/src/Wire/API/Routes/Internal/Brig/SearchIndex.hs deleted file mode 100644 index 016c41f082f..00000000000 --- a/libs/wire-api/src/Wire/API/Routes/Internal/Brig/SearchIndex.hs +++ /dev/null @@ -1,41 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.API.Routes.Internal.Brig.SearchIndex where - -import Data.Id (UserId) -import Servant (JSON) -import Servant hiding (Handler, JSON, Tagged, addHeader, respond) -import Servant.OpenApi.Internal.Orphans () -import Wire.API.Routes.Named (Named) - -type ISearchIndexAPI = - Named - "indexRefresh" - ( Summary "make index updates visible (e.g. for integration testing)" - :> "index" - :> "refresh" - :> Post '[JSON] NoContent - ) - :<|> Named - "update-search-index" - ( Summary "updates the search index for a single user" - :> "index" - :> "update" - :> Capture "userId" UserId - :> Post '[JSON] NoContent - ) diff --git a/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs b/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs index fd3c3901731..dcb61b0494b 100644 --- a/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs +++ b/libs/wire-api/src/Wire/API/Routes/Public/Brig.hs @@ -1480,7 +1480,6 @@ type ConnectionAPI = \
  • prefix-match the normalized user display name.\ \ \ \

    NB: '@' Does NOT do anything special, ignoring user display names.

    \ - \

    See also: [authoritative ElasticSearch query](https://github.com/wireapp/wire-server/blob/83c25cca6a5e9d2205c102410b452eb78fc50a00/libs/wire-subsystems/src/Wire/IndexedUserStore/ElasticSearch.hs#L251-L288)

    \ \" ] "q" diff --git a/libs/wire-api/src/Wire/API/User/Search.hs b/libs/wire-api/src/Wire/API/User/Search.hs index 13325e91915..5ea1ea6453a 100644 --- a/libs/wire-api/src/Wire/API/User/Search.hs +++ b/libs/wire-api/src/Wire/API/User/Search.hs @@ -35,6 +35,11 @@ module Wire.API.User.Search UserTypeFilter (..), userTypeFilterToText, userTypeFilterToUserType, + TeamSearchInfo (..), + SearchVisibilityInbound (..), + defaultSearchVisibilityInbound, + searchVisibilityInboundFromFeatureStatus, + BrowseTeamFilters (..), ) where @@ -60,7 +65,9 @@ import Data.Text.Ascii (AsciiBase64Url, toText, validateBase64Url) import Data.Text.Encoding qualified as TE import Imports import Servant.API (FromHttpApiData, ToHttpApiData (..)) +import Test.QuickCheck (arbitrary, elements) import Web.Internal.HttpApiData (parseQueryParam) +import Wire.API.Team.Feature (FeatureStatus (..)) import Wire.API.Team.Role (Role) import Wire.API.User (ManagedBy, UserType (..)) import Wire.API.User.Identity (EmailAddress) @@ -429,3 +436,86 @@ instance ToSchema SetSearchable where object $ SetSearchable <$> setSearchable .= field "set_searchable" schema + +-------------------------------------------------------------------------------- +-- TeamSearchInfo / SearchVisibilityInbound + +-- | Outbound search restrictions configured by team admin of the searcher. This +-- value restricts the set of user that are searched. +-- +-- See 'optionallySearchWithinTeam' for the effect on full-text search. +-- +-- See 'mkTeamSearchInfo' for the business logic that defines the TeamSearchInfo +-- value. +-- +-- Search results might be affected by the inbound search restriction settings of +-- the searched user. ('SearchVisibilityInbound') +data TeamSearchInfo + = -- | Only users that are not part of any team are searched + NoTeam + | -- | Only users from the same team as the searcher are searched + TeamOnly TeamId + | -- | No search restrictions, all users are searched + AllUsers + +-- | Inbound search restrictions configured by team to-be-searched. Affects only +-- full-text search (i.e. search on the display name and the handle), not exact +-- handle search. +data SearchVisibilityInbound + = -- | The user can only be found by users from the same team + SearchableByOwnTeam + | -- | The user can by found by any user of any team + SearchableByAllTeams + deriving (Eq, Show) + +instance Arbitrary SearchVisibilityInbound where + arbitrary = elements [SearchableByOwnTeam, SearchableByAllTeams] + +instance ToByteString SearchVisibilityInbound where + builder SearchableByOwnTeam = "searchable-by-own-team" + builder SearchableByAllTeams = "searchable-by-all-teams" + +instance FromByteString SearchVisibilityInbound where + parser = + SearchableByOwnTeam + <$ string "searchable-by-own-team" + <|> SearchableByAllTeams + <$ string "searchable-by-all-teams" + +-- | Integral representation used for persistence (Cassandra and Postgres). +instance C.Cql SearchVisibilityInbound where + ctype = C.Tagged C.IntColumn + + toCql SearchableByOwnTeam = C.CqlInt 0 + toCql SearchableByAllTeams = C.CqlInt 1 + + fromCql (C.CqlInt 0) = pure SearchableByOwnTeam + fromCql (C.CqlInt 1) = pure SearchableByAllTeams + fromCql n = Left $ "Unexpected SearchVisibilityInbound: " ++ show n + +defaultSearchVisibilityInbound :: SearchVisibilityInbound +defaultSearchVisibilityInbound = SearchableByOwnTeam + +searchVisibilityInboundFromFeatureStatus :: FeatureStatus -> SearchVisibilityInbound +searchVisibilityInboundFromFeatureStatus FeatureStatusDisabled = SearchableByOwnTeam +searchVisibilityInboundFromFeatureStatus FeatureStatusEnabled = SearchableByAllTeams + +instance ToJSON SearchVisibilityInbound where + toJSON = String . TE.decodeUtf8 . BS.toByteString' . builder + +instance FromJSON SearchVisibilityInbound where + parseJSON = withText "SearchVisibilityInbound" $ \str -> + case AP.parseOnly (parser @SearchVisibilityInbound) (TE.encodeUtf8 str) of + Left _ -> fail "Invalid SearchVisibilityInbound" + Right result -> pure result + +data BrowseTeamFilters = BrowseTeamFilters + { teamId :: TeamId, + mQuery :: Maybe Text, + mRoleFilter :: Maybe RoleFilter, + mSortBy :: Maybe TeamUserSearchSortBy, + mSortOrder :: Maybe TeamUserSearchSortOrder, + mEmailVerificationFilter :: Maybe EmailVerificationFilter, + mSearchable :: Maybe Bool + } + deriving (Eq, Show) diff --git a/libs/wire-api/wire-api.cabal b/libs/wire-api/wire-api.cabal index e7f4e0886e0..d539f48c6e0 100644 --- a/libs/wire-api/wire-api.cabal +++ b/libs/wire-api/wire-api.cabal @@ -179,7 +179,6 @@ library Wire.API.Routes.Internal.Brig.EJPD Wire.API.Routes.Internal.Brig.EnterpriseLogin Wire.API.Routes.Internal.Brig.OAuth - Wire.API.Routes.Internal.Brig.SearchIndex Wire.API.Routes.Internal.Cannon Wire.API.Routes.Internal.Cargohold Wire.API.Routes.Internal.Enterprise diff --git a/libs/wire-subsystems/default.nix b/libs/wire-subsystems/default.nix index 4d3b1d9cbd0..f40d9d5a481 100644 --- a/libs/wire-subsystems/default.nix +++ b/libs/wire-subsystems/default.nix @@ -21,7 +21,6 @@ , base64-bytestring , bilge , bimap -, bloodhound , bytestring , bytestring-conversion , case-insensitive @@ -170,7 +169,6 @@ mkDerivation { base64-bytestring bilge bimap - bloodhound bytestring bytestring-conversion case-insensitive @@ -306,7 +304,6 @@ mkDerivation { base16-bytestring base64-bytestring bilge - bloodhound bytestring bytestring-conversion case-insensitive diff --git a/libs/wire-subsystems/postgres-migrations/20260911000000-search-store-without-elasticsearch.sql b/libs/wire-subsystems/postgres-migrations/20260911000000-search-store-without-elasticsearch.sql new file mode 100644 index 00000000000..04dcacf4280 --- /dev/null +++ b/libs/wire-subsystems/postgres-migrations/20260911000000-search-store-without-elasticsearch.sql @@ -0,0 +1,24 @@ +-- Stores the ICU-transliterated, lowercased display name +-- (see 'Wire.UserSearch.Normalize.normalized'). Required for +-- case/diacritic-insensitive prefix search of user names, which plain +-- lower() cannot provide ("Björn" -> "bjorn"). +ALTER TABLE wire_user ADD COLUMN name_normalized text; + +-- Prefix search indexes (LIKE 'abc%', no pg_trgm needed). +CREATE INDEX wire_user_name_normalized_pattern_idx ON wire_user (name_normalized text_pattern_ops); +CREATE INDEX wire_user_lower_handle_pattern_idx ON wire_user (lower(handle) text_pattern_ops); + +CREATE INDEX wire_user_team_idx ON wire_user (team); + +-- Replaces the per-user search_visibility_inbound field formerly +-- denormalized into the ElasticSearch user documents. One row per team; +-- a missing row means 'SearchableByOwnTeam'. +CREATE TABLE team_search_visibility ( + team uuid PRIMARY KEY, + search_visibility_inbound integer NOT NULL +); + +-- Deployment note: rows created before this migration have a NULL +-- name_normalized. Run `brig-index backfill-normalized-names` once before +-- switching user search over to Postgres, otherwise name search misses +-- every pre-migration user (handle/email search are unaffected). diff --git a/libs/wire-subsystems/src/Wire/AppSubsystem/Interpreter.hs b/libs/wire-subsystems/src/Wire/AppSubsystem/Interpreter.hs index e1dd13ff30f..6203079547e 100644 --- a/libs/wire-subsystems/src/Wire/AppSubsystem/Interpreter.hs +++ b/libs/wire-subsystems/src/Wire/AppSubsystem/Interpreter.hs @@ -56,7 +56,7 @@ import Wire.TeamSubsystem import Wire.TeamSubsystem.Util import Wire.UserStore (UserStore) import Wire.UserStore qualified as Store -import Wire.UserSubsystem (UserSubsystem, internalUpdateSearchIndex) +import Wire.UserSubsystem (UserSubsystem) runAppSubsystem :: ( Member UserStore r, @@ -96,7 +96,6 @@ createAppImpl :: Member TeamSubsystem r, Member NotificationSubsystem r, Member AuthenticationSubsystem r, - Member UserSubsystem r, Member Random r ) => Local UserId -> @@ -129,7 +128,6 @@ createAppImpl lusr tid newApp = do Store.createUser u Nothing now <- toUTCTimeMillis <$> get void $ addTeamMember u.id tid (Just (tUnqualified lusr, now)) R.RoleMember - internalUpdateSearchIndex u.id -- generate a team event generateTeamEvents creator.id tid [EdMemberJoin u.id] @@ -199,7 +197,6 @@ updateAppImpl :: Member (Error AppSubsystemError) r, Member Events r, Member GalleyAPIAccess r, - Member UserSubsystem r, Member UserStore r ) => Local UserId -> @@ -214,7 +211,6 @@ updateAppImpl lusr tid appid upd = do Right () -> pure () Left Store.NotFound -> throw AppSubsystemErrorNoApp Store.updateUser appid (def {Store.name = upd.name, Store.assets = upd.assets, Store.accentId = upd.accentId}) - internalUpdateSearchIndex appid generateUserEvent appid Nothing $ UserUpdated $ (emptyUserUpdatedData appid) diff --git a/libs/wire-subsystems/src/Wire/BrigAPIAccess.hs b/libs/wire-subsystems/src/Wire/BrigAPIAccess.hs index c23b679ed72..969d4d18e2b 100644 --- a/libs/wire-subsystems/src/Wire/BrigAPIAccess.hs +++ b/libs/wire-subsystems/src/Wire/BrigAPIAccess.hs @@ -111,7 +111,6 @@ data BrigAPIAccess m a where BrigAPIAccess m () GetUserExportData :: UserId -> BrigAPIAccess m (Maybe TeamExportUser) DeleteBot :: ConvId -> BotId -> BrigAPIAccess m () - UpdateSearchIndex :: UserId -> BrigAPIAccess m () GetAccountsBy :: GetBy -> BrigAPIAccess m [User] GetUsersByVariousKeys :: [UserId] -> [Handle] -> [EmailAddress] -> HavePendingInvitations -> BrigAPIAccess m [User] CreateGroupInternal :: ManagedBy -> TeamId -> Maybe UserId -> NewUserGroup -> BrigAPIAccess m (Either Wai.Error UserGroup) diff --git a/libs/wire-subsystems/src/Wire/BrigAPIAccess/Rpc.hs b/libs/wire-subsystems/src/Wire/BrigAPIAccess/Rpc.hs index e42a4791392..748ae85eafa 100644 --- a/libs/wire-subsystems/src/Wire/BrigAPIAccess/Rpc.hs +++ b/libs/wire-subsystems/src/Wire/BrigAPIAccess/Rpc.hs @@ -126,7 +126,6 @@ interpretBrigAccess brigEndpoint = updateSearchVisibilityInbound status DeleteBot convId botId -> deleteBot convId botId - UpdateSearchIndex uid -> updateSearchIndex uid GetAccountsBy localGetBy -> getAccountsBy localGetBy GetUsersByVariousKeys uids handles emails includePendingInvitations -> @@ -581,16 +580,6 @@ getLocalMLSClient lusr cid suite = ) >>= decodeBodyOrThrow "brig" -updateSearchIndex :: - (Member Rpc r, Member (Input Endpoint) r) => - UserId -> - Sem r () -updateSearchIndex uid = do - void . brigRequest $ - method POST - . paths ["i", "index", "update", toByteString' uid] - . expect2xx - -- | Calls 'Brig.API.Internal.getAccountsByInternalH'. getAccountsBy :: (Member Rpc r, Member (Input Endpoint) r, Member (Error ParseException) r) => diff --git a/libs/wire-subsystems/src/Wire/IndexedUserStore.hs b/libs/wire-subsystems/src/Wire/IndexedUserStore.hs deleted file mode 100644 index 2ff09936afb..00000000000 --- a/libs/wire-subsystems/src/Wire/IndexedUserStore.hs +++ /dev/null @@ -1,63 +0,0 @@ -{-# LANGUAGE TemplateHaskell #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.IndexedUserStore where - -import Data.Id -import Database.Bloodhound qualified as ES -import Database.Bloodhound.Types hiding (SearchResult) -import Imports -import Polysemy -import Wire.API.Team.Size -import Wire.API.User.Search -import Wire.UserSearch.Types - -data IndexedUserStoreError - = IndexUpdateError ES.EsError - | IndexLookupError ES.EsError - | IndexError Text - deriving (Show) - -instance Exception IndexedUserStoreError - -data IndexedUserStore m a where - Upsert :: DocId -> UserDoc -> VersionControl -> IndexedUserStore m () - UpdateTeamSearchVisibilityInbound :: - TeamId -> - SearchVisibilityInbound -> - IndexedUserStore m () - -- | Will only be applied to main ES index and not the additional one - BulkUpsert :: [(DocId, UserDoc, VersionControl)] -> IndexedUserStore m () - DoesIndexExist :: IndexedUserStore m Bool - SearchUsers :: - UserId -> - Maybe TeamId -> - TeamSearchInfo -> - Text -> - Int -> - Maybe [UserTypeFilter] -> - IndexedUserStore m (SearchResult UserDoc) - PaginateTeamMembers :: - BrowseTeamFilters -> - Int -> - Maybe PagingState -> - IndexedUserStore m (SearchResult UserDoc) - GetTeamSize :: TeamId -> IndexedUserStore m TeamSize - -makeSem ''IndexedUserStore diff --git a/libs/wire-subsystems/src/Wire/IndexedUserStore/Bulk/ElasticSearch.hs b/libs/wire-subsystems/src/Wire/IndexedUserStore/Bulk/ElasticSearch.hs deleted file mode 100644 index 6317ed7ba2d..00000000000 --- a/libs/wire-subsystems/src/Wire/IndexedUserStore/Bulk/ElasticSearch.hs +++ /dev/null @@ -1,188 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.IndexedUserStore.Bulk.ElasticSearch where - -import Cassandra.Exec (paginateWithStateC) -import Cassandra.Util (Writetime (Writetime)) -import Conduit (ConduitT, runConduit, (.|)) -import Control.Error (headMay) -import Control.Exception (try) -import Control.Monad.Extra (mapMaybeM) -import Data.Conduit.Combinators qualified as Conduit -import Data.Conduit.Internal (zipSources) -import Data.Conduit.List qualified as CL -import Data.Id -import Data.Json.Util (UTCTimeMillis (fromUTCTimeMillis)) -import Data.Map qualified as Map -import Database.Bloodhound qualified as ES -import Imports -import Polysemy -import Polysemy.Error hiding (try) -import Polysemy.TinyLog -import Polysemy.TinyLog qualified as Log -import System.Logger.Message qualified as Log -import UnliftIO (pooledForConcurrentlyN) -import Wire.API.Team.Feature -import Wire.API.Team.Member.Info -import Wire.API.Team.Role -import Wire.GalleyAPIAccess -import Wire.IndexedUserStore (IndexedUserStore) -import Wire.IndexedUserStore qualified as IndexedUserStore -import Wire.IndexedUserStore.MigrationStore -import Wire.IndexedUserStore.MigrationStore qualified as MigrationStore -import Wire.UserSearch.Migration -import Wire.UserSearch.Types -import Wire.UserStore -import Wire.UserStore.IndexUser - -type IOInterpreter r = forall a. Sem r a -> IO a - --- | Increase this number any time you want to force reindexing. -expectedMigrationVersion :: MigrationVersion -expectedMigrationVersion = MigrationVersion 6 - -syncAllUsers :: (Member UserStore r, Member IndexedUserStore r, Member TinyLog r, Member GalleyAPIAccess r) => IOInterpreter r -> Int32 -> IO () -syncAllUsers interpreter pageSize = syncAllUsersWithVersion interpreter pageSize ES.ExternalGT - -forceSyncAllUsers :: (Member UserStore r, Member IndexedUserStore r, Member TinyLog r, Member GalleyAPIAccess r) => IOInterpreter r -> Int32 -> IO () -forceSyncAllUsers interpreter pageSize = syncAllUsersWithVersion interpreter pageSize ES.ExternalGTE - -syncAllUsersWithVersion :: (Member UserStore r, Member IndexedUserStore r, Member TinyLog r, Member GalleyAPIAccess r) => IOInterpreter r -> Int32 -> (ES.ExternalDocVersion -> ES.VersionControl) -> IO () -syncAllUsersWithVersion interpreter pageSize mkVersion = - runConduit $ - zipSources (CL.sourceList [1 ..]) (paginateWithStateC (interpreter . getIndexUsersPaginated pageSize)) - .| logPage - .| mkUserDocs - .| Conduit.mapM_ (interpreter . IndexedUserStore.bulkUpsert) - where - logPage :: ConduitT (Int32, [IndexUser]) [IndexUser] IO () - logPage = Conduit.mapM $ \(pageNumber, page) -> do - interpreter $ - info $ - Log.field "estimatedUserSoFar" (length page + fromIntegral (pageSize * pageNumber)) - . Log.msg (Log.val "Received user page") - . Log.field "firstUser" (maybe "N/A" (idToText . (.userId)) (headMay page)) - pure page - - mkUserDocs :: ConduitT [IndexUser] [(ES.DocId, UserDoc, ES.VersionControl)] IO () - mkUserDocs = Conduit.mapM $ \page -> do - -- FUTUREWORK: extract team visibilities, roles and user type - -- more efficiently sending one query per page - - -- FUTUREWORK: introduce type ExtendedUser (or something), which - -- contains User, Maybe Role, UserType, ..., and pass around - -- ExtendedUser. this should make the code less convoluted. - - let teams :: Map TeamId [IndexUser] = Map.fromListWith (<>) $ mapMaybe (\u -> (,[u]) <$> u.teamId) page - teamIds = Map.keys teams - - visMap <- fmap Map.fromList . pooledForConcurrentlyN 16 teamIds $ \t -> do - x <- try $ interpreter $ teamSearchVisibilityInbound t - pure (t, x) - - let getRoles :: TeamId -> [UserId] -> IO (Map UserId (Either SomeException (WithWritetime Role))) - getRoles tid uids = do - eithMembers <- try $ interpreter $ (.members) <$> selectTeamMemberInfos tid uids - case eithMembers of - Left e -> do - let lenUids = length uids - if lenUids <= 1 - then pure . Map.fromList $ map (,Left e) uids - else do - let (uids1, uids2) = splitAt (lenUids `div` 2) uids - roles1 <- getRoles tid uids1 - roles2 <- getRoles tid uids2 - pure $ Map.union roles1 roles2 - Right tms -> pure . Map.fromList $ mapMaybe (fmap rightSecond . mkRoleWithWriteTime) tms - - roles :: Map UserId (Either SomeException (WithWritetime Role)) <- - fmap Map.unions . pooledForConcurrentlyN 16 (Map.toList teams) $ \(t, us) -> - getRoles t (fmap (.userId) us) - - let vis :: IndexUser -> Either SomeException SearchVisibilityInbound - vis indexUser = - fromMaybe (Right defaultSearchVisibilityInbound) $ flip Map.lookup visMap =<< indexUser.teamId - - mkUserDoc :: IndexUser -> Either SomeException UserDoc - mkUserDoc indexUser = do - currentVis <- vis indexUser - currentRole <- sequence $ Map.lookup indexUser.userId roles - pure $ indexUserToDoc currentVis ((.value) <$> currentRole) indexUser - - mkDocVersion :: IndexUser -> Either SomeException ES.VersionControl - mkDocVersion u = do - roleWithTime <- sequence (Map.lookup u.userId roles) - pure . mkVersion . ES.ExternalDocVersion . docVersion $ indexUserToVersion roleWithTime u - - let docsWithErrors = map (\u -> (userIdToDocId u.userId, mkUserDoc u, mkDocVersion u)) page - interpreter . flip mapMaybeM docsWithErrors $ logAndHush - - rightSecond :: (a, b) -> (a, Either c b) - rightSecond (a, b) = (a, Right b) - - logAndHush :: (Member TinyLog r) => (ES.DocId, Either SomeException UserDoc, Either SomeException ES.VersionControl) -> Sem r (Maybe (ES.DocId, UserDoc, ES.VersionControl)) - logAndHush (docId@(ES.DocId idText), eithUserDoc, eithVersion) = - case (,) <$> eithUserDoc <*> eithVersion of - Left e -> do - Log.err $ - Log.msg (Log.val "Error ocurred while indexing user") - . Log.field "userId" idText - . Log.field "error" (show e) - pure Nothing - Right (userDoc, version) -> pure $ Just (docId, userDoc, version) - - mkRoleWithWriteTime :: TeamMemberInfo -> Maybe (UserId, WithWritetime Role) - mkRoleWithWriteTime tmi = - ( \role -> - ( tmi.userId, - WithWriteTime - { value = role, - writetime = Writetime $ fromUTCTimeMillis tmi.permissionsWriteTime - } - ) - ) - <$> permissionsToRole tmi.permissions - -migrateData :: - (Member (Embed IO) r, Member IndexedUserStore r, Member (Error MigrationException) r, Member IndexedUserMigrationStore r, Member TinyLog r, Member UserStore r, Member GalleyAPIAccess r) => - IOInterpreter r -> - Int32 -> - IO () -migrateData interpreter pageSize = interpreter $ do - unlessM IndexedUserStore.doesIndexExist $ - throw TargetIndexAbsent - MigrationStore.ensureMigrationIndex - foundVersion <- MigrationStore.getLatestMigrationVersion - if expectedMigrationVersion > foundVersion - then do - Log.info $ - Log.msg (Log.val "Migration necessary.") - . Log.field "expectedVersion" expectedMigrationVersion - . Log.field "foundVersion" foundVersion - embed $ forceSyncAllUsers interpreter pageSize - MigrationStore.persistMigrationVersion expectedMigrationVersion - else do - Log.info $ - Log.msg (Log.val "No migration necessary.") - . Log.field "expectedVersion" expectedMigrationVersion - . Log.field "foundVersion" foundVersion - -teamSearchVisibilityInbound :: (Member GalleyAPIAccess r) => TeamId -> Sem r SearchVisibilityInbound -teamSearchVisibilityInbound tid = - searchVisibilityInboundFromFeatureStatus . (.status) - <$> getFeatureConfigForTeam @_ @SearchVisibilityInboundConfig tid diff --git a/libs/wire-subsystems/src/Wire/IndexedUserStore/ElasticSearch.hs b/libs/wire-subsystems/src/Wire/IndexedUserStore/ElasticSearch.hs deleted file mode 100644 index 156f8f6e479..00000000000 --- a/libs/wire-subsystems/src/Wire/IndexedUserStore/ElasticSearch.hs +++ /dev/null @@ -1,689 +0,0 @@ -{-# LANGUAGE RecordWildCards #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.IndexedUserStore.ElasticSearch where - -import Control.Error (lastMay) -import Control.Exception (throwIO) -import Data.Aeson -import Data.Aeson.Key qualified as Key -import Data.Aeson.Types (parseMaybe) -import Data.ByteString qualified as LBS -import Data.ByteString.Builder -import Data.ByteString.Conversion -import Data.Id -import Data.List.NonEmpty (NonEmpty (..)) -import Data.Map.Strict qualified as M -import Data.Text qualified as Text -import Data.Text.Ascii -import Data.Text.Encoding qualified as Text -import Database.Bloodhound qualified as ES -import Imports -import Network.HTTP.Client -import Network.HTTP.Types -import Numeric.Natural (Natural) -import Polysemy -import Wire.API.Team.Role (roleName) -import Wire.API.Team.Size (TeamSize (TeamSize)) -import Wire.API.User.Search -import Wire.IndexedUserStore -import Wire.Sem.Metrics (Metrics) -import Wire.Sem.Metrics qualified as Metrics -import Wire.UserSearch.Metrics -import Wire.UserSearch.Types -import Wire.UserStore.IndexUser - -data ESConn = ESConn - { env :: ES.BHEnv, - indexName :: ES.IndexName - } - -data IndexedUserStoreConfig = IndexedUserStoreConfig - { conn :: ESConn, - additionalConn :: Maybe ESConn - } - -interpretIndexedUserStoreES :: - ( Member (Embed IO) r, - Member Metrics r - ) => - IndexedUserStoreConfig -> - InterpreterFor IndexedUserStore r -interpretIndexedUserStoreES cfg = - interpret $ \case - Upsert docId userDoc versioning -> upsertImpl cfg docId userDoc versioning - UpdateTeamSearchVisibilityInbound tid vis -> - updateTeamSearchVisibilityInboundImpl cfg tid vis - BulkUpsert docs -> bulkUpsertImpl cfg docs - DoesIndexExist -> doesIndexExistImpl cfg - SearchUsers searcherId mSearcherTeam teamSearchInfo term maxResults mTypes -> - searchUsersImpl cfg searcherId mSearcherTeam teamSearchInfo term maxResults mTypes - PaginateTeamMembers filters maxResults mPagingState -> - paginateTeamMembersImpl cfg filters maxResults mPagingState - GetTeamSize tid -> getTeamSizeImpl cfg tid - -getTeamSizeImpl :: - (Member (Embed IO) r) => - IndexedUserStoreConfig -> - TeamId -> - Sem r TeamSize -getTeamSizeImpl cfg tid = do - r <- embed $ ES.runBH cfg.conn.env $ do - res <- ES.searchByType cfg.conn.indexName mappingName search - liftIO $ ES.parseEsResponse res - result <- either (embed . throwIO . IndexLookupError) pure (r :: Either ES.EsError (ES.SearchResult UserDoc)) - let aggs = fromMaybe mempty (ES.aggregations result) - getCount name = maybe 0 (.filterDocCount) $ M.lookup name aggs >>= parseMaybe (parseJSON @FilterResult) - pure $ TeamSize (getCount "regulars") (getCount "apps") - where - teamQ = termQ "team" (idToText tid) - - -- Regular users: type = "regular" or type field absent (legacy documents) - regularQuery = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ teamQ, - ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = - [ termQ "type" "regular", - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = [ES.QueryExistsQuery (ES.FieldName "type")] - } - ] - } - ] - } - - appQuery = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = [teamQ, termQ "type" "app"] - } - - search = - (ES.mkSearch Nothing Nothing) - { ES.size = ES.Size 0, - ES.aggBody = - Just $ - ES.mkAggregations "regulars" (ES.FilterAgg (ES.FilterAggregation (ES.Filter regularQuery) Nothing)) - <> ES.mkAggregations "apps" (ES.FilterAgg (ES.FilterAggregation (ES.Filter appQuery) Nothing)) - } - -upsertImpl :: - forall r. - ( Member (Embed IO) r, - Member Metrics r - ) => - IndexedUserStoreConfig -> - ES.DocId -> - UserDoc -> - ES.VersionControl -> - Sem r () -upsertImpl cfg docId userDoc versioning = do - void $ runInBothES cfg indexDoc - where - indexDoc :: ES.IndexName -> ES.BH (Sem r) () - indexDoc idx = do - r <- ES.indexDocument idx mappingName settings userDoc docId - unless (ES.isSuccess r || ES.isVersionConflict r) $ do - lift $ Metrics.incCounter indexUpdateErrorCounter - res <- liftIO $ ES.parseEsResponse r - liftIO . throwIO . IndexUpdateError . either id id $ res - lift $ Metrics.incCounter indexUpdateSuccessCounter - - settings = ES.defaultIndexDocumentSettings {ES.idsVersionControl = versioning} - -updateTeamSearchVisibilityInboundImpl :: forall r. (Member (Embed IO) r) => IndexedUserStoreConfig -> TeamId -> SearchVisibilityInbound -> Sem r () -updateTeamSearchVisibilityInboundImpl cfg tid vis = - void $ runInBothES cfg updateAllDocs - where - updateAllDocs :: ES.IndexName -> ES.BH (Sem r) () - updateAllDocs idx = do - r <- ES.updateByQuery idx query (Just script) - unless (ES.isSuccess r || ES.isVersionConflict r) $ do - res <- liftIO $ ES.parseEsResponse r - liftIO . throwIO . IndexUpdateError . either id id $ res - - query :: ES.Query - query = ES.TermQuery (ES.Term "team" $ idToText tid) Nothing - - script :: ES.Script - script = ES.Script (Just (ES.ScriptLanguage "painless")) (Just (ES.ScriptInline scriptText)) Nothing Nothing - - -- Unfortunately ES disallows updating ctx._version with a "Update By Query" - scriptText = - "ctx._source." - <> Key.toText searchVisibilityInboundFieldName - <> " = '" - <> Text.decodeUtf8 (toByteString' vis) - <> "';" - -bulkUpsertImpl :: (Member (Embed IO) r) => IndexedUserStoreConfig -> [(ES.DocId, UserDoc, ES.VersionControl)] -> Sem r () -bulkUpsertImpl cfg docs = do - let bhe = cfg.conn.env - ES.IndexName idx = cfg.conn.indexName - ES.MappingName mpp = mappingName - (ES.Server base) = ES.bhServer bhe - baseReq <- embed $ parseRequest (Text.unpack $ base <> "/" <> idx <> "/" <> mpp <> "/_bulk") - let reqWithoutCreds = - baseReq - { method = "POST", - requestHeaders = [(hContentType, "application/x-ndjson")], - requestBody = RequestBodyLBS (toLazyByteString (foldMap encodeActionAndData docs)) - } - req <- embed $ bhe.bhRequestHook reqWithoutCreds - res <- embed $ httpLbs req (ES.bhManager bhe) - unless (ES.isSuccess res) $ do - parsedRes <- liftIO $ ES.parseEsResponse res - liftIO . throwIO . IndexUpdateError . either id id $ parsedRes - where - encodeJSONToString :: (ToJSON a) => a -> Builder - encodeJSONToString = fromEncoding . toEncoding - - encodeActionAndData :: (ES.DocId, UserDoc, ES.VersionControl) -> Builder - encodeActionAndData (docId, userDoc, versionControl) = - encodeJSONToString (bulkIndexAction docId versionControl) - <> "\n" - <> encodeJSONToString userDoc - <> "\n" - - bulkIndexAction :: ES.DocId -> ES.VersionControl -> Value - bulkIndexAction docId versionControl = - let (versionType :: Maybe Text, version) = case versionControl of - ES.NoVersionControl -> (Nothing, Nothing) - ES.InternalVersion v -> (Nothing, Just v) - ES.ExternalGT (ES.ExternalDocVersion v) -> (Just "external", Just v) - ES.ExternalGTE (ES.ExternalDocVersion v) -> (Just "external_gte", Just v) - ES.ForceVersion (ES.ExternalDocVersion v) -> (Just "force", Just v) - in object - [ "index" - .= object - [ "_id" .= docId, - "version_type" .= versionType, - "version" .= version - ] - ] - -doesIndexExistImpl :: (Member (Embed IO) r) => IndexedUserStoreConfig -> Sem r Bool -doesIndexExistImpl cfg = do - (mainExists, fromMaybe True -> additionalExists) <- runInBothES cfg ES.indexExists - pure $ mainExists && additionalExists - -searchUsersImpl :: - (Member (Embed IO) r) => - IndexedUserStoreConfig -> - UserId -> - Maybe TeamId -> - TeamSearchInfo -> - Text -> - Int -> - Maybe [UserTypeFilter] -> - Sem r (SearchResult UserDoc) -searchUsersImpl cfg searcherId mSearcherTeam teamSearchInfo term maxResults mTypes = do - queryIndex cfg maxResults $ - defaultUserQuery searcherId mSearcherTeam teamSearchInfo mTypes term - --- | The default or canonical 'IndexQuery'. --- --- The intention behind parameterising 'queryIndex' over the 'IndexQuery' is that --- it allows to experiment with different queries (perhaps in an A/B context). --- --- FUTUREWORK: Drop legacyPrefixMatch -defaultUserQuery :: UserId -> Maybe TeamId -> TeamSearchInfo -> Maybe [UserTypeFilter] -> Text -> IndexQuery Contact -defaultUserQuery searcher mSearcherTeamId teamSearchInfo mTypes (normalized -> term') = - let matchPhraseOrPrefix = - ES.QueryMultiMatchQuery $ - ( ES.mkMultiMatchQuery - [ ES.FieldName "handle.prefix^2", - ES.FieldName "normalized.prefix", - ES.FieldName "normalized^3" - ] - (ES.QueryString term') - ) - { ES.multiMatchQueryType = Just ES.MultiMatchMostFields, - ES.multiMatchQueryOperator = ES.And - } - query = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = [matchPhraseOrPrefix], - -- This removes exact handle matches, as they are fetched from cassandra - ES.boolQueryMustNotMatch = [termQ "handle" term'] - } - ], - ES.boolQueryShouldMatch = [ES.QueryExistsQuery (ES.FieldName "handle")] - } - -- This reduces relevance on users not in team of search by 90% (no - -- science behind that number). If the searcher is not part of a team the - -- relevance is not reduced for any users. - queryWithBoost = - ES.QueryBoostingQuery - ES.BoostingQuery - { ES.positiveQuery = query, - ES.negativeQuery = maybe ES.QueryMatchNoneQuery matchUsersNotInTeam mSearcherTeamId, - ES.negativeBoost = ES.Boost 0.1 - } - in mkUserQuery searcher mSearcherTeamId teamSearchInfo mTypes queryWithBoost - -paginateTeamMembersImpl :: - (Member (Embed IO) r) => - IndexedUserStoreConfig -> - BrowseTeamFilters -> - Int -> - Maybe PagingState -> - Sem r (SearchResult UserDoc) -paginateTeamMembersImpl cfg BrowseTeamFilters {..} maxResults mPagingState = do - let (IndexQuery q f sortSpecs) = - teamUserSearchQuery teamId mQuery mRoleFilter mSortBy mSortOrder mEmailVerificationFilter mSearchable - let search = - (ES.mkSearch (Just q) (Just f)) - { -- we are requesting one more result than the page size to determine if there is a next page - ES.size = ES.Size (fromIntegral maxResults + 1), - ES.sortBody = Just (fmap ES.DefaultSortSpec sortSpecs), - ES.searchAfterKey = toSearchAfterKey =<< mPagingState - } - mkResult <$> searchInMainIndex cfg search - where - toSearchAfterKey ps = decode' . LBS.fromStrict =<< (decodeBase64Url . unPagingState) ps - - fromSearchAfterKey :: ES.SearchAfterKey -> PagingState - fromSearchAfterKey = PagingState . encodeBase64Url . LBS.toStrict . encode - - mkResult es = - let hitsPlusOne = ES.hits . ES.searchHits $ es - hits = take (fromIntegral maxResults) hitsPlusOne - mps = fromSearchAfterKey <$> lastMay (mapMaybe ES.hitSort hits) - results = mapMaybe ES.hitSource hits - in SearchResult - { searchFound = ES.hitsTotalValue . ES.hitsTotal . ES.searchHits $ es, - searchReturned = length results, - searchTook = ES.took es, - searchResults = results, - searchPolicy = FullSearch, - searchPagingState = mps, - searchHasMore = Just $ length hitsPlusOne > length hits - } - -searchInMainIndex :: forall r. (Member (Embed IO) r) => IndexedUserStoreConfig -> ES.Search -> Sem r (ES.SearchResult UserDoc) -searchInMainIndex cfg search = do - r <- ES.runBH cfg.conn.env $ do - res <- ES.searchByType cfg.conn.indexName mappingName search - liftIO $ ES.parseEsResponse res - either (embed . throwIO . IndexLookupError) pure r - -queryIndex :: - (Member (Embed IO) r) => - IndexedUserStoreConfig -> - Int -> - IndexQuery x -> - Sem r (SearchResult UserDoc) -queryIndex cfg s (IndexQuery q f _) = do - let search = (ES.mkSearch (Just q) (Just f)) {ES.size = ES.Size (fromIntegral s)} - mkResult <$> searchInMainIndex cfg search - where - mkResult es = - let results = mapMaybe ES.hitSource . ES.hits . ES.searchHits $ es - in SearchResult - { searchFound = ES.hitsTotalValue . ES.hitsTotal . ES.searchHits $ es, - searchReturned = length results, - searchTook = ES.took es, - searchResults = results, - searchPolicy = FullSearch, - searchPagingState = Nothing, - searchHasMore = Nothing - } - -teamUserSearchQuery :: - TeamId -> - Maybe Text -> - Maybe RoleFilter -> - Maybe TeamUserSearchSortBy -> - Maybe TeamUserSearchSortOrder -> - Maybe EmailVerificationFilter -> - Maybe Bool -> - IndexQuery TeamContact -teamUserSearchQuery tid mbSearchText mRoleFilter mSortBy mSortOrder mEmailFilter mSearchable = - IndexQuery - ( maybe - (ES.MatchAllQuery Nothing) - matchPhraseOrPrefix - mbQStr - ) - teamFilter - -- in combination with pagination a non-unique search specification can lead to missing results - -- therefore we use the unique `_doc` value as a tie breaker - -- - see https://www.elastic.co/guide/en/elasticsearch/reference/6.8/search-request-sort.html for details on `_doc` - -- - see https://www.elastic.co/guide/en/elasticsearch/reference/6.8/search-request-search-after.html for details on pagination and tie breaker - -- in the latter article it "is advised to duplicate (client side or [...]) the content of the _id field - -- in another field that has doc value enabled and to use this new field as the tiebreaker for the sort" - -- so alternatively we could use the user ID as a tie breaker, but this would require a change in the index mapping - (sorting ++ sortingTieBreaker) - where - sorting :: [ES.DefaultSort] - sorting = - maybe - [defaultSort SortByCreatedAt SortOrderDesc | isNothing mbQStr] - (\tuSortBy -> [defaultSort tuSortBy (fromMaybe SortOrderAsc mSortOrder)]) - mSortBy - sortingTieBreaker :: [ES.DefaultSort] - sortingTieBreaker = [ES.DefaultSort (ES.FieldName "_doc") ES.Ascending Nothing Nothing Nothing Nothing] - - mbQStr :: Maybe Text - mbQStr = - case mbSearchText of - Nothing -> Nothing - Just q -> - case normalized q of - "" -> Nothing - term' -> Just term' - - matchPhraseOrPrefix term' = - ES.QueryMultiMatchQuery $ - ( ES.mkMultiMatchQuery - [ ES.FieldName "email^4", - ES.FieldName "handle^4", - ES.FieldName "normalized^3", - ES.FieldName "email.prefix^3", - ES.FieldName "handle.prefix^2", - ES.FieldName "normalized.prefix" - ] - (ES.QueryString term') - ) - { ES.multiMatchQueryType = Just ES.MultiMatchMostFields, - ES.multiMatchQueryOperator = ES.And - } - - teamFilter :: ES.Filter - teamFilter = ES.Filter $ ES.QueryBoolQuery boolQuery {ES.boolQueryMustMatch = mustMatch} - where - mustMatch :: [ES.Query] - mustMatch = ES.TermQuery (ES.Term "team" $ idToText tid) Nothing : roleFilter <> emailFilter <> searchableFilter - - searchableFilter :: [ES.Query] - searchableFilter = case mSearchable of - Just False -> [ES.TermQuery (ES.Term "searchable" "false") Nothing] - Just True -> [ES.QueryBoolQuery boolQuery {ES.boolQueryMustNotMatch = [ES.TermQuery (ES.Term "searchable" "false") Nothing]}] - Nothing -> [] - - roleFilter :: [ES.Query] - roleFilter = - case mRoleFilter of - Nothing -> [] - Just (RoleFilter []) -> [] - Just (RoleFilter (r : rs)) -> [ES.TermsQuery "role" (roleName <$> r :| rs)] - - emailFilter :: [ES.Query] - emailFilter = - case mEmailFilter of - Nothing -> [] - -- Verified: must have a verified email and must NOT have an unverified email - Just EmailVerified -> - [ ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = [ES.QueryExistsQuery (ES.FieldName "email")], - ES.boolQueryMustNotMatch = [ES.QueryExistsQuery (ES.FieldName "email_unvalidated")] - } - ] - -- Unverified: must have an unverified email, regardless of verified - Just EmailUnverified -> [ES.QueryExistsQuery (ES.FieldName "email_unvalidated")] - - defaultSort :: TeamUserSearchSortBy -> TeamUserSearchSortOrder -> ES.DefaultSort - defaultSort tuSortBy sortOrder = - ES.DefaultSort - ( case tuSortBy of - SortByName -> ES.FieldName "name" - SortByHandle -> ES.FieldName "handle.keyword" - SortByEmail -> ES.FieldName "email.keyword" - SortBySAMLIdp -> ES.FieldName "saml_idp" - SortByManagedBy -> ES.FieldName "managed_by" - SortByRole -> ES.FieldName "role" - SortByCreatedAt -> ES.FieldName "created_at" - ) - ( case sortOrder of - SortOrderAsc -> ES.Ascending - SortOrderDesc -> ES.Descending - ) - Nothing - Nothing - Nothing - Nothing - -mkUserQuery :: UserId -> Maybe TeamId -> TeamSearchInfo -> Maybe [UserTypeFilter] -> ES.Query -> IndexQuery Contact -mkUserQuery searcher mSearcherTeamId teamSearchInfo mTypes q = - IndexQuery - q - ( ES.Filter - . ES.QueryBoolQuery - $ boolQuery - { ES.boolQueryMustNotMatch = - maybeToList (matchSelf searcher) - <> - -- The following matches both where searchable is true - -- or where the field is missing. There didn't seem to - -- be a more readable way to express - -- "not(exists(searchable)) or searchable = true" in - -- Elastic Search. - [ES.TermQuery (ES.Term "searchable" "false") Nothing] - <> - -- Exclude apps from other teams - maybeToList (matchAppsFromOtherTeams mSearcherTeamId), - ES.boolQueryMustMatch = - [ restrictSearchSpaceByTeam mSearcherTeamId teamSearchInfo, - restrictSearchSpaceByUserType mTypes, - ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = - [ termQ "account_status" "active", - -- Also match entries where the account_status field is not present. - -- These must have been inserted before we added the account_status - -- and at that time we only inserted active users in the first place. - -- This should be unnecessary after re-indexing, but let's be lenient - -- here for a while. - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = - [ES.QueryExistsQuery (ES.FieldName "account_status")] - } - ] - } - ] - } - ) - [] - -termQ :: Text -> Text -> ES.Query -termQ f v = - ES.TermQuery - ES.Term - { ES.termField = f, - ES.termValue = v - } - Nothing - -matchSelf :: UserId -> Maybe ES.Query -matchSelf searcher = Just (termQ "_id" (idToText searcher)) - --- | Exclude apps from other teams. --- Apps should only be searchable within their own team. -matchAppsFromOtherTeams :: Maybe TeamId -> Maybe ES.Query -matchAppsFromOtherTeams mSearcherTeamId = - Just $ - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ -- Match apps (type = "app") - termQ "type" "app", - -- That are from a different team than the searcher - case mSearcherTeamId of - -- If searcher has no team, exclude all apps - Nothing -> - ES.QueryExistsQuery (ES.FieldName "team") - -- If searcher has a team, exclude apps from other teams or with no team - Just searcherTeam -> - ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = - [ -- Apps with no team - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = - [ES.QueryExistsQuery (ES.FieldName "team")] - }, - -- Apps from a different team - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ES.QueryExistsQuery (ES.FieldName "team")], - ES.boolQueryMustNotMatch = - [termQ "team" (idToText searcherTeam)] - } - ] - } - ] - } - --- | See 'TeamSearchInfo' -restrictSearchSpaceByTeam :: Maybe TeamId -> TeamSearchInfo -> ES.Query --- --- FUTUREWORK(fisx): can this commented-out code be removed? if not, --- there should be a comment explaining why. --- --- restrictSearchSpace (FederatedSearch Nothing) = --- ES.QueryBoolQuery --- boolQuery --- { ES.boolQueryShouldMatch = --- [ matchNonTeamMemberUsers, --- matchTeamMembersSearchableByAllTeams --- ] --- } --- restrictSearchSpace (FederatedSearch (Just [])) = --- ES.QueryBoolQuery --- boolQuery --- { ES.boolQueryMustMatch = --- [ -- if the list of allowed teams is empty, this is impossible to fulfill, and no results will be returned --- -- this case should be handled earlier, so this is just a safety net --- ES.TermQuery (ES.Term "team" "must not match any team") Nothing --- ] --- } --- restrictSearchSpace (FederatedSearch (Just teams)) = --- ES.QueryBoolQuery --- boolQuery --- { ES.boolQueryMustMatch = --- [ matchTeamMembersSearchableByAllTeams, --- onlyInTeams --- ] --- } --- where --- onlyInTeams = ES.QueryBoolQuery boolQuery {ES.boolQueryShouldMatch = map matchTeamMembersOf teams} -restrictSearchSpaceByTeam mteam searchInfo = - case (mteam, searchInfo) of - (Nothing, _) -> matchNonTeamMemberUsers - (Just _, NoTeam) -> matchNonTeamMemberUsers - (Just searcherTeam, TeamOnly team) -> - if searcherTeam == team - then matchTeamMembersOf team - else ES.QueryMatchNoneQuery - (Just searcherTeam, AllUsers) -> - ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = - [ matchNonTeamMemberUsers, - matchTeamMembersSearchableByAllTeams, - matchTeamMembersOf searcherTeam - ] - } - -restrictSearchSpaceByUserType :: Maybe [UserTypeFilter] -> ES.Query -restrictSearchSpaceByUserType = \case - -- Nothing (param omitted) and Just [] (param present but empty) both mean - -- "no filter" to avoid surprising empty-result regressions when clients send - -- `type=` without values. - Nothing -> ES.MatchAllQuery Nothing - Just [] -> ES.MatchAllQuery Nothing - Just uts@(utsH : utsT) -> - if UserTypeFilterRegular `elem` uts - then - ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = - [ ES.TermsQuery "type" (userTypeFilterToText <$> (utsH :| utsT)), - -- Older index entries may lack the "type" field; treat those as regular. - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = - [ES.QueryExistsQuery (ES.FieldName "type")] - } - ] - } - else ES.TermsQuery "type" (userTypeFilterToText <$> (utsH :| utsT)) - -matchTeamMembersOf :: TeamId -> ES.Query -matchTeamMembersOf team = ES.TermQuery (ES.Term "team" $ idToText team) Nothing - -matchTeamMembersSearchableByAllTeams :: ES.Query -matchTeamMembersSearchableByAllTeams = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ ES.QueryExistsQuery $ ES.FieldName "team", - ES.TermQuery (ES.Term (Key.toText searchVisibilityInboundFieldName) "searchable-by-all-teams") Nothing - ] - } - -matchNonTeamMemberUsers :: ES.Query -matchNonTeamMemberUsers = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = [ES.QueryExistsQuery $ ES.FieldName "team"] - } - -matchUsersNotInTeam :: TeamId -> ES.Query -matchUsersNotInTeam tid = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = [ES.TermQuery (ES.Term "team" $ idToText tid) Nothing] - } - --------------------------------------------- --- Utils - -runInBothES :: (Monad m) => IndexedUserStoreConfig -> (ES.IndexName -> ES.BH m a) -> m (a, Maybe a) -runInBothES cfg f = do - x <- ES.runBH cfg.conn.env $ f cfg.conn.indexName - y <- forM cfg.additionalConn $ \additional -> - ES.runBH additional.env $ f additional.indexName - pure (x, y) - -mappingName :: ES.MappingName -mappingName = ES.MappingName "user" - -boolQuery :: ES.BoolQuery -boolQuery = ES.mkBoolQuery [] [] [] [] - --- | (or can something like this be found in bloodhound?) -newtype FilterResult = FilterResult {filterDocCount :: Natural} - -instance FromJSON FilterResult where - parseJSON = withObject "FilterResult" $ \o -> FilterResult <$> o .: "doc_count" diff --git a/libs/wire-subsystems/src/Wire/IndexedUserStore/MigrationStore.hs b/libs/wire-subsystems/src/Wire/IndexedUserStore/MigrationStore.hs deleted file mode 100644 index 6c1fbbcc44a..00000000000 --- a/libs/wire-subsystems/src/Wire/IndexedUserStore/MigrationStore.hs +++ /dev/null @@ -1,30 +0,0 @@ -{-# LANGUAGE TemplateHaskell #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.IndexedUserStore.MigrationStore where - -import Polysemy -import Wire.UserSearch.Migration - -data IndexedUserMigrationStore m a where - EnsureMigrationIndex :: IndexedUserMigrationStore m () - GetLatestMigrationVersion :: IndexedUserMigrationStore m MigrationVersion - PersistMigrationVersion :: MigrationVersion -> IndexedUserMigrationStore m () - -makeSem ''IndexedUserMigrationStore diff --git a/libs/wire-subsystems/src/Wire/IndexedUserStore/MigrationStore/ElasticSearch.hs b/libs/wire-subsystems/src/Wire/IndexedUserStore/MigrationStore/ElasticSearch.hs deleted file mode 100644 index 927238702dc..00000000000 --- a/libs/wire-subsystems/src/Wire/IndexedUserStore/MigrationStore/ElasticSearch.hs +++ /dev/null @@ -1,90 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.IndexedUserStore.MigrationStore.ElasticSearch where - -import Data.Aeson -import Data.Text qualified as Text -import Database.Bloodhound qualified as ES -import Imports -import Polysemy -import Polysemy.Error -import Polysemy.TinyLog -import System.Logger.Message qualified as Log -import Wire.IndexedUserStore.MigrationStore -import Wire.Sem.Logger qualified as Log -import Wire.UserSearch.Migration - -interpretIndexedUserMigrationStoreES :: (Member (Embed IO) r, Member (Error MigrationException) r, Member TinyLog r) => ES.BHEnv -> ES.IndexName -> InterpreterFor IndexedUserMigrationStore r -interpretIndexedUserMigrationStoreES env migrationIndexName = interpret $ \case - EnsureMigrationIndex -> ensureMigrationIndexImpl env migrationIndexName - GetLatestMigrationVersion -> getLatestMigrationVersionImpl env migrationIndexName - PersistMigrationVersion v -> persistMigrationVersionImpl env v migrationIndexName - -ensureMigrationIndexImpl :: (Member TinyLog r, Member (Embed IO) r, Member (Error MigrationException) r) => ES.BHEnv -> ES.IndexName -> Sem r () -ensureMigrationIndexImpl env migrationIndexName = do - unlessM (ES.runBH env $ ES.indexExists migrationIndexName) $ do - Log.info $ - Log.msg (Log.val "Creating migrations index, used for tracking which migrations have run") - ES.runBH env (ES.createIndexWith [] 1 migrationIndexName) - >>= throwIfNotCreated CreateMigrationIndexFailed - ES.runBH env (ES.putNamedMapping migrationIndexName migrationMappingName migrationIndexMapping) - >>= throwIfNotCreated PutMappingFailed - where - throwIfNotCreated mkErr response = - unless (ES.isSuccess response) $ - throw $ - mkErr (show response) - -getLatestMigrationVersionImpl :: (Member (Embed IO) r, Member (Error MigrationException) r) => ES.BHEnv -> ES.IndexName -> Sem r MigrationVersion -getLatestMigrationVersionImpl env migrationIndexName = do - reply <- ES.runBH env $ ES.searchByIndex migrationIndexName (ES.mkSearch Nothing Nothing) - resp <- liftIO $ ES.parseEsResponse reply - result <- either (throw . FetchMigrationVersionsFailed . show) pure resp - let versions = map ES.hitSource $ ES.hits . ES.searchHits $ result - case versions of - [] -> - pure $ MigrationVersion 0 - vs -> - if any isNothing vs - then throw $ VersionSourceMissing result - else pure $ maximum $ catMaybes vs - -persistMigrationVersionImpl :: (Member (Embed IO) r, Member TinyLog r, Member (Error MigrationException) r) => ES.BHEnv -> MigrationVersion -> ES.IndexName -> Sem r () -persistMigrationVersionImpl env v migrationIndexName = do - let docId = ES.DocId . Text.pack . show $ migrationVersion v - persistResponse <- ES.runBH env $ ES.indexDocument migrationIndexName migrationMappingName ES.defaultIndexDocumentSettings v docId - if ES.isCreated persistResponse - then do - Log.info $ - Log.msg (Log.val "Migration success recorded") - . Log.field "migrationVersion" v - else throw $ PersistVersionFailed v $ show persistResponse - -defaultMigrationIndexName :: ES.IndexName -defaultMigrationIndexName = ES.IndexName "wire_brig_migrations" - -migrationMappingName :: ES.MappingName -migrationMappingName = ES.MappingName "wire_brig_migrations" - -migrationIndexMapping :: Value -migrationIndexMapping = - object - [ "properties" - .= object - ["migration_version" .= object ["index" .= True, "type" .= ("integer" :: Text)]] - ] diff --git a/libs/wire-subsystems/src/Wire/UserSearch/Metrics.hs b/libs/wire-subsystems/src/Wire/UserSearch/Metrics.hs deleted file mode 100644 index f35b5dab2f1..00000000000 --- a/libs/wire-subsystems/src/Wire/UserSearch/Metrics.hs +++ /dev/null @@ -1,61 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.UserSearch.Metrics where - -import Imports -import Prometheus qualified as Prom - -{-# NOINLINE indexUpdateCounter #-} -indexUpdateCounter :: Prom.Counter -indexUpdateCounter = - Prom.unsafeRegister $ - Prom.counter - Prom.Info - { Prom.metricName = "user_index_update_count", - Prom.metricHelp = "Number of updates on user index" - } - -{-# NOINLINE indexUpdateErrorCounter #-} -indexUpdateErrorCounter :: Prom.Counter -indexUpdateErrorCounter = - Prom.unsafeRegister $ - Prom.counter - Prom.Info - { Prom.metricName = "user_index_update_err", - Prom.metricHelp = "Number of errors during user index update" - } - -{-# NOINLINE indexUpdateSuccessCounter #-} -indexUpdateSuccessCounter :: Prom.Counter -indexUpdateSuccessCounter = - Prom.unsafeRegister $ - Prom.counter - Prom.Info - { Prom.metricName = "user_index_update_ok", - Prom.metricHelp = "Number of successful user index updates" - } - -{-# NOINLINE indexDeleteCounter #-} -indexDeleteCounter :: Prom.Counter -indexDeleteCounter = - Prom.unsafeRegister $ - Prom.counter - Prom.Info - { Prom.metricName = "user_index_delete_count", - Prom.metricHelp = "Number of deletes on user index" - } diff --git a/libs/wire-subsystems/src/Wire/UserSearch/Migration.hs b/libs/wire-subsystems/src/Wire/UserSearch/Migration.hs deleted file mode 100644 index 817d10370e9..00000000000 --- a/libs/wire-subsystems/src/Wire/UserSearch/Migration.hs +++ /dev/null @@ -1,47 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.UserSearch.Migration where - -import Data.Aeson -import Database.Bloodhound.Types qualified as ES -import Imports -import Numeric.Natural -import System.Logger.Class (ToBytes (..)) - -newtype MigrationVersion = MigrationVersion {migrationVersion :: Natural} - deriving (Show, Eq, Ord) - -instance ToJSON MigrationVersion where - toJSON (MigrationVersion v) = object ["migration_version" .= v] - -instance FromJSON MigrationVersion where - parseJSON = withObject "MigrationVersion" $ \o -> MigrationVersion <$> o .: "migration_version" - -instance ToBytes MigrationVersion where - bytes = bytes . toInteger . migrationVersion - -data MigrationException - = CreateMigrationIndexFailed String - | FetchMigrationVersionsFailed String - | PersistVersionFailed MigrationVersion String - | PutMappingFailed String - | TargetIndexAbsent - | VersionSourceMissing (ES.SearchResult MigrationVersion) - deriving (Show) - -instance Exception MigrationException diff --git a/services/brig/src/Brig/Index/Types.hs b/libs/wire-subsystems/src/Wire/UserSearch/Normalize.hs similarity index 51% rename from services/brig/src/Brig/Index/Types.hs rename to libs/wire-subsystems/src/Wire/UserSearch/Normalize.hs index ef6c2da80c1..012828c47df 100644 --- a/services/brig/src/Brig/Index/Types.hs +++ b/libs/wire-subsystems/src/Wire/UserSearch/Normalize.hs @@ -1,6 +1,6 @@ -- This file is part of the Wire Server implementation. -- --- Copyright (C) 2022 Wire Swiss GmbH +-- Copyright (C) 2026 Wire Swiss GmbH -- -- This program is free software: you can redistribute it and/or modify it under -- the terms of the GNU Affero General Public License as published by the Free @@ -15,14 +15,20 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module Brig.Index.Types where +module Wire.UserSearch.Normalize + ( normalized, + ) +where -import Database.Bloodhound qualified as ES -import Imports +import Data.Text (Text) +import Data.Text.ICU.Translit (trans, transliterate) -data CreateIndexSettings = CreateIndexSettings - { _cisIndexSettings :: [ES.UpdatableIndexSetting], - _cisShardCount :: Int, - _cisDeleteTemplate :: Maybe ES.TemplateName - } - deriving (Show) +-- | Normalizes a name (or search term) for matching: transliterate to +-- Latin, strip diacritics, lowercase. ("Björn" -> "bjorn") +-- +-- This is the same function that used to be applied when writing the +-- ElasticSearch user documents (formerly 'Wire.UserStore.IndexUser.normalized'); +-- it is now applied when writing @wire_user.name_normalized@ and when +-- building search queries. +normalized :: Text -> Text +normalized = transliterate (trans "Any-Latin; Latin-ASCII; Lower") diff --git a/libs/wire-subsystems/src/Wire/UserSearch/Types.hs b/libs/wire-subsystems/src/Wire/UserSearch/Types.hs deleted file mode 100644 index 5e8dcac765e..00000000000 --- a/libs/wire-subsystems/src/Wire/UserSearch/Types.hs +++ /dev/null @@ -1,275 +0,0 @@ -{-# LANGUAGE RecordWildCards #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.UserSearch.Types where - -import Cassandra qualified as C -import Cassandra.Util -import Data.Aeson -import Data.Attoparsec.ByteString -import Data.ByteString.Builder -import Data.ByteString.Conversion -import Data.ByteString.Lazy -import Data.Handle -import Data.Id -import Data.Json.Util -import Data.Qualified -import Data.Text.Encoding -import Database.Bloodhound.Types -import Imports -import Test.QuickCheck -import Wire.API.Team.Feature -import Wire.API.Team.Role -import Wire.API.User -import Wire.API.User.Search -import Wire.Arbitrary -import Wire.StoredUser - -newtype IndexVersion = IndexVersion {docVersion :: DocVersion} - -mkIndexVersion :: [Maybe (Writetime x)] -> IndexVersion -mkIndexVersion writetimes = - let maxVersion = getMax . mconcat . fmap (Max . writetimeToInt64) $ catMaybes writetimes - in -- This minBound case would only get triggered when the maxVersion is <= 0 - -- or >= 9.2e+18. First case can happen when the writetimes list is empty - -- or contains a timestamp before the unix epoch, which is unlikely. - -- Second case will happen in a few billion years. It is also not really a - -- restriction in ES, Bloodhound's authors' interpretation of the the ES - -- documentation caused this limiation, otherwise `maxBound :: Int64`, - -- would be acceptable by ES. - IndexVersion . fromMaybe minBound . mkDocVersion . fromIntegral $ maxVersion - --- | Represents an ES *document*, ie. the subset of user attributes stored in ES. --- See also 'IndexUser'. --- --- If a user is not searchable, e.g. because the account got --- suspended, all fields except for the user id are set to 'Nothing' and --- consequently removed from the index. -data UserDoc = UserDoc - { udId :: UserId, - udType :: Maybe UserType, - udTeam :: Maybe TeamId, - udName :: Maybe Name, - udNormalized :: Maybe Text, - udHandle :: Maybe Handle, - udEmail :: Maybe EmailAddress, - udColourId :: Maybe ColourId, - udAccountStatus :: Maybe AccountStatus, - udSAMLIdP :: Maybe Text, - udManagedBy :: Maybe ManagedBy, - udCreatedAt :: Maybe UTCTimeMillis, - udRole :: Maybe Role, - udSearchVisibilityInbound :: Maybe SearchVisibilityInbound, - udScimExternalId :: Maybe Text, - udSso :: Maybe Sso, - udEmailUnvalidated :: Maybe EmailAddress, - udSearchable :: Maybe Bool - } - deriving (Eq, Show, Generic) - deriving (Arbitrary) via (GenericUniform UserDoc) - -instance ToJSON UserDoc where - toJSON ud = - object - [ "id" .= udId ud, - "type" .= udType ud, - "team" .= udTeam ud, - "name" .= udName ud, - "normalized" .= udNormalized ud, - "handle" .= udHandle ud, - "email" .= udEmail ud, - "accent_id" .= udColourId ud, - "account_status" .= udAccountStatus ud, - "saml_idp" .= udSAMLIdP ud, - "managed_by" .= udManagedBy ud, - "created_at" .= udCreatedAt ud, - "role" .= udRole ud, - searchVisibilityInboundFieldName .= udSearchVisibilityInbound ud, - "scim_external_id" .= udScimExternalId ud, - "sso" .= udSso ud, - "email_unvalidated" .= udEmailUnvalidated ud, - "searchable" .= udSearchable ud - ] - -instance FromJSON UserDoc where - parseJSON = withObject "UserDoc" $ \o -> - UserDoc - <$> o .: "id" - <*> o .:? "type" - <*> o .:? "team" - <*> o .:? "name" - <*> o .:? "normalized" - <*> o .:? "handle" - <*> o .:? "email" - <*> o .:? "accent_id" - <*> o .:? "account_status" - <*> o .:? "saml_idp" - <*> o .:? "managed_by" - <*> o .:? "created_at" - <*> o .:? "role" - <*> o .:? searchVisibilityInboundFieldName - <*> o .:? "scim_external_id" - <*> o .:? "sso" - <*> o .:? "email_unvalidated" - <*> o .:? "searchable" - -searchVisibilityInboundFieldName :: Key -searchVisibilityInboundFieldName = "search_visibility_inbound" - --- Qualified UserId is not included in `UserDoc`, so it needs to be --- provided here. Monad will most likely be Identity (I promise we'll --- always make up some name if missing) or Maybe (if no name, then no --- contact). -userDocToContact :: (Monad m) => Qualified UserId -> (Maybe Name -> m Text) -> UserDoc -> m Contact -userDocToContact contactQualifiedId getName userDoc = - getName userDoc.udName <&> \name -> - Contact - { contactQualifiedId, - contactName = name, - contactColorId = fromIntegral . fromColourId <$> userDoc.udColourId, - contactHandle = fromHandle <$> userDoc.udHandle, - contactTeam = userDoc.udTeam, - contactType = - -- users of type `UserTypeBot` are not searchable as - -- contacts, so we can assume this is either - -- `UserTypeRegular` or `UserTypeApp`. - inferUserType Nothing userDoc.udType - } - -userDocToTeamContact :: [UserGroupId] -> UserDoc -> TeamContact -userDocToTeamContact userGroups UserDoc {..} = - TeamContact - { teamContactUserId = udId, - teamContactUserType = - -- bots are not searchable as contacts, so we can assume this is not one. - inferUserType Nothing udType, - teamContactTeam = udTeam, - teamContactSso = udSso, - teamContactScimExternalId = udScimExternalId, - teamContactSAMLIdp = udSAMLIdP, - teamContactRole = udRole, - teamContactName = maybe "" fromName udName, - teamContactManagedBy = udManagedBy, - teamContactHandle = fromHandle <$> udHandle, - teamContactEmailUnvalidated = udEmailUnvalidated, - teamContactEmail = udEmail, - teamContactCreatedAt = udCreatedAt, - teamContactColorId = fromIntegral . fromColourId <$> udColourId, - teamContactUserGroups = userGroups, - teamContactSearchable = fromMaybe True udSearchable - } - --- | Outbound search restrictions configured by team admin of the searcher. This --- value restricts the set of user that are searched. --- --- See 'optionallySearchWithinTeam' for the effect on full-text search. --- --- See 'mkTeamSearchInfo' for the business logic that defines the TeamSearchInfo --- value. --- --- Search results might be affected by the inbound search restriction settings of --- the searched user. ('SearchVisibilityInbound') -data TeamSearchInfo - = -- | Only users that are not part of any team are searched - NoTeam - | -- | Only users from the same team as the searcher are searched - TeamOnly TeamId - | -- | No search restrictions, all users are searched - AllUsers - --- | Inbound search restrictions configured by team to-be-searched. Affects only --- full-text search (i.e. search on the display name and the handle), not exact --- handle search. -data SearchVisibilityInbound - = -- | The user can only be found by users from the same team - SearchableByOwnTeam - | -- | The user can by found by any user of any team - SearchableByAllTeams - deriving (Eq, Show) - -instance Arbitrary SearchVisibilityInbound where - arbitrary = elements [SearchableByOwnTeam, SearchableByAllTeams] - -instance ToByteString SearchVisibilityInbound where - builder SearchableByOwnTeam = "searchable-by-own-team" - builder SearchableByAllTeams = "searchable-by-all-teams" - -instance FromByteString SearchVisibilityInbound where - parser = - SearchableByOwnTeam - <$ string "searchable-by-own-team" - <|> SearchableByAllTeams - <$ string "searchable-by-all-teams" - -instance C.Cql SearchVisibilityInbound where - ctype = C.Tagged C.IntColumn - - toCql SearchableByOwnTeam = C.CqlInt 0 - toCql SearchableByAllTeams = C.CqlInt 1 - - fromCql (C.CqlInt 0) = pure SearchableByOwnTeam - fromCql (C.CqlInt 1) = pure SearchableByAllTeams - fromCql n = Left $ "Unexpected SearchVisibilityInbound: " ++ show n - -defaultSearchVisibilityInbound :: SearchVisibilityInbound -defaultSearchVisibilityInbound = SearchableByOwnTeam - -searchVisibilityInboundFromFeatureStatus :: FeatureStatus -> SearchVisibilityInbound -searchVisibilityInboundFromFeatureStatus FeatureStatusDisabled = SearchableByOwnTeam -searchVisibilityInboundFromFeatureStatus FeatureStatusEnabled = SearchableByAllTeams - -instance ToJSON SearchVisibilityInbound where - toJSON = String . decodeUtf8 . toStrict . toLazyByteString . builder - -instance FromJSON SearchVisibilityInbound where - parseJSON = withText "SearchVisibilityInbound" $ \str -> - case runParser (parser @SearchVisibilityInbound) (encodeUtf8 str) of - Left err -> fail err - Right result -> pure result - -data IndexQuery r = IndexQuery Query Filter [DefaultSort] - -data BrowseTeamFilters = BrowseTeamFilters - { teamId :: TeamId, - mQuery :: Maybe Text, - mRoleFilter :: Maybe RoleFilter, - mSortBy :: Maybe TeamUserSearchSortBy, - mSortOrder :: Maybe TeamUserSearchSortOrder, - mEmailVerificationFilter :: Maybe EmailVerificationFilter, - mSearchable :: Maybe Bool - } - deriving (Eq, Show) - -userIdToDocId :: UserId -> DocId -userIdToDocId uid = DocId (idToText uid) - --- | We use cassandra writetimes to construct the ES index version. Since nulling fields in --- cassandra also nulls the writetime, re-indexing does not happen when nulling a field, and --- the old search key can still effectively be used. --- --- `write_time_bumper type int` is an extra field that we can update whenever we null a field --- and want to update the write time of the table. `WriteTimeBumper` writes to 'int' fields, --- but only cares about the field's writetime. -data WriteTimeBumper = WriteTimeBumper - deriving (Eq, Show) - -instance C.Cql WriteTimeBumper where - ctype = C.Tagged C.IntColumn - toCql WriteTimeBumper = C.CqlInt 0 - fromCql _ = pure WriteTimeBumper diff --git a/libs/wire-subsystems/src/Wire/UserSearchStore.hs b/libs/wire-subsystems/src/Wire/UserSearchStore.hs new file mode 100644 index 00000000000..250757582ce --- /dev/null +++ b/libs/wire-subsystems/src/Wire/UserSearchStore.hs @@ -0,0 +1,70 @@ +{-# LANGUAGE TemplateHaskell #-} + +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2026 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +module Wire.UserSearchStore where + +import Data.Id +import Data.Qualified (Local) +import Imports +import Polysemy +import Wire.API.Team.Size +import Wire.API.User.Search + +-- | User search queries backed directly by brig's user store (Postgres), +-- replacing the former ElasticSearch-backed 'Wire.IndexedUserStore'. +data UserSearchStore m a where + -- | Full-text search over user names and handles (prefix matching, see the + -- swagger docs of @/users/search@ for rank ordering). Excludes the searcher + -- and exact-handle matches (the latter are fetched from the user store by + -- the caller). + SearchUsers :: + -- | The searcher; its domain qualifies the returned contacts. + Local UserId -> + Maybe TeamId -> + TeamSearchInfo -> + Text -> + Int -> + Maybe [UserTypeFilter] -> + UserSearchStore m (SearchResult Contact) + -- | Team member browse (formerly @/teams/:tid/browse@ via ES). Fills + -- 'TeamContact.teamContactRole'; 'teamContactUserGroups' is left empty and is + -- filled by the caller. + PaginateTeamMembers :: + BrowseTeamFilters -> + Int -> + Maybe PagingState -> + UserSearchStore m (SearchResult TeamContact) + -- | Number of activated, non-deleted team members, split by regular users + -- and apps. Used for max-team-size enforcement. + GetTeamSize :: TeamId -> UserSearchStore m TeamSize + -- | Inbound federated search (a remote backend searching this backend). + -- @Nothing@ allows non-team users and members of teams opted in to + -- searchable-by-all-teams; @Just []@ matches nothing; @Just teams@ allows + -- only opted-in members of the given teams. + SearchUsersFederated :: + Maybe [TeamId] -> + Text -> + Int -> + Maybe [UserTypeFilter] -> + UserSearchStore m (SearchResult Contact) + -- | Upsert the inbound search visibility setting of a team (pushed from + -- galley when the corresponding team feature changes). + SetTeamSearchVisibilityInbound :: TeamId -> SearchVisibilityInbound -> UserSearchStore m () + +makeSem ''UserSearchStore diff --git a/libs/wire-subsystems/src/Wire/UserSearchStore/Postgres.hs b/libs/wire-subsystems/src/Wire/UserSearchStore/Postgres.hs new file mode 100644 index 00000000000..07b0d16c185 --- /dev/null +++ b/libs/wire-subsystems/src/Wire/UserSearchStore/Postgres.hs @@ -0,0 +1,864 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2026 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +-- | Postgres interpreter for 'UserSearchStore'. +-- +-- Queries brig's @wire_user@ table directly, replacing the former +-- ElasticSearch projection ('Wire.IndexedUserStore.ElasticSearch' is the +-- semantic spec for the predicates below): +-- +-- * candidate filter: activated, status active/suspended, no service +-- accounts (mirrors the former @shouldIndex@ write filter); +-- +-- * 'searchUsers': prefix matching on @name_normalized@ (ICU-folded) and +-- handle, rank-ordered like the documented swagger ordering (exact +-- handle, exact name, handle prefix, name prefix), visibility via the +-- @team_search_visibility@ table (left join; a missing row defaults to +-- own-team-only); +-- +-- * 'paginateTeamMembers': keyset pagination on @(sort_value, id)@ for +-- SQL-side sorts; role filter and role/SAML-idp sorts are resolved via +-- the galley role map and paginated by offset in Haskell (team +-- membership is bounded by @hardTruncationLimit@); +-- +-- * 'getTeamSize': count of activated, active/suspended team members. +module Wire.UserSearchStore.Postgres + ( interpretUserSearchStorePostgres, + ) +where + +import Control.Lens ((^.)) +import Data.Aeson qualified as Aeson +import Data.Bifunctor (first) +import Data.ByteString.Conversion qualified as BSC +import Data.ByteString.Lazy qualified as LBS +import Data.Domain (Domain) +import Data.Functor.Contravariant ((>$<)) +import Data.Id +import Data.Json.Util (toUTCTimeMillis) +import Data.Map.Strict qualified as Map +import Data.Qualified (Local, Qualified (..), tDomain, tUnqualified) +import Data.Text qualified as Text +import Data.Text.Ascii (decodeBase64Url, encodeBase64Url) +import Data.Text.Encoding qualified as TE +import Data.Time (UTCTime) +import Hasql.Decoders qualified as Dec +import Hasql.Encoders qualified as Enc +import Hasql.Statement (Statement, refineResult) +import Hasql.TH +import Imports +import Polysemy +import Polysemy.Input +import Polysemy.TinyLog (TinyLog) +import Polysemy.TinyLog qualified as TinyLog +import System.Logger.Class qualified as Log +import Wire.API.PostgresMarshall +import Wire.API.Team.HardTruncationLimit (hardTruncationLimit, hardTruncationLimitRange) +import Wire.API.Team.Member qualified as Team +import Wire.API.Team.Member.Info (TeamMemberInfo (..), TeamMemberInfoList (members)) +import Wire.API.Team.Role (Role, permissionsToRole, roleName) +import Wire.API.Team.Size (TeamSize (..)) +import Wire.API.User +import Wire.API.User.Search +import Wire.GalleyAPIAccess (GalleyAPIAccess) +import Wire.GalleyAPIAccess qualified as GalleyAPIAccess +import Wire.Postgres +import Wire.StoredUser (inferUserType) +import Wire.UserSearch.Normalize (normalized) +import Wire.UserSearchStore + +-- | Only users with these account statuses were indexed into ElasticSearch +-- (see the former @shouldIndex@); encoded as in +-- @instance PostgresMarshall Int32 AccountStatus@. +accountStatusIndexed :: [AccountStatus] +accountStatusIndexed = [Active, Suspended] + +interpretUserSearchStorePostgres :: + ( PGConstraints r, + Member GalleyAPIAccess r, + Member TinyLog r, + Member (Input (Local ())) r + ) => + InterpreterFor UserSearchStore r +interpretUserSearchStorePostgres = + interpret $ \case + SearchUsers lSearcher mSearcherTeam teamSearchInfo term maxResults mTypes -> + searchUsersImpl lSearcher mSearcherTeam teamSearchInfo term maxResults mTypes + PaginateTeamMembers filters maxResults mPagingState -> + paginateTeamMembersImpl filters maxResults mPagingState + SearchUsersFederated mOnlyInTeams term maxResults mTypes -> + searchUsersFederatedImpl mOnlyInTeams term maxResults mTypes + GetTeamSize tid -> getTeamSizeImpl tid + SetTeamSearchVisibilityInbound tid vis -> + runStatement (tid, searchVisibilityInboundToInt vis) upsertTeamSearchVisibilityStatement + +-------------------------------------------------------------------------------- +-- SearchUsers + +searchUsersImpl :: + (PGConstraints r) => + Local UserId -> + Maybe TeamId -> + TeamSearchInfo -> + Text -> + Int -> + Maybe [UserTypeFilter] -> + Sem r (SearchResult Contact) +searchUsersImpl lSearcher mSearcherTeam teamSearchInfo searchTerm maxResults mTypes = + case (Text.words term, visibilityCondition mSearcherTeam teamSearchInfo) of + -- The former ES query could not match the empty term either. + ([], _) -> pure emptyResult + (_, VisImpossible) -> pure emptyResult + (tokens, VisCondition vis) -> + runContactSearch + localDomain + (contactSearchQuery (Just searcher) mSearcherTeam tokens term vis maxResults mTypes) + rankedOrder + where + searcher = tUnqualified lSearcher + localDomain = tDomain lSearcher + term = Text.unwords (analyzeTerm searchTerm) + + rankedOrder = literal "order by" <> rankFragment term <> literal "asc, wu.id asc" + + emptyResult = SearchResult 0 0 0 [] FullSearch Nothing Nothing + +-- | Inbound federated search: the searcher is unknown, so no self +-- exclusion applies and results are not relevance-ordered (the former ES +-- implementation scored all matches equally here). +searchUsersFederatedImpl :: + (PGConstraints r, Member (Input (Local ())) r) => + Maybe [TeamId] -> + Text -> + Int -> + Maybe [UserTypeFilter] -> + Sem r (SearchResult Contact) +searchUsersFederatedImpl mOnlyInTeams searchTerm maxResults mTypes = + case Text.words term of + [] -> pure emptyResult + tokens -> case federatedVisibility mOnlyInTeams of + VisImpossible -> pure emptyResult + VisCondition vis -> do + loc <- input + runContactSearch (tDomain loc) (contactSearchQuery Nothing Nothing tokens term vis maxResults mTypes) (literal "order by wu.id asc") + where + term = normalized searchTerm + + emptyResult = SearchResult 0 0 0 [] FullSearch Nothing Nothing + + federatedVisibility = \case + Nothing -> + VisCondition $ + literal "(wu.team is null or tsv.search_visibility_inbound =" + <> intParam searchVisibilityInboundAllTeams + <> literal ")" + Just [] -> + -- Impossible to fulfill (safety net, handled earlier by the caller). + VisImpossible + Just teams -> + VisCondition $ + andList + [ paramLiteral + (const (map toUUID teams) >$< Enc.param (Enc.nonNullable (Enc.foldableArray (Enc.nonNullable Enc.uuid)))) + (\i -> "wu.team = any(" <> argPattern "uuid[]" i <> ")"), + literal "tsv.search_visibility_inbound =" <> intParam searchVisibilityInboundAllTeams + ] + +runContactSearch :: + (PGConstraints r) => + Domain -> + QueryFragment -> + QueryFragment -> + Sem r (SearchResult Contact) +runContactSearch localDomain query orderFragment = do + rows <- + runStatement + () + ( refineResult + (traverse (\(total, raw) -> (,) (fromIntegral (total :: Int64)) <$> rawToContact localDomain raw)) + (buildStatement (query <> orderFragment) (Dec.rowList contactRow)) + ) + pure + SearchResult + { searchFound = maybe 0 fst (listToMaybe rows), + searchReturned = length rows, + searchTook = 0, + searchResults = map snd rows, + searchPolicy = FullSearch, + searchPagingState = Nothing, + searchHasMore = Nothing + } + +-- | Shared filter and matching logic of 'SearchUsers' and +-- 'SearchUsersFederated' (the former @defaultUserQuery@ / @mkUserQuery@). +-- The ordering clause is supplied by the caller. +contactSearchQuery :: + Maybe UserId -> + Maybe TeamId -> + [Text] -> + Text -> + QueryFragment -> + Int -> + Maybe [UserTypeFilter] -> + QueryFragment +contactSearchQuery mSearcher mSearcherTeam tokens term vis maxResults mTypes = + literal "select (count(*) over ()) :: bigint, wu.id :: uuid, wu.name :: text, wu.accent_id :: int, wu.handle :: text, wu.team :: uuid, wu.user_type :: int" + <> literal "from wire_user wu left join team_search_visibility tsv on tsv.team = wu.team" + <> literal "where" + <> andList conditions + <> limitParam maxResults + where + conditions = + candidateCondition + ++ [ -- Exact handle matches are fetched by the caller (user store + -- lookup for local search, exact-handle search for federation). + -- Handle-less users pass (they are still findable by name). + literal "(wu.handle is null or lower(wu.handle) <>" + <> textParam term + <> literal ")", + literal "(wu.searchable is null or wu.searchable)", + appExclusionCondition mSearcherTeam, + vis + ] + ++ catMaybes [selfExclusion mSearcher, userTypeCondition mTypes] + ++ map (tokenMatchCondition False) tokens + + selfExclusion = \case + Nothing -> Nothing + Just searcher -> Just (clause1 "wu.id" "<>" searcher) + +-- | Rank ordering documented in the swagger docs of @/users/search@: +-- exact name, handle prefix, name prefix. (Exact handle matches are +-- excluded from this query entirely - the swagger's first tier is +-- resolved by the caller's exact-handle lookup.) +rankFragment :: Text -> QueryFragment +rankFragment term = + literal "case when wu.name_normalized =" + <> textParam term + <> literal "then 0 when lower(wu.handle) like" + <> textParam (escapeLike term <> "%") + <> literal "then 1 when wu.name_normalized like" + <> textParam (escapeLike term <> "%") + <> literal "then 2 else 3 end" + +contactRow :: Dec.Row (Int64, (UserId, Text, Int32, Maybe Text, Maybe TeamId, Int32)) +contactRow = + (,) + <$> Dec.column (Dec.nonNullable Dec.int8) + <*> ( (,,,,,) + <$> (Id <$> Dec.column (Dec.nonNullable Dec.uuid)) + <*> Dec.column (Dec.nonNullable Dec.text) + <*> Dec.column (Dec.nonNullable Dec.int4) + <*> Dec.column (Dec.nullable Dec.text) + <*> ((Id <$>) <$> Dec.column (Dec.nullable Dec.uuid)) + <*> Dec.column (Dec.nonNullable Dec.int4) + ) + +rawToContact :: Domain -> (UserId, Text, Int32, Maybe Text, Maybe TeamId, Int32) -> Either Text Contact +rawToContact dom (uid, name, accent, mHandle, mTeam, utype) = do + ty <- postgresUnmarshall utype + pure + Contact + { contactQualifiedId = Qualified uid dom, + contactName = name, + contactColorId = Just (fromIntegral accent), + contactHandle = mHandle, + contactTeam = mTeam, + contactType = inferUserType Nothing (Just ty) + } + +-------------------------------------------------------------------------------- +-- PaginateTeamMembers + +data BrowseCursor + = -- | Offset into the Haskell-sorted result set (role / SAML idp sorts). + OffsetCursor Int + | -- | Keyset: null-flag of the sort value, the value itself, and the id + -- of the last returned row. + KeysetCursor Bool Aeson.Value UserId + +-- | Raw @wire_user@ row served to the browse endpoint. +data TeamRow = TeamRow + { trFound :: Int64, + trId :: UserId, + trType :: Int32, + trName :: Text, + trAccent :: Int32, + trHandle :: Maybe Text, + trTeam :: Maybe TeamId, + trEmail :: Maybe Text, + trEmailUnvalidated :: Maybe Text, + trCreatedAt :: UTCTime, + trManagedBy :: Maybe Int32, + trSsoId :: Maybe UserSSOId, + trSearchable :: Maybe Bool + } + +teamRow :: Dec.Row TeamRow +teamRow = + TeamRow + <$> Dec.column (Dec.nonNullable Dec.int8) + <*> (Id <$> Dec.column (Dec.nonNullable Dec.uuid)) + <*> Dec.column (Dec.nonNullable Dec.int4) + <*> Dec.column (Dec.nonNullable Dec.text) + <*> Dec.column (Dec.nonNullable Dec.int4) + <*> Dec.column (Dec.nullable Dec.text) + <*> ((Id <$>) <$> Dec.column (Dec.nullable Dec.uuid)) + <*> Dec.column (Dec.nullable Dec.text) + <*> Dec.column (Dec.nullable Dec.text) + <*> Dec.column (Dec.nonNullable Dec.timestamptz) + <*> Dec.column (Dec.nullable Dec.int4) + <*> Dec.column (Dec.nullable (Dec.jsonbBytes decodeJsonStrict)) + <*> Dec.column (Dec.nullable Dec.bool) + +decodeJsonStrict :: (Aeson.FromJSON a) => ByteString -> Either Text a +decodeJsonStrict = first Text.pack . Aeson.eitherDecodeStrict' + +teamRowToTeamContact :: TeamRow -> Either Text TeamContact +teamRowToTeamContact row = do + ty <- postgresUnmarshall row.trType + managedBy <- traverse postgresUnmarshall row.trManagedBy + email <- traverse decodeEmail row.trEmail + emailUnvalidated <- traverse decodeEmail row.trEmailUnvalidated + pure + TeamContact + { teamContactUserId = row.trId, + teamContactUserType = ty, + teamContactName = row.trName, + teamContactColorId = Just (fromIntegral row.trAccent), + teamContactHandle = row.trHandle, + teamContactTeam = row.trTeam, + teamContactEmail = email, + teamContactCreatedAt = Just (toUTCTimeMillis row.trCreatedAt), + teamContactManagedBy = managedBy, + teamContactSAMLIdp = fst <$> (ssoIssuerAndNameId =<< row.trSsoId), + teamContactRole = Nothing, + teamContactScimExternalId = join (scimExternalId <$> managedBy <*> row.trSsoId), + teamContactSso = fmap (uncurry Sso) (row.trSsoId >>= ssoIssuerAndNameId), + teamContactEmailUnvalidated = emailUnvalidated, + teamContactUserGroups = [], + teamContactSearchable = fromMaybe True row.trSearchable + } + where + decodeEmail = first Text.pack . BSC.runParser BSC.parser . TE.encodeUtf8 + +-- | @refineResult@ step keeping the raw row alongside the contact, so that +-- keyset cursors are built from the exact DB values. +withRaw :: TeamRow -> Either Text (TeamContact, TeamRow) +withRaw row = (,row) <$> teamRowToTeamContact row + +paginateTeamMembersImpl :: + forall r. + (PGConstraints r, Member GalleyAPIAccess r, Member TinyLog r) => + BrowseTeamFilters -> + Int -> + Maybe PagingState -> + Sem r (SearchResult TeamContact) +paginateTeamMembersImpl filters maxResults mPagingState = do + -- The role filter is resolved via the galley role map; sorts on role and + -- SAML idp are not SQL-expressible on wire_user. + let needRoleMap = isJust filters.mRoleFilter || filters.mSortBy `elem` [Just SortByRole, Just SortBySAMLIdp] + mRoleMap <- + if needRoleMap + then Just <$> fetchRoleMap filters.teamId + else pure Nothing + let mCursor = mPagingState >>= decodeCursor + case filters.mSortBy of + Just SortByRole -> haskellSortPage (Just SortByRole) mRoleMap mCursor + Just SortBySAMLIdp -> haskellSortPage (Just SortBySAMLIdp) mRoleMap mCursor + _ -> sqlKeysetPage mRoleMap mCursor + where + -- Full result set, filtered/sorted/paged in Haskell. Only used for + -- sorts that depend on the galley role map; team membership is bounded + -- by `hardTruncationLimit` (see `fetchRoleMap`). + haskellSortPage :: + Maybe TeamUserSearchSortBy -> + Maybe (Map UserId Role) -> + Maybe BrowseCursor -> + Sem r (SearchResult TeamContact) + haskellSortPage mSortBy' mRoleMap mCursor = do + let startOffset = case mCursor of + Just (OffsetCursor n) -> n + _ -> 0 + rows <- + runStatement + () + (refineResult (traverse withRaw) (buildStatement (browseQuery filters mRoleMap Nothing) (Dec.rowList teamRow))) + let total = maybe 0 (trFound . snd) (listToMaybe rows) + dir = fromMaybe SortOrderAsc filters.mSortOrder + sorted = sortTeamContacts mSortBy' dir (map fst rows) + page = take maxResults (drop startOffset sorted) + hasMore = length sorted > startOffset + length page + applyRoleFill mRoleMap (mkSearchResult (fromIntegral total) page (Just (OffsetCursor (startOffset + length page))) hasMore) + + -- Keyset-paginated SQL path for all sorts directly expressible on + -- wire_user. + sqlKeysetPage :: + Maybe (Map UserId Role) -> + Maybe BrowseCursor -> + Sem r (SearchResult TeamContact) + sqlKeysetPage mRoleMap mCursor = do + let sortSpec@(_, dir) = effectiveSort filters + keyset = mCursor >>= keysetPredicate (Just (fst sortSpec)) dir + rows <- + runStatement + () + ( refineResult + (traverse withRaw) + ( buildStatement + ( browseQuery filters mRoleMap keyset + <> browseOrderBy sortSpec + <> limitParam (maxResults + 1) + ) + (Dec.rowList teamRow) + ) + ) + let total = maybe 0 (trFound . snd) (listToMaybe rows) + pageRows = take maxResults rows + hasMore = length rows > maxResults + nextCursor = keysetCursor sortSpec . snd <$> listToMaybe (reverse pageRows) + applyRoleFill mRoleMap (mkSearchResult (fromIntegral total) (map fst pageRows) nextCursor hasMore) + + -- Fill roles from the role map when one was fetched (role filter/sort), + -- otherwise resolve the page roles via the galley batch RPC. + applyRoleFill :: + Maybe (Map UserId Role) -> + SearchResult TeamContact -> + Sem r (SearchResult TeamContact) + applyRoleFill (Just roleMap) result = + pure result {searchResults = map setRole (searchResults result)} + where + setRole tc = tc {teamContactRole = Map.lookup tc.teamContactUserId roleMap} + applyRoleFill Nothing result = do + results <- fillRolesFromGalley filters.teamId (searchResults result) + pure result {searchResults = results} + +-- | Browse query on @wire_user wu@; @keyset@ (if given) is the pagination +-- continuation predicate, @mRoleMap@ provides the optional role filter. +browseQuery :: + BrowseTeamFilters -> + Maybe (Map UserId Role) -> + Maybe QueryFragment -> + QueryFragment +browseQuery filters mRoleMap keyset = + literal "select (count(*) over ()) :: bigint, wu.id :: uuid, wu.user_type :: int, wu.name :: text, wu.accent_id :: int, wu.handle :: text, wu.team :: uuid, wu.email :: text, wu.email_unvalidated :: text, wu.created_at :: timestamptz, wu.managed_by :: int, wu.sso_id :: jsonb, wu.searchable :: bool" + <> literal "from wire_user wu" + <> literal "where" + <> andList + ( candidateCondition + ++ [ clause1 "wu.team" "=" filters.teamId, + searchableCondition filters.mSearchable, + emailVerificationCondition filters.mEmailVerificationFilter + ] + ++ catMaybes [roleFilterCondition mRoleMap filters.mRoleFilter, keyset] + ++ map (tokenMatchCondition True) (maybe [] analyzeTerm filters.mQuery) + ) + +-- | Keyset continuation cursor for the last returned row. +keysetCursor :: (TeamUserSearchSortBy, TeamUserSearchSortOrder) -> TeamRow -> BrowseCursor +keysetCursor (mSortBy', _) row = + KeysetCursor (isNothing val) (maybe (Aeson.toJSON ()) sortValToAeson val) row.trId + where + val = rowSortVal (Just mSortBy') row + +rowSortVal :: Maybe TeamUserSearchSortBy -> TeamRow -> Maybe SortVal +rowSortVal mSortBy' row = case mSortBy' of + Just SortByName -> Just (SortText row.trName) + Just SortByHandle -> SortText <$> row.trHandle + Just SortByEmail -> SortText <$> row.trEmail + Just SortByManagedBy -> SortInt32 <$> row.trManagedBy + Just SortByCreatedAt -> Just (SortTime row.trCreatedAt) + _ -> Nothing + +-- | The effective SQL sort: explicit @sort-by@ wins (default direction +-- ascending); without it, browse is ordered by creation date, newest first +-- (same as the former ES query). +effectiveSort :: BrowseTeamFilters -> (TeamUserSearchSortBy, TeamUserSearchSortOrder) +effectiveSort filters = case filters.mSortBy of + Just sb -> (sb, fromMaybe SortOrderAsc filters.mSortOrder) + Nothing -> (SortByCreatedAt, SortOrderDesc) + +-- | Ordering clause for the SQL keyset path. Nulls sort last when +-- ascending and first when descending (the former ES implementation sorted +-- missing values last in BOTH directions - a small, documented divergence); +-- the user id is the deterministic tie breaker. +browseOrderBy :: (TeamUserSearchSortBy, TeamUserSearchSortOrder) -> QueryFragment +browseOrderBy (mSortBy', dir) = + literal "order by" + <> literal ("(" <> col <> " is null)") + <> literal dirSql + <> literal "," + <> literal col + <> literal dirSql + <> literal "," + <> literal ("wu.id " <> dirSql) + where + col = fromMaybe "wu.created_at" (sortColumnExpr (Just mSortBy')) + dirSql = case dir of SortOrderAsc -> "asc"; SortOrderDesc -> "desc" + +-- | SQL expressions of the SQL-side sort columns. +sortColumnExpr :: Maybe TeamUserSearchSortBy -> Maybe Text +sortColumnExpr = \case + Just SortByName -> Just "wu.name" + Just SortByHandle -> Just "wu.handle" + Just SortByEmail -> Just "wu.email" + Just SortByManagedBy -> Just "wu.managed_by" + Just SortByCreatedAt -> Just "wu.created_at" + _ -> Nothing + +-- | Keyset continuation predicate for @(null-flag, sort value, id)@ tuples. +-- @Nothing@ when the cursor does not match the requested sort (the page +-- restarts). +keysetPredicate :: + Maybe TeamUserSearchSortBy -> + TeamUserSearchSortOrder -> + BrowseCursor -> + Maybe QueryFragment +keysetPredicate mSortBy dir = \case + OffsetCursor _ -> Nothing + KeysetCursor flag val uid -> do + col <- sortColumnExpr mSortBy + typed <- sortValFromAeson col val + let (op, idOp, dirSql) = case dir of + SortOrderAsc -> (">", ">", "asc") + SortOrderDesc -> ("<", "<", "desc") + pure $ + literal "(((" + <> literal ("(" <> col <> " is null)") + <> literal dirSql + <> literal "," + <> literal col + <> literal dirSql + <> literal "," + <> literal ("wu.id " <> dirSql) + <> literal (") " <> op <> " (") + <> boolParam flag + <> literal "," + <> sortValParam typed + <> literal "," + <> uidParam uid + <> literal "))" + <> literal "or (" + <> literal col + <> literal "is null and" + <> boolParam flag + <> literal ("and wu.id " <> idOp) + <> uidParam uid + <> literal ")" + +-- | Sort value of a row, used for keyset pagination cursors. +data SortVal = SortText Text | SortInt32 Int32 | SortTime UTCTime + +sortValToAeson :: SortVal -> Aeson.Value +sortValToAeson = \case + SortText t -> Aeson.toJSON t + SortInt32 n -> Aeson.toJSON n + SortTime t -> Aeson.toJSON t + +-- | Decode a cursor sort value according to the SQL type of the column. +sortValFromAeson :: Text -> Aeson.Value -> Maybe SortVal +sortValFromAeson col v = case col of + "wu.managed_by" -> tryVal SortInt32 + "wu.created_at" -> tryVal SortTime + _ -> tryVal SortText + where + tryVal :: forall a. (Aeson.FromJSON a) => (a -> SortVal) -> Maybe SortVal + tryVal mk = case Aeson.fromJSON v of + Aeson.Success x -> Just (mk x) + Aeson.Error _ -> Nothing + +sortValParam :: SortVal -> QueryFragment +sortValParam = \case + SortText t -> textParam t + SortInt32 n -> intParam n + SortTime t -> timeParam t + +sortTeamContacts :: Maybe TeamUserSearchSortBy -> TeamUserSearchSortOrder -> [TeamContact] -> [TeamContact] +sortTeamContacts mSortBy' dir = arrange dir . sortOn key + where + key tc = case mSortBy' of + Just SortByRole -> roleName @Text <$> tc.teamContactRole + Just SortBySAMLIdp -> tc.teamContactSAMLIdp + _ -> Nothing + arrange = \case + SortOrderAsc -> id + SortOrderDesc -> reverse + +-- | Role filter as an @id = any(...)@ condition over the members with a +-- matching role in the galley role map. +roleFilterCondition :: Maybe (Map UserId Role) -> Maybe RoleFilter -> Maybe QueryFragment +roleFilterCondition mRoleMap mRoleFilter = do + roleMap <- mRoleMap + RoleFilter roles <- mRoleFilter + -- The former ES query treated an empty role list as "no filter". + if null roles + then pure (literal "true") + else + let roleNames = map (roleName @Text) roles + matching = + [toUUID uid | (uid, role) <- Map.toList roleMap, roleName @Text role `elem` roleNames] + in pure $ + paramLiteral + (const matching >$< Enc.param (Enc.nonNullable (Enc.foldableArray (Enc.nonNullable Enc.uuid)))) + (\i -> "wu.id = any(" <> argPattern "uuid[]" i <> ")") + +searchableCondition :: Maybe Bool -> QueryFragment +searchableCondition = \case + Nothing -> literal "true" + -- Former `searchableFilter`: false = exactly false; true = not false + -- (i.e. true or unset). + Just False -> literal "wu.searchable is false" + Just True -> literal "not (wu.searchable is false)" + +-- | Former @emailFilter@: verified = has a verified email and no unvalidated +-- one; unverified = has an unvalidated email. +emailVerificationCondition :: Maybe EmailVerificationFilter -> QueryFragment +emailVerificationCondition = \case + Nothing -> literal "true" + Just EmailVerified -> literal "(wu.email is not null and wu.email_unvalidated is null)" + Just EmailUnverified -> literal "wu.email_unvalidated is not null" + +-- | Fetches the team member role map from galley. Like the other +-- team-member endpoints, the list is truncated at @hardTruncationLimit@; +-- the truncation is logged and the (possibly incomplete) map is used. +fetchRoleMap :: (Member GalleyAPIAccess r, Member TinyLog r) => TeamId -> Sem r (Map UserId Role) +fetchRoleMap tid = do + teamMemberList <- GalleyAPIAccess.getTeamMembersWithLimit tid (Just (hardTruncationLimitRange @Int32)) + let teamMembers = teamMemberList ^. Team.teamMembers + when (length teamMembers >= hardTruncationLimit) $ + TinyLog.warn $ + Log.msg (Log.val "UserSearchStore: team member list truncated, browse role filter/sort may be incomplete") + . Log.field "team" (idToText tid) + pure $ + Map.fromList + [ (m ^. Team.userId, role) + | m <- teamMembers, + Just role <- [permissionsToRole (m ^. Team.permissions)] + ] + +-- | Fills 'TeamContact.teamContactRole' for the returned page via the +-- galley batch RPC (only needed for pages produced without a role map). +fillRolesFromGalley :: (Member GalleyAPIAccess r) => TeamId -> [TeamContact] -> Sem r [TeamContact] +fillRolesFromGalley tid contacts = do + infos <- members <$> GalleyAPIAccess.selectTeamMemberInfos tid (map (.teamContactUserId) contacts) + let roleOf = Map.fromList [(i.userId, permissionsToRole i.permissions) | i <- infos] + pure [tc {teamContactRole = join (Map.lookup tc.teamContactUserId roleOf)} | tc <- contacts] + +-------------------------------------------------------------------------------- +-- GetTeamSize + +-- | Counts activated team members with an active/suspended status, split by +-- regulars and apps (the former ES implementation aggregated over index +-- documents, which never contained deactivated/deleted/service users). +-- +-- The user type literals mirror @instance PostgresMarshall Int32 UserType@ +-- (regular = 0, app = 2); the status literals mirror +-- @instance PostgresMarshall Int32 AccountStatus@ (active = 0, suspended = 1). +getTeamSizeImpl :: (PGConstraints r) => TeamId -> Sem r TeamSize +getTeamSizeImpl tid = do + (regulars, apps) <- + runStatement tid select + pure TeamSize {regulars = fromIntegral regulars, apps = fromIntegral apps} + where + select :: Statement TeamId (Int64, Int64) + select = + dimapPG + [singletonStatement| + select + count(*) filter (where wu.user_type = 0) :: bigint, + count(*) filter (where wu.user_type = 2) :: bigint + from wire_user wu + where wu.team = ($1 :: uuid) + and wu.activated + and (wu.account_status is null or wu.account_status in (0, 1)) + and wu.service is null + |] + +-------------------------------------------------------------------------------- +-- SetTeamSearchVisibilityInbound + +-- | Integral encoding shared with @instance C.Cql SearchVisibilityInbound@. +searchVisibilityInboundToInt :: SearchVisibilityInbound -> Int32 +searchVisibilityInboundToInt = \case + SearchableByOwnTeam -> 0 + SearchableByAllTeams -> 1 + +upsertTeamSearchVisibilityStatement :: Statement (TeamId, Int32) () +upsertTeamSearchVisibilityStatement = + dimapPG + [resultlessStatement| + insert into team_search_visibility (team, search_visibility_inbound) + values ($1 :: uuid, $2 :: int) + on conflict (team) do update set search_visibility_inbound = excluded.search_visibility_inbound + |] + +-------------------------------------------------------------------------------- +-- Shared query fragments + +-- | Normalizes a search term and splits it into tokens, dropping the +-- leading '@' of each token (the former ES analyzer did this implicitly; +-- the swagger documents that '@' does nothing special). +analyzeTerm :: Text -> [Text] +analyzeTerm = map (Text.dropWhile (== '@')) . Text.words . normalized + +-- | Prefix match on a single (whitespace-split) token of the normalized +-- search term. A name token must start at the beginning of the name or be +-- preceded by whitespace; handle starts with the token. Email is matched +-- only for team browse: matching email in contact search would enable +-- email-based user enumeration on those broader surfaces, which the +-- former ES contact-search queries did not do either. +tokenMatchCondition :: Bool -> Text -> QueryFragment +tokenMatchCondition matchEmail tok = + literal "(" + <> likeParam "wu.name_normalized" prefix + <> literal "or" + <> likeParam "wu.name_normalized" midWord + <> literal "or" + <> likeParam "lower(wu.handle)" prefix + <> (if matchEmail then literal "or" <> likeParam "lower(wu.email)" prefix else literal "true") + <> literal ")" + where + prefix = escapeLike tok <> "%" + midWord = "% " <> escapeLike tok <> "%" + +-- | Mirrors the former @shouldIndex@ write filter: only activated users with +-- an active/suspended status and no service account are searchable. +candidateCondition :: [QueryFragment] +candidateCondition = + [ literal "wu.activated", + literal "(wu.account_status is null or wu.account_status in" + <> commaList (intParam . postgresMarshall @Int32 <$> accountStatusIndexed) + <> literal "))", + literal "wu.service is null" + ] + +-- | The user_type filter semantics of the former ES implementation: no +-- filter for @Nothing@ and @Just []@. +userTypeCondition :: Maybe [UserTypeFilter] -> Maybe QueryFragment +userTypeCondition = \case + Nothing -> Nothing + Just [] -> Nothing + Just uts -> + Just $ + literal "(" + <> foldr1 + (\a b -> a <> literal "or" <> b) + [clause1 "wu.user_type" "=" (postgresMarshall @Int32 (userTypeFilterToUserType ut)) | ut <- uts] + <> literal ")" + +-- | Apps are only searchable within their own team (former +-- @matchAppsFromOtherTeams@, expressed as a negated condition). +appExclusionCondition :: Maybe TeamId -> QueryFragment +appExclusionCondition = \case + Nothing -> + literal "not (wu.user_type =" + <> intParam (postgresMarshall @Int32 UserTypeApp) + <> literal "and wu.team is not null)" + Just st -> + literal "not (wu.user_type =" + <> intParam (postgresMarshall @Int32 UserTypeApp) + <> literal "and (wu.team is null or wu.team <>" + <> uidParam st + <> literal "))" + +-- | Inbound search visibility (former @restrictSearchSpaceByTeam@). +data Visibility = VisImpossible | VisCondition QueryFragment + +visibilityCondition :: Maybe TeamId -> TeamSearchInfo -> Visibility +visibilityCondition mSearcherTeam teamSearchInfo = case (mSearcherTeam, teamSearchInfo) of + (Nothing, _) -> VisCondition (literal "wu.team is null") + (Just _, NoTeam) -> VisCondition (literal "wu.team is null") + (Just searcherTeam, TeamOnly t) + | searcherTeam == t -> VisCondition (clause1 "wu.team" "=" t) + | otherwise -> VisImpossible + (Just searcherTeam, AllUsers) -> + -- Team members of other teams are only visible if their team set + -- search_visibility_inbound = searchable-by-all-teams. The left join + -- yields null (own-team-only default) when the team has no row. + VisCondition $ + literal "(wu.team is null or" + <> clause1 "wu.team" "=" searcherTeam + <> literal "or tsv.search_visibility_inbound =" + <> intParam searchVisibilityInboundAllTeams + <> literal ")" + +searchVisibilityInboundAllTeams :: Int32 +searchVisibilityInboundAllTeams = 1 + +andList :: [QueryFragment] -> QueryFragment +andList = foldr1 (\a b -> a <> literal "and" <> b) + +commaList :: [QueryFragment] -> QueryFragment +commaList = foldr1 (\a b -> a <> literal "," <> b) + +-- | Single (non-null) parameter fragments. +intParam :: Int32 -> QueryFragment +intParam n = paramLiteral (const n >$< Enc.param (Enc.nonNullable Enc.int4)) (argPattern "int") + +textParam :: Text -> QueryFragment +textParam t = paramLiteral (const t >$< Enc.param (Enc.nonNullable Enc.text)) (argPattern "text") + +timeParam :: UTCTime -> QueryFragment +timeParam t = paramLiteral (const t >$< Enc.param (Enc.nonNullable Enc.timestamptz)) (argPattern "timestamptz") + +boolParam :: Bool -> QueryFragment +boolParam b = paramLiteral (const b >$< Enc.param (Enc.nonNullable Enc.bool)) (argPattern "bool") + +uidParam :: Id a -> QueryFragment +uidParam u = paramLiteral (const (toUUID u) >$< Enc.param (Enc.nonNullable Enc.uuid)) (argPattern "uuid") + +limitParam :: Int -> QueryFragment +limitParam n = paramLiteral (const (fromIntegral n) >$< Enc.param (Enc.nonNullable Enc.int4)) (\i -> "limit " <> argPattern "int" i) + +-- | @ like $n@ with the given (already escaped) pattern. +likeParam :: Text -> Text -> QueryFragment +likeParam field pat = paramLiteral (const pat >$< Enc.param (Enc.nonNullable Enc.text)) (\i -> field <> " like " <> argPattern "text" i) + +escapeLike :: Text -> Text +escapeLike = Text.replace "_" "\\_" . Text.replace "%" "\\%" . Text.replace "\\" "\\\\" + +mkSearchResult :: Int -> [a] -> Maybe BrowseCursor -> Bool -> SearchResult a +mkSearchResult found results cursor hasMore = + SearchResult + { searchFound = found, + searchReturned = length results, + searchTook = 0, + searchResults = results, + searchPolicy = FullSearch, + searchPagingState = encodeCursor <$> cursor, + searchHasMore = Just hasMore + } + +encodeCursor :: BrowseCursor -> PagingState +encodeCursor c = + PagingState + . encodeBase64Url + . LBS.toStrict + . Aeson.encode + $ case c of + OffsetCursor n -> Aeson.toJSON n + KeysetCursor flag val uid -> Aeson.toJSON (flag, val, uid) + +decodeCursor :: PagingState -> Maybe BrowseCursor +decodeCursor (PagingState ps) = do + bs <- decodeBase64Url ps + v <- either (const Nothing) Just (Aeson.eitherDecode (LBS.fromStrict bs)) + case Aeson.fromJSON @Int v of + Aeson.Success n -> Just (OffsetCursor n) + Aeson.Error _ -> case Aeson.fromJSON @(Bool, Aeson.Value, UserId) v of + Aeson.Success (flag, val, uid) -> Just (KeysetCursor flag val uid) + Aeson.Error _ -> Nothing diff --git a/libs/wire-subsystems/src/Wire/UserStore.hs b/libs/wire-subsystems/src/Wire/UserStore.hs index 46bc9e2ff57..bf73a365a65 100644 --- a/libs/wire-subsystems/src/Wire/UserStore.hs +++ b/libs/wire-subsystems/src/Wire/UserStore.hs @@ -33,7 +33,6 @@ import Wire.API.User.RichInfo import Wire.API.User.Search (SetSearchable) import Wire.Arbitrary import Wire.StoredUser -import Wire.UserStore.IndexUser -- | Update of any "simple" attributes (ones that do not involve locking, like handle, or -- validation protocols, like email). @@ -65,15 +64,11 @@ data StoredUserHandleUpdate = MkStoredUserHandleUpdate data StoredUserUpdateError = StoredUserUpdateHandleExists -data UserPageMarker = PagingExitingUsers UserId | PagingDeletedUsers UserId - -- | Effect containing database logic around 'StoredUser'. (Example: claim handle lock is -- database logic; validate handle is application logic.) data UserStore m a where CreateUser :: NewStoredUser -> Maybe (ConvId, Maybe TeamId) -> UserStore m () - GetIndexUser :: UserId -> UserStore m (Maybe IndexUser) DoesUserExist :: UserId -> UserStore m Bool - GetIndexUsersPaginated :: Int32 -> Maybe (GeneralPaginationState UserPageMarker) -> UserStore m (PageWithState UserPageMarker IndexUser) GetUsers :: [UserId] -> UserStore m [StoredUser] UpdateUser :: UserId -> StoredUserUpdate -> UserStore m () UpdateEmail :: UserId -> EmailAddress -> UserStore m () diff --git a/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs b/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs index 54fefc2d485..425c43b12c3 100644 --- a/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs +++ b/libs/wire-subsystems/src/Wire/UserStore/Cassandra.hs @@ -22,7 +22,6 @@ module Wire.UserStore.Cassandra where import Cassandra -import Cassandra.Exec (prepared) import Control.Lens ((^.)) import Data.Handle import Data.Id @@ -49,7 +48,6 @@ import Wire.Postgres (PGConstraints) import Wire.StoredUser import Wire.UserStore import Wire.UserStore qualified as UserStore -import Wire.UserStore.IndexUser hiding (userId) import Wire.UserStore.Postgres (interpretUserStorePostgres) import Wire.UserStore.Unique @@ -60,8 +58,6 @@ interpretUserStoreCassandra casClient = CreateUser new mbConv -> createUserImpl new mbConv GetUsers uids -> getUsersImpl uids DoesUserExist uid -> doesUserExistImpl uid - GetIndexUser uid -> getIndexUserImpl uid - GetIndexUsersPaginated pageSize mPagingState -> getIndexUserPaginatedImpl pageSize (paginationStateCassandra =<< mPagingState) UpdateUser uid update -> updateUserImpl uid update UpdateEmail uid email -> updateEmailImpl uid email UpdateEmailUnvalidated uid email -> updateEmailUnvalidatedImpl uid email @@ -128,15 +124,6 @@ interpretUserStoreToCassandraAndPostgres casClient = if isUserInPg then pure True else interpretUserStoreCassandra casClient $ UserStore.doesUserExist uid - GetIndexUser uid -> - runAppropriateInterpreter casClient uid $ UserStore.getIndexUser uid - GetIndexUsersPaginated pageSize mPagingState -> do - paginateOverCassandraAndPostgres - (\size state -> interpretUserStoreCassandra casClient $ UserStore.getIndexUsersPaginated size state) - (\size state -> interpretUserStorePostgres $ UserStore.getIndexUsersPaginated size state) - (PagingExitingUsers $ Id UUID.nil) - pageSize - mPagingState UpdateUser uid update -> runAppropriateInterpreter casClient uid $ UserStore.updateUser uid update UpdateEmail uid email -> @@ -334,43 +321,6 @@ doesUserExistImpl uid = idSelect :: PrepQuery R (Identity UserId) (Identity UserId) idSelect = "SELECT id FROM user WHERE id = ?" -getIndexUserImpl :: UserId -> Client (Maybe IndexUser) -getIndexUserImpl u = do - mIndexUserTuple <- retry x1 $ query1 cql (params LocalQuorum (Identity u)) - pure $ indexUserFromTuple <$> mIndexUserTuple - where - cql :: PrepQuery R (Identity UserId) (TupleType IndexUser) - cql = prepared . QueryString $ getIndexUserBaseQuery <> " WHERE id = ?" - -getIndexUserPaginatedImpl :: Int32 -> Maybe PagingState -> Client (PageWithState x IndexUser) -getIndexUserPaginatedImpl pageSize mPagingState = - indexUserFromTuple <$$> paginateWithState cql (paramsPagingState LocalQuorum () pageSize mPagingState) x1 - where - cql :: PrepQuery R () (TupleType IndexUser) - cql = prepared $ QueryString getIndexUserBaseQuery - -getIndexUserBaseQuery :: LText -getIndexUserBaseQuery = - [sql| - SELECT - id, - user_type, - team, writetime(team), - name, writetime(name), - status, writetime(status), - handle, writetime(handle), - email, writetime(email), - accent_id, writetime(accent_id), - activated, writetime(activated), - service, writetime(service), - managed_by, writetime(managed_by), - sso_id, writetime(sso_id), - email_unvalidated, writetime(email_unvalidated), - searchable, writetime(searchable), - writetime(write_time_bumper) - FROM user - |] - updateUserImpl :: UserId -> StoredUserUpdate -> Client () updateUserImpl uid update = retry x5 $ batch do diff --git a/libs/wire-subsystems/src/Wire/UserStore/IndexUser.hs b/libs/wire-subsystems/src/Wire/UserStore/IndexUser.hs deleted file mode 100644 index 09ac630d191..00000000000 --- a/libs/wire-subsystems/src/Wire/UserStore/IndexUser.hs +++ /dev/null @@ -1,213 +0,0 @@ -{-# LANGUAGE RecordWildCards #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.UserStore.IndexUser where - -import Cassandra.Util -import Data.ByteString.Builder -import Data.ByteString.Lazy qualified as LBS -import Data.Default -import Data.Handle -import Data.Id -import Data.Json.Util -import Data.Text.Encoding qualified as Text -import Data.Text.Encoding.Error qualified as Text -import Data.Text.ICU.Translit -import Data.Time -import Database.CQL.Protocol -import Imports -import SAML2.WebSSO qualified as SAML -import URI.ByteString -import Wire.API.Team.Role (Role) -import Wire.API.User hiding (userId) -import Wire.API.User.Search -import Wire.UserSearch.Types - -type Activated = Bool - -data WithWritetime a = WithWriteTime {value :: a, writetime :: Writetime a} - deriving (Eq, Show) - -data IndexUser = IndexUser - { userId :: UserId, - userType :: UserType, - teamId :: Maybe TeamId, - name :: Name, - accountStatus :: Maybe AccountStatus, - handle :: Maybe Handle, - email :: Maybe EmailAddress, - colourId :: ColourId, - activated :: Activated, - serviceId :: Maybe ServiceId, - managedBy :: Maybe ManagedBy, - ssoId :: Maybe UserSSOId, - unverifiedEmail :: Maybe EmailAddress, - searchable :: Maybe Bool, - createdAt :: UTCTime, - updatedAt :: UTCTime - } - deriving (Eq, Show) - -{- ORMOLU_DISABLE -} -type instance - TupleType IndexUser = - ( UserId, - Maybe UserType, - Maybe TeamId, Maybe (Writetime TeamId), - Name, Writetime Name, - Maybe AccountStatus, Maybe (Writetime AccountStatus), - Maybe Handle, Maybe (Writetime Handle), - Maybe EmailAddress, Maybe (Writetime EmailAddress), - ColourId, Writetime ColourId, - Activated, Writetime Activated, - Maybe ServiceId, Maybe (Writetime ServiceId), - Maybe ManagedBy, Maybe (Writetime ManagedBy), - Maybe UserSSOId, Maybe (Writetime UserSSOId), - Maybe EmailAddress, Maybe (Writetime EmailAddress), - Maybe Bool, Maybe (Writetime Bool), - Maybe (Writetime WriteTimeBumper) - ) - -indexUserFromTuple :: TupleType IndexUser -> IndexUser -indexUserFromTuple - ( userId, - mbUserType, - teamId, tTeam, - name, tName, - accountStatus, tStatus, - handle, tHandle, - email, tEmail, - colourId, tColour, - activated, tActivated, - serviceId, tService, - managedBy, tManagedBy, - ssoId, tSsoId, - unverifiedEmail, tEmailUnvalidated, - searchable, tSearchable, - tWriteTimeBumper - ) = IndexUser { - createdAt = writetimeToUTC tActivated, - updatedAt = maximum $ catMaybes [writetimeToUTC <$> tTeam, - Just $ writetimeToUTC tName, - writetimeToUTC <$> tStatus, - writetimeToUTC <$> tHandle, - writetimeToUTC <$> tEmail, - Just $ writetimeToUTC tColour, - Just $ writetimeToUTC tActivated, - writetimeToUTC <$> tService, - writetimeToUTC <$> tManagedBy, - writetimeToUTC <$> tSsoId, - writetimeToUTC <$> tEmailUnvalidated, - writetimeToUTC <$> tSearchable, - writetimeToUTC <$> tWriteTimeBumper - ], - userType = fromMaybe def mbUserType, - .. - } -{- ORMOLU_ENABLE -} - -indexUserToVersion :: Maybe (WithWritetime Role) -> IndexUser -> IndexVersion -indexUserToVersion role iu = - mkIndexVersion [Just $ Writetime iu.updatedAt, const () <$$> fmap writetime role] - -indexUserToDoc :: SearchVisibilityInbound -> Maybe Role -> IndexUser -> UserDoc -indexUserToDoc searchVisInbound mRole IndexUser {..} = - if shouldIndex - then - UserDoc - { udId = userId, - udType = Just userType, - udSearchable = searchable, - udEmailUnvalidated = unverifiedEmail, - udSso = sso =<< ssoId, - udScimExternalId = join $ scimExternalId <$> (managedBy) <*> (ssoId), - udSearchVisibilityInbound = Just searchVisInbound, - udRole = mRole, - udCreatedAt = Just . toUTCTimeMillis $ createdAt, - udManagedBy = managedBy, - udSAMLIdP = idpUrl =<< ssoId, - udAccountStatus = accountStatus, - udColourId = Just colourId, - udEmail = email, - udHandle = handle, - udNormalized = Just $ normalized name.fromName, - udName = Just name, - udTeam = teamId - } - else -- We insert a tombstone-style user here, as it's easier than - -- deleting the old one. It's mostly empty, but having the status here - -- might be useful in the future. - emptyUserDoc userId - where - shouldIndex = - ( case accountStatus of - Nothing -> True - Just Active -> True - Just Suspended -> True - Just Deleted -> False - Just Ephemeral -> False - Just PendingInvitation -> False - ) - && activated -- FUTUREWORK: how is this adding to the first case? - && isNothing serviceId - - idpUrl :: UserSSOId -> Maybe Text - idpUrl (UserSSOId (SAML.UserRef (SAML.Issuer uri) _subject)) = - Just $ fromUri uri - idpUrl (UserScimExternalId _) = Nothing - - fromUri :: URI -> Text - fromUri = - Text.decodeUtf8With Text.lenientDecode - . LBS.toStrict - . toLazyByteString - . serializeURIRef - - sso :: UserSSOId -> Maybe Sso - sso userSsoId = do - (issuer, nameid) <- ssoIssuerAndNameId userSsoId - pure $ Sso {ssoIssuer = issuer, ssoNameId = nameid} - --- Transliteration could also be done by ElasticSearch (ICU plugin), but this would --- require a data migration. -normalized :: Text -> Text -normalized = transliterate (trans "Any-Latin; Latin-ASCII; Lower") - -emptyUserDoc :: UserId -> UserDoc -emptyUserDoc uid = - UserDoc - { udType = Nothing, - udSearchable = Nothing, - udEmailUnvalidated = Nothing, - udSso = Nothing, - udScimExternalId = Nothing, - udSearchVisibilityInbound = Nothing, - udRole = Nothing, - udCreatedAt = Nothing, - udManagedBy = Nothing, - udSAMLIdP = Nothing, - udAccountStatus = Nothing, - udColourId = Nothing, - udEmail = Nothing, - udHandle = Nothing, - udNormalized = Nothing, - udName = Nothing, - udTeam = Nothing, - udId = uid - } diff --git a/libs/wire-subsystems/src/Wire/UserStore/Postgres.hs b/libs/wire-subsystems/src/Wire/UserStore/Postgres.hs index ca9e002bfc5..960188a8a1b 100644 --- a/libs/wire-subsystems/src/Wire/UserStore/Postgres.hs +++ b/libs/wire-subsystems/src/Wire/UserStore/Postgres.hs @@ -36,7 +36,6 @@ import Data.Time import Data.Tuple.Extra (fst3) import Data.Vector (Vector) import Data.Vector qualified as V -import Data.Vector qualified as Vector import Hasql.Pipeline qualified as Pipeline import Hasql.Statement qualified as Hasql import Hasql.TH @@ -56,8 +55,8 @@ import Wire.API.User.Search import Wire.Postgres import Wire.Sem.Logger import Wire.StoredUser +import Wire.UserSearch.Normalize (normalized) import Wire.UserStore -import Wire.UserStore.IndexUser interpretUserStorePostgres :: (PGConstraints r, Member TinyLog r) => InterpreterFor UserStore r interpretUserStorePostgres = @@ -67,8 +66,6 @@ interpretUserStorePostgres = DeactivateUser uid -> deactivateUserImpl uid GetUsers uids -> getUsersImpl uids DoesUserExist uid -> doesUserExistImpl uid - GetIndexUser uid -> getIndexUserImpl uid - GetIndexUsersPaginated pageSize mPagingState -> getIndexUsersPaginatedImpl pageSize (paginationStatePostgres =<< mPagingState) UpdateUser uid update -> updateUserImpl uid update UpdateEmail uid email -> updateEmailImpl uid (Just email) DeleteEmail uid -> updateEmailImpl uid Nothing @@ -101,7 +98,7 @@ interpretUserStorePostgres = {- ORMOLU_DISABLE -} type InsertUserRow = - ( UserId, Name, Maybe TextStatus, Pict, Maybe EmailAddress, + ( UserId, Name, Text, Maybe TextStatus, Pict, Maybe EmailAddress, Maybe UserSSOId, ColourId, Maybe Password, Bool, AccountStatus, Maybe UTCTimeMillis, Language, Maybe Country, Maybe ProviderId, Maybe ServiceId, Maybe TeamId, ManagedBy, Set BaseProtocolTag, Bool, UserType @@ -127,40 +124,7 @@ storedUserFromRow (id_, name, textStatus, pict, email, emailUnvalidated, .. } -type SelectIndexUserRow = - (UserId, Maybe TeamId, Name, Maybe AccountStatus, Maybe Handle, - Maybe EmailAddress, Maybe EmailAddress, ColourId, Bool, Maybe ServiceId, - Maybe ManagedBy, Maybe UserSSOId, Maybe Bool, UTCTime, UTCTime, - UserType) - -indexUserFromRow :: SelectIndexUserRow -> IndexUser -indexUserFromRow ( uid, teamId, name, accountStatus, handle, - email, unverifiedEmail, colourId, activated, serviceId, - managedBy, ssoId, searchable, createdAt, updatedAt, - userType - ) = IndexUser{userId = uid, ..} -{- ORMOLU_ENABLE -} - -indexUserFromDeletedRow :: (UserId, Maybe TeamId, UTCTime, UTCTime) -> IndexUser -indexUserFromDeletedRow (uid, teamId, createdAt, deletedAt) = - IndexUser - { userId = uid, - teamId = teamId, - createdAt = createdAt, - updatedAt = deletedAt, - name = Name "default", - accountStatus = Just Deleted, - handle = Nothing, - email = Nothing, - colourId = defaultAccentId, - activated = False, - serviceId = Nothing, - managedBy = Nothing, - ssoId = Nothing, - unverifiedEmail = Nothing, - searchable = Nothing, - userType = UserTypeRegular - } + createUserImpl :: (PGConstraints r) => NewStoredUser -> Maybe (ConvId, Maybe TeamId) -> Sem r () createUserImpl new mbConv = @@ -174,6 +138,7 @@ createUserImpl new mbConv = userRow = ( new.id, new.name, + normalized (fromName new.name), new.textStatus, new.pict, new.email, @@ -199,17 +164,18 @@ createUserImpl new mbConv = lmapPG [resultlessStatement| INSERT INTO wire_user - (id, name, text_status, picture, email, + (id, name, name_normalized, text_status, picture, email, sso_id, accent_id, password, activated, account_status, expires, language, country, provider, service, team, managed_by, supported_protocols, searchable, user_type) VALUES - ($1 :: uuid, $2 :: text, $3 :: text?, $4 :: jsonb, $5 :: text?, - $6 :: jsonb?, $7 :: integer, $8 :: text?, $9 :: boolean, $10 :: integer, - $11 :: timestamptz?, $12 :: text, $13 :: text?, $14 :: uuid?, $15 :: uuid?, - $16 :: uuid?, $17 :: integer, $18 :: integer, $19 :: boolean, $20 :: integer) + ($1 :: uuid, $2 :: text, $3 :: text, $4 :: text?, $5 :: jsonb, $6 :: text?, + $7 :: jsonb?, $8 :: integer, $9 :: text?, $10 :: boolean, $11 :: integer, + $12 :: timestamptz?, $13 :: text, $14 :: text?, $15 :: uuid?, $16 :: uuid?, + $17 :: uuid?, $18 :: integer, $19 :: integer, $20 :: boolean, $21 :: integer) ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name, + name_normalized = EXCLUDED.name_normalized, text_status = EXCLUDED.text_status, picture = EXCLUDED.picture, email = EXCLUDED.email, @@ -397,113 +363,12 @@ deactivateUserImpl uid = WHERE id = $1 :: uuid |] -getIndexUserImpl :: (PGConstraints r) => UserId -> Sem r (Maybe IndexUser) -getIndexUserImpl uid = do - indexUserFromRow <$$> runStatement uid selectUser - where - selectUser :: Hasql.Statement UserId (Maybe SelectIndexUserRow) - selectUser = - dimapPG - [maybeStatement| - SELECT - id :: uuid, team :: uuid?, name :: text, account_status :: integer?, handle :: text?, - email :: text?, email_unvalidated :: text?, accent_id :: integer, activated :: Bool, service :: uuid?, - managed_by :: integer?, sso_id :: jsonb?, searchable :: boolean?, created_at :: timestamptz, updated_at :: timestamptz, - user_type :: integer - FROM wire_user - WHERE id = $1 :: uuid - |] - -getIndexUsersPaginatedImpl :: forall r. (PGConstraints r) => Int32 -> Maybe UserPageMarker -> Sem r (PageWithState UserPageMarker IndexUser) -getIndexUsersPaginatedImpl lim mState = do - case mState of - Nothing -> getExistingUserPage Nothing - Just (PagingExitingUsers startId) -> getExistingUserPage (Just startId) - Just (PagingDeletedUsers startId) -> getDeletedUserPage mempty lim (Just startId) - where - getExistingUserPage :: Maybe UserId -> Sem r (PageWithState UserPageMarker IndexUser) - getExistingUserPage mLastUserId = do - rows <- case mLastUserId of - Nothing -> runStatement lim selectStart - Just startId -> runStatement (startId, lim) selectFrom - let results = indexUserFromRow <$> rows - if fromIntegral (Vector.length results) >= lim - then do - pure - PageWithState - { pwsResults = Vector.toList results, - pwsState = PaginationStatePostgres . PagingExitingUsers . (.userId) <$> results Vector.!? (Vector.length results - 1) - } - else getDeletedUserPage results (lim - fromIntegral (Vector.length results)) Nothing - - getDeletedUserPage :: Vector IndexUser -> Int32 -> Maybe UserId -> Sem r (PageWithState UserPageMarker IndexUser) - getDeletedUserPage prevResults remainingLim mLastStartId = do - rows <- case mLastStartId of - Nothing -> runStatement remainingLim selectDeletedStart - Just startId -> runStatement (startId, remainingLim) selectDeletedFrom - let results = indexUserFromDeletedRow <$> rows - pure - PageWithState - { pwsResults = Vector.toList $ prevResults <> results, - pwsState = PaginationStatePostgres . PagingDeletedUsers . (.userId) <$> results Vector.!? (Vector.length results - 1) - } - - selectStart :: Hasql.Statement Int32 (Vector SelectIndexUserRow) - selectStart = - dimapPG - [vectorStatement| - SELECT - id :: uuid, team :: uuid?, name :: text, account_status :: integer?, handle :: text?, - email :: text?, email_unvalidated :: text?, accent_id :: integer, activated :: Bool, service :: uuid?, - managed_by :: integer?, sso_id :: jsonb?, searchable :: boolean?, created_at :: timestamptz, updated_at :: timestamptz, - user_type :: integer - FROM wire_user - ORDER BY id ASC - LIMIT ($1 :: integer) - |] - - selectFrom :: Hasql.Statement (UserId, Int32) (Vector SelectIndexUserRow) - selectFrom = - dimapPG - [vectorStatement| - SELECT - id :: uuid, team :: uuid?, name :: text, account_status :: integer?, handle :: text?, - email :: text?, email_unvalidated :: text?, accent_id :: integer, activated :: Bool, service :: uuid?, - managed_by :: integer?, sso_id :: jsonb?, searchable :: boolean?, created_at :: timestamptz, updated_at :: timestamptz, - user_type :: integer - FROM wire_user - WHERE id > ($1 :: uuid) - ORDER BY id ASC - LIMIT ($2 :: integer) - |] - - selectDeletedStart :: Hasql.Statement Int32 (Vector (UserId, Maybe TeamId, UTCTime, UTCTime)) - selectDeletedStart = - dimapPG - [vectorStatement| - SELECT id :: uuid, team :: uuid?, created_at :: timestamptz, deleted_at :: timestamptz - FROM deleted_user - ORDER BY id ASC - LIMIT ($1 :: integer) - |] - - selectDeletedFrom :: Hasql.Statement (UserId, Int32) (Vector (UserId, Maybe TeamId, UTCTime, UTCTime)) - selectDeletedFrom = - dimapPG - [vectorStatement| - SELECT id :: uuid, team :: uuid?, created_at :: timestamptz, deleted_at :: timestamptz - FROM deleted_user - WHERE id > ($1 :: uuid) - ORDER BY id ASC - LIMIT ($2 :: integer) - |] - updateUserImpl :: (PGConstraints r, Member TinyLog r) => UserId -> StoredUserUpdate -> Sem r () updateUserImpl uid MkStoredUserUpdate {..} = do warn $ Log.msg (Log.val "Updating user") . Log.field "locale" (show locale) runTransaction Serializable Write $ do Transaction.statement - (uid, name, textStatus, pict, accentId, supportedProtocols) + (uid, name, normalized . fromName <$> name, textStatus, pict, accentId, supportedProtocols) updateUserFields for_ locale $ \newLocale -> Transaction.statement (uid, newLocale.lLanguage, newLocale.lCountry) updateLocale @@ -511,16 +376,19 @@ updateUserImpl uid MkStoredUserUpdate {..} = do Transaction.statement uid deleteAssetsStatement Transaction.statement (mkAssetRows uid newAssets) insertAssetsStatement where - updateUserFields :: Hasql.Statement (UserId, Maybe Name, Maybe TextStatus, Maybe Pict, Maybe ColourId, Maybe (Set BaseProtocolTag)) () + updateUserFields :: Hasql.Statement (UserId, Maybe Name, Maybe Text, Maybe TextStatus, Maybe Pict, Maybe ColourId, Maybe (Set BaseProtocolTag)) () updateUserFields = lmapPG [resultlessStatement| UPDATE wire_user SET name = COALESCE($2 :: text?, name), - text_status = COALESCE($3 :: text?, text_status), - picture = COALESCE($4 :: jsonb?, picture), - accent_id = COALESCE($5 :: integer?, accent_id), - supported_protocols = COALESCE($6 :: integer?, supported_protocols) + name_normalized = CASE WHEN $2 :: text? IS NULL + THEN name_normalized + ELSE COALESCE($3 :: text?, name_normalized) END, + text_status = COALESCE($4 :: text?, text_status), + picture = COALESCE($5 :: jsonb?, picture), + accent_id = COALESCE($6 :: integer?, accent_id), + supported_protocols = COALESCE($7 :: integer?, supported_protocols) WHERE id = ($1 :: uuid) |] updateLocale :: Hasql.Statement (UserId, Language, Maybe Country) () diff --git a/libs/wire-subsystems/src/Wire/UserSubsystem.hs b/libs/wire-subsystems/src/Wire/UserSubsystem.hs index 0f5f428ae73..af74c182e70 100644 --- a/libs/wire-subsystems/src/Wire/UserSubsystem.hs +++ b/libs/wire-subsystems/src/Wire/UserSubsystem.hs @@ -66,7 +66,6 @@ import Wire.SparAPIAccess (SparAPIAccess, getIdentityProviders) import Wire.StoredUser qualified as SU import Wire.TeamSubsystem import Wire.UserKeyStore -import Wire.UserSearch.Types import Wire.UserStore import Wire.UserStore qualified as UserStore import Wire.UserSubsystem.Error @@ -178,9 +177,6 @@ data UserSubsystem m a where UserSubsystem m (SearchResult TeamContact) -- | (... or does `AcceptTeamInvitation` belong into `TeamInvitationSubsystems`?) AcceptTeamInvitation :: Local UserId -> PlainTextPassword6 -> InvitationCode -> UserSubsystem m () - -- | The following "internal" functions exists to support migration in this susbystem, after the - -- migration this would just be an internal detail of the subsystem - InternalUpdateSearchIndex :: UserId -> UserSubsystem m () InternalFindTeamInvitation :: Maybe EmailKey -> InvitationCode -> UserSubsystem m StoredInvitation GetUserExportData :: UserId -> UserSubsystem m (Maybe TeamExportUser) RemoveEmailEither :: Local UserId -> UserSubsystem m (Either UserSubsystemError ()) @@ -272,7 +268,6 @@ requestEmailChange lusr email allowScim = do ChangeEmailNeedsActivation (usr, adata, en) -> do sendOutEmail usr adata en updateEmailUnvalidated u email - internalUpdateSearchIndex u pure ChangeEmailResponseNeedsActivation where throwGuardFailed :: diff --git a/libs/wire-subsystems/src/Wire/UserSubsystem/Interpreter.hs b/libs/wire-subsystems/src/Wire/UserSubsystem/Interpreter.hs index d5cb2dfee62..d16698b581b 100644 --- a/libs/wire-subsystems/src/Wire/UserSubsystem/Interpreter.hs +++ b/libs/wire-subsystems/src/Wire/UserSubsystem/Interpreter.hs @@ -24,7 +24,6 @@ module Wire.UserSubsystem.Interpreter ) where -import Cassandra.Util (Writetime (Writetime)) import Control.Error.Util (hush) import Control.Lens (view, (^.)) import Control.Monad.Extra (partitionM) @@ -43,7 +42,6 @@ import Data.Qualified import Data.Range import Data.Set qualified as Set import Data.Time.Clock -import Database.Bloodhound qualified as ES import Imports import Polysemy import Polysemy.Error @@ -63,9 +61,8 @@ import Wire.API.Routes.Internal.Galley.TeamFeatureNoConfigMulti (TeamStatus (..) import Wire.API.Team.Export import Wire.API.Team.Feature import Wire.API.Team.Member -import Wire.API.Team.Member.Info (TeamMemberInfo (..), TeamMemberInfoList (members)) import Wire.API.Team.Permission qualified as Permission -import Wire.API.Team.Role (Role, defaultRole, permissionsToRole) +import Wire.API.Team.Role (defaultRole) import Wire.API.Team.SearchVisibility import Wire.API.Team.Size import Wire.API.User as User @@ -90,25 +87,19 @@ import Wire.FederationAPIAccess import Wire.FederationConfigStore import Wire.GalleyAPIAccess import Wire.GalleyAPIAccess qualified as GalleyAPIAccess -import Wire.IndexedUserStore (IndexedUserStore) -import Wire.IndexedUserStore qualified as IndexedUserStore -import Wire.IndexedUserStore.Bulk.ElasticSearch (teamSearchVisibilityInbound) import Wire.InvitationStore import Wire.MlsKeyPackageSubsystem (MlsKeyPackageSubsystem) import Wire.MlsKeyPackageSubsystem qualified as Mls import Wire.Sem.Concurrency -import Wire.Sem.Metrics -import Wire.Sem.Metrics qualified as Metrics import Wire.Sem.Now (Now) import Wire.Sem.Now qualified as Now import Wire.StoredUser import Wire.TeamSubsystem import Wire.UserGroupStore (UserGroupStore, getUserGroupIdsForUsers) import Wire.UserKeyStore -import Wire.UserSearch.Metrics -import Wire.UserSearch.Types +import Wire.UserSearchStore (UserSearchStore) +import Wire.UserSearchStore qualified as UserSearchStore import Wire.UserStore as UserStore -import Wire.UserStore.IndexUser import Wire.UserSubsystem as UserSubsystem import Wire.UserSubsystem.Error import Wire.UserSubsystem.HandleBlacklist @@ -134,9 +125,8 @@ runUserSubsystem :: RunClient (fedM 'Brig), FederationMonad fedM, Typeable fedM, - Member IndexedUserStore r, + Member UserSearchStore r, Member FederationConfigStore r, - Member Metrics r, Member InvitationStore r, Member TinyLog r, Member (Input UserSubsystemConfig) r, @@ -194,8 +184,6 @@ runUserSubsystem authInterpreter appInterpreter clientInterpreter = isUsersContactableImpl users mlsAvailable allowedCipherSuites BrowseTeam uid browseTeamFilters mMaxResults mPagingState -> browseTeamImpl uid browseTeamFilters mMaxResults mPagingState - InternalUpdateSearchIndex uid -> - syncUserIndex uid AcceptTeamInvitation luid pwd code -> acceptTeamInvitationImpl luid pwd code InternalFindTeamInvitation mEmailKey code -> @@ -279,7 +267,7 @@ internalFindTeamInvitationImpl :: Member (Error UserSubsystemError) r, Member (Input UserSubsystemConfig) r, Member (GalleyAPIAccess) r, - Member IndexedUserStore r, + Member UserSearchStore r, Member TinyLog r, Member DRS.DomainRegistrationStore r ) => @@ -309,7 +297,7 @@ internalFindTeamInvitationImpl (Just e) c = NotAllowed -> throwGuardFailed TeamInviteSetToNotAllowed maxSize <- maxTeamSize <$> input - teamSize <- teamSizeTotal <$> IndexedUserStore.getTeamSize tid + teamSize <- teamSizeTotal <$> UserSearchStore.getTeamSize tid when (teamSize >= fromIntegral maxSize) $ throw UserSubsystemTooManyTeamMembers -- FUTUREWORK: The above can easily be done/tested in the intra call. @@ -709,9 +697,7 @@ updateUserProfileImpl :: ( Member UserStore r, Member (Error UserSubsystemError) r, Member Events r, - Member GalleyAPIAccess r, - Member IndexedUserStore r, - Member Metrics r + Member GalleyAPIAccess r ) => Local UserId -> Maybe ConnId -> @@ -724,8 +710,6 @@ updateUserProfileImpl (tUnqualified -> uid) mconn updateOrigin update = do guardLockedFields user updateOrigin update mapError (\StoredUserUpdateHandleExists -> UserSubsystemHandleExists) $ updateUser uid (storedUserUpdate update) - let interestingToUpdateIndex = isJust update.name || isJust update.accentId - when interestingToUpdateIndex $ syncUserIndex uid generateUserEvent uid mconn (mkProfileUpdateEvent uid update) where guardMlsSupport user = for_ update.supportedProtocols $ \protocols -> do @@ -770,9 +754,7 @@ updateHandleImpl :: ( Member (Error UserSubsystemError) r, Member GalleyAPIAccess r, Member Events r, - Member UserStore r, - Member IndexedUserStore r, - Member Metrics r + Member UserStore r ) => Local UserId -> Maybe ConnId -> @@ -789,7 +771,6 @@ updateHandleImpl (tUnqualified -> uid) mconn updateOrigin uhandle = do throw UserSubsystemNoIdentity mapError (\StoredUserUpdateHandleExists -> UserSubsystemHandleExists) $ UserStore.updateUserHandle uid (MkStoredUserHandleUpdate user.handle newHandle) - syncUserIndex uid generateUserEvent uid mconn (mkProfileUpdateHandleEvent uid newHandle) checkHandleImpl :: (Member (Error UserSubsystemError) r, Member UserStore r) => Text -> Sem r CheckHandleResp @@ -834,55 +815,9 @@ checkHandlesImpl check num = reverse <$> collectFree [] check num ------------------------------------------------------------------------------- -- Search -syncUserIndex :: - forall r. - ( Member UserStore r, - Member GalleyAPIAccess r, - Member IndexedUserStore r, - Member Metrics r - ) => - UserId -> - Sem r () -syncUserIndex uid = - getIndexUser uid - >>= maybe deleteFromIndex upsert - where - deleteFromIndex :: Sem r () - deleteFromIndex = do - Metrics.incCounter indexDeleteCounter - IndexedUserStore.upsert (userIdToDocId uid) (emptyUserDoc uid) ES.NoVersionControl - - upsert :: IndexUser -> Sem r () - upsert indexUser = do - vis <- - maybe - (pure defaultSearchVisibilityInbound) - teamSearchVisibilityInbound - indexUser.teamId - tm <- maybe (pure Nothing) selectTeamMember indexUser.teamId - let mRole = tm >>= mkRoleWithWriteTime - userDoc = indexUserToDoc vis (value <$> mRole) indexUser - version = ES.ExternalGT . ES.ExternalDocVersion . docVersion $ indexUserToVersion mRole indexUser - Metrics.incCounter indexUpdateCounter - IndexedUserStore.upsert (userIdToDocId uid) userDoc version - - selectTeamMember :: TeamId -> Sem r (Maybe TeamMemberInfo) - selectTeamMember tid = do - listToMaybe . members <$> selectTeamMemberInfos tid [uid] - - mkRoleWithWriteTime :: TeamMemberInfo -> Maybe (WithWritetime Role) - mkRoleWithWriteTime info = - ( \role -> - WithWriteTime - { value = role, - writetime = Writetime $ fromUTCTimeMillis info.permissionsWriteTime - } - ) - <$> permissionsToRole info.permissions - -updateTeamSearchVisibilityInboundImpl :: (Member IndexedUserStore r) => TeamStatus SearchVisibilityInboundConfig -> Sem r () +updateTeamSearchVisibilityInboundImpl :: (Member UserSearchStore r) => TeamStatus SearchVisibilityInboundConfig -> Sem r () updateTeamSearchVisibilityInboundImpl teamStatus = - IndexedUserStore.updateTeamSearchVisibilityInbound teamStatus.team $ + UserSearchStore.setTeamSearchVisibilityInbound teamStatus.team $ searchVisibilityInboundFromFeatureStatus teamStatus.status searchUsersImpl :: @@ -890,7 +825,7 @@ searchUsersImpl :: ( Member UserStore r, Member GalleyAPIAccess r, Member (Error UserSubsystemError) r, - Member IndexedUserStore r, + Member UserSearchStore r, Member FederationConfigStore r, RunClient (fedM 'Brig), Member (FederationAPIAccess fedM) r, @@ -926,7 +861,7 @@ searchLocally :: forall r. ( Member GalleyAPIAccess r, Member UserStore r, - Member IndexedUserStore r, + Member UserSearchStore r, Member (Input UserSubsystemConfig) r ) => Local (UserId, Maybe TeamId) -> @@ -946,8 +881,8 @@ searchLocally searcher searchTerm maybeMaxResults mTypes = do esResult <- if esMaxResults > 0 then - IndexedUserStore.searchUsers - (tUnqualified searcherId) + UserSearchStore.searchUsers + searcherId (tUnqualified searcherTeamId) teamSearchInfo searchTerm @@ -955,8 +890,8 @@ searchLocally searcher searchTerm maybeMaxResults mTypes = do mTypes else pure $ SearchResult 0 0 0 [] FullSearch Nothing Nothing - let esContacts = map userDocToContact' (searchResults esResult) - -- Prepend results matching exact handle and results from ES. + let esContacts = searchResults esResult + -- Prepend results matching exact handle and results from the search store. allContacts = case maybeExactHandleMatch of Nothing -> esContacts Just exactHandleMatch -> exactHandleMatch : filter (\c -> c.contactQualifiedId /= exactHandleMatch.contactQualifiedId) esContacts @@ -971,14 +906,6 @@ searchLocally searcher searchTerm maybeMaxResults mTypes = do handleTeamVisibility _ SearchVisibilityStandard = AllUsers handleTeamVisibility t SearchVisibilityNoNameOutsideTeam = TeamOnly t - userDocToContact' :: UserDoc -> Contact - userDocToContact' userDoc = - runIdentity $ - userDocToContact - (tUntagged $ qualifyAs searcher userDoc.udId) - (Identity . maybe "" fromName) - userDoc - mkTeamSearchInfo :: Maybe TeamId -> Sem r TeamSearchInfo mkTeamSearchInfo searcherTeamId = do config <- input @@ -1059,7 +986,7 @@ searchRemotely rDom mTid searchTerm mTypes = do browseTeamImpl :: ( Member (Error UserSubsystemError) r, - Member IndexedUserStore r, + Member UserSearchStore r, Member TeamSubsystem r, Member UserGroupStore r ) => @@ -1075,13 +1002,13 @@ browseTeamImpl uid filters mMaxResults mPagingState = do ensurePermissions uid filters.teamId [Permission.AddTeamMember] let maxResults = maybe 15 fromRange mMaxResults - result <- IndexedUserStore.paginateTeamMembers filters maxResults mPagingState - let docs = result.searchResults - uids = fmap (.udId) docs + result <- UserSearchStore.paginateTeamMembers filters maxResults mPagingState + let uids = map (.teamContactUserId) result.searchResults ugMap <- getUserGroupIdsForUsers (toList uids) - for result $ \userDoc -> do - let ugids = fromMaybe [] (Map.lookup userDoc.udId ugMap) - pure $ userDocToTeamContact ugids userDoc + pure $ + fmap + (\teamContact -> teamContact {teamContactUserGroups = fromMaybe [] (Map.lookup teamContact.teamContactUserId ugMap)}) + result getAccountsByEmailNoFilterImpl :: forall r. @@ -1179,8 +1106,7 @@ acceptTeamInvitationImpl :: Member GalleyAPIAccess r, Member (Error UserSubsystemError) r, Member InvitationStore r, - Member IndexedUserStore r, - Member Metrics r, + Member UserSearchStore r, Member Events r, Member AuthenticationSubsystem r, Member TinyLog r, @@ -1208,7 +1134,6 @@ acceptTeamInvitationImpl luid pw code = do deleteInvitation inv.teamId inv.invitationId for_ (userEmail . selfUser =<< mSelfProfile) $ \email -> deletePendingScimUser tid email uid - syncUserIndex uid generateUserEvent uid Nothing (teamUpdated uid tid) getUserExportDataImpl :: (Member UserStore r, Member ClientSubsystem r) => UserId -> Sem r (Maybe TeamExportUser) @@ -1243,10 +1168,8 @@ removeEmailEitherImpl :: ( Member UserKeyStore r, Member UserStore r, Member Events r, - Member IndexedUserStore r, Member (Input UserSubsystemConfig) r, - Member GalleyAPIAccess r, - Member Metrics r + Member GalleyAPIAccess r ) => Local UserId -> Sem r (Either UserSubsystemError ()) @@ -1258,7 +1181,6 @@ removeEmailEitherImpl lusr = runError $ do deleteKey $ mkEmailKey e deleteEmail uid generateUserEvent uid Nothing (emailRemoved uid e) - syncUserIndex uid Just _ -> throw UserSubsystemLastIdentity Nothing -> throw UserSubsystemNoIdentity @@ -1277,10 +1199,7 @@ checkUserIsAdminImpl uid = do setUserSearchableImpl :: ( Member UserStore r, Member (Error UserSubsystemError) r, - Member TeamSubsystem r, - Member GalleyAPIAccess r, - Member IndexedUserStore r, - Member Metrics r + Member TeamSubsystem r ) => Local UserId -> UserId -> @@ -1290,4 +1209,3 @@ setUserSearchableImpl luid uid searchable = do tid <- maybe (throw UserSubsystemInsufficientPermissions) pure =<< UserStore.getUserTeam uid ensurePermissions (tUnqualified luid) tid [SetMemberSearchable] UserStore.setUserSearchable uid searchable - syncUserIndex uid diff --git a/libs/wire-subsystems/test/resources/elasticsearch-ca.pem b/libs/wire-subsystems/test/resources/elasticsearch-ca.pem deleted file mode 120000 index ed6d4718bf2..00000000000 --- a/libs/wire-subsystems/test/resources/elasticsearch-ca.pem +++ /dev/null @@ -1 +0,0 @@ -../../../../deploy/dockerephemeral/docker/elasticsearch-ca.pem \ No newline at end of file diff --git a/libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml b/libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml deleted file mode 100644 index 47846ea1017..00000000000 --- a/libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml +++ /dev/null @@ -1,2 +0,0 @@ -username: "elastic" -password: changeme diff --git a/libs/wire-subsystems/test/unit/Wire/MiniBackend.hs b/libs/wire-subsystems/test/unit/Wire/MiniBackend.hs index 1324d919db3..740a33b80ff 100644 --- a/libs/wire-subsystems/test/unit/Wire/MiniBackend.hs +++ b/libs/wire-subsystems/test/unit/Wire/MiniBackend.hs @@ -117,7 +117,6 @@ import Wire.FederationAPIAccess.Interpreter as FI import Wire.FederationConfigStore import Wire.GalleyAPIAccess import Wire.HashPassword (HashPassword) -import Wire.IndexedUserStore import Wire.InternalEvent hiding (DeleteUser) import Wire.InvitationStore import Wire.MlsKeyPackageSubsystem @@ -143,6 +142,7 @@ import Wire.TeamSubsystem.GalleyAPI import Wire.UserClientIndexStore (UserClientIndexStore) import Wire.UserGroupStore (UserGroupStore) import Wire.UserKeyStore +import Wire.UserSearchStore import Wire.UserStore import Wire.UserSubsystem import Wire.UserSubsystem.Error @@ -285,7 +285,7 @@ type MiniBackendLowerEffects = AppStore, TeamCollaboratorsStore, UserKeyStore, - IndexedUserStore, + UserSearchStore, FederationConfigStore, DRS.DomainRegistrationStore, PasswordResetCodeStore, @@ -341,7 +341,7 @@ miniBackendLowerEffectsInterpreters mb@(MiniBackendParams {..}) = . runInMemoryPasswordResetCodeStore . inMemoryDomainRegistrationStoreInterpreter . runFederationConfigStoreInMemory - . inMemoryIndexedUserStoreInterpreter + . inMemoryUserSearchStoreInterpreter . inMemoryUserKeyStoreInterpreter . inMemoryTeamCollaboratorsStoreInterpreter . inMemoryAppStoreInterpreter @@ -399,7 +399,7 @@ type StateEffects = State (Map UserId Password), State UserGroupInMemState, State [StoredApp], - State UserIndex, + State UserSearchIndex, State (Map EmailKey UserId), State [DRS.StoredDomainRegistration], State [InternalNotification], @@ -416,7 +416,7 @@ stateEffectsInterpreters MiniBackendParams {..} = . evalState [] . evalState [] . liftUserKeyStoreState - . liftIndexedUserStoreState + . liftUserSearchStoreState . liftAppStoreState . liftUserGroupStoreState . liftUserPasswordState @@ -510,7 +510,7 @@ data MiniBackend = MkMiniBackend users :: [StoredUser], userPasswords :: Map UserId Password, apps :: [StoredApp], - userIndex :: UserIndex, + userIndex :: UserSearchIndex, userKeys :: Map EmailKey UserId, passwordResetCodes :: Map PasswordResetKey (PRQueryData Identity), blockList :: [EmailKey], @@ -531,7 +531,7 @@ instance Default MiniBackend where { users = mempty, userPasswords = mempty, apps = mempty, - userIndex = emptyIndex, + userIndex = emptyUserSearchIndex, userKeys = mempty, passwordResetCodes = mempty, blockList = mempty, @@ -867,13 +867,13 @@ liftAppStoreState = interpret $ \case Polysemy.State.Get -> gets (.apps) Put newApps -> modify $ \b -> (b :: MiniBackend) {apps = newApps} -liftUserGroupStoreState :: Sem (State UserGroupInMemState : State [StoredApp] : State UserIndex : State (Map EmailKey UserId) : State [DRS.StoredDomainRegistration] : State [InternalNotification] : State MiniBackend : State [MiniEvent] : r) a -> Sem (State [StoredApp] : State UserIndex : State (Map EmailKey UserId) : State [DRS.StoredDomainRegistration] : State [InternalNotification] : State MiniBackend : State [MiniEvent] : r) a +liftUserGroupStoreState :: Sem (State UserGroupInMemState : State [StoredApp] : State UserSearchIndex : State (Map EmailKey UserId) : State [DRS.StoredDomainRegistration] : State [InternalNotification] : State MiniBackend : State [MiniEvent] : r) a -> Sem (State [StoredApp] : State UserSearchIndex : State (Map EmailKey UserId) : State [DRS.StoredDomainRegistration] : State [InternalNotification] : State MiniBackend : State [MiniEvent] : r) a liftUserGroupStoreState = interpret $ \case Polysemy.State.Get -> Polysemy.State.gets @MiniBackend (.userGroups) Put newState -> modify $ \b -> (b :: MiniBackend) {userGroups = newState} -liftIndexedUserStoreState :: (Member (State MiniBackend) r) => Sem (State UserIndex : r) a -> Sem r a -liftIndexedUserStoreState = interpret $ \case +liftUserSearchStoreState :: (Member (State MiniBackend) r) => Sem (State UserSearchIndex : r) a -> Sem r a +liftUserSearchStoreState = interpret $ \case Polysemy.State.Get -> gets (.userIndex) Put newUserIndex -> modify $ \b -> (b :: MiniBackend) {userIndex = newUserIndex} diff --git a/libs/wire-subsystems/test/unit/Wire/MockInterpreters.hs b/libs/wire-subsystems/test/unit/Wire/MockInterpreters.hs index 4630c0c7f77..2b43f6fa18b 100644 --- a/libs/wire-subsystems/test/unit/Wire/MockInterpreters.hs +++ b/libs/wire-subsystems/test/unit/Wire/MockInterpreters.hs @@ -39,7 +39,6 @@ import Wire.MockInterpreters.Events as MockInterpreters import Wire.MockInterpreters.FederationConfigStore as MockInterpreters import Wire.MockInterpreters.GalleyAPIAccess as MockInterpreters import Wire.MockInterpreters.HashPassword as MockInterpreters -import Wire.MockInterpreters.IndexedUserStore as MockInterpreters import Wire.MockInterpreters.InvitationStore as MockInterpreters import Wire.MockInterpreters.MeetingsStore as MockInterpreters import Wire.MockInterpreters.NotificationSubsystem as MockInterpreters @@ -55,6 +54,7 @@ import Wire.MockInterpreters.TeamCollaboratorsStore as MockInterpreters import Wire.MockInterpreters.TinyLog as MockInterpreters import Wire.MockInterpreters.UserGroupStore as MockInterpreters import Wire.MockInterpreters.UserKeyStore as MockInterpreters +import Wire.MockInterpreters.UserSearchStore as MockInterpreters import Wire.MockInterpreters.UserStore as MockInterpreters import Wire.MockInterpreters.UserSubsystem as MockInterpreters import Wire.MockInterpreters.VerificationCodeStore as MockInterpreters diff --git a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/IndexedUserStore.hs b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/IndexedUserStore.hs deleted file mode 100644 index b77869840fe..00000000000 --- a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/IndexedUserStore.hs +++ /dev/null @@ -1,191 +0,0 @@ -{-# LANGUAGE RecordWildCards #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.MockInterpreters.IndexedUserStore where - -import Data.Handle -import Data.Id -import Data.Map qualified as Map -import Data.Text qualified as Text -import Data.Tuple.Extra -import Database.Bloodhound.Internal.Client qualified as ES -import Database.Bloodhound.Types qualified as ES -import Imports -import Polysemy -import Polysemy.State -import Wire.API.Team.Size -import Wire.API.User -import Wire.API.User.Search -import Wire.IndexedUserStore -import Wire.UserSearch.Types - -newtype OrdDocId = OrdDocId Text - deriving (Show, Eq, Ord) - -data UserIndex = UserIndex - { nextVersion :: ES.DocVersion, - docs :: Map OrdDocId (UserDoc, ES.DocVersion) - } - deriving (Show, Eq) - -fromDocId :: ES.DocId -> OrdDocId -fromDocId (ES.DocId docId) = OrdDocId docId - -emptyIndex :: UserIndex -emptyIndex = - UserIndex - { nextVersion = (ES.DocVersion 0), - docs = mempty - } - -runInMemoryIndexedUserStoreIntepreter :: InterpreterFor IndexedUserStore r -runInMemoryIndexedUserStoreIntepreter = - evalState emptyIndex - . inMemoryIndexedUserStoreInterpreter - . raiseUnder - -inMemoryIndexedUserStoreInterpreter :: (Member (State UserIndex) r) => InterpreterFor IndexedUserStore r -inMemoryIndexedUserStoreInterpreter = - interpret $ \case - Upsert docId userDoc versionControl -> - upsertImpl docId userDoc versionControl - UpdateTeamSearchVisibilityInbound tid visibility -> - modify $ \index -> - index - { docs = - Map.map - ( first - ( \doc -> - if doc.udTeam == Just tid - then doc {udSearchVisibilityInbound = Just visibility} - else doc - ) - ) - index.docs - } - BulkUpsert upserts -> mapM_ (uncurry3 upsertImpl) upserts - DoesIndexExist -> pure True - SearchUsers searcher mTeam teamSearchInfo query maxResults mTypes -> - searchImpl searcher mTeam teamSearchInfo query maxResults mTypes - PaginateTeamMembers {} -> - error "IndexedUserStore: unimplemented in memory interpreter" - GetTeamSize tid -> - gets $ \index -> - let regulars = help [Just UserTypeRegular, Nothing] - apps = help [Just UserTypeApp] - help allowedTypes = - fromIntegral - . length - $ Map.filter (\(doc, _) -> doc.udTeam == Just tid && doc.udType `elem` allowedTypes) index.docs - in TeamSize {..} - -upsertImpl :: (Member (State UserIndex) r) => ES.DocId -> UserDoc -> ES.VersionControl -> Sem r () -upsertImpl docId userDoc versionControl = - modify $ \index -> - let mOldDoc = Map.lookup (fromDocId docId) index.docs - insertedDocs ver = Map.insert (fromDocId docId) (userDoc, ver) index.docs - insertWithVersionCheck newVer comp = - case mOldDoc of - (Just (_, oldVer)) - | newVer `comp` oldVer -> - index {docs = insertedDocs newVer} - _ -> index - in case (versionControl) of - (ES.NoVersionControl) -> - index - { nextVersion = succ index.nextVersion, - docs = insertedDocs index.nextVersion - } - (ES.InternalVersion newVer) -> - insertWithVersionCheck newVer (>) - (ES.ExternalGT (ES.ExternalDocVersion newVer)) -> - insertWithVersionCheck newVer (>) - (ES.ExternalGTE (ES.ExternalDocVersion newVer)) -> - insertWithVersionCheck newVer (>=) - (ES.ForceVersion (ES.ExternalDocVersion ver)) -> - index {docs = insertedDocs ver} - -data MatchType = Reject | NonTeamMember | TeamMate | NameMatch | HandleMatch - deriving (Show) - -matchScore :: MatchType -> Int -matchScore = \case - Reject -> 0 - NonTeamMember -> 1 - TeamMate -> 2 - NameMatch -> 3 - HandleMatch -> 4 - -searchImpl :: (Member (State UserIndex) r) => UserId -> Maybe TeamId -> TeamSearchInfo -> Text -> Int -> Maybe [UserTypeFilter] -> Sem r (SearchResult UserDoc) -searchImpl searcher mTeam teamSearchInfo query maxResults = \case - Nothing -> runSearch - Just [] -> runSearch - Just _ -> error "filtering contacts search for user types is not supposed by this mock interpreter." - where - runSearch = do - let teamFilter (doc :: UserDoc) = case (mTeam, teamSearchInfo) of - (Nothing, _) -> maybe NonTeamMember (const Reject) doc.udTeam - (Just _, NoTeam) -> maybe NonTeamMember (const Reject) doc.udTeam - (Just searcherTeam, AllUsers) -> - if Just searcherTeam == doc.udTeam then TeamMate else NonTeamMember - (Just searcherTeam, TeamOnly team) -> - if (searcherTeam == team && Just searcherTeam == doc.udTeam) - then TeamMate - else Reject - searcherFilter (doc :: UserDoc) = listToMaybe [Reject | doc.udId == searcher] - tokens = Text.splitOn " " query - nameFilter (doc :: UserDoc) = - case doc.udNormalized of - Nothing -> Reject - Just normalizedName -> - let isMatch = flip all tokens $ \token -> any (token `Text.isPrefixOf`) $ Text.splitOn " " normalizedName - in if isMatch then NameMatch else Reject - handleFilter (doc :: UserDoc) = - case doc.udHandle of - Nothing -> Reject - Just handle -> - if (query `Text.isPrefixOf` fromHandle handle) - then HandleMatch - else Reject - totalScore (doc :: UserDoc) = - matchScore (teamFilter doc) - * maybe 1 matchScore (searcherFilter doc) - * ( matchScore (nameFilter doc) - + matchScore (handleFilter doc) - ) - allDocs <- map fst . Map.elems <$> gets (.docs) - pure - . mkResult maxResults - . map fst - . sortOn snd - . filter (\(_, score) -> score /= 0) - . map (\doc -> (doc, totalScore doc)) - $ filter (\u -> fromMaybe True u.udSearchable) allDocs - -mkResult :: Int -> [a] -> SearchResult a -mkResult maxResults results = - SearchResult - { searchTook = 0, - searchReturned = min maxResults (length results), - searchResults = take maxResults results, - searchPagingState = Nothing, - searchHasMore = Just False, - searchFound = length results, - searchPolicy = FullSearch - } diff --git a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserSearchStore.hs b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserSearchStore.hs new file mode 100644 index 00000000000..c7612bbda26 --- /dev/null +++ b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserSearchStore.hs @@ -0,0 +1,407 @@ +-- This file is part of the Wire Server implementation. +-- +-- Copyright (C) 2026 Wire Swiss GmbH +-- +-- This program is free software: you can redistribute it and/or modify it under +-- the terms of the GNU Affero General Public License as published by the Free +-- Software Foundation, either version 3 of the License, or (at your option) any +-- later version. +-- +-- This program is distributed in the hope that it will be useful, but WITHOUT +-- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +-- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +-- details. +-- +-- You should have received a copy of the GNU Affero General Public License along +-- with this program. If not, see . + +-- | In-memory interpreter for 'UserSearchStore', used by 'MiniBackend' unit +-- tests. Ports the search semantics of the former +-- 'Wire.MockInterpreters.IndexedUserStore'. +-- +-- Note: like the former mock, 'SearchUsers' does not apply the +-- 'SearchVisibilityInbound' restriction of the searched user's team; the +-- setting is recorded per team (and applied by the Postgres interpreter in +-- production). +module Wire.MockInterpreters.UserSearchStore where + +import Data.Aeson qualified as Aeson +import Data.ByteString.Conversion (toByteString') +import Data.ByteString.Lazy qualified as LBS +import Data.Domain (Domain) +import Data.Id +import Data.Json.Util (toUTCTimeMillis) +import Data.Map.Strict qualified as Map +import Data.Qualified (Local, Qualified (..), tDomain, tUnqualified) +import Data.Text qualified as Text +import Data.Text.Ascii (decodeBase64Url, encodeBase64Url) +import Data.Text.Encoding qualified as TE +import Data.Time.Clock (UTCTime) +import Imports +import Polysemy +import Polysemy.Input +import Polysemy.State +import Wire.API.Team.Role (Role, roleName) +import Wire.API.Team.Size (TeamSize (..)) +import Wire.API.User +import Wire.API.User.Search +import Wire.UserSearch.Normalize (normalized) +import Wire.UserSearchStore + +-- | A user as seen by the mock search store. Roughly the document that +-- used to be indexed into ElasticSearch, plus the fields needed to apply +-- the former @shouldIndex@ candidate filter. +data SearchUserDoc = SearchUserDoc + { sdId :: UserId, + sdType :: UserType, + sdTeam :: Maybe TeamId, + sdName :: Text, + sdAccentId :: Maybe Int, + sdHandle :: Maybe Text, + sdEmail :: Maybe EmailAddress, + sdEmailUnvalidated :: Maybe EmailAddress, + sdNormalized :: Maybe Text, + sdSearchable :: Maybe Bool, + sdRole :: Maybe Role, + sdCreatedAt :: Maybe UTCTime, + sdManagedBy :: Maybe ManagedBy, + sdSAMLIdp :: Maybe Text, + sdScimExternalId :: Maybe Text, + sdSso :: Maybe Sso, + sdAccountStatus :: Maybe AccountStatus, + sdActivated :: Bool, + sdService :: Maybe ServiceId + } + deriving (Show, Eq) + +data UserSearchIndex = UserSearchIndex + { docs :: Map UserId SearchUserDoc, + teamVisibility :: Map TeamId SearchVisibilityInbound + } + deriving (Show, Eq) + +emptyUserSearchIndex :: UserSearchIndex +emptyUserSearchIndex = UserSearchIndex {docs = mempty, teamVisibility = mempty} + +runInMemoryUserSearchStoreInterpreter :: + (Member (Input (Local ())) r) => + InterpreterFor UserSearchStore r +runInMemoryUserSearchStoreInterpreter = + evalState emptyUserSearchIndex + . inMemoryUserSearchStoreInterpreter + . raiseUnder + +inMemoryUserSearchStoreInterpreter :: + ( Member (State UserSearchIndex) r, + Member (Input (Local ())) r + ) => + InterpreterFor UserSearchStore r +inMemoryUserSearchStoreInterpreter = + interpret $ \case + SearchUsers lSearcher mTeam teamSearchInfo query maxResults mTypes -> + searchImpl lSearcher mTeam teamSearchInfo query maxResults mTypes + PaginateTeamMembers filters maxResults mPagingState -> + paginateTeamMembersImpl filters maxResults mPagingState + SearchUsersFederated mOnlyInTeams query maxResults mTypes -> + federatedSearchImpl mOnlyInTeams query maxResults mTypes + GetTeamSize tid -> + gets $ \index -> + let isCounted doc = + doc.sdTeam == Just tid + && doc.sdActivated + && maybe True (`elem` [Active, Suspended]) doc.sdAccountStatus + && isNothing doc.sdService + countTyped ty = + fromIntegral . length $ + filter (\doc -> isCounted doc && mType doc == Just ty) (Map.elems index.docs) + mType doc = case doc.sdType of + UserTypeRegular -> Just UserTypeRegular + UserTypeApp -> Just UserTypeApp + UserTypeBot -> Nothing + in TeamSize {regulars = countTyped UserTypeRegular, apps = countTyped UserTypeApp} + SetTeamSearchVisibilityInbound tid visibility -> + modify $ \index -> + index {teamVisibility = Map.insert tid visibility index.teamVisibility} + +emailText :: EmailAddress -> Text +emailText = TE.decodeUtf8 . toByteString' + +toContact :: Domain -> SearchUserDoc -> Contact +toContact dom doc = + Contact + { contactQualifiedId = Qualified doc.sdId dom, + contactName = doc.sdName, + contactColorId = doc.sdAccentId, + contactHandle = doc.sdHandle, + contactTeam = doc.sdTeam, + contactType = doc.sdType + } + +data MatchType = Reject | NonTeamMember | TeamMate | NameMatch | HandleMatch + deriving (Show) + +matchScore :: MatchType -> Int +matchScore = \case + Reject -> 0 + NonTeamMember -> 1 + TeamMate -> 2 + NameMatch -> 3 + HandleMatch -> 4 + +searchImpl :: + forall r. + (Member (State UserSearchIndex) r) => + Local UserId -> + Maybe TeamId -> + TeamSearchInfo -> + Text -> + Int -> + Maybe [UserTypeFilter] -> + Sem r (SearchResult Contact) +searchImpl lSearcher mTeam teamSearchInfo query maxResults mTypes = do + allDocs <- gets (Map.elems . (.docs)) + pure + . mkContactResult maxResults + . map (toContact (tDomain lSearcher)) + . map fst + . sortOn snd + . filter (\(_, score) -> score /= 0) + . map (\doc -> (doc, totalScore doc)) + . filter candidateDoc + . filter (\u -> fromMaybe True u.sdSearchable) + . filter (typeMatches mTypes) + $ allDocs + where + typeMatches = \case + Nothing -> const True + Just [] -> const True + Just uts -> \doc -> userTypeToFilter doc.sdType `elem` uts + + userTypeToFilter UserTypeRegular = UserTypeFilterRegular + userTypeToFilter UserTypeApp = UserTypeFilterApp + userTypeToFilter UserTypeBot = UserTypeFilterRegular + + candidateDoc doc = + doc.sdActivated + && maybe True (`elem` [Active, Suspended]) doc.sdAccountStatus + && isNothing doc.sdService + && doc.sdId /= tUnqualified lSearcher + + teamFilter (doc :: SearchUserDoc) = case (mTeam, teamSearchInfo) of + (Nothing, _) -> maybe NonTeamMember (const Reject) doc.sdTeam + (Just _, NoTeam) -> maybe NonTeamMember (const Reject) doc.sdTeam + (Just searcherTeam, AllUsers) -> + if Just searcherTeam == doc.sdTeam then TeamMate else NonTeamMember + (Just searcherTeam, TeamOnly team) -> + if searcherTeam == team && Just searcherTeam == doc.sdTeam + then TeamMate + else Reject + tokens = Text.splitOn " " (normalized query) + nameFilter (doc :: SearchUserDoc) = + case doc.sdNormalized of + Nothing -> Reject + Just normalizedName -> + let isMatch = all (\token -> any (token `Text.isPrefixOf`) $ Text.splitOn " " normalizedName) tokens + in if isMatch then NameMatch else Reject + handleFilter (doc :: SearchUserDoc) = + case doc.sdHandle of + Nothing -> Reject + Just handle -> + if normalized query `Text.isPrefixOf` handle + then HandleMatch + else Reject + totalScore (doc :: SearchUserDoc) = + matchScore (teamFilter doc) + * (matchScore (nameFilter doc) + matchScore (handleFilter doc)) + +paginateTeamMembersImpl :: + (Member (State UserSearchIndex) r) => + BrowseTeamFilters -> + Int -> + Maybe PagingState -> + Sem r (SearchResult TeamContact) +paginateTeamMembersImpl filters maxResults mPagingState = do + allDocs <- gets (Map.elems . (.docs)) + let offset = fromMaybe 0 (mPagingState >>= decodeOffset) + teamDocs = + filter candidateDoc + . filter (filterTeam filters.teamId) + . filter (filterSearchable filters.mSearchable) + . filter (filterEmail filters.mEmailVerificationFilter) + . filter (filterRole filters.mRoleFilter) + . filter (filterQuery filters.mQuery) + $ allDocs + sorted = sortDocs teamDocs + page = take maxResults (drop offset sorted) + hasMore = length sorted > offset + length page + pure (mkResult maxResults (map toTeamContact page) hasMore (offset + length page)) + where + candidateDoc doc = + doc.sdActivated + && maybe True (`elem` [Active, Suspended]) doc.sdAccountStatus + && isNothing doc.sdService + + filterTeam tid doc = doc.sdTeam == Just tid + + filterSearchable = \case + Nothing -> const True + Just False -> \doc -> doc.sdSearchable == Just False + Just True -> \doc -> fromMaybe True doc.sdSearchable + + filterRole = \case + Nothing -> const True + Just (RoleFilter rs) -> \doc -> maybe False (`elem` rs) doc.sdRole + + filterEmail = \case + Nothing -> const True + Just EmailVerified -> \doc -> isJust doc.sdEmail && isNothing doc.sdEmailUnvalidated + Just EmailUnverified -> \doc -> isJust doc.sdEmailUnvalidated + + filterQuery mQuery doc = case normalized <$> mQuery of + Nothing -> True + Just q | Text.null q -> True + Just q -> all (`tokenMatches` doc) (Text.splitOn " " q) + + tokenMatches token doc = + maybe False (any (token `Text.isPrefixOf`) . Text.splitOn " ") doc.sdNormalized + || maybe False (token `Text.isPrefixOf`) doc.sdHandle + || maybe False ((token `Text.isPrefixOf`) . emailText) doc.sdEmail + + -- Without an explicit sort, browse is ordered by creation date, newest + -- first (as in the former ES query and the PG interpreter). With one, + -- missing keys sort last when ascending and first when descending, and + -- the user id is the deterministic tie breaker. + sortDocs :: [SearchUserDoc] -> [SearchUserDoc] + sortDocs docs = case filters.mSortBy of + Nothing -> sortOn (\doc -> (Down doc.sdCreatedAt, doc.sdId)) docs + Just _ -> arrange (fromMaybe SortOrderAsc filters.mSortOrder) (sortOn ascKey docs) + where + ascKey doc = (isNothing (keyOf doc), keyOf doc, doc.sdId) + + keyOf :: SearchUserDoc -> Maybe Text + keyOf doc = case filters.mSortBy of + Just SortByName -> Just doc.sdName + Just SortByHandle -> doc.sdHandle + Just SortByEmail -> emailText <$> doc.sdEmail + Just SortBySAMLIdp -> doc.sdSAMLIdp + Just SortByRole -> roleName @Text <$> doc.sdRole + Just SortByManagedBy -> Text.pack . show <$> doc.sdManagedBy + Just SortByCreatedAt -> Text.pack . show <$> doc.sdCreatedAt + Nothing -> Text.pack . show <$> doc.sdCreatedAt + + arrange = \case + SortOrderAsc -> id + SortOrderDesc -> reverse + + toTeamContact :: SearchUserDoc -> TeamContact + toTeamContact doc = + TeamContact + { teamContactUserId = doc.sdId, + teamContactUserType = doc.sdType, + teamContactName = doc.sdName, + teamContactColorId = doc.sdAccentId, + teamContactHandle = doc.sdHandle, + teamContactTeam = doc.sdTeam, + teamContactEmail = doc.sdEmail, + teamContactCreatedAt = toUTCTimeMillis <$> doc.sdCreatedAt, + teamContactManagedBy = doc.sdManagedBy, + teamContactSAMLIdp = doc.sdSAMLIdp, + teamContactRole = doc.sdRole, + teamContactScimExternalId = doc.sdScimExternalId, + teamContactSso = doc.sdSso, + teamContactEmailUnvalidated = doc.sdEmailUnvalidated, + teamContactUserGroups = [], + teamContactSearchable = fromMaybe True doc.sdSearchable + } + +federatedSearchImpl :: + (Member (State UserSearchIndex) r, Member (Input (Local ())) r) => + Maybe [TeamId] -> + Text -> + Int -> + Maybe [UserTypeFilter] -> + Sem r (SearchResult Contact) +federatedSearchImpl mOnlyInTeams query maxResults mTypes = do + loc <- input + index <- get + let allDocs = Map.elems index.docs + pure + . mkContactResult maxResults + . map (toContact (tDomain loc)) + . filter candidateDoc + . filter (\u -> fromMaybe True u.sdSearchable) + . filter (typeMatches mTypes) + . filter (visibility index) + . filter matchesQuery + $ allDocs + where + term = normalized query + tokens = Text.splitOn " " term + + typeMatches = \case + Nothing -> const True + Just [] -> const True + Just uts -> \doc -> userTypeToFilter doc.sdType `elem` uts + + userTypeToFilter UserTypeRegular = UserTypeFilterRegular + userTypeToFilter UserTypeApp = UserTypeFilterApp + userTypeToFilter UserTypeBot = UserTypeFilterRegular + + candidateDoc doc = + doc.sdActivated + && maybe True (`elem` [Active, Suspended]) doc.sdAccountStatus + && isNothing doc.sdService + + visOf idx tid = Map.findWithDefault SearchableByOwnTeam tid idx.teamVisibility + + visibility idx doc = case mOnlyInTeams of + Nothing -> case doc.sdTeam of + Nothing -> True + Just tid -> visOf idx tid == SearchableByAllTeams + Just [] -> False + Just teams -> + maybe False (\tid -> tid `elem` teams && visOf idx tid == SearchableByAllTeams) doc.sdTeam + + matchesQuery doc = + not (Text.null term) + && all + ( \token -> + any (token `Text.isPrefixOf`) (maybe [] (Text.splitOn " ") doc.sdNormalized) + || maybe False (token `Text.isPrefixOf`) doc.sdHandle + ) + tokens + +decodeOffset :: PagingState -> Maybe Int +decodeOffset (PagingState ps) = do + bs <- decodeBase64Url ps + case Aeson.eitherDecode (LBS.fromStrict bs) of + Right n -> Just n + Left _ -> Nothing + +mkResult :: Int -> [a] -> Bool -> Int -> SearchResult a +mkResult maxResults results hasMore nextOffset = + SearchResult + { searchTook = 0, + searchReturned = min maxResults (length results), + searchResults = take maxResults results, + searchPagingState = + if hasMore + then Just . PagingState . encodeBase64Url . LBS.toStrict . Aeson.encode $ nextOffset + else Nothing, + searchHasMore = Just hasMore, + searchFound = length results, + searchPolicy = FullSearch + } + +-- | Result builder for searches without paging. +mkContactResult :: Int -> [a] -> SearchResult a +mkContactResult maxResults results = + SearchResult + { searchTook = 0, + searchReturned = min maxResults (length results), + searchResults = take maxResults results, + searchPagingState = Nothing, + searchHasMore = Nothing, + searchFound = length results, + searchPolicy = FullSearch + } diff --git a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserStore.hs b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserStore.hs index d30c8546589..f89a4530c7f 100644 --- a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserStore.hs +++ b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserStore.hs @@ -23,8 +23,6 @@ import Control.Monad.Trans.Maybe (MaybeT (..)) import Data.Handle import Data.Id import Data.Map qualified as Map -import Data.Time -import Data.Time.Calendar.OrdinalDate import Imports import Polysemy import Polysemy.Error @@ -35,7 +33,6 @@ import Wire.API.User qualified as User import Wire.API.User.Search (SetSearchable (SetSearchable)) import Wire.StoredUser import Wire.UserStore -import Wire.UserStore.IndexUser runInMemoryUserStoreInterpreter :: [StoredUser] -> Map UserId Password -> InterpreterFor UserStore r runInMemoryUserStoreInterpreter users passwords = @@ -111,11 +108,6 @@ inMemoryUserStoreInterpreterWithDeleteHook onDelete = interpret $ \case DeactivateUser uid -> updateUserInStore uid (\u -> u {activated = False}) UpdateFeatureConferenceCalling {} -> error "UpdateFeatureConferenceCalling: Not implemented" LookupFeatureConferenceCalling {} -> error "FeatureConferenceCalling: Not implemented" - GetIndexUser uid -> do - mUser <- gets @[StoredUser] $ find (\user -> user.id == uid) - pure $ storedUserToIndexUser <$> mUser - GetIndexUsersPaginated _pageSize _pagingState -> - error "GetIndexUsersPaginated not implemented in inMemoryUserStoreInterpreter" UpdateUserHandleEither uid hUpdate -> runError $ modifyLocalUsers (traverse doUpdate) where doUpdate :: StoredUser -> Sem (Error StoredUserUpdateError : r) StoredUser @@ -177,31 +169,7 @@ inMemoryUserStoreInterpreterWithDeleteHook onDelete = interpret $ \case LookupServiceUsers {} -> error "lookupServiceUsers: Not implemented" LookupServiceUsersForTeam {} -> error "lookupServiceUsersForteam: Not implemented" -storedUserToIndexUser :: StoredUser -> IndexUser -storedUserToIndexUser storedUser = - -- If we really care about this, we could start storing the writetimes, but we - -- don't need it right now - let defaultTime = UTCTime (YearDay 0 1) 0 - in IndexUser - { userId = storedUser.id, - userType = inferUserType storedUser.serviceId storedUser.userType, - teamId = storedUser.teamId, - name = storedUser.name, - accountStatus = storedUser.status, - handle = storedUser.handle, - email = storedUser.email, - colourId = storedUser.accentId, - activated = storedUser.activated, - serviceId = storedUser.serviceId, - managedBy = storedUser.managedBy, - ssoId = storedUser.ssoId, - unverifiedEmail = Nothing, - searchable = storedUser.searchable, - createdAt = defaultTime, - updatedAt = defaultTime - } - -lookupLocaleImpl :: (Member (State [StoredUser]) r) => UserId -> Sem r (Maybe ((Maybe Language, Maybe Country))) +lookupLocaleImpl :: (Member (State [StoredUser]) r) => UserId -> Sem r (Maybe (Maybe Language, Maybe Country)) lookupLocaleImpl uid = do users <- get let mUser = find ((== uid) . (.id)) users diff --git a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserSubsystem.hs b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserSubsystem.hs index 786f733240f..143996c6f2e 100644 --- a/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserSubsystem.hs +++ b/libs/wire-subsystems/test/unit/Wire/MockInterpreters/UserSubsystem.hs @@ -87,7 +87,6 @@ inMemoryUserSubsystemInterpreter = BlockListInsert _ -> error "BlockListInsert: implement on demand (userSubsystemInterpreter)" UpdateTeamSearchVisibilityInbound _ -> error "UpdateTeamSearchVisibilityInbound: implement on demand (userSubsystemInterpreter)" AcceptTeamInvitation {} -> error "AcceptTeamInvitation: implement on demand (userSubsystemInterpreter)" - InternalUpdateSearchIndex _ -> error "InternalUpdateSearchIndex: implement on demand (userSubsystemInterpreter)" InternalFindTeamInvitation {} -> error "InternalFindTeamInvitation: implement on demand (userSubsystemInterpreter)" GetUserExportData _ -> error "GetUserExportData: implement on demand (userSubsystemInterpreter)" RemoveEmailEither _ -> error "RemoveEmailEither: implement on demand (userSubsystemInterpreter)" diff --git a/libs/wire-subsystems/test/unit/Wire/UserSearch/TypesSpec.hs b/libs/wire-subsystems/test/unit/Wire/UserSearch/TypesSpec.hs deleted file mode 100644 index a09d56bd8ff..00000000000 --- a/libs/wire-subsystems/test/unit/Wire/UserSearch/TypesSpec.hs +++ /dev/null @@ -1,73 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2025 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Wire.UserSearch.TypesSpec where - -import Control.Error (hush) -import Data.Aeson as Aeson -import Data.Fixed -import Data.Handle -import Data.Id -import Data.Json.Util -import Data.Time -import Data.Time.Clock.POSIX -import Imports -import Test.Hspec -import Test.Hspec.QuickCheck -import Test.QuickCheck -import Wire.API.Team.Role -import Wire.API.User -import Wire.UserSearch.Types - -spec :: Spec -spec = describe "UserDoc" $ do - describe "JSON" $ do - prop "roundrip to/fromJSON" $ \(userDoc :: UserDoc) -> - fromJSON (toJSON userDoc) === Aeson.Success userDoc - - it "should be backwards comptibile" $ do - eitherDecode (userDoc1ByteString) `shouldBe` Right userDoc1 - -mkTime :: Int -> UTCTime -mkTime = posixSecondsToUTCTime . secondsToNominalDiffTime . MkFixed . (* 1000000000) . fromIntegral - -userDoc1 :: UserDoc -userDoc1 = - UserDoc - { udId = fromJust . hush . parseIdFromText $ "0a96b396-57d6-11ea-a04b-7b93d1a5c19c", - udTeam = hush . parseIdFromText $ "17c59b18-57d6-11ea-9220-8bbf5eee961a", - udName = Just . Name $ "Carl Phoomp", - udNormalized = Just $ "carl phoomp", - udHandle = Just . fromJust . parseHandle $ "phoompy", - udEmail = Just $ unsafeEmailAddress "phoompy" "example.com", - udColourId = Just . ColourId $ 32, - udAccountStatus = Just Active, - udSAMLIdP = Just "https://issuer.net/214234", - udManagedBy = Just ManagedByScim, - udCreatedAt = Just (toUTCTimeMillis (mkTime 1598737800000)), - udRole = Just RoleAdmin, - udSearchVisibilityInbound = Nothing, - udScimExternalId = Nothing, - udSso = Nothing, - udEmailUnvalidated = Nothing, - udSearchable = Nothing, - udType = Nothing - } - --- Dont touch this. This represents serialized legacy data. -userDoc1ByteString :: LByteString -userDoc1ByteString = "{\"email\":\"phoompy@example.com\",\"account_status\":\"active\",\"handle\":\"phoompy\",\"managed_by\":\"scim\",\"role\":\"admin\",\"accent_id\":32,\"name\":\"Carl Phoomp\",\"created_at\":\"2020-08-29T21:50:00.000Z\",\"team\":\"17c59b18-57d6-11ea-9220-8bbf5eee961a\",\"id\":\"0a96b396-57d6-11ea-a04b-7b93d1a5c19c\",\"normalized\":\"carl phoomp\",\"saml_idp\":\"https://issuer.net/214234\"}" diff --git a/libs/wire-subsystems/test/unit/Wire/UserSubsystem/InterpreterSpec.hs b/libs/wire-subsystems/test/unit/Wire/UserSubsystem/InterpreterSpec.hs index 68942a77cb3..d7a5682f284 100644 --- a/libs/wire-subsystems/test/unit/Wire/UserSubsystem/InterpreterSpec.hs +++ b/libs/wire-subsystems/test/unit/Wire/UserSubsystem/InterpreterSpec.hs @@ -38,7 +38,6 @@ import Data.Set (insert, member, notMember) import Data.Set qualified as S import Data.String.Conversions (cs) import Data.Text.Encoding (encodeUtf8) -import Database.Bloodhound.Internal.Client qualified as ES import Imports import Polysemy import Polysemy.Error @@ -64,7 +63,6 @@ import Wire.AppSubsystem import Wire.AuthenticationSubsystem.Error import Wire.ClientSubsystem.Error (ClientError) import Wire.DomainRegistrationStore qualified as DRS -import Wire.IndexedUserStore qualified as IU import Wire.InvitationStore (InsertInvitation, StoredInvitation) import Wire.InvitationStore qualified as InvitationStore import Wire.MiniBackend @@ -72,8 +70,7 @@ import Wire.MockInterpreters import Wire.RateLimit import Wire.StoredUser import Wire.UserKeyStore -import Wire.UserSearch.Types -import Wire.UserStore.IndexUser +import Wire.UserSearch.Normalize (normalized) import Wire.UserSubsystem import Wire.UserSubsystem.Error import Wire.UserSubsystem.HandleBlacklist @@ -1109,14 +1106,36 @@ spec = describe "UserSubsystem.Interpreter" do let teamMember = mkTeamMember searcher.id fullPermissions Nothing defUserLegalHoldStatus searchee = searcheeNoHandle {handle = Just searcheeHandle} :: StoredUser - storedUserToDoc :: StoredUser -> UserDoc - storedUserToDoc user = indexUserToDoc defaultSearchVisibilityInbound Nothing (storedUserToIndexUser user) + storedUserToDoc :: StoredUser -> SearchUserDoc + storedUserToDoc user = + let msso = user.identity >>= ssoIdentity + in SearchUserDoc + { sdId = user.id, + sdType = inferUserType user.serviceId user.userType, + sdTeam = user.teamId, + sdName = fromName user.name, + sdAccentId = Just (fromIntegral (fromColourId user.accentId)), + sdHandle = fromHandle <$> user.handle, + sdEmailUnvalidated = user.emailUnvalidated, + sdEmail = user.email, + sdNormalized = Just (normalized (fromName user.name)), + sdSearchable = user.searchable, + sdRole = Nothing, + sdCreatedAt = Nothing, + sdManagedBy = user.managedBy, + sdSAMLIdp = fst <$> (msso >>= ssoIssuerAndNameId), + sdScimExternalId = join (scimExternalId <$> user.managedBy <*> msso), + sdSso = fmap (uncurry Sso) (msso >>= ssoIssuerAndNameId), + sdAccountStatus = user.status, + sdActivated = True, + sdService = user.serviceId + } - indexFromStoredUsers :: [StoredUser] -> UserIndex - indexFromStoredUsers storedUsers = do - run . execState emptyIndex . inMemoryIndexedUserStoreInterpreter $ do - for_ storedUsers $ \storedUser -> - IU.upsert (userIdToDocId storedUser.id) (storedUserToDoc storedUser) ES.NoVersionControl + indexFromStoredUsers :: [StoredUser] -> UserSearchIndex + indexFromStoredUsers storedUsers = + emptyUserSearchIndex + { docs = Map.fromList [(storedUser.id, storedUserToDoc storedUser) | storedUser <- storedUsers] + } localBackend = def diff --git a/libs/wire-subsystems/wire-subsystems.cabal b/libs/wire-subsystems/wire-subsystems.cabal index 66662926e3d..6742ea35e9b 100644 --- a/libs/wire-subsystems/wire-subsystems.cabal +++ b/libs/wire-subsystems/wire-subsystems.cabal @@ -100,7 +100,6 @@ common common-all , base16-bytestring , base64-bytestring , bilge - , bloodhound , bytestring , bytestring-conversion , case-insensitive @@ -374,11 +373,6 @@ library Wire.IdPRawMetadataStore.Mem Wire.IdPSubsystem Wire.IdPSubsystem.Interpreter - Wire.IndexedUserStore - Wire.IndexedUserStore.Bulk.ElasticSearch - Wire.IndexedUserStore.ElasticSearch - Wire.IndexedUserStore.MigrationStore - Wire.IndexedUserStore.MigrationStore.ElasticSearch Wire.InternalEvent Wire.InvitationStore Wire.InvitationStore.Cassandra @@ -494,12 +488,11 @@ library Wire.UserList Wire.UserPendingActivationStore Wire.UserPendingActivationStore.Cassandra - Wire.UserSearch.Metrics - Wire.UserSearch.Migration - Wire.UserSearch.Types + Wire.UserSearch.Normalize Wire.UserStore Wire.UserStore.Cassandra - Wire.UserStore.IndexUser + Wire.UserSearchStore + Wire.UserSearchStore.Postgres Wire.UserStore.Migration Wire.UserStore.Migration.Types Wire.UserStore.Postgres @@ -535,13 +528,11 @@ library , base64-bytestring , bilge , bimap - , bloodhound , bytestring , bytestring-conversion , case-insensitive , cassandra-util , conduit - , containers , cql , crypton , crypton-x509 @@ -667,7 +658,7 @@ test-suite wire-subsystems-tests Wire.MockInterpreters.FederationConfigStore Wire.MockInterpreters.GalleyAPIAccess Wire.MockInterpreters.HashPassword - Wire.MockInterpreters.IndexedUserStore + Wire.MockInterpreters.UserSearchStore Wire.MockInterpreters.InvitationStore Wire.MockInterpreters.MeetingsStore Wire.MockInterpreters.NotificationSubsystem @@ -696,7 +687,6 @@ test-suite wire-subsystems-tests Wire.TeamCollaboratorsSubsystem.InterpreterSpec Wire.TeamInvitationSubsystem.InterpreterSpec Wire.UserGroupSubsystem.InterpreterSpec - Wire.UserSearch.TypesSpec Wire.UserStoreSpec Wire.UserSubsystem.InterpreterSpec Wire.Util diff --git a/nix/haskell-pins.nix b/nix/haskell-pins.nix index 13ebf988279..b2ca13419f0 100644 --- a/nix/haskell-pins.nix +++ b/nix/haskell-pins.nix @@ -71,10 +71,6 @@ let }; }; - bloodhound = { - src = inputs.bloodhound; - }; - # Our fork because we need to a few special things http-client = { diff --git a/nix/local-haskell-packages.nix b/nix/local-haskell-packages.nix index 5562225f673..f8c24fefa00 100644 --- a/nix/local-haskell-packages.nix +++ b/nix/local-haskell-packages.nix @@ -45,7 +45,6 @@ hsuper: hself: { wire-server-enterprise = hself.callPackage ../services/wire-server-enterprise/default.nix { }; assets = hself.callPackage ../tools/db/assets/default.nix { }; auto-whitelist = hself.callPackage ../tools/db/auto-whitelist/default.nix { }; - find-undead = hself.callPackage ../tools/db/find-undead/default.nix { }; inconsistencies = hself.callPackage ../tools/db/inconsistencies/default.nix { }; migrate-features = hself.callPackage ../tools/db/migrate-features/default.nix { }; migrate-sso-feature-flag = hself.callPackage ../tools/db/migrate-sso-feature-flag/default.nix { }; diff --git a/nix/manual-overrides.nix b/nix/manual-overrides.nix index 7ace4edb661..75a28c1c901 100644 --- a/nix/manual-overrides.nix +++ b/nix/manual-overrides.nix @@ -8,9 +8,6 @@ hself: hsuper: { # FUTUREWORK: investigate whether all of these tests need to fail # ---------------- - # test suite doesn't compile and needs network access - bloodhound = hlib.dontCheck hsuper.bloodhound; - # tests need network access, cabal2nix disables haddocks cql-io = hlib.doHaddock (hlib.dontCheck hsuper.cql-io); diff --git a/services/brig/.env b/services/brig/.env index 2366ad06134..212b719cf8e 100644 --- a/services/brig/.env +++ b/services/brig/.env @@ -2,8 +2,6 @@ BRIG_WEB_HOST=127.0.0.1 BRIG_CASSANDRA_HOST=127.0.0.1 BRIG_CASSANDRA_PORT=9042 BRIG_CASSANDRA_KEYSPACE=brig_test -BRIG_ELASTICSEARCH_URL=http://localhost:9200 -BRIG_ELASTICSEARCH_USER_INDEX=directory_test AWS_SQS_ENDPOINT=http://localhost:4568 AWS_SES_ENDPOINT=http://localhost:4569 AWS_DYNAMODB_ENDPOINT=http://localhost:4567 diff --git a/services/brig/brig.cabal b/services/brig/brig.cabal index 613e931c9ef..9a2ebfcd806 100644 --- a/services/brig/brig.cabal +++ b/services/brig/brig.cabal @@ -110,9 +110,6 @@ library Brig.DeleteQueue.Interpreter Brig.Effects.ConnectionStore Brig.Effects.ConnectionStore.Cassandra - Brig.Index.Eval - Brig.Index.Options - Brig.Index.Types Brig.InternalEvent.Process Brig.InternalEvent.Types Brig.IO.Intra @@ -192,8 +189,6 @@ library Brig.User.Auth.Cookie Brig.User.Client Brig.User.EJPD - Brig.User.Search.Index - Brig.User.Search.SearchIndex Brig.User.Template Brig.Version @@ -216,7 +211,7 @@ library , base16-bytestring >=0.1 , base64-bytestring >=1.0 , bilge >=0.21.1 - , bloodhound >=0.13 + , bytestring >=0.10 , bytestring-conversion >=0.2 , cassandra-util >=0.16.2 @@ -328,9 +323,13 @@ executable brig-index , base , brig , extended + , hasql + , uuid , imports , optparse-applicative + , text , tinylog + , wire-subsystems executable brig-integration import: common-all @@ -361,7 +360,6 @@ executable brig-integration API.User.Util Federation.End2end Federation.Util - Index.Create Run SMTP Util @@ -378,7 +376,6 @@ executable brig-integration , base , base16-bytestring , bilge - , bloodhound , brig , bytestring >=0.9 , bytestring-conversion @@ -402,7 +399,6 @@ executable brig-integration , http-client , http-client-tls >=0.3 , http-media - , http-reverse-proxy , http-types , imports , jose @@ -427,7 +423,6 @@ executable brig-integration , random-shuffle , raw-strings-qq , retry >=0.6 - , safe , saml2-web-sso , servant , servant-client diff --git a/services/brig/default.nix b/services/brig/default.nix index e431f9c93db..c60bf9e2cb8 100644 --- a/services/brig/default.nix +++ b/services/brig/default.nix @@ -18,7 +18,6 @@ , base64-bytestring , bilge , binary -, bloodhound , bytestring , bytestring-conversion , case-insensitive @@ -168,7 +167,6 @@ mkDerivation { base16-bytestring base64-bytestring bilge - bloodhound bytestring bytestring-conversion cassandra-util @@ -270,7 +268,6 @@ mkDerivation { base base16-bytestring bilge - bloodhound bytestring bytestring-conversion case-insensitive diff --git a/services/brig/index/src/Main.hs b/services/brig/index/src/Main.hs index 99ea1f0c595..7e38a1d5acc 100644 --- a/services/brig/index/src/Main.hs +++ b/services/brig/index/src/Main.hs @@ -1,6 +1,8 @@ +{-# LANGUAGE OverloadedStrings #-} + -- This file is part of the Wire Server implementation. -- --- Copyright (C) 2022 Wire Swiss GmbH +-- Copyright (C) 2026 Wire Swiss GmbH -- -- This program is free software: you can redistribute it and/or modify it under -- the terms of the GNU Affero General Public License as published by the Free @@ -15,28 +17,93 @@ -- You should have received a copy of the GNU Affero General Public License along -- with this program. If not, see . -module Main - ( main, - ) -where +-- | One-off backfill tool for @wire_user.name_normalized@ (required by +-- 'Wire.UserSearchStore' after the ElasticSearch user index was removed). +-- Run once per deployment before switching user search over to Postgres: +-- +-- > brig-index backfill-normalized-names --pg-settings "host=... dbname=... user=... password=..." +module Main (main) where -import Brig.Index.Eval -import Brig.Index.Options +import Data.Functor.Contravariant ((>$<)) +import Data.Text qualified as Text +import Data.Text.IO qualified as TextIO +import Data.UUID (UUID) +import Hasql.Connection qualified as Hasql +import Hasql.Connection.Settings qualified as HasqlSettings +import Hasql.Decoders qualified as Decoders +import Hasql.Encoders qualified as Encoders +import Hasql.Errors (IsError (..), toDetailedText) +import Hasql.Session qualified as Session +import Hasql.Statement (Statement) +import Hasql.Statement qualified as Statement import Imports import Options.Applicative -import System.Exit -import System.Logger.Extended qualified as Log +import System.Exit (exitFailure) +import Wire.UserSearch.Normalize (normalized) + +data Opts = Opts + { pgSettings :: Text, + batchSize :: Int32 + } + +optsParser :: Parser Opts +optsParser = + Opts + <$> strOption + ( long "pg-settings" + <> metavar "SETTINGS" + <> help "libpq connection settings, e.g. \"host=... dbname=... user=... password=...\"" + ) + <*> option + auto + ( long "batch-size" + <> metavar "N" + <> value 500 + <> showDefault + <> help "number of users per batch" + ) + +-- | Fetches up to N users with a missing @name_normalized@. +selectBatch :: Statement Int32 [(UUID, Text)] +selectBatch = + Statement.preparable + "SELECT id :: uuid, name :: text FROM wire_user WHERE name_normalized IS NULL AND name IS NOT NULL ORDER BY id LIMIT ($1 :: int4)" + (const (0 :: Int32) >$< Encoders.param (Encoders.nonNullable Encoders.int4)) + (Decoders.rowList ((,) <$> Decoders.column (Decoders.nonNullable Decoders.uuid) <*> Decoders.column (Decoders.nonNullable Decoders.text))) + +updateOne :: Statement (UUID, Text) () +updateOne = + Statement.preparable + "UPDATE wire_user SET name_normalized = ($2 :: text) WHERE id = ($1 :: uuid)" + ( (fst >$< Encoders.param (Encoders.nonNullable Encoders.uuid)) + <> (snd >$< Encoders.param (Encoders.nonNullable Encoders.text)) + ) + Decoders.noResult main :: IO () main = do - cmd <- execParser (info (helper <*> commandParser) desc) - lgr <- initLogger - runCommand lgr cmd - -- TODO: dump metrics in a suitable format (NOT json) - exitSuccess - where - desc = - header "brig-index" - <> progDesc "Brig Search Index Utilities" - <> fullDesc - initLogger = Log.mkLogger Log.Debug Nothing (Just $ Last Log.JSON) + opts <- execParser (info (optsParser <**> helper) (fullDesc <> progDesc "Backfill wire_user.name_normalized with the ICU-folded lowercase display name")) + conn <- do + r <- Hasql.acquire (HasqlSettings.connectionString opts.pgSettings) + either (failWith "connecting to postgres") pure r + let loop total = do + rows <- runSession conn (Session.statement opts.batchSize selectBatch) + case rows of + [] -> + TextIO.putStrLn + ("backfill-normalized-names: done, updated " <> Text.pack (show (total :: Int)) <> " users") + batch -> do + forM_ batch $ \(uid, name) -> + runSession conn (Session.statement (uid, normalized name) updateOne) + loop (total + length batch) + loop 0 + +failWith :: (IsError e) => Text -> e -> IO a +failWith context err = do + TextIO.putStrLn ("backfill-normalized-names: " <> context <> ": " <> toDetailedText err) + exitFailure + +runSession :: Hasql.Connection -> Session.Session a -> IO a +runSession conn sess = do + r <- Hasql.use conn sess + either (failWith "postgres query") pure r diff --git a/services/brig/src/Brig/API/Auth.hs b/services/brig/src/Brig/API/Auth.hs index 5051540aacd..9263ec202eb 100644 --- a/services/brig/src/Brig/API/Auth.hs +++ b/services/brig/src/Brig/API/Auth.hs @@ -77,7 +77,6 @@ import Wire.UserSubsystem.UserSubsystemConfig accessH :: ( Member TinyLog r, - Member UserSubsystem r, Member Events r, Member (Input AuthenticationSubsystemConfig) r, Member (Embed IO) r, @@ -103,7 +102,6 @@ accessH mcid ut' mat' = do access :: ( Member TinyLog r, - Member UserSubsystem r, Member Events r, UserTokenLike u, AccessTokenLike a, @@ -244,7 +242,6 @@ removeCookies lusr (RemoveCookies pw lls ids) = legalHoldLogin :: ( Member GalleyAPIAccess r, Member TinyLog r, - Member UserSubsystem r, Member Events r, Member AuthenticationSubsystem r, Member (Input AuthenticationSubsystemConfig) r, @@ -264,7 +261,6 @@ legalHoldLogin lhl = do ssoLogin :: ( Member TinyLog r, Member AuthenticationSubsystem r, - Member UserSubsystem r, Member Events r, Member (Input AuthenticationSubsystemConfig) r, Member (Concurrency Unsafe) r, diff --git a/services/brig/src/Brig/API/Federation.hs b/services/brig/src/Brig/API/Federation.hs index d109db22531..f235ba08e79 100644 --- a/services/brig/src/Brig/API/Federation.hs +++ b/services/brig/src/Brig/API/Federation.hs @@ -31,7 +31,6 @@ import Brig.Data.Connection qualified as Data import Brig.IO.Intra (notify) import Brig.Options import Brig.User.API.Handle -import Brig.User.Search.SearchIndex qualified as Q import Control.Error.Util import Control.Monad.Trans.Except import Data.Domain @@ -75,6 +74,7 @@ import Wire.GalleyAPIAccess (GalleyAPIAccess) import Wire.MlsKeyPackageSubsystem (MlsKeyPackageSubsystem) import Wire.NotificationSubsystem import Wire.Sem.Concurrency +import Wire.UserSearchStore qualified as UserSearchStore import Wire.UserStore import Wire.UserStore qualified as UserStore import Wire.UserSubsystem (UserSubsystem) @@ -91,7 +91,8 @@ federationSitemap :: Member UserStore r, Member ClientStore r, Member MlsKeyPackageSubsystem r, - Member ClientSubsystem r + Member ClientSubsystem r, + Member UserSearchStore.UserSearchStore r ) => ServerT FederationAPI (Handler r) federationSitemap = @@ -226,7 +227,8 @@ searchUsers :: forall r. ( Member FederationConfigStore r, Member UserSubsystem r, - Member UserStore r + Member UserStore r, + Member UserSearchStore.UserSearchStore r ) => Domain -> SearchRequest -> @@ -255,7 +257,7 @@ searchUsers domain (SearchRequest searchTerm mTeam mOnlyInTeams mbUserTypeFilter fullSearch :: Int -> ExceptT HttpError (AppT r) [Contact] fullSearch n - | n > 0 = lift $ searchResults <$> Q.searchIndex (Q.FederatedSearch mOnlyInTeams mbUserTypeFilter) searchTerm n + | n > 0 = lift . liftSem $ searchResults <$> UserSearchStore.searchUsersFederated mOnlyInTeams searchTerm n mbUserTypeFilter | otherwise = pure [] exactHandleSearch :: Int -> ExceptT HttpError (AppT r) [Contact] diff --git a/services/brig/src/Brig/API/Internal.hs b/services/brig/src/Brig/API/Internal.hs index 86f9f2e9b2d..38e23e1e8c5 100644 --- a/services/brig/src/Brig/API/Internal.hs +++ b/services/brig/src/Brig/API/Internal.hs @@ -38,7 +38,6 @@ import Brig.Options hiding (internalEvents) import Brig.Provider.API qualified as Provider import Brig.Team.API qualified as Team import Brig.User.EJPD qualified -import Brig.User.Search.Index qualified as Search import Control.Error hiding (bool) import Control.Lens (preview, to, _Just) import Control.Lens.Extras (is) @@ -115,7 +114,6 @@ import Wire.FederationConfigStore import Wire.FederationConfigStore qualified as E import Wire.GalleyAPIAccess (GalleyAPIAccess) import Wire.HashPassword (HashPassword) -import Wire.IndexedUserStore (IndexedUserStore, getTeamSize) import Wire.InvitationStore import Wire.MlsKeyPackageSubsystem (MlsKeyPackageSubsystem) import Wire.MlsKeyPackageSubsystem qualified as Mls @@ -135,6 +133,7 @@ import Wire.TeamSubsystem (TeamSubsystem) import Wire.UserGroupSubsystem import Wire.UserKeyStore import Wire.UserPendingActivationStore (UserPendingActivationStore) +import Wire.UserSearchStore qualified as UserSearchStore import Wire.UserStore as UserStore import Wire.UserSubsystem import Wire.UserSubsystem qualified as User @@ -157,6 +156,7 @@ servantSitemap :: Member UserSubsystem r, Member UserGroupSubsystem r, Member TeamSubsystem r, + Member UserSearchStore.UserSearchStore r, Member TeamInvitationSubsystem r, Member UserStore r, Member InvitationStore r, @@ -170,7 +170,6 @@ servantSitemap :: Member PasswordResetCodeStore r, Member PropertySubsystem r, Member (Input (Local ())) r, - Member IndexedUserStore r, Member (Polysemy.Error UserSubsystemError) r, Member HashPassword r, Member (Embed IO) r, @@ -204,7 +203,6 @@ servantSitemap = :<|> clientAPI :<|> authAPI :<|> internalOauthAPI - :<|> internalSearchIndexAPI :<|> federationRemotesAPI :<|> Provider.internalProviderAPI :<|> enterpriseLoginApi @@ -320,7 +318,7 @@ teamsAPI :: Member (Polysemy.Error UserSubsystemError) r, Member Events r, Member (Input (Local ())) r, - Member IndexedUserStore r, + Member UserSearchStore.UserSearchStore r, Member AuthenticationSubsystem r ) => ServerT BrigIRoutes.TeamsAPI (Handler r) @@ -330,7 +328,7 @@ teamsAPI = :<|> Named @"get-invitation-code" (\tid iid -> lift . liftSem $ Team.getInvitationCode tid iid) :<|> Named @"suspend-team" Team.suspendTeam :<|> Named @"unsuspend-team" Team.unsuspendTeam - :<|> Named @"team-size" (lift . liftSem . getTeamSize) + :<|> Named @"team-size" (lift . liftSem . UserSearchStore.getTeamSize) :<|> Named @"create-invitations-via-scim" Team.createInvitationViaScim userAPI :: (Member UserSubsystem r) => ServerT BrigIRoutes.UserAPI (Handler r) @@ -348,7 +346,6 @@ authAPI :: ( Member GalleyAPIAccess r, Member TinyLog r, Member Events r, - Member UserSubsystem r, Member AuthenticationSubsystem r, Member (Input AuthenticationSubsystemConfig) r, Member (Concurrency Unsafe) r, @@ -503,11 +500,6 @@ getVerificationCode uid action = runMaybeT do code <- MaybeT . lift . liftSem $ internalLookupCode key (scopeFromAction action) pure code.codeValue -internalSearchIndexAPI :: forall r. (Member UserSubsystem r) => ServerT BrigIRoutes.ISearchIndexAPI (Handler r) -internalSearchIndexAPI = - Named @"indexRefresh" (NoContent <$ lift (wrapClient Search.refreshIndexes)) - :<|> Named @"update-search-index" (\uid -> lift $ liftSem $ UserSubsystem.internalUpdateSearchIndex uid $> NoContent) - enterpriseLoginApi :: ( Member EnterpriseLoginSubsystem r, Member (Polysemy.Error EnterpriseLoginSubsystemError) r @@ -788,8 +780,7 @@ getPasswordResetCode email = >>= maybe (throwStd (errorToWai @'E.InvalidPasswordResetKey)) pure changeAccountStatusH :: - ( Member UserSubsystem r, - Member Events r, + ( Member Events r, Member (Concurrency Unsafe) r, Member AuthenticationSubsystem r, Member UserStore r @@ -866,8 +857,7 @@ addBlacklist :: (Member BlockListStore r) => EmailAddress -> Handler r NoContent addBlacklist email = lift $ NoContent <$ API.blacklistInsert email updateSSOIdH :: - ( Member UserSubsystem r, - Member Events r, + ( Member Events r, Member UserStore r ) => UserId -> @@ -878,14 +868,12 @@ updateSSOIdH uid ssoid = lift $ do liftSem $ if success then do - UserSubsystem.internalUpdateSearchIndex uid Events.generateUserEvent uid Nothing (UserUpdated ((emptyUserUpdatedData uid) {eupSSOId = Just ssoid})) pure UpdateSSOIdSuccess else pure UpdateSSOIdNotFound deleteSSOIdH :: - ( Member UserSubsystem r, - Member Events r, + ( Member Events r, Member UserStore r ) => UserId -> @@ -894,7 +882,6 @@ deleteSSOIdH uid = lift $ do success <- liftSem $ UserStore.updateSSOId uid Nothing if success then liftSem $ do - UserSubsystem.internalUpdateSearchIndex uid Events.generateUserEvent uid Nothing (UserUpdated ((emptyUserUpdatedData uid) {eupSSOIdRemoved = True})) pure UpdateSSOIdSuccess else pure UpdateSSOIdNotFound diff --git a/services/brig/src/Brig/API/Public.hs b/services/brig/src/Brig/API/Public.hs index 60cc7a53bf6..3cce81aa029 100644 --- a/services/brig/src/Brig/API/Public.hs +++ b/services/brig/src/Brig/API/Public.hs @@ -173,7 +173,6 @@ import Wire.FederationConfigStore (FederationConfigStore) import Wire.GalleyAPIAccess import Wire.GalleyAPIAccess qualified as GalleyAPIAccess import Wire.HashPassword (HashPassword) -import Wire.IndexedUserStore (IndexedUserStore) import Wire.InvitationStore import Wire.JwtTools (JwtTools) import Wire.MlsKeyPackageSubsystem (MlsKeyPackageSubsystem) @@ -197,7 +196,7 @@ import Wire.UserGroupSubsystem (UserGroupSubsystem) import Wire.UserGroupSubsystem qualified as UserGroup import Wire.UserKeyStore import Wire.UserPendingActivationStore (UserPendingActivationStore) -import Wire.UserSearch.Types +import Wire.UserSearchStore (UserSearchStore) import Wire.UserStore (UserStore) import Wire.UserStore qualified as UserStore import Wire.UserSubsystem hiding (checkHandle, checkHandles, requestEmailChange) @@ -389,7 +388,7 @@ servantSitemap :: Member VerificationCodeSubsystem r, Member (Concurrency 'Unsafe) r, Member BlockListStore r, - Member IndexedUserStore r, + Member UserSearchStore r, Member (ConnectionStore InternalPaging) r, Member HashPassword r, Member (Input UserSubsystemConfig) r, @@ -651,7 +650,7 @@ browseTeamHandler :: Maybe Bool -> Handler r (Public.SearchResult Public.TeamContact) browseTeamHandler uid tid mQuery mRoleFilter mTeamUserSearchSortBy mTeamUserSearchSortOrder mMaxResults mPagingState mEmailFilter mSearchable = do - let browseTeamFilters = BrowseTeamFilters tid mQuery mRoleFilter mTeamUserSearchSortBy mTeamUserSearchSortOrder mEmailFilter mSearchable + let browseTeamFilters = Public.BrowseTeamFilters tid mQuery mRoleFilter mTeamUserSearchSortBy mTeamUserSearchSortOrder mEmailFilter mSearchable lift . liftSem $ User.browseTeam uid browseTeamFilters mMaxResults mPagingState setPropertyH :: (Member PropertySubsystem r) => UserId -> ConnId -> Public.PropertyKey -> Public.RawPropertyValue -> Handler r () diff --git a/services/brig/src/Brig/API/User.hs b/services/brig/src/Brig/API/User.hs index d26a22d5fa6..c628543433b 100644 --- a/services/brig/src/Brig/API/User.hs +++ b/services/brig/src/Brig/API/User.hs @@ -250,7 +250,6 @@ createUserSpar new = do for_ new.newUserSparRichInfo $ UserStore.updateRichInfo uid . unRichInfo GalleyAPIAccess.createSelfConv uid - User.internalUpdateSearchIndex uid Events.generateUserEvent uid Nothing (UserCreated u) -- Add to team @@ -323,7 +322,6 @@ upgradePersonalToTeam luid bNewTeam = do liftSem $ GalleyAPIAccess.changeTeamStatus tid Team.Active bNewTeam.bnuCurrency liftSem $ UserStore.updateUserTeam uid tid - liftSem $ User.internalUpdateSearchIndex uid liftSem $ Intra.sendUserEvent uid Nothing (teamUpdated uid tid) initAccountFeatureConfig uid @@ -702,7 +700,6 @@ revokeIdentity key = do changeAccountStatus :: forall r. ( Member (Concurrency 'Unsafe) r, - Member UserSubsystem r, Member Events r, Member AuthenticationSubsystem r, Member UserStore r @@ -720,12 +717,10 @@ changeAccountStatus usrs status = do Sem r () update ev u = do UserStore.updateAccountStatus u status - User.internalUpdateSearchIndex u Events.generateUserEvent u Nothing (ev u) changeSingleAccountStatus :: - ( Member UserSubsystem r, - Member Events r, + ( Member Events r, Member (Concurrency Unsafe) r, Member AuthenticationSubsystem r, Member UserStore r @@ -738,7 +733,6 @@ changeSingleAccountStatus uid status = do ev <- mkUserEvent (NonEmpty.singleton uid) status lift . liftSem $ do UserStore.updateAccountStatus uid status - User.internalUpdateSearchIndex uid Events.generateUserEvent uid Nothing (ev uid) mkUserEvent :: @@ -847,7 +841,6 @@ preverify tgt code = do onActivated :: ( Member TinyLog r, - Member UserSubsystem r, Member Events r, Member UserStore r ) => @@ -857,13 +850,11 @@ onActivated (AccountActivated account) = liftSem $ do let uid = userId account Log.debug $ field "user" (toByteString uid) . field "action" (val "User.onActivated") Log.info $ field "user" (toByteString uid) . msg (val "User activated") - User.internalUpdateSearchIndex uid Events.generateUserEvent uid Nothing $ UserActivated account -- userIdentity is always Just at the time of writing this comment, -- since account has been activated already. pure (uid, userIdentity account, True) onActivated (EmailActivated uid email) = liftSem $ do - User.internalUpdateSearchIndex uid Events.generateUserEvent uid Nothing (emailUpdated uid email) UserStore.deleteEmailUnvalidated uid pure (uid, Just (EmailIdentity email), False) @@ -1180,7 +1171,6 @@ deleteAccount :: Member UserStore r, Member InvitationStore r, Member PropertySubsystem r, - Member UserSubsystem r, Member Events r, Member AuthenticationSubsystem r, Member UserGroupSubsystem r, @@ -1208,7 +1198,6 @@ deleteAccount user = do Intra.rmUser uid (userAssets user) ClientStore.lookupClients uid >>= mapM_ (ClientStore.delete uid . (.clientId)) luid <- embed $ qualifyLocal uid - User.internalUpdateSearchIndex uid Events.generateUserEvent uid Nothing (UserDeleted (tUntagged luid)) embed do -- Note: Connections can only be deleted afterwards, since diff --git a/services/brig/src/Brig/App.hs b/services/brig/src/Brig/App.hs index 6c2145ea8cd..c7ee953ec30 100644 --- a/services/brig/src/Brig/App.hs +++ b/services/brig/src/Brig/App.hs @@ -27,7 +27,6 @@ module Brig.App -- * App Environment Env (..), - mkIndexEnv, newEnv, closeEnv, providerTemplatesWithLocale, @@ -66,7 +65,6 @@ module Brig.App zauthEnvLens, digestSHA256Lens, digestMD5Lens, - indexEnvLens, randomPrekeyLocalLockLens, keyPackageLocalLockLens, rabbitmqChannelLens, @@ -108,7 +106,7 @@ import Bilge.RPC (HasRequestId (..)) import Brig.AWS qualified as AWS import Brig.Calling qualified as Calling import Brig.DeleteQueue.Interpreter -import Brig.Options (ElasticSearchOpts, Opts, Settings (..)) +import Brig.Options (Opts, Settings (..)) import Brig.Options qualified as Opt import Brig.Provider.Template import Brig.Queue.Stomp qualified as Stomp @@ -116,7 +114,6 @@ import Brig.Queue.Types import Brig.Schema.Run qualified as Migrations import Brig.Team.Template import Brig.Template (InvitationUrlTemplates (..), genTemplateBranding, genTemplateBrandingMap) -import Brig.User.Search.Index (IndexEnv (..), MonadIndexIO (..), runIndexIO) import Brig.User.Template import Cassandra (runClient) import Cassandra qualified as Cas @@ -129,7 +126,6 @@ import Control.Monad.Catch import Control.Monad.Trans.Resource import Data.ByteString qualified as BS import Data.ByteString.Conversion (fromByteString) -import Data.Credentials (Credentials (..)) import Data.Domain import Data.Id import Data.Misc @@ -140,7 +136,6 @@ import Data.Text.Encoding (encodeUtf8) import Data.Text.Encoding qualified as Text import Data.Text.IO qualified as Text import Data.Time.Clock -import Database.Bloodhound qualified as ES import HTTP2.Client.Manager (Http2Manager, http2ManagerWithSSLCtx) import Hasql.Pool.Extended (initPostgresPool) import Hasql.Pool.Extended qualified as HasqlPool @@ -219,7 +214,6 @@ data Env = Env zauthEnv :: ZAuthEnv, digestSHA256 :: Digest, digestMD5 :: Digest, - indexEnv :: IndexEnv, randomPrekeyLocalLock :: Maybe (MVar ()), keyPackageLocalLock :: MVar (), rabbitmqChannel :: MVar Q.Channel, @@ -297,7 +291,6 @@ newEnv opts = do kpLock <- newMVar () rabbitChan <- Q.mkRabbitMqChannelMVar lgr (Just "brig") opts.rabbitmq let allDisabledVersions = foldMap expandVersionExp opts.settings.disabledAPIVersions - idxEnv <- mkIndexEnv opts.elasticsearch lgr (Opt.galley opts) mgr rateLimitEnv <- newRateLimitEnv opts.settings.passwordHashingRateLimit hasqlPool <- initPostgresPool opts.postgresqlPool opts.postgresql opts.postgresqlPassword amqpJobsPublisherChannel <- Q.mkRabbitMqChannelMVar lgr (Just "brig") opts.rabbitmq @@ -337,7 +330,6 @@ newEnv opts = do zauthEnv = zau, digestMD5 = md5, digestSHA256 = sha256, - indexEnv = idxEnv, randomPrekeyLocalLock = prekeyLocalLock, keyPackageLocalLock = kpLock, rabbitmqChannel = rabbitChan, @@ -360,33 +352,6 @@ newEnv opts = do pure (Nothing, Just smtp) mkEndpoint service = RPC.host (encodeUtf8 service.host) . RPC.port service.port $ RPC.empty -mkIndexEnv :: ElasticSearchOpts -> Logger -> Endpoint -> Manager -> IO IndexEnv -mkIndexEnv esOpts logger galleyEp rpcHttpManager = do - mEsCreds :: Maybe Credentials <- for esOpts.credentials initCredentials - mEsAddCreds :: Maybe Credentials <- for esOpts.additionalCredentials initCredentials - - let mkBhEnv skipVerifyTls mCustomCa mCreds url = do - mgr <- initHttpManagerWithTLSConfig skipVerifyTls mCustomCa - let bhe = ES.mkBHEnv url mgr - pure $ maybe bhe (\creds -> bhe {ES.bhRequestHook = ES.basicAuthHook (ES.EsUsername creds.username) (ES.EsPassword creds.password)}) mCreds - esLogger = Log.clone (Just "index.brig") logger - bhEnv <- mkBhEnv esOpts.insecureSkipVerifyTls esOpts.caCert mEsCreds esOpts.url - additionalBhEnv <- - for esOpts.additionalWriteIndexUrl $ - mkBhEnv esOpts.additionalInsecureSkipVerifyTls esOpts.additionalCaCert mEsAddCreds - pure $ - IndexEnv - { idxLogger = esLogger, - idxElastic = bhEnv, - idxRequest = Nothing, - idxName = esOpts.index, - idxAdditionalName = esOpts.additionalWriteIndex, - idxAdditionalElastic = additionalBhEnv, - idxGalley = galleyEp, - idxRpcHttpManager = rpcHttpManager, - idxCredentials = mEsCreds - } - initZAuth :: Opts -> IO ZAuthEnv initZAuth o = do let zOpts = Opt.zauth o @@ -611,8 +576,7 @@ newtype HttpClientIO a = HttpClientIO MonadThrow, MonadCatch, MonadMask, - MonadUnliftIO, - MonadIndexIO + MonadUnliftIO ) runHttpClientIO :: (MonadIO m) => Env -> HttpClientIO a -> m a @@ -642,16 +606,6 @@ wrapHttpClient = wrapHttp wrapHttpClientE :: ExceptT e HttpClientIO a -> ExceptT e (AppT r) a wrapHttpClientE = mapExceptT wrapHttpClient -instance (MonadIO m) => MonadIndexIO (ReaderT Env m) where - liftIndexIO m = asks (.indexEnv) >>= \e -> runIndexIO e m - -instance MonadIndexIO (AppT r) where - liftIndexIO m = do - AppT $ mapReaderT (embedToFinal @IO) $ liftIndexIO m - -instance (MonadIndexIO (AppT r)) => MonadIndexIO (ExceptT err (AppT r)) where - liftIndexIO m = asks (.indexEnv) >>= \e -> runIndexIO e m - instance HasRequestId (AppT r) where getRequestId = asks (.requestId) diff --git a/services/brig/src/Brig/CanonicalInterpreter.hs b/services/brig/src/Brig/CanonicalInterpreter.hs index d44c91648d3..6eeae8b7d6d 100644 --- a/services/brig/src/Brig/CanonicalInterpreter.hs +++ b/services/brig/src/Brig/CanonicalInterpreter.hs @@ -26,7 +26,6 @@ import Brig.IO.Intra (runEvents) import Brig.Options (Settings (consumableNotifications), federationDomainConfigs, federationStrategy) import Brig.Options qualified as Opt import Brig.Template (InvitationUrlTemplates) -import Brig.User.Search.Index (IndexEnv (..)) import Cassandra qualified as Cas import Control.Exception (ErrorCall) import Control.Lens (to, (^.), _Just) @@ -105,8 +104,6 @@ import Wire.GalleyAPIAccess.Rpc import Wire.GundeckAPIAccess import Wire.HashPassword import Wire.HashPassword.Interpreter -import Wire.IndexedUserStore -import Wire.IndexedUserStore.ElasticSearch import Wire.InvitationStore (InvitationStore) import Wire.InvitationStore.Cassandra (interpretInvitationStoreToCassandra) import Wire.JwtTools @@ -166,6 +163,8 @@ import Wire.UserKeyStore import Wire.UserKeyStore.Cassandra import Wire.UserPendingActivationStore (UserPendingActivationStore) import Wire.UserPendingActivationStore.Cassandra (userPendingActivationStoreToCassandra) +import Wire.UserSearchStore (UserSearchStore) +import Wire.UserSearchStore.Postgres (interpretUserSearchStorePostgres) import Wire.UserStore import Wire.UserStore.Cassandra import Wire.UserStore.Postgres (interpretUserStorePostgres) @@ -217,6 +216,7 @@ type BrigLowerLevelEffects = MlsKeyPackageStore, UserStore, UserGroupStore, + UserSearchStore, DomainRegistrationStore, DomainVerificationChallengeStore, Error AppSubsystemError, @@ -239,7 +239,6 @@ type BrigLowerLevelEffects = CryptoSign, HashPassword, ClientStore, - IndexedUserStore, SessionStore, PasswordStore, VerificationCodeStore, @@ -364,21 +363,6 @@ runBrigToIO e (AppT ma) = do userCookieLimit = e.settings.userCookieLimit, userCookieThrottle = e.settings.userCookieThrottle } - mainESEnv = e.indexEnv ^. to idxElastic - indexedUserStoreConfig = - IndexedUserStoreConfig - { conn = - ESConn - { env = mainESEnv, - indexName = e.indexEnv ^. to idxName - }, - additionalConn = - (e.indexEnv ^. to idxAdditionalName) <&> \additionalIndexName -> - ESConn - { env = e.indexEnv ^. to idxAdditionalElastic . to (fromMaybe mainESEnv), - indexName = additionalIndexName - } - } clientStoreCassandraEnv = ClientStoreCassandraEnv { prekeyLocking = @@ -469,7 +453,6 @@ runBrigToIO e (AppT ma) = do . interpretVerificationCodeStoreCassandra e.casClient . interpretPasswordStore e.casClient . interpretSessionStoreCassandra e.casClient - . interpretIndexedUserStoreES indexedUserStoreConfig . interpretClientStoreCassandra clientStoreCassandraEnv . runHashPassword e.settings.passwordHashingOptions . runCryptoSign @@ -492,6 +475,18 @@ runBrigToIO e (AppT ma) = do . mapError appSubsystemErrorToHttpError . domainVerificationChallengeStore . domainRegistrationStore + -- User search reads brig's Postgres user store directly; a + -- Cassandra-only user store does not support user search. + . ( case e.postgresMigration.user of + CassandraStorage -> + error $ + "UserSearchStore requires the brig user store in" + <> " Postgres: set postgresMigration.user to" + <> " PostgresqlStorage or MigrationToPostgresql" + <> " (user search is unsupported with a" + <> " Cassandra-only user store)." + _ -> interpretUserSearchStorePostgres + ) . interpretUserGroupStoreToPostgres . userStoreInterpreter . interpretMlsKeyPackageStoreToCassandra e.casClient diff --git a/services/brig/src/Brig/Index/Eval.hs b/services/brig/src/Brig/Index/Eval.hs deleted file mode 100644 index f931a032769..00000000000 --- a/services/brig/src/Brig/Index/Eval.hs +++ /dev/null @@ -1,271 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Brig.Index.Eval - ( runCommand, - initIndex, - ) -where - -import Brig.App (initHttpManagerWithTLSConfig, mkIndexEnv) -import Brig.Index.Options as IxOpts -import Brig.Options as Opt -import Brig.User.Search.Index -import Cassandra (ClientState) -import Cassandra.Options -import Cassandra.Util (defInitCassandra) -import Control.Exception (throwIO) -import Control.Lens -import Control.Monad.Catch -import Control.Retry -import Data.Aeson (FromJSON) -import Data.Aeson qualified as Aeson -import Data.ByteString.Lazy.UTF8 qualified as UTF8 -import Data.Credentials (Credentials (..)) -import Data.Id -import Database.Bloodhound qualified as ES -import Database.Bloodhound.Internal.Client (BHEnv (..)) -import Hasql.Pool (UsageError) -import Hasql.Pool.Extended -import Hasql.Pool.Extended qualified as Hasql -import Imports -import Network.HTTP.Client (Manager) -import Polysemy -import Polysemy.Async (Async, asyncToIOFinal) -import Polysemy.Conc (Race, interpretRace) -import Polysemy.Error -import Polysemy.Input -import Polysemy.Resource (Resource, runResource) -import Polysemy.TinyLog (TinyLog) -import System.Logger qualified as Log -import System.Logger.Class (Logger) -import Util.Options -import Wire.ClientSubsystem.Error (ClientError) -import Wire.GalleyAPIAccess (GalleyAPIAccess) -import Wire.GalleyAPIAccess.Rpc -import Wire.IndexedUserStore -import Wire.IndexedUserStore.Bulk.ElasticSearch qualified as IndexedUserStoreBulk -import Wire.IndexedUserStore.ElasticSearch -import Wire.IndexedUserStore.MigrationStore (IndexedUserMigrationStore) -import Wire.IndexedUserStore.MigrationStore.ElasticSearch -import Wire.MigrationLock -import Wire.ParseException -import Wire.PostgresMigrationOpts -import Wire.Rpc -import Wire.Sem.Logger.TinyLog -import Wire.Sem.Metrics (Metrics) -import Wire.Sem.Metrics.IO -import Wire.UserKeyStore (UserKeyStore) -import Wire.UserKeyStore.Cassandra -import Wire.UserSearch.Migration (MigrationException) -import Wire.UserStore (UserStore) -import Wire.UserStore.Cassandra -import Wire.UserStore.Postgres (interpretUserStorePostgres) - -type BrigIndexEffectStack = - [ UserKeyStore, - UserStore, - IndexedUserStore, - Error IndexedUserStoreError, - IndexedUserMigrationStore, - Error MigrationException, - Error MigrationLockError, - GalleyAPIAccess, - Error ParseException, - Rpc, - Metrics, - TinyLog, - Input Hasql.Pool, - Error UsageError, - Error ClientError, - Resource, - Race, - Async, - Embed IO, - Final IO - ] - -type SemDeps = (Manager, ClientState, Hasql.Pool, BHEnv, IndexedUserStoreConfig, RequestId, IndexName) - -newtype PostgresUsageException = PostgresUsageException UsageError - deriving (Show) - -instance Exception PostgresUsageException - -mkSemDeps :: ESConnectionSettings -> CassandraSettings -> PostgresSettings -> Logger -> IO SemDeps -mkSemDeps esConn cas pg logger = do - mgr <- initHttpManagerWithTLSConfig esConn.esInsecureSkipVerifyTls esConn.esCaCert - mEsCreds :: Maybe Credentials <- for esConn.esCredentials initCredentials - casClient <- defInitCassandra (toCassandraOpts cas) logger - pgPool <- initPostgresPool pg.pool pg.settings pg.passwordFile - let bhEnv = - BHEnv - { bhServer = toESServer esConn.esServer, - bhManager = mgr, - bhRequestHook = maybe pure (\creds -> ES.basicAuthHook (ES.EsUsername creds.username) (ES.EsPassword creds.password)) mEsCreds - } - indexedUserStoreConfig = - IndexedUserStoreConfig - { conn = - ESConn - { indexName = esConn.esIndex, - env = bhEnv - }, - additionalConn = Nothing - } - reqId = (RequestId "brig-index") - migrationIndexName = fromMaybe defaultMigrationIndexName (esMigrationIndexName esConn) - pure (mgr, casClient, pgPool, bhEnv, indexedUserStoreConfig, reqId, migrationIndexName) - -runSem :: SemDeps -> UserStorageLocation -> Endpoint -> Logger -> Sem BrigIndexEffectStack a -> IO a -runSem (mgr, casClient, pgPool, bhEnv, indexedUserStoreConfig, reqId, migrationIndexName) userStorage galleyEndpoint logger action = do - let userStoreInterpreter = case userStorage.userStorageLocation of - CassandraStorage -> interpretUserStoreCassandra casClient - MigrationToPostgresql -> interpretUserStoreToCassandraAndPostgres casClient - PostgresqlStorage -> interpretUserStorePostgres - runFinal - . embedToFinal - . asyncToIOFinal - . interpretRace - . runResource - . throwErrorToIOFinal @ClientError - . throwPostgresUsageErrorToIOFinal - . runInputConst pgPool - . loggerToTinyLogReqId reqId logger - . ignoreMetrics - . runRpcWithHttp mgr reqId - . throwErrorToIOFinal @ParseException - . interpretGalleyAPIAccessToRpc mempty galleyEndpoint - . throwErrorToIOFinal @MigrationLockError - . throwErrorToIOFinal @MigrationException - . interpretIndexedUserMigrationStoreES bhEnv migrationIndexName - . throwErrorToIOFinal @IndexedUserStoreError - . interpretIndexedUserStoreES indexedUserStoreConfig - . userStoreInterpreter - . interpretUserKeyStoreCassandra casClient - $ action - -throwErrorToIOFinal :: (Exception e, Member (Final IO) r) => InterpreterFor (Error e) r -throwErrorToIOFinal action = do - runError action >>= \case - Left e -> embedFinal $ throwIO e - Right a -> pure a - -throwPostgresUsageErrorToIOFinal :: (Member (Final IO) r) => InterpreterFor (Error UsageError) r -throwPostgresUsageErrorToIOFinal action = do - runError action >>= \case - Left e -> embedFinal $ throwIO (PostgresUsageException e) - Right a -> pure a - -runCommand :: Logger -> Command -> IO () -runCommand l = \case - Create es galley -> do - e <- initIndex l (es ^. esConnection) galley - runIndexIO e $ createIndexIfNotPresent (mkCreateIndexSettings es) - Reset es galley -> do - e <- initIndex l (es ^. esConnection) galley - runIndexIO e $ resetIndex (mkCreateIndexSettings es) - Reindex es cas pg userStorageLocation galley pageSize -> do - semDeps <- mkSemDeps (es ^. esConnection) cas pg l - IndexedUserStoreBulk.syncAllUsers (runSem semDeps userStorageLocation galley l) pageSize - ReindexSameOrNewer es cas pg userStorageLocation galley pageSize -> do - semDeps <- mkSemDeps (es ^. esConnection) cas pg l - IndexedUserStoreBulk.forceSyncAllUsers (runSem semDeps userStorageLocation galley l) pageSize - UpdateMapping esConn galley -> do - e <- initIndex l esConn galley - runIndexIO e updateMapping - Migrate es cas pg userStorageLocation galley pageSize -> do - semDeps <- mkSemDeps (es ^. esConnection) cas pg l - IndexedUserStoreBulk.migrateData (runSem semDeps userStorageLocation galley l) pageSize - ReindexFromAnotherIndex reindexSettings -> do - mgr <- - initHttpManagerWithTLSConfig - (reindexSettings ^. reindexEsConnection . to esInsecureSkipVerifyTls) - (reindexSettings ^. reindexEsConnection . to esCaCert) - mCreds <- for (reindexSettings ^. reindexEsConnection . to esCredentials) initCredentials - let bhEnv = initES (reindexSettings ^. reindexEsConnection . to esServer) mgr mCreds - ES.runBH bhEnv $ do - let src = reindexSettings ^. reindexEsConnection . to esIndex - dest = view reindexDestIndex reindexSettings - timeoutSeconds = view reindexTimeoutSeconds reindexSettings - - srcExists <- ES.indexExists src - unless srcExists $ do - throwM $ ReindexFromAnotherIndexError $ "Source index " <> show src <> " doesn't exist" - - destExists <- ES.indexExists dest - unless destExists $ do - throwM $ ReindexFromAnotherIndexError $ "Destination index " <> show dest <> " doesn't exist" - - Log.info l $ Log.msg ("Reindexing" :: ByteString) . Log.field "from" (show src) . Log.field "to" (show dest) - eitherTaskNodeId <- ES.reindexAsync $ ES.mkReindexRequest src dest - case eitherTaskNodeId of - Left e -> throwM $ ReindexFromAnotherIndexError $ "Error occurred while running reindex: " <> show e - Right taskNodeId -> do - Log.info l $ Log.field "taskNodeId" (show taskNodeId) - waitForTaskToComplete @ES.ReindexResponse timeoutSeconds taskNodeId - Log.info l $ Log.msg ("Finished reindexing" :: ByteString) - where - initES esURI mgr mCreds = - let env = ES.mkBHEnv (toESServer esURI) mgr - in maybe env (\(creds :: Credentials) -> env {ES.bhRequestHook = ES.basicAuthHook (ES.EsUsername creds.username) (ES.EsPassword creds.password)}) mCreds - -initIndex :: Logger -> ESConnectionSettings -> Endpoint -> IO IndexEnv -initIndex l esConn gly = do - mgr <- initHttpManagerWithTLSConfig esConn.esInsecureSkipVerifyTls esConn.esCaCert - let esOpts = - ElasticSearchOpts - { url = toESServer esConn.esServer, - index = esConn.esIndex, - credentials = esConn.esCredentials, - insecureSkipVerifyTls = esConn.esInsecureSkipVerifyTls, - caCert = esConn.esCaCert, - additionalWriteIndex = Nothing, - additionalWriteIndexUrl = Nothing, - additionalCredentials = Nothing, - additionalInsecureSkipVerifyTls = False, - additionalCaCert = Nothing - } - - mkIndexEnv esOpts l gly mgr - -waitForTaskToComplete :: forall a m. (ES.MonadBH m, MonadThrow m, FromJSON a) => Int -> ES.TaskNodeId -> m () -waitForTaskToComplete timeoutSeconds taskNodeId = do - -- Delay is 0.1 seconds, so retries are limited to timeoutSeconds * 10 - let policy = constantDelay 100000 <> limitRetries (timeoutSeconds * 10) - let retryCondition _ = fmap not . isTaskComplete - taskEither <- retrying policy retryCondition (const $ ES.getTask @m @a taskNodeId) - task <- either errTaskGet pure taskEither - unless (ES.taskResponseCompleted task) $ do - throwM $ ReindexFromAnotherIndexError $ "Timed out waiting for task: " <> show taskNodeId - when (isJust $ ES.taskResponseError task) $ do - throwM $ - ReindexFromAnotherIndexError $ - "Task failed with error: " - <> UTF8.toString (Aeson.encode $ ES.taskResponseError task) - where - isTaskComplete :: Either ES.EsError (ES.TaskResponse a) -> m Bool - isTaskComplete (Left e) = throwM $ ReindexFromAnotherIndexError $ "Error response while getting task: " <> show e - isTaskComplete (Right taskRes) = pure $ ES.taskResponseCompleted taskRes - - errTaskGet :: ES.EsError -> m x - errTaskGet e = throwM $ ReindexFromAnotherIndexError $ "Error response while getting task: " <> show e - -newtype ReindexFromAnotherIndexError = ReindexFromAnotherIndexError String - deriving (Show) - -instance Exception ReindexFromAnotherIndexError diff --git a/services/brig/src/Brig/Index/Options.hs b/services/brig/src/Brig/Index/Options.hs deleted file mode 100644 index b19c1f3124f..00000000000 --- a/services/brig/src/Brig/Index/Options.hs +++ /dev/null @@ -1,548 +0,0 @@ -{-# LANGUAGE ApplicativeDo #-} -{-# LANGUAGE StrictData #-} -{-# LANGUAGE TemplateHaskell #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Brig.Index.Options - ( Command (..), - ElasticSettings, - ESConnectionSettings (..), - esConnection, - esIndexShardCount, - esIndexReplicas, - esIndexRefreshInterval, - esDeleteTemplate, - CassandraSettings, - toCassandraOpts, - cHost, - cPort, - cTlsCa, - cKeyspace, - PostgresSettings (..), - UserStorageLocation (..), - localElasticSettings, - brigOptsToPostgresSettings, - localCassandraSettings, - commandParser, - mkCreateIndexSettings, - toESServer, - ReindexFromAnotherIndexSettings (..), - reindexDestIndex, - reindexTimeoutSeconds, - reindexEsConnection, - ) -where - -import Brig.Index.Types (CreateIndexSettings (..)) -import Brig.Options qualified as Opts -import Cassandra qualified as C -import Control.Lens -import Data.Aeson as Aeson -import Data.Aeson.Key qualified as AKey -import Data.Aeson.KeyMap qualified as AKM -import Data.Aeson.Text qualified as Aeson -import Data.ByteString.Lens -import Data.Map qualified as Map -import Data.Misc -import Data.Text qualified as Text -import Data.Text.Encoding (encodeUtf8) -import Data.Text.Lazy qualified as LText -import Data.Text.Strict.Lens -import Data.Time (NominalDiffTime) -import Database.Bloodhound qualified as ES -import Hasql.Pool.Extended -import Imports -import Options.Applicative -import URI.ByteString -import URI.ByteString.QQ -import Util.Options (CassandraOpts (..), Endpoint (..), FilePathSecrets) -import Wire.PostgresMigrationOpts - -data Command - = Create ElasticSettings Endpoint - | Reset ElasticSettings Endpoint - | Reindex ElasticSettings CassandraSettings PostgresSettings UserStorageLocation Endpoint Int32 - | ReindexSameOrNewer ElasticSettings CassandraSettings PostgresSettings UserStorageLocation Endpoint Int32 - | -- | 'ElasticSettings' has shards and other settings that are not needed here. - UpdateMapping ESConnectionSettings Endpoint - | Migrate ElasticSettings CassandraSettings PostgresSettings UserStorageLocation Endpoint Int32 - | ReindexFromAnotherIndex ReindexFromAnotherIndexSettings - deriving (Show) - -data ESConnectionSettings = ESConnectionSettings - { esServer :: URIRef Absolute, - esIndex :: ES.IndexName, - esCaCert :: Maybe FilePath, - esInsecureSkipVerifyTls :: Bool, - esCredentials :: Maybe FilePathSecrets, - esMigrationIndexName :: Maybe ES.IndexName - } - deriving (Show) - -data ElasticSettings = ElasticSettings - { _esConnection :: ESConnectionSettings, - _esIndexShardCount :: Int, - _esIndexReplicas :: ES.ReplicaCount, - _esIndexRefreshInterval :: NominalDiffTime, - _esDeleteTemplate :: Maybe ES.TemplateName - } - deriving (Show) - -data PostgresSettings = PostgresSettings - { pool :: !PoolConfig, - passwordFile :: !(Maybe FilePathSecrets), - -- | Postgresql settings, the key values must be in libpq format. - -- https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS - settings :: !(Map Text Text) - } - deriving (Show) - -data CassandraSettings = CassandraSettings - { _cHost :: String, - _cPort :: Word16, - _cKeyspace :: C.Keyspace, - _cTlsCa :: Maybe FilePath - } - deriving (Show) - -data ReindexFromAnotherIndexSettings = ReindexFromAnotherIndexSettings - { _reindexEsConnection :: ESConnectionSettings, - _reindexDestIndex :: ES.IndexName, - _reindexTimeoutSeconds :: Int - } - deriving (Show) - -newtype UserStorageLocation = UserStorageLocation {userStorageLocation :: StorageLocation} - deriving (Show) - -makeLenses ''ElasticSettings - -makeLenses ''CassandraSettings - -makeLenses ''ReindexFromAnotherIndexSettings - -toCassandraOpts :: CassandraSettings -> CassandraOpts -toCassandraOpts cas = - CassandraOpts - { endpoint = Endpoint (Text.pack (cas ^. cHost)) (cas ^. cPort), - keyspace = C.unKeyspace (cas ^. cKeyspace), - filterNodesByDatacentre = Nothing, - tlsCa = cas ^. cTlsCa - } - -mkCreateIndexSettings :: ElasticSettings -> CreateIndexSettings -mkCreateIndexSettings es = - CreateIndexSettings - [ ES.NumberOfReplicas $ _esIndexReplicas es, - ES.RefreshInterval $ _esIndexRefreshInterval es - ] - (_esIndexShardCount es) - (_esDeleteTemplate es) - -localElasticSettings :: ElasticSettings -localElasticSettings = - ElasticSettings - { _esConnection = - ESConnectionSettings - { esServer = [uri|https://localhost:9200|], - esIndex = ES.IndexName "directory_test", - esCaCert = Just "../../libs/wire-subsystems/test/resources/elasticsearch-ca.pem", - esInsecureSkipVerifyTls = False, - esCredentials = Just "../../libs/wire-subsystems/test/resources/elasticsearch-credentials.yaml", - esMigrationIndexName = Nothing - }, - _esIndexShardCount = 1, - _esIndexReplicas = ES.ReplicaCount 1, - _esIndexRefreshInterval = 1, - _esDeleteTemplate = Nothing - } - -brigOptsToPostgresSettings :: Opts.Opts -> PostgresSettings -brigOptsToPostgresSettings opts = - PostgresSettings - { pool = opts.postgresqlPool, - passwordFile = opts.postgresqlPassword, - settings = opts.postgresql - } - -localCassandraSettings :: CassandraSettings -localCassandraSettings = - CassandraSettings - { _cHost = "localhost", - _cPort = 9042, - _cKeyspace = C.Keyspace "brig_test", - _cTlsCa = Nothing - } - -elasticServerParser :: Parser (URIRef Absolute) -elasticServerParser = - option - url - ( long "elasticsearch-server" - <> metavar "URL" - <> help "Base URL of the Elasticsearch Server." - <> value localElasticSettings._esConnection.esServer - <> showDefaultWith (view unpackedChars . serializeURIRef') - ) - where - url = - eitherReader - (over _Left show . parseURI strictURIParserOptions . view packedChars) - -restrictedElasticSettingsParser :: Parser ElasticSettings -restrictedElasticSettingsParser = do - server <- elasticServerParser - prefix <- - strOption - ( long "elasticsearch-index-prefix" - <> metavar "PREFIX" - <> help "Elasticsearch Index Prefix. The actual index name will be PREFIX_test." - <> value "directory" - <> showDefault - ) - mCreds <- credentialsPathParser - mCaCert <- caCertParser - verifyCa <- verifyCaParser - pure $ - localElasticSettings - { _esConnection = - localElasticSettings._esConnection - { esServer = server, - esIndex = ES.IndexName (prefix <> "_test"), - esCredentials = mCreds, - esCaCert = mCaCert, - esInsecureSkipVerifyTls = verifyCa - } - } - -indexNameParser :: Parser ES.IndexName -indexNameParser = - ES.IndexName . view packed - <$> strOption - ( long "elasticsearch-index" - <> metavar "STRING" - <> help "Elasticsearch Index Name." - <> value (view (_IndexName . unpacked) localElasticSettings._esConnection.esIndex) - <> showDefault - ) - -connectionSettingsParser :: Parser ESConnectionSettings -connectionSettingsParser = - ESConnectionSettings - <$> elasticServerParser - <*> indexNameParser - <*> caCertParser - <*> verifyCaParser - <*> credentialsPathParser - <*> pure Nothing - -caCertParser :: Parser (Maybe FilePath) -caCertParser = - optional - ( option - str - ( long "elasticsearch-ca-cert" - <> metavar "FILE" - <> help "Path to CA Certitificate for TLS validation, system CA bundle is used when unspecified" - ) - ) - -verifyCaParser :: Parser Bool -verifyCaParser = - flag - False -- the default is False - True - ( long "elasticsearch-insecure-skip-tls-verify" - <> help "Skip TLS verification when connecting to Elasticsearch (not recommended)" - ) - -elasticSettingsParser :: Parser ElasticSettings -elasticSettingsParser = - ElasticSettings - <$> connectionSettingsParser - <*> indexShardCountParser - <*> indexReplicaCountParser - <*> indexRefreshIntervalParser - <*> templateParser - where - indexShardCountParser = - option - auto - ( long "elasticsearch-shards" - <> metavar "INT" - <> help "Number of Shards for the Elasticsearch Index." - <> value 1 - <> showDefault - ) - indexReplicaCountParser = - ES.ReplicaCount - <$> option - auto - ( long "elasticsearch-replicas" - <> metavar "INT" - <> help "Number of Replicas for the Elasticsearch Index." - <> value 1 - <> showDefault - ) - indexRefreshIntervalParser = - fromInteger - <$> option - auto - ( long "elasticsearch-refresh-interval" - <> metavar "SECONDS" - <> help "Refresh interval for the Elasticsearch Index in seconds" - <> value 1 - <> showDefault - ) - templateParser :: Parser (Maybe ES.TemplateName) = - ES.TemplateName - <$$> optional - ( option - str - ( long "delete-template" - <> metavar "TEMPLATE_NAME" - <> help "Delete this ES template before creating a new index" - ) - ) - -credentialsPathParser :: Parser (Maybe FilePathSecrets) -credentialsPathParser = - optional - ( strOption - ( long "elasticsearch-credentials" - <> metavar "FILE" - <> help "Location of a file containing the Elasticsearch credentials" - ) - ) - -postgresSettingsParser :: Parser PostgresSettings -postgresSettingsParser = - PostgresSettings - <$> poolConfigParser - <*> optional - ( strOption - ( long "pg-password-file" - <> metavar "FILE" - <> help "File containing PostgreSQL password" - ) - ) - <*> option - (eitherReader parseJsonMap) - ( long "pg-settings" - <> metavar "JSON" - <> help "PostgreSQL connection parameters as JSON object" - <> value Map.empty - ) - -poolConfigParser :: Parser PoolConfig -poolConfigParser = - PoolConfig - <$> option - auto - ( long "pg-pool-size" - <> metavar "INT" - <> help "Connection pool size" - <> value 10 - ) - <*> option - (eitherReader (parseDuration . Text.pack)) - ( long "pg-pool-acquisition-timeout" - <> metavar "Duration" - <> help "Pool acquisition timeout in seconds" - <> value (unsafeParseDuration "10s") - ) - <*> option - (eitherReader (parseDuration . Text.pack)) - ( long "pg-pool-idleness-timeout" - <> metavar "Duration" - <> help "Pool idleness timeout in seconds" - <> value (unsafeParseDuration "10m") - ) - -parseJsonMap :: String -> Either String (Map Text Text) -parseJsonMap s = do - Aeson.eitherDecodeStrict' (encodeUtf8 (Text.pack s)) >>= \case - Object hmap -> pure $ Map.fromList $ bimap AKey.toText valueToText <$> AKM.toList hmap - bad -> Left $ "invalid json object: " <> show bad - where - valueToText :: Value -> Text - valueToText (String t) = t - valueToText (Bool b) = (if b then "true" else "false") - valueToText (Number n) = (Text.pack (show n)) - valueToText Null = "null" - valueToText v = LText.toStrict (Aeson.encodeToLazyText v) - -cassandraSettingsParser :: Parser CassandraSettings -cassandraSettingsParser = - CassandraSettings - <$> strOption - ( long "cassandra-host" - <> metavar "HOST" - <> help "Cassandra Host." - <> value (_cHost localCassandraSettings) - <> showDefault - ) - <*> option - auto - ( long "cassandra-port" - <> metavar "PORT" - <> help "Cassandra Port." - <> value (_cPort localCassandraSettings) - <> showDefault - ) - <*> ( C.Keyspace . view packed - <$> strOption - ( long "cassandra-keyspace" - <> metavar "STRING" - <> help "Cassandra Keyspace." - <> value (view (cKeyspace . _Keyspace . unpacked) localCassandraSettings) - <> showDefault - ) - ) - <*> ( (optional . strOption) - ( long "cassandra-ca-cert" - <> metavar "FILE" - <> help "Location of a PEM encoded list of CA certificates to be used when verifying the Cassandra server's certificate" - ) - ) - -reindexToAnotherIndexSettingsParser :: Parser ReindexFromAnotherIndexSettings -reindexToAnotherIndexSettingsParser = - ReindexFromAnotherIndexSettings - <$> connectionSettingsParser - <*> ( ES.IndexName . view packed - <$> strOption - ( long "destination-index" - <> metavar "STRING" - <> help "Elasticsearch index name to reindex to" - ) - ) - <*> option - auto - ( long "timeout" - <> metavar "SECONDS" - <> help "Number of seconds to wait for reindexing to complete. The reindexing will not be cancelled when this timeout expires." - <> value 600 - <> showDefault - ) - -userStorageLocationParser :: Parser UserStorageLocation -userStorageLocationParser = - UserStorageLocation - <$> option - (eitherReader parseStorageLocation) - ( long "user-storage-location" - <> help "Storage location of user, valid options: cassandra, postgersql, migration-to-postgresql" - <> value CassandraStorage - <> showDefaultWith storageLocationString - ) - -galleyEndpointParser :: Parser Endpoint -galleyEndpointParser = - Endpoint - <$> strOption - ( long "galley-host" - <> help "Hostname or IP address of galley" - <> metavar "HOSTNAME" - <> value "localhost" - <> showDefault - ) - <*> option - auto - ( long "galley-port" - <> help "Port number of galley" - <> metavar "PORT" - <> value 8085 - <> showDefault - ) - -pageSizeParser :: Parser Int32 -pageSizeParser = - option - auto - ( long "page-size" - <> help "Page size for reading users" - <> metavar "PAGE_SIZE" - <> value 10000 - <> showDefault - ) - -commandParser :: Parser Command -commandParser = - hsubparser - ( command - "create" - ( info - (Create <$> elasticSettingsParser <*> galleyEndpointParser) - (progDesc "Create the ES user index, if it doesn't already exist. ") - ) - <> command - "update-mapping" - ( info - (UpdateMapping <$> connectionSettingsParser <*> galleyEndpointParser) - (progDesc "Update mapping of the user index.") - ) - <> command - "reset" - ( info - (Reset <$> restrictedElasticSettingsParser <*> galleyEndpointParser) - (progDesc "Delete and re-create the ES user index. Only works on a test index (directory_test).") - ) - <> command - "reindex" - ( info - (Reindex <$> elasticSettingsParser <*> cassandraSettingsParser <*> postgresSettingsParser <*> userStorageLocationParser <*> galleyEndpointParser <*> pageSizeParser) - (progDesc "Reindex all users from Cassandra if there is a new version.") - ) - <> command - "reindex-if-same-or-newer" - ( info - (ReindexSameOrNewer <$> elasticSettingsParser <*> cassandraSettingsParser <*> postgresSettingsParser <*> userStorageLocationParser <*> galleyEndpointParser <*> pageSizeParser) - (progDesc "Reindex all users from Cassandra, even if the version has not changed.") - ) - <> command - "migrate-data" - ( info - (Migrate <$> elasticSettingsParser <*> cassandraSettingsParser <*> postgresSettingsParser <*> userStorageLocationParser <*> galleyEndpointParser <*> pageSizeParser) - (progDesc "Migrate data in elastic search") - ) - <> command - "reindex-from-another-index" - ( info - (ReindexFromAnotherIndex <$> reindexToAnotherIndexSettingsParser) - ( progDesc - "Reindex data from an index to another. More about migrating to a new index here: https://github.com/wireapp/wire-server/blob/develop/docs/reference/elastic-search.md" - ) - ) - ) - -_IndexName :: Iso' ES.IndexName Text -_IndexName = iso (\(ES.IndexName n) -> n) ES.IndexName - -_Keyspace :: Iso' C.Keyspace Text -_Keyspace = iso C.unKeyspace C.Keyspace - -toESServer :: URIRef Absolute -> ES.Server -toESServer = - ES.Server - . view utf8 - . serializeURIRef' - . set pathL mempty - . set queryL mempty - . set fragmentL mempty diff --git a/services/brig/src/Brig/Options.hs b/services/brig/src/Brig/Options.hs index e69f515a391..900ed0441d6 100644 --- a/services/brig/src/Brig/Options.hs +++ b/services/brig/src/Brig/Options.hs @@ -42,7 +42,6 @@ import Data.Range import Data.Schema import Data.Text qualified as Text import Data.Text.Encoding qualified as Text -import Database.Bloodhound.Types qualified as ES import Hasql.Pool.Extended import Imports import Network.AMQP.Extended @@ -63,38 +62,6 @@ import Wire.EmailSubsystem.Template (TeamOpts) import Wire.PostgresMigrationOpts import Wire.RateLimit.Interpreter -data ElasticSearchOpts = ElasticSearchOpts - { -- | ElasticSearch URL - url :: !ES.Server, - -- | The name of the ElasticSearch user index - index :: !ES.IndexName, - -- | An additional index to write user data, useful while migrating to a new - -- index. - -- There is a bug hidden when using this option. Sometimes a user won't get - -- deleted from the index. Attempts at reproducing this issue in a simpler - -- environment have failed. As a workaround, there is a tool in - -- tools/db/find-undead which can be used to find the undead users right - -- after the migration, if they exist, we can run the reindexing to get data - -- in elasticsearch in a consistent state. - additionalWriteIndex :: !(Maybe ES.IndexName), - -- | An additional ES URL to write user data, useful while migrating to a - -- new instance of ES. It is necessary to provide 'additionalWriteIndex' for - -- this to be used. If this is 'Nothing' and 'additionalWriteIndex' is - -- configured, the 'url' field will be used. - additionalWriteIndexUrl :: !(Maybe ES.Server), - -- | Elasticsearch credentials - credentials :: !(Maybe FilePathSecrets), - -- | Credentials for additional ES index (maily used for migrations) - additionalCredentials :: !(Maybe FilePathSecrets), - insecureSkipVerifyTls :: Bool, - caCert :: Maybe FilePath, - additionalInsecureSkipVerifyTls :: Bool, - additionalCaCert :: Maybe FilePath - } - deriving (Show, Generic) - -instance FromJSON ElasticSearchOpts - data AWSOpts = AWSOpts { -- | Event journal queue for user events -- (e.g. user deletion) @@ -332,8 +299,6 @@ data Opts = Opts -- | Cassandra settings cassandra :: !CassandraOpts, - -- | ElasticSearch settings - elasticsearch :: !ElasticSearchOpts, -- | Postgresql settings, the key values must be in libpq format. -- https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS postgresql :: !(Map Text Text), @@ -833,6 +798,4 @@ makeLensesWith (lensRules & lensField .~ suffixNamer) ''Opts makeLensesWith (lensRules & lensField .~ suffixNamer) ''Settings -makeLensesWith (lensRules & lensField .~ suffixNamer) ''ElasticSearchOpts - makeLensesWith (lensRules & lensField .~ suffixNamer) ''TurnOpts diff --git a/services/brig/src/Brig/Team/API.hs b/services/brig/src/Brig/Team/API.hs index ce88b50a63f..628d55ed17d 100644 --- a/services/brig/src/Brig/Team/API.hs +++ b/services/brig/src/Brig/Team/API.hs @@ -75,7 +75,6 @@ import Wire.Error import Wire.Events (Events) import Wire.GalleyAPIAccess (GalleyAPIAccess, ShowOrHideInvitationUrl (..)) import Wire.GalleyAPIAccess qualified as GalleyAPIAccess -import Wire.IndexedUserStore (IndexedUserStore, getTeamSize) import Wire.InvitationStore (InvitationStore (..), PaginatedResult (..), StoredInvitation (..)) import Wire.InvitationStore qualified as Store import Wire.NotificationSubsystem (NotificationSubsystem) @@ -91,6 +90,8 @@ import Wire.UserGroupSubsystem (UserGroupSubsystem) import Wire.UserKeyStore import Wire.UserPendingActivationStore (UserPendingActivationStore) import Wire.UserPendingActivationStore qualified as UserPendingActivationStore +import Wire.UserSearchStore (UserSearchStore) +import Wire.UserSearchStore qualified as UserSearchStore import Wire.UserStore import Wire.UserSubsystem import Wire.UserSubsystem.Error @@ -105,7 +106,7 @@ servantAPI :: Member (Input InvitationUrlTemplates) r, Member (Input (Local ())) r, Member (Error UserSubsystemError) r, - Member IndexedUserStore r, + Member UserSearchStore r, Member TeamSubsystem r, Member SparAPIAccess r, Member (Embed App.HttpClientIO) r, @@ -133,7 +134,7 @@ servantAPI = teamSizePublic :: ( Member (Error UserSubsystemError) r, - Member IndexedUserStore r, + Member UserSearchStore r, Member TeamSubsystem r ) => UserId -> @@ -142,7 +143,7 @@ teamSizePublic :: teamSizePublic uid tid = do -- limit this to team admins to reduce risk of involuntary DOS attacks ensurePermissions uid tid [AddTeamMember] - getTeamSize tid + UserSearchStore.getTeamSize tid getInvitationCode :: ( Member Store.InvitationStore r, @@ -445,7 +446,6 @@ getInvitationByEmail email = do suspendTeam :: ( Member (Concurrency 'Unsafe) r, Member GalleyAPIAccess r, - Member UserSubsystem r, Member TeamSubsystem r, Member Events r, Member TinyLog r, @@ -468,7 +468,6 @@ suspendTeam tid = do unsuspendTeam :: ( Member (Concurrency 'Unsafe) r, Member GalleyAPIAccess r, - Member UserSubsystem r, Member TeamSubsystem r, Member Events r, Member AuthenticationSubsystem r, @@ -488,7 +487,6 @@ changeTeamAccountStatuses :: ( Member (Concurrency 'Unsafe) r, Member GalleyAPIAccess r, Member TeamSubsystem r, - Member UserSubsystem r, Member Events r, Member AuthenticationSubsystem r, Member UserStore r diff --git a/services/brig/src/Brig/User/Auth.hs b/services/brig/src/Brig/User/Auth.hs index 51880fca8b1..85320a61243 100644 --- a/services/brig/src/Brig/User/Auth.hs +++ b/services/brig/src/Brig/User/Auth.hs @@ -177,7 +177,6 @@ logout uts at = do renewAccess :: forall r u a. ( Member TinyLog r, - Member UserSubsystem r, Member Events r, ZAuth.UserTokenLike u, ZAuth.AccessTokenLike a, @@ -233,7 +232,6 @@ revokeAccess luid@(tUnqualified -> u) pw cc ll = do catchSuspendInactiveUser :: ( Member TinyLog r, - Member UserSubsystem r, Member Events r, Member (Concurrency 'Unsafe) r, Member AuthenticationSubsystem r, @@ -261,7 +259,6 @@ catchSuspendInactiveUser uid errval = do newAccess :: forall u a r. ( Member TinyLog r, - Member UserSubsystem r, Member Events r, ZAuth.UserTokenLike u, ZAuth.AccessTokenLike a, @@ -392,7 +389,6 @@ validateToken ut at = do -- | Allow to login as any user without having the credentials. ssoLogin :: ( Member TinyLog r, - Member UserSubsystem r, Member Events r, Member AuthenticationSubsystem r, Member (Input AuthenticationSubsystemConfig) r, @@ -431,7 +427,6 @@ ssoLogin (SsoLogin uid label) typ = do legalHoldLogin :: ( Member GalleyAPIAccess r, Member TinyLog r, - Member UserSubsystem r, Member AuthenticationSubsystem r, Member Events r, Member (Input AuthenticationSubsystemConfig) r, diff --git a/services/brig/src/Brig/User/Search/Index.hs b/services/brig/src/Brig/User/Search/Index.hs deleted file mode 100644 index 4c4919729d9..00000000000 --- a/services/brig/src/Brig/User/Search/Index.hs +++ /dev/null @@ -1,538 +0,0 @@ -{-# LANGUAGE GeneralizedNewtypeDeriving #-} -{-# LANGUAGE StrictData #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Brig.User.Search.Index - ( boolQuery, - - -- * Monad - IndexEnv (..), - IndexIO, - runIndexIO, - MonadIndexIO (..), - - -- * Administrative - createIndex, - createIndexIfNotPresent, - createIndexWithoutMapping, - resetIndex, - refreshIndexes, - updateMapping, - indexMapping, - - -- * Re-exports - ES.IndexSettings (..), - ES.IndexName (..), - ) -where - -import Bilge.IO (MonadHttp) -import Bilge.IO qualified as RPC -import Brig.Index.Types (CreateIndexSettings (..)) -import Control.Lens hiding ((#), (.=)) -import Control.Monad.Catch (MonadCatch, MonadMask, MonadThrow, throwM) -import Control.Monad.Except -import Data.Aeson as Aeson -import Data.Credentials -import Data.Id -import Data.Map qualified as Map -import Data.Text qualified as Text -import Data.Text.Encoding -import Database.Bloodhound qualified as ES -import Imports hiding (log, searchable) -import Network.HTTP.Client hiding (host, path, port) -import Network.HTTP.Types (statusCode) -import Prometheus (MonadMonitor) -import System.Logger qualified as Log -import System.Logger.Class (Logger, MonadLogger (..), field, info, msg, val, (+++), (~~)) -import Util.Options (Endpoint) -import Wire.IndexedUserStore (IndexedUserStoreError (..)) -import Wire.IndexedUserStore.ElasticSearch (mappingName) -import Wire.UserSearch.Types (searchVisibilityInboundFieldName) - --------------------------------------------------------------------------------- --- IndexIO Monad - -data IndexEnv = IndexEnv - { idxLogger :: Logger, - idxElastic :: ES.BHEnv, - idxRequest :: Maybe RequestId, - idxName :: ES.IndexName, - idxAdditionalName :: Maybe ES.IndexName, - idxAdditionalElastic :: Maybe ES.BHEnv, - idxGalley :: Endpoint, - -- | Used to make RPC calls to other wire-server services - idxRpcHttpManager :: Manager, - -- credentials for reindexing have to be passed via the env because bulk API requests are not supported by bloodhound - idxCredentials :: Maybe Credentials - } - -newtype IndexIO a = IndexIO (ReaderT IndexEnv IO a) - deriving - ( Functor, - Applicative, - Monad, - MonadIO, - MonadReader IndexEnv, - MonadThrow, - MonadCatch, - MonadMask, - MonadMonitor - ) - -runIndexIO :: (MonadIO m) => IndexEnv -> IndexIO a -> m a -runIndexIO e (IndexIO m) = liftIO $ runReaderT m e - -class (MonadIO m) => MonadIndexIO m where - liftIndexIO :: IndexIO a -> m a - -instance MonadIndexIO IndexIO where - liftIndexIO = id - -instance MonadLogger IndexIO where - log l m = do - g <- asks idxLogger - r <- asks idxRequest - Log.log g l $ maybe id (field "request" . unRequestId) r ~~ m - -instance MonadLogger (ExceptT e IndexIO) where - log l m = lift (log l m) - -instance ES.MonadBH IndexIO where - getBHEnv = asks idxElastic - -instance MonadHttp IndexIO where - handleRequestWithCont req handler = do - manager <- asks idxRpcHttpManager - liftIO $ withResponse req manager handler - --------------------------------------------------------------------------------- --- Administrative - --- | Refresh ElasticSearch index and the additional one if it's configured --- Only used in tests. In production, the addtional index is used write-only. -refreshIndexes :: (MonadIndexIO m) => m () -refreshIndexes = liftIndexIO $ do - idx <- asks idxName - void $ ES.refreshIndex idx - mbAddIdx <- asks idxAdditionalName - mbAddElasticEnv <- asks idxAdditionalElastic - case (mbAddIdx, mbAddElasticEnv) of - (Just addIdx, Just addElasticEnv) -> - -- Refresh additional index on a separate ElasticSearch instance. - ES.runBH addElasticEnv ((void . ES.refreshIndex) addIdx) - (Just addIdx, Nothing) -> - -- Refresh additional index on the same ElasticSearch instance. - void $ ES.refreshIndex addIdx - (Nothing, _) -> - -- No additional index - pure () - -createIndexIfNotPresent :: - (MonadIndexIO m) => - CreateIndexSettings -> - m () -createIndexIfNotPresent = createIndex' False - -createIndex :: - (MonadIndexIO m) => - CreateIndexSettings -> - m () -createIndex = createIndex' True - -createIndexWithoutMapping :: - (MonadIndexIO m) => - -- | Fail if index alredy exists - Bool -> - CreateIndexSettings -> - m () -createIndexWithoutMapping failIfExists (CreateIndexSettings settings shardCount mbDeleteTemplate) = liftIndexIO $ do - idx <- asks idxName - ex <- ES.indexExists idx - when (failIfExists && ex) $ - throwM (IndexError "Index already exists.") - unless ex $ do - let fullSettings = settings ++ [ES.AnalysisSetting analysisSettings] - - -- A previous release added an ES Index Template that matched all indices - -- named 'directory*'. This template is deprecated now, but it might still - -- be present in production instances. If present then it causes the update mapping - -- step to fail. - -- FUTUREWORK: remove this block and the --delete-template option, - -- after this has been released. - for_ mbDeleteTemplate $ \templateName@(ES.TemplateName tname) -> do - tExists <- ES.templateExists templateName - when tExists $ do - dr <- - traceES - ( encodeUtf8 - ("Delete index template " <> "\"" <> tname <> "\"") - ) - $ ES.deleteTemplate templateName - unless (ES.isSuccess dr) $ - throwM (IndexError "Deleting index template failed.") - - cr <- traceES "Create index" $ ES.createIndexWith fullSettings shardCount idx - unless (ES.isSuccess cr) $ - throwM (IndexError $ "Index creation failed: " <> Text.pack (show cr)) - -createIndex' :: - (MonadIndexIO m) => - -- | Fail if index alredy exists - Bool -> - CreateIndexSettings -> - m () -createIndex' failIfExists (CreateIndexSettings settings shardCount mbDeleteTemplate) = do - idx <- liftIndexIO $ asks idxName - -- Check if the index already exists before attempting creation. - -- If it already exists, we should not update anything (including mappings). - existedBefore <- liftIndexIO $ ES.indexExists idx - createIndexWithoutMapping failIfExists (CreateIndexSettings settings shardCount mbDeleteTemplate) - -- Only put the mapping when we actually created the index above. - unless existedBefore $ do - liftIndexIO $ do - mr <- - traceES "Put mapping" $ - ES.putNamedMapping idx mappingName indexMapping - unless (ES.isSuccess mr) $ - throwM (IndexError $ "Put Mapping failed: " <> Text.pack (show mr)) - -analysisSettings :: ES.Analysis -analysisSettings = - let analyzerDef = - Map.fromList - [ ("prefix_index", ES.AnalyzerDefinition (Just $ ES.Tokenizer "whitespace") [ES.TokenFilter "edge_ngram_1_30"] []), - ("prefix_search", ES.AnalyzerDefinition (Just $ ES.Tokenizer "whitespace") [ES.TokenFilter "truncate_30"] []) - ] - filterDef = - Map.fromList - [ ("edge_ngram_1_30", ES.TokenFilterDefinitionEdgeNgram (ES.NgramFilter 1 30) Nothing), - ("truncate_30", ES.TokenFilterTruncate 30) - ] - in ES.Analysis analyzerDef mempty filterDef mempty - -updateMapping :: (MonadIndexIO m) => m () -updateMapping = liftIndexIO $ do - idx <- asks idxName - ex <- ES.indexExists idx - unless ex $ - throwM (IndexError "Index does not exist.") - -- FUTUREWORK: check return code (ES.isSuccess) and fail if appropriate. - -- But to do that we have to consider the consequences of this failing in our helm chart: - -- https://github.com/wireapp/wire-server-deploy/blob/92311d189818ffc5e26ff589f81b95c95de8722c/charts/elasticsearch-index/templates/create-index.yaml - void $ - traceES "Put mapping" $ - ES.putNamedMapping idx mappingName indexMapping - -resetIndex :: - (MonadIndexIO m) => - CreateIndexSettings -> - m () -resetIndex ciSettings = liftIndexIO $ do - idx <- asks idxName - gone <- - ES.indexExists idx >>= \case - True -> ES.isSuccess <$> traceES "Delete Index" (ES.deleteIndex idx) - False -> pure True - if gone - then createIndex ciSettings - else throwM (IndexError "Index deletion failed.") - --------------------------------------------------------------------------------- --- Internal - -traceES :: (MonadIndexIO m) => ByteString -> IndexIO ES.Reply -> m ES.Reply -traceES descr act = liftIndexIO $ do - info (msg descr) - r <- act - info . msg $ (r & statusCode . responseStatus) +++ val " - " +++ responseBody r - pure r - --- | This mapping defines how elasticsearch will treat each field in a document. Here --- is how it treats each field: --- name: Not indexed, as it is only meant to be shown to user, for querying we use --- normalized --- team: Used to ensure only teammates can find each other --- accent_id: Not indexed, we cannot search by this. --- normalized: This is transliterated version of the name to ASCII Latin characters, --- this is used for searching by name --- handle: Used for searching by handle --- normalized.prefix: Used for searching by name prefix --- handle.prefix: Used for searching by handle prefix --- saml_idp: URL of SAML issuer, not indexed, used for sorting --- managed_by: possible values "scim" or "wire", indexed as keyword --- created_at: date when "activated" state last chagned in epoch-millis, not indexed, used for sorting --- searchable: Used to filter searchable users --- --- The prefix fields use "prefix_index" analyzer for indexing and "prefix_search" --- analyzer for searching. The "prefix_search" analyzer uses "edge_ngram" filter, this --- indexes the handles and normalized names by prefixes. For example: "alice" will be --- indexed as "a", "al", "ali", "alic" and "alice". While searching for say "ali", we --- do not want to again use the "prefix" analyzer, otherwise we would get a match for --- "a", "al" and "ali" each, this skews the scoring in elasticsearch a lot and exact --- matches get pushed behind prefix matches. --- --- The "prefix_index" analyzer is defined as a combination of the "whitespace" --- tokenizer and "edge_ngram_1_30" filter. The edge_ngram_1_30 filter generates tokens --- of from length 1 to 30 and the whitespace tokenizer ensures words separated by --- whitespaces are tokenized separately. So, tokens for "Alice Charlie" would be: --- ["a", "al", "ali", "alic", "alice", "c", "ch", "cha", "char", "charl", "charlie"] --- This makes searching for somebody by just their last or middle name possible. --- Additionally one could look for "ali char" and still expect to find "Alice Charlie" --- --- The "prefix_search" analyzer is defined as a combination of the "whitespace" --- tokenizer and "truncate_30" filter. The truncate_30 filter ensures that the --- searched tokens are not bigger than 30 characters by truncating them, this is --- necessary as our "prefix_index" analyzer only creates edge_ngrams until 30 --- characters. --- --- About the dynamic field: When this is not set and we add another field to our --- user document, elasticsearch will try to guess how it is supposed to be indexed. --- Changes to this require creating a new index and a cumbersome migration. So it is --- important that we set this field to `false`. This will make new fields will just --- not be indexed. After we decide what they should look like, we can just run a --- reindex to make them usable. More info: --- https://www.elastic.co/guide/en/elasticsearch/reference/7.7/dynamic.html -indexMapping :: Value -indexMapping = - object - [ "dynamic" .= False, - "properties" - .= object - [ "normalized" -- normalized user name - .= MappingProperty - { mpType = MPText, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = - Map.fromList [("prefix", MappingField MPText (Just "prefix_index") (Just "prefix_search"))] - }, - "name" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = False, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "handle" - .= MappingProperty - { mpType = MPText, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = - Map.fromList - [ ("prefix", MappingField MPText (Just "prefix_index") (Just "prefix_search")), - ("keyword", MappingField MPKeyword Nothing Nothing) - ] - }, - "email" - .= MappingProperty - { mpType = MPText, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = - Map.fromList - [ ("prefix", MappingField MPText (Just "prefix_index") (Just "prefix_search")), - ("keyword", MappingField MPKeyword Nothing Nothing) - ] - }, - "team" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "accent_id" - .= MappingProperty - { mpType = MPByte, - mpStore = False, - mpIndex = False, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "account_status" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "saml_idp" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = False, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "managed_by" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "created_at" - .= MappingProperty - { mpType = MPDate, - mpStore = False, - mpIndex = False, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "role" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = mempty - }, - searchVisibilityInboundFieldName - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "searchable" - .= MappingProperty - { mpType = MPBoolean, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "scim_external_id" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = False, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "sso" - .= object - [ "type" .= Aeson.String "nested", - "properties" - .= object - [ "issuer" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = False, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "nameid" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = False, - mpAnalyzer = Nothing, - mpFields = mempty - } - ] - ], - "email_unvalidated" - .= MappingProperty - { mpType = MPText, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = mempty - }, - "type" - .= MappingProperty - { mpType = MPKeyword, - mpStore = False, - mpIndex = True, - mpAnalyzer = Nothing, - mpFields = mempty - } - ] - ] - -data MappingProperty = MappingProperty - { mpType :: MappingPropertyType, - mpStore :: Bool, - mpIndex :: Bool, - mpAnalyzer :: Maybe Text, - mpFields :: Map Text MappingField - } - -data MappingField = MappingField - { mfType :: MappingPropertyType, - mfAnalyzer :: Maybe Text, - mfSearchAnalyzer :: Maybe Text - } - -data MappingPropertyType = MPText | MPKeyword | MPByte | MPDate | MPBoolean - deriving (Eq) - -instance ToJSON MappingProperty where - toJSON mp = - object - ( [ "type" .= mpType mp, - "store" .= mpStore mp, - "index" .= mpIndex mp - ] - <> ["analyzer" .= mpAnalyzer mp | isJust $ mpAnalyzer mp] - <> ["fields" .= mpFields mp | not . Map.null $ mpFields mp] - ) - -instance ToJSON MappingPropertyType where - toJSON MPText = Aeson.String "text" - toJSON MPKeyword = Aeson.String "keyword" - toJSON MPByte = Aeson.String "byte" - toJSON MPDate = Aeson.String "date" - toJSON MPBoolean = Aeson.String "boolean" - -instance ToJSON MappingField where - toJSON mf = - object $ - ["type" .= mfType mf] - <> ["analyzer" .= mfAnalyzer mf | isJust (mfAnalyzer mf)] - <> ["search_analyzer" .= mfSearchAnalyzer mf | isJust (mfSearchAnalyzer mf)] - -boolQuery :: ES.BoolQuery -boolQuery = ES.mkBoolQuery [] [] [] [] - -data ParseException = ParseException - { _parseExceptionRemote :: !Text, - _parseExceptionMsg :: String - } - -instance Show ParseException where - show (ParseException r m) = - "Failed to parse response from remote " - ++ Text.unpack r - ++ " with message: " - ++ m - -instance Exception ParseException diff --git a/services/brig/src/Brig/User/Search/SearchIndex.hs b/services/brig/src/Brig/User/Search/SearchIndex.hs deleted file mode 100644 index 360baa24430..00000000000 --- a/services/brig/src/Brig/User/Search/SearchIndex.hs +++ /dev/null @@ -1,279 +0,0 @@ -{-# LANGUAGE StrictData #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Brig.User.Search.SearchIndex - ( searchIndex, - SearchSetting (..), - ) -where - -import Brig.App (Env, viewFederationDomain) -import Brig.User.Search.Index -import Control.Lens hiding (setting, (#), (.=)) -import Control.Monad.Catch (MonadThrow, throwM) -import Data.Aeson.Key qualified as Key -import Data.Domain (Domain) -import Data.Id -import Data.Qualified (Qualified (Qualified)) -import Database.Bloodhound qualified as ES -import Imports hiding (log, searchable) -import Wire.API.User (Name (fromName)) -import Wire.API.User.Search -import Wire.IndexedUserStore (IndexedUserStoreError (..)) -import Wire.IndexedUserStore.ElasticSearch (mappingName, restrictSearchSpaceByUserType) -import Wire.UserSearch.Types -import Wire.UserStore.IndexUser (normalized) - -data SearchSetting - = FederatedSearch - { _teamsPartial :: Maybe [TeamId], - types :: Maybe [UserTypeFilter] - } - | LocalSearch - { -- | User that is performing the search - _searcherPartial :: UserId, - -- | Team of user that is performing the search - _teamPartial :: Maybe TeamId, - -- | Types of users that should be returned (regular, app) - types :: Maybe [UserTypeFilter], - -- | Outgoing search restrictions - _infoPartial :: TeamSearchInfo - } - -searchSettingTeam :: SearchSetting -> Maybe TeamId -searchSettingTeam (FederatedSearch _ _) = Nothing -searchSettingTeam (LocalSearch _ mbTeam _ _) = mbTeam - -searchIndex :: - (MonadIndexIO m, MonadReader Env m) => - -- | The user performing the search. - SearchSetting -> - -- | The search query - Text -> - -- | The maximum number of results. - Int -> - m (SearchResult Contact) -searchIndex setting q = queryIndex (defaultUserQuery setting q) - -queryIndex :: - (MonadIndexIO m, MonadReader Env m) => - IndexQuery r -> - Int -> - m (SearchResult Contact) -queryIndex (IndexQuery q f _) s = do - localDomain <- viewFederationDomain - liftIndexIO $ do - idx <- asks idxName - let search = (ES.mkSearch (Just q) (Just f)) {ES.size = ES.Size (fromIntegral s)} - r <- - ES.searchByType idx mappingName search - >>= ES.parseEsResponse @_ @(ES.SearchResult UserDoc) - either (throwM . IndexLookupError) (traverse (userDocToContact' localDomain) . mkResult) r - where - mkResult es = - let results = mapMaybe ES.hitSource . ES.hits . ES.searchHits $ es - in SearchResult - { searchFound = ES.hitsTotalValue . ES.hitsTotal . ES.searchHits $ es, - searchReturned = length results, - searchTook = ES.took es, - searchResults = results, - searchPolicy = FullSearch, - searchPagingState = Nothing, - searchHasMore = Nothing - } - - userDocToContact' :: (MonadThrow m) => Domain -> UserDoc -> m Contact - userDocToContact' localDomain userDoc = do - userDocToContact - (Qualified userDoc.udId localDomain) - (maybe (throwM $ IndexError "Name not found") (pure . fromName)) - userDoc - --- | The default or canonical 'IndexQuery'. --- --- The intention behind parameterising 'queryIndex' over the 'IndexQuery' is that --- it allows to experiment with different queries (perhaps in an A/B context). --- --- FUTUREWORK: Drop legacyPrefixMatch -defaultUserQuery :: SearchSetting -> Text -> IndexQuery Contact -defaultUserQuery setting (normalized -> term') = - let matchPhraseOrPrefix = - ES.QueryMultiMatchQuery $ - ( ES.mkMultiMatchQuery - [ ES.FieldName "handle.prefix^2", - ES.FieldName "normalized.prefix", - ES.FieldName "normalized^3" - ] - (ES.QueryString term') - ) - { ES.multiMatchQueryType = Just ES.MultiMatchMostFields, - ES.multiMatchQueryOperator = ES.And - } - query = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = [matchPhraseOrPrefix], - -- This removes exact handle matches, as they are fetched from cassandra - ES.boolQueryMustNotMatch = [termQ "handle" term'] - } - ], - ES.boolQueryShouldMatch = [ES.QueryExistsQuery (ES.FieldName "handle")] - } - -- This reduces relevance on users not in team of search by 90% (no - -- science behind that number). If the searcher is not part of a team the - -- relevance is not reduced for any users. - queryWithBoost setting' = - ES.QueryBoostingQuery - ES.BoostingQuery - { ES.positiveQuery = query, - ES.negativeQuery = maybe ES.QueryMatchNoneQuery matchUsersNotInTeam (searchSettingTeam setting'), - ES.negativeBoost = ES.Boost 0.1 - } - in mkUserQuery setting (queryWithBoost setting) - -mkUserQuery :: SearchSetting -> ES.Query -> IndexQuery Contact -mkUserQuery setting q = - IndexQuery - q - ( ES.Filter - . ES.QueryBoolQuery - $ boolQuery - { ES.boolQueryMustNotMatch = - maybeToList (matchSelf setting) - <> - -- Federated search must respect the same "searchable" flag as - -- local contact search. - [ES.TermQuery (ES.Term "searchable" "false") Nothing], - ES.boolQueryMustMatch = - [ restrictSearchSpaceByTeam setting, - restrictSearchSpaceByUserType setting.types, - ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = - [ termQ "account_status" "active", - -- Also match entries where the account_status field is not present. - -- These must have been inserted before we added the account_status - -- and at that time we only inserted active users in the first place. - -- This should be unnecessary after re-indexing, but let's be lenient - -- here for a while. - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = - [ES.QueryExistsQuery (ES.FieldName "account_status")] - } - ] - } - ] - } - ) - [] - -termQ :: Text -> Text -> ES.Query -termQ f v = - ES.TermQuery - ES.Term - { ES.termField = f, - ES.termValue = v - } - Nothing - -matchSelf :: SearchSetting -> Maybe ES.Query -matchSelf (FederatedSearch _ _) = Nothing -matchSelf (LocalSearch searcher _tid _mtypes _searchInfo) = Just (termQ "_id" (idToText searcher)) - --- | See 'TeamSearchInfo' --- --- FUTUREWORK(fisx): there is at least *some* overlap with --- restrictSearchSpaceByTeam in Wire.IndexedUserStore.ElasticSearch, --- can that be resolved? -restrictSearchSpaceByTeam :: SearchSetting -> ES.Query -restrictSearchSpaceByTeam (FederatedSearch Nothing _) = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = - [ matchNonTeamMemberUsers, - matchTeamMembersSearchableByAllTeams - ] - } -restrictSearchSpaceByTeam (FederatedSearch (Just []) _) = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ -- if the list of allowed teams is empty, this is impossible to fulfill, and no results will be returned - -- this case should be handled earlier, so this is just a safety net - ES.TermQuery (ES.Term "team" "must not match any team") Nothing - ] - } -restrictSearchSpaceByTeam (FederatedSearch (Just teams) _) = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ matchTeamMembersSearchableByAllTeams, - onlyInTeams - ] - } - where - onlyInTeams = ES.QueryBoolQuery boolQuery {ES.boolQueryShouldMatch = map matchTeamMembersOf teams} -restrictSearchSpaceByTeam (LocalSearch _uid mteam _ searchInfo) = - case (mteam, searchInfo) of - (Nothing, _) -> matchNonTeamMemberUsers - (Just _, NoTeam) -> matchNonTeamMemberUsers - (Just searcherTeam, TeamOnly team) -> - if searcherTeam == team - then matchTeamMembersOf team - else ES.QueryMatchNoneQuery - (Just searcherTeam, AllUsers) -> - ES.QueryBoolQuery - boolQuery - { ES.boolQueryShouldMatch = - [ matchNonTeamMemberUsers, - matchTeamMembersSearchableByAllTeams, - matchTeamMembersOf searcherTeam - ] - } - -matchTeamMembersOf :: TeamId -> ES.Query -matchTeamMembersOf team = ES.TermQuery (ES.Term "team" $ idToText team) Nothing - -matchTeamMembersSearchableByAllTeams :: ES.Query -matchTeamMembersSearchableByAllTeams = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustMatch = - [ ES.QueryExistsQuery $ ES.FieldName "team", - ES.TermQuery (ES.Term (Key.toText searchVisibilityInboundFieldName) "searchable-by-all-teams") Nothing - ] - } - -matchNonTeamMemberUsers :: ES.Query -matchNonTeamMemberUsers = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = [ES.QueryExistsQuery $ ES.FieldName "team"] - } - -matchUsersNotInTeam :: TeamId -> ES.Query -matchUsersNotInTeam tid = - ES.QueryBoolQuery - boolQuery - { ES.boolQueryMustNotMatch = [ES.TermQuery (ES.Term "team" $ idToText tid) Nothing] - } diff --git a/services/brig/test/integration/API/Federation.hs b/services/brig/test/integration/API/Federation.hs index fe4173888eb..d1d153b875d 100644 --- a/services/brig/test/integration/API/Federation.hs +++ b/services/brig/test/integration/API/Federation.hs @@ -35,7 +35,6 @@ module API.Federation where -import API.Search.Util (refreshIndex) import Bilge hiding (head) import Bilge.Assert import Brig.Options qualified as Opt @@ -100,7 +99,6 @@ allowFullSearch domain opts = testSearchSuccess :: Opt.Opts -> Brig -> Http () testSearchSuccess opts brig = do (handle, user) <- createUserWithHandle brig - refreshIndex brig let quid = userQualifiedId user let domain = Domain "example.com" @@ -117,7 +115,6 @@ testSearchSuccess opts brig = do testFulltextSearchSuccess :: Opt.Opts -> Brig -> Http () testFulltextSearchSuccess opts brig = do (_, user) <- createUserWithHandle brig - refreshIndex brig let quid = userQualifiedId user let domain = Domain "example.com" @@ -145,7 +142,6 @@ testFulltextSearchMultipleUsers opts brig = do update = RequestBodyLBS . encode $ update' put (brig . path "/self" . contentJson . zUser (User.userId identityThief) . zConn "c" . body update) !!! const 200 === statusCode - refreshIndex brig let domain = Domain "example.com" @@ -189,7 +185,6 @@ testSearchRestrictions opts brig = do (handle, user) <- createUserWithHandle brig let quid = userQualifiedId user - refreshIndex brig let opts' = opts @@ -233,7 +228,6 @@ testGetUserByHandleRestrictions opts brig = do (handle, user) <- createUserWithHandle brig let quid = userQualifiedId user - refreshIndex brig let opts' = opts diff --git a/services/brig/test/integration/API/Search.hs b/services/brig/test/integration/API/Search.hs index 7729573dc7d..046f63b81c0 100644 --- a/services/brig/test/integration/API/Search.hs +++ b/services/brig/test/integration/API/Search.hs @@ -1,10 +1,6 @@ {-# LANGUAGE OverloadedRecordDot #-} -{-# LANGUAGE PartialTypeSignatures #-} -{-# LANGUAGE QuasiQuotes #-} -{-# LANGUAGE RecordWildCards #-} {-# OPTIONS_GHC -Wno-incomplete-patterns #-} {-# OPTIONS_GHC -Wno-incomplete-uni-patterns #-} -{-# OPTIONS_GHC -Wno-partial-type-signatures #-} {-# OPTIONS_GHC -Wno-redundant-constraints #-} -- This file is part of the Wire Server implementation. @@ -26,109 +22,61 @@ module API.Search ( tests, - testWithBothIndices, ) where import API.Search.Util -import API.Search.Util qualified as Search import API.Team.Util import API.User.Util import Bilge import Bilge.Assert -import Brig.App (initHttpManagerWithTLSConfig) -import Brig.Index.Eval (initIndex, runCommand) -import Brig.Index.Options -import Brig.Index.Options qualified as IndexOpts -import Brig.Options import Brig.Options qualified as Opt -import Brig.Options qualified as Opts -import Brig.User.Search.Index -import Cassandra qualified as C -import Cassandra.Options qualified as CassOpts -import Control.Lens ((.~), (?~), (^.), (^?), (^?!)) +import Control.Lens ((?~)) import Control.Monad.Catch (MonadCatch) -import Data.Aeson (Value, decode) import Data.Aeson qualified as Aeson -import Data.Aeson.Lens qualified as Aeson import Data.Domain (Domain (Domain)) import Data.Handle (fromHandle) import Data.Id import Data.Qualified (Qualified (qDomain, qUnqualified)) -import Data.String.Conversions import Data.Text qualified as Text -import Data.Text.Encoding qualified as Text -import Data.UUID qualified as UUID -import Database.Bloodhound qualified as ES import Federation.Util import Imports -import Network.HTTP.ReverseProxy (waiProxyTo) -import Network.HTTP.ReverseProxy qualified as Wai -import Network.HTTP.Types qualified as HTTP -import Network.Wai qualified as Wai -import Network.Wai.Handler.Warp qualified as Warp -import Network.Wai.Test qualified as WaiTest -import Safe (headMay) -import System.Logger qualified as Log import Test.QuickCheck (Arbitrary (arbitrary), generate) import Test.Tasty import Test.Tasty.HUnit -import Text.RawString.QQ (r) -import URI.ByteString qualified as URI -import UnliftIO (Concurrently (..), async, bracket, cancel, runConcurrently) import Util -import Util.Options (Endpoint) import Wire.API.Federation.API.Brig (SearchResponse (SearchResponse)) import Wire.API.Team.Feature -import Wire.API.Team.Member qualified as Member -import Wire.API.Team.Permission -import Wire.API.Team.Role import Wire.API.Team.SearchVisibility import Wire.API.User as User import Wire.API.User.Search import Wire.API.User.Search qualified as Search -import Wire.IndexedUserStore.ElasticSearch (mappingName) -import Wire.IndexedUserStore.MigrationStore.ElasticSearch (defaultMigrationIndexName) -import Wire.PostgresMigrationOpts -tests :: Opt.Opts -> ES.Server -> Manager -> Galley -> Brig -> IO TestTree -tests opts additionalElasticSearch mgr galley brig = do +tests :: Opt.Opts -> Manager -> Galley -> Brig -> IO TestTree +tests opts mgr galley brig = do testSetupOutboundOnly <- runHttpT mgr prepareUsersForSearchVisibilityNoNameOutsideTeamTests pure $ testGroup "search" $ - [ testWithBothIndices opts mgr "by-name" $ testSearchByName brig, - testWithBothIndices opts mgr "by-handle" $ testSearchByHandle brig, - testWithBothIndices opts mgr "size - when exact handle matches a team user" $ testSearchSize brig True, - testWithBothIndices opts mgr "size - when exact handle matches a non team user" $ testSearchSize brig False, + [ test mgr "by-name" $ testSearchByName brig, + test mgr "by-handle" $ testSearchByHandle brig, + test mgr "size - when exact handle matches a team user" $ testSearchSize brig True, + test mgr "size - when exact handle matches a non team user" $ testSearchSize brig False, test mgr "empty query" $ testSearchEmpty brig, - flakyTest mgr "reindex" $ testReindex brig, - testWithBothIndices opts mgr "no match" $ testSearchNoMatch brig, - testWithBothIndices opts mgr "no extra results" $ testSearchNoExtraResults brig, - testWithBothIndices opts mgr "order-handle (prefix match)" $ testOrderHandle brig, - testWithBothIndices opts mgr "by-first/middle/last name" $ testSearchByLastOrMiddleName brig, - testWithBothIndices opts mgr "Non ascii names" $ testSearchNonAsciiNames brig, - testWithBothIndices opts mgr "user with umlaut" $ testSearchWithUmlaut brig, - testWithBothIndices opts mgr "user with japanese name" $ testSearchCJK brig, - testGroup "index migration" $ - [ testGroup "same ElasticSearch instance" $ - let esServer = (opts ^. Opt.elasticsearchLens . Opt.urlLens) - in [ test mgr "migration to new index from existing index" $ testMigrationToNewIndex opts brig esServer runReindexFromAnotherIndex, - test mgr "migration to new index from database" $ testMigrationToNewIndex opts brig esServer (runReindexFromDatabase Reindex), - test mgr "migration to new index from database (force sync)" $ testMigrationToNewIndex opts brig esServer (runReindexFromDatabase ReindexSameOrNewer) - ], - testGroup "different ElasticSearch instance" $ - [ test mgr "migration to new index from database" $ - testMigrationToNewIndex opts brig additionalElasticSearch (runReindexFromDatabase Migrate) - ] - ], + test mgr "no match" $ testSearchNoMatch brig, + test mgr "no extra results" $ testSearchNoExtraResults brig, + test mgr "order-handle (prefix match)" $ testOrderHandle brig, + test mgr "by-first/middle/last name" $ testSearchByLastOrMiddleName brig, + test mgr "Non ascii names" $ testSearchNonAsciiNames brig, + test mgr "user with umlaut" $ testSearchWithUmlaut brig, + test mgr "user with japanese name" $ testSearchCJK brig, testGroup "team A: SearchVisibilityStandard (= unrestricted outbound search)" $ [ testGroup "team A: SearchableByOwnTeam (= restricted inbound search)" $ - [ testWithBothIndices opts mgr " I. non-team user cannot find team A member by display name" $ testSearchTeamMemberAsNonMemberDisplayName mgr brig galley FeatureStatusDisabled, - testWithBothIndices opts mgr " II. non-team user can find team A member by exact handle" $ testSearchTeamMemberAsNonMemberExactHandle mgr brig galley FeatureStatusDisabled, - testWithBothIndices opts mgr "III. team B member cannot find team A member by display name" $ testSearchTeamMemberAsOtherMemberDisplayName mgr brig galley FeatureStatusDisabled, - testWithBothIndices opts mgr " IV. team B member can find team A member by exact handle" $ testSearchTeamMemberAsOtherMemberExactHandle mgr brig galley FeatureStatusDisabled, - testWithBothIndices opts mgr " V. team A member can find team A member by display name" $ testSearchTeamMemberAsSameMember mgr brig galley FeatureStatusDisabled, - testWithBothIndices opts mgr " VI. team A member can find non-team user by display name" $ testSeachNonMemberAsTeamMember brig, + [ test mgr " I. non-team user cannot find team A member by display name" $ testSearchTeamMemberAsNonMemberDisplayName mgr brig galley FeatureStatusDisabled, + test mgr " II. non-team user can find team A member by exact handle" $ testSearchTeamMemberAsNonMemberExactHandle mgr brig galley FeatureStatusDisabled, + test mgr "III. team B member cannot find team A member by display name" $ testSearchTeamMemberAsOtherMemberDisplayName mgr brig galley FeatureStatusDisabled, + test mgr " IV. team B member can find team A member by exact handle" $ testSearchTeamMemberAsOtherMemberExactHandle mgr brig galley FeatureStatusDisabled, + test mgr " V. team A member can find team A member by display name" $ testSearchTeamMemberAsSameMember mgr brig galley FeatureStatusDisabled, + test mgr " VI. team A member can find non-team user by display name" $ testSeachNonMemberAsTeamMember brig, testGroup "order" $ [ test mgr "team-mates are listed before team-outsiders (exact match)" $ testSearchOrderingAsTeamMemberExactMatch brig, test mgr "team-mates are listed before team-outsiders (prefix match)" $ testSearchOrderingAsTeamMemberPrefixMatch brig, @@ -145,7 +93,7 @@ tests opts additionalElasticSearch mgr galley brig = do ] ], testGroup "searchSameTeamOnly == true (server setting)" $ - [ testWithBothIndicesAndOpts opts mgr "any team user cannot find any non-team user by display name or exact handle" $ testSearchSameTeamOnly brig + [ test mgr "any team user cannot find any non-team user by display name or exact handle" $ testSearchSameTeamOnly brig opts ], testGroup "team A: SearchVisibilityNoNameOutsideTeam (restricted outbound search)" $ [ testGroup "team A: SearchableByOwnTeam (= restricted inbound search)" $ @@ -164,9 +112,7 @@ tests opts additionalElasticSearch mgr galley brig = do -- failure/error cases on search (augment the federatorMock?) -- wire-api-federation Servant-Api vs protobuf-client interactions ], - test mgr "user with unvalidated email" $ testSearchWithUnvalidatedEmail brig, - test mgr "testSearchableMissing: searchable field missing defaults to true" $ - testSearchableMissing opts brig galley + test mgr "user with unvalidated email" $ testSearchWithUnvalidatedEmail brig ] where -- Since the tests are about querying only, we only need 1 creation @@ -178,60 +124,10 @@ tests opts additionalElasticSearch mgr galley brig = do setTeamSearchVisibility galley tidA SearchVisibilityNoNameOutsideTeam (tidB, ownerB, memberB : _) <- createPopulatedBindingTeamWithNamesAndHandles brig 1 regularUser <- randomUserWithHandle brig - refreshIndex brig pure ((tidA, ownerA, memberA), (tidB, ownerB, memberB), regularUser) type TestConstraints m = (MonadFail m, MonadCatch m, MonadIO m, MonadHttp m) -testSearchableMissing :: Opts.Opts -> Brig -> Galley -> Http () -testSearchableMissing opts brig galley = do - (owner, tid) <- createUserWithTeam brig - - let mkTeamMember :: Permissions -> Http User - mkTeamMember perms = do - member <- createTeamMember brig galley owner tid perms - selfUser <$> (responseJsonError =<< get (brig . path "/self" . zUser (userId member))) - - -- create user, this by default has searchable = True - user <- mkTeamMember (Member.rolePermissions RoleMember) - let uid = userId user - liftIO $ assertBool "created users are searchable by default" $ userSearchable user - - -- remove searchable field from elasticsearch - let indexName = opts.elasticsearch.index - docId = ES.DocId $ UUID.toText $ toUUID uid - userJson :: Aeson.Value <- do - resp <- liftIO $ runBH opts $ ES.getDocument indexName mappingName docId - responseJsonError $ fmap Just resp - liftIO $ - assertBool "Newly created users have searchable field set" $ - isJust $ - userJson ^? Aeson.key "_source" . Aeson.key "searchable" - let userJson' = userJson ^?! Aeson.key "_source" - userJsonLegacy = userJson' & Aeson.atKey "searchable" .~ Nothing -- this raw JSON has now "searchable" field removed - void $ liftIO $ runBH opts $ ES.deleteDocument indexName mappingName docId - void $ liftIO $ runBH opts $ ES.indexDocument indexName mappingName ES.defaultIndexDocumentSettings userJsonLegacy docId - refreshIndex brig - - -- get updated raw JSON and double-check that "searchable" field is gone - userJsonLegacyCheck :: Aeson.Value <- do - resp <- liftIO (runBH opts $ ES.getDocument indexName mappingName docId) - responseJsonError $ fmap Just resp - liftIO $ - assertBool "Updated user has no searchable field" $ - isNothing $ - userJsonLegacyCheck ^? Aeson.key "_source" . Aeson.key "searchable" - - -- perform search and still get the user - searcher <- userId <$> mkTeamMember (Member.rolePermissions RoleMember) - s' <- Search.executeSearch brig searcher $ fromName $ userDisplayName user - liftIO $ - assertBool "User with no searchable field is still found via /search/contacts" $ - uid `elem` map contactUid (searchResults s') - where - contactUid :: Contact -> UserId - contactUid = qUnqualified . contactQualifiedId - testSearchWithUnvalidatedEmail :: (TestConstraints m) => Brig -> m () testSearchWithUnvalidatedEmail brig = do (tid, owner, user : _) <- createPopulatedBindingTeamWithNamesAndHandles brig 1 @@ -240,21 +136,18 @@ testSearchWithUnvalidatedEmail brig = do ownerId = userId owner let searchForUserAndCheckThat = searchAndCheckResult brig tid ownerId uid email <- randomEmail - refreshIndex brig searchForUserAndCheckThat ( \tc -> do Search.teamContactEmail tc @?= Just oldEmail assertBool "unvalidated email should be null" (isNothing . Search.teamContactEmailUnvalidated $ tc) ) initiateEmailUpdateLogin brig email (emailLogin oldEmail defPassword Nothing) uid !!! const 202 === statusCode - refreshIndex brig searchForUserAndCheckThat ( \tc -> do Search.teamContactEmail tc @?= Just oldEmail Search.teamContactEmailUnvalidated tc @?= Just email ) activateEmail brig email - refreshIndex brig searchForUserAndCheckThat ( \tc -> do Search.teamContactEmail tc @?= Just email @@ -276,7 +169,6 @@ testSearchByName :: (TestConstraints m) => Brig -> m () testSearchByName brig = do u1 <- randomUser brig u2 <- randomUser brig - refreshIndex brig let uid1 = userId u1 quid1 = userQualifiedId u1 uid2 = userId u2 @@ -295,7 +187,6 @@ testSearchByLastOrMiddleName brig = do lastName <- randomHandle searchedUser <- createUser' True (firstName <> " " <> middleName <> " " <> lastName) brig let searched = userQualifiedId searchedUser - refreshIndex brig assertCanFind brig searcher searched firstName assertCanFind brig searcher searched middleName assertCanFind brig searcher searched lastName @@ -307,7 +198,6 @@ testSearchNonAsciiNames brig = do suffix <- randomHandle searchedUser <- createUser' True ("शक्तिमान" <> suffix) brig let searched = userQualifiedId searchedUser - refreshIndex brig assertCanFind brig searcher searched ("शक्तिमान" <> suffix) -- This is pathetic transliteration, but it is what we have. assertCanFind brig searcher searched ("saktimana" <> suffix) @@ -318,7 +208,6 @@ testSearchCJK brig = do user <- createUser' True "藤崎詩織" brig user' <- createUser' True "さおり" brig user'' <- createUser' True "ジョン" brig - refreshIndex brig assertCanFind brig (User.userId searcher) user.userQualifiedId "藤崎詩織" assertCanFind brig (User.userId searcher) user'.userQualifiedId "saori" @@ -333,7 +222,6 @@ testSearchWithUmlaut :: (TestConstraints m) => Brig -> m () testSearchWithUmlaut brig = do searcher <- randomUser brig user <- createUser' True "Özi Müller" brig - refreshIndex brig assertCanFind brig (User.userId searcher) user.userQualifiedId "ozi muller" assertCanFind brig (User.userId searcher) user.userQualifiedId "Özi Müller" @@ -341,7 +229,6 @@ testSearchByHandle :: (TestConstraints m) => Brig -> m () testSearchByHandle brig = do u1 <- randomUserWithHandle brig u2 <- randomUser brig - refreshIndex brig let quid1 = userQualifiedId u1 uid2 = userId u2 Just h = fromHandle <$> userHandle u1 @@ -352,7 +239,6 @@ testSearchEmpty brig = do -- This user exists just in case empty string starts matching everything _someUser <- randomUserWithHandle brig searcher <- randomUser brig - refreshIndex brig res <- searchResults <$> executeSearch brig (userId searcher) "" liftIO $ assertEqual "nothing should be returned" [] res @@ -369,7 +255,6 @@ testSearchSize brig exactHandleInTeam = do let handle = fromHandle . fromMaybe (error "impossible") $ userHandle nonTeamHandleMatch pure (nonTeamHandleMatch, handle) replicateM_ 6 $ createUser' True searchTerm brig - refreshIndex brig self <- userId <$> randomUser brig res <- searchResults <$> executeSearch' brig self searchTerm Nothing (Just 5) @@ -391,7 +276,6 @@ testSearchNoMatch brig = do _ <- randomUser brig let uid1 = userId u1 -- _uid2 = userId u2 - refreshIndex brig result <- searchResults <$> executeSearch brig uid1 "nomatch" liftIO $ assertEqual "Expected 0 results" 0 (length result) @@ -403,58 +287,10 @@ testSearchNoExtraResults brig = do u2 <- createUser' True u2Handle brig let uid1 = userId u1 quid2 = userQualifiedId u2 - refreshIndex brig resultUIds <- map contactQualifiedId . searchResults <$> executeSearch brig uid1 u2Handle liftIO $ assertEqual "Expected search returns only the searched" [quid2] resultUIds -testReindex :: Brig -> Http () -testReindex brig = do - u <- randomUser brig - ((), regular) <- - runConcurrently $ - (,) - <$> Concurrently (reindex brig) - <*> Concurrently (replicateM 5 $ delayed *> mkRegularUser) - refreshIndex brig - for_ regular $ \u' -> do - let Just h = fromHandle <$> userHandle u' - assertCanFind brig (userId u) (userQualifiedId u') h - (found : _) <- searchResults <$> executeSearch brig (userId u) h - liftIO $ do - assertEqual "Unexpected UserId" (contactQualifiedId found) (userQualifiedId u') - assertEqual "Unexpected Name" (contactName found) (fromName $ userDisplayName u') - assertEqual "Unexpected Colour" (contactColorId found) (Just . fromIntegral . fromColourId $ userAccentId u') - assertEqual "Unexpected Handle" (contactHandle found) (fromHandle <$> userHandle u') - where - -- note: delaying user creation a bit to increase the chance of actually - -- happen concurrently to the reindex on a small test database - delayed = liftIO $ threadDelay 10000 - mkRegularUser = randomUserWithHandle brig - --- This test is currently disabled, because it fails sporadically, probably due --- to imprecisions in ES exact match scoring. --- FUTUREWORK: Find the reason for the failures and fix ES behaviour. --- See also the "cassandra writetime hypothesis": --- https://wearezeta.atlassian.net/browse/BE-523 --- https://github.com/wireapp/wire-server/pull/1798#issuecomment-933174913 -_testOrderName :: (TestConstraints m) => Brig -> m () -_testOrderName brig = do - searcher <- userId <$> randomUser brig - Name searchedWord <- randomNameWithMaxLen 122 - nameMatch <- userQualifiedId <$> createUser' True searchedWord brig - namePrefixMatch <- userQualifiedId <$> createUser' True (searchedWord <> "suffix") brig - refreshIndex brig - results <- searchResults <$> executeSearch brig searcher searchedWord - let resultUIds = map contactQualifiedId results - let expectedOrder = [nameMatch, namePrefixMatch] - let dbg = "results: " <> show results <> "\nsearchedWord: " <> cs searchedWord - liftIO $ - assertEqual - ("Expected order: name match, name prefix match.\n\nSince this test fails sporadically for unknown reasons here is some debug info:\n" <> dbg) - expectedOrder - resultUIds - testOrderHandle :: (TestConstraints m) => Brig -> m () testOrderHandle brig = do searcher <- userId <$> randomUser brig @@ -463,7 +299,6 @@ testOrderHandle brig = do void $ putHandle brig (qUnqualified handleMatch) searchedWord handlePrefixMatch <- userQualifiedId <$> createUser' True "handle prefix match" brig void $ putHandle brig (qUnqualified handlePrefixMatch) (searchedWord <> "suffix") - refreshIndex brig results <- searchResults <$> executeSearch brig searcher searchedWord let resultUIds = map contactQualifiedId results let expectedOrder = [handleMatch, handlePrefixMatch] @@ -478,9 +313,7 @@ testSearchTeamMemberAsNonMemberDisplayName mgr brig galley inboundVisibility = d nonTeamMember <- randomUser brig (tid, _, [teamMember, teamBTargetReindexedAfter]) <- createPopulatedBindingTeamWithNamesAndHandles brig 2 circumventSettingsOverride mgr $ setTeamSearchVisibilityInboundAvailable galley tid inboundVisibility - -- we set a random handle here to force a reindexing of that user void $ setRandomHandle brig teamBTargetReindexedAfter - refreshIndex brig assertCan'tFind brig (userId nonTeamMember) (userQualifiedId teamMember) (fromName (userDisplayName teamMember)) assertCan'tFind brig (userId nonTeamMember) (userQualifiedId teamBTargetReindexedAfter) (fromName (userDisplayName teamBTargetReindexedAfter)) @@ -489,11 +322,9 @@ testSearchTeamMemberAsNonMemberExactHandle mgr brig galley inboundVisibility = d nonTeamMember <- randomUser brig (tid, _, [teamMember, teamMemberReindexedAfter]) <- createPopulatedBindingTeamWithNamesAndHandles brig 2 circumventSettingsOverride mgr $ setTeamSearchVisibilityInboundAvailable galley tid inboundVisibility - -- we set a random handle here to force a reindexing of that user teamMemberReindexedAfterHandle <- do teamMemberReindexedAfter' <- setRandomHandle brig teamMemberReindexedAfter pure $ fromMaybe (error "teamATargetReindexedAfter must have a handle") (userHandle teamMemberReindexedAfter') - refreshIndex brig let teamMemberHandle = fromMaybe (error "teamMember must have a handle") (userHandle teamMember) assertCanFind brig (userId nonTeamMember) (userQualifiedId teamMember) (fromHandle teamMemberHandle) assertCanFind brig (userId nonTeamMember) (userQualifiedId teamMemberReindexedAfter) (fromHandle teamMemberReindexedAfterHandle) @@ -502,11 +333,8 @@ testSearchTeamMemberAsOtherMemberDisplayName :: (TestConstraints m) => Manager - testSearchTeamMemberAsOtherMemberDisplayName mgr brig galley inboundVisibility = do (_, _, [teamBSearcher]) <- createPopulatedBindingTeamWithNamesAndHandles brig 1 (tidA, _, [teamATarget, teamATargetReindexedAfter]) <- createPopulatedBindingTeamWithNamesAndHandles brig 2 - refreshIndex brig circumventSettingsOverride mgr $ setTeamSearchVisibilityInboundAvailable galley tidA inboundVisibility void $ setRandomHandle brig teamATargetReindexedAfter - hFlush stdout - refreshIndex brig assertion brig (userId teamBSearcher) (userQualifiedId teamATarget) (fromName (userDisplayName teamATarget)) assertion brig (userId teamBSearcher) (userQualifiedId teamATargetReindexedAfter) (fromName (userDisplayName teamATargetReindexedAfter)) where @@ -522,7 +350,6 @@ testSearchTeamMemberAsOtherMemberExactHandle mgr brig galley inboundVisibility = (tidA, _, [teamATarget, teamATargetReindexedAfter]) <- createPopulatedBindingTeamWithNamesAndHandles brig 2 circumventSettingsOverride mgr $ setTeamSearchVisibilityInboundAvailable galley tidA inboundVisibility teamATargetReindexedAfter' <- setRandomHandle brig teamATargetReindexedAfter - refreshIndex brig let teamATargetHandle = fromMaybe (error "teamATarget must have a handle") (userHandle teamATarget) assertCanFind brig (userId teamASearcher) (userQualifiedId teamATarget) (fromHandle teamATargetHandle) assertCanFind brig (userId teamASearcher) (userQualifiedId teamATargetReindexedAfter) (fromHandle (fromJust (userHandle teamATargetReindexedAfter'))) @@ -531,14 +358,12 @@ testSearchTeamMemberAsSameMember :: (TestConstraints m) => Manager -> Brig -> Ga testSearchTeamMemberAsSameMember mgr brig galley inboundVisibility = do (tid, _, [teamASearcher, teamATarget]) <- createPopulatedBindingTeam brig 2 circumventSettingsOverride mgr $ setTeamSearchVisibilityInboundAvailable galley tid inboundVisibility - refreshIndex brig assertCanFind brig (userId teamASearcher) (userQualifiedId teamATarget) (fromName (userDisplayName teamATarget)) testSeachNonMemberAsTeamMember :: (TestConstraints m) => Brig -> m () testSeachNonMemberAsTeamMember brig = do nonTeamMember <- randomUser brig (_, _, [teamMember]) <- createPopulatedBindingTeam brig 1 - refreshIndex brig assertCanFind brig (userId teamMember) (userQualifiedId nonTeamMember) (fromName (userDisplayName nonTeamMember)) testSearchOrderingAsTeamMemberExactMatch :: (TestConstraints m) => Brig -> m () @@ -546,7 +371,6 @@ testSearchOrderingAsTeamMemberExactMatch brig = do searchedName <- randomName mapM_ (\(_ :: Int) -> createUser' True (fromName searchedName) brig) [0 .. 99] (_, _, [searcher, teamSearchee]) <- createPopulatedBindingTeamWithNames brig [Name "Searcher", searchedName] - refreshIndex brig result <- executeSearch brig (userId searcher) (fromName searchedName) let resultUserIds = contactQualifiedId <$> searchResults result liftIO $ @@ -559,7 +383,6 @@ testSearchOrderingAsTeamMemberPrefixMatch brig = do searchedName <- randomNameWithMaxLen 122 -- 6 characters for "suffix" mapM_ (\(i :: Int) -> createUser' True (fromName searchedName <> Text.pack (show i)) brig) [0 .. 99] (_, _, [searcher, teamSearchee]) <- createPopulatedBindingTeamWithNames brig [Name "Searcher", Name $ fromName searchedName <> "suffix"] - refreshIndex brig result <- executeSearch brig (userId searcher) (fromName searchedName) let resultUserIds = contactQualifiedId <$> searchResults result liftIO $ @@ -572,7 +395,6 @@ testSearchOrderingAsTeamMemberWorseNameMatch brig = do searchedTerm <- randomHandle _ <- createUser' True searchedTerm brig (_, _, [searcher, teamSearchee]) <- createPopulatedBindingTeamWithNames brig [Name "Searcher", Name (searchedTerm <> "Suffix")] - refreshIndex brig result <- executeSearch brig (userId searcher) searchedTerm let resultUserIds = contactQualifiedId <$> searchResults result liftIO $ @@ -586,7 +408,6 @@ testSearchOrderingAsTeamMemberWorseHandleMatch brig = do nonTeamSearchee <- createUser' True searchedTerm brig void $ putHandle brig (userId nonTeamSearchee) searchedTerm (_, _, [searcher, teamSearchee]) <- createPopulatedBindingTeamWithNames brig [Name "Searcher", Name (searchedTerm <> "Suffix")] - refreshIndex brig result <- executeSearch brig (userId searcher) searchedTerm let resultUserIds = contactQualifiedId <$> searchResults result liftIO $ do @@ -602,7 +423,6 @@ testSearchSameTeamOnly brig opts = do nonTeamMember' <- randomUser brig nonTeamMember <- setRandomHandle brig nonTeamMember' (_, _, [teamMember]) <- createPopulatedBindingTeam brig 1 - refreshIndex brig let newOpts = opts & Opt.settingsLens . Opt.searchSameTeamOnlyLens ?~ True withSettingsOverrides newOpts $ do assertCan'tFind brig (userId teamMember) (userQualifiedId nonTeamMember) (fromName (userDisplayName nonTeamMember)) @@ -627,7 +447,7 @@ testSearchTeamMemberAsSameMemberOutboundOnly brig ((_, teamAOwner, teamAMember), let teamAMemberHandle = fromMaybe (error "teamAMember must have a handle") (userHandle teamAMember) assertCanFind brig (userId teamAOwner) (userQualifiedId teamAMember) (fromName (userDisplayName teamAMember)) assertCanFind brig (userId teamAOwner) (userQualifiedId teamAMember) (fromHandle teamAMemberHandle) - let teamAOwnerHandle = fromMaybe (error "teamAMember must have a handle") (userHandle teamAOwner) + let teamAOwnerHandle = fromMaybe (error "teamAOwner must have a handle") (userHandle teamAOwner) assertCanFind brig (userId teamAMember) (userQualifiedId teamAOwner) (fromName (userDisplayName teamAOwner)) assertCanFind brig (userId teamAMember) (userQualifiedId teamAOwner) (fromHandle teamAOwnerHandle) @@ -644,7 +464,6 @@ testSearchWithDomain :: (TestConstraints m) => Brig -> m () testSearchWithDomain brig = do searcher <- randomUser brig searchee <- randomUser brig - refreshIndex brig let searcherId = userId searcher searcheeQid = userQualifiedId searchee searcheeName = fromName (userDisplayName searchee) @@ -675,427 +494,3 @@ testSearchOtherDomain opts brig = do } liftIO $ do assertEqual "The search request should get its result from federator" expectedResult searchResult - --- | Migration sequence: --- 1. A migration is planned, in this time brig writes to two indices --- 2. A migration is triggered, users are copied from old index to new index --- 3. Brig is configured to write to only the new index --- --- So, we have four time frames ("phases") in which a user could be created/updated: --- 1. Before migration is even planned --- 2. When brig is writing to both indices --- 3. While/After reindexing is done from old index to new index --- 4. After brig is writing to only the new index --- --- Note: The new index can be on another cluster of ES, but we have only one ES --- cluster. This test spins up a proxy server to pass requests to our only ES --- server. The proxy server ensures that only requests to the 'old' index go --- through. -testMigrationToNewIndex :: - (HasCallStack, TestConstraints m, MonadUnliftIO m) => - Opt.Opts -> - Brig -> - ES.Server -> - (Log.Logger -> Opt.Opts -> ES.IndexName -> ES.IndexName -> Int32 -> IO ()) -> - m () -testMigrationToNewIndex opts brig additionalIndexServer migrateIndexCommand = do - logger <- Log.create Log.StdOut - migrationIndexName <- ES.IndexName <$> randomHandle - -- running brig with `withSettingsOverride` to direct it to the expected index(es). it's - -- important to make both old and new index name/url explicit via `withESProxy`, or the - -- calls to `refreshIndex` in this test will interfere with parallel test runs of this test. - withESProxy logger opts migrationIndexName $ \oldESUrl oldESIndex -> - withESProxy logger (opts & Opt.elasticsearchLens . Opt.urlLens .~ additionalIndexServer) migrationIndexName $ - \newESUrl newESIndex -> do - let optsWithIndex :: Text -> Opt.Opts - optsWithIndex "old" = - opts - & Opt.elasticsearchLens . Opt.indexLens .~ oldESIndex - & Opt.elasticsearchLens . Opt.urlLens .~ oldESUrl - optsWithIndex "new" = - opts - & Opt.elasticsearchLens . Opt.indexLens .~ newESIndex - & Opt.elasticsearchLens . Opt.urlLens .~ newESUrl - optsWithIndex "both" = - optsWithIndex "old" - & Opt.elasticsearchLens . Opt.additionalWriteIndexLens ?~ newESIndex - & Opt.elasticsearchLens . Opt.additionalWriteIndexUrlLens ?~ newESUrl - -- 'additionalCaCertLens' needs to be added in order for brig to be able to reach both indices. - & Opt.elasticsearchLens . Opt.additionalCaCertLens .~ (opts ^. Opt.elasticsearchLens . Opt.caCertLens) - & Opt.elasticsearchLens . Opt.additionalInsecureSkipVerifyTlsLens .~ (opts ^. Opt.elasticsearchLens . Opt.insecureSkipVerifyTlsLens) - optsWithIndex "oldAccessToBoth" = - -- Configure only the old index. However, allow HTTP access to both - -- (such that jobs can create and fill the new one). - optsWithIndex "old" & Opt.elasticsearchLens . Opt.urlLens .~ additionalIndexServer - - -- Phase 1: Using old index only - (phase1NonTeamUser, teamOwner, phase1TeamUser1, phase1TeamUser2, tid) <- withSettingsOverrides (optsWithIndex "old") $ do - nonTeamUser <- randomUser brig - (tid, teamOwner, [teamUser1, teamUser2]) <- createPopulatedBindingTeam brig 2 - pure (nonTeamUser, teamOwner, teamUser1, teamUser2, tid) - - -- Phase 2: Using old index for search, writing to both indices, migrations have not run - (phase2NonTeamUser, phase2TeamUser) <- withSettingsOverrides (optsWithIndex "both") $ do - phase2NonTeamUser <- randomUser brig - phase2TeamUser <- inviteAndRegisterUser teamOwner tid brig - refreshIndex brig - - -- searching phase1 users should work - assertEventuallyCanFindByName brig phase1TeamUser1 phase1TeamUser2 - assertEventuallyCanFindByName brig phase1TeamUser1 phase1NonTeamUser - - -- searching phase2 users should work - assertEventuallyCanFindByName brig phase1TeamUser1 phase2NonTeamUser - assertEventuallyCanFindByName brig phase1TeamUser1 phase2TeamUser - - pure (phase2NonTeamUser, phase2TeamUser) - - withSettingsOverrides (optsWithIndex "new") $ do - -- Before migration the phase1 users shouldn't be found in the new index - assertEventuallyCan'tFindByName brig phase1TeamUser1 phase1TeamUser2 - assertEventuallyCan'tFindByName brig phase1TeamUser1 phase1NonTeamUser - - -- Before migration the phase2 users should be found in the new index - assertEventuallyCanFindByName brig phase1TeamUser1 phase2NonTeamUser - assertEventuallyCanFindByName brig phase1TeamUser1 phase2TeamUser - - -- Run Migrations - liftIO $ migrateIndexCommand logger (optsWithIndex "oldAccessToBoth") newESIndex migrationIndexName 5 - - -- Phase 3: Using old index for search, writing to both indices, migrations have run - (phase3NonTeamUser, phase3TeamUser) <- withSettingsOverrides (optsWithIndex "both") $ do - refreshIndex brig - phase3NonTeamUser <- randomUser brig - phase3TeamUser <- inviteAndRegisterUser teamOwner tid brig - refreshIndex brig - - -- searching phase1/2 users should work - assertEventuallyCanFindByName brig phase1TeamUser1 phase1TeamUser2 - assertEventuallyCanFindByName brig phase1TeamUser1 phase1NonTeamUser - assertEventuallyCanFindByName brig phase1TeamUser1 phase2TeamUser - assertEventuallyCanFindByName brig phase1TeamUser1 phase2NonTeamUser - - -- searching new phase3 should also work - assertEventuallyCanFindByName brig phase1TeamUser1 phase3NonTeamUser - assertEventuallyCanFindByName brig phase1TeamUser1 phase3TeamUser - pure (phase3NonTeamUser, phase3TeamUser) - - -- Phase 4: Using only new index - withSettingsOverrides (optsWithIndex "new") $ do - refreshIndex brig - -- Searching should work for phase1 users - assertEventuallyCanFindByName brig phase1TeamUser1 phase1TeamUser2 - assertEventuallyCanFindByName brig phase1TeamUser1 phase1NonTeamUser - - -- Searching should work for phase2 users - assertEventuallyCanFindByName brig phase1TeamUser1 phase2TeamUser - assertEventuallyCanFindByName brig phase1TeamUser1 phase2NonTeamUser - - -- Searching should work for phase3 users - assertEventuallyCanFindByName brig phase1TeamUser1 phase3NonTeamUser - assertEventuallyCanFindByName brig phase1TeamUser1 phase3TeamUser - -runReindexFromAnotherIndex :: Log.Logger -> Opt.Opts -> ES.IndexName -> ES.IndexName -> Int32 -> IO () -runReindexFromAnotherIndex logger opts newIndexName migrationIndexName _pageSize = - let esOldOpts :: Opt.ElasticSearchOpts = opts ^. Opt.elasticsearchLens - esOldConnectionSettings :: ESConnectionSettings = toESConnectionSettings esOldOpts migrationIndexName - reindexSettings = ReindexFromAnotherIndexSettings esOldConnectionSettings newIndexName 5 - in runCommand logger $ ReindexFromAnotherIndex reindexSettings - -runReindexFromDatabase :: - (ElasticSettings -> CassandraSettings -> PostgresSettings -> UserStorageLocation -> Endpoint -> Int32 -> Command) -> - Log.Logger -> - Opt.Opts -> - ES.IndexName -> - ES.IndexName -> - Int32 -> - IO () -runReindexFromDatabase syncCommand logger opts newIndexName migrationIndexName pageSize = - let esNewOpts :: Opt.ElasticSearchOpts = (opts ^. Opt.elasticsearchLens) & (Opt.indexLens .~ newIndexName) - esNewConnectionSettings :: ESConnectionSettings = toESConnectionSettings esNewOpts migrationIndexName - replicas = 2 - shards = 2 - refreshInterval = 5 - elasticSettings :: ElasticSettings = - IndexOpts.localElasticSettings - & IndexOpts.esConnection .~ esNewConnectionSettings - & IndexOpts.esIndexReplicas .~ ES.ReplicaCount replicas - & IndexOpts.esIndexShardCount .~ shards - & IndexOpts.esIndexRefreshInterval .~ refreshInterval - cassandraSettings :: CassandraSettings = - localCassandraSettings - & IndexOpts.cHost .~ (Text.unpack opts.cassandra.endpoint.host) - & IndexOpts.cPort .~ (opts.cassandra.endpoint.port) - & IndexOpts.cKeyspace .~ (C.Keyspace opts.cassandra.keyspace) - postgresSettings :: PostgresSettings = - brigOptsToPostgresSettings opts - endpoint :: Endpoint = opts.galley - in runCommand logger $ syncCommand elasticSettings cassandraSettings postgresSettings (UserStorageLocation opts.postgresMigration.user) endpoint pageSize - -toESConnectionSettings :: ElasticSearchOpts -> ES.IndexName -> ESConnectionSettings -toESConnectionSettings opts migrationIndexName = ESConnectionSettings {..} - where - toText (ES.Server url) = url - esServer = (fromRight undefined . URI.parseURI URI.strictURIParserOptions . Text.encodeUtf8 . toText) opts.url - esIndex = opts.index - esCaCert = opts.caCert - esInsecureSkipVerifyTls = opts.insecureSkipVerifyTls - esCredentials = opts.credentials - esMigrationIndexName = Just migrationIndexName - -withESProxy :: - (TestConstraints m, MonadUnliftIO m, HasCallStack) => - Log.Logger -> - Opt.Opts -> - ES.IndexName -> - (ES.Server -> ES.IndexName -> m a) -> - m a -withESProxy lg opts migrationIndexName f = do - indexName <- ES.IndexName <$> randomHandle - liftIO $ createEsIndexCommand lg opts indexName migrationIndexName - withESProxyOnly [indexName] opts $ flip f indexName - -mkElasticSettings :: Opts.Opts -> IndexName -> IndexName -> ElasticSettings -mkElasticSettings opts newIndexName migrationIndexName = - let esNewOpts = (opts ^. Opt.elasticsearchLens) & (Opt.indexLens .~ newIndexName) - replicas = 2 - shards = 2 - refreshInterval = 5 - esSettings = - IndexOpts.localElasticSettings - & IndexOpts.esConnection .~ toESConnectionSettings esNewOpts migrationIndexName - & IndexOpts.esIndexReplicas .~ ES.ReplicaCount replicas - & IndexOpts.esIndexShardCount .~ shards - & IndexOpts.esIndexRefreshInterval .~ refreshInterval - in esSettings - -createEsIndexCommand :: Log.Logger -> Opt.Opts -> ES.IndexName -> ES.IndexName -> IO () -createEsIndexCommand logger opts newIndexName migrationIndexName = - let esSettings = mkElasticSettings opts newIndexName migrationIndexName - in runCommand logger $ Create esSettings opts.galley - --- | Gives a URL to a HTTP proxy server to the continuation. The proxy is only --- configured for ES calls for the given @indexNames@ (and some other ES --- specific endpoints.) -withESProxyOnly :: (TestConstraints m, MonadUnliftIO m, HasCallStack) => [ES.IndexName] -> Opt.Opts -> (ES.Server -> m a) -> m a -withESProxyOnly indexNames opts f = do - mgr <- liftIO $ initHttpManagerWithTLSConfig opts.elasticsearch.insecureSkipVerifyTls opts.elasticsearch.caCert - (proxyPort, sock) <- liftIO Warp.openFreePort - bracket - (async $ liftIO $ Warp.runSettingsSocket Warp.defaultSettings sock $ indexProxyServer indexNames opts mgr) - cancel - (\_ -> f (ES.Server ("http://localhost:" <> Text.pack (show proxyPort)))) - --- | Create a `Wai.Application` that acts as a proxy to ElasticSearch. Requests --- are only forwarded for specified index names (and some technical endpoints.) -indexProxyServer :: [ES.IndexName] -> Opt.Opts -> Manager -> Wai.Application -indexProxyServer idxs opts mgr = - let toUri (ES.Server url) = either (error . show) id $ URI.parseURI URI.strictURIParserOptions (Text.encodeUtf8 url) - proxyURI = toUri (Opts.url (Opts.elasticsearch opts)) - proxyToHost = URI.hostBS . URI.authorityHost . fromMaybe (error "No Host") . URI.uriAuthority $ proxyURI - proxyToPort = URI.portNumber . fromMaybe (URI.Port 9200) . URI.authorityPort . fromMaybe (error "No Host") . URI.uriAuthority $ proxyURI - forwardRequest = Wai.WPRProxyDestSecure (Wai.ProxyDest proxyToHost proxyToPort) - denyRequest req = - Wai.WPRResponse - ( Wai.responseLBS HTTP.status400 [] $ - "Refusing to proxy to path=" <> cs (Wai.rawPathInfo req) <> ". Proxy configured for indices: " <> cs (show idxs) - ) - proxyApp req - | (headMay (Wai.pathInfo req)) `elem` [Just "_reindex", Just "_tasks"] = - forwardRequest - | (any (\(ES.IndexName idx) -> (headMay (Wai.pathInfo req) == Just idx)) idxs) = - forwardRequest - | otherwise = - denyRequest req - in waiProxyTo (pure . proxyApp) Wai.defaultOnExc mgr - -testWithBothIndices :: Opt.Opts -> Manager -> TestName -> WaiTest.Session a -> TestTree -testWithBothIndices opts mgr name f = do - testGroup - name - [ test mgr "new-index" $ withSettingsOverrides opts f, - test mgr "old-index" $ withOldIndex opts defaultMigrationIndexName f - ] - -testWithBothIndicesAndOpts :: Opt.Opts -> Manager -> TestName -> ((HasCallStack) => Opt.Opts -> Http ()) -> TestTree -testWithBothIndicesAndOpts opts mgr name f = - testGroup - name - [ test mgr "new-index" (f opts), - test mgr "old-index" $ do - (newOpts, indexName) <- optsForOldIndex opts defaultMigrationIndexName - f newOpts <* deleteIndex opts indexName - ] - -withOldIndex :: (MonadIO m, HasCallStack) => Opt.Opts -> ES.IndexName -> WaiTest.Session a -> m a -withOldIndex opts migrationIndexName f = do - lg <- Log.create Log.StdOut - indexName <- randomHandle - createIndexWithMapping lg opts migrationIndexName indexName oldMapping - let newOpts = opts & Opt.elasticsearchLens . Opt.indexLens .~ (ES.IndexName indexName) - withSettingsOverrides newOpts f <* deleteIndex opts indexName - -optsForOldIndex :: (MonadIO m, HasCallStack) => Opt.Opts -> ES.IndexName -> m (Opt.Opts, Text) -optsForOldIndex opts migrationIndexName = do - lg <- Log.create Log.StdOut - indexName <- randomHandle - createIndexWithMapping lg opts migrationIndexName indexName oldMapping - pure (opts & Opt.elasticsearchLens . Opt.indexLens .~ (ES.IndexName indexName), indexName) - -createIndexWithMapping :: (MonadIO m, HasCallStack) => Log.Logger -> Opt.Opts -> ES.IndexName -> Text -> Value -> m () -createIndexWithMapping lg opts migrationIndexName name val = do - let indexName = ES.IndexName name - let elasticSettings = mkElasticSettings opts indexName migrationIndexName - settings = mkCreateIndexSettings elasticSettings - conn = elasticSettings ^. esConnection - - e <- liftIO $ initIndex lg conn opts.galley - runIndexIO e $ createIndexWithoutMapping True settings - mappingReply <- runBH opts $ ES.putNamedMapping indexName mappingName val - unless (ES.isCreated mappingReply || ES.isSuccess mappingReply) $ do - liftIO $ assertFailure $ "failed to create mapping: " <> show name <> ", error: " <> show mappingReply - --- | This doesn't fail if ES returns error because we don't really want to fail the tests for this -deleteIndex :: (MonadIO m, HasCallStack) => Opt.Opts -> Text -> m () -deleteIndex opts name = do - let indexName = ES.IndexName name - void $ runBH opts $ ES.deleteIndex indexName - -runBH :: (MonadIO m, HasCallStack) => Opt.Opts -> ES.BH m a -> m a -runBH opts action = do - let (ES.Server esURL) = opts ^. Opt.elasticsearchLens . Opt.urlLens - mgr <- liftIO $ initHttpManagerWithTLSConfig opts.elasticsearch.insecureSkipVerifyTls opts.elasticsearch.caCert - let bEnv = mkBHEnv esURL mgr - ES.runBH bEnv action - --- | This was generated from Brig.User.Search.Index.indexMapping at commit 18885bc --- how to generate: --- - run `cabal repl brig` --- - ghci> import Brig.User.Search.Index --- ghci> import Data.Aeson --- ghci> import qualified Data.ByteString.Lazy.Char8 as BL --- ghci> BL.putStrLn $ encode indexMapping --- - copy the output, format and paste it here -oldMapping :: Value -oldMapping = - fromJust $ - decode - [r| -{ - "dynamic": false, - "properties": { - "accent_id": { - "index": false, - "store": false, - "type": "byte" - }, - "account_status": { - "index": true, - "store": false, - "type": "keyword" - }, - "created_at": { - "index": false, - "store": false, - "type": "date" - }, - "email": { - "fields": { - "keyword": { - "type": "keyword" - }, - "prefix": { - "analyzer": "prefix_index", - "search_analyzer": "prefix_search", - "type": "text" - } - }, - "index": true, - "store": false, - "type": "text" - }, - "email_unvalidated": { - "index": false, - "store": false, - "type": "text" - }, - "handle": { - "fields": { - "keyword": { - "type": "keyword" - }, - "prefix": { - "analyzer": "prefix_index", - "search_analyzer": "prefix_search", - "type": "text" - } - }, - "index": true, - "store": false, - "type": "text" - }, - "managed_by": { - "index": true, - "store": false, - "type": "keyword" - }, - "name": { - "index": false, - "store": false, - "type": "keyword" - }, - "normalized": { - "fields": { - "prefix": { - "analyzer": "prefix_index", - "search_analyzer": "prefix_search", - "type": "text" - } - }, - "index": true, - "store": false, - "type": "text" - }, - "role": { - "index": true, - "store": false, - "type": "keyword" - }, - "saml_idp": { - "index": false, - "store": false, - "type": "keyword" - }, - "scim_external_id": { - "index": false, - "store": false, - "type": "keyword" - }, - "search_visibility_inbound": { - "index": true, - "store": false, - "type": "keyword" - }, - "sso": { - "properties": { - "issuer": { - "index": false, - "store": false, - "type": "keyword" - }, - "nameid": { - "index": false, - "store": false, - "type": "keyword" - } - }, - "type": "nested" - }, - "team": { - "index": true, - "store": false, - "type": "keyword" - } - } -} -|] diff --git a/services/brig/test/integration/API/Search/Util.hs b/services/brig/test/integration/API/Search/Util.hs index 9a7bdba8d44..259951de902 100644 --- a/services/brig/test/integration/API/Search/Util.hs +++ b/services/brig/test/integration/API/Search/Util.hs @@ -19,8 +19,7 @@ module API.Search.Util where import Bilge import Bilge.Assert -import Control.Monad.Catch (MonadCatch, MonadMask) -import Control.Retry +import Control.Monad.Catch (MonadCatch) import Data.ByteString.Conversion (toByteString') import Data.ByteString.Conversion.To (toByteString) import Data.Domain (Domain) @@ -29,9 +28,7 @@ import Data.Qualified (Qualified (..)) import Data.Range (Range) import Data.String.Conversions import Data.Text.Encoding (encodeUtf8) -import Database.Bloodhound qualified as ES import Imports -import Network.HTTP.Client qualified as HTTP import Test.Tasty.HUnit import Util import Wire.API.User @@ -61,15 +58,6 @@ searchRequest brig self q maybeDomain maybeSize = do . maybe id (queryItem "size" . toByteString') maybeSize ) --- | ES is only refreshed occasionally; we don't want to wait for that in tests. -refreshIndex :: (MonadCatch m, MonadIO m, MonadHttp m, HasCallStack) => Brig -> m () -refreshIndex brig = - post (brig . path "/i/index/refresh") !!! const 200 === statusCode - -reindex :: (MonadCatch m, MonadIO m, MonadHttp m, HasCallStack) => Brig -> m () -reindex brig = - post (brig . path "/i/index/reindex") !!! const 200 === statusCode - assertCanFindByName :: (MonadCatch m, MonadIO m, MonadHttp m, HasCallStack) => Brig -> User -> User -> m () assertCanFindByName brig self expected = assertCanFind brig (userId self) (userQualifiedId expected) (fromName $ userDisplayName expected) @@ -78,15 +66,6 @@ assertCan'tFindByName :: (MonadCatch m, MonadIO m, MonadHttp m, HasCallStack) => assertCan'tFindByName brig self expected = assertCan'tFind brig (userId self) (userQualifiedId expected) (fromName $ userDisplayName expected) -ourRetryPol :: Int -> RetryPolicy -ourRetryPol to = limitRetriesByCumulativeDelay (to * 1_000_000) (exponentialBackoff 50000) - -assertEventuallyCanFindByName :: (MonadMask m, MonadIO m, MonadHttp m, HasCallStack) => Brig -> User -> User -> m () -assertEventuallyCanFindByName brig self expected = recoverAll (ourRetryPol 5) (\_ -> assertCanFindByName brig self expected) - -assertEventuallyCan'tFindByName :: (MonadMask m, MonadIO m, MonadHttp m, HasCallStack) => Brig -> User -> User -> m () -assertEventuallyCan'tFindByName brig self expected = recoverAll (ourRetryPol 5) (\_ -> assertCan'tFindByName brig self expected) - assertCanFind :: (MonadCatch m, MonadIO m, MonadHttp m, HasCallStack) => Brig -> UserId -> Qualified UserId -> Text -> m () assertCanFind brig self expected q = do r <- searchResults <$> executeSearch brig self q @@ -154,6 +133,3 @@ executeTeamUserSearchWithMaybeState brig teamid self mbSearchText mRoleFilter mS === statusCode responseJsonError r -mkBHEnv :: Text -> HTTP.Manager -> ES.BHEnv -mkBHEnv url mgr = do - (ES.mkBHEnv (ES.Server url) mgr) {ES.bhRequestHook = ES.basicAuthHook (ES.EsUsername "elastic") (ES.EsPassword "changeme")} diff --git a/services/brig/test/integration/API/Team.hs b/services/brig/test/integration/API/Team.hs index 66e1ead9e28..f19c3480282 100644 --- a/services/brig/test/integration/API/Team.hs +++ b/services/brig/test/integration/API/Team.hs @@ -22,7 +22,6 @@ module API.Team ) where -import API.Search.Util qualified as SearchUtil import API.Team.Util import API.User.Util as Util import Bilge hiding (accept, head, timeout) @@ -139,14 +138,12 @@ testTeamSizePublic brig = do testTeamSize :: Brig -> (TeamId -> UserId -> Request -> Request) -> Http () testTeamSize brig req = do (tid, owner, _) <- createPopulatedBindingTeam brig 10 - SearchUtil.refreshIndex brig -- 10 Team Members and an admin let expectedSize = 11 assertSize tid owner expectedSize -- Even suspended teams should report correct size suspendTeam brig tid !!! const 200 === statusCode - SearchUtil.refreshIndex brig assertSize tid owner expectedSize where assertSize :: (HasCallStack) => TeamId -> UserId -> Natural -> Http () @@ -692,7 +689,6 @@ testInvitationTooManyMembers brig galley (TeamSizeLimit limit) = do (creator, tid) <- createUserWithTeam brig pooledForConcurrentlyN_ 16 [1 .. limit - 1] $ \_ -> do void $ createTeamMember brig galley creator tid fullPermissions - SearchUtil.refreshIndex brig let invite email = stdInvitationRequest email email <- randomEmail inv :: Invitation <- responseJsonError =<< postInvitation brig tid creator (invite email) diff --git a/services/brig/test/integration/API/TeamUserSearch.hs b/services/brig/test/integration/API/TeamUserSearch.hs index d6ddbecfbf9..bda8a7002b0 100644 --- a/services/brig/test/integration/API/TeamUserSearch.hs +++ b/services/brig/test/integration/API/TeamUserSearch.hs @@ -17,14 +17,12 @@ module API.TeamUserSearch (tests) where -import API.Search (testWithBothIndices) -import API.Search.Util (executeTeamUserSearch, executeTeamUserSearchWithMaybeState, refreshIndex) +import API.Search.Util (executeTeamUserSearch, executeTeamUserSearchWithMaybeState) import API.Team.Util (createPopulatedBindingTeamWithNamesAndHandles) import API.User.Util (initiateEmailUpdateAutoActivate) import Bilge (Manager, MonadHttp) import Brig.Options qualified as Opt import Control.Monad.Catch (MonadCatch) -import Control.Retry () import Data.ByteString.Conversion (toByteString) import Data.Handle (fromHandle) import Data.Id (TeamId, UserId) @@ -34,7 +32,7 @@ import Imports import System.Random.Shuffle (shuffleM) import Test.Tasty (TestTree, testGroup) import Test.Tasty.HUnit (assertBool, assertEqual, (@?=)) -import Util (Brig, Galley, randomEmail) +import Util (Brig, Galley, randomEmail, test) import Wire.API.User (User (..), userEmail, userId) import Wire.API.User.Identity hiding (toByteString) import Wire.API.User.Search @@ -42,22 +40,21 @@ import Wire.API.User.Search type TestConstraints m = (MonadFail m, MonadCatch m, MonadIO m, MonadHttp m) tests :: Opt.Opts -> Manager -> Galley -> Brig -> IO TestTree -tests opts mgr _galley brig = do +tests _opts mgr _galley brig = do pure $ testGroup "teams user search" $ - [ testWithBothIndices opts mgr "can find user by email" (testSearchByEmailSameTeam brig), - testWithBothIndices opts mgr "empty query returns the whole team sorted" (testEmptyQuerySorted brig), - testWithBothIndices opts mgr "sorting by some properties works" (testSort brig), - testWithBothIndices opts mgr "call to search with remaining properties succeeds" (testSortCallSucceeds brig), - testWithBothIndices opts mgr "query with paging state" (testEmptyQuerySortedWithPagination brig) + [ test mgr "can find user by email" (testSearchByEmailSameTeam brig), + test mgr "empty query returns the whole team sorted" (testEmptyQuerySorted brig), + test mgr "sorting by some properties works" (testSort brig), + test mgr "call to search with remaining properties succeeds" (testSortCallSucceeds brig), + test mgr "query with paging state" (testEmptyQuerySortedWithPagination brig) ] -testSearchByEmail :: (HasCallStack, TestConstraints m) => Brig -> m (TeamId, UserId, User) -> Bool -> m () +testSearchByEmail :: (TestConstraints m) => Brig -> m (TeamId, UserId, User) -> Bool -> m () testSearchByEmail brig mkSearcherAndSearchee canFind = do (tid, searcher, searchee) <- mkSearcherAndSearchee eml <- randomEmail _ <- initiateEmailUpdateAutoActivate brig eml (userId searchee) - refreshIndex brig let check = if canFind then assertTeamUserSearchCanFind else assertTeamUserSearchCannotFind check brig tid searcher (userId searchee) (fromEmail eml) @@ -87,7 +84,6 @@ assertTeamUserSearchCannotFind brig teamid self expected q = do testEmptyQuerySorted :: (TestConstraints m) => Brig -> m () testEmptyQuerySorted brig = do (tid, userId -> ownerId, users) <- createPopulatedBindingTeamWithNamesAndHandles brig 4 - refreshIndex brig r <- searchResults <$> executeTeamUserSearch brig tid ownerId (Just "") Nothing Nothing Nothing let creationDates = fmap teamContactCreatedAt r liftIO $ @@ -102,14 +98,13 @@ testSort brig = do (tid, userId -> ownerId, usersImplicitOrder) <- createPopulatedBindingTeamWithNamesAndHandles brig 4 -- Shuffle here to guard against false positives in this test. -- This might happen due to buggy data generation, where all users share the same value in the sort property, - -- resulting in an implicit order, which might coincide in the DB and ES, resulting in false positive test + -- resulting in an implicit order, which might coincide in the store and + -- the search result, resulting in false positive test -- result. users <- liftIO $ shuffleM usersImplicitOrder - refreshIndex brig let sortByProperty' :: (TestConstraints m, Ord a) => TeamUserSearchSortBy -> (User -> a) -> TeamUserSearchSortOrder -> m () sortByProperty' = sortByProperty tid users ownerId for_ [SortOrderAsc, SortOrderDesc] $ \sortOrder -> do - -- FUTUREWORK: Test SortByRole when role is available in index sortByProperty' SortByEmail userEmail sortOrder sortByProperty' SortByName userDisplayName sortOrder sortByProperty' SortByHandle (fmap fromHandle . userHandle) sortOrder @@ -132,7 +127,6 @@ testSort brig = do testSortCallSucceeds :: (TestConstraints m) => Brig -> m () testSortCallSucceeds brig = do (tid, userId -> ownerId, users) <- createPopulatedBindingTeamWithNamesAndHandles brig 4 - refreshIndex brig let n = length users + 1 for_ [SortByManagedBy, SortBySAMLIdp] $ \tuSortBy -> do r <- searchResults <$> executeTeamUserSearch brig tid ownerId Nothing Nothing (Just tuSortBy) (Just SortOrderAsc) @@ -141,7 +135,6 @@ testSortCallSucceeds brig = do testEmptyQuerySortedWithPagination :: (TestConstraints m) => Brig -> m () testEmptyQuerySortedWithPagination brig = do (tid, userId -> ownerId, _) <- createPopulatedBindingTeamWithNamesAndHandles brig 20 - refreshIndex brig let teamUserSearch mPs = executeTeamUserSearchWithMaybeState brig tid ownerId (Just "") Nothing Nothing Nothing (Just $ unsafeRange 10) mPs searchResultFirst10 <- teamUserSearch Nothing searchResultNext10 <- teamUserSearch (searchPagingState searchResultFirst10) diff --git a/services/brig/test/integration/API/User/Account.hs b/services/brig/test/integration/API/User/Account.hs index da547287827..382b748906c 100644 --- a/services/brig/test/integration/API/User/Account.hs +++ b/services/brig/test/integration/API/User/Account.hs @@ -348,7 +348,6 @@ testCreateUserAnon brig galley = do assertOnlySelfConversations galley uid -- should not appear in search suid <- userId <$> randomUser brig - Search.refreshIndex brig Search.assertCan'tFind brig suid quid "Mr. Pink" testCreateUserPending :: Opt.Opts -> Brig -> Http () @@ -389,7 +388,6 @@ testCreateUserPending _ brig = do pure $! isNothing (userIdentity (selfUser self)) -- should not appear in search suid <- userId <$> randomUser brig - Search.refreshIndex brig Search.assertCan'tFind brig suid quid "Mr. Pink" -- The testCreateUserConflict test conforms to the following testing standards: @@ -946,7 +944,6 @@ testUserUpdate brig cannon userJournalWatcher = do . responseJsonMaybe -- should appear in search by 'newName' suid <- userId <$> randomUser brig - Search.refreshIndex brig Search.assertCanFind brig suid aliceQ (fromName aliceNewName) -- This tests the behavior of `/i/self/email` instead of `/self/email` or @@ -1039,13 +1036,11 @@ testSuspendUser brig = do chkStatus brig uid Suspended -- should not appear in search suid <- userId <$> randomUser brig - Search.refreshIndex brig Search.assertCan'tFind brig suid quid (fromName (userDisplayName u)) -- re-activate setStatus brig uid Active chkStatus brig uid Active -- should appear in search again - Search.refreshIndex brig Search.assertCanFind brig suid quid (fromName (userDisplayName u)) testGetByIdentity :: Brig -> Http () @@ -1545,7 +1540,6 @@ execAndAssertUserDeletion brig cannon u hdl others userJournalWatcher execDelete const (Just "invalid-credentials") === fmap Error.label . responseJsonMaybe -- Deleted flag appears in self profile; email, handle and picture are gone get (brig . path "/self" . zUser uid) !!! assertDeletedProfileSelf - Search.refreshIndex brig -- Does not appear in search; public profile shows the user as deleted forM_ others $ \usr -> do get (apiVersion "v1" . brig . paths ["users", toByteString' uid] . zUser usr) !!! assertDeletedProfilePublic diff --git a/services/brig/test/integration/API/User/Handles.hs b/services/brig/test/integration/API/User/Handles.hs index 289cb8113bd..e1626b4f401 100644 --- a/services/brig/test/integration/API/User/Handles.hs +++ b/services/brig/test/integration/API/User/Handles.hs @@ -113,7 +113,6 @@ testHandleUpdate brig cannon = do const 409 === statusCode const (Just "handle-exists") === fmap Error.label . responseJsonMaybe -- The owner appears by that handle in search - Search.refreshIndex brig Search.assertCanFind brig uid2 quid hdl -- Change the handle again, thus freeing the old handle hdl2 <- randomHandle @@ -123,7 +122,6 @@ testHandleUpdate brig cannon = do Bilge.head (brig . paths ["handles", toByteString' hdl] . zUser uid) !!! const 404 === statusCode -- The owner appears by the new handle in search - Search.refreshIndex brig Search.assertCan'tFind brig uid2 quid hdl Search.assertCanFind brig uid2 quid hdl2 -- Other users can immediately claim the old handle (the claim of the old handle is diff --git a/services/brig/test/integration/Federation/Util.hs b/services/brig/test/integration/Federation/Util.hs index c2cb0fbf558..4dc423d6714 100644 --- a/services/brig/test/integration/Federation/Util.hs +++ b/services/brig/test/integration/Federation/Util.hs @@ -43,7 +43,6 @@ import Data.Map.Strict qualified as Map import Data.Qualified (Qualified (..)) import Data.Text qualified as T import Data.Text qualified as Text -import Database.Bloodhound qualified as ES import Federator.MockServer qualified as Mock import Foreign.C.Error (Errno (..), eCONNREFUSED) import GHC.IO.Exception (IOException (ioe_errno)) diff --git a/services/brig/test/integration/Index/Create.hs b/services/brig/test/integration/Index/Create.hs deleted file mode 100644 index 54a847735ee..00000000000 --- a/services/brig/test/integration/Index/Create.hs +++ /dev/null @@ -1,150 +0,0 @@ --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Index.Create where - -import API.Search.Util (mkBHEnv) -import Brig.App (initHttpManagerWithTLSConfig) -import Brig.Index.Eval qualified as IndexEval -import Brig.Index.Options -import Brig.Index.Options qualified as IndexOpts -import Brig.Options (Opts (galley)) -import Brig.Options qualified as BrigOpts -import Control.Lens ((.~)) -import Control.Lens.Combinators (none) -import Data.Text qualified as Text -import Data.Text.Encoding qualified as Text -import Database.Bloodhound qualified as ES -import Imports -import System.Logger.Class qualified as Log -import System.Random as Random -import Test.Tasty -import Test.Tasty.HUnit -import URI.ByteString - --- FUTUREWORK: Make Bloodhound capable of getting a mapping and add test here to make sure --- the intended mapping is set on the created index. Also add a test to ensure when the index --- already exists, the existing mapping is not updated. -spec :: BrigOpts.Opts -> IO TestTree -spec brigOpts = - pure $ - testGroup - "brig-index create" - [ testCase "should create an index when it is not present" $ testCreateIndexWhenNotPresent brigOpts, - testCase "should not update anything when index already exists" $ testCreateIndexWhenPresent brigOpts - ] - -testCreateIndexWhenNotPresent :: BrigOpts.Opts -> Assertion -testCreateIndexWhenNotPresent brigOpts = do - let (ES.Server esURL) = brigOpts.elasticsearch.url - case parseURI strictURIParserOptions (Text.encodeUtf8 esURL) of - Left e -> fail $ "Invalid ES URL: " <> show esURL <> "\nerror: " <> show e - Right esURI -> do - indexName <- ES.IndexName . Text.pack <$> replicateM 20 (Random.randomRIO ('a', 'z')) - let replicas = 2 - shards = 2 - refreshInterval = 5 - let connSettings = - ESConnectionSettings - { esServer = esURI, - esIndex = indexName, - esCaCert = brigOpts.elasticsearch.caCert, - esInsecureSkipVerifyTls = brigOpts.elasticsearch.insecureSkipVerifyTls, - esCredentials = brigOpts.elasticsearch.credentials, - esMigrationIndexName = Nothing - } - let esSettings = - IndexOpts.localElasticSettings - & IndexOpts.esConnection .~ connSettings - & IndexOpts.esIndexReplicas .~ ES.ReplicaCount replicas - & IndexOpts.esIndexShardCount .~ shards - & IndexOpts.esIndexRefreshInterval .~ refreshInterval - devNullLogger <- Log.create (Log.Path "/dev/null") - IndexEval.runCommand devNullLogger (IndexOpts.Create esSettings (galley brigOpts)) - mgr <- liftIO $ initHttpManagerWithTLSConfig connSettings.esInsecureSkipVerifyTls connSettings.esCaCert - let bEnv = (mkBHEnv esURL mgr) {ES.bhRequestHook = ES.basicAuthHook (ES.EsUsername "elastic") (ES.EsPassword "changeme")} - ES.runBH bEnv $ do - indexExists <- ES.indexExists indexName - lift $ - assertBool "Index should exist" indexExists - eitherIndexSettings <- ES.getIndexSettings indexName - lift $ do - case eitherIndexSettings of - Left err -> fail $ "Failed to fetch index settings with error: " <> show err - Right indexSettings -> do - assertEqual "Shard count should be set" (ES.ShardCount replicas) (ES.indexShards . ES.sSummaryFixedSettings $ indexSettings) - assertEqual "Replica count should be set" (ES.ReplicaCount replicas) (ES.indexReplicas . ES.sSummaryFixedSettings $ indexSettings) - -- Check if the `RefreshInterval` is part of `UpdateIndexSettings`. - -- There can be more settings. E.g. ElasticSearch 7 has these: - -- `[RefreshInterval 5s, RoutingAllocationInclude (NodeAttrFilter {nodeAttrFilterName = NodeAttrName "_tier_preference", nodeAttrFilterValues = "data_content" :| []} :| [])]` - assertBool "Refresh interval should be set" $ (ES.RefreshInterval refreshInterval) `elem` (ES.sSummaryUpdateable indexSettings) - -testCreateIndexWhenPresent :: BrigOpts.Opts -> Assertion -testCreateIndexWhenPresent brigOpts = do - let (ES.Server esURL) = brigOpts.elasticsearch.url - case parseURI strictURIParserOptions (Text.encodeUtf8 esURL) of - Left e -> fail $ "Invalid ES URL: " <> show esURL <> "\nerror: " <> show e - Right esURI -> do - indexName <- ES.IndexName . Text.pack <$> replicateM 20 (Random.randomRIO ('a', 'z')) - let replicas = 2 - shards = 2 - refreshInterval = 5 - connSettings = - ESConnectionSettings - { esServer = esURI, - esIndex = indexName, - esCaCert = brigOpts.elasticsearch.caCert, - esInsecureSkipVerifyTls = brigOpts.elasticsearch.insecureSkipVerifyTls, - esCredentials = brigOpts.elasticsearch.credentials, - esMigrationIndexName = Nothing - } - esSettings = - IndexOpts.localElasticSettings - & IndexOpts.esConnection .~ connSettings - & IndexOpts.esIndexReplicas .~ ES.ReplicaCount replicas - & IndexOpts.esIndexShardCount .~ shards - & IndexOpts.esIndexRefreshInterval .~ refreshInterval - mgr <- liftIO $ initHttpManagerWithTLSConfig connSettings.esInsecureSkipVerifyTls connSettings.esCaCert - let bEnv = (mkBHEnv esURL mgr) {ES.bhRequestHook = ES.basicAuthHook (ES.EsUsername "elastic") (ES.EsPassword "changeme")} - ES.runBH bEnv $ do - _ <- ES.createIndex (ES.IndexSettings (ES.ShardCount 1) (ES.ReplicaCount 1)) indexName - indexExists <- ES.indexExists indexName - lift $ - assertBool "Index should exist" indexExists - devNullLogger <- Log.create (Log.Path "/dev/null") - IndexEval.runCommand devNullLogger (IndexOpts.Create esSettings (galley brigOpts)) - ES.runBH bEnv $ do - indexExists <- ES.indexExists indexName - lift $ - assertBool "Index should still exist" indexExists - eitherIndexSettings <- ES.getIndexSettings indexName - lift $ do - case eitherIndexSettings of - Left err -> fail $ "Failed to fetch index settings with error: " <> show err - Right indexSettings -> do - assertEqual "Shard count should not be updated" (ES.ShardCount 1) (ES.indexShards . ES.sSummaryFixedSettings $ indexSettings) - assertEqual "Replica count should not be updated" (ES.ReplicaCount 1) (ES.indexReplicas . ES.sSummaryFixedSettings $ indexSettings) - -- Ensure that the `RefreshInterval` is not part of `UpdateIndexSettings`. - -- There can be more settings. E.g. ElasticSearch 7 has this by default: - -- `[RoutingAllocationInclude (NodeAttrFilter {nodeAttrFilterName = NodeAttrName "_tier_preference", nodeAttrFilterValues = "data_content" :| []} :| [])]` - assertBool "Refresh interval should not be updated" $ - none - ( \case - ES.RefreshInterval _ -> True - _otherwise -> False - ) - (ES.sSummaryUpdateable indexSettings) diff --git a/services/brig/test/integration/Run.hs b/services/brig/test/integration/Run.hs index 205c48714df..088d1794520 100644 --- a/services/brig/test/integration/Run.hs +++ b/services/brig/test/integration/Run.hs @@ -41,10 +41,8 @@ import Data.Aeson import Data.ByteString.Char8 qualified as B8 import Data.Text.Encoding (encodeUtf8) import Data.Yaml (decodeFileEither) -import Database.Bloodhound.Types qualified as ES import Federation.End2end qualified import Imports hiding (local) -import Index.Create qualified import Network.HTTP.Client qualified as HTTP import Network.URI (pathSegments) import OpenSSL (withOpenSSL) @@ -98,12 +96,7 @@ data Config = Config -- external provider provider :: Provider.Config, -- for federation - backendTwo :: BackendConf, - -- The additional ElasticSearch server is configured like the main one - -- (regarding passwords, certificated, etc.). Thus, we only need the - -- additional endpoint and can deduce the rest from the main instance's - -- configuration. - additionalElasticSearch :: ES.Server + backendTwo :: BackendConf } deriving (Show, Generic) @@ -136,12 +129,11 @@ runTests iConf brigOpts otherArgs = do mUserJournalWatcher <- for (Opts.userJournalQueue awsOpts) $ SQS.watchSQSQueue (view AWS.amazonkaEnv awsEnv) userApi <- User.tests brigOpts fedBrigClient mg b c ch g n awsEnv db mUserJournalWatcher providerApi <- Provider.tests localDomain brigOpts (provider iConf) mg db b c g n - searchApis <- Search.tests brigOpts iConf.additionalElasticSearch mg g b + searchApis <- Search.tests brigOpts mg g b teamApis <- Team.tests brigOpts mg n b c g mUserJournalWatcher turnApi <- Calling.tests mg b brigOpts turnFile turnFileV2 metricsApi <- Metrics.tests mg brigOpts b settingsApi <- Settings.tests brigOpts mg b g - createIndex <- Index.Create.spec brigOpts browseTeam <- TeamUserSearch.tests brigOpts mg g b federationEnd2End <- Federation.End2end.spec brigOpts mg b g ch c f brigTwo galleyTwo ch2 cannonTwo federationEndpoints <- API.Federation.tests mg brigOpts b fedBrigClient @@ -159,7 +151,6 @@ runTests iConf brigOpts otherArgs = do turnApi, metricsApi, settingsApi, - createIndex, browseTeam, federationEndpoints, smtp, diff --git a/services/brig/test/integration/Util.hs b/services/brig/test/integration/Util.hs index 75520d234a1..06cffe698f9 100644 --- a/services/brig/test/integration/Util.hs +++ b/services/brig/test/integration/Util.hs @@ -973,12 +973,12 @@ randomName = randomNameWithMaxLen 128 -- | For testing purposes we restrict ourselves to code points in the -- Basic Multilingual Plane that are considered to be numbers, letters, -- punctuation or symbols and ensure the name starts with a "letter". --- That is in order for the name to be searchable at all, since the standard --- ElasticSearch tokenizer may otherwise produce an empty list of tokens, --- e.g. if the name is entirely made of characters from categories that --- the standard tokenizer considers as word boundaries (or which are --- simply unassigned code points), yielding no tokens to match and thus --- no results in search queries. +-- That is in order for the name to be searchable at all, since the search +-- tokenizer may otherwise produce an empty list of tokens, e.g. if the name +-- is entirely made of characters from categories that the tokenizer +-- considers as word boundaries (or which are simply unassigned code +-- points), yielding no tokens to match and thus no results in search +-- queries. randomNameWithMaxLen :: (MonadIO m) => Word -> m Name randomNameWithMaxLen maxLen = liftIO $ do len <- randomRIO (2, maxLen) diff --git a/services/galley/src/Galley/API/Teams.hs b/services/galley/src/Galley/API/Teams.hs index 181a720ae8f..3d2c39ca0db 100644 --- a/services/galley/src/Galley/API/Teams.hs +++ b/services/galley/src/Galley/API/Teams.hs @@ -113,7 +113,6 @@ import Wire.API.User qualified as U import Wire.API.User.Search import Wire.BoundedQueue qualified as E import Wire.BrigAPIAccess -import Wire.BrigAPIAccess qualified as Brig import Wire.BrigAPIAccess qualified as E import Wire.ConversationStore (ConversationStore) import Wire.ConversationStore qualified as E @@ -633,7 +632,6 @@ uncheckedUpdateTeamMember mlzusr mZcon tid newMem = do transient = True } ] - Brig.updateSearchIndex targetId updateTeamMember :: forall r. diff --git a/services/integration.yaml b/services/integration.yaml index acb0e595b23..73995c52ee9 100644 --- a/services/integration.yaml +++ b/services/integration.yaml @@ -328,6 +328,5 @@ federation-v2: integrationTestHostName: "localhost" -additionalElasticSearch: https://localhost:9201 cellsEventQueue: cells_events diff --git a/tools/db/find-undead/.ormolu b/tools/db/find-undead/.ormolu deleted file mode 120000 index ffc2ca9745e..00000000000 --- a/tools/db/find-undead/.ormolu +++ /dev/null @@ -1 +0,0 @@ -../../../.ormolu \ No newline at end of file diff --git a/tools/db/find-undead/README.md b/tools/db/find-undead/README.md deleted file mode 100644 index 739d49c1ce8..00000000000 --- a/tools/db/find-undead/README.md +++ /dev/null @@ -1,23 +0,0 @@ -## Find certain inconsistencies between ES and Cassandra user data - -Context: https://github.com/zinfra/backend-issues/issues/1493 - -This script identifies users that are still visible on ES, but are -marked as deleted on C*. - -It outputs the time at which users have been marked as deleted in C* -so that you can decide whether what you are seeing may be a race -condition (eg., big team is being deleted while you run the script, -and users will be gone from ES a moment after you log them as -inconsistencies). - -### How to run this - -```sh -export BRIG_HOST=... # ip address of galley cassandra DB node -export BRIG_KEYSPACE=brig - -ssh -v -f ubuntu@${BRIG_HOST} -L 2021:${BRIG_HOST}:9042 -N - -./dist/find-undead --cassandra-host-brig=localhost --cassandra-port-brig=2021 --cassandra-keyspace-brig=${BRIG_KEYSPACE} -``` diff --git a/tools/db/find-undead/default.nix b/tools/db/find-undead/default.nix deleted file mode 100644 index 40a367dca80..00000000000 --- a/tools/db/find-undead/default.nix +++ /dev/null @@ -1,47 +0,0 @@ -# WARNING: GENERATED FILE, DO NOT EDIT. -# This file is generated by running hack/bin/generate-local-nix-packages.sh and -# must be regenerated whenever local packages are added or removed, or -# dependencies are added or removed. -{ mkDerivation -, aeson -, base -, bloodhound -, cassandra-util -, conduit -, containers -, http-client -, imports -, lens -, lib -, optparse-applicative -, text -, tinylog -, uuid -, wire-api -}: -mkDerivation { - pname = "find-undead"; - version = "1.0.0"; - src = ./.; - isLibrary = false; - isExecutable = true; - executableHaskellDepends = [ - aeson - base - bloodhound - cassandra-util - conduit - containers - http-client - imports - lens - optparse-applicative - text - tinylog - uuid - wire-api - ]; - description = "Backfill billing_team_member table"; - license = lib.licenses.agpl3Only; - mainProgram = "find-undead"; -} diff --git a/tools/db/find-undead/find-undead.cabal b/tools/db/find-undead/find-undead.cabal deleted file mode 100644 index 16a7035e1ed..00000000000 --- a/tools/db/find-undead/find-undead.cabal +++ /dev/null @@ -1,84 +0,0 @@ -cabal-version: 1.12 -name: find-undead -version: 1.0.0 -synopsis: Backfill billing_team_member table -category: Network -author: Wire Swiss GmbH -maintainer: Wire Swiss GmbH -copyright: (c) 2020 Wire Swiss GmbH -license: AGPL-3 -build-type: Simple - -executable find-undead - main-is: Main.hs - other-modules: - Options - Paths_find_undead - Work - - hs-source-dirs: src - default-extensions: - AllowAmbiguousTypes - BangPatterns - ConstraintKinds - DataKinds - DefaultSignatures - DeriveFunctor - DeriveGeneric - DeriveLift - DeriveTraversable - DerivingStrategies - DerivingVia - DuplicateRecordFields - EmptyCase - FlexibleContexts - FlexibleInstances - FunctionalDependencies - GADTs - InstanceSigs - KindSignatures - LambdaCase - MultiParamTypeClasses - MultiWayIf - NamedFieldPuns - NoImplicitPrelude - OverloadedRecordDot - OverloadedStrings - PackageImports - PatternSynonyms - PolyKinds - QuasiQuotes - RankNTypes - ScopedTypeVariables - StandaloneDeriving - TupleSections - TypeApplications - TypeFamilies - TypeFamilyDependencies - TypeOperators - UndecidableInstances - ViewPatterns - - ghc-options: - -O2 -Wall -Wincomplete-uni-patterns -Wincomplete-record-updates - -Wpartial-fields -fwarn-tabs -optP-Wno-nonportable-include-path - -funbox-strict-fields -threaded "-with-rtsopts=-N -T" -rtsopts - -Wredundant-constraints -Wunused-packages - - build-depends: - aeson - , base - , bloodhound - , cassandra-util - , conduit - , containers - , http-client - , imports - , lens - , optparse-applicative - , text - , tinylog - , uuid - , wire-api - - default-language: GHC2021 diff --git a/tools/db/find-undead/src/Main.hs b/tools/db/find-undead/src/Main.hs deleted file mode 100644 index 54e22f9d2bf..00000000000 --- a/tools/db/find-undead/src/Main.hs +++ /dev/null @@ -1,62 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Main - ( main, - ) -where - -import Cassandra as C -import Cassandra.Settings as C -import Data.Text as Text hiding (show) -import Database.Bloodhound qualified as ES -import Imports -import Network.HTTP.Client qualified as HTTP -import Options as O -import Options.Applicative -import System.Logger qualified as Log -import Work - -main :: IO () -main = do - s <- execParser (info (helper <*> settingsParser) desc) - lgr <- initLogger - cas <- initCas (setCasBrig s) lgr - mgr <- HTTP.newManager HTTP.defaultManagerSettings - let es = initES (setESBrig s) mgr - runCommand lgr cas es (esIndex $ setESBrig s) (esMapping $ setESBrig s) - where - desc = - header "find-undead" - <> progDesc "finds users which are in ES but not in cassandra" - <> fullDesc - initLogger = - Log.new - . Log.setOutput Log.StdOut - . Log.setBufSize 0 - $ Log.defSettings - initCas cas l = - C.init - . C.setLogger (C.mkLogger Nothing l) - . C.setContacts (cHosts cas) [] - . C.setPortNumber (fromIntegral $ cPort cas) - . C.setKeyspace (cKeyspace cas) - . C.setProtocolVersion C.V4 - $ C.defSettings - initES es = ES.mkBHEnv (ES.Server . Text.pack $ "http://" <> esHost es <> ":" <> show (esPort es)) diff --git a/tools/db/find-undead/src/Options.hs b/tools/db/find-undead/src/Options.hs deleted file mode 100644 index ea27268afc2..00000000000 --- a/tools/db/find-undead/src/Options.hs +++ /dev/null @@ -1,88 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Options where - -import Cassandra qualified as C -import Data.Text qualified as Text -import Imports -import Options.Applicative - -data MigratorSettings = MigratorSettings - { setCasBrig :: CassandraSettings, - setESBrig :: ElasticSettings - } - deriving (Show) - -data CassandraSettings = CassandraSettings - { cHosts :: !String, - cPort :: !Word16, - cKeyspace :: !C.Keyspace - } - deriving (Show) - -data ElasticSettings = ElasticSettings - { esHost :: !String, - esPort :: !Word16, - esIndex :: !String, - esMapping :: !String - } - deriving (Show) - -settingsParser :: Parser MigratorSettings -settingsParser = - MigratorSettings - <$> cassandraSettingsParser "brig" - <*> esSettingsParser - -cassandraSettingsParser :: String -> Parser CassandraSettings -cassandraSettingsParser ks = - CassandraSettings - <$> strOption - ( long ("cassandra-host-" ++ ks) - <> metavar "HOST" - <> help ("Cassandra Host for: " ++ ks) - <> value "localhost" - <> showDefault - ) - <*> option - auto - ( long ("cassandra-port-" ++ ks) - <> metavar "PORT" - <> help ("Cassandra Port for: " ++ ks) - <> value 9042 - <> showDefault - ) - <*> ( C.Keyspace . Text.pack - <$> strOption - ( long ("cassandra-keyspace-" ++ ks) - <> metavar "STRING" - <> help ("Cassandra Keyspace for: " ++ ks) - <> value (ks ++ "_test") - <> showDefault - ) - ) - -esSettingsParser :: Parser ElasticSettings -esSettingsParser = - ElasticSettings - <$> strOption (long "es-host" <> value "localhost") - <*> option auto (long "es-port" <> value 9200) - <*> strOption (long "es-index" <> value "directory_test") - <*> strOption (long "es-mapping" <> value "user") diff --git a/tools/db/find-undead/src/Work.hs b/tools/db/find-undead/src/Work.hs deleted file mode 100644 index 87fceb70e64..00000000000 --- a/tools/db/find-undead/src/Work.hs +++ /dev/null @@ -1,125 +0,0 @@ -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE ScopedTypeVariables #-} -{-# OPTIONS_GHC -fno-warn-orphans #-} - --- This file is part of the Wire Server implementation. --- --- Copyright (C) 2022 Wire Swiss GmbH --- --- This program is free software: you can redistribute it and/or modify it under --- the terms of the GNU Affero General Public License as published by the Free --- Software Foundation, either version 3 of the License, or (at your option) any --- later version. --- --- This program is distributed in the hope that it will be useful, but WITHOUT --- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS --- FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more --- details. --- --- You should have received a copy of the GNU Affero General Public License along --- with this program. If not, see . - -module Work where - -import Cassandra -import Cassandra.Util (Writetime, writetimeToUTC) -import Conduit -import Control.Lens (view, _1, _2) -import Data.Aeson (FromJSON, (.:)) -import Data.Aeson qualified as Aeson -import Data.Conduit.List qualified as C -import Data.Set qualified as Set -import Data.Text qualified as Text -import Data.UUID -import Database.Bloodhound qualified as ES -import Imports -import System.Logger (Logger) -import System.Logger qualified as Log -import Wire.API.User (AccountStatus (..)) - -runCommand :: Logger -> ClientState -> ES.BHEnv -> String -> String -> IO () -runCommand l cas es indexStr mappingStr = do - let index = ES.IndexName $ Text.pack indexStr - mapping = ES.MappingName $ Text.pack mappingStr - runConduit $ - transPipe (ES.runBH es) $ - getScrolled index mapping - .| C.iterM (logProgress l) - .| C.mapM - ( \uuids -> do - fromCas <- runClient cas $ usersInCassandra uuids - pure (uuids, fromCas) - ) - .| C.mapM_ (logDifference l) - ----------------------------------------------------------------------------- --- Queries - -logProgress :: (MonadIO m) => Logger -> [UUID] -> m () -logProgress l uuids = Log.info l $ Log.field "Progress" (show $ length uuids) - -logDifference :: Logger -> ([UUID], [(UUID, Maybe AccountStatus, Maybe (Writetime ()))]) -> ES.BH IO () -logDifference l (uuidsFromES, fromCas) = do - let noStatusUuidsFromCas = filter (isNothing . view _2) fromCas - deletedUuidsFromCas = filter ((== Just Deleted) . view _2) fromCas - extraUuids = Set.difference (Set.fromList uuidsFromES) (Set.fromList $ map (view _1) fromCas) - mapM_ (logUUID l "NoStatus") noStatusUuidsFromCas - mapM_ (logUUID l "Deleted") deletedUuidsFromCas - mapM_ (logUUID l "Extra" . (,Nothing,Nothing)) extraUuids - -logUUID :: (MonadIO m) => Logger -> ByteString -> (UUID, Maybe AccountStatus, Maybe (Writetime ())) -> m () -logUUID l f (uuid, _, time) = - Log.info l $ - Log.msg f - . Log.field "uuid" (show uuid) - . Log.field "write time" (show $ writetimeToUTC <$> time) - -getScrolled :: (ES.MonadBH m, MonadThrow m) => ES.IndexName -> ES.MappingName -> ConduitM () [UUID] m () -getScrolled index mapping = processRes =<< lift (ES.getInitialScroll index mapping esSearch) - where - processRes :: (ES.MonadBH m, MonadThrow m) => Either ES.EsError (ES.SearchResult User) -> ConduitM () [UUID] m () - processRes = \case - Left e -> throwM $ EsError e - Right res -> - case map docId $ extractHits res of - [] -> pure () - ids -> do - yield ids - processRes - =<< (\scrollId -> lift (ES.advanceScroll scrollId 120)) - =<< extractScrollId res - -esFilter :: ES.Filter -esFilter = ES.Filter $ ES.QueryExistsQuery (ES.FieldName "normalized") - -chunkSize :: Int -chunkSize = 10000 - -esSearch :: ES.Search -esSearch = (ES.mkSearch Nothing (Just esFilter)) {ES.size = ES.Size chunkSize} - -extractHits :: ES.SearchResult User -> [User] -extractHits = mapMaybe ES.hitSource . ES.hits . ES.searchHits - -extractScrollId :: (MonadThrow m) => ES.SearchResult a -> m ES.ScrollId -extractScrollId res = maybe (throwM NoScrollId) pure (ES.scrollId res) - -usersInCassandra :: [UUID] -> Client [(UUID, Maybe AccountStatus, Maybe (Writetime ()))] -usersInCassandra users = retry x1 $ query cql (params LocalQuorum (Identity users)) - where - cql :: PrepQuery R (Identity [UUID]) (UUID, Maybe AccountStatus, Maybe (Writetime ())) - cql = "SELECT id, status, writetime(status) from user where id in ?" - -newtype User = User {docId :: UUID} - -instance FromJSON User where - parseJSON = Aeson.withObject "User" $ \o -> User <$> o .: "id" - -data WorkError - = NoScrollId - | EsError ES.EsError - deriving (Show, Eq) - -instance Exception WorkError - -type Name = Text diff --git a/tools/stern/src/Stern/Intra.hs b/tools/stern/src/Stern/Intra.hs index 3bd2dd34cc7..1d25e46e7cd 100644 --- a/tools/stern/src/Stern/Intra.hs +++ b/tools/stern/src/Stern/Intra.hs @@ -125,6 +125,7 @@ import Wire.API.Routes.Named import Wire.API.Routes.Version import Wire.API.Routes.Versioned import Wire.API.Team +import Wire.API.Team qualified as Team import Wire.API.Team.Feature import Wire.API.Team.Feature qualified as Public import Wire.API.Team.Member @@ -419,7 +420,7 @@ getUserBindingTeam u = do teams <- parseResponse (mkError status502 "bad-upstream") r pure $ listToMaybe $ - fmap (view teamId) $ + fmap (view Team.teamId) $ filter ((== Binding) . view teamBinding) $ teams ^. teamListTeams