diff --git a/.gitignore b/.gitignore index c781ab91..f5fb47b9 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,5 @@ site/ **/*.env !*.env.template !**/*.env.template +!deploy/ollama.env +!deploy/openwebui.env diff --git a/deploy/Makefile b/deploy/Makefile index bcbf31be..3f618642 100644 --- a/deploy/Makefile +++ b/deploy/Makefile @@ -351,6 +351,29 @@ start-filebeat-3: start-kibana: $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) kibana +start-ollama: ## Start the Ollama inference service + $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) ollama + +pull-ollama-model: start-ollama ## Pull the model configured by OLLAMA_MODEL + $(WITH_ENV) docker compose -f services.yml exec -T ollama ollama pull "$${OLLAMA_MODEL}" + +setup-ollama: pull-ollama-model ## Start Ollama and pull its configured model + +opensearch-ai-init: ## Create or update Ollama-backed OpenSearch chat and PPL assistants + $(WITH_ENV) ../scripts/opensearch-ai-init.sh + +start-openwebui: ## Start Open WebUI and its Ollama dependency + $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) open-webui + +restart-openwebui: ## Restart the Open WebUI container + $(WITH_ENV) docker compose -f services.yml restart open-webui + +status-openwebui: ## Show Open WebUI and Ollama container status + $(WITH_ENV) docker compose -f services.yml ps open-webui ollama + +logs-openwebui: ## Follow the Open WebUI logs + $(WITH_ENV) docker compose -f services.yml logs --tail=100 -f open-webui + start-samples: ## Start the sample database service $(WITH_ENV) docker compose -f services.yml $(DC_START_CMD) samples-db @@ -379,7 +402,7 @@ start-data-infra: start-nifi start-elastic start-samples ## Start core data infr start-all: start-data-infra start-medcat-service start-ocr-services ## Start all major services -.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-samples start-jupyter +.PHONY: start-all start-data-infra start-nifi start-nifi-dev start-nifi-dev-build build-nifi-image start-elastic start-ollama pull-ollama-model setup-ollama opensearch-ai-init start-openwebui restart-openwebui status-openwebui logs-openwebui start-samples start-jupyter # stop services @@ -423,6 +446,12 @@ delete-elastic: ## Remove Elasticsearch and Kibana containers delete-elastic-volumes: ## Delete Elasticsearch and Kibana volumes $(WITH_ENV) docker compose -f services.yml $(DC_DOWN_CMD) -v elasticsearch-1 elasticsearch-2 elasticsearch-3 kibana +delete-ollama: ## Remove the Ollama container but preserve downloaded models + $(WITH_ENV) docker compose -f services.yml rm -f -s ollama + +delete-openwebui: ## Remove the Open WebUI container but preserve its data + $(WITH_ENV) docker compose -f services.yml rm -f -s open-webui + delete-databank: $(WITH_ENV) docker compose -f services.yml rm -f -s cogstack-databank-db cogstack-databank-db-mssql @@ -494,6 +523,12 @@ stop-filebeat-3: stop-kibana: $(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) kibana +stop-ollama: ## Stop the Ollama inference service + $(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) ollama + +stop-openwebui: ## Stop the Open WebUI service + $(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) open-webui + stop-samples: ## Stop the sample database service $(WITH_ENV) docker compose -f services.yml $(DC_STOP_CMD) samples-db @@ -522,7 +557,7 @@ stop-data-infra: stop-nifi stop-elastic stop-samples ## Stop core data infrastru stop-all: stop-data-infra stop-jupyter stop-medcat-service stop-ocr-services ## Stop all major services -.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images delete-nifi-volumes delete-elastic delete-elastic-volumes delete-databank delete-databank-volumes delete-samples-db delete-samples-db-volumes delete-medcat-trainer delete-medcat-trainer-volumes delete-jupyter delete-medcat-service delete-medcat-service-deid delete-ocr-services make-delete-jupyter make-delete-medcat-service make-delete-medcat-service-deid make-delete-ocr-services stop-elastic stop-samples stop-jupyter +.PHONY: stop-data-infra stop-nifi stop-nifi-dev delete-nifi delete-nifi-containers delete-nifi-dev-containers delete-nifi-images delete-nifi-dev-images delete-nifi-volumes delete-elastic delete-elastic-volumes delete-ollama delete-openwebui delete-databank delete-databank-volumes delete-samples-db delete-samples-db-volumes delete-medcat-trainer delete-medcat-trainer-volumes delete-jupyter delete-medcat-service delete-medcat-service-deid delete-ocr-services make-delete-jupyter make-delete-medcat-service make-delete-medcat-service-deid make-delete-ocr-services stop-elastic stop-ollama stop-openwebui stop-samples stop-jupyter # cleanup diff --git a/deploy/elasticsearch.env b/deploy/elasticsearch.env index 56fa8c61..edad8df1 100644 --- a/deploy/elasticsearch.env +++ b/deploy/elasticsearch.env @@ -106,6 +106,29 @@ ELASTICSEARCH_DOCKER_SHM_SIZE=512m ELASTICSEARCH_DOCKER_LOG_SIZE_PER_FILE="1000m" ELASTICSEARCH_DOCKER_LOG_NUM_FILES=10 +# OpenSearch ML Commons bootstrap settings. Connector timeouts are seconds; +# the Dashboards opensearch.requestTimeout setting is milliseconds. +OPENSEARCH_AI_OPENSEARCH_CONTAINER=elasticsearch-1 +# Local development credentials; update these when the OpenSearch admin login +# is rotated or override them when invoking the bootstrap script. +OPENSEARCH_AI_OPENSEARCH_USER=admin +OPENSEARCH_AI_OPENSEARCH_PASSWORD=admin +OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS=120 +OPENSEARCH_AI_READ_TIMEOUT_SECONDS=360 +OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS=300 +OPENSEARCH_AI_MAX_TOKENS=256 +OPENSEARCH_AI_REASONING_EFFORT=none + +# Stable resource names let scripts/opensearch-ai-init.sh safely find and +# update resources instead of creating duplicates on every run. +OPENSEARCH_AI_CHAT_CONNECTOR_NAME="Ollama Qwen Agent Connector" +OPENSEARCH_AI_CHAT_MODEL_NAME="Qwen 3.5 Ollama Agent Model" +OPENSEARCH_AI_CHAT_AGENT_NAME="Qwen 3.5 Conversational Agent v2" +OPENSEARCH_AI_ROOT_AGENT_NAME="Qwen Dashboards Root Agent" +OPENSEARCH_AI_PPL_CONNECTOR_NAME="Ollama Qwen PPL Connector" +OPENSEARCH_AI_PPL_MODEL_NAME="Qwen 3.5 Ollama PPL Model" +OPENSEARCH_AI_PPL_AGENT_NAME="Qwen PPL Query Assist Agent" + # disables swapping, imporving r/w performance (at the expense of RAM) ELASTICSEARCH_BOOTSTRAP_MEMORY_LOCK=true diff --git a/deploy/export_env_vars.sh b/deploy/export_env_vars.sh index 1d05060b..8528034b 100755 --- a/deploy/export_env_vars.sh +++ b/deploy/export_env_vars.sh @@ -29,6 +29,8 @@ env_files=( "$DEPLOY_DIR/general.env" "$DEPLOY_DIR/nifi.env" "$DEPLOY_DIR/elasticsearch.env" + "$DEPLOY_DIR/ollama.env" + "$DEPLOY_DIR/openwebui.env" "$DEPLOY_DIR/database.env" "$DEPLOY_DIR/network_settings.env" "$DEPLOY_DIR/project.env" diff --git a/deploy/general.env b/deploy/general.env index fd51869a..768710c2 100644 --- a/deploy/general.env +++ b/deploy/general.env @@ -21,6 +21,6 @@ VOLUME_PREFIX=deploy_ DEPLOYMENT_ENV_FILE_PATH_GENERAL=../../../deploy/general.env # this is the default, for containers which dont rely on individual settings -DOCKER_SHM_SIZE=1g +DOCKER_SHM_SIZE=512m DOCKER_LOG_SIZE_PER_FILE="100m" DOCKER_LOG_NUM_FILES=10 diff --git a/deploy/ollama.env b/deploy/ollama.env new file mode 100644 index 00000000..48fa3df5 --- /dev/null +++ b/deploy/ollama.env @@ -0,0 +1,31 @@ +####################################################################### Ollama Env vars ####################################################################### + +# Official Ollama server image. +OLLAMA_DOCKER_IMAGE=ollama/ollama:latest +OLLAMA_CONTAINER_NAME=cogstack-ollama + +# Ollama listens on every container interface so OpenSearch can reach it over +# cogstack-net. The published host port remains bound to loopback in services.yml. +OLLAMA_HOST=0.0.0.0:11434 +OLLAMA_OUTPUT_HOST=127.0.0.1 +OLLAMA_OUTPUT_PORT=11434 + +# Default model for CogStack Q&A setup. This variable documents the selected +# model; pull it once after starting Ollama (see the command below). +OLLAMA_MODEL=qwen3.5:9b-q4_K_M + +# A 16K context provides room for OpenSearch agent instructions, mappings, +# tool definitions, and retrieved documents without paying for the full 256K +# context window in memory. +OLLAMA_CONTEXT_LENGTH=16384 + +# Keep one model resident and avoid excessive memory use alongside OpenSearch. +OLLAMA_KEEP_ALIVE=5m +OLLAMA_MAX_LOADED_MODELS=1 +OLLAMA_NUM_PARALLEL=1 + +# Docker resource allocation. +OLLAMA_DOCKER_CPU_MIN=1 +OLLAMA_DOCKER_CPU_MAX=4 +OLLAMA_DOCKER_RAM_MIN=2g +OLLAMA_DOCKER_RAM=16g diff --git a/deploy/openwebui.env b/deploy/openwebui.env new file mode 100644 index 00000000..48f01ccb --- /dev/null +++ b/deploy/openwebui.env @@ -0,0 +1,39 @@ +###################################################################### Open WebUI Env vars ###################################################################### + +# Official Open WebUI image. Pin this tag to a numbered release when a +# reproducible production deployment is required. +OPENWEBUI_DOCKER_IMAGE=ghcr.io/open-webui/open-webui:main +OPENWEBUI_CONTAINER_NAME=cogstack-open-webui + +# Open WebUI reaches Ollama through the private Compose network. Do not use +# localhost here because localhost inside this container is Open WebUI itself. +OLLAMA_BASE_URL=http://ollama:11434 + +# The first account registered becomes the administrator. After creating it, +# signups can be disabled from the Open WebUI Admin Panel. +WEBUI_AUTH=true +ENABLE_SIGNUP=true +DEFAULT_USER_ROLE=user +WEBUI_NAME=CogStack + +# Leave this empty when using the official image entrypoint. It generates a +# random key and persists it in the openwebui-data volume on first startup. +WEBUI_SECRET_KEY= + +# Browser-facing address. Port 3001 avoids the existing Gitea port 3000. +WEBUI_URL=http://localhost:3001 +OPENWEBUI_OUTPUT_HOST=127.0.0.1 +OPENWEBUI_OUTPUT_PORT=3001 + +# Open WebUI is lightweight compared with model inference, which stays in the +# separate Ollama container. +OPENWEBUI_DOCKER_CPU_MIN=0.25 +OPENWEBUI_DOCKER_CPU_MAX=1 +OPENWEBUI_DOCKER_RAM_MIN=256m +OPENWEBUI_DOCKER_RAM=2g + +# Disable optional usage reporting and update checks in this local deployment. +ANONYMIZED_TELEMETRY=false +DO_NOT_TRACK=true +SCARF_NO_ANALYTICS=true +ENABLE_VERSION_UPDATE_CHECK=false diff --git a/deploy/services.yml b/deploy/services.yml index 41bfe036..ce0f42ec 100644 --- a/deploy/services.yml +++ b/deploy/services.yml @@ -139,6 +139,9 @@ x-es-common-volumes: &es-common-volumes - ../security/es_roles/opensearch/internal_users.yml:/usr/share/opensearch/config/opensearch-security/internal_users.yml:ro - ../security/es_roles/opensearch/roles_mapping.yml:/usr/share/opensearch/config/opensearch-security/roles_mapping.yml:ro - ../security/es_roles/opensearch/roles.yml:/usr/share/opensearch/config/opensearch-security/roles.yml:ro + # Shared local ML model repository. Register files with a + # file:///usr/share/opensearch/models/.zip URL. + - ../services/elasticsearch/models:/usr/share/opensearch/models:ro # Shared system settings - ../services/elasticsearch/sysctl.conf:/etc/sysctl.conf:ro # Shared snapshot mounts @@ -475,6 +478,73 @@ services: extra_hosts: *common-hosts logging: *es-logging-common +#---------------------------------------------------------------------------# +# Ollama local model inference # +#---------------------------------------------------------------------------# + ollama: + image: ${OLLAMA_DOCKER_IMAGE:-ollama/ollama:latest} + container_name: ${OLLAMA_CONTAINER_NAME:-cogstack-ollama} + platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} + restart: unless-stopped + env_file: + - ./ollama.env + deploy: + resources: + limits: + cpus: "${OLLAMA_DOCKER_CPU_MAX:-4}" + memory: "${OLLAMA_DOCKER_RAM:-12g}" + reservations: + cpus: "${OLLAMA_DOCKER_CPU_MIN:-1}" + memory: "${OLLAMA_DOCKER_RAM_MIN:-2g}" + volumes: + - ollama-models:/root/.ollama + ports: + # Keep the unauthenticated Ollama API local to the Docker host. + - "${OLLAMA_OUTPUT_HOST:-127.0.0.1}:${OLLAMA_OUTPUT_PORT:-11434}:11434" + expose: + - "11434" + networks: + - cognet + logging: *logging-common + healthcheck: + test: ["CMD", "ollama", "list"] + interval: 30s + timeout: 10s + retries: 5 + start_period: 20s + +#---------------------------------------------------------------------------# +# Open WebUI for local Ollama chat and model management # +#---------------------------------------------------------------------------# + open-webui: + image: ${OPENWEBUI_DOCKER_IMAGE:-ghcr.io/open-webui/open-webui:main} + container_name: ${OPENWEBUI_CONTAINER_NAME:-cogstack-open-webui} + platform: ${DOCKER_DEFAULT_PLATFORM:-linux/amd64} + restart: unless-stopped + env_file: + - ./openwebui.env + depends_on: + ollama: + condition: service_healthy + deploy: + resources: + limits: + cpus: "${OPENWEBUI_DOCKER_CPU_MAX:-1}" + memory: "${OPENWEBUI_DOCKER_RAM:-2g}" + reservations: + cpus: "${OPENWEBUI_DOCKER_CPU_MIN:-0.25}" + memory: "${OPENWEBUI_DOCKER_RAM_MIN:-256m}" + volumes: + - openwebui-data:/app/backend/data + ports: + # Port 3000 is already used by Gitea in this Compose project. + - "${OPENWEBUI_OUTPUT_HOST:-127.0.0.1}:${OPENWEBUI_OUTPUT_PORT:-3001}:8080" + expose: + - "8080" + networks: + - cognet + logging: *logging-common + #---------------------------------------------------------------------------# # NiFi webapp # #---------------------------------------------------------------------------# @@ -632,6 +702,14 @@ volumes: filebeat-data-3: driver: local + # Ollama model cache + ollama-models: + driver: local + + # Open WebUI database, uploads, configuration, and generated secret key + openwebui-data: + driver: local + # NiFi related nifi-vol-logs: driver: local diff --git a/docs/deploy/configuration.md b/docs/deploy/configuration.md index c4240be5..0c63b0e0 100644 --- a/docs/deploy/configuration.md +++ b/docs/deploy/configuration.md @@ -24,6 +24,7 @@ The core Compose file, `deploy/services.yml`, loads these files: | `deploy/general.env` | General Docker logging and shared defaults | | `deploy/nifi.env` | NiFi image, ports, resources, proxy settings, and paths | | `deploy/elasticsearch.env` | Elasticsearch/OpenSearch, Dashboards/Kibana, and Beats | +| `deploy/ollama.env` | Ollama image, model selection, port, and resource limits | | `deploy/database.env` | Sample and production database resources | | `deploy/gitea.env` | Gitea image, ports, and application settings | | `deploy/nginx.env` | NiFi nginx proxy settings | diff --git a/scripts/opensearch-ai-init.sh b/scripts/opensearch-ai-init.sh new file mode 100755 index 00000000..f7890214 --- /dev/null +++ b/scripts/opensearch-ai-init.sh @@ -0,0 +1,416 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +log() { + printf '[opensearch-ai-init] %s\n' "$*" >&2 +} + +die() { + printf '[opensearch-ai-init] ERROR: %s\n' "$*" >&2 + exit 1 +} + +load_env_file() { + local env_file="$1" + if [[ -f "$env_file" ]]; then + set -a + # shellcheck disable=SC1090 + source "$env_file" + set +a + fi +} + +# Make exports these variables already. Loading the files here also makes the +# script convenient to run directly from the repository root. Preserve values +# supplied by the caller so one-off command-line overrides take precedence. +ENV_OVERRIDE_NAMES=( + ELASTIC_USER + ELASTIC_PASSWORD + OLLAMA_CONTAINER_NAME + OLLAMA_MODEL + OPENSEARCH_AI_OPENSEARCH_CONTAINER + OPENSEARCH_AI_OPENSEARCH_USER + OPENSEARCH_AI_OPENSEARCH_PASSWORD + OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS + OPENSEARCH_AI_READ_TIMEOUT_SECONDS + OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS + OPENSEARCH_AI_MAX_TOKENS + OPENSEARCH_AI_REASONING_EFFORT + OPENSEARCH_AI_CHAT_CONNECTOR_NAME + OPENSEARCH_AI_CHAT_MODEL_NAME + OPENSEARCH_AI_CHAT_AGENT_NAME + OPENSEARCH_AI_ROOT_AGENT_NAME + OPENSEARCH_AI_PPL_CONNECTOR_NAME + OPENSEARCH_AI_PPL_MODEL_NAME + OPENSEARCH_AI_PPL_AGENT_NAME +) +ENV_OVERRIDES=() +for variable_name in "${ENV_OVERRIDE_NAMES[@]}"; do + if [[ -n "${!variable_name+x}" ]]; then + ENV_OVERRIDES+=("$variable_name=${!variable_name}") + fi +done +load_env_file "$PROJECT_DIR/security/env/users_elasticsearch.env" +load_env_file "$PROJECT_DIR/deploy/elasticsearch.env" +load_env_file "$PROJECT_DIR/deploy/ollama.env" +for variable_assignment in "${ENV_OVERRIDES[@]}"; do + variable_name="${variable_assignment%%=*}" + printf -v "$variable_name" '%s' "${variable_assignment#*=}" + export "${variable_name?}" +done + +OPENSEARCH_CONTAINER="${OPENSEARCH_AI_OPENSEARCH_CONTAINER:-elasticsearch-1}" +OPENSEARCH_USER="${OPENSEARCH_AI_OPENSEARCH_USER:-${ELASTIC_USER:-admin}}" +OPENSEARCH_PASSWORD="${OPENSEARCH_AI_OPENSEARCH_PASSWORD:-${ELASTIC_PASSWORD:-}}" +OLLAMA_CONTAINER="${OLLAMA_CONTAINER_NAME:-cogstack-ollama}" +OLLAMA_MODEL_NAME="${OLLAMA_MODEL:-qwen3.5:9b-q4_K_M}" + +CONNECTION_TIMEOUT="${OPENSEARCH_AI_CONNECTION_TIMEOUT_SECONDS:-120}" +READ_TIMEOUT="${OPENSEARCH_AI_READ_TIMEOUT_SECONDS:-360}" +DEPLOY_TIMEOUT="${OPENSEARCH_AI_DEPLOY_TIMEOUT_SECONDS:-300}" +MAX_TOKENS="${OPENSEARCH_AI_MAX_TOKENS:-256}" +REASONING_EFFORT="${OPENSEARCH_AI_REASONING_EFFORT:-none}" + +CHAT_CONNECTOR_NAME="${OPENSEARCH_AI_CHAT_CONNECTOR_NAME:-Ollama Qwen Agent Connector}" +CHAT_MODEL_NAME="${OPENSEARCH_AI_CHAT_MODEL_NAME:-Qwen 3.5 Ollama Agent Model}" +CHAT_AGENT_NAME="${OPENSEARCH_AI_CHAT_AGENT_NAME:-Qwen 3.5 Conversational Agent v2}" +ROOT_AGENT_NAME="${OPENSEARCH_AI_ROOT_AGENT_NAME:-Qwen Dashboards Root Agent}" +PPL_CONNECTOR_NAME="${OPENSEARCH_AI_PPL_CONNECTOR_NAME:-Ollama Qwen PPL Connector}" +PPL_MODEL_NAME="${OPENSEARCH_AI_PPL_MODEL_NAME:-Qwen 3.5 Ollama PPL Model}" +PPL_AGENT_NAME="${OPENSEARCH_AI_PPL_AGENT_NAME:-Qwen PPL Query Assist Agent}" + +OPENSEARCH_URL="https://localhost:9200" +OPENSEARCH_CA="/usr/share/opensearch/config/root-ca.crt" + +[[ -n "$OPENSEARCH_PASSWORD" ]] \ + || die "set OPENSEARCH_AI_OPENSEARCH_PASSWORD or ELASTIC_PASSWORD" + +for command_name in docker jq; do + command -v "$command_name" >/dev/null 2>&1 || die "$command_name is required" +done + +for integer_value in "$CONNECTION_TIMEOUT" "$READ_TIMEOUT" "$DEPLOY_TIMEOUT" "$MAX_TOKENS"; do + [[ "$integer_value" =~ ^[1-9][0-9]*$ ]] || die "timeout and token settings must be positive integers" +done + +[[ "$(docker inspect --format '{{.State.Running}}' "$OPENSEARCH_CONTAINER" 2>/dev/null)" == "true" ]] \ + || die "OpenSearch container '$OPENSEARCH_CONTAINER' is not running" +[[ "$(docker inspect --format '{{.State.Running}}' "$OLLAMA_CONTAINER" 2>/dev/null)" == "true" ]] \ + || die "Ollama container '$OLLAMA_CONTAINER' is not running" + +os_request() { + local method="$1" + local path="$2" + local payload="${3-}" + local response body status + local curl_args=( + --silent + --show-error + --cacert "$OPENSEARCH_CA" + --user "$OPENSEARCH_USER:$OPENSEARCH_PASSWORD" + --request "$method" + --header 'Content-Type: application/json' + --write-out $'\n%{http_code}' + ) + + if [[ -n "$payload" ]]; then + response="$(printf '%s' "$payload" | docker exec -i "$OPENSEARCH_CONTAINER" \ + curl "${curl_args[@]}" --data-binary @- "$OPENSEARCH_URL$path")" || return + else + response="$(docker exec "$OPENSEARCH_CONTAINER" \ + curl "${curl_args[@]}" "$OPENSEARCH_URL$path")" || return + fi + + status="${response##*$'\n'}" + body="${response%$'\n'*}" + if [[ ! "$status" =~ ^2[0-9][0-9]$ ]]; then + log "$method $path failed with HTTP $status" + [[ -n "$body" ]] && printf '%s\n' "$body" >&2 + return 1 + fi + printf '%s' "$body" +} + +wait_for_opensearch() { + local deadline=$((SECONDS + DEPLOY_TIMEOUT)) + local response + log "Waiting for OpenSearch to reach yellow health" + while (( SECONDS < deadline )); do + if response="$(os_request GET '/_cluster/health?wait_for_status=yellow&timeout=5s' 2>/dev/null)" \ + && [[ "$(jq -r '.status // empty' <<<"$response")" =~ ^(yellow|green)$ ]]; then + return 0 + fi + sleep 5 + done + die "OpenSearch did not reach yellow health within ${DEPLOY_TIMEOUT}s" +} + +wait_for_ollama() { + local deadline=$((SECONDS + DEPLOY_TIMEOUT)) + log "Waiting for Ollama and ensuring model '$OLLAMA_MODEL_NAME' is available" + while (( SECONDS < deadline )); do + if docker exec "$OLLAMA_CONTAINER" ollama list >/dev/null 2>&1; then + if ! docker exec "$OLLAMA_CONTAINER" ollama show "$OLLAMA_MODEL_NAME" >/dev/null 2>&1; then + docker exec "$OLLAMA_CONTAINER" ollama pull "$OLLAMA_MODEL_NAME" + fi + return 0 + fi + sleep 5 + done + die "Ollama did not become ready within ${DEPLOY_TIMEOUT}s" +} + +search_resource_id() { + local resource="$1" + local name="$2" + local response + response="$(os_request POST "/_plugins/_ml/${resource}/_search" '{"query":{"match_all":{}},"size":1000}')" + jq -r --arg name "$name" '.hits.hits[]? | select(._source.name == $name) | ._id' \ + <<<"$response" | head -n 1 +} + +models_for_connector() { + local connector_id="$1" + local response + response="$(os_request POST '/_plugins/_ml/models/_search' '{"query":{"match_all":{}},"size":1000}')" + jq -r --arg connector_id "$connector_id" \ + '.hits.hits[]? | select(._source.connector_id == $connector_id) | [._id, (._source.model_state // "")] | @tsv' \ + <<<"$response" +} + +wait_for_model_state() { + local model_id="$1" + local deadline=$((SECONDS + DEPLOY_TIMEOUT)) + local response state + while (( SECONDS < deadline )); do + response="$(os_request GET "/_plugins/_ml/models/$model_id")" + state="$(jq -r '.model_state // empty' <<<"$response")" + case "$state" in + DEPLOYED) + return 0 + ;; + DEPLOY_FAILED) + die "model '$model_id' failed to deploy" + ;; + esac + sleep 3 + done + die "model '$model_id' did not deploy within ${DEPLOY_TIMEOUT}s" +} + +wait_for_model_undeployed() { + local model_id="$1" + local deadline=$((SECONDS + DEPLOY_TIMEOUT)) + local response state + while (( SECONDS < deadline )); do + response="$(os_request GET "/_plugins/_ml/models/$model_id")" + state="$(jq -r '.model_state // empty' <<<"$response")" + if [[ "$state" != "DEPLOYED" && "$state" != "DEPLOYING" ]]; then + return 0 + fi + sleep 2 + done + die "model '$model_id' did not undeploy within ${DEPLOY_TIMEOUT}s" +} + +deploy_model() { + local model_id="$1" + local state + state="$(os_request GET "/_plugins/_ml/models/$model_id" | jq -r '.model_state // empty')" + if [[ "$state" != "DEPLOYED" ]]; then + log "Deploying model $model_id" + os_request POST "/_plugins/_ml/models/$model_id/_deploy" '{}' >/dev/null + fi + wait_for_model_state "$model_id" +} + +undeploy_connector_models() { + local connector_id="$1" + local model_id state + while IFS=$'\t' read -r model_id state; do + [[ -n "$model_id" ]] || continue + if [[ "$state" == "DEPLOYED" || "$state" == "DEPLOYING" ]]; then + log "Undeploying model $model_id before updating connector $connector_id" + os_request POST "/_plugins/_ml/models/$model_id/_undeploy" '{}' >/dev/null + wait_for_model_undeployed "$model_id" + fi + done < <(models_for_connector "$connector_id") +} + +redeploy_connector_models() { + local connector_id="$1" + local model_id state + while IFS=$'\t' read -r model_id state; do + [[ -n "$model_id" ]] || continue + deploy_model "$model_id" + done < <(models_for_connector "$connector_id") +} + +upsert_connector() { + local name="$1" + local payload="$2" + local connector_id + connector_id="$(search_resource_id connectors "$name")" + if [[ -n "$connector_id" ]]; then + undeploy_connector_models "$connector_id" + log "Updating connector '$name' ($connector_id)" + os_request PUT "/_plugins/_ml/connectors/$connector_id" "$payload" >/dev/null + redeploy_connector_models "$connector_id" + else + log "Creating connector '$name'" + connector_id="$(os_request POST '/_plugins/_ml/connectors/_create' "$payload" | jq -r '.connector_id // empty')" + [[ -n "$connector_id" ]] || die "connector '$name' was created without a connector_id" + fi + printf '%s\n' "$connector_id" +} + +register_model() { + local name="$1" + local connector_id="$2" + local payload response model_id task_id task_response task_state + payload="$(jq -nc \ + --arg name "$name" \ + --arg connector_id "$connector_id" \ + '{name:$name, description:("Ollama remote model for " + $name), function_name:"remote", connector_id:$connector_id}')" + response="$(os_request POST '/_plugins/_ml/models/_register' "$payload")" + model_id="$(jq -r '.model_id // empty' <<<"$response")" + task_id="$(jq -r '.task_id // empty' <<<"$response")" + + if [[ -z "$model_id" && -n "$task_id" ]]; then + local deadline=$((SECONDS + DEPLOY_TIMEOUT)) + while (( SECONDS < deadline )); do + task_response="$(os_request GET "/_plugins/_ml/tasks/$task_id")" + task_state="$(jq -r '.state // empty' <<<"$task_response")" + model_id="$(jq -r '.model_id // empty' <<<"$task_response")" + [[ "$task_state" == "FAILED" ]] && die "model registration task '$task_id' failed" + [[ "$task_state" == "COMPLETED" && -n "$model_id" ]] && break + sleep 2 + done + fi + + [[ -n "$model_id" ]] || die "model '$name' was registered without a model_id" + printf '%s\n' "$model_id" +} + +ensure_model() { + local name="$1" + local connector_id="$2" + local model_id current_connector_id + model_id="$(search_resource_id models "$name")" + if [[ -n "$model_id" ]]; then + current_connector_id="$(os_request GET "/_plugins/_ml/models/$model_id" | jq -r '.connector_id // empty')" + if [[ "$current_connector_id" != "$connector_id" ]]; then + die "model '$name' already exists with connector '$current_connector_id', expected '$connector_id'" + fi + log "Reusing model '$name' ($model_id)" + else + log "Registering model '$name'" + model_id="$(register_model "$name" "$connector_id")" + fi + deploy_model "$model_id" + printf '%s\n' "$model_id" +} + +upsert_agent() { + local name="$1" + local payload="$2" + local agent_id + agent_id="$(search_resource_id agents "$name")" + if [[ -n "$agent_id" ]]; then + log "Updating agent '$name' ($agent_id)" + # Agent type and top-level interface parameters are creation-time fields. + # Existing agents retain them while supported mutable fields are refreshed. + os_request PUT "/_plugins/_ml/agents/$agent_id" \ + "$(jq -c 'del(.type, .parameters)' <<<"$payload")" >/dev/null + else + log "Registering agent '$name'" + agent_id="$(os_request POST '/_plugins/_ml/agents/_register' "$payload" | jq -r '.agent_id // empty')" + [[ -n "$agent_id" ]] || die "agent '$name' was registered without an agent_id" + fi + printf '%s\n' "$agent_id" +} + +write_ml_config() { + local config_id="$1" + local config_type="$2" + local agent_id="$3" + local payload + payload="$(jq -nc --arg type "$config_type" --arg agent_id "$agent_id" \ + '{type:$type, configuration:{agent_id:$agent_id}}')" + log "Writing ML config '$config_id' -> $agent_id" + os_request PUT "/.plugins-ml-config/_doc/$config_id?refresh=true" "$payload" >/dev/null +} + +wait_for_opensearch +wait_for_ollama + +# The single quotes deliberately preserve ML Commons ${parameters.*} +# placeholders until connector inference time. +# shellcheck disable=SC2016 +printf -v CHAT_REQUEST_BODY \ + '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"${parameters.system_prompt}"},${parameters._chat_history:-}{"role":"user","content":"${parameters.prompt}"}${parameters._interactions:-}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s"${parameters.tool_configs:-} }' \ + "$MAX_TOKENS" "$REASONING_EFFORT" + +# shellcheck disable=SC2016 +printf -v PPL_REQUEST_BODY \ + '{ "model": "${parameters.model}", "messages": [{"role":"system","content":"Translate the request into a valid OpenSearch PPL query. Return only the PPL query."},{"role":"user","content":"${parameters.prompt}"}], "stream": false, "max_tokens": %d, "reasoning_effort": "%s" }' \ + "$MAX_TOKENS" "$REASONING_EFFORT" + +CLIENT_CONFIG="$(jq -nc \ + --argjson connection_timeout "$CONNECTION_TIMEOUT" \ + --argjson read_timeout "$READ_TIMEOUT" \ + '{max_connection:10, connection_timeout:$connection_timeout, read_timeout:$read_timeout, max_retry_times:0}')" + +CHAT_CONNECTOR_PAYLOAD="$(jq -nc \ + --arg name "$CHAT_CONNECTOR_NAME" \ + --arg model "$OLLAMA_MODEL_NAME" \ + --arg request_body "$CHAT_REQUEST_BODY" \ + --argjson client_config "$CLIENT_CONFIG" \ + '{name:$name, description:"OpenAI-compatible Ollama connector for OpenSearch conversational agents", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')" + +PPL_CONNECTOR_PAYLOAD="$(jq -nc \ + --arg name "$PPL_CONNECTOR_NAME" \ + --arg model "$OLLAMA_MODEL_NAME" \ + --arg request_body "$PPL_REQUEST_BODY" \ + --argjson client_config "$CLIENT_CONFIG" \ + '{name:$name, description:"Prompt-compatible Ollama connector for OpenSearch PPL query assist", version:"1", protocol:"http", parameters:{endpoint:"ollama:11434", model:$model, response_filter:"$.choices[0].message.content"}, credential:{ollama_key:"local"}, client_config:$client_config, actions:[{action_type:"predict", method:"POST", url:"http://${parameters.endpoint}/v1/chat/completions", headers:{"Content-Type":"application/json"}, request_body:$request_body}]}')" + +CHAT_CONNECTOR_ID="$(upsert_connector "$CHAT_CONNECTOR_NAME" "$CHAT_CONNECTOR_PAYLOAD")" +CHAT_MODEL_ID="$(ensure_model "$CHAT_MODEL_NAME" "$CHAT_CONNECTOR_ID")" + +CHAT_AGENT_PAYLOAD="$(jq -nc \ + --arg name "$CHAT_AGENT_NAME" \ + --arg model_id "$CHAT_MODEL_ID" \ + '{name:$name, description:"Local Qwen assistant served through Ollama", type:"conversational", app_type:"os_chat", memory:{type:"conversation_index"}, llm:{model_id:$model_id, parameters:{max_iteration:"3", response_filter:"$.response", system_prompt:"You are a helpful OpenSearch assistant. Keep answers concise unless the user requests detail.", prompt:"${parameters.question}", message_history_limit:"5"}}, parameters:{_llm_interface:"openai/v1/chat/completions"}}')" +CHAT_AGENT_ID="$(upsert_agent "$CHAT_AGENT_NAME" "$CHAT_AGENT_PAYLOAD")" + +ROOT_AGENT_PAYLOAD="$(jq -nc \ + --arg name "$ROOT_AGENT_NAME" \ + --arg agent_id "$CHAT_AGENT_ID" \ + '{name:$name, description:"Root agent for OpenSearch Assistant", type:"flow", app_type:"os_chat", tools:[{type:"AgentTool", name:"LLMResponseGenerator", include_output_in_agent_response:true, parameters:{agent_id:$agent_id}}]}')" +ROOT_AGENT_ID="$(upsert_agent "$ROOT_AGENT_NAME" "$ROOT_AGENT_PAYLOAD")" + +PPL_CONNECTOR_ID="$(upsert_connector "$PPL_CONNECTOR_NAME" "$PPL_CONNECTOR_PAYLOAD")" +PPL_MODEL_ID="$(ensure_model "$PPL_MODEL_NAME" "$PPL_CONNECTOR_ID")" + +PPL_AGENT_PAYLOAD="$(jq -nc \ + --arg name "$PPL_AGENT_NAME" \ + --arg model_id "$PPL_MODEL_ID" \ + '{name:$name, description:"Generate PPL queries from natural-language questions", type:"flow", app_type:"query_assist", tools:[{type:"PPLTool", name:"TransferQuestionToPPLAndExecuteTool", description:"Translate a natural-language question into an OpenSearch PPL query for the supplied index. Inputs: {index:IndexName, question:UserQuestion}.", include_output_in_agent_response:true, parameters:{model_id:$model_id, model_type:"OPENAI", response_filter:"$.choices[0].message.content", execute:false}}]}')" +PPL_AGENT_ID="$(upsert_agent "$PPL_AGENT_NAME" "$PPL_AGENT_PAYLOAD")" + +write_ml_config os_chat os_chat_root_agent "$ROOT_AGENT_ID" +write_ml_config os_query_assist_ppl os_query_assist_ppl_agent "$PPL_AGENT_ID" + +log "Bootstrap complete" +printf ' chat_connector_id=%s\n' "$CHAT_CONNECTOR_ID" +printf ' chat_model_id=%s\n' "$CHAT_MODEL_ID" +printf ' chat_agent_id=%s\n' "$CHAT_AGENT_ID" +printf ' root_agent_id=%s\n' "$ROOT_AGENT_ID" +printf ' ppl_connector_id=%s\n' "$PPL_CONNECTOR_ID" +printf ' ppl_model_id=%s\n' "$PPL_MODEL_ID" +printf ' ppl_agent_id=%s\n' "$PPL_AGENT_ID" diff --git a/security/es_roles/opensearch/roles.yml b/security/es_roles/opensearch/roles.yml index 77693806..f0e4c6c1 100644 --- a/security/es_roles/opensearch/roles.yml +++ b/security/es_roles/opensearch/roles.yml @@ -328,6 +328,25 @@ ml_full_access: allowed_actions: - 'indices_monitor' +# OpenSearch 3.x protects ML Commons' backing indices as system indices. The +# standard ml_full_access role grants the ML API actions but not the explicit +# system-index permission needed when system-index permissions are enabled. +cogstack_ml_full_access: + reserved: false + hidden: false + cluster_permissions: + - 'cluster_monitor' + - 'cluster:admin/opensearch/ml/*' + index_permissions: + - index_patterns: + - '*' + allowed_actions: + - 'indices_monitor' + - index_patterns: + - '.plugins-ml-*' + allowed_actions: + - 'system:admin/system_index' + # Allows users to use all Notifications functionality notifications_full_access: reserved: true diff --git a/security/es_roles/opensearch/roles_mapping.yml b/security/es_roles/opensearch/roles_mapping.yml index ba604379..76592a84 100644 --- a/security/es_roles/opensearch/roles_mapping.yml +++ b/security/es_roles/opensearch/roles_mapping.yml @@ -31,6 +31,43 @@ all_access: and_backend_roles: [""] description: "Maps admin to all_access" +# Grant the current administrator ML Commons access and provide a backend role +# that can be assigned to dedicated ML API/UI users. +ml_full_access: + reserved: false + hidden: false + backend_roles: + - "ml_full_access" + hosts: [] + users: + - "admin" + and_backend_roles: [] + description: "Maps ML administrators to ml_full_access" + +# Companion role required by OpenSearch 3.x when system-index permissions are +# enabled. Keep its users/backend roles aligned with ml_full_access above. +cogstack_ml_full_access: + reserved: false + hidden: false + backend_roles: + - "ml_full_access" + hosts: [] + users: + - "admin" + and_backend_roles: [] + description: "Allows ML Commons APIs to access their protected system indices" + +# Read-only ML users can inspect models, connectors, tasks, and statistics. +ml_read_access: + reserved: false + hidden: false + backend_roles: + - "ml_read_access" + hosts: [] + users: [] + and_backend_roles: [] + description: "Maps ML observers to ml_read_access" + own_index: reserved: false hidden: false diff --git a/services/elasticsearch/config/opensearch.yml b/services/elasticsearch/config/opensearch.yml index bbc9fd3c..274b4ee9 100644 --- a/services/elasticsearch/config/opensearch.yml +++ b/services/elasticsearch/config/opensearch.yml @@ -182,6 +182,41 @@ plugins.security.audit.config.enable_ssl_client_auth: true plugins.security.system_indices.enabled: true plugins.security.system_indices.permission.enabled: true +# ================= ML Commons ================= + +plugins.ml_commons.local_model.enabled: true +plugins.ml_commons.remote_inference.enabled: true +plugins.ml_commons.only_run_on_ml_node: false +plugins.ml_commons.allow_registering_model_via_url: true +plugins.ml_commons.allow_registering_model_via_local_file: true +# Permit HTTPS model repositories and only the dedicated read-only local model +# directory; do not allow arbitrary HTTP, FTP, or file-system paths. +plugins.ml_commons.trusted_url_regex: '^https://.*$|^file:///usr/share/opensearch/models/[^?#]+\.zip$' +# Permit ML Commons connectors to call only the Ollama service on the private +# CogStack Docker network. Docker resolves this stable service name to a +# private container IP, so both the general and private endpoint allowlists +# are required by OpenSearch 3.7 and later. +plugins.ml_commons.connector.private_ip_enabled: true +plugins.ml_commons.trusted_connector_endpoints_regex: + - '^http://ollama:11434/.*$' +plugins.ml_commons.trusted_connector_private_endpoints_regex: + - '^http://ollama:11434/.*$' +plugins.ml_commons.model_access_control_enabled: true +plugins.ml_commons.connector_access_control_enabled: true +plugins.ml_commons.safe_delete_model: true +plugins.ml_commons.agent_framework_enabled: true +plugins.ml_commons.rag_pipeline_feature_enabled: true +# Upper bound for asynchronous ML Commons tasks. Remote connector connection +# and response timeouts are configured per connector by opensearch-ai-init.sh. +plugins.ml_commons.ml_task_timeout_in_seconds: 600 + +# This cluster has no dedicated ML nodes, so allow both local and remote models +# to execute on data nodes. +plugins.ml_commons.task_dispatcher.eligible_node_role.local_model: + - data +plugins.ml_commons.task_dispatcher.eligible_node_role.remote_model: + - data + # Enable SQL-like queries, allows you to join across indices etc. plugins.sql.enabled: true diff --git a/services/elasticsearch/models/.gitignore b/services/elasticsearch/models/.gitignore new file mode 100644 index 00000000..7c9d611b --- /dev/null +++ b/services/elasticsearch/models/.gitignore @@ -0,0 +1,3 @@ +* +!.gitignore +!README.md diff --git a/services/elasticsearch/models/README.md b/services/elasticsearch/models/README.md new file mode 100644 index 00000000..1683d704 --- /dev/null +++ b/services/elasticsearch/models/README.md @@ -0,0 +1,18 @@ +# Local OpenSearch ML models + +Place custom TorchScript or ONNX model ZIP files in this directory before +starting the OpenSearch containers. The directory is mounted read-only at the +same path on every OpenSearch node: + +```text +/usr/share/opensearch/models +``` + +Register a model through the ML Commons REST API or OpenSearch Dashboards Dev +Tools using a URL such as: + +```text +file:///usr/share/opensearch/models/my-model.zip +``` + +Do not commit model binaries to the repository. diff --git a/services/kibana/config/opensearch.yml b/services/kibana/config/opensearch.yml index 72a3103d..f37d5131 100644 --- a/services/kibana/config/opensearch.yml +++ b/services/kibana/config/opensearch.yml @@ -85,7 +85,7 @@ opensearch.pingTimeout: 10000 # Time in milliseconds to wait for responses from the back end or OpenSearch. This value # must be a positive integer. -opensearch.requestTimeout: 200000 +opensearch.requestTimeout: 420000 # List of OpenSearch Dashboards client-side headers to send to OpenSearch. To send *no* client-side # headers, set this value to [] (an empty list). @@ -132,12 +132,16 @@ opensearch.requestHeadersAllowlist: ["securitytenant", "Authorization"] # for your map rendering services. # map.showRegionBlockedWarning: false +# Enable ML Commons model monitoring and management. Model registration and +# deployment are performed through the REST API or the Dashboards Dev Tools UI. ml_commons_dashboards.enabled: true +assistant.chat.enabled: true +observability.query_assist.enabled: true savedObjects.permission.enabled: true # Set the value to true to enable multiple data source feature -data_source.enabled: true +data_source.enabled: false # Set the value to true to enable workspace feature workspace.enabled: true # Set the value to true to enable explore feature