Mqtt

.github/workflows/cdb-base-mqtt.yml

@@ -0,0 +1,45 @@
+name: CDB Base MQTT
+
+on:
+  push:
+    paths:
+      - "tools/developer_tools/cdb-base-mqtt/**"
+      - ".github/workflows/cdb-base-mqtt.yml"
+  pull_request:
+    paths:
+      - "tools/developer_tools/cdb-base-mqtt/**"
+      - ".github/workflows/cdb-base-mqtt.yml"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: tools/developer_tools/cdb-base-mqtt
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Java 8 and Maven cache
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: "8"
+          cache: maven
+          cache-dependency-path: tools/developer_tools/cdb-base-mqtt/pom.xml
+
+      - name: Run tests
+        run: mvn --batch-mode --no-transfer-progress verify
+
+      - name: Publish test results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cdb-base-mqtt-test-results
+          path: tools/developer_tools/cdb-base-mqtt/target/surefire-reports/
+          if-no-files-found: warn

CLAUDE.md

@@ -128,6 +128,7 @@ MariaDB 10.5. Authoritative schema is the set of `create_cdb_tables.sql`, `creat
 - **Generated files that look like sources** are common: `cdb.portal.properties`, `web/WEB-INF/glassfish-web.xml`, `web/WEB-INF/web.xml`, `setup/glassfish-resources.xml`, `nbproject/private/private.properties`, and everything under `tools/developer_tools/python-client/packages/api/cdbApi/`. They are produced from `*.template` files (or via openapi-generator) by `dev-config` / `configure-web-portal` / `dist` targets and are in `.gitignore`. Edit the template, regenerate, then commit only the template.
 - **`cdb` vs `cdb_dev`**: every db, deploy, undeploy, and backup target has a `-dev` variant that targets a separate database name and Payara domain, so a developer can run both side-by-side. When adding new admin scripts, follow the same pattern (first arg is DB name, defaulting to `cdb`).
 - **Single canonical version string**: run `make prepare-release` (`sbin/cdb_prepare_release.py`) to bump `etc/version`, `openapi.yaml`, the web `?v=` cache-busters, the python-client `packages/*/pyproject.toml`/conda recipes, and scaffold `docs/release-notes/<VERSION>.md` together. It also flags any file that drifted out of sync in a prior manual release, and refreshes `uv.lock`.
+- **Release notes as a plan milestone**: when planning a feature, include updating `docs/release-notes/<VERSION>.md` as the final milestone. Use the in-progress version (unreleased versions may already have notes; `etc/version` lags until `make prepare-release`), and create the file if it doesn't exist. Match the style of existing notes: `#` headings per area (e.g. `# MQTT`, `# API`, `# Testing`, `# Bug Fixes`), concise user-facing `*` bullets with nested sub-bullets for detail, and no commit hashes or file lists. Only list fixes to previously released behavior under Bug Fixes.
 - **Plugins** under `tools/developer_tools/cdb_plugins/plugins/<name>/` extend both the Java portal (deployed via `make deploy-cdb-plugin` → `update_plugin_generated_files.py`) and the Python web service. `pluginTemplates/` is the scaffold.
 - **REST changes touch three places**: the Java route, `openapi.yaml`, and (after `generatePyClient.sh`) the regenerated `cdbApi/` package. The Python API tests under `tools/developer_tools/python-client/test/` are the integration check.
 

Makefile

@@ -9,6 +9,7 @@ TOP = .
 SUBDIRS = src
 
 .PHONY: support support-mysql dev-config prepare-release release-python-client
+.PHONY: mqtt-configuration configure-mqtt mqtt-configuration-dev configure-mqtt-dev
 .PHONY: db backup db-dev deploy-web-portal undeploy-web-portal deploy-web-service undeploy-web-service
 .PHONY: db-dev backup-dev deploy-web-portal-dev undeploy-web-portal-dev deploy-web-service-dev undeploy-web-service-dev
 
@@ -28,6 +29,12 @@ release-python-client:
 configuration:
 	$(TOP)/sbin/cdb_create_configuration.sh
 
+mqtt-configuration:
+	$(TOP)/sbin/cdb_create_mqtt_configuration.sh cdb
+
+configure-mqtt:
+	$(TOP)/sbin/cdb_configure_mqtt_service.sh cdb
+
 support:
 	$(TOP)/sbin/cdb_install_support.sh
 
@@ -79,6 +86,12 @@ undeploy-web-service:
 configuration-dev:
 	$(TOP)/sbin/cdb_create_configuration.sh cdb_dev
 
+mqtt-configuration-dev:
+	$(TOP)/sbin/cdb_create_mqtt_configuration.sh cdb_dev
+
+configure-mqtt-dev:
+	$(TOP)/sbin/cdb_configure_mqtt_service.sh cdb_dev
+
 db-dev:
 	$(TOP)/sbin/cdb_create_db.sh cdb_dev
 

docs/release-notes/3.20.0.md

@@ -0,0 +1,44 @@
+# MQTT
+
+* Added MQTT integration. The portal publishes change events to an MQTT broker so external systems (e.g. BELY) can react to CDB changes.
+  * Shared `cdb-base-mqtt` library for MQTT connection and event serialization, used by both BELY and CDB.
+  * Events include the affected entity, the user who triggered the change, and details of what changed.
+  * Consistent operation names across events (add, update, delete, trash, restore).
+* Entity events
+  * Item add, update, and delete.
+* Property events
+  * Property value add, update, and delete, with the before and after values.
+* Log events
+  * Log entry add, update, and delete, with the before and after text.
+  * Log attachment add.
+* Machine design events
+  * New machine design, including ones created from a template.
+  * Assigned item update.
+  * Move within the machine hierarchy.
+  * Represented element update, including changes made through a template.
+  * Trash and restore.
+* Inventory events
+  * Location update.
+  * Assembly element assignment and clearing.
+* Configuration
+  * `make mqtt-configuration` and `make configure-mqtt` (with `-dev` variants) to create and deploy the MQTT connection configuration.
+
+# API
+
+* Ability to update an existing log entry.
+* Ability to delete a log entry.
+* Attachments added via the API, utilities, and the portal now use the same workflow.
+
+# Testing
+
+* Added unit tests for all MQTT event models and change detection.
+* Added an MQTT test framework that listens to the broker and verifies published events.
+  * Python API tests for events that can be triggered via the API.
+  * Selenium tests for events only available in the portal.
+  * MQTT tests are included in `make test`.
+* CI runs the shared MQTT library tests when it changes.
+* Documented the new tests in the python client and Selenium READMEs.
+
+# Bug Fixes
+
+* Destroy is now called after a successful `updateOnRemoval`, removing the need for workarounds in the log list views.

sbin/cdb_configure_mqtt_service.sh

@@ -0,0 +1,170 @@
+#!/bin/bash
+
+# Copyright (c) UChicago Argonne, LLC. All rights reserved.
+# See LICENSE file.
+
+
+#
+# Script used for configuring MQTT connector for CDB webapp
+# Deploys MQTT Resource Adapter and creates connection pool/resource
+#
+# Usage:
+#
+# $0 [CDB_DB_NAME] [mqtt_config_file]
+#
+# If no config file is specified, defaults to $CDB_INSTALL_DIR/etc/<CDB_DB_NAME>.mqtt.conf
+#
+# Sample MQTT configuration file contents (mqtt.conf):
+# MQTT_HOST=localhost              # MQTT broker hostname (default: localhost)
+# MQTT_PORT=1883                   # MQTT broker port (default: 1883)
+# MQTT_USERNAME=admin              # MQTT username (optional)
+# MQTT_PASSWORD=admin              # MQTT password (optional)
+# MQTT_CLEAN_SESSION=true          # Clean session flag (optional)
+# MQTT_QOS=1                       # Quality of Service level (optional)
+# MQTT_KEEP_ALIVE_INTERVAL=60      # Keep alive interval in seconds (optional)
+# MQTT_CONNECTION_TIMEOUT=30       # Connection timeout in seconds (optional)
+# MQTT_MAX_INFLIGHT=10             # Maximum number of messages in flight (optional)
+# MQTT_AUTOMATIC_RECONNECT=true    # Automatic reconnection on disconnect (optional)
+# MQTT_FILE_PERSISTANCE=false      # Enable file-based message persistence (optional)
+# MQTT_PERSISTENCE_DIRECTORY=.     # Directory for persistent message storage (optional)
+# MQTT_TOPIC_FILTER                # MQTT topic filter for subscriptions (optional)
+
+MY_DIR=`dirname $0` && cd $MY_DIR && MY_DIR=`pwd`
+if [ -z "${CDB_ROOT_DIR}" ]; then
+    CDB_ROOT_DIR=$MY_DIR/..
+fi
+CDB_ENV_FILE=${CDB_ROOT_DIR}/setup.sh
+if [ ! -f ${CDB_ENV_FILE} ]; then
+    echo "Environment file ${CDB_ENV_FILE} does not exist."
+    exit 2
+fi
+. ${CDB_ENV_FILE} > /dev/null
+
+# Constants
+MQTT_POOL_NAME="cdb/MQTT/pool"
+MQTT_RESOURCE_NAME="cdb/MQTT/resource"
+MQTT_RAR_NAME="mqtt-rar-0.8.0"
+MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME="cdb-mqtt-rar-deployment"
+MQTT_RAR_PATH=$CDB_ROOT_DIR/src/lib/${MQTT_RAR_NAME}.rar
+
+# Look for the deployment-specific MQTT configuration file
+CDB_DB_NAME=${1:-${CDB_DB_NAME:-cdb}}
+if [ ! -z "$2" ]; then
+    mqttConfigFile=$2
+else
+    mqttConfigFile=$CDB_INSTALL_DIR/etc/${CDB_DB_NAME}.mqtt.conf
+fi
+
+if [ -f $mqttConfigFile ]; then
+    echo "Using MQTT config file: $mqttConfigFile"
+    . $mqttConfigFile
+else
+    echo "Error: MQTT config file $mqttConfigFile not found."
+    echo "You can create one using cdb_create_mqtt_configuration.sh ${CDB_DB_NAME}"
+    exit 1
+fi
+
+CDB_HOST_ARCH=$(uname -sm | tr -s '[:upper:][:blank:]' '[:lower:][\-]')
+GLASSFISH_DIR=$CDB_SUPPORT_DIR/payara/$CDB_HOST_ARCH
+
+ASADMIN_CMD=$GLASSFISH_DIR/bin/asadmin
+
+# MQTT Configuration defaults
+MQTT_HOST=${MQTT_HOST:=localhost}
+MQTT_PORT=${MQTT_PORT:=1883}
+
+# Build properties string for connection pool
+PROPERTIES="serverURIs=tcp\\://${MQTT_HOST}\\:${MQTT_PORT}"
+
+if [ ! -z "$MQTT_USERNAME" ]; then
+    PROPERTIES="${PROPERTIES}:userName=${MQTT_USERNAME}"
+fi
+
+if [ ! -z "$MQTT_PASSWORD" ]; then
+    PROPERTIES="${PROPERTIES}:password=${MQTT_PASSWORD}"
+fi
+
+if [ ! -z "$MQTT_CLEAN_SESSION" ]; then
+    PROPERTIES="${PROPERTIES}:cleanSession=${MQTT_CLEAN_SESSION}"
+fi
+
+if [ ! -z "$MQTT_KEEP_ALIVE_INTERVAL" ]; then
+    PROPERTIES="${PROPERTIES}:keepAliveInterval=${MQTT_KEEP_ALIVE_INTERVAL}"
+fi
+
+if [ ! -z "$MQTT_CONNECTION_TIMEOUT" ]; then
+    PROPERTIES="${PROPERTIES}:connectionTimeout=${MQTT_CONNECTION_TIMEOUT}"
+fi
+
+if [ ! -z "$MQTT_MAX_INFLIGHT" ]; then
+    PROPERTIES="${PROPERTIES}:maxInflight=${MQTT_MAX_INFLIGHT}"
+fi
+
+if [ ! -z "$MQTT_AUTOMATIC_RECONNECT" ]; then
+    PROPERTIES="${PROPERTIES}:automaticReconnect=${MQTT_AUTOMATIC_RECONNECT}"
+fi
+
+if [ ! -z "$MQTT_FILE_PERSISTANCE" ]; then
+    PROPERTIES="${PROPERTIES}:filePersistance=${MQTT_FILE_PERSISTANCE}"
+fi
+
+if [ ! -z "$MQTT_PERSISTENCE_DIRECTORY" ]; then
+    PROPERTIES="${PROPERTIES}:persistenceDirectory=${MQTT_PERSISTENCE_DIRECTORY}"
+fi
+
+if [ ! -z "$MQTT_QOS" ]; then
+    PROPERTIES="${PROPERTIES}:qos=${MQTT_QOS}"
+fi
+
+if [ ! -z "$MQTT_TOPIC_FILTER" ]; then
+    PROPERTIES="${PROPERTIES}:topicFilter=${MQTT_TOPIC_FILTER}"
+fi
+
+# Deploy MQTT RAR
+echo "Deploying MQTT RAR"
+if [ -f "$MQTT_RAR_PATH" ]; then
+    # Check if already deployed and undeploy if needed
+    $ASADMIN_CMD list-applications | grep -q ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} && {
+        echo "Undeploying existing MQTT RAR"
+        # Check if resource exists and delete it
+        $ASADMIN_CMD list-connector-resources | grep -q ${MQTT_RESOURCE_NAME} && {
+            echo "Deleting existing MQTT resource"
+            $ASADMIN_CMD delete-connector-resource ${MQTT_RESOURCE_NAME} || exit 1
+        }
+
+        # Check if connection pool exists and delete it
+        $ASADMIN_CMD list-connector-connection-pools | grep -q ${MQTT_POOL_NAME} && {
+            echo "Deleting existing MQTT connection pool"
+            $ASADMIN_CMD delete-connector-connection-pool ${MQTT_POOL_NAME} || exit 1
+        }
+
+        echo "Undeploying existing MQTT RAR"
+        $ASADMIN_CMD undeploy ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} || exit 1
+    }
+    $ASADMIN_CMD deploy --name ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} $MQTT_RAR_PATH || exit 1
+else
+    echo "Warning: MQTT RAR file not found at $MQTT_RAR_PATH"
+    exit 1
+fi
+
+# Create MQTT connection pool
+echo "Creating MQTT connection pool ${MQTT_POOL_NAME}"
+$ASADMIN_CMD create-connector-connection-pool \
+    --raname ${MQTT_CONNECTOR_RAR_DEPLOYMENT_NAME} \
+    --connectiondefinition fish.payara.cloud.connectors.mqtt.api.MQTTConnectionFactory \
+    --property "${PROPERTIES}" \
+    ${MQTT_POOL_NAME} || exit 1
+# Create MQTT resource
+echo "Creating MQTT resource ${MQTT_RESOURCE_NAME}"
+$ASADMIN_CMD create-connector-resource \
+    --poolname ${MQTT_POOL_NAME} \
+    ${MQTT_RESOURCE_NAME} || exit 1
+
+# The connector logs allocation failures with a full stack trace before returning null.
+$ASADMIN_CMD set-log-levels fish.payara.cloud.connectors.mqtt.api.outbound.MQTTConnectionFactoryImpl=OFF || exit 1
+
+# Test MQTT connection pool
+echo "Testing MQTT connection pool"
+$ASADMIN_CMD ping-connection-pool ${MQTT_POOL_NAME} || { echo "Warning: MQTT connection pool ping failed"; exit 1; }
+
+echo "Restart or redeploy CDB."