Retention policy

obp-api/src/main/resources/props/sample.props.template

@@ -516,6 +516,9 @@ mail.smtp.port=25
 mail.smtp.auth=false
 mail.smtp.user=
 mail.smtp.password=
+## Milliseconds to wait for the SMTP server to connect, answer and accept data (Default: 30000).
+## Mail such as the consent challenge is sent inside the request, so a server that goes quiet must not hold it.
+#mail.smtp.timeout_millis=30000
 
 ## TLS/SSL Configuration
 ## ---------------------

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerJSONFactory.scala

@@ -707,7 +707,13 @@ object SwaggerJSONFactory extends MdcLoggable {
         case _ => paramValue
       }
 
-      val definition = buildSwaggerSchema(paramType, exampleValue)
+      // Name the entity and field, so an unusable example can be found from the message alone.
+      val definition =
+        try buildSwaggerSchema(paramType, exampleValue)
+        catch {
+          case e: RuntimeException =>
+            throw new RuntimeException(s"${entityType.typeSymbol.name}.$paramName: ${e.getMessage}", e)
+        }
 
       s""" "$paramName": $definition """
     })

obp-api/src/main/scala/code/api/constant/constant.scala

@@ -374,6 +374,10 @@ object Constant extends MdcLoggable {
   // The Glossary as each instance holds it in memory. Bumping it reloads the Glossary at once and
   // rebuilds every cached resource-docs document, because their keys carry the Glossary version.
   final val GLOSSARY_NAMESPACE = "glossary"
+  // Dynamic entity definitions: the definition lists memoized in Redis and the definitions map each
+  // instance builds from them. A definition change bumps it once committed, so every instance reads
+  // the definitions again.
+  final val DYNAMIC_ENTITY_NAMESPACE = "dynamic_entity"
 
   // List of all versioned cache namespaces
   final val ALL_CACHE_NAMESPACES = List(
@@ -393,7 +397,8 @@ object Constant extends MdcLoggable {
     FINANCIAL_PRODUCTS_NAMESPACE,
     API_PRODUCTS_NAMESPACE,
     MESSAGE_DOCS_NAMESPACE,
-    GLOSSARY_NAMESPACE
+    GLOSSARY_NAMESPACE,
+    DYNAMIC_ENTITY_NAMESPACE
   )
 
   // Cache key prefixes with global namespace and versioning for easy invalidation

obp-api/src/main/scala/code/api/dynamic/entity/helper/DefinitionsMapCache.scala

@@ -0,0 +1,68 @@
+/**
+Open Bank Project - API
+Copyright (C) 2011-2026, TESOBE GmbH.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Email: contact@tesobe.com
+TESOBE GmbH.
+Osloer Strasse 16/17
+Berlin 13359, Germany
+
+This product includes software developed at
+TESOBE (http://www.tesobe.com/)
+
+  */
+
+package code.api.dynamic.entity.helper
+
+/**
+ * Keeps one built value (the dynamic entity definitions map) for up to `ttlMillis`, and only while
+ * `currentVersion` still returns the version it was built under.
+ *
+ * The version is read before building, so a value built while the version changed is kept under the
+ * old version and dropped on the next read. [[forget]] drops the value on this instance alone, for when
+ * the version cannot change (Redis unavailable); a build that started before it is not kept after it.
+ * Nothing is kept, and the version is not read, when `ttlMillis` is not positive.
+ */
+private[helper] final class DefinitionsMapCache[M](ttlMillis: Long, currentVersion: () => Long, build: () => M) {
+
+  // (built at, version it was built under, value)
+  @volatile private var entry: Option[(Long, Long, M)] = None
+  // Replaced by forget; a build keeps its value only if the token it started under is still current.
+  @volatile private var token: AnyRef = new Object
+
+  def get: M =
+    if (ttlMillis <= 0) build()
+    else {
+      val now = System.currentTimeMillis()
+      val version = currentVersion()
+      entry match {
+        case Some((builtAt, builtUnder, value)) if builtUnder == version && now - builtAt < ttlMillis => value
+        case _ =>
+          val startedUnder = token
+          val value = build()
+          if (token eq startedUnder) entry = Some((now, version, value))
+          value
+      }
+    }
+
+  def forget(): Unit = {
+    token = new Object
+    entry = None
+  }
+
+  /** The value kept now, if any, whatever its version. */
+  def kept: Option[M] = entry.map(_._3)
+}

obp-api/src/main/scala/code/api/dynamic/entity/helper/DynamicEntityHelper.scala

@@ -27,6 +27,7 @@ TESOBE (http://www.tesobe.com/)
 
 package code.api.dynamic.entity.helper
 
+import code.api.Constant
 import code.api.util.APIUtil.{EmptyBody, ResourceDoc, userAuthenticationMessage}
 import code.api.util.ApiRole.getOrCreateDynamicApiRole
 import code.api.util.ApiTag._
@@ -202,35 +203,44 @@ object DynamicEntityHelper {
   // It is asked for many times per request (the access checks, each field's read check, the resource
   // docs), and building it reads every definition and parses each one's JSON: with 120 definitions
   // that was ~5 ms a time, and a Dynamic Query returning 11 records spent ~0.5 s rebuilding it. So
-  // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and
-  // forgotten the moment a definition is created, updated or deleted on this node
-  // (MappedDynamicEntityProvider calls forgetDefinitions). Another node sees such a change when its
-  // copy expires. Not kept in test mode, where a test may change the table behind the provider.
+  // the built map is kept for dynamicEntity.definitions_map.cache.ttl.seconds (default 300), and only
+  // while the dynamic_entity cache namespace version is the one it was built under: a definition change
+  // bumps that version once committed (forgetDefinitions), and every instance, which re-reads the version
+  // at most once a second, then builds the map again. Not kept in test mode, where a test may change the
+  // table behind the provider.
   private val definitionsMapTtlMillis: Long =
     if (net.liftweb.util.Props.testMode) 0L
     else APIUtil.getPropsAsLongValue("dynamicEntity.definitions_map.cache.ttl.seconds", 300L) * 1000L
-  // (built at, map). Each change replaces the token, and a build keeps its map only while the token it
-  // started under is still the current one, so a build that started before a change is not kept after it.
-  @volatile private var definitionsMapCache: Option[(Long, Map[(String, String), DynamicEntityInfo])] = None
-  @volatile private var definitionsToken: AnyRef = new Object
-
-  /** Forget the kept definitions map, so the next use reads the definitions again. */
-  def forgetDefinitions(): Unit = {
-    definitionsToken = new Object
-    definitionsMapCache = None
-  }
 
-  def definitionsMap: Map[(String, String), DynamicEntityInfo] = {
-    val now = System.currentTimeMillis()
-    definitionsMapCache match {
-      case Some((builtAt, map)) if now - builtAt < definitionsMapTtlMillis => map
-      case _ =>
-        val token = definitionsToken
-        val map = buildDefinitionsMap
-        if (definitionsMapTtlMillis > 0 && (definitionsToken eq token)) definitionsMapCache = Some((now, map))
-        map
+  private val definitionsMapCache = new DefinitionsMapCache[Map[(String, String), DynamicEntityInfo]](
+    definitionsMapTtlMillis,
+    () => Constant.recentCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE),
+    () => buildDefinitionsMap)
+
+  /**
+   * Tell every instance that a definition was created, updated or deleted, once the request's transaction
+   * has committed (at once outside a transaction): bump the dynamic_entity cache namespace version, which
+   * the definitions map and the definition lists NewStyle.getDynamicEntities memoizes in Redis both carry,
+   * so all of them read the definitions again, and then the dynamic resource-doc caches, which hold the
+   * docs generated from the definitions. Not earlier, because until the commit the change is not visible
+   * to other connections: an instance that saw a bump first could read the old rows and keep them under the
+   * new version. Nothing happens if the transaction rolls back.
+   *
+   * This instance also drops its map directly, which is what keeps it current when Redis is unavailable
+   * and the version cannot change. Another instance follows the dynamic_entity version within a second, so
+   * for that second it can still build resource docs from its previous map.
+   */
+  def forgetDefinitions(): Unit =
+    code.api.util.http4s.RequestScopeConnection.afterCommit {
+      Constant.incrementCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE)
+      definitionsMapCache.forget()
+      NewStyle.function.invalidateDynamicResourceDocCaches()
     }
-  }
+
+  def definitionsMap: Map[(String, String), DynamicEntityInfo] = definitionsMapCache.get
+
+  /** The number of definitions in this instance's kept map, for the cache info page. */
+  def keptDefinitionCount: Int = definitionsMapCache.kept.map(_.size).getOrElse(0)
 
   private def buildDefinitionsMap: Map[(String, String), DynamicEntityInfo] = NewStyle.function.getDynamicEntities(None, true).map(it => ((DynamicEntitySpace.bankIdOrSystem(it.bankId), it.entityName), DynamicEntityInfo(it.metadataJson, it.entityName, it.bankId, it.hasPersonalEntity, it.hasPublicAccess, it.hasCommunityAccess, it.personalRequiresRole, it.useRowLevelAccess, it.authMode))).toMap
 

obp-api/src/main/scala/code/api/util/CommonsEmailWrapper.scala

@@ -51,9 +51,17 @@ object CommonsEmailWrapper extends MdcLoggable {
     useTLS: Boolean = true,
     useSSL: Boolean = false,
     debug: Boolean = false,
-    tlsProtocols: String = "TLSv1.2"
+    tlsProtocols: String = "TLSv1.2",
+    timeoutMillis: Int = CommonsEmailWrapper.defaultTimeoutMillis
   )
 
+  val defaultTimeoutMillis = 30000
+
+  // mail.smtp.timeout_millis as a positive number of milliseconds; anything else falls back to the default,
+  // because Jakarta Mail reads 0 (and Lift reads an unparseable value as 0) as "wait forever".
+  def timeoutMillisFrom(value: Option[String]): Int =
+    value.flatMap(v => scala.util.Try(v.trim.toInt).toOption).filter(_ > 0).getOrElse(defaultTimeoutMillis)
+
   case class EmailContent(
     from: String,
     to: List[String],
@@ -80,13 +88,33 @@ object CommonsEmailWrapper extends MdcLoggable {
       useTLS = APIUtil.getPropsValue("mail.smtp.starttls.enable", "false").toBoolean,
       useSSL = APIUtil.getPropsValue("mail.smtp.ssl.enable", "false").toBoolean,
       debug = APIUtil.getPropsValue("mail.debug", "false").toBoolean,
-      tlsProtocols = APIUtil.getPropsValue("mail.smtp.ssl.protocols", "TLSv1.2")
+      tlsProtocols = APIUtil.getPropsValue("mail.smtp.ssl.protocols", "TLSv1.2"),
+      timeoutMillis = timeoutMillisFrom(APIUtil.getPropsValue("mail.smtp.timeout_millis").toOption)
     )
   }
 
   def isTestMode: Boolean = APIUtil.getPropsValue("mail.test.mode", "false").toBoolean
 
+  // What logEmailInTestMode logs (sendTextEmail, sendHtmlEmail, sendEmailWithAttachments), oldest first and
+  // bounded, so tests can check what OBP has sent and when. Kept only when OBP runs its tests (Props.testMode): a
+  // sandbox in mail test mode must not hold challenge codes and reset links in memory.
+  private val testModeMailLimit = 200
+  private val testModeMail = new java.util.concurrent.ConcurrentLinkedQueue[EmailContent]()
+  def testModeSentMail: List[EmailContent] = {
+    import scala.jdk.CollectionConverters._
+    testModeMail.asScala.toList
+  }
+  def clearTestModeSentMail(): Unit = testModeMail.clear()
+
   private def logEmailInTestMode(content: EmailContent): Box[String] = {
+    if (net.liftweb.util.Props.testMode) {
+      // mail.test.mode.delay_millis makes a send take that long, as an SMTP round trip does, so a test can tell
+      // whether a request waited for its mail.
+      val delay = APIUtil.getPropsAsIntValue("mail.test.mode.delay_millis", 0)
+      if (delay > 0) Thread.sleep(delay.toLong)
+      testModeMail.add(content)
+      while (testModeMail.size > testModeMailLimit) testModeMail.poll()
+    }
     logger.info("=" * 80)
     logger.info("EMAIL TEST MODE - Email would be sent with the following content:")
     logger.info("=" * 80)
@@ -291,7 +319,7 @@ object CommonsEmailWrapper extends MdcLoggable {
     }
   }
 
-  private def createSession(config: EmailConfig): Session = {
+  def sessionProperties(config: EmailConfig): Properties = {
     val props = new Properties()
     props.put("mail.smtp.host", config.smtpHost)
     props.put("mail.smtp.port", config.smtpPort.toString)
@@ -300,6 +328,15 @@ object CommonsEmailWrapper extends MdcLoggable {
     props.put("mail.smtp.ssl.enable", config.useSSL.toString)
     props.put("mail.debug", config.debug.toString)
     props.put("mail.smtp.ssl.protocols", config.tlsProtocols)
+    // Jakarta Mail waits without limit by default; mail is sent inside requests, which must not hang on SMTP.
+    props.put("mail.smtp.connectiontimeout", config.timeoutMillis.toString)
+    props.put("mail.smtp.timeout", config.timeoutMillis.toString)
+    props.put("mail.smtp.writetimeout", config.timeoutMillis.toString)
+    props
+  }
+
+  private def createSession(config: EmailConfig): Session = {
+    val props = sessionProperties(config)
     val authenticator = new Authenticator() {
       override def getPasswordAuthentication: PasswordAuthentication =
         new PasswordAuthentication(config.username, config.password)

obp-api/src/main/scala/code/api/util/NewStyle.scala

@@ -3543,12 +3543,14 @@ object NewStyle extends MdcLoggable{
      * next `/resource-docs` request regenerates from the database instead of
      * serving a pre-change snapshot (TTL is 1 hour by default).
      *
-     * Call after a Dynamic Entity, Dynamic Endpoint or Dynamic Resource Doc is
-     * created, updated, or deleted: each one generates resource docs. The static
-     * resource-doc / swagger caches are not touched because dynamic docs only
-     * reach them through the content=all documents, which are left to their TTL.
+     * Call after a Dynamic Endpoint or Dynamic Resource Doc is created, updated,
+     * or deleted: each one generates resource docs. For a Dynamic Entity,
+     * DynamicEntityHelper.forgetDefinitions calls it once the change has
+     * committed. The static resource-doc / swagger caches are not touched because
+     * dynamic docs only reach them through the content=all documents, which are
+     * left to their TTL.
      */
-    private def invalidateDynamicResourceDocCaches(): Unit = {
+    private[code] def invalidateDynamicResourceDocCaches(): Unit = {
       Constant.incrementCacheNamespaceVersion(Constant.RD_DYNAMIC_NAMESPACE)
       Constant.incrementCacheNamespaceVersion(Constant.RD_ALL_NAMESPACE)
     }
@@ -3589,9 +3591,9 @@ object NewStyle extends MdcLoggable{
       }
 
       Future {
-        val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity)
-        if (result.isDefined) invalidateDynamicResourceDocCaches()
-        result
+        // The provider calls DynamicEntityHelper.forgetDefinitions, which also drops the dynamic
+        // resource-doc caches once the change has committed.
+        DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity)
       }
     }
 
@@ -3630,9 +3632,9 @@ object NewStyle extends MdcLoggable{
       }
 
       Future {
-        val result = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity)
-        if (result.isDefined) invalidateDynamicResourceDocCaches()
-        result
+        // The provider calls DynamicEntityHelper.forgetDefinitions, which also drops the dynamic
+        // resource-doc caches once the change has committed.
+        DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(dynamicEntity)
       }
 
     }
@@ -3674,8 +3676,8 @@ object NewStyle extends MdcLoggable{
           }
           deleteEntitleMentResult
         }
-        // The entity (and its generated resource docs) are gone — drop the stale doc caches.
-        if (result.exists(deleted => deleted)) invalidateDynamicResourceDocCaches()
+        // The entity's generated resource docs go with it: the provider's delete calls
+        // DynamicEntityHelper.forgetDefinitions, which drops the doc caches once the delete has committed.
         result
       }
     }
@@ -3704,9 +3706,27 @@ object NewStyle extends MdcLoggable{
     }
 
     def getDynamicEntities(bankId: Option[String], returnBothBankAndSystemLevel: Boolean): List[DynamicEntityT] = {
-      import scala.concurrent.duration._
-
       validateBankId(bankId, None)
+      getDynamicEntities(bankId, returnBothBankAndSystemLevel, dynamicEntityDefinitionsVersion)
+    }
+
+    /**
+     * The dynamic_entity cache namespace version, which the memoized definition lists below carry in their
+     * keys. DynamicEntityHelper.forgetDefinitions bumps it once a definition change has committed, so every
+     * list memoized before the change becomes unreachable on every instance, including one that a read still
+     * in flight writes after the bump: it is written under the version that read started with.
+     */
+    private def dynamicEntityDefinitionsVersion: Long =
+      if (dynamicEntityTTL == 0) 0L // nothing is memoized, so no need to ask Redis
+      else Constant.recentCacheNamespaceVersion(Constant.DYNAMIC_ENTITY_NAMESPACE)
+
+    // The version is a parameter so that CacheKeyFromArguments renders it into the key; the name stays
+    // getDynamicEntities so the key and its Telemetry label stay the same apart from the version. Keep
+    // buildCacheKey as the tail expression (see getEndpointMappingsCached): bound to a val first, the macro
+    // renders no arguments and every list would share one key whatever the version. Test mode memoizes
+    // nothing, so no test would notice; the rendering was checked with javap.
+    private def getDynamicEntities(bankId: Option[String], returnBothBankAndSystemLevel: Boolean, definitionsVersion: Long): List[DynamicEntityT] = {
+      import scala.concurrent.duration._
 
       var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)
       CacheKeyFromArguments.buildCacheKey {
@@ -3715,8 +3735,11 @@ object NewStyle extends MdcLoggable{
         }
       }
     }
-
-    def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] = {
+
+    def getDynamicEntitiesByUserId(userId: String): List[DynamicEntityT] =
+      getDynamicEntitiesByUserId(userId, dynamicEntityDefinitionsVersion)
+
+    private def getDynamicEntitiesByUserId(userId: String, definitionsVersion: Long): List[DynamicEntityT] = {
       import scala.concurrent.duration._
 
       var cacheKey = (randomUUID().toString, randomUUID().toString, randomUUID().toString)