Merge pull request #2903 from simonredfern/develop

auth_type metrics column etc.

Author: simonredfern

obp-api/src/main/scala/code/api/ResourceDocs1_4_0/SwaggerDefinitionsJSON.scala

@@ -3202,6 +3202,7 @@ object SwaggerDefinitionsJSON {
     operation_id = "OBPv4.0.0-getBanks",
     api_instance_id = "obp_node_a",
     consent_reference_id = Some(ExampleValue.consentReferenceIdExample.value),
+    auth_type = Some("Consent"),
     certificate_trust = Some("forwarded"),
     certificate_trust_detail = Some("cn=nginx-prod-1,ou=edge,o=tesobe gmbh,c=de")
   )

obp-api/src/main/scala/code/api/constant/constant.scala

@@ -16,6 +16,21 @@ object Constant extends MdcLoggable {
 
   final val directLoginHeaderName = "DirectLogin"
 
+  // createdByProcess of entitlement rows the consent engine copies onto a consent user —
+  // the per-consent principal a Consent-JWT authenticates as (its ResourceUser row carries
+  // CreatedByConsentId). Only rows tagged with this value may target a consent user:
+  // addEntitlement redirects any other grant to the consent's granting human, so durable
+  // roles (e.g. bank-creator grants) can never strand on a principal that dies with its
+  // consent. Also the marker for cleaning these rows up when the consent is revoked.
+  final val consent_user = "consent_user"
+
+  // createdByProcess of entitlement rows granted through group membership (the Groups
+  // feature). The value predates this constant: the Groups feature originally wrote it to
+  // its own `process` column, a duplicate of createdByProcess since retired — provenance
+  // now lives in createdByProcess like every other granting mechanism, and group rows are
+  // identified by their group_id.
+  final val group_membership = "GROUP_MEMBERSHIP"
+
   object Pagination {
     final val offset = 0
     final val limit = 50

obp-api/src/main/scala/code/api/util/ApiSession.scala

@@ -39,7 +39,7 @@ case class CallContext(
                         // the creator is the granting human (they create their own consent in the Portal).
                         // Not set by Berlin Group / UK flows, where the consent may be created by a TPP flow
                         // with no human logged in — see `consenter` for those.
-                        // Read via humanUser / effectiveHumanUserId, where it takes precedence over consenter.
+                        // Read via humanUser / accountableUserId, where it takes precedence over consenter.
                         onBehalfOfUser: Box[User] = Empty,
                         // The human (PSU) who AUTHORISED the consent this request runs under — the owner of
                         // record, from the consent table's userId (bound by updateConsentUser during the
@@ -113,7 +113,7 @@ case class CallContext(
    * Anything that must name a human rather than a principal reads this instead: the CBS adapter,
    * which tells the core banking system who is asking, and the consent ownership checks.
    * Stored data (metric rows included) always carries the authenticated principal; the human is
-   * resolved at read time via the consent table (see effectiveHumanUserId).
+   * resolved at read time via the consent table (see accountableUserId).
    */
   def humanUser: Box[User] = onBehalfOfUser.or(consenter).or(user)
 
@@ -182,7 +182,7 @@ case class CallContext(
       // (CallContext.user), never a resolved human. Under a consent that principal is the
       // consent's own shadow user (a per-consent UUID with an empty name) — the on-behalf-of
       // human is not stored here but resolved at read time via the consent table
-      // (consentReferenceId below -> consent.userId), see CallContext.effectiveHumanUserId.
+      // (consentReferenceId below -> consent.userId), see CallContext.accountableUserId.
       userId = this.user.map(_.userId).toOption,
       userName = this.user.map(_.name).toOption,
       consumerId = this.consumer.map(_.consumerId.get).toOption,
@@ -217,22 +217,31 @@ case class CallContext(
   def userId: String  = user.map(_.userId).openOrThrowException(AuthenticatedUserIsRequired)
 
   /**
-   * The human User this request is really about.
+   * The ACCOUNTABLE identity this request is really about — the user_id that durable
+   * state (creator role grants, account holders, entitlement requests) and attribution
+   * (metrics families, "my" queries) bind to. "Accountable" deliberately hints at a
+   * legal person: today resolution always ends at the human who granted the consent,
+   * but the contract is accountability, not species — if durable, sponsored agent
+   * identities are ever admitted as principals in their own right, resolution may stop
+   * at such an agent without this name becoming a lie (unlike the previous name,
+   * effectiveHumanUserId).
    *
-   * The authenticated `user` may be the human themselves, or an agent user minted by a
-   * Consent the human granted (e.g. Opey / MCP acting under a consent). Resolution order:
+   * The authenticated `user` may be the accountable party themselves, or a consent user
+   * minted by a Consent they granted (e.g. Opey / MCP acting under a consent) — consent
+   * users are ephemeral and must never hold durable state (see addEntitlement's guard).
+   * Resolution order:
    *  1. `onBehalfOfUser` or `consenter`, when a middleware populated them (free);
    *  2. otherwise resolve via the delegation registry: the caller's ResourceUser row's
    *     CreatedByConsentId names the Consent that minted it, and that Consent's userId
    *     names the granting human;
-   *  3. otherwise the caller IS the human.
+   *  3. otherwise the caller IS the accountable party.
    *
    * IMPORTANT: this reads only the authenticated user and server-written columns
    * (ResourceUser.CreatedByConsentId, MappedConsent.mUserId). It deliberately takes no
    * parameters so nothing caller-asserted (body/header/query values) can ever influence
    * the resolution — identity-sensitive queries (e.g. /my/banks) depend on that.
    */
-  def effectiveHumanUserId: String = {
+  def accountableUserId: String = {
     val delegatedHumanUserId = onBehalfOfUser.or(consenter).map(_.userId).filter(_.nonEmpty)
     delegatedHumanUserId.openOr {
       val authenticatedUserId = user.map(_.userId).openOr("")

obp-api/src/main/scala/code/api/util/ConsentUtil.scala

@@ -380,7 +380,10 @@ object Consent extends MdcLoggable {
           existingEntitlements.exists(_.roleName == entitlement.role_name) match { // Check is a role already added to a user
             case false =>
               val bankId = if (role.requiresBankId) entitlement.bank_id else ""
-              Entitlement.entitlement.vend.addEntitlement(bankId, user.userId, entitlement.role_name) match {
+              // Tagged consent_user: this is the ONE writer allowed to target a consent
+              // user — addEntitlement redirects untagged grants to the granting human.
+              Entitlement.entitlement.vend.addEntitlement(bankId, user.userId, entitlement.role_name,
+                createdByProcess = Constant.consent_user) match {
                 case Full(_) => (entitlement, "AddedOrExisted")
                 case _ =>
                   (entitlement, CannotAddEntitlement + entitlement)
@@ -905,7 +908,7 @@ object Consent extends MdcLoggable {
       } yield {
         (principal, callContext.copy(
           // The PSU stays reachable for everything that needs a human: the CBS adapter, metric
-          // attribution, and CallContext.effectiveHumanUserId.
+          // attribution, and CallContext.accountableUserId.
           consenter = Full(psu),
           ukConsentId = Some(storedConsent.consentId),
           consentReferenceId = Some(storedConsent.consentReferenceId)

obp-api/src/main/scala/code/api/util/WriteMetricUtil.scala

@@ -45,7 +45,8 @@ object WriteMetricUtil extends MdcLoggable {
                                    duration: Long,
                                    responseBodyToWrite: String,
                                    sourceIp: String,
-                                   targetIp: String)
+                                   targetIp: String,
+                                   authType: String)
 
   private def persistAndPublishMetric(responseBody: Any, cc: CallContextLight): Unit = {
     val fields = MetricFields(
@@ -58,7 +59,8 @@ object WriteMetricUtil extends MdcLoggable {
       duration = callDuration(cc),
       responseBodyToWrite = responseBodyForMetric(responseBody, cc),
       sourceIp = requestHeaderValue(cc, "x-forwarded-for"),
-      targetIp = requestHeaderValue(cc, "x-forwarded-host")
+      targetIp = requestHeaderValue(cc, "x-forwarded-host"),
+      authType = deriveAuthType(cc)
     )
 
     // enqueue synchronously so flush() in tests reliably drains this metric before assertions
@@ -74,6 +76,30 @@ object WriteMetricUtil extends MdcLoggable {
     }
   }
 
+  /**
+   * Authentication SCHEME of the call — never the credential itself. "Consent" wins
+   * outright: when a consent authenticated the call, the Authorization header (if any)
+   * was not what authorized it. The rest is read off the Authorization header shape,
+   * with the gateway payload / direct-login params as fallbacks for flows that
+   * populate those without a header.
+   */
+  private[util] def deriveAuthType(cc: CallContextLight): String = {
+    if (cc.consentReferenceId.isDefined) "Consent"
+    else cc.authReqHeaderField.map(_.trim) match {
+      case Some(h) if h.startsWith("DirectLogin") => "DirectLogin"
+      case Some(h) if h.startsWith("Bearer") => "OAuth2"
+      case Some(h) if h.startsWith("GatewayLogin") => "GatewayLogin"
+      case Some(h) if h.startsWith("DAuth") => "DAuth"
+      case Some(h) if h.startsWith("OAuth") => "OAuth1"
+      case Some(_) => "Other"
+      case None =>
+        if (cc.gatewayLoginRequestPayload.isDefined) "GatewayLogin"
+        else if (cc.directLoginToken != null && cc.directLoginToken.nonEmpty) "DirectLogin"
+        else if (cc.userId.isDefined) "Other"
+        else "Anonymous"
+    }
+  }
+
   private def callDuration(cc: CallContextLight): Long =
     (cc.startTime, cc.endTime) match {
       case (Some(s), Some(e)) => e.getTime - s.getTime
@@ -116,7 +142,8 @@ object WriteMetricUtil extends MdcLoggable {
         code.api.Constant.ApiInstanceId,
         cc.consentReferenceId.orNull,
         cc.certificateTrust.orNull,
-        cc.certificateTrustDetail.orNull
+        cc.certificateTrustDetail.orNull,
+        authType
       )
     } catch {
       case NonFatal(e) =>

obp-api/src/main/scala/code/api/util/migration/Migration.scala

@@ -157,6 +157,7 @@ object Migration extends MdcLoggable {
       migrateChatRoomCreatedByAndLastMessageSender()
       migrateConsentReferenceIdToUuid(startedBeforeSchemifier)
       migrateMetricConsentReferenceId(startedBeforeSchemifier)
+      migrateMetricAuthType(startedBeforeSchemifier)
       migrateMetricCertificateTrust(startedBeforeSchemifier)
       dropFastFirehoseAccountsViews(startedBeforeSchemifier)
       alterDynamicResourceDocBodyFieldsLength()
@@ -808,6 +809,18 @@ object Migration extends MdcLoggable {
       }
     }
 
+    private def migrateMetricAuthType(startedBeforeSchemifier: Boolean): Boolean = {
+      if(startedBeforeSchemifier == true) {
+        logger.warn(s"Migration.database.migrateMetricAuthType(true) cannot be run before Schemifier.")
+        true
+      } else {
+        val name = nameOf(migrateMetricAuthType(startedBeforeSchemifier))
+        runOnce(name) {
+          MigrationOfMetricAuthType.migrate(name)
+        }
+      }
+    }
+
     private def migrateMetricCertificateTrust(startedBeforeSchemifier: Boolean): Boolean = {
       if(startedBeforeSchemifier == true) {
         logger.warn(s"Migration.database.migrateMetricCertificateTrust(true) cannot be run before Schemifier.")

obp-api/src/main/scala/code/api/util/migration/MigrationOfActivityDashboardIndexes.scala

@@ -88,7 +88,7 @@ object MigrationOfActivityDashboardIndexes {
    * Index on resourceuser.createdbyconsentid.
    *
    * The delegation registry: consent-agent fan-down (/my/metrics, /my/banks) and
-   * CallContext.effectiveHumanUserId look up agent users by the consent that minted them.
+   * CallContext.accountableUserId look up agent users by the consent that minted them.
    * Unindexed this is a full scan of resourceuser on every such request, which matters on
    * consent-heavy instances where every consent mints a user row.
    */
@@ -130,7 +130,7 @@ object MigrationOfActivityDashboardIndexes {
           s"""Added index on resourceuser.createdbyconsentid
              |Executed SQL:
              |$executedSql
-             |Serves the consent-agent delegation fan-down (/my/metrics, /my/banks, effectiveHumanUserId).
+             |Serves the consent-agent delegation fan-down (/my/metrics, /my/banks, accountableUserId).
              |""".stripMargin
         isSuccessful = true
         saveLog(name, commitId, isSuccessful, startDate, endDate, comment)

obp-api/src/main/scala/code/api/util/migration/MigrationOfMetricAuthType.scala

@@ -0,0 +1,74 @@
+package code.api.util.migration
+
+import code.api.util.APIUtil
+import code.api.util.migration.Migration.{DbFunction, saveLog}
+import code.metrics.MappedMetric
+import net.liftweb.mapper.Schemifier
+
+/**
+ * Migration: add `auth_type VARCHAR(32)` to both the live `Metric` table and the
+ * `metricarchive` table — the authentication SCHEME of each call ("Consent",
+ * "OAuth2", "OAuth1", "DirectLogin", "GatewayLogin", "DAuth", "Anonymous",
+ * "Other"), never the credential itself.
+ *
+ * No backup and no backfill: the column is additive and nullable — historical rows
+ * legitimately predate it and stay null. No index: always queried alongside the
+ * indexed date range.
+ *
+ * Lift's Schemifier auto-creates the column on fresh deploys from the updated model;
+ * this migration handles existing deploys. Table name note as in
+ * MigrationOfMetricConsentReferenceId: unquoted lowercase `metric` everywhere.
+ */
+object MigrationOfMetricAuthType {
+
+  def migrate(name: String): Boolean = {
+    DbFunction.tableExists(MappedMetric) match {
+      case true =>
+        val startDate = System.currentTimeMillis()
+        val commitId: String = APIUtil.gitCommit
+        val dbDriver = APIUtil.getPropsValue("db.driver") openOr "org.h2.Driver"
+        val isMssql = dbDriver.contains("com.microsoft.sqlserver.jdbc.SQLServerDriver")
+        var isSuccessful = false
+        val sqlLog = new StringBuilder()
+
+        try {
+          val addColumnMetric = if (isMssql) {
+            "ALTER TABLE metric ADD auth_type VARCHAR(32) NULL;"
+          } else {
+            "ALTER TABLE metric ADD COLUMN IF NOT EXISTS auth_type VARCHAR(32);"
+          }
+          sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnMetric)).append("\n")
+
+          val addColumnArchive = if (isMssql) {
+            "ALTER TABLE metricarchive ADD auth_type VARCHAR(32) NULL;"
+          } else {
+            "ALTER TABLE metricarchive ADD COLUMN IF NOT EXISTS auth_type VARCHAR(32);"
+          }
+          sqlLog.append(DbFunction.maybeWrite(true, Schemifier.infoF _)(() => addColumnArchive)).append("\n")
+
+          isSuccessful = true
+        } catch {
+          case e: Exception =>
+            isSuccessful = false
+            sqlLog.append(s"\nException: ${e.getMessage}\n")
+        }
+
+        val endDate = System.currentTimeMillis()
+        val comment: String =
+          s"""Executed SQL:
+             |$sqlLog
+             |""".stripMargin
+        saveLog(name, commitId, isSuccessful, startDate, endDate, comment)
+        isSuccessful
+
+      case false =>
+        val startDate = System.currentTimeMillis()
+        val commitId: String = APIUtil.gitCommit
+        val isSuccessful = false
+        val endDate = System.currentTimeMillis()
+        val comment: String = s"""${MappedMetric._dbTableNameLC} table does not exist""".stripMargin
+        saveLog(name, commitId, isSuccessful, startDate, endDate, comment)
+        isSuccessful
+    }
+  }
+}

obp-api/src/main/scala/code/api/v2_0_0/Http4s200.scala

@@ -845,8 +845,14 @@ object Http4s200 {
               isValidID(bank.bankId.value)
             }
             loggedInUserId   = user.userId
-            userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else loggedInUserId
+            // Implicit owner resolves to the HUMAN: under a Consent the caller is the
+            // per-consent shadow, and an account held by it strands when the consent dies.
+            userIdAccountOwner = if (body.user_id.nonEmpty) body.user_id else cc.accountableUserId
             (postedOrLoggedInUser, cc2) <- NewStyle.function.findByUserId(userIdAccountOwner, Some(cc))
+            // Explicit target: fail loud rather than redirect (see the entitlement endpoints).
+            _ <- code.util.Helper.booleanToFuture(
+              s"$InvalidUserId user_id names a consent user (an agent identity minted by a Consent). Accounts are held by humans - use the granting user's USER_ID.",
+              failCode = 400, cc = cc2)(!postedOrLoggedInUser.isConsentUser)
             _ <- if (userIdAccountOwner == loggedInUserId) Future.successful(Full(()))
                  else code.util.Helper.booleanToFuture(
                    s"${UserHasMissingRoles} $canCreateAccount or create account for self", failCode = 403, cc = Some(cc)) {
@@ -1188,7 +1194,13 @@ object Http4s200 {
       case req @ POST -> `prefixPath` / "users" / userId / "entitlements" =>
         EndpointHelpers.withUserAndBodyCreated[CreateEntitlementJSON, EntitlementJSON](req) { (user, body, cc) =>
           for {
-            (_, cc2) <- NewStyle.function.findByUserId(userId, Some(cc))
+            (targetUser, cc2) <- NewStyle.function.findByUserId(userId, Some(cc))
+            // Explicit target: fail loud rather than redirect. A consent user (an agent
+            // identity minted by a Consent) cannot hold durable roles — grant to the
+            // granting human instead.
+            _ <- code.util.Helper.booleanToFuture(
+              s"$InvalidUserId USER_ID names a consent user (an agent identity minted by a Consent). Entitlements target humans - use the granting user's USER_ID.",
+              failCode = 400, cc = cc2)(!targetUser.isConsentUser)
             role <- Future {
               unboxFullOrFail(
                 net.liftweb.util.Helpers.tryo { ApiRole.valueOf(body.role_name) },

obp-api/src/main/scala/code/api/v2_2_0/Http4s220.scala

@@ -465,17 +465,20 @@ object Http4s220 {
               bank.swift_bic, bank.national_identifier,
               bank.bank_routing.scheme, bank.bank_routing.address, Some(cc)
             )
+            // Creator grants target the HUMAN (see v6.0.0 createBank): under a Consent the
+            // authenticated user is a per-consent shadow, and roles granted to it are stranded.
+            humanUserId = cc.accountableUserId
             entitlements <- Future {
               unboxFullOrFail(
-                code.entitlement.Entitlement.entitlement.vend.getEntitlementsByUserId(user.userId),
+                code.entitlement.Entitlement.entitlement.vend.getEntitlementsByUserId(humanUserId),
                 Some(cc), UnknownError)
             }
             _ <- Future {
               val bankEntitlements = entitlements.filter(_.bankId == bank.id)
               if (!bankEntitlements.exists(_.roleName == canCreateEntitlementAtOneBank.toString()))
-                code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, user.userId, canCreateEntitlementAtOneBank.toString())
+                code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, humanUserId, canCreateEntitlementAtOneBank.toString(), grantedByUserId = Some(user.userId))
               if (!bankEntitlements.exists(_.roleName == canReadDynamicResourceDocsAtOneBank.toString()))
-                code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, user.userId, canReadDynamicResourceDocsAtOneBank.toString())
+                code.entitlement.Entitlement.entitlement.vend.addEntitlement(bank.id, humanUserId, canReadDynamicResourceDocsAtOneBank.toString(), grantedByUserId = Some(user.userId))
             }
           } yield JSONFactory220.createBankJSON(success)
         }