Merge remote-tracking branch 'Simon/develop' into develop-Simon

# Conflicts:
#	.gitignore

Author: hongwei1

.gitignore

@@ -8,6 +8,9 @@
 .settings
 .metals
 .vscode
+*.code-workspace
+.zed
+.cursor
 .classpath
 .project
 .cache
@@ -30,9 +33,8 @@ obp-api/src/main/scala/code/api/v3_0_0/custom/
 marketing_diagram_generation/outputs/*
 
 .bloop
+!.bloop/*.json
 .bsp
 .specstory
 project/project
-coursier
-*.code-workspace
-.cursor
+coursier

.metals-config.json

@@ -0,0 +1,76 @@
+{
+  "maven": {
+    "enabled": true
+  },
+  "metals": {
+    "serverVersion": "1.0.0",
+    "javaHome": "/usr/lib/jvm/java-17-openjdk-amd64",
+    "bloopVersion": "2.0.0",
+    "superMethodLensesEnabled": true,
+    "enableSemanticHighlighting": true,
+    "compileOnSave": true,
+    "testUserInterface": "Code Lenses",
+    "inlayHints": {
+      "enabled": true,
+      "hintsInPatternMatch": {
+        "enabled": true
+      },
+      "implicitArguments": {
+        "enabled": true
+      },
+      "implicitConversions": {
+        "enabled": true
+      },
+      "inferredTypes": {
+        "enabled": true
+      },
+      "typeParameters": {
+        "enabled": true
+      }
+    }
+  },
+  "buildTargets": [
+    {
+      "id": "obp-commons",
+      "displayName": "obp-commons",
+      "baseDirectory": "file:///home/marko/Tesobe/GitHub/constantine2nd/OBP-API/obp-commons/",
+      "tags": ["library"],
+      "languageIds": ["scala", "java"],
+      "dependencies": [],
+      "capabilities": {
+        "canCompile": true,
+        "canTest": true,
+        "canRun": false,
+        "canDebug": true
+      },
+      "dataKind": "scala",
+      "data": {
+        "scalaOrganization": "org.scala-lang",
+        "scalaVersion": "2.12.20",
+        "scalaBinaryVersion": "2.12",
+        "platform": "jvm"
+      }
+    },
+    {
+      "id": "obp-api",
+      "displayName": "obp-api",
+      "baseDirectory": "file:///home/marko/Tesobe/GitHub/constantine2nd/OBP-API/obp-api/",
+      "tags": ["application"],
+      "languageIds": ["scala", "java"],
+      "dependencies": ["obp-commons"],
+      "capabilities": {
+        "canCompile": true,
+        "canTest": true,
+        "canRun": true,
+        "canDebug": true
+      },
+      "dataKind": "scala",
+      "data": {
+        "scalaOrganization": "org.scala-lang",
+        "scalaVersion": "2.12.20",
+        "scalaBinaryVersion": "2.12",
+        "platform": "jvm"
+      }
+    }
+  ]
+}

README.md

@@ -46,11 +46,22 @@ This project is dual licensed under the AGPL V3 (see NOTICE) and commercial lice
 The project uses Maven 3 as its build tool.
 
 To compile and run Jetty, install Maven 3, create your configuration in `obp-api/src/main/resources/props/default.props` and execute:
+To compile and run Jetty, install Maven 3, create your configuration in `obp-api/src/main/resources/props/`, copy `sample.props.template` to `default.props` and edit the latter. Then:
 
 ```sh
 mvn install -pl .,obp-commons && mvn jetty:run -pl obp-api
 ```
 
+### ZED IDE Setup
+
+For ZED IDE users, we provide a complete development environment with Scala language server support:
+
+```bash
+./zed/setup-zed-ide.sh
+```
+
+This sets up automated build tasks, code navigation, and real-time error checking. See [`zed/README.md`](zed/README.md) for complete documentation.
+
 In case the above command fails try the next one:
 
 ```sh
@@ -206,6 +217,23 @@ Once Postgres is installed (On macOS, use `brew`):
 
 1.  Grant all on database `obpdb` to `obp`; (So OBP-API can create tables etc.)
 
+#### For newer versions of postgres 16 and above, you need to follow the following instructions
+
+-- Connect to the sandbox database
+\c sandbox;
+
+-- Grant schema usage and creation privileges
+GRANT USAGE ON SCHEMA public TO obp;
+GRANT CREATE ON SCHEMA public TO obp;
+
+-- Grant all privileges on existing tables (if any)
+GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO obp;
+GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO obp;
+
+-- Grant privileges on future tables and sequences
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO obp;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO obp;
+
 1.  Then, set the `db.url` in your Props:
 
     ```
@@ -638,6 +666,59 @@ allow_oauth2_login=true
 oauth2.jwk_set.url=https://www.googleapis.com/oauth2/v3/certs
 ```
 
+### OAuth2 JWKS URI Configuration
+
+The `oauth2.jwk_set.url` property is critical for OAuth2 JWT token validation. OBP-API uses this to verify the authenticity of JWT tokens by fetching the JSON Web Key Set (JWKS) from the specified URI(s).
+
+#### Configuration Methods
+
+The `oauth2.jwk_set.url` property is resolved in the following order of priority:
+
+1. **Environment Variable**
+
+   ```bash
+   export OBP_OAUTH2_JWK_SET_URL="https://your-oidc-server.com/jwks"
+   ```
+
+2. **Properties Files** (located in `obp-api/src/main/resources/props/`)
+   - `production.default.props` (for production deployments)
+   - `default.props` (for development)
+   - `test.default.props` (for testing)
+
+#### Supported Formats
+
+- **Single URL**: `oauth2.jwk_set.url=http://localhost:9000/obp-oidc/jwks`
+- **Multiple URLs**: `oauth2.jwk_set.url=http://localhost:8080/jwk.json,https://www.googleapis.com/oauth2/v3/certs`
+
+#### Common OAuth2 Provider Examples
+
+- **Google**: `https://www.googleapis.com/oauth2/v3/certs`
+- **OBP-OIDC**: `http://localhost:9000/obp-oidc/jwks`
+- **Keycloak**: `http://localhost:7070/realms/master/protocol/openid-connect/certs`
+- **Azure AD**: `https://login.microsoftonline.com/common/discovery/v2.0/keys`
+
+#### Troubleshooting OBP-20208 Error
+
+If you encounter the error "OBP-20208: Cannot match the issuer and JWKS URI at this server instance", check the following:
+
+1. **Verify JWT Issuer Claim**: The JWT token's `iss` (issuer) claim must match one of the configured identity providers
+2. **Check JWKS URL Configuration**: Ensure `oauth2.jwk_set.url` contains URLs that correspond to your JWT issuer
+3. **Case-Insensitive Matching**: OBP-API performs case-insensitive substring matching between the issuer and JWKS URLs
+4. **URL Format Consistency**: Check for trailing slashes or URL formatting differences
+
+**Debug Logging**: Enable debug logging to see detailed information about the matching process:
+
+```properties
+# Add to your logging configuration
+logger.code.api.OAuth2=DEBUG
+```
+
+The debug logs will show:
+
+- Expected identity provider vs actual JWT issuer claim
+- Available JWKS URIs from configuration
+- Matching logic results
+
 ---
 
 ## Frozen APIs

obp-api/src/main/resources/props/sample.props.template

@@ -652,13 +652,18 @@ defaultBank.bank_id=OBP
 
 
 ################################################################################
-## Super Admin Users are used to boot strap User Entitlements (access to Roles).
-## Super Admins are receive **ONLY TWO** implicit entitlements which are:
+## Super Admin Users are used to boot-strap User Entitlements (access to Roles).
+## Super Admins listed below can grant them selves the following entitlements:
 ## CanCreateEntitlementAtAnyBank
 ## and
 ## CanCreateEntitlementAtOneBank
+## After they have granted these roles, they can grant further roles and remove their
+# user_id from the super_admin_user_ids list because its redundant.
+## Once you have the roles above you can grant any other system or bank related roles to yourself.
+##
 ## THUS, probably the first thing a Super Admin will do is to grant themselves or other users a number of Roles
-## For instance, a Super Admin *CANNOT delete an entitlement* unless they grant themselves CanDeleteEntitlementAtAnyBank or CanDeleteEntitlementAtOneBank
+## For instance, a Super Admin defined by their user_id in super_admin_user_ids CANNOT carry out actions unless they first give themselves an actual Entitlment to a Role.
+
 ## List the Users here, with their user_id(s), that should be Super Admins
 super_admin_user_ids=USER_ID1,USER_ID2,
 ################################################################################
@@ -1491,6 +1496,8 @@ validate_iban=false
 # sample props regulated_entities = [{"certificate_authority_ca_owner_id":"CY_CBC","entity_certificate_public_key":"-----BEGIN CERTIFICATE-----MIICsjCCAZqgAwIBAgIGAYwQ62R0MA0GCSqGSIb3DQEBCwUAMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTAeFw0yMzExMjcxMzE1MTFaFw0yNTExMjYxMzE1MTFaMBoxGDAWBgNVBAMMD2FwcC5leGFtcGxlLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAK9WIodZHWzKyCcf9YfWEhPURbfO6zKuMqzHN27GdqHsVVEGxP4F/J4mso+0ENcRr6ur4u81iREaVdCc40rHDHVJNEtniD8Icbz7tcsqAewIVhc/q6WXGqImJpCq7hA0m247dDsaZT0lb/MVBiMoJxDEmAE/GYYnWTEn84R35WhJsMvuQ7QmLvNg6RkChY6POCT/YKe9NKwa1NqI1U+oA5RFzAaFtytvZCE3jtp+aR0brL7qaGfgxm6B7dEpGyhg0NcVCV7xMQNq2JxZTVdAr6lcsRGaAFulakmW3aNnmK+L35Wu8uW+OxNxwUuC6f3b4FVBa276FMuUTRfu7gc+k6kCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAAU5CjEyAoyTn7PgFpQD48ZNPuUsEQ19gzYgJvHMzFIoZ7jKBodjO5mCzWBcR7A4mpeAsdyiNBl2sTiZscSnNqxk61jVzP5Ba1D7XtOjjr7+3iqowrThj6BY40QqhYh/6BSY9fDzVZQiHnvlo6ZUM5kUK6OavZOovKlp5DIl5sGqoP0qAJnpQ4nhB2WVVsKfPlOXc+2KSsbJ23g9l8zaTMr+X0umlvfEKqyEl1Fa2L1dO0y/KFQ+ILmxcZLpRdq1hRAjd0quq9qGC8ucXhRWDgM4hslVpau0da68g0aItWNez3mc5lB82b3dcZpFMzO41bgw7gvw10AvvTfQDqEYIuQ==-----END CERTIFICATE-----","entity_code":"PSD_PICY_CBC!12345","entity_type":"PSD_PI","entity_address":"EXAMPLE COMPANY LTD, 5 SOME STREET","entity_town_city":"SOME CITY","entity_post_code":"1060","entity_country":"CY","entity_web_site":"www.example.com","services":[{"CY":["PS_010","PS_020","PS_03C","PS_04C"]}]}]
 regulated_entities = []
 
+
+# Trusted Consumer pairs
 #In OBP Create Consent if the app that is creating the consent (grantor_consumer_id) wants to create a consent for the grantee_consumer_id App, then we should skip SCA.
 #The use case is API Explorer II giving a consent to Opey . In such a case API Explorer II and Opey are effectively the same App as far as the user is concerned.
 
@@ -1511,3 +1518,48 @@ regulated_entities = []
 
 
 # Note: For secure and http only settings for cookies see resources/web.xml which is mentioned in the README.md
+
+
+
+##########################################################
+#                      Redis Logging                     #
+##########################################################
+## Enable Redis logging (true/false)
+redis_logging_enabled = false
+
+## Batch size for sending logs to Redis
+## Smaller batch size reduces latency for logging critical messages.
+redis_logging_batch_size = 50
+
+## Flush interval for batch logs in milliseconds
+## Flush every 500ms to keep Redis queues up-to-date without too much delay.
+redis_logging_flush_interval_ms = 500
+
+## Maximum number of retries for failed log writes
+## Ensures transient Redis errors are retried before failing.
+redis_logging_max_retries = 3
+
+## Number of consecutive failures before opening circuit breaker
+## Prevents hammering Redis when it is down.
+redis_logging_circuit_breaker_threshold = 10
+
+## Number of threads for asynchronous Redis operations
+## Keep small for lightweight logging; adjust if heavy logging is expected.
+redis_logging_thread_pool_size = 2
+
+## SIX different FIFO Redis queues. Each queue has a maximum number of entries.
+## These control how many messages are kept per log level.
+## 1000 is a reasonable default; adjust if you expect higher traffic.
+redis_logging_trace_queue_max_entries = 1000     # Max TRACE messages
+redis_logging_debug_queue_max_entries = 1000     # Max DEBUG messages
+redis_logging_info_queue_max_entries = 1000      # Max INFO messages
+redis_logging_warning_queue_max_entries = 1000   # Max WARNING messages
+redis_logging_error_queue_max_entries = 1000     # Max ERROR messages
+redis_logging_all_queue_max_entries = 1000       # Max ALL messages
+
+## Optional: Circuit breaker reset interval (ms)
+## How long before retrying after circuit breaker opens. Default 60s.
+redis_logging_circuit_breaker_reset_ms = 60000
+##########################################################
+#                      Redis Logging                     #
+##########################################################

obp-api/src/main/scala/bootstrap/liftweb/CustomDBVendor.scala

@@ -1,6 +1,7 @@
 package bootstrap.liftweb
 
 import code.api.util.APIUtil
+import code.util.Helper.MdcLoggable
 import com.zaxxer.hikari.pool.ProxyConnection
 import com.zaxxer.hikari.{HikariConfig, HikariDataSource}
 
@@ -21,19 +22,17 @@ import net.liftweb.util.Helpers.tryo
 class CustomDBVendor(driverName: String,
                      dbUrl: String,
                      dbUser: Box[String],
-                     dbPassword: Box[String]) extends CustomProtoDBVendor {
-
-  private val logger = Logger(classOf[CustomDBVendor])
+                     dbPassword: Box[String]) extends CustomProtoDBVendor with MdcLoggable {
 
   object HikariDatasource {
     val config = new HikariConfig()
-    
+
     val connectionTimeout = APIUtil.getPropsAsLongValue("hikari.connectionTimeout")
     val maximumPoolSize = APIUtil.getPropsAsIntValue("hikari.maximumPoolSize")
     val idleTimeout = APIUtil.getPropsAsLongValue("hikari.idleTimeout")
     val keepaliveTime = APIUtil.getPropsAsLongValue("hikari.keepaliveTime")
     val maxLifetime = APIUtil.getPropsAsLongValue("hikari.maxLifetime")
-    
+
     if(connectionTimeout.isDefined){
       config.setConnectionTimeout(connectionTimeout.head)
     }
@@ -63,7 +62,7 @@ class CustomDBVendor(driverName: String,
       case _ =>
         config.setJdbcUrl(dbUrl)
     }
-    
+
     config.addDataSourceProperty("cachePrepStmts", "true")
     config.addDataSourceProperty("prepStmtCacheSize", "250")
     config.addDataSourceProperty("prepStmtCacheSqlLimit", "2048")
@@ -79,8 +78,7 @@ class CustomDBVendor(driverName: String,
   def closeAllConnections_!(): Unit = HikariDatasource.ds.close()
 }
 
-trait CustomProtoDBVendor extends ConnectionManager {
-  private val logger = Logger(classOf[CustomProtoDBVendor])
+trait CustomProtoDBVendor extends ConnectionManager with MdcLoggable {
 
   def createOne: Box[Connection]
 
@@ -90,4 +88,4 @@ trait CustomProtoDBVendor extends ConnectionManager {
 
   def releaseConnection(conn: Connection): Unit = {conn.asInstanceOf[ProxyConnection].close()}
 
-}
+}

obp-api/src/main/scala/code/accountattribute/AccountAttribute.scala

@@ -7,6 +7,7 @@ import com.openbankproject.commons.model.enums.AccountAttributeType
 import com.openbankproject.commons.model.{AccountAttribute, AccountId, BankId, BankIdAccountId, ProductAttribute, ProductCode, ViewId}
 import net.liftweb.common.{Box, Logger}
 import net.liftweb.util.SimpleInjector
+import code.util.Helper.MdcLoggable
 
 import scala.collection.immutable.List
 import scala.concurrent.Future
@@ -16,7 +17,7 @@ object AccountAttributeX extends SimpleInjector {
   val accountAttributeProvider = new Inject(buildOne _) {}
 
   def buildOne: AccountAttributeProvider = MappedAccountAttributeProvider
-  
+
   // Helper to get the count out of an option
   def countOfAccountAttribute(listOpt: Option[List[AccountAttribute]]): Int = {
     val count = listOpt match {
@@ -29,17 +30,15 @@ object AccountAttributeX extends SimpleInjector {
 
 }
 
-trait AccountAttributeProvider {
-
-  private val logger = Logger(classOf[AccountAttributeProvider])
+trait AccountAttributeProvider extends MdcLoggable {
 
   def getAccountAttributesFromProvider(accountId: AccountId, productCode: ProductCode): Future[Box[List[AccountAttribute]]]
   def getAccountAttributesByAccount(bankId: BankId,
                                     accountId: AccountId): Future[Box[List[AccountAttribute]]]
-  def getAccountAttributesByAccountCanBeSeenOnView(bankId: BankId, 
-                                                   accountId: AccountId, 
+  def getAccountAttributesByAccountCanBeSeenOnView(bankId: BankId,
+                                                   accountId: AccountId,
                                                    viewId: ViewId): Future[Box[List[AccountAttribute]]]
-  def getAccountAttributesByAccountsCanBeSeenOnView(accounts: List[BankIdAccountId], 
+  def getAccountAttributesByAccountsCanBeSeenOnView(accounts: List[BankIdAccountId],
                                                     viewId: ViewId): Future[Box[List[AccountAttribute]]]
 
   def getAccountAttributeById(productAttributeId: String): Future[Box[AccountAttribute]]
@@ -58,10 +57,10 @@ trait AccountAttributeProvider {
                               productCode: ProductCode,
                               accountAttributes: List[ProductAttribute],
                               productInstanceCode: Option[String]): Future[Box[List[AccountAttribute]]]
-  
+
   def deleteAccountAttribute(accountAttributeId: String): Future[Box[Boolean]]
 
   def getAccountIdsByParams(bankId: BankId, params: Map[String, List[String]]): Future[Box[List[String]]]
 
   // End of Trait
-}
+}